def setUp(self): model.Repo(key_name='haiti').put() config.set_for_repo( 'haiti', repo_titles={'en': 'Haiti Earthquake'}, language_menu_options=['en', 'ht', 'fr', 'es']) self.original_get_rendered = resources.get_rendered
def setUp(self): model.Repo(key_name='haiti').put() config.set_for_repo('haiti', repo_titles={'en': 'Haiti Earthquake'}, language_menu_options=['en', 'ht', 'fr', 'es'], referrer_whitelist=[]) self.original_get_rendered = resources.get_rendered
def _set_site_info_config(self): config.set_for_repo( '*', brand=self.params.brand, privacy_policy_url=self.params.privacy_policy_url, tos_url=self.params.tos_url, feedback_url=self.params.feedback_url)
def _set_timezone_config(self): if self._category_permissions['everything_else']: values = {} values['time_zone_offset'] = self.params.time_zone_offset values['time_zone_abbreviation'] = ( self.params.time_zone_abbreviation) config.set_for_repo(self.env.repo, **values)
def run_task_for_repo(self, repo): start_time = utils.get_utcnow() timestamp = self.params.timestamp or start_time is_first = not self.params.cursor query = model.Person.all_in_repo(repo).order('entry_date') if self.params.cursor: query.with_cursor(self.params.cursor) filtered_writer = record_writer.PersonWithNoteCsvWriter( StringIO.StringIO(), write_header=is_first) full_writer = record_writer.PersonWithNoteCsvWriter( StringIO.StringIO(), write_header=is_first) has_data = False scan_completed = False while True: persons = query.fetch(limit=FETCH_LIMIT) if persons: has_data = True else: scan_completed = True break full_records = self.get_person_records_with_notes(repo, persons) full_writer.write(full_records) filtered_records = copy.deepcopy(full_records) utils.filter_sensitive_fields(filtered_records) filtered_writer.write(filtered_records) if utils.get_utcnow() >= start_time + self.MAX_FETCH_TIME: break query.with_cursor(query.cursor()) for kind, writer in [ ('filtered', filtered_writer), ('full', full_writer)]: base_name = '%s-persons-%s-%s' % ( repo, kind, timestamp.strftime('%Y-%m-%d-%H%M%S')) final_csv_name = '%s.csv' % base_name temp_csv_name = '%s.temp.csv' % base_name if is_first: self.storage.insert_object( final_csv_name, 'text/csv', writer.io.getvalue()) elif has_data: # Creates a temporary CSV file with new records, and append it to # the final CSV file. self.storage.insert_object( temp_csv_name, 'text/csv', writer.io.getvalue()) self.storage.compose_objects( [final_csv_name, temp_csv_name], final_csv_name, 'text/csv') if scan_completed: key = 'latest_%s_csv_object_name' % kind config.set_for_repo(repo, **{key: final_csv_name}) if not scan_completed: self.schedule_next_task(query.cursor(), timestamp)
def run_task_for_repo(self, repo): start_time = utils.get_utcnow() timestamp = self.params.timestamp or start_time is_first = not self.params.cursor query = model.Person.all_in_repo(repo).order('entry_date') if self.params.cursor: query.with_cursor(self.params.cursor) filtered_writer = record_writer.PersonWithNoteCsvWriter( StringIO.StringIO(), write_header=is_first) full_writer = record_writer.PersonWithNoteCsvWriter( StringIO.StringIO(), write_header=is_first) has_data = False scan_completed = False while True: persons = query.fetch(limit=FETCH_LIMIT) if persons: has_data = True else: scan_completed = True break full_records = self.get_person_records_with_notes(repo, persons) full_writer.write(full_records) filtered_records = copy.deepcopy(full_records) utils.filter_sensitive_fields(filtered_records) filtered_writer.write(filtered_records) if utils.get_utcnow() >= start_time + self.MAX_FETCH_TIME: break query.with_cursor(query.cursor()) for kind, writer in [('filtered', filtered_writer), ('full', full_writer)]: base_name = '%s-persons-%s-%s' % ( repo, kind, timestamp.strftime('%Y-%m-%d-%H%M%S')) final_csv_name = '%s.csv' % base_name temp_csv_name = '%s.temp.csv' % base_name if is_first: self.storage.insert_object(final_csv_name, 'text/csv', writer.io.getvalue()) elif has_data: # Creates a temporary CSV file with new records, and append it to # the final CSV file. self.storage.insert_object(temp_csv_name, 'text/csv', writer.io.getvalue()) self.storage.compose_objects([final_csv_name, temp_csv_name], final_csv_name, 'text/csv') if scan_completed: key = 'latest_%s_csv_object_name' % kind config.set_for_repo(repo, **{key: final_csv_name}) if not scan_completed: self.schedule_next_task(query.cursor(), timestamp)
def _set_api_access_control_config(self): if self._category_permissions['everything_else']: values = {} values['search_auth_key_required'] = ( self.params.search_auth_key_required) values['read_auth_key_required'] = ( self.params.read_auth_key_required) config.set_for_repo(self.env.repo, **values)
def setUp(self): super(SitemapViewTests, self).setUp() self.data_generator.repo(repo_id='haiti') self.data_generator.repo(repo_id='japan') self.data_generator.repo(repo_id='minnesota') # Set two of the repos as launched; the unlaunched one shouldn't appear # in the sitemap. config.set_for_repo('haiti', launched=True) config.set_for_repo('japan', launched=True)
def _set_language_config(self): values = {} if self._category_permissions['everything_else']: values['language_menu_options'] = self.params.lang_list values['repo_titles'] = self.params.repo_titles if self._category_permissions['custom_messages']: for config_key, value in self.params.custom_messages.items(): values[config_key] = value config.set_for_repo(self.env.repo, **values)
def _set_map_config(self): if self._category_permissions['everything_else']: values = {} values['map_default_center'] = simplejson.loads( self.params.map_default_center) values['map_default_zoom'] = self.params.map_default_zoom values['map_size_pixels'] = simplejson.loads( self.params.map_size_pixels) config.set_for_repo(self.env.repo, **values)
def test_set_allow_believed_dead_via_ui(self): """Verify the configuration of allow_believed_dead_via_ui.""" # Set allow_believed_dead_via_ui to be True config.set_for_repo('haiti', allow_believed_dead_via_ui=True) _, response, handler = self.handler_for_url('/haiti/start') assert handler.config.allow_believed_dead_via_ui == True # Set allow_believed_dead_via_ui to be False config.set_for_repo('haiti', allow_believed_dead_via_ui=False) _, response, handler = self.handler_for_url('/haiti/start') assert handler.config.allow_believed_dead_via_ui == False
def setUp(self): self.testbed = testbed.Testbed() self.testbed.activate() self.testbed.init_user_stub() model.Repo(key_name='haiti').put() config.set_for_repo( 'haiti', repo_titles={'en': 'Haiti Earthquake'}, language_menu_options=['en', 'ht', 'fr', 'es'], referrer_whitelist=[]) self.original_get_rendered = resources.get_rendered
def _set_data_retention_config(self): if self._category_permissions['everything_else']: if self._repo_obj.test_mode != self.params.test_mode: # TODO(nworden): stop setting test_mode in the config once we've # switched to using the field on the Repo object exclusively. self._repo_obj.test_mode = self.params.test_mode self._repo_obj.put() config.set_for_repo( self.env.repo, test_mode=self.params.test_mode, updated_date=utils.get_utcnow_timestamp())
def setUp(self): # Make it look like the dev server so HTTPS isn't required. os.environ['APPLICATION_ID'] = 'dev~abc' self.testbed = testbed.Testbed() self.testbed.activate() self.testbed.init_user_stub() model.Repo(key_name='haiti').put() config.set_for_repo('haiti', repo_titles={'en': 'Haiti Earthquake'}, language_menu_options=['en', 'ht', 'fr', 'es'], referrer_whitelist=[]) self.original_get_rendered = resources.get_rendered
def _set_activation_config(self): if self._category_permissions['everything_else']: if (self._repo_obj.activation_status != self.params.activation_status): self._repo_obj.activation_status = self.params.activation_status self._repo_obj.put() config.set_for_repo( self.env.repo, updated_date=utils.get_utcnow_timestamp()) config.set_for_repo( self.env.repo, deactivation_message_html=self.params.deactivation_message_html)
def setUp(self): # Make it look like the dev server so HTTPS isn't required. os.environ['APPLICATION_ID'] = 'dev~abc' self.testbed = testbed.Testbed() self.testbed.activate() self.testbed.init_user_stub() model.Repo(key_name='haiti').put() config.set_for_repo( 'haiti', repo_titles={'en': 'Haiti Earthquake'}, language_menu_options=['en', 'ht', 'fr', 'es'], referrer_whitelist=[]) self.original_get_rendered = resources.get_rendered
def test_default_language(self): """Verify that language_menu_options[0] is used as the default.""" request = setup_request("/haiti/start") handler = main.Main(request, webapp.Response()) assert handler.env.lang == "en" # first language in the options list assert django.utils.translation.get_language() == "en" config.set_for_repo("haiti", language_menu_options=["fr", "ht", "es"]) request = setup_request("/haiti/start") handler = main.Main(request, webapp.Response()) assert handler.env.lang == "fr" # first language in the options list assert django.utils.translation.get_language() == "fr"
def test_default_language(self): """Verify that language_menu_options[0] is used as the default.""" request = setup_request('/haiti/start') handler = main.Main(request, webapp.Response()) assert handler.env.lang == 'en' # first language in the options list assert django.utils.translation.get_language() == 'en' config.set_for_repo('haiti', language_menu_options=['fr', 'ht', 'es']) request = setup_request('/haiti/start') handler = main.Main(request, webapp.Response()) assert handler.env.lang == 'fr' # first language in the options list assert django.utils.translation.get_language() == 'fr'
def run_task_for_repo(self, repo): start_time = utils.get_utcnow() timestamp = self.params.timestamp or start_time base_name = '%s-persons-%s' % (repo, timestamp.strftime('%Y-%m-%d-%H%M%S')) is_first = not self.params.cursor query = model.Person.all_in_repo(repo).order('entry_date') if self.params.cursor: query.with_cursor(self.params.cursor) csv_io = StringIO.StringIO() writer = record_writer.PersonWithNoteCsvWriter(csv_io, write_header=is_first) has_data = False scan_completed = False while True: persons = query.fetch(limit=FETCH_LIMIT) if persons: has_data = True else: scan_completed = True break records = self.get_person_records_with_notes(repo, persons) # So far it only supports dump of records without sensitive fields. utils.filter_sensitive_fields(records) writer.write(records) if utils.get_utcnow() >= start_time + self.MAX_FETCH_TIME: break query.with_cursor(query.cursor()) final_csv_name = '%s.csv' % base_name temp_csv_name = '%s.temp.csv' % base_name if is_first: self.storage.insert_object(final_csv_name, 'text/csv', csv_io.getvalue()) elif has_data: # Creates a temporary CSV file with new records, and append it to # the final CSV file. self.storage.insert_object(temp_csv_name, 'text/csv', csv_io.getvalue()) self.storage.compose_objects([final_csv_name, temp_csv_name], final_csv_name, 'text/csv') if scan_completed: config.set_for_repo(repo, latest_csv_object_name=final_csv_name) else: self.schedule_next_task(query.cursor(), timestamp)
def test_repo_feed_deactivated_repo(self): config.set_for_repo('haiti', deactivated=True) doc = self.go('/haiti/feeds/repo') expected_content = '''\ <?xml version="1.0" encoding="UTF-8"?> <feed xmlns="http://www.w3.org/2005/Atom" xmlns:gpf="http://schemas.google.com/personfinder/2012" xmlns:georss="http://www.georss.org/georss"> <id>http://%s/personfinder/haiti/feeds/repo</id> <title>Person Finder Repository Feed</title> </feed> ''' % self.hostport assert expected_content == doc.content, \ text_diff(expected_content, doc.content)
def setup_repo_config(self, repo_id='haiti', updated_date=DEFAULT_REPO_UPDATED_DATE, language_menu_options=['en', 'es'], repo_titles={ 'en': 'Haiti', 'es': 'Haití' }, map_default_center=[123.45, 67.89]): config.set_for_repo(repo_id, updated_date=updated_date, language_menu_options=language_menu_options, repo_titles=repo_titles, map_default_center=map_default_center)
def _set_forms_config(self): if self._category_permissions['everything_else']: values = {} values['use_family_name'] = self.params.use_family_name values['family_name_first'] = self.params.family_name_first values['use_alternate_names'] = self.params.use_alternate_names values['use_postal_code'] = self.params.use_postal_code values['allow_believed_dead_via_ui'] = ( self.params.allow_believed_dead_via_ui) values['min_query_word_length'] = self.params.min_query_word_length values['show_profile_entry'] = self.params.show_profile_entry values['profile_websites'] = simplejson.loads( self.params.profile_websites) config.set_for_repo(self.env.repo, **values)
def test_config_namespaces(self): # Tests the cache's ability to retrieve global or repository-specific # configuration entries. cfg_sub = config.Configuration('_foo') cfg_global = config.Configuration('*') config.set_for_repo('*', captcha_private_key='global_abcd', captcha_public_key='global_efgh', translate_api_key='global_hijk') assert cfg_global.captcha_private_key == 'global_abcd' assert cfg_global.captcha_public_key == 'global_efgh' assert cfg_global.translate_api_key == 'global_hijk' config.set_for_repo('_foo', captcha_private_key='abcd', captcha_public_key='efgh') assert cfg_sub.captcha_private_key == 'abcd' assert cfg_sub.captcha_public_key == 'efgh' # If a key isn't present for a repository, its value for # the global domain is retrieved. assert cfg_sub.translate_api_key == 'global_hijk'
def test_repo_feed_all_launched_repos(self): config.set_for_repo('haiti', deactivated=True, launched=True, test_mode=False) config.set_for_repo('japan', deactivated=False, launched=True, test_mode=True, updated_date=utils.get_timestamp( datetime.datetime(2012, 03, 11))) config.set_for_repo('pakistan', deactivated=False, launched=False, test_mode=False) # 'haiti', 'japan', and 'pakistan' exist in the datastore. Only those # which are 'launched' and not 'deactivated' i.e., only 'japan' should # appear in the feed. doc = self.go('/global/feeds/repo') expected_content = u'''\ <?xml version="1.0" encoding="UTF-8"?> <feed xmlns="http://www.w3.org/2005/Atom" xmlns:gpf="http://schemas.google.com/personfinder/2012" xmlns:georss="http://www.georss.org/georss"> <id>http://%s/personfinder/global/feeds/repo</id> <title>Person Finder Repository Feed</title> <updated>2012-03-11T00:00:00Z</updated> <entry> <id>%s/japan</id> <published>2011-03-11T00:00:00Z</published> <updated>2012-03-11T00:00:00Z</updated> <title xml:lang="ja">2011 日本地震</title> <content type="text/xml"> <gpf:repo> <gpf:title xml:lang="ja">2011 日本地震</gpf:title> <gpf:title xml:lang="en">2011 Japan Earthquake</gpf:title> <gpf:title xml:lang="ko"></gpf:title> <gpf:title xml:lang="zh-CN">2011 日本地震</gpf:title> <gpf:title xml:lang="zh-TW">2011 日本地震</gpf:title> <gpf:title xml:lang="pt-BR">2011 Terremoto no Japão</gpf:title> <gpf:title xml:lang="es">2011 Terremoto en Japón</gpf:title> <gpf:read_auth_key_required>true</gpf:read_auth_key_required> <gpf:search_auth_key_required>true</gpf:search_auth_key_required> <gpf:test_mode>true</gpf:test_mode> <gpf:location> <georss:point>38 140.7</georss:point> </gpf:location> </gpf:repo> </content> </entry> </feed> ''' % (self.hostport, ROOT_URL) assert expected_content == doc.content, \ text_diff(expected_content, doc.content) # verify we logged the repo read. self.verify_api_log(ApiActionLog.REPO, api_key='')
def create_repo(self, repo): logging.info('Create repo: %s', repo) db.put([model.Repo(key_name=repo)]) # Provides some defaults. config.set_for_repo( repo, language_menu_options=['en'], repo_titles={'en': repo}, keywords='', use_family_name=True, use_alternate_names=True, use_postal_code=True, allow_believed_dead_via_ui=False, min_query_word_length=1, show_profile_entry=False, profile_websites=[], map_default_zoom=6, map_default_center=[0, 0], map_size_pixels=[400, 280], read_auth_key_required=True, search_auth_key_required=True, deactivated=False, launched=False, deactivation_message_html='', start_page_custom_htmls={}, results_page_custom_htmls={}, view_page_custom_htmls={}, seek_query_form_custom_htmls={}, footer_custom_htmls={}, bad_words='', published_date=0.0, updated_date=0.0, test_mode=False, force_https=False, zero_rating_mode=False, )
def configure_api_logging(self, repo='*', enable=True): db.delete(ApiActionLog.all()) config.set_for_repo(repo, api_action_logging=enable)
def _set_notification_config(self): config.set_for_repo( '*', notification_email=self.params.notification_email, unreviewed_notes_threshold=self.params.unreviewed_notes_threshold)
def test_whitelisted_referrer(self): config.set_for_repo('haiti', referrer_whitelist=['a.org']) _, _, handler = self.handler_for_url('/haiti?referrer=a.org') assert handler.params.referrer == 'a.org'
def _set_gtranslate_config(self): config.set_for_repo( '*', translate_api_key=self.params.translate_api_key)
def tearDown(self): config.set_for_repo('haiti', deactivated=False) config.set_for_repo('japan', test_mode=False) ServerTestsBase.tearDown(self)
def post(self, request, *args, **kwargs): """Serves POST requests, creating a new repo. Creates a new repository and sets some default values (assuming the user has permission and a valid XSRF token). Args: request: Unused. *args: Unused. **kwargs: Unused. Returns: HttpResponse: A redirect to the new repo's admin page. """ del request, args, kwargs # unused self.enforce_xsrf(self.ACTION_ID) new_repo = self.params.new_repo model.Repo( key_name=new_repo, activation_status=model.Repo.ActivationStatus.STAGING, test_mode=False).put() # Provide some defaults. config.set_for_repo( new_repo, language_menu_options=['en', 'fr'], repo_titles={ 'en': 'Earthquake', 'fr': u'S\xe9isme' }, keywords='person finder, people finder, person, people, ' + 'crisis, survivor, family', use_family_name=True, use_alternate_names=True, use_postal_code=True, allow_believed_dead_via_ui=False, min_query_word_length=2, show_profile_entry=False, profile_websites=const.DEFAULT_PROFILE_WEBSITES, map_default_zoom=6, map_default_center=[0, 0], map_size_pixels=[400, 280], read_auth_key_required=True, search_auth_key_required=True, deactivated=False, launched=False, deactivation_message_html='', start_page_custom_htmls={ 'en': '', 'fr': '' }, results_page_custom_htmls={ 'en': '', 'fr': '' }, view_page_custom_htmls={ 'en': '', 'fr': '' }, seek_query_form_custom_htmls={ 'en': '', 'fr': '' }, footer_custom_htmls={ 'en': '', 'fr': '' }, bad_words='', published_date=utils.get_utcnow_timestamp(), updated_date=utils.get_utcnow_timestamp(), test_mode=False, force_https=True, zero_rating_mode=False, time_zone_offset=0, time_zone_abbreviation='UTC', ) return django.shortcuts.redirect( self.build_absolute_path('/%s/admin' % new_repo))
def _set_gmaps_config(self): config.set_for_repo('*', maps_api_key=self.params.maps_api_key)
def setUp(self): super(AdminGlobalIndexViewTests, self).setUp() self.data_generator.repo() config.set_for_repo('*', **AdminGlobalIndexViewTests._PRIOR_CONFIG) self.login_as_superadmin()
def test_whitelisted_referrer(self): config.set_for_repo('haiti', referrer_whitelist=['a.org']) _, _, handler = self.handler_for_url( '/haiti?referrer=a.org') assert handler.params.referrer == 'a.org'
def setUp(self): ServerTestsBase.setUp(self) config.set_for_repo( 'haiti', api_action_logging=True) self.filename = None
def setUp(self): super(AdminRepoIndexViewTests, self).setUp() self.data_generator.repo() config.set_for_repo('haiti', **AdminRepoIndexViewTests._PRIOR_CONFIG)