def test_data_expires_after_ttl(self): cache = resources.RamCache() cache.put('a', 'b', 10) utils.set_utcnow_for_test(9.99) assert cache.get('a') == 'b' utils.set_utcnow_for_test(10.01) assert cache.get('a') is None
def test_get_localized(self): get_localized = resources.get_localized # These three fetches should load resources into the cache. self.fetched = [] assert get_localized('static.html', 'es').content == 'hello' assert self.fetched == ['static.html:es', 'static.html'] self.fetched = [] assert get_localized('static.html', 'en').content == 'hello' assert self.fetched == ['static.html:en', 'static.html'] self.fetched = [] assert get_localized('static.html', 'fr').content == 'bonjour' assert self.fetched == ['static.html:fr'] # These should now be cache hits, and shouldn't touch the datastore. self.fetched = [] assert get_localized('static.html', 'es').content == 'hello' assert get_localized('static.html', 'en').content == 'hello' assert get_localized('static.html', 'fr').content == 'bonjour' assert self.fetched == [] # Expire static.html:fr from the cache. utils.set_utcnow_for_test(21) self.fetched = [] assert get_localized('static.html', 'es').content == 'hello' assert get_localized('static.html', 'en').content == 'hello' assert self.fetched == [] assert get_localized('static.html', 'fr').content == 'bonjour' assert self.fetched == ['static.html:fr']
def test_put_expiry_flags(self): # Try put_expiry_flags when the record has not expired yet. assert not self.p1.is_expired self.p1.put_expiry_flags() # Both entities should be unexpired. p1 = db.get(self.p1.key()) assert p1.expiry_date assert not p1.is_expired assert p1.given_name == 'John' n1_1 = db.get(self.n1_1.key()) assert not n1_1.is_expired # Advance past the expiry date and try again. set_utcnow_for_test(datetime(2010, 2, 3)) p1.put_expiry_flags() # Both entities should be expired. p1 = db.get(self.p1.key()) assert p1.is_expired assert p1.given_name == 'John' assert p1.source_date == datetime(2010, 2, 3) assert p1.entry_date == datetime(2010, 2, 3) assert p1.expiry_date == datetime(2010, 2, 1) n1_1 = db.get(self.n1_1.key()) assert n1_1.is_expired
def test_get_default_expiration_date(self): """Tests that the expected default expiration date is correctly.""" self.login_as_manager() utils.set_utcnow_for_test(datetime.datetime(2010, 1, 5)) res = self.client.get('/haiti/admin/acls/', secure=True) self.assertEqual(res.context['default_expiration_date'], datetime.datetime(2011, 1, 5))
def test_data_expires_after_ttl(self): cache = resources.RamCache() cache.put("a", "b", 10) utils.set_utcnow_for_test(9.99) assert cache.get("a") == "b" utils.set_utcnow_for_test(10.01) assert cache.get("a") is None
def testRejectsExpiredToken(self): config.set(xsrf_token_key='abcdef') tool = utils.XsrfTool() token = tool.generate_token(12345, 'test_action') utils.set_utcnow_for_test(XsrfToolTests.TEST_NOW + datetime.timedelta(hours=4, minutes=1)) self.assertFalse(tool.verify_token(token, 12345, 'test_action'))
def test_expired_records_with_unremoved_top_level_note(self): """validate_expired_records_removed should return a list with messages for each expired record that still had a note referring to its person_record_id""" validator = (self.set_up_validator( PfifXml.XML_EXPIRE_99_HAS_NOTE_DATA)) utils.set_utcnow_for_test(ValidatorTests.EXPIRED_TIME) self.assertEqual(len(validator.validate_expired_records_removed()), 1)
def test_expiration_placeholder_with_bad_source_entry_date(self): """validate_expired_records_removed should return a list with the person_record_ids of all expired records whose source_date and entry_date are not the same value and are not created within a day after expiration""" validator = self.set_up_validator( PfifXml.XML_EXPIRE_99_NO_DATA_NONSYNCED_DATES) utils.set_utcnow_for_test(ValidatorTests.EXPIRED_TIME) self.assertEqual(len(validator.validate_expired_records_removed()), 2)
def test_rejects_expired_token(self): """Tests that an expired token is rejected.""" config.set(xsrf_token_key='abcdef') tool = utils.XsrfTool() token = tool.generate_token(12345, 'test_action') utils.set_utcnow_for_test(XsrfToolTests.TEST_NOW + datetime.timedelta(hours=4, minutes=1)) self.assertFalse(tool.verify_token(token, 12345, 'test_action'))
def test_expired_records_with_unremoved_top_level_note(self): """validate_expired_records_removed should return a list with messages for each expired record that still had a note referring to its person_record_id""" validator = ( self.set_up_validator(PfifXml.XML_EXPIRE_99_HAS_NOTE_DATA)) utils.set_utcnow_for_test(ValidatorTests.EXPIRED_TIME) self.assertEqual(len(validator.validate_expired_records_removed()), 1)
def test_get_default_expiration_date(self): """Tests that the expected default expiration date is correctly.""" self.login_as_manager() utils.set_utcnow_for_test(datetime.datetime(2010, 1, 5)) res = self.client.get('/haiti/admin/acls/', secure=True) self.assertEqual( res.context['default_expiration_date'], datetime.datetime(2011, 1, 5))
def test_past_due(self): """Make sure Person records are detected as past due correctly.""" def assert_past_due_count(expected): assert len(list(model.Person.past_due_records())) == expected assert_past_due_count(0) set_utcnow_for_test(datetime(2010, 2, 15)) assert_past_due_count(1) set_utcnow_for_test(datetime(2010, 3, 15)) assert_past_due_count(2)
def tearDown(self): utils.set_utcnow_for_test(None) resources.clear_caches() Resource.get_by_key_name = self.resource_get_by_key_name_original webapp.template.Template.__init__ = self.template_init_original webapp.template.Template.render = self.template_render_original db.delete(self.temp_entity_keys)
def test_clean_up_in_test_mode_multi_tasks(self): """Test the clean up in test mode when it is broken into multiple tasks.""" class Listener(object): def before_deletion(self, person): # This will be implemented later using mock. assert False tasks.CleanUpInTestMode.DELETION_AGE_SECONDS = 2 * 3600 # 2 hours utcnow = datetime.datetime(2010, 1, 1, 7, 0, 0) set_utcnow_for_test(utcnow) self.mox = mox.Mox() cleanup = \ test_handler.initialize_handler(tasks.CleanUpInTestMode, tasks.CleanUpInTestMode.ACTION) listener = Listener() cleanup.set_listener(listener) # Simulates add_task_for_repo() because it doesn't work in unit tests. def add_task_for_repo(repo, task_name, action, **kwargs): cleanup = test_handler.initialize_handler(tasks.CleanUpInTestMode, action, repo=repo, params=kwargs) cleanup.set_listener(listener) cleanup.get() self.mox.StubOutWithMock(cleanup, 'add_task_for_repo') (cleanup.add_task_for_repo('haiti', mox.IsA(str), mox.IsA(str), utcnow=str( calendar.timegm(utcnow.utctimetuple())), cursor=mox.IsA(str), queue_name=mox.IsA(str)).WithSideEffects( add_task_for_repo).MultipleTimes()) def raise_deadline_exceeded_error(_): raise runtime.DeadlineExceededError() self.mox.StubOutWithMock(listener, 'before_deletion') listener.before_deletion(self.key_p1) listener.before_deletion( self.key_p2).WithSideEffects(raise_deadline_exceeded_error) listener.before_deletion(self.key_p2) self.mox.ReplayAll() config.set(test_mode=True, repo='haiti') # This should run multiple tasks and finally deletes all records. cleanup.get() assert db.get(self.key_p1) is None assert db.get(self.key_p2) is None self.mox.UnsetStubs() self.mox.VerifyAll()
def test_no_expiration_without_date(self): """validate_expired_records_removed should return an empty list when the there isn't an expiry_date""" validator = self.set_up_validator(PfifXml.XML_NO_EXPIRY_DATE) utils.set_utcnow_for_test(ValidatorTests.EXPIRED_TIME) self.assertEqual(len(validator.validate_expired_records_removed()), 0) validator = self.set_up_validator(PfifXml.XML_EMPTY_EXPIRY_DATE) utils.set_utcnow_for_test(ValidatorTests.EXPIRED_TIME) self.assertEqual(len(validator.validate_expired_records_removed()), 0)
def test_clean_up_in_test_mode_multi_tasks(self): """Test the clean up in test mode when it is broken into multiple tasks.""" class Listener(object): def before_deletion(self, person): # This will be implemented later using mock. assert False tasks.CleanUpInTestMode.DELETION_AGE_SECONDS = 2 * 3600 # 2 hours utcnow = datetime.datetime(2010, 1, 1, 7, 0, 0) set_utcnow_for_test(utcnow) self.mox = mox.Mox() cleanup = \ test_handler.initialize_handler(tasks.CleanUpInTestMode, tasks.CleanUpInTestMode.ACTION) listener = Listener() cleanup.set_listener(listener) # Simulates add_task_for_repo() because it doesn't work in unit tests. def add_task_for_repo(repo, task_name, action, **kwargs): cleanup = test_handler.initialize_handler( tasks.CleanUpInTestMode, action, repo=repo, params=kwargs) cleanup.set_listener(listener) cleanup.get() self.mox.StubOutWithMock(cleanup, 'add_task_for_repo') (cleanup.add_task_for_repo( 'haiti', mox.IsA(str), mox.IsA(str), utcnow=str(calendar.timegm(utcnow.utctimetuple())), cursor=mox.IsA(str), queue_name=mox.IsA(str)). WithSideEffects(add_task_for_repo).MultipleTimes()) def raise_deadline_exceeded_error(_): raise runtime.DeadlineExceededError() self.mox.StubOutWithMock(listener, 'before_deletion') listener.before_deletion(self.key_p1) listener.before_deletion(self.key_p2).WithSideEffects( raise_deadline_exceeded_error) listener.before_deletion(self.key_p2) self.mox.ReplayAll() config.set(test_mode=True, repo='haiti') # This should run multiple tasks and finally deletes all records. cleanup.get() assert db.get(self.key_p1) is None assert db.get(self.key_p2) is None self.mox.UnsetStubs() self.mox.VerifyAll()
def setUp(self): set_utcnow_for_test(datetime(2010, 1, 1)) self.p1 = model.Person.create_original( 'haiti', first_name='John', last_name='Smith', home_street='Washington St.', home_city='Los Angeles', home_state='California', home_postal_code='11111', home_neighborhood='Good Neighborhood', author_name='Alice Smith', author_phone='111-111-1111', author_email='*****@*****.**', source_url='https://www.source.com', source_date=datetime(2010, 1, 1), source_name='Source Name', entry_date=datetime(2010, 1, 1), expiry_date=datetime(2010, 2, 1), other='') self.p2 = model.Person.create_original( 'haiti', first_name='Tzvika', last_name='Hartman', home_street='Herzl St.', home_city='Tel Aviv', home_state='Israel', entry_date=datetime(2010, 1, 1), expiry_date=datetime(2010, 3, 1), other='') self.key_p1 = db.put(self.p1) self.key_p2 = db.put(self.p2) self.n1_1 = model.Note.create_original( 'haiti', person_record_id=self.p1.record_id, linked_person_record_id=self.p2.record_id, status=u'believed_missing', found=False, entry_date=get_utcnow(), source_date=datetime(2000, 1, 1)) self.n1_2 = model.Note.create_original( 'haiti', person_record_id=self.p1.record_id, found=True, entry_date=get_utcnow(), source_date=datetime(2000, 2, 2)) self.key_n1_1 = db.put(self.n1_1) self.key_n1_2 = db.put(self.n1_2) # Update the Person entity according to the Note. self.p1.update_from_note(self.n1_1) self.p1.update_from_note(self.n1_2) db.put(self.p1)
def test_past_due(self): """Make sure Person records are detected as past due correctly.""" def assert_past_due_count(expected): actual = len(list(model.Person.past_due_records(repo='haiti'))) assert actual == expected assert_past_due_count(0) set_utcnow_for_test(datetime(2010, 2, 15)) assert_past_due_count(1) set_utcnow_for_test(datetime(2010, 3, 15)) assert_past_due_count(2)
def test_unexpired_records(self): """validate_expired_records_removed should return an empty list when no records are expired""" validator = self.set_up_validator( PfifXml.XML_EXPIRE_99_HAS_DATA_NONSYNCED_DATES) not_expired_1998 = datetime.datetime(1998, 11, 1, 1, 1, 1, 1) utils.set_utcnow_for_test(not_expired_1998) self.assertEqual(len(validator.validate_expired_records_removed()), 0) just_not_expired = datetime.datetime(1999, 2, 4, 4, 5, 5, 0) utils.set_utcnow_for_test(just_not_expired) self.assertEqual(len(validator.validate_expired_records_removed()), 0)
def setUp(self): logging.basicConfig(level=logging.INFO, stream=sys.stderr) self.mox = None # Setup cheerfully stolen from test_model. set_utcnow_for_test(datetime.datetime(2010, 1, 1)) self.photo = model.Photo.create('haiti', image_data='xyz') self.photo.put() self.photo_key = self.photo.key() self.p1 = model.Person.create_original( 'haiti', given_name='John', family_name='Smith', home_street='Washington St.', home_city='Los Angeles', home_state='California', home_postal_code='11111', home_neighborhood='Good Neighborhood', author_name='Alice Smith', author_phone='111-111-1111', author_email='*****@*****.**', photo_url='', photo=self.photo, source_url='https://www.source.com', source_date=datetime.datetime(2010, 1, 1), source_name='Source Name', entry_date=datetime.datetime(2010, 1, 1), expiry_date=datetime.datetime(2010, 2, 1), other='') self.p2 = model.Person.create_original( 'haiti', given_name='Tzvika', family_name='Hartman', home_street='Herzl St.', home_city='Tel Aviv', home_state='Israel', source_date=datetime.datetime(2010, 1, 1), entry_date=datetime.datetime(2010, 1, 1), expiry_date=datetime.datetime(2010, 3, 1), other='') self.key_p1 = db.put(self.p1) self.key_p2 = db.put(self.p2) self.n1_1 = model.Note.create_original( 'haiti', person_record_id=self.p1.record_id, linked_person_record_id=self.p2.record_id, status=u'believed_missing', author_made_contact=False, entry_date=get_utcnow(), source_date=datetime.datetime(2010, 1, 2)) self.note_id = self.n1_1.note_record_id db.put(self.n1_1) self.to_delete = [self.p1, self.p2, self.n1_1, self.photo]
def test_task(self): utils.set_utcnow_for_test(datetime.datetime(2010, 4, 2)) self.run_task('/haiti/tasks/cleanup_stray_notes', data={}, method='POST') notes_q = model.Note.all() # Note #1 should be kept because it's associated with an existing Person # record, and note #2 should be kept because it's within the grace # period. self.assertEqual(2, notes_q.count()) notes = notes_q[:2] self.assertEqual(sorted([n.key() for n in notes]), sorted([self.note1.key(), self.note2.key()]))
def test_edit_data_retention_mode_config(self): # Set the time to an hour past the original update_date. utils.set_utcnow_for_test(datetime.datetime(2019, 5, 10, 12, 15, 0)) self.login_as_superadmin() self._post_with_params(test_mode=True) repo = model.Repo.get_by_key_name('haiti') self.assertTrue(repo.test_mode) repo_conf = config.Configuration('haiti') self.assertIs(repo_conf.test_mode, True) self.assertEqual( repo_conf.updated_date, utils.get_timestamp(datetime.datetime(2019, 5, 10, 12, 15, 0)))
def setUp(self): utils.set_utcnow_for_test(0) resources.clear_caches() resources.set_active_bundle_name('1') self.temp_entity_keys = [] self.put_resource('1', 'base.html.template', 50, 'hi! {% block foo %}{% endblock foo %}') self.put_resource('1', 'base.html.template:es', 40, '\xc2\xa1hola! {% block foo %}{% endblock foo %}') self.put_resource( '1', 'page.html.template', 30, '{% extends "base.html.template" %} ' '{% block foo %}default{% endblock foo %}') self.put_resource( '1', 'page.html.template:fr', 20, '{% extends "base.html.template" %} ' '{% block foo %}fran\xc3\xa7ais{% endblock foo %}') self.put_resource('1', 'static.html', 30, 'hello') self.put_resource('1', 'static.html:fr', 20, 'bonjour') self.put_resource('1', 'data', 10, '\xff\xfe\xfd\xfc') self.fetched = [] self.compiled = [] self.rendered = [] self.resource_get_by_key_name_original = Resource.get_by_key_name self.template_init_original = django.template.Template.__init__ self.template_render_original = django.template.Template.render test_self = self @staticmethod def resource_get_by_key_name_for_test(key_name, parent, *args, **kwargs): test_self.fetched.append(key_name) # track datastore fetches return test_self.resource_get_by_key_name_original( key_name, parent, *args, **kwargs) def template_init_for_test(self, content, origin, name, *args, **kwargs): test_self.compiled.append(name) # track template compilations return test_self.template_init_original(self, content, origin, name, *args, **kwargs) def template_render_for_test(self, context, *args, **kwargs): test_self.rendered.append(self.name) # track render calls return test_self.template_render_original(self, context, *args, **kwargs) Resource.get_by_key_name = resource_get_by_key_name_for_test django.template.Template.__init__ = template_init_for_test django.template.Template.render = template_render_for_test
def get(self): utcnow_before_change = get_utcnow() utcnow = self.params.utcnow if self.is_test_mode(): try: logging.info('Setting utcnow to %r' % utcnow) set_utcnow_for_test(utcnow) self.render('templates/set_utcnow.html', utcnow=get_utcnow(), utcbefore=utcnow_before_change) except Exception, e: # bad param. return self.error(400, 'bad timestamp %s, e=%s' % (utcnow, e))
def test_task(self): utils.set_utcnow_for_test(datetime.datetime(2010, 4, 2)) self.run_task('/haiti/tasks/cleanup_stray_subscriptions', data={}, method='POST') subs_q = model.Subscription.all() # Subscription #1 should be kept because it's associated with an # existing Person record, and subscription #2 should be kept because # it's within the grace period. self.assertEqual(2, subs_q.count()) subs = subs_q[:2] self.assertEqual(sorted([s.key() for s in subs]), sorted([self.subscription1.key(), self.subscription2.key()]))
def setUp(self): utils.set_utcnow_for_test(0) resources.clear_caches() resources.set_active_bundle_name('1') self.temp_entity_keys = [] self.put_resource('1', 'base.html.template', 50, 'hi! {% block foo %}{% endblock foo %}') self.put_resource('1', 'base.html.template:es', 40, '\xc2\xa1hola! {% block foo %}{% endblock foo %}') self.put_resource('1', 'page.html.template', 30, '{% extends "base.html.template" %} ' '{% block foo %}default{% endblock foo %}') self.put_resource('1', 'page.html.template:fr', 20, '{% extends "base.html.template" %} ' '{% block foo %}fran\xc3\xa7ais{% endblock foo %}') self.put_resource('1', 'static.html', 30, 'hello') self.put_resource('1', 'static.html:fr', 20, 'bonjour') self.put_resource('1', 'data', 10, '\xff\xfe\xfd\xfc') self.fetched = [] self.compiled = [] self.rendered = [] self.resource_get_by_key_name_original = Resource.get_by_key_name self.template_init_original = django.template.Template.__init__ self.template_render_original = django.template.Template.render test_self = self @staticmethod def resource_get_by_key_name_for_test( key_name, parent, *args, **kwargs): test_self.fetched.append(key_name) # track datastore fetches return test_self.resource_get_by_key_name_original( key_name, parent, *args, **kwargs) def template_init_for_test( self, content, origin, name, *args, **kwargs): test_self.compiled.append(name) # track template compilations return test_self.template_init_original( self, content, origin, name, *args, **kwargs) def template_render_for_test(self, context, *args, **kwargs): test_self.rendered.append(self.name) # track render calls return test_self.template_render_original( self, context, *args, **kwargs) Resource.get_by_key_name = resource_get_by_key_name_for_test django.template.Template.__init__ = template_init_for_test django.template.Template.render = template_render_for_test
def test_set_utcnow_for_test(self): max_delta = datetime.timedelta(0, 0, 100) utcnow = datetime.datetime.utcnow() utilsnow = utils.get_utcnow() # max sure we're getting the current time. assert (utilsnow - utcnow) < max_delta # now set the utils time. test_time = datetime.datetime(2011, 1, 1, 0, 0) utils.set_utcnow_for_test(test_time) assert utils.get_utcnow() == test_time # now unset. utils.set_utcnow_for_test(None) assert utils.get_utcnow() assert utils.get_utcnow() != test_time
def test_set_utcnow_for_test(self): max_delta = datetime.timedelta(0,0,100) utcnow = datetime.datetime.utcnow() utilsnow = utils.get_utcnow() # max sure we're getting the current time. assert (utilsnow - utcnow) < max_delta # now set the utils time. test_time = datetime.datetime(2011, 1, 1, 0, 0) utils.set_utcnow_for_test(test_time) assert utils.get_utcnow() == test_time # now unset. utils.set_utcnow_for_test(None) assert utils.get_utcnow() assert utils.get_utcnow() != test_time
def setUp(self): utils.set_utcnow_for_test(0) resources.clear_caches() resources.set_active_bundle_name("1") self.temp_entity_keys = [] self.put_resource("1", "base.html.template", 50, "hi! {% block foo %}{% endblock foo %}") self.put_resource("1", "base.html.template:es", 40, "\xc2\xa1hola! {% block foo %}{% endblock foo %}") self.put_resource( "1", "page.html.template", 30, '{% extends "base.html.template" %} ' "{% block foo %}default{% endblock foo %}", ) self.put_resource( "1", "page.html.template:fr", 20, '{% extends "base.html.template" %} ' "{% block foo %}fran\xc3\xa7ais{% endblock foo %}", ) self.put_resource("1", "static.html", 30, "hello") self.put_resource("1", "static.html:fr", 20, "bonjour") self.put_resource("1", "data", 10, "\xff\xfe\xfd\xfc") self.fetched = [] self.compiled = [] self.rendered = [] self.resource_get_by_key_name_original = Resource.get_by_key_name self.template_init_original = django.template.Template.__init__ self.template_render_original = django.template.Template.render test_self = self @staticmethod def resource_get_by_key_name_for_test(key_name, parent): test_self.fetched.append(key_name) # track datastore fetches return test_self.resource_get_by_key_name_original(key_name, parent) def template_init_for_test(self, content, origin, name): test_self.compiled.append(name) # track template compilations return test_self.template_init_original(self, content, origin, name) def template_render_for_test(self, context): test_self.rendered.append(self.name) # track render calls return test_self.template_render_original(self, context) Resource.get_by_key_name = resource_get_by_key_name_for_test django.template.Template.__init__ = template_init_for_test django.template.Template.render = template_render_for_test
def test_task(self): utils.set_utcnow_for_test(datetime.datetime(2010, 4, 2)) self.run_task('/haiti/tasks/cleanup_stray_subscriptions', data={}, method='POST') subs_q = model.Subscription.all() # Subscription #1 should be kept because it's associated with an # existing Person record, and subscription #2 should be kept because # it's within the grace period. self.assertEqual(2, subs_q.count()) subs = subs_q[:2] self.assertEqual( sorted([s.key() for s in subs]), sorted([self.subscription1.key(), self.subscription2.key()]))
def test_wipe_contents(self): # Advance past the expiry date. set_utcnow_for_test(datetime(2010, 2, 3)) self.p1.put_expiry_flags() # Try wiping the contents. self.p1.wipe_contents() p1 = db.get(self.p1.key()) assert p1.is_expired assert p1.first_name == None assert p1.source_date == datetime(2010, 2, 3) assert p1.entry_date == datetime(2010, 2, 3) assert p1.expiry_date == datetime(2010, 2, 1) assert not db.get(self.n1_1.key())
def test_edit_activation_status_config(self): # Set the time to an hour past the original update_date. utils.set_utcnow_for_test(datetime.datetime(2019, 5, 10, 12, 15, 0)) self.login_as_superadmin() self._post_with_params(activation_status=str( model.Repo.ActivationStatus.DEACTIVATED), deactivation_message_html='it is deactivated') repo = model.Repo.get_by_key_name('haiti') self.assertEqual(repo.activation_status, model.Repo.ActivationStatus.DEACTIVATED) repo_conf = config.Configuration('haiti') self.assertEqual(repo_conf.deactivation_message_html, 'it is deactivated') self.assertEqual( repo_conf.updated_date, utils.get_timestamp(datetime.datetime(2019, 5, 10, 12, 15, 0)))
def set_utcnow_for_test(self, new_utcnow, flush=''): """Sets the utils.get_utcnow() clock locally and on the server, and optionally also flushes caches on the server. Args: new_utcnow: A datetime, timestamp, or None to revert to real time. flush: Names of caches to flush (see main.flush_caches). """ if new_utcnow is None: param = 'real' elif isinstance(new_utcnow, (int, float)): param = str(new_utcnow) else: param = calendar.timegm(new_utcnow.utctimetuple()) path = '/?utcnow=%s&flush=%s' % (param, flush) # Requesting '/' gives a fast redirect; to save time, don't follow it. scrape.Session(verbose=0).go(self.path_to_url(path), redirects=0) utils.set_utcnow_for_test(new_utcnow)
def initialize(self, request, response): webapp.RequestHandler.initialize(self, request, response) # If requested, set the clock before doing anything clock-related. # Only works on localhost for testing. Specify ?utcnow=1293840000 to # set the clock to 2011-01-01, or ?utcnow=real to revert to real time. utcnow = request.get('utcnow') if request.remote_addr == '127.0.0.1' and utcnow: if utcnow == 'real': utils.set_utcnow_for_test(None) else: utils.set_utcnow_for_test(float(utcnow)) # If requested, flush caches before we touch anything that uses them. flush_caches(*request.get('flush', '').split(',')) # check for legacy redirect: # TODO(lschumacher|kpy): remove support for legacy URLS Q1 2012. if legacy_redirect.do_redirect(self): # stub out get/head to prevent failures. self.get = self.head = lambda *args: None return legacy_redirect.redirect(self) # Gather commonly used information into self.env. self.env = setup_env(request) # Force a redirect if requested, except where https is not supported: # - for cron jobs # - for task queue jobs # - in development if (self.env.force_https and self.env.scheme == 'http' and not is_cron_task(self.request) and not is_task_queue_task(self.request) and not is_development_server()): self.redirect(self.env.url.replace('http:', 'https:')) # Activate the selected language. response.headers['Content-Language'] = self.env.lang response.headers['Set-Cookie'] = \ 'django_language=%s; path=/' % self.env.lang django_setup.activate(self.env.lang) # Activate the appropriate resource bundle. resources.set_active_bundle_name(self.env.resource_bundle)
def setUp(self): super(ProcessExpirationsTaskTests, self).setUp() self.data_generator.repo(repo_id='haiti') logging.basicConfig(level=logging.INFO, stream=sys.stderr) self.mox = None utils.set_utcnow_for_test(datetime.datetime(2010, 1, 1)) # TODO(nworden): factor out a general-purpose utility for creation of # test entities. self.photo = self.data_generator.photo() self.photo_key = self.photo.key() self.p1 = self.data_generator.person_a(photo=self.photo) self.p2 = self.data_generator.person_b() self.key_p1 = self.p1.key() self.key_p2 = self.p2.key() self.n1_1 = self.data_generator.note(person_id=self.p1.record_id) self.note_id = self.n1_1.note_record_id self.to_delete = [self.p1, self.p2, self.n1_1, self.photo]
def test_wipe_contents(self): # Advance past the expiry date. set_utcnow_for_test(datetime(2010, 2, 3)) # original_creation_date is auto_now_add, so we override it here. self.p1.original_creation_date = datetime(2010, 1, 3) self.p1.put() self.p1.put_expiry_flags() # Try wiping the contents. self.p1.wipe_contents() p1 = db.get(self.p1.key()) assert p1.is_expired assert p1.given_name == None assert p1.source_date == datetime(2010, 2, 3) assert p1.entry_date == datetime(2010, 2, 3) # verify we preserve the original_creation_date assert p1.original_creation_date == datetime(2010, 1, 3) assert p1.expiry_date == datetime(2010, 2, 1) assert not db.get(self.n1_1.key())
def test_get_localized(self): get_localized = resources.get_localized # These three fetches should load resources into the cache. self.fetched = [] assert get_localized("static.html", "es").content == "hello" assert self.fetched == ["static.html:es", "static.html"] self.fetched = [] assert get_localized("static.html", "en").content == "hello" assert self.fetched == ["static.html:en", "static.html"] self.fetched = [] assert get_localized("static.html", "fr").content == "bonjour" assert self.fetched == ["static.html:fr"] # These should now be cache hits, and shouldn't touch the datastore. self.fetched = [] assert get_localized("static.html", "es").content == "hello" assert get_localized("static.html", "en").content == "hello" assert get_localized("static.html", "fr").content == "bonjour" assert self.fetched == [] # Expire static.html:fr from the cache. utils.set_utcnow_for_test(21) self.fetched = [] assert get_localized("static.html", "es").content == "hello" assert get_localized("static.html", "en").content == "hello" assert self.fetched == [] assert get_localized("static.html", "fr").content == "bonjour" assert self.fetched == ["static.html:fr"] # Expire static.html:es from the cache (static.html:fr remains cached). utils.set_utcnow_for_test(31) self.fetched = [] assert get_localized("static.html", "es").content == "hello" assert self.fetched == ["static.html:es", "static.html"] self.fetched = [] assert get_localized("static.html", "en").content == "hello" assert self.fetched == ["static.html:en", "static.html"] self.fetched = [] assert get_localized("static.html", "fr").content == "bonjour" assert self.fetched == []
def test_clean_up_in_test_mode_multi_tasks(self): """Test the clean up in test mode when it is broken into multiple tasks.""" tasks.CleanUpInTestMode.DELETION_AGE_SECONDS = 2 * 3600 # 2 hours utcnow = datetime.datetime(2010, 1, 1, 7, 0, 0) set_utcnow_for_test(utcnow) self.mox = mox.Mox() cleanup = self.initialize_handler(tasks.CleanUpInTestMode()) # Simulates add_task_for_repo() because it doesn't work in unit tests. def add_task_for_repo(repo, task_name, action, **kwargs): test_handler.initialize_handler( cleanup, action, repo=repo, params=kwargs) cleanup.get() self.mox.StubOutWithMock(cleanup, 'add_task_for_repo') (cleanup.add_task_for_repo( 'haiti', mox.IsA(str), mox.IsA(str), utcnow=str(calendar.timegm(utcnow.utctimetuple())), cursor=mox.IsA(str), queue_name=mox.IsA(str)). WithSideEffects(add_task_for_repo).MultipleTimes()) # Always pretends that we have consumed more CPU than threshold, # so that it creates a new task for each entry. self.mox.StubOutWithMock(quota, 'get_request_cpu_usage') quota.get_request_cpu_usage().MultipleTimes().AndReturn( tasks.CPU_MEGACYCLES_PER_REQUEST + 1) self.mox.ReplayAll() config.set(test_mode=True, repo='haiti') # This should run multiple tasks and finally deletes all records. cleanup.get() assert db.get(self.key_p1) is None assert db.get(self.key_p2) is None self.mox.UnsetStubs() self.mox.VerifyAll()
def test_get_localized(self): get_localized = resources.get_localized # These three fetches should load resources into the cache. self.fetched = [] assert get_localized('static.html', 'es').content == 'hello' assert self.fetched == ['static.html:es', 'static.html'] self.fetched = [] assert get_localized('static.html', 'en').content == 'hello' assert self.fetched == ['static.html:en', 'static.html'] self.fetched = [] assert get_localized('static.html', 'fr').content == 'bonjour' assert self.fetched == ['static.html:fr'] # These should now be cache hits, and shouldn't touch the datastore. self.fetched = [] assert get_localized('static.html', 'es').content == 'hello' assert get_localized('static.html', 'en').content == 'hello' assert get_localized('static.html', 'fr').content == 'bonjour' assert self.fetched == [] # Expire static.html:fr from the cache. utils.set_utcnow_for_test(21) self.fetched = [] assert get_localized('static.html', 'es').content == 'hello' assert get_localized('static.html', 'en').content == 'hello' assert self.fetched == [] assert get_localized('static.html', 'fr').content == 'bonjour' assert self.fetched == ['static.html:fr'] # Expire static.html:es from the cache (static.html:fr remains cached). utils.set_utcnow_for_test(31) self.fetched = [] assert get_localized('static.html', 'es').content == 'hello' assert self.fetched == ['static.html:es', 'static.html'] self.fetched = [] assert get_localized('static.html', 'en').content == 'hello' assert self.fetched == ['static.html:en', 'static.html'] self.fetched = [] assert get_localized('static.html', 'fr').content == 'bonjour' assert self.fetched == []
def setUp(self): self.mox = mox.Mox() self.mox.StubOutWithMock(urlfetch, 'fetch') self.mox.StubOutWithMock(random, 'shuffle') random.shuffle(mox.IsA(list)) self.orig_person = model.Person model.Person = MockPerson logger = logging.getLogger() # Don't log to stderr... self.original_handlers = logger.handlers logger.handlers = [] # ...instead log to our mock logging handler. self.mock_logging_handler = MockLoggingHandler() logger.addHandler(self.mock_logging_handler) logger.setLevel(logging.INFO) # The first two calls of utils.get_utcnow_seconds() at line 45 and 49 in # external_search.py consult the following date setting for debug. utils.set_utcnow_for_test(datetime.datetime(2011, 1, 1))
def test_expired_records_with_unremoved_data(self): """validate_expired_records_removed should return a list with the person_record_ids of all expired records that have data that should be removed""" validator = self.set_up_validator( PfifXml.XML_EXPIRE_99_HAS_NOTE_SYNCED_DATES) utils.set_utcnow_for_test(ValidatorTests.EXPIRED_TIME) self.assertEqual(len(validator.validate_expired_records_removed()), 1) just_expired = datetime.datetime(1999, 2, 4, 4, 5, 7) utils.set_utcnow_for_test(just_expired) self.assertEqual(len(validator.validate_expired_records_removed()), 1) validator = self.set_up_validator( PfifXml.XML_EXPIRE_99_HAS_DATA_SYNCED_DATES) utils.set_utcnow_for_test(ValidatorTests.EXPIRED_TIME) self.assertEqual(len(validator.validate_expired_records_removed()), 1)
def setUp(self): set_utcnow_for_test(datetime(2010, 1, 1)) self.p1 = model.Person.create_original( 'haiti', given_name='John', family_name='Smith', home_street='Washington St.', home_city='Los Angeles', home_state='California', home_postal_code='11111', home_neighborhood='Good Neighborhood', author_name='Alice Smith', author_phone='111-111-1111', author_email='*****@*****.**', source_url='https://www.source.com', source_date=datetime(2010, 1, 1), source_name='Source Name', entry_date=datetime(2010, 1, 1), expiry_date=datetime(2010, 2, 1)) self.p2 = model.Person.create_original('haiti', given_name='Tzvika', family_name='Hartman', home_street='Herzl St.', home_city='Tel Aviv', home_state='Israel', entry_date=datetime(2010, 1, 1), expiry_date=datetime( 2010, 3, 1)) self.p3 = model.Person.create_original('haiti', given_name='Third', family_name='Person', home_street='Main St.', home_city='San Francisco', home_state='California', entry_date=datetime(2010, 1, 1), expiry_date=datetime( 2020, 3, 1)) self.key_p1 = db.put(self.p1) self.key_p2 = db.put(self.p2) self.key_p3 = db.put(self.p3) # Link p2 and p3 to p1 self.n1_1 = model.Note.create_original( 'haiti', person_record_id=self.p1.record_id, linked_person_record_id=self.p2.record_id, status=u'believed_missing', author_email='*****@*****.**', author_made_contact=False, photo_url='http://example.com/note1.photo.jpg', entry_date=get_utcnow(), source_date=datetime(2000, 1, 1)) self.n1_2 = model.Note.create_original( 'haiti', person_record_id=self.p1.record_id, author_made_contact=True, entry_date=get_utcnow(), source_date=datetime(2000, 2, 2)) self.n1_3 = model.Note.create_original( 'haiti', person_record_id=self.p1.record_id, linked_person_record_id=self.p3.record_id, entry_date=get_utcnow(), source_date=datetime(2000, 3, 3)) # Link p1 and p3 to p2 self.n2_1 = model.Note.create_original( 'haiti', person_record_id=self.p2.record_id, linked_person_record_id=self.p1.record_id, entry_date=get_utcnow(), source_date=datetime(2000, 1, 1)) self.n2_2 = model.Note.create_original( 'haiti', person_record_id=self.p2.record_id, linked_person_record_id=self.p3.record_id, entry_date=get_utcnow(), source_date=datetime(2000, 2, 2)) # Link p2 and p1 to p3 self.n3_1 = model.Note.create_original( 'haiti', person_record_id=self.p3.record_id, linked_person_record_id=self.p2.record_id, entry_date=get_utcnow(), source_date=datetime(2000, 1, 1)) self.n3_2 = model.Note.create_original( 'haiti', person_record_id=self.p3.record_id, linked_person_record_id=self.p1.record_id, entry_date=get_utcnow(), source_date=datetime(2000, 2, 2)) self.key_n1_1 = db.put(self.n1_1) self.key_n1_2 = db.put(self.n1_2) self.key_n1_3 = db.put(self.n1_3) self.key_n2_1 = db.put(self.n2_1) self.key_n2_2 = db.put(self.n2_2) self.key_n3_1 = db.put(self.n3_1) self.key_n3_2 = db.put(self.n3_2) # Update the Person entity according to the Note. self.p1.update_from_note(self.n1_1) self.p1.update_from_note(self.n1_2) self.p1.update_from_note(self.n1_3) self.p2.update_from_note(self.n2_1) self.p2.update_from_note(self.n2_2) self.p3.update_from_note(self.n3_1) self.p3.update_from_note(self.n3_2) db.put(self.p1) db.put(self.p2) db.put(self.p3) self.to_delete = [ self.p1, self.p2, self.p3, self.n1_1, self.n1_2, self.n1_3, self.n2_1, self.n2_2, self.n3_1, self.n3_2 ]
def tearDown(self): utils.set_utcnow_for_test(None)
def setUp(self): utils.set_utcnow_for_test(0)
def test_get_rendered(self): get_rendered = resources.get_rendered eq = self.assertEquals # There's no es-specific page but there is an es-specific base template. self.fetched, self.compiled, self.rendered = [], [], [] assert get_rendered('page.html', 'es') == u'\xa1hola! default' assert self.fetched == ['page.html:es', 'page.html', 'page.html.template:es', 'page.html.template', 'base.html.template:es'] assert self.compiled == ['page.html.template', 'base.html.template:es'] assert self.rendered == ['page.html.template'] # There's an fr-specific page but no fr-specific base template. self.fetched, self.compiled, self.rendered = [], [], [] assert get_rendered('page.html', 'fr') == u'hi! fran\xe7ais' assert self.fetched == ['page.html:fr', 'page.html', 'page.html.template:fr', 'base.html.template:fr', 'base.html.template'] assert self.compiled == ['page.html.template:fr', 'base.html.template'] assert self.rendered == ['page.html.template:fr'] # There's no en-specific page and no en-specific base template. self.fetched, self.compiled, self.rendered = [], [], [] assert get_rendered('page.html', 'en') == u'hi! default' assert self.fetched == ['page.html:en', 'page.html', 'page.html.template:en', 'page.html.template', 'base.html.template:en', 'base.html.template'] assert self.compiled == ['page.html.template', 'base.html.template'] assert self.rendered == ['page.html.template'] # These should be cache hits, and shouldn't fetch, compile, or render. self.fetched, self.compiled, self.rendered = [], [], [] assert get_rendered('page.html', 'es') == u'\xa1hola! default' assert get_rendered('page.html', 'fr') == u'hi! fran\xe7ais' assert get_rendered('page.html', 'en') == u'hi! default' assert self.fetched == [] assert self.compiled == [] assert self.rendered == [] # Expire the pages but not the base templates. utils.set_utcnow_for_test(31) # Should fetch and recompile the pages but not the base templates. self.fetched, self.compiled, self.rendered = [], [], [] assert get_rendered('page.html', 'es') == u'\xa1hola! default' assert self.fetched == ['page.html:es', 'page.html', 'page.html.template:es', 'page.html.template'] assert self.compiled == ['page.html.template'] assert self.rendered == ['page.html.template'] # Should fetch and recompile the pages but not the base templates. self.fetched, self.compiled, self.rendered = [], [], [] assert get_rendered('page.html', 'fr') == u'hi! fran\xe7ais' assert self.fetched == ['page.html:fr', 'page.html', 'page.html.template:fr'] assert self.compiled == ['page.html.template:fr'] assert self.rendered == ['page.html.template:fr'] # Should fetch and recompile the pages but not the base templates. self.fetched, self.compiled, self.rendered = [], [], [] assert get_rendered('page.html', 'en') == u'hi! default' assert self.fetched == ['page.html:en', 'page.html', 'page.html.template:en', 'page.html.template'] assert self.compiled == ['page.html.template'] assert self.rendered == ['page.html.template'] # Expire the base templates and page.html.template:fr # (page.html.template:en and page.html.template:es remain cached). utils.set_utcnow_for_test(52) # Should fetch and recompile the base template but not the page. self.fetched, self.compiled, self.rendered = [], [], [] assert get_rendered('page.html', 'es') == u'\xa1hola! default' assert self.fetched == ['page.html:es', 'page.html', 'base.html.template:es'] assert self.compiled == ['base.html.template:es'] assert self.rendered == ['page.html.template'] # Should fetch and recompile both the fr page and the base template. self.fetched, self.compiled, self.rendered = [], [], [] assert get_rendered('page.html', 'fr') == u'hi! fran\xe7ais' assert self.fetched == ['page.html:fr', 'page.html', 'page.html.template:fr', 'base.html.template:fr', 'base.html.template'] assert self.compiled == ['page.html.template:fr', 'base.html.template'] assert self.rendered == ['page.html.template:fr'] # Should fetch and recompile the base template but not the page. self.fetched, self.compiled, self.rendered = [], [], [] assert get_rendered('page.html', 'en') == u'hi! default' assert self.fetched == ['page.html:en', 'page.html', 'base.html.template:en', 'base.html.template'] assert self.compiled == ['base.html.template'] assert self.rendered == ['page.html.template'] # Ensure binary data is preserved. assert get_rendered('data', 'en') == '\xff\xfe\xfd\xfc'