def test_purge(self): now = datetime.datetime.now() # Create 5 objs of each type at various date ranges. for i in [1, 160, 180, 200, 220]: ResponseEmailFactory( opinion__created=(now - datetime.timedelta(days=i)) ) ResponseContextFactory( opinion__created=(now - datetime.timedelta(days=i)) ) ResponsePIFactory( opinion__created=(now - datetime.timedelta(days=i)) ) # Note: This creates 3 * 5 = 15 Response objects. # Since creating the objects and indexing them happens very # quickly in tests, we hit a race condition and the has_email # column ends up being false. So instead we just drop the # index and rebuild it. self.setup_indexes() # Make sure everything is in the db assert Response.objects.count() == 15 assert ResponseEmail.objects.count() == 5 assert ResponseContext.objects.count() == 5 assert ResponsePI.objects.count() == 5 # Make sure everything is in the index resp_s = ResponseDocType.docs.search() assert resp_s.count() == 15 assert resp_s.filter('term', has_email=True).count() == 5 # Now purge everything older than 180 days and make sure # things got removed that should have gotten removed. purge_data() self.refresh() # All the Response objects should still be there assert Response.objects.count() == 15 # For the other objects, 2 should be there for each type assert ResponseEmail.objects.count() == 2 assert ResponseContext.objects.count() == 2 assert ResponsePI.objects.count() == 2 # Everything should still be in the index, but the number of # things with has_email=True should go down resp_s = ResponseDocType.docs.search() assert resp_s.count() == 15 assert resp_s.filter('term', has_email=True).count() == 2
def test_purge(self): now = datetime.datetime.now() cutoff = now - datetime.timedelta(days=5) # Create 10 ResponseEmail objs--one for each day for the last # 10 days. for i in range(10): ResponseEmailFactory( opinion__created=(now - datetime.timedelta(days=i)) ) ResponseContextFactory( opinion__created=(now - datetime.timedelta(days=i)) ) # Since creating the objects and indexing them happens very # quickly in tests, we hit a race condition and the has_email # column ends up being false. So instead we just drop the # index and rebuild it. self.setup_indexes() # Make sure everything is in the db eq_(Response.objects.count(), 20) eq_(ResponseEmail.objects.count(), 10) eq_(ResponseContext.objects.count(), 10) # Make sure everything is in the index resp_s = ResponseMappingType.search() eq_(resp_s.count(), 20) eq_(resp_s.filter(has_email=True).count(), 10) # Now purge everything older than 5 days and make sure things # got removed that should have gotten removed cutoff = now - datetime.timedelta(days=5) purge_data(cutoff=cutoff) self.refresh() eq_(Response.objects.count(), 20) eq_(ResponseEmail.objects.count(), 5) eq_(ResponseEmail.objects.filter( opinion__created__gte=cutoff).count(), 5) eq_(ResponseContext.objects.count(), 5) eq_(ResponseContext.objects.filter( opinion__created__gte=cutoff).count(), 5) # Everything should still be in the index, but the number of # things with has_email=True should go down resp_s = ResponseMappingType.search() eq_(resp_s.count(), 20) eq_(resp_s.filter(has_email=True).count(), 5)
def test_purge(self): now = datetime.datetime.now() cutoff = now - datetime.timedelta(days=5) # Create 10 objs of each type--one for each day for the last # 10 days. for i in range(10): ResponseEmailFactory( opinion__created=(now - datetime.timedelta(days=i)) ) ResponseContextFactory( opinion__created=(now - datetime.timedelta(days=i)) ) ResponsePIFactory( opinion__created=(now - datetime.timedelta(days=i)) ) # Note that this creates 30 Response objects. # Since creating the objects and indexing them happens very # quickly in tests, we hit a race condition and the has_email # column ends up being false. So instead we just drop the # index and rebuild it. self.setup_indexes() # Make sure everything is in the db eq_(Response.objects.count(), 30) eq_(ResponseEmail.objects.count(), 10) eq_(ResponseContext.objects.count(), 10) eq_(ResponsePI.objects.count(), 10) # Make sure everything is in the index resp_s = ResponseMappingType.search() eq_(resp_s.count(), 30) eq_(resp_s.filter(has_email=True).count(), 10) # Now purge everything older than 5 days and make sure things # got removed that should have gotten removed. Also check if # there is a journal entry for the purge operation. cutoff = now - datetime.timedelta(days=5) purge_data(cutoff=cutoff) self.refresh() eq_(Response.objects.count(), 30) eq_(ResponseEmail.objects.count(), 5) eq_(ResponseEmail.objects.filter( opinion__created__gte=cutoff).count(), 5) eq_(ResponseContext.objects.count(), 5) eq_(ResponseContext.objects.filter( opinion__created__gte=cutoff).count(), 5) eq_(ResponsePI.objects.count(), 5) eq_(ResponsePI.objects.filter( opinion__created__gte=cutoff).count(), 5) eq_(1, Record.objects.filter(action='purge_data').count()) expected_msg = ('feedback_responseemail: 5, ' 'feedback_responsecontext: 5, ' 'feedback_responsepi: 5') eq_(expected_msg, Record.objects.get(action='purge_data').msg) # Everything should still be in the index, but the number of # things with has_email=True should go down resp_s = ResponseMappingType.search() eq_(resp_s.count(), 30) eq_(resp_s.filter(has_email=True).count(), 5)
def handle(self, *args, **options): verbose = (int(options.get('verbosity', 1)) >= 1) purge_data(verbose=verbose)
def test_purge(self): now = datetime.datetime.now() cutoff = now - datetime.timedelta(days=5) # Create 10 objs of each type--one for each day for the last # 10 days. for i in range(10): ResponseEmailFactory(opinion__created=(now - datetime.timedelta(days=i))) ResponseContextFactory( opinion__created=(now - datetime.timedelta(days=i))) ResponsePIFactory(opinion__created=(now - datetime.timedelta(days=i))) # Note that this creates 30 Response objects. # Since creating the objects and indexing them happens very # quickly in tests, we hit a race condition and the has_email # column ends up being false. So instead we just drop the # index and rebuild it. self.setup_indexes() # Make sure everything is in the db eq_(Response.objects.count(), 30) eq_(ResponseEmail.objects.count(), 10) eq_(ResponseContext.objects.count(), 10) eq_(ResponsePI.objects.count(), 10) # Make sure everything is in the index resp_s = ResponseMappingType.search() eq_(resp_s.count(), 30) eq_(resp_s.filter(has_email=True).count(), 10) # Now purge everything older than 5 days and make sure things # got removed that should have gotten removed. Also check if # there is a journal entry for the purge operation. cutoff = now - datetime.timedelta(days=5) purge_data(cutoff=cutoff) self.refresh() eq_(Response.objects.count(), 30) eq_(ResponseEmail.objects.count(), 5) eq_( ResponseEmail.objects.filter(opinion__created__gte=cutoff).count(), 5) eq_(ResponseContext.objects.count(), 5) eq_( ResponseContext.objects.filter( opinion__created__gte=cutoff).count(), 5) eq_(ResponsePI.objects.count(), 5) eq_(ResponsePI.objects.filter(opinion__created__gte=cutoff).count(), 5) eq_(1, Record.objects.filter(action='purge_data').count()) expected_msg = ('feedback_responseemail: 5, ' 'feedback_responsecontext: 5, ' 'feedback_responsepi: 5') eq_(expected_msg, Record.objects.get(action='purge_data').msg) # Everything should still be in the index, but the number of # things with has_email=True should go down resp_s = ResponseMappingType.search() eq_(resp_s.count(), 30) eq_(resp_s.filter(has_email=True).count(), 5)