def test_metadata_created(self, get_records, open): get_records.return_value = [] with patch('civil_registry.utils.mirror_database') as mirror_database: mirror_database.return_value = MirrorStats() import_citizen_dump(input_filename=None) mirror_database.assert_called() self.assertEqual(1, CitizenMetadata.objects.count())
def test_too_many_changes(self, get_records, open): cit1 = CitizenFactory() cit2 = CitizenFactory() citizen = CitizenFactory() data = model_to_dict(citizen) new_name = 'Jim Bob' data['first_name'] = new_name get_records.return_value = [model_to_dict(cit1), model_to_dict(cit2), data] # Import raises an exception: with self.assertRaises(TooManyChanges): import_citizen_dump(input_filename=None, max_change_percent=10) # Citizen is unchanged: new_citizen = refresh_model(citizen) self.assertEqual(citizen.first_name, new_citizen.first_name)
def test_metadata_updated(self, get_records, open): start_time = now() last_time = start_time - timedelta(days=2) CitizenMetadata.objects.create(dump_time=last_time) get_records.return_value = [] sleep(0.001) # ensure the clock has advanced very slightly with patch('civil_registry.utils.mirror_database') as mirror_database: mirror_database.return_value = MirrorStats() import_citizen_dump(input_filename=None) sleep(0.001) meta = CitizenMetadata.objects.get() self.assertNotEqual(last_time, meta.dump_time) self.assertTrue(meta.dump_time > last_time) self.assertTrue(meta.dump_time > start_time) self.assertTrue(meta.dump_time < now())
def test_too_many_changes(self, get_records, open): cit1 = CitizenFactory() cit2 = CitizenFactory() citizen = CitizenFactory() data = model_to_dict(citizen) new_name = 'Jim Bob' data['first_name'] = new_name get_records.return_value = [ model_to_dict(cit1), model_to_dict(cit2), data ] # Import raises an exception: with self.assertRaises(TooManyChanges): import_citizen_dump(input_filename=None, max_change_percent=10) # Citizen is unchanged: new_citizen = refresh_model(citizen) self.assertEqual(citizen.first_name, new_citizen.first_name)
def test_citizen_created(self, get_records, open): citizen = CitizenFactory() data = model_to_dict(citizen) citizen.delete() get_records.return_value = [data] self.assertFalse(Citizen.objects.exists()) stats = import_citizen_dump(input_filename=None) self.assertEqual(1, stats.records_read) new_cit = Citizen.objects.get() self.assertEqual(data, model_to_dict(new_cit))
def test_citizen_updated(self, get_records, open): cit1 = CitizenFactory() cit2 = CitizenFactory() citizen = CitizenFactory() data = model_to_dict(citizen) new_name = 'Jim Bob' data['first_name'] = new_name get_records.return_value = [model_to_dict(cit1), model_to_dict(cit2), data] stats = import_citizen_dump(input_filename=None, max_change_percent=34) self.assertEqual(3, stats.records_read) updated_cit = Citizen.objects.get(pk=citizen.pk) self.assertEqual(new_name, updated_cit.first_name)
def test_citizen_missing(self, get_records, open): # Citizen in db but not in import citizen = CitizenFactory() get_records.return_value = [] stats = import_citizen_dump(input_filename=None, max_change_percent=100) self.assertEqual(0, stats.records_read) # The record is still there self.assertEqual(0, Citizen.objects.count()) # we don't see missing records by default self.assertEqual(1, Citizen.objects.unfiltered().count()) # but it's really there self.assertEqual(1, stats.not_there_anymore_count) self.assertIn(citizen.pk, stats.missing_pks) # But it's been marked missing citizen = Citizen.objects.unfiltered().get(pk=citizen.pk) self.assertTrue(citizen.missing)
def test_citizen_updated(self, get_records, open): cit1 = CitizenFactory() cit2 = CitizenFactory() citizen = CitizenFactory() data = model_to_dict(citizen) new_name = 'Jim Bob' data['first_name'] = new_name get_records.return_value = [ model_to_dict(cit1), model_to_dict(cit2), data ] stats = import_citizen_dump(input_filename=None, max_change_percent=34) self.assertEqual(3, stats.records_read) updated_cit = Citizen.objects.get(pk=citizen.pk) self.assertEqual(new_name, updated_cit.first_name)
def test_citizen_missing(self, get_records, open): # Citizen in db but not in import citizen = CitizenFactory() get_records.return_value = [] stats = import_citizen_dump(input_filename=None, max_change_percent=100) self.assertEqual(0, stats.records_read) # The record is still there self.assertEqual( 0, Citizen.objects.count()) # we don't see missing records by default self.assertEqual( 1, Citizen.objects.unfiltered().count()) # but it's really there self.assertEqual(1, stats.not_there_anymore_count) self.assertIn(citizen.pk, stats.missing_pks) # But it's been marked missing citizen = Citizen.objects.unfiltered().get(pk=citizen.pk) self.assertTrue(citizen.missing)
def test_temp_citizen_created(self, get_records, open): citizen = CitizenFactory() data = model_to_dict(citizen) citizen.delete() get_records.return_value = [data] self.assertFalse(Citizen.objects.exists()) self.assertFalse(TempCitizen.objects.exists()) with patch('civil_registry.utils.delete_all'): with patch('civil_registry.utils.mirror_database') as mirror_database: mirror_database.return_value = MirrorStats() stats = import_citizen_dump(input_filename=None) # We mocked mirror_database, so nothing actually got copied to Citizen # but it should have "read" the data into TempCitizen, and we # mocked delete_all, so it should still be there. self.assertFalse(Citizen.objects.exists()) self.assertEqual(1, stats.records_read) mirror_database.assert_called() temp_cit = TempCitizen.objects.get() self.assertEqual(data, model_to_dict(temp_cit))
def test_temp_citizen_created(self, get_records, open): citizen = CitizenFactory() data = model_to_dict(citizen) citizen.delete() get_records.return_value = [data] self.assertFalse(Citizen.objects.exists()) self.assertFalse(TempCitizen.objects.exists()) with patch('civil_registry.utils.delete_all'): with patch( 'civil_registry.utils.mirror_database') as mirror_database: mirror_database.return_value = MirrorStats() stats = import_citizen_dump(input_filename=None) # We mocked mirror_database, so nothing actually got copied to Citizen # but it should have "read" the data into TempCitizen, and we # mocked delete_all, so it should still be there. self.assertFalse(Citizen.objects.exists()) self.assertEqual(1, stats.records_read) mirror_database.assert_called() temp_cit = TempCitizen.objects.get() self.assertEqual(data, model_to_dict(temp_cit))
def handle_label(self, label, **options): input_filename = label if not os.path.exists(input_filename): raise CommandError("File does not exist: %s" % input_filename) # Lots of output on stdout logging.getLogger('civil_registry').setLevel(logging.DEBUG) logging.getLogger('civil_registry').addHandler(logging.StreamHandler()) logger.info("Starting at %s" % now()) try: stats = import_citizen_dump( input_filename, max_change_percent=options['max_change_percent'], encoding=options['encoding'], ) except TooManyChanges as e: raise CommandError(e.args[0]) # Say what we did. logger.info("Records read: %10d" % stats.records_read) logger.info("Unchanged records: %10d" % stats.unchanged_count) logger.info("Modified records: %10d" % stats.modified_record_count) logger.info("New records: %10d" % stats.new_record_count) previous_total = (stats.unchanged_count + stats.modified_record_count + stats.not_there_anymore_count) new_total = previous_total + stats.new_record_count logger.info("Records no longer there: %10d" % stats.not_there_anymore_count) logger.info("Previous total: %10d" % previous_total) logger.info("New total: %10d" % new_total) logger.info("Finished at: %s" % CitizenMetadata.objects.get().dump_time)
def test_input_filename(self, get_records, open): filename = 'my_dump_file' import_citizen_dump(input_filename=filename) open.assert_called_with(filename, encoding=ANY)
def test_encoding(self, get_records, open): encoding = 'ASCII' import_citizen_dump(input_filename=None, encoding=encoding) open.assert_called_with(ANY, encoding=encoding)