Esempio n. 1
0
 def test_new_record(self):
     cit = TempCitizenFactory()
     stats = mirror_database(from_model=TempCitizen, to_model=Citizen)
     self.assertEqual(0, stats.unchanged_count)
     self.assertEqual(0, stats.modified_record_count)
     self.assertEqual(1, stats.new_record_count)
     self.assertEqual(0, stats.not_there_anymore_count)
     self.assertTrue(Citizen.objects.filter(pk=cit.pk).exists())
Esempio n. 2
0
    def test_unchanged_record(self):
        cit = CitizenFactory()
        TempCitizenFactory(**model_to_dict(cit))

        stats = mirror_database(from_model=TempCitizen, to_model=Citizen)
        self.assertEqual(1, stats.unchanged_count)
        self.assertEqual(0, stats.modified_record_count)
        self.assertEqual(0, stats.new_record_count)
        self.assertEqual(0, stats.not_there_anymore_count)
Esempio n. 3
0
 def test_empty_dbs(self):
     stats = mirror_database(
         from_model=TempCitizen,
         to_model=Citizen
     )
     self.assertEqual(0, stats.unchanged_count)
     self.assertEqual(0, stats.modified_record_count)
     self.assertEqual(0, stats.new_record_count)
     self.assertEqual(0, stats.not_there_anymore_count)
Esempio n. 4
0
 def test_changed_record(self):
     cit = CitizenFactory()
     temp_cit = TempCitizenFactory(pk=cit.pk,
                                   first_name=cit.first_name + '_CHANGED')
     stats = mirror_database(from_model=TempCitizen, to_model=Citizen)
     self.assertEqual(0, stats.unchanged_count)
     self.assertEqual(1, stats.modified_record_count)
     self.assertEqual(0, stats.new_record_count)
     self.assertEqual(0, stats.not_there_anymore_count)
     c = Citizen.objects.get(pk=cit.pk)
     self.assertEqual(temp_cit.first_name, c.first_name)
Esempio n. 5
0
 def test_new_record(self):
     cit = TempCitizenFactory()
     stats = mirror_database(
         from_model=TempCitizen,
         to_model=Citizen
     )
     self.assertEqual(0, stats.unchanged_count)
     self.assertEqual(0, stats.modified_record_count)
     self.assertEqual(1, stats.new_record_count)
     self.assertEqual(0, stats.not_there_anymore_count)
     self.assertTrue(Citizen.objects.filter(pk=cit.pk).exists())
Esempio n. 6
0
    def test_unchanged_record(self):
        cit = CitizenFactory()
        TempCitizenFactory(**model_to_dict(cit))

        stats = mirror_database(
            from_model=TempCitizen,
            to_model=Citizen
        )
        self.assertEqual(1, stats.unchanged_count)
        self.assertEqual(0, stats.modified_record_count)
        self.assertEqual(0, stats.new_record_count)
        self.assertEqual(0, stats.not_there_anymore_count)
Esempio n. 7
0
 def test_record_gone_do_not_delete_with_existing(self):
     existing = CitizenFactory(pk=1)
     TempCitizenFactory(pk=1, **model_to_dict(existing))
     cit = CitizenFactory(pk=2)
     existing2 = CitizenFactory(pk=3)
     TempCitizenFactory(pk=3, **model_to_dict(existing2))
     stats = mirror_database(from_model=TempCitizen, to_model=Citizen)
     self.assertEqual(2, stats.unchanged_count)
     self.assertEqual(0, stats.modified_record_count)
     self.assertEqual(0, stats.new_record_count)
     self.assertEqual(1, stats.not_there_anymore_count)
     # NOT deleted
     self.assertTrue(Citizen.objects.filter(pk=cit.pk).exists())
Esempio n. 8
0
 def test_changed_record(self):
     cit = CitizenFactory()
     temp_cit = TempCitizenFactory(pk=cit.pk, first_name=cit.first_name + '_CHANGED')
     stats = mirror_database(
         from_model=TempCitizen,
         to_model=Citizen
     )
     self.assertEqual(0, stats.unchanged_count)
     self.assertEqual(1, stats.modified_record_count)
     self.assertEqual(0, stats.new_record_count)
     self.assertEqual(0, stats.not_there_anymore_count)
     c = Citizen.objects.get(pk=cit.pk)
     self.assertEqual(temp_cit.first_name, c.first_name)
Esempio n. 9
0
 def test_new_record_with_existing(self):
     nid = 100000000000
     existing = CitizenFactory(pk=1, national_id=nid)
     TempCitizenFactory(pk=1, **model_to_dict(existing))
     nid += 1
     cit = TempCitizenFactory(pk=2, national_id=nid)
     nid += 1
     existing2 = CitizenFactory(pk=3, national_id=nid)
     TempCitizenFactory(pk=3, **model_to_dict(existing2))
     stats = mirror_database(from_model=TempCitizen, to_model=Citizen)
     self.assertEqual(2, stats.unchanged_count)
     self.assertEqual(0, stats.modified_record_count)
     self.assertEqual(1, stats.new_record_count)
     self.assertEqual(0, stats.not_there_anymore_count)
     self.assertTrue(Citizen.objects.filter(pk=cit.pk).exists())
Esempio n. 10
0
 def test_record_gone_do_not_delete_with_existing(self):
     existing = CitizenFactory(pk=1)
     TempCitizenFactory(pk=1, **model_to_dict(existing))
     cit = CitizenFactory(pk=2)
     existing2 = CitizenFactory(pk=3)
     TempCitizenFactory(pk=3, **model_to_dict(existing2))
     stats = mirror_database(
         from_model=TempCitizen,
         to_model=Citizen
     )
     self.assertEqual(2, stats.unchanged_count)
     self.assertEqual(0, stats.modified_record_count)
     self.assertEqual(0, stats.new_record_count)
     self.assertEqual(1, stats.not_there_anymore_count)
     # NOT deleted
     self.assertTrue(Citizen.objects.filter(pk=cit.pk).exists())
Esempio n. 11
0
 def test_new_record_with_existing(self):
     nid = 100000000000
     existing = CitizenFactory(pk=1, national_id=nid)
     TempCitizenFactory(pk=1, **model_to_dict(existing))
     nid += 1
     cit = TempCitizenFactory(pk=2, national_id=nid)
     nid += 1
     existing2 = CitizenFactory(pk=3, national_id=nid)
     TempCitizenFactory(pk=3, **model_to_dict(existing2))
     stats = mirror_database(
         from_model=TempCitizen,
         to_model=Citizen
     )
     self.assertEqual(2, stats.unchanged_count)
     self.assertEqual(0, stats.modified_record_count)
     self.assertEqual(1, stats.new_record_count)
     self.assertEqual(0, stats.not_there_anymore_count)
     self.assertTrue(Citizen.objects.filter(pk=cit.pk).exists())
Esempio n. 12
0
 def test_changed_record_with_existing(self):
     nid = 100000000000
     existing = CitizenFactory(pk=1, national_id=nid)
     nid += 1
     TempCitizenFactory(pk=existing.pk, **model_to_dict(existing))
     nid += 1
     cit = CitizenFactory(pk=2, national_id=nid)
     temp_cit = TempCitizenFactory(pk=cit.pk,
                                   national_id=nid,
                                   first_name=cit.first_name + '_CHANGED')
     existing2 = CitizenFactory(pk=3)
     TempCitizenFactory(pk=existing2.pk, **model_to_dict(existing2))
     stats = mirror_database(from_model=TempCitizen, to_model=Citizen)
     self.assertEqual(2, stats.unchanged_count)
     self.assertEqual(1, stats.modified_record_count)
     self.assertEqual(0, stats.new_record_count)
     self.assertEqual(0, stats.not_there_anymore_count)
     c = Citizen.objects.get(pk=cit.pk)
     self.assertEqual(temp_cit.first_name, c.first_name)
Esempio n. 13
0
 def test_changed_record_with_existing(self):
     nid = 100000000000
     existing = CitizenFactory(pk=1, national_id=nid)
     nid += 1
     TempCitizenFactory(pk=existing.pk, **model_to_dict(existing))
     nid += 1
     cit = CitizenFactory(pk=2, national_id=nid)
     temp_cit = TempCitizenFactory(pk=cit.pk, national_id=nid,
                                   first_name=cit.first_name + '_CHANGED')
     existing2 = CitizenFactory(pk=3)
     TempCitizenFactory(pk=existing2.pk, **model_to_dict(existing2))
     stats = mirror_database(
         from_model=TempCitizen,
         to_model=Citizen
     )
     self.assertEqual(2, stats.unchanged_count)
     self.assertEqual(1, stats.modified_record_count)
     self.assertEqual(0, stats.new_record_count)
     self.assertEqual(0, stats.not_there_anymore_count)
     c = Citizen.objects.get(pk=cit.pk)
     self.assertEqual(temp_cit.first_name, c.first_name)
Esempio n. 14
0
 def test_empty_dbs(self):
     stats = mirror_database(from_model=TempCitizen, to_model=Citizen)
     self.assertEqual(0, stats.unchanged_count)
     self.assertEqual(0, stats.modified_record_count)
     self.assertEqual(0, stats.new_record_count)
     self.assertEqual(0, stats.not_there_anymore_count)
Esempio n. 15
0
def import_citizen_dump(input_filename,
                        max_change_percent=DEFAULT_MAX_CHANGE_PERCENT,
                        encoding='UTF-8'):

    with transaction.atomic():

        # Clear out TempCitizen table. (We clear it at the end too, but this makes
        # extra sure that we start with it empty.)
        delete_all('default', [TempCitizen])

        num_records_at_start = Citizen.objects.count()

        #
        # 1. Fill our temp table with the data from the latest dump
        #
        logger.info("Loading data from dump")
        input_file = codecs.open(input_filename, encoding=encoding)
        logger.info("Reading %s" % input_filename)
        batch = BatchOperations(TempCitizen)
        records_read = 0
        for record in get_records(input_file):
            records_read += 1
            batch.add(record)
        batch.flush()

        #
        # 2. Sync data from temp table to our real table
        #
        logger.info("Updating our own database")
        stats = mirror_database(from_model=TempCitizen,
                                to_model=Citizen)

        # See what % of the records we're changing
        if num_records_at_start > 0:
            num_changes = (stats.modified_record_count + stats.new_record_count
                           + stats.not_there_anymore_count)
            percent_changed = 100 * (num_changes / num_records_at_start)
            if percent_changed > max_change_percent:
                raise TooManyChanges(
                    "Too many changes, aborting Citizen data import. Max change is %f%% but "
                    "the import would have changed %f%% records (%d/%d).  Use "
                    "--max-change-percent=NN to override this limit if necessary."
                    % (max_change_percent, percent_changed, num_changes, num_records_at_start))

        # Add our data
        stats.records_read = records_read

        # Make a note of when we did it
        timestamp = now()
        CitizenMetadata.objects.update_or_create(defaults=dict(dump_time=timestamp))

        # Flag any records that turned up missing
        if stats.missing_pks:
            Citizen.objects.filter(pk__in=stats.missing_pks, missing=None).update(missing=timestamp)

        # And we're done!

        # Clear out our temp table (no point in taking up disk space)
        delete_all('default', [TempCitizen])

    return stats