def test_duplicate_entries(): """We insert a duplicate entry, and check that it wasn"t in fact duplicated.""" zipcode_utils.create_zipcode("20852", "Rockville", "MD", "2013") results = ZipcodeCityStateYear.objects.filter(state="MD") assert results.count() == 1 zipcode_utils.create_zipcode("20852", "Rockville", "MD", "2013") results = ZipcodeCityStateYear.objects.filter(state="MD") assert results.count() == 1
def test_createzipcode(): zipcode_utils.create_zipcode("20852", "Rockville", "MD", "2013") results = ZipcodeCityStateYear.objects.filter(state="MD") assert results.count() == 1 assert results[0].zip_code == 20852 assert results[0].city == "Rockville" assert results[0].state == "MD" assert results[0].year == 2013
def test_createzipcode(self): ZipcodeCityState.objects.all().delete() zipcode_utils.create_zipcode('20852', 'Rockville', 'MD') results = ZipcodeCityState.objects.filter(state='MD') self.assertEqual(1, len(results)) self.assertEqual(results[0].zip_code, 20852) self.assertEqual(results[0].city, 'Rockville') self.assertEqual(results[0].state, 'MD')
def handle(self, *args, **options): transmittal_filename = args[0] agencies = Agency.objects.get_all_by_code() with open(transmittal_filename) as institutioncsv: transmittal_reader = csv.reader(institutioncsv, delimiter='\t') institutions = [] for inst_line in transmittal_reader: zip_code = inst_line[8] state = inst_line[7] city = inst_line[6] zipcode_city = create_zipcode(zip_code, city, state) agency = agencies[int(inst_line[2])] inst = Institution( year=inst_line[0], respondent_id=inst_line[1], agency=agency, institution_id = inst_line[2]+inst_line[1], tax_id=inst_line[3], name=inst_line[4], mailing_address=inst_line[5], zip_code=zipcode_city, assets=int(inst_line[17]), ) institutions.append(inst) Institution.objects.bulk_create(institutions)
def handle(self, *args, **options): transmittal_filename = args[0] agencies = Agency.objects.get_all_by_code() with open(transmittal_filename) as institutioncsv: transmittal_reader = csv.reader(institutioncsv, delimiter='\t') institutions = [] for inst_line in transmittal_reader: zip_code = inst_line[8] state = inst_line[7] city = inst_line[6] zipcode_city = create_zipcode(zip_code, city, state) agency = agencies[int(inst_line[2])] inst = Institution( year=inst_line[0], respondent_id=inst_line[1], agency=agency, institution_id=inst_line[2] + inst_line[1], tax_id=inst_line[3], name=inst_line[4], mailing_address=inst_line[5], zip_code=zipcode_city, assets=int(inst_line[17]), ) institutions.append(inst) Institution.objects.bulk_create(institutions)
def load_from_csv( agencies: Dict[int, Agency], transmittal_reader: Iterator[List[str]]): for zero_line_number, line in enumerate(transmittal_reader): line_number = zero_line_number + 1 line = fixup(line) if len(line) not in (15, 22): logger.warning( "Line %s is invalid, has length %s (expected 15 or 22)", line_number, len(line), ) break year = int(line[0]) zipcode_city = create_zipcode( city=line[6], state=line[7], year=year, zip_code=line[8]) agency_id = line[2] if not agency_id.isdigit() or int(agency_id) not in agencies: logger.warning("Line %s has an invalid agency: %s", line_number, agency_id) break if len(line) == 22 and not line[17].isdigit(): logger.warning("Line %s has an invalid asset amount: %s", line_number, line[17]) break respondent_id = line[1] inst = Institution( year=year, respondent_id=line[1], agency=agencies[int(agency_id)], institution_id=f"{year}{agency_id}{respondent_id}", tax_id=line[3], name=line[4], mailing_address=line[5], zip_code=zipcode_city, assets=int(line[17]) if len(line) == 22 else None, ) try: inst.full_clean(exclude=["agency"], validate_unique=False) except ValidationError: logger.exception("Line %s has invalid data:", line_number) break yield inst
def handle(self, *args, **options): transmittal_filename = args[0] agencies = Agency.objects.get_all_by_code() with open(transmittal_filename) as institutioncsv: transmittal_reader = csv.reader(institutioncsv, delimiter='\t') institutions = [] # count = 1 # use if want to see which item failed, see comment below where we create institution individually for inst_line in transmittal_reader: year = inst_line[0] zip_code = inst_line[8] state = inst_line[7] city = inst_line[6] zipcode_city = create_zipcode(zip_code, city, state, year) agency = agencies[int(inst_line[2])] inst = Institution( year=year, respondent_id=inst_line[1], agency=agency, institution_id=year + inst_line[2] + inst_line[1], tax_id=inst_line[3], name=inst_line[4], mailing_address=inst_line[5], zip_code=zipcode_city, assets=int(inst_line[17]), ) # This can be used to figure out which exact item was failing, will need to disable bulk create below to use this # Institution.objects.create(inst) # inst.save() # count += 1 institutions.append(inst) Institution.objects.bulk_create(institutions)
def handle(self, *args, **options): transmittal_filename = args[0] agencies = Agency.objects.get_all_by_code() with open(transmittal_filename) as institutioncsv: transmittal_reader = csv.reader(institutioncsv, delimiter='\t') institutions = [] # count = 1 # use if want to see which item failed, see comment below where we create institution individually for inst_line in transmittal_reader: year = inst_line[0] zip_code = inst_line[8] state = inst_line[7] city = inst_line[6] zipcode_city = create_zipcode(zip_code, city, state, year) agency = agencies[int(inst_line[2])] inst = Institution( year=year, respondent_id=inst_line[1], agency=agency, institution_id=year+inst_line[2]+inst_line[1], tax_id=inst_line[3], name=inst_line[4], mailing_address=inst_line[5], zip_code=zipcode_city, assets=int(inst_line[17]), ) # This can be used to figure out which exact item was failing, will need to disable bulk create below to use this # Institution.objects.create(inst) # inst.save() # count += 1 institutions.append(inst) Institution.objects.bulk_create(institutions)
def test_duplicate_entries(self): """ We insert a duplicate entry, and check that it wasn't in fact duplicated. """ zipcode_utils.create_zipcode('20852', 'Rockville', 'MD') results = ZipcodeCityState.objects.filter(state='MD') self.assertEqual(1, len(results))
def test_duplicate_entries(self): """ We insert a duplicate entry, and check that it wasn't in fact duplicated. """ zipcode_utils.create_zipcode('20852', 'Rockville', 'MD', '2013') results = ZipcodeCityStateYear.objects.filter(state='MD') self.assertEqual(1, len(results))