def test_duplicate_entries():
"""We insert a duplicate entry, and check that it wasn"t in fact
duplicated."""
zipcode_utils.create_zipcode("20852", "Rockville", "MD", "2013")
results = ZipcodeCityStateYear.objects.filter(state="MD")
assert results.count() == 1
zipcode_utils.create_zipcode("20852", "Rockville", "MD", "2013")
results = ZipcodeCityStateYear.objects.filter(state="MD")
assert results.count() == 1
def test_createzipcode():
zipcode_utils.create_zipcode("20852", "Rockville", "MD", "2013")
results = ZipcodeCityStateYear.objects.filter(state="MD")
assert results.count() == 1
assert results[0].zip_code == 20852
assert results[0].city == "Rockville"
assert results[0].state == "MD"
assert results[0].year == 2013
def test_createzipcode(self):
ZipcodeCityState.objects.all().delete()
zipcode_utils.create_zipcode('20852', 'Rockville', 'MD')
results = ZipcodeCityState.objects.filter(state='MD')
self.assertEqual(1, len(results))
self.assertEqual(results[0].zip_code, 20852)
self.assertEqual(results[0].city, 'Rockville')
self.assertEqual(results[0].state, 'MD')
def test_createzipcode(self):
ZipcodeCityState.objects.all().delete()
zipcode_utils.create_zipcode('20852', 'Rockville', 'MD')
results = ZipcodeCityState.objects.filter(state='MD')
self.assertEqual(1, len(results))
self.assertEqual(results[0].zip_code, 20852)
self.assertEqual(results[0].city, 'Rockville')
self.assertEqual(results[0].state, 'MD')
def handle(self, *args, **options):
transmittal_filename = args[0]
agencies = Agency.objects.get_all_by_code()
with open(transmittal_filename) as institutioncsv:
transmittal_reader = csv.reader(institutioncsv, delimiter='\t')
institutions = []
for inst_line in transmittal_reader:
zip_code = inst_line[8]
state = inst_line[7]
city = inst_line[6]
zipcode_city = create_zipcode(zip_code, city, state)
agency = agencies[int(inst_line[2])]
inst = Institution(
year=inst_line[0],
respondent_id=inst_line[1],
agency=agency,
institution_id = inst_line[2]+inst_line[1],
tax_id=inst_line[3],
name=inst_line[4],
mailing_address=inst_line[5],
zip_code=zipcode_city,
assets=int(inst_line[17]),
)
institutions.append(inst)
Institution.objects.bulk_create(institutions)
def handle(self, *args, **options):
transmittal_filename = args[0]
agencies = Agency.objects.get_all_by_code()
with open(transmittal_filename) as institutioncsv:
transmittal_reader = csv.reader(institutioncsv, delimiter='\t')
institutions = []
for inst_line in transmittal_reader:
zip_code = inst_line[8]
state = inst_line[7]
city = inst_line[6]
zipcode_city = create_zipcode(zip_code, city, state)
agency = agencies[int(inst_line[2])]
inst = Institution(
year=inst_line[0],
respondent_id=inst_line[1],
agency=agency,
institution_id=inst_line[2] + inst_line[1],
tax_id=inst_line[3],
name=inst_line[4],
mailing_address=inst_line[5],
zip_code=zipcode_city,
assets=int(inst_line[17]),
)
institutions.append(inst)
Institution.objects.bulk_create(institutions)
def load_from_csv(
agencies: Dict[int, Agency], transmittal_reader: Iterator[List[str]]):
for zero_line_number, line in enumerate(transmittal_reader):
line_number = zero_line_number + 1
line = fixup(line)
if len(line) not in (15, 22):
logger.warning(
"Line %s is invalid, has length %s (expected 15 or 22)",
line_number,
len(line),
)
break
year = int(line[0])
zipcode_city = create_zipcode(
city=line[6], state=line[7], year=year, zip_code=line[8])
agency_id = line[2]
if not agency_id.isdigit() or int(agency_id) not in agencies:
logger.warning("Line %s has an invalid agency: %s",
line_number, agency_id)
break
if len(line) == 22 and not line[17].isdigit():
logger.warning("Line %s has an invalid asset amount: %s",
line_number, line[17])
break
respondent_id = line[1]
inst = Institution(
year=year,
respondent_id=line[1],
agency=agencies[int(agency_id)],
institution_id=f"{year}{agency_id}{respondent_id}",
tax_id=line[3],
name=line[4],
mailing_address=line[5],
zip_code=zipcode_city,
assets=int(line[17]) if len(line) == 22 else None,
)
try:
inst.full_clean(exclude=["agency"], validate_unique=False)
except ValidationError:
logger.exception("Line %s has invalid data:", line_number)
break
yield inst
def handle(self, *args, **options):
transmittal_filename = args[0]
agencies = Agency.objects.get_all_by_code()
with open(transmittal_filename) as institutioncsv:
transmittal_reader = csv.reader(institutioncsv, delimiter='\t')
institutions = []
# count = 1 # use if want to see which item failed, see comment below where we create institution individually
for inst_line in transmittal_reader:
year = inst_line[0]
zip_code = inst_line[8]
state = inst_line[7]
city = inst_line[6]
zipcode_city = create_zipcode(zip_code, city, state, year)
agency = agencies[int(inst_line[2])]
inst = Institution(
year=year,
respondent_id=inst_line[1],
agency=agency,
institution_id=year + inst_line[2] + inst_line[1],
tax_id=inst_line[3],
name=inst_line[4],
mailing_address=inst_line[5],
zip_code=zipcode_city,
assets=int(inst_line[17]),
)
# This can be used to figure out which exact item was failing, will need to disable bulk create below to use this
# Institution.objects.create(inst)
# inst.save()
# count += 1
institutions.append(inst)
Institution.objects.bulk_create(institutions)
def handle(self, *args, **options):
transmittal_filename = args[0]
agencies = Agency.objects.get_all_by_code()
with open(transmittal_filename) as institutioncsv:
transmittal_reader = csv.reader(institutioncsv, delimiter='\t')
institutions = []
# count = 1 # use if want to see which item failed, see comment below where we create institution individually
for inst_line in transmittal_reader:
year = inst_line[0]
zip_code = inst_line[8]
state = inst_line[7]
city = inst_line[6]
zipcode_city = create_zipcode(zip_code, city, state, year)
agency = agencies[int(inst_line[2])]
inst = Institution(
year=year,
respondent_id=inst_line[1],
agency=agency,
institution_id=year+inst_line[2]+inst_line[1],
tax_id=inst_line[3],
name=inst_line[4],
mailing_address=inst_line[5],
zip_code=zipcode_city,
assets=int(inst_line[17]),
)
# This can be used to figure out which exact item was failing, will need to disable bulk create below to use this
# Institution.objects.create(inst)
# inst.save()
# count += 1
institutions.append(inst)
Institution.objects.bulk_create(institutions)
def test_duplicate_entries(self):
""" We insert a duplicate entry, and check that it wasn't in fact
duplicated. """
zipcode_utils.create_zipcode('20852', 'Rockville', 'MD')
results = ZipcodeCityState.objects.filter(state='MD')
self.assertEqual(1, len(results))
def test_duplicate_entries(self):
""" We insert a duplicate entry, and check that it wasn't in fact
duplicated. """
zipcode_utils.create_zipcode('20852', 'Rockville', 'MD', '2013')
results = ZipcodeCityStateYear.objects.filter(state='MD')
self.assertEqual(1, len(results))
