def testDownloadCSV(self):
"""Downloading bulk data on a state in CSV format"""
zip_file = BytesIO()
pyopenstates.download_bulk_data("AK", zip_file, data_format="csv")
zip = ZipFile(zip_file)
for filename in zip.namelist():
self.assertTrue(filename.endswith(".csv"))
def testDownloadJSON(self):
"""Downloading bulk data on a state in JSON format"""
zip_file = BytesIO()
pyopenstates.download_bulk_data("AK", zip_file)
zip = ZipFile(zip_file)
self.assertIn("metadata.json", zip.namelist())
def update_state_bulk(state_name, openstates_state_abbrev):
"""
This method is deprecated
Update state information using bulk data, which appears to be aggregiously outdated
"""
tempdir = None
try:
tempdir = tempfile.mkdtemp()
sys.stderr.write("temporary directory: {}\n".format(tempdir))
workingdir = tempdir + "/openstates_data_workingdir"
jsonzip = tempdir + "/state-json.zip"
num_new_legislators = 0
num_updated_legislators = 0
num_legislator_exceptions = 0
num_new_bills = 0
num_updated_bills = 0
num_bill_exceptions = 0
sys.stdout.write(state_name + "\n")
sys.stdout.flush()
state_geotag = cm.GeoTag.objects.get(feature_type="SP",
name=state_name)
with open(jsonzip, "wb") as state_zip_file:
sys.stdout.write("downloading...\n")
pyopenstates.download_bulk_data(openstates_state_abbrev,
state_zip_file)
sys.stdout.write("unzipping...\n")
zip_ref = zipfile.ZipFile(jsonzip, 'r')
zip_ref.extractall(workingdir)
zip_ref.close()
legislator_files = os.listdir(workingdir + "/legislators")
sys.stdout.write("number of legislators:" +
str(len(legislator_files)) + "\n")
for f in legislator_files:
with open(workingdir + "/legislators/" + f, 'r') as leg_file:
leg_json = json.loads(leg_file.read())
result = update_legislator(leg_json, state_geotag)
if result == "update":
num_updated_legislators += 1
elif result == "new":
num_new_legislators += 1
sys.stdout.write(
"Number of legislators added: {}\n".format(num_new_legislators))
sys.stdout.write("Number of legislators updated: {}\n".format(
num_updated_legislators))
sys.stdout.write(
"Number of exceptions updating a legislator: {}\n".format(
num_legislator_exceptions))
bill_files = [
os.path.join(root, name)
for root, dirs, files in os.walk(workingdir + "/bills")
for name in files
]
sys.stdout.write("number of bills:" + str(len(bill_files)) + "\n")
for f in bill_files:
with open(f, 'r') as bill_file:
bill_json = json.loads(bill_file.read())
result = update_bill(bill_json, state_geotag)
if result == "update":
num_updated_bills += 1
if num_updated_bills % 100 == 0:
sys.stderr.write(
"Updating bill #{}\n".format(num_updated_bills))
elif result == "new":
num_new_bills += 1
if num_new_bills % 100 == 0:
sys.stderr.write(
"Adding new bill #{}\n".format(num_new_bills))
committee_files = os.listdir(workingdir + "/committees")
sys.stdout.write("number of committees:" + str(len(committee_files)) +
"\n")
sys.stdout.write("Number of bills added: {}\n".format(num_new_bills))
sys.stdout.write(
"Number of bills updated: {}\n".format(num_updated_bills))
sys.stdout.write("DONE\n")
except cm.GeoTag.DoesNotExist:
sys.stdout.write("state not in geotags\n")
except zipfile.BadZipfile:
sys.stdout.write("there was a problem reading this state's zipfile\n")
finally:
sys.stdout.flush()
shutil.rmtree(tempdir)