def dump_bill_csvs(level, abbr): bill_fields = ('level', 'country', 'state', 'session', 'chamber', 'bill_id', 'title', 'created_at', 'updated_at', 'type', 'subjects') bill_csv_fname, bill_csv = _make_csv(abbr, 'bills.csv', bill_fields) action_fields = ('level', 'country', 'state', 'session', 'chamber', 'bill_id', 'date', 'action', 'actor', 'type') action_csv_fname, action_csv = _make_csv(abbr, 'bill_actions.csv', action_fields) sponsor_fields = ('level', 'country', 'state', 'session', 'chamber', 'bill_id', 'type', 'name', 'leg_id') sponsor_csv_fname, sponsor_csv = _make_csv(abbr, 'bill_sponsors.csv', sponsor_fields) vote_fields = ('level', 'country', 'state', 'session', 'chamber', 'bill_id', 'vote_id', 'vote_chamber', 'motion', 'date', 'type', 'yes_count', 'no_count', 'other_count') vote_csv_fname, vote_csv = _make_csv(abbr, 'bill_votes.csv', vote_fields) legvote_fields = ('vote_id', 'leg_id', 'name', 'vote') legvote_csv_fname, legvote_csv = _make_csv(abbr, 'bill_legislator_votes.csv', legvote_fields) for bill in db.bills.find({'level': level, level: abbr}): bill_csv.writerow(extract_fields(bill, bill_fields)) bill_info = extract_fields(bill, ('bill_id', 'level', 'country', 'state', 'session', 'chamber')) # basically same behavior for actions, sponsors and votes: # extract fields, update with bill_info, write to csv for action in bill['actions']: adict = extract_fields(action, action_fields) adict.update(bill_info) action_csv.writerow(adict) for sponsor in bill['sponsors']: sdict = extract_fields(sponsor, sponsor_fields) sdict.update(bill_info) sponsor_csv.writerow(sdict) for vote in bill['votes']: vdict = extract_fields(vote, vote_fields) # copy chamber from vote into vote_chamber vdict['vote_chamber'] = vdict['chamber'] vdict.update(bill_info) vote_csv.writerow(vdict) for vtype in ('yes', 'no', 'other'): for leg_vote in vote[vtype + '_votes']: legvote_csv.writerow({'vote_id': vote['vote_id'], 'leg_id': leg_vote['leg_id'], 'name': leg_vote['name'].encode('utf8'), 'vote': vtype}) return (bill_csv_fname, action_csv_fname, sponsor_csv_fname, vote_csv_fname, legvote_csv_fname)
def dump_legislator_csvs(state): leg_fields = ( "leg_id", "full_name", "first_name", "middle_name", "last_name", "suffixes", "nickname", "active", "state", "chamber", "district", "party", "votesmart_id", "transparencydata_id", "photo_url", "created_at", "updated_at", ) leg_csv_fname, leg_csv = _make_csv(state, "legislators.csv", leg_fields) role_fields = ( "leg_id", "type", "term", "district", "chamber", "state", "party", "committee_id", "committee", "subcommittee", "start_date", "end_date", ) role_csv_fname, role_csv = _make_csv(state, "legislator_roles.csv", role_fields) com_fields = ("id", "state", "chamber", "committee", "subcommittee", "parent_id") com_csv_fname, com_csv = _make_csv(state, "committees.csv", com_fields) for legislator in db.legislators.find({"state": state}): leg_csv.writerow(extract_fields(legislator, leg_fields)) # go through roles to create role csv all_roles = legislator["roles"] for roles in legislator.get("old_roles", {}).values(): all_roles.extend(roles) for role in all_roles: d = extract_fields(role, role_fields) d.update({"leg_id": legislator["leg_id"]}) role_csv.writerow(d) for committee in db.committees.find({"state": state}): cdict = extract_fields(committee, com_fields) cdict["id"] = committee["_id"] com_csv.writerow(cdict) return leg_csv_fname, role_csv_fname, com_csv_fname
def dump_legislator_csvs(state): leg_fields = ('leg_id', 'full_name', 'first_name', 'middle_name', 'last_name', 'suffixes', 'nickname', 'active', 'state', 'chamber', 'district', 'party', 'votesmart_id', 'transparencydata_id', 'photo_url', 'created_at', 'updated_at') leg_csv_fname, leg_csv = _make_csv(state, 'legislators.csv', leg_fields) role_fields = ('leg_id', 'type', 'term', 'district', 'chamber', 'state', 'party', 'committee_id', 'committee', 'subcommittee', 'start_date', 'end_date') role_csv_fname, role_csv = _make_csv(state, 'legislator_roles.csv', role_fields) com_fields = ('id', 'state', 'chamber', 'committee', 'subcommittee', 'parent_id') com_csv_fname, com_csv = _make_csv(state, 'committees.csv', com_fields) for legislator in db.legislators.find({'state': state}): leg_csv.writerow(extract_fields(legislator, leg_fields)) # go through roles to create role csv all_roles = legislator['roles'] for roles in legislator.get('old_roles', {}).values(): all_roles.extend(roles) for role in all_roles: d = extract_fields(role, role_fields) d.update({'leg_id':legislator['leg_id']}) role_csv.writerow(d) for committee in db.committees.find({'state': state}): cdict = extract_fields(committee, com_fields) cdict['id'] = committee['_id'] com_csv.writerow(cdict) return leg_csv_fname, role_csv_fname, com_csv_fname
def dump_legislator_csvs(level, abbr): leg_fields = ('leg_id', 'full_name', 'first_name', 'middle_name', 'last_name', 'suffixes', 'nickname', 'active', 'level', 'country', 'state', 'chamber', 'district', 'party', 'votesmart_id', 'transparencydata_id', 'photo_url', 'created_at', 'updated_at') leg_csv_fname, leg_csv = _make_csv(abbr, 'legislators.csv', leg_fields) role_fields = ('leg_id', 'type', 'term', 'district', 'chamber', 'level', 'country', 'state', 'party', 'committee_id', 'committee', 'subcommittee', 'start_date', 'end_date') role_csv_fname, role_csv = _make_csv(abbr, 'legislator_roles.csv', role_fields) com_fields = ('id', 'level', 'country', 'state', 'chamber', 'committee', 'subcommittee', 'parent_id') com_csv_fname, com_csv = _make_csv(abbr, 'committees.csv', com_fields) for legislator in db.legislators.find({'level': level, level: abbr}): leg_csv.writerow(extract_fields(legislator, leg_fields)) # go through roles to create role csv all_roles = legislator['roles'] for roles in legislator.get('old_roles', {}).values(): all_roles.extend(roles) for role in all_roles: d = extract_fields(role, role_fields) d.update({'leg_id': legislator['leg_id']}) role_csv.writerow(d) for committee in db.committees.find({'level': level, level: abbr}): cdict = extract_fields(committee, com_fields) cdict['id'] = committee['_id'] com_csv.writerow(cdict) return leg_csv_fname, role_csv_fname, com_csv_fname
def dump_bill_csvs(state): bill_fields = ("state", "session", "chamber", "bill_id", "title", "created_at", "updated_at", "type", "subjects") bill_csv_fname, bill_csv = _make_csv(state, "bills.csv", bill_fields) action_fields = ("state", "session", "chamber", "bill_id", "date", "action", "actor", "type") action_csv_fname, action_csv = _make_csv(state, "bill_actions.csv", action_fields) sponsor_fields = ("state", "session", "chamber", "bill_id", "type", "name", "leg_id") sponsor_csv_fname, sponsor_csv = _make_csv(state, "bill_sponsors.csv", sponsor_fields) vote_fields = ( "state", "session", "chamber", "bill_id", "vote_id", "vote_chamber", "motion", "date", "type", "yes_count", "no_count", "other_count", ) vote_csv_fname, vote_csv = _make_csv(state, "bill_votes.csv", vote_fields) legvote_fields = ("vote_id", "leg_id", "name", "vote") legvote_csv_fname, legvote_csv = _make_csv(state, "bill_legislator_votes.csv", legvote_fields) for bill in db.bills.find({"state": state}): bill_csv.writerow(extract_fields(bill, bill_fields)) bill_info = extract_fields(bill, ("bill_id", "state", "session", "chamber")) # basically same behavior for actions, sponsors and votes: # extract fields, update with bill_info, write to csv for action in bill["actions"]: adict = extract_fields(action, action_fields) adict.update(bill_info) action_csv.writerow(adict) for sponsor in bill["sponsors"]: sdict = extract_fields(sponsor, sponsor_fields) sdict.update(bill_info) sponsor_csv.writerow(sdict) for vote in bill["votes"]: vdict = extract_fields(vote, vote_fields) # copy chamber from vote into vote_chamber vdict["vote_chamber"] = vdict["chamber"] vdict.update(bill_info) vote_csv.writerow(vdict) for vtype in ("yes", "no", "other"): for leg_vote in vote[vtype + "_votes"]: legvote_csv.writerow( { "vote_id": vote["vote_id"], "leg_id": leg_vote["leg_id"], "name": leg_vote["name"].encode("utf8"), "vote": vtype, } ) return (bill_csv_fname, action_csv_fname, sponsor_csv_fname, vote_csv_fname, legvote_csv_fname)