def test_unmatched_voters():
session, org, person = create_data()
b = Bill.objects.create(identifier='HB2', title='Two', legislative_session_id=session)
v1 = VoteEvent.objects.create(legislative_session_id=session, motion_text='Passage', bill=b,
organization=org)
v2 = VoteEvent.objects.create(legislative_session_id=session, motion_text='Override', bill=b,
organization=org)
report = generate_session_report(session)
assert report.unmatched_voters == {}
# add voters
v1.votes.create(option='yes', voter_name='Roy')
v1.votes.create(option='yes', voter_name='Wendy')
v2.votes.create(option='yes', voter_name='Wendy')
report = generate_session_report(session)
assert len(report.unmatched_voters) == 2
assert report.unmatched_voters['Roy'] == 1
assert report.unmatched_voters['Wendy'] == 2
# ensure that Roy goes away when linked
voter = v1.votes.get(voter_name='Roy')
voter.voter_id = person.id
voter.save()
report = generate_session_report(session)
assert report.unmatched_voters == {'Wendy': 2}
def test_bills_missing_versions():
session, org, person = create_data()
Bill.objects.create(identifier='HB1', title='One', legislative_session_id=session)
b = Bill.objects.create(identifier='HB2', title='Two', legislative_session_id=session)
report = generate_session_report(session)
assert report.bills_missing_versions == 2
b.versions.create(note='Final Copy')
report = generate_session_report(session)
assert report.bills_missing_versions == 1
def test_bills_missing_sponsors():
session, org, person = create_data()
Bill.objects.create(identifier='HB1', title='One', legislative_session_id=session)
b = Bill.objects.create(identifier='HB2', title='Two', legislative_session_id=session)
report = generate_session_report(session)
assert report.bills_missing_sponsors == 2
b.sponsorships.create(name='Roy', entity_type='person')
report = generate_session_report(session)
assert report.bills_missing_sponsors == 1
def test_bills_missing_actions():
session, org, person = create_data()
Bill.objects.create(identifier='HB1', title='One', legislative_session_id=session)
b = Bill.objects.create(identifier='HB2', title='Two', legislative_session_id=session)
report = generate_session_report(session)
assert report.bills_missing_actions == 2
b.actions.create(description='Introduced', order=1, organization=org)
report = generate_session_report(session)
assert report.bills_missing_actions == 1
def test_votes_missing_voters():
session, org, person = create_data()
b = Bill.objects.create(identifier='HB2', title='Two', legislative_session_id=session)
v = VoteEvent.objects.create(legislative_session_id=session, motion_text='Passage', bill=b,
organization=org)
VoteEvent.objects.create(legislative_session_id=session, motion_text='Amendment', bill=b,
organization=org)
report = generate_session_report(session)
assert report.votes_missing_voters == 2
v.votes.create(option='yes', voter_name='Speaker')
report = generate_session_report(session)
assert report.votes_missing_voters == 1
def test_votes_missing_bill():
session, org, person = create_data()
b = Bill.objects.create(identifier='HB2', title='Two', legislative_session_id=session)
v = VoteEvent.objects.create(legislative_session_id=session, motion_text='Passage',
organization=org)
VoteEvent.objects.create(legislative_session_id=session, motion_text='Amendment',
organization=org)
report = generate_session_report(session)
assert report.votes_missing_bill == 2
v.bill = b
v.save()
report = generate_session_report(session)
assert report.votes_missing_bill == 1
def do_import(self, juris, args):
# import inside here because to avoid loading Django code unnecessarily
from pupa.importers import (JurisdictionImporter, OrganizationImporter, PersonImporter,
PostImporter, MembershipImporter, BillImporter,
VoteEventImporter, EventImporter)
from pupa.reports import generate_session_report
from pupa.models import SessionDataQualityReport
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id, person_importer,
org_importer, post_importer)
bill_importer = BillImporter(juris.jurisdiction_id, org_importer, person_importer)
vote_event_importer = VoteEventImporter(juris.jurisdiction_id, person_importer,
org_importer, bill_importer)
event_importer = EventImporter(juris.jurisdiction_id,
org_importer,
person_importer,
bill_importer,
vote_event_importer)
report = {}
with transaction.atomic():
print('import jurisdictions...')
report.update(juris_importer.import_directory(datadir))
if settings.ENABLE_PEOPLE_AND_ORGS:
print('import organizations...')
report.update(org_importer.import_directory(datadir))
print('import people...')
report.update(person_importer.import_directory(datadir))
print('import posts...')
report.update(post_importer.import_directory(datadir))
print('import memberships...')
report.update(membership_importer.import_directory(datadir))
if settings.ENABLE_BILLS:
print('import bills...')
report.update(bill_importer.import_directory(datadir))
if settings.ENABLE_EVENTS:
print('import events...')
report.update(event_importer.import_directory(datadir))
if settings.ENABLE_VOTES:
print('import vote events...')
report.update(vote_event_importer.import_directory(datadir))
# compile info on all sessions that were updated in this run
seen_sessions = set()
seen_sessions.update(bill_importer.get_seen_sessions())
seen_sessions.update(vote_event_importer.get_seen_sessions())
for session in seen_sessions:
new_report = generate_session_report(session)
with transaction.atomic():
SessionDataQualityReport.objects.filter(legislative_session=session).delete()
new_report.save()
return report
def test_missing_yes_no_counts():
session, org, person = create_data()
b = Bill.objects.create(identifier='HB2', title='Two', legislative_session_id=session)
v = VoteEvent.objects.create(legislative_session_id=session, motion_text='Passage', bill=b,
organization=org)
VoteEvent.objects.create(legislative_session_id=session, motion_text='Amendment', bill=b,
organization=org)
report = generate_session_report(session)
assert report.votes_missing_yes_count == 2
assert report.votes_missing_no_count == 2
v.counts.create(option='yes', value=1)
report = generate_session_report(session)
assert report.votes_missing_yes_count == 1
assert report.votes_missing_no_count == 2
v.counts.create(option='no', value=0)
report = generate_session_report(session)
assert report.votes_missing_yes_count == 1
assert report.votes_missing_no_count == 1
def test_votes_with_bad_counts():
session, org, person = create_data()
b = Bill.objects.create(identifier='HB2', title='Two', legislative_session_id=session)
v = VoteEvent.objects.create(legislative_session_id=session, motion_text='Passage', bill=b,
organization=org)
report = generate_session_report(session)
assert report.votes_with_bad_counts == 0
# add count, breaking
v.counts.create(option='yes', value=1)
report = generate_session_report(session)
assert report.votes_with_bad_counts == 1
# add voter, fixing
v.votes.create(option='yes', voter_name='One')
report = generate_session_report(session)
assert report.votes_with_bad_counts == 0
# add voter, breaking
v.votes.create(option='no', voter_name='Two')
report = generate_session_report(session)
assert report.votes_with_bad_counts == 1
# add count, still not equal
v.counts.create(option='no', value=2)
report = generate_session_report(session)
assert report.votes_with_bad_counts == 1
# add voter, fixing
v.votes.create(option='no', voter_name='Three')
report = generate_session_report(session)
assert report.votes_with_bad_counts == 0
def test_unmatched_sponsors():
session, org, person = create_data()
b1 = Bill.objects.create(identifier='HB1', title='One', legislative_session_id=session)
b2 = Bill.objects.create(identifier='HB2', title='Two', legislative_session_id=session)
b1.sponsorships.create(name='Roy', entity_type='person')
b1.sponsorships.create(name='Wendy', entity_type='person')
b1.sponsorships.create(name='Committee On Legislation', entity_type='organization')
b2.sponsorships.create(name='Wendy', entity_type='person')
report = generate_session_report(session)
assert len(report.unmatched_sponsor_people) == 2
assert report.unmatched_sponsor_people['Roy'] == 1
assert report.unmatched_sponsor_people['Wendy'] == 2
assert report.unmatched_sponsor_organizations == {'Committee On Legislation': 1}
# ensure that Roy goes away when linked
sp = b1.sponsorships.get(name='Roy')
sp.person_id = person.id
sp.save()
report = generate_session_report(session)
assert report.unmatched_sponsor_people == {'Wendy': 2}
def do_import(self, juris, args):
# import inside here because to avoid loading Django code unnecessarily
from pupa.importers import (JurisdictionImporter, OrganizationImporter,
PersonImporter, PostImporter,
MembershipImporter, BillImporter,
VoteEventImporter, EventImporter)
from pupa.reports import generate_session_report
from pupa.models import SessionDataQualityReport
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id,
person_importer, org_importer,
post_importer)
bill_importer = BillImporter(juris.jurisdiction_id, org_importer,
person_importer)
vote_event_importer = VoteEventImporter(juris.jurisdiction_id,
person_importer, org_importer,
bill_importer)
event_importer = EventImporter(juris.jurisdiction_id, org_importer,
person_importer, bill_importer,
vote_event_importer)
report = {}
with transaction.atomic():
print('import jurisdictions...')
report.update(juris_importer.import_directory(datadir))
if settings.ENABLE_PEOPLE_AND_ORGS:
print('import organizations...')
report.update(org_importer.import_directory(datadir))
print('import people...')
report.update(person_importer.import_directory(datadir))
print('import posts...')
report.update(post_importer.import_directory(datadir))
print('import memberships...')
report.update(membership_importer.import_directory(datadir))
if settings.ENABLE_BILLS:
print('import bills...')
report.update(bill_importer.import_directory(datadir))
if settings.ENABLE_EVENTS:
print('import events...')
report.update(event_importer.import_directory(datadir))
if settings.ENABLE_VOTES:
print('import vote events...')
report.update(vote_event_importer.import_directory(datadir))
# compile info on all sessions that were updated in this run
seen_sessions = set()
seen_sessions.update(bill_importer.get_seen_sessions())
seen_sessions.update(vote_event_importer.get_seen_sessions())
for session in seen_sessions:
new_report = generate_session_report(session)
with transaction.atomic():
SessionDataQualityReport.objects.filter(
legislative_session=session).delete()
new_report.save()
return report