def test_unmatched_voters(): session, org, person = create_data() b = Bill.objects.create(identifier='HB2', title='Two', legislative_session_id=session) v1 = VoteEvent.objects.create(legislative_session_id=session, motion_text='Passage', bill=b, organization=org) v2 = VoteEvent.objects.create(legislative_session_id=session, motion_text='Override', bill=b, organization=org) report = generate_session_report(session) assert report.unmatched_voters == {} # add voters v1.votes.create(option='yes', voter_name='Roy') v1.votes.create(option='yes', voter_name='Wendy') v2.votes.create(option='yes', voter_name='Wendy') report = generate_session_report(session) assert len(report.unmatched_voters) == 2 assert report.unmatched_voters['Roy'] == 1 assert report.unmatched_voters['Wendy'] == 2 # ensure that Roy goes away when linked voter = v1.votes.get(voter_name='Roy') voter.voter_id = person.id voter.save() report = generate_session_report(session) assert report.unmatched_voters == {'Wendy': 2}
def test_bills_missing_versions(): session, org, person = create_data() Bill.objects.create(identifier='HB1', title='One', legislative_session_id=session) b = Bill.objects.create(identifier='HB2', title='Two', legislative_session_id=session) report = generate_session_report(session) assert report.bills_missing_versions == 2 b.versions.create(note='Final Copy') report = generate_session_report(session) assert report.bills_missing_versions == 1
def test_bills_missing_sponsors(): session, org, person = create_data() Bill.objects.create(identifier='HB1', title='One', legislative_session_id=session) b = Bill.objects.create(identifier='HB2', title='Two', legislative_session_id=session) report = generate_session_report(session) assert report.bills_missing_sponsors == 2 b.sponsorships.create(name='Roy', entity_type='person') report = generate_session_report(session) assert report.bills_missing_sponsors == 1
def test_bills_missing_actions(): session, org, person = create_data() Bill.objects.create(identifier='HB1', title='One', legislative_session_id=session) b = Bill.objects.create(identifier='HB2', title='Two', legislative_session_id=session) report = generate_session_report(session) assert report.bills_missing_actions == 2 b.actions.create(description='Introduced', order=1, organization=org) report = generate_session_report(session) assert report.bills_missing_actions == 1
def test_votes_missing_voters(): session, org, person = create_data() b = Bill.objects.create(identifier='HB2', title='Two', legislative_session_id=session) v = VoteEvent.objects.create(legislative_session_id=session, motion_text='Passage', bill=b, organization=org) VoteEvent.objects.create(legislative_session_id=session, motion_text='Amendment', bill=b, organization=org) report = generate_session_report(session) assert report.votes_missing_voters == 2 v.votes.create(option='yes', voter_name='Speaker') report = generate_session_report(session) assert report.votes_missing_voters == 1
def test_votes_missing_bill(): session, org, person = create_data() b = Bill.objects.create(identifier='HB2', title='Two', legislative_session_id=session) v = VoteEvent.objects.create(legislative_session_id=session, motion_text='Passage', organization=org) VoteEvent.objects.create(legislative_session_id=session, motion_text='Amendment', organization=org) report = generate_session_report(session) assert report.votes_missing_bill == 2 v.bill = b v.save() report = generate_session_report(session) assert report.votes_missing_bill == 1
def do_import(self, juris, args): # import inside here because to avoid loading Django code unnecessarily from pupa.importers import (JurisdictionImporter, OrganizationImporter, PersonImporter, PostImporter, MembershipImporter, BillImporter, VoteEventImporter, EventImporter) from pupa.reports import generate_session_report from pupa.models import SessionDataQualityReport datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module) juris_importer = JurisdictionImporter(juris.jurisdiction_id) org_importer = OrganizationImporter(juris.jurisdiction_id) person_importer = PersonImporter(juris.jurisdiction_id) post_importer = PostImporter(juris.jurisdiction_id, org_importer) membership_importer = MembershipImporter(juris.jurisdiction_id, person_importer, org_importer, post_importer) bill_importer = BillImporter(juris.jurisdiction_id, org_importer, person_importer) vote_event_importer = VoteEventImporter(juris.jurisdiction_id, person_importer, org_importer, bill_importer) event_importer = EventImporter(juris.jurisdiction_id, org_importer, person_importer, bill_importer, vote_event_importer) report = {} with transaction.atomic(): print('import jurisdictions...') report.update(juris_importer.import_directory(datadir)) if settings.ENABLE_PEOPLE_AND_ORGS: print('import organizations...') report.update(org_importer.import_directory(datadir)) print('import people...') report.update(person_importer.import_directory(datadir)) print('import posts...') report.update(post_importer.import_directory(datadir)) print('import memberships...') report.update(membership_importer.import_directory(datadir)) if settings.ENABLE_BILLS: print('import bills...') report.update(bill_importer.import_directory(datadir)) if settings.ENABLE_EVENTS: print('import events...') report.update(event_importer.import_directory(datadir)) if settings.ENABLE_VOTES: print('import vote events...') report.update(vote_event_importer.import_directory(datadir)) # compile info on all sessions that were updated in this run seen_sessions = set() seen_sessions.update(bill_importer.get_seen_sessions()) seen_sessions.update(vote_event_importer.get_seen_sessions()) for session in seen_sessions: new_report = generate_session_report(session) with transaction.atomic(): SessionDataQualityReport.objects.filter(legislative_session=session).delete() new_report.save() return report
def test_missing_yes_no_counts(): session, org, person = create_data() b = Bill.objects.create(identifier='HB2', title='Two', legislative_session_id=session) v = VoteEvent.objects.create(legislative_session_id=session, motion_text='Passage', bill=b, organization=org) VoteEvent.objects.create(legislative_session_id=session, motion_text='Amendment', bill=b, organization=org) report = generate_session_report(session) assert report.votes_missing_yes_count == 2 assert report.votes_missing_no_count == 2 v.counts.create(option='yes', value=1) report = generate_session_report(session) assert report.votes_missing_yes_count == 1 assert report.votes_missing_no_count == 2 v.counts.create(option='no', value=0) report = generate_session_report(session) assert report.votes_missing_yes_count == 1 assert report.votes_missing_no_count == 1
def test_votes_with_bad_counts(): session, org, person = create_data() b = Bill.objects.create(identifier='HB2', title='Two', legislative_session_id=session) v = VoteEvent.objects.create(legislative_session_id=session, motion_text='Passage', bill=b, organization=org) report = generate_session_report(session) assert report.votes_with_bad_counts == 0 # add count, breaking v.counts.create(option='yes', value=1) report = generate_session_report(session) assert report.votes_with_bad_counts == 1 # add voter, fixing v.votes.create(option='yes', voter_name='One') report = generate_session_report(session) assert report.votes_with_bad_counts == 0 # add voter, breaking v.votes.create(option='no', voter_name='Two') report = generate_session_report(session) assert report.votes_with_bad_counts == 1 # add count, still not equal v.counts.create(option='no', value=2) report = generate_session_report(session) assert report.votes_with_bad_counts == 1 # add voter, fixing v.votes.create(option='no', voter_name='Three') report = generate_session_report(session) assert report.votes_with_bad_counts == 0
def test_unmatched_sponsors(): session, org, person = create_data() b1 = Bill.objects.create(identifier='HB1', title='One', legislative_session_id=session) b2 = Bill.objects.create(identifier='HB2', title='Two', legislative_session_id=session) b1.sponsorships.create(name='Roy', entity_type='person') b1.sponsorships.create(name='Wendy', entity_type='person') b1.sponsorships.create(name='Committee On Legislation', entity_type='organization') b2.sponsorships.create(name='Wendy', entity_type='person') report = generate_session_report(session) assert len(report.unmatched_sponsor_people) == 2 assert report.unmatched_sponsor_people['Roy'] == 1 assert report.unmatched_sponsor_people['Wendy'] == 2 assert report.unmatched_sponsor_organizations == {'Committee On Legislation': 1} # ensure that Roy goes away when linked sp = b1.sponsorships.get(name='Roy') sp.person_id = person.id sp.save() report = generate_session_report(session) assert report.unmatched_sponsor_people == {'Wendy': 2}
def do_import(self, juris, args): # import inside here because to avoid loading Django code unnecessarily from pupa.importers import (JurisdictionImporter, OrganizationImporter, PersonImporter, PostImporter, MembershipImporter, BillImporter, VoteEventImporter, EventImporter) from pupa.reports import generate_session_report from pupa.models import SessionDataQualityReport datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module) juris_importer = JurisdictionImporter(juris.jurisdiction_id) org_importer = OrganizationImporter(juris.jurisdiction_id) person_importer = PersonImporter(juris.jurisdiction_id) post_importer = PostImporter(juris.jurisdiction_id, org_importer) membership_importer = MembershipImporter(juris.jurisdiction_id, person_importer, org_importer, post_importer) bill_importer = BillImporter(juris.jurisdiction_id, org_importer, person_importer) vote_event_importer = VoteEventImporter(juris.jurisdiction_id, person_importer, org_importer, bill_importer) event_importer = EventImporter(juris.jurisdiction_id, org_importer, person_importer, bill_importer, vote_event_importer) report = {} with transaction.atomic(): print('import jurisdictions...') report.update(juris_importer.import_directory(datadir)) if settings.ENABLE_PEOPLE_AND_ORGS: print('import organizations...') report.update(org_importer.import_directory(datadir)) print('import people...') report.update(person_importer.import_directory(datadir)) print('import posts...') report.update(post_importer.import_directory(datadir)) print('import memberships...') report.update(membership_importer.import_directory(datadir)) if settings.ENABLE_BILLS: print('import bills...') report.update(bill_importer.import_directory(datadir)) if settings.ENABLE_EVENTS: print('import events...') report.update(event_importer.import_directory(datadir)) if settings.ENABLE_VOTES: print('import vote events...') report.update(vote_event_importer.import_directory(datadir)) # compile info on all sessions that were updated in this run seen_sessions = set() seen_sessions.update(bill_importer.get_seen_sessions()) seen_sessions.update(vote_event_importer.get_seen_sessions()) for session in seen_sessions: new_report = generate_session_report(session) with transaction.atomic(): SessionDataQualityReport.objects.filter( legislative_session=session).delete() new_report.save() return report