def test_related_people_event(): create_jurisdiction() george = Person.objects.create(id='gw', name='George Washington') john = Person.objects.create(id='jqp', name='John Q. Public') o = Organization.objects.create(name='Merica', jurisdiction_id='jid') Membership.objects.create(person=george, organization=o) Membership.objects.create(person=john, organization=o) event1 = ge() event2 = ge() for event in [event1, event2]: item = event.add_agenda_item("Cookies will be served") item.add_person(person="John Q. Public") event.add_person("George Washington") result = EventImporter('jid', oi, pi, bi, vei).import_data([event1.as_dict()]) assert result['event']['insert'] == 1 result = EventImporter('jid', oi, pi, bi, vei).import_data([event2.as_dict()]) assert result['event']['noop'] == 1 assert Event.objects.get(name="America's Birthday").participants.first().person_id == 'gw' assert Event.objects.get(name="America's Birthday" ).agenda.first().related_entities.first().person_id == 'jqp'
def test_full_event(): create_jurisdiction() george = Person.objects.create(id='gw', name='George Washington') o = Organization.objects.create(name='Merica', jurisdiction_id='jid') Membership.objects.create(person=george, organization=o) event = ge() result = EventImporter('jid', oi, pi, bi, vei).import_data([event.as_dict()]) assert result['event']['insert'] == 1 event = ge() result = EventImporter('jid', oi, pi, bi, vei).import_data([event.as_dict()]) assert result['event']['noop'] == 1 event = ge() event.location['name'] = "United States of America" result = EventImporter('jid', oi, pi, bi, vei).import_data([event.as_dict()]) assert result['event']['update'] == 1 event.location = None result = EventImporter('jid', oi, pi, bi, vei).import_data([event.as_dict()]) assert result['event']['update'] == 1
def test_pupa_identifier_event(): create_jurisdiction() create_other_jurisdiction() george = Person.objects.create(id='gw', name='George Washington') o = Organization.objects.create(name='Merica', jurisdiction_id='jid') Membership.objects.create(person=george, organization=o) event = ge() event.pupa_id = 'foo' result = EventImporter('jid', oi, pi, bi, vei).import_data([event.as_dict()]) assert result['event']['insert'] == 1 result = EventImporter('jid', oi, pi, bi, vei).import_data([event.as_dict()]) assert result['event']['noop'] == 1 event.name = "America's Anniversary", event.location['name'] = "United States of America" result = EventImporter('jid', oi, pi, bi, vei).import_data([event.as_dict()]) assert result['event']['update'] == 1 event.pupa_id = 'bar' result = EventImporter('jid', oi, pi, bi, vei).import_data([event.as_dict()]) assert result['event']['insert'] == 1 result = EventImporter('ojid', oi, pi, bi, vei).import_data([event.as_dict()]) assert result['event']['insert'] == 1
def test_related_committee_event(): j = create_jurisdiction() j.legislative_sessions.create(name='1900', identifier='1900') org = Organization.objects.create(id='org-id', name='House', classification='lower', jurisdiction=j) Organization.objects.create(id='fiscal', name="Fiscal Committee", classification='committee', parent=org, jurisdiction=j) event1 = ge() event2 = ge() for event in [event1, event2]: item = event.add_agenda_item("Cookies will be served") item.add_committee(committee="Fiscal Committee") result = EventImporter('jid', oi, pi, bi, vei).import_data([event1.as_dict()]) assert result['event']['insert'] == 1 result = EventImporter('jid', oi, pi, bi, vei).import_data([event2.as_dict()]) assert result['event']['noop'] == 1 assert Event.objects.get(name="America's Birthday" ).agenda.first().related_entities.first().organization_id == 'fiscal'
def test_related_vote_event(): j = create_jurisdiction() session = j.legislative_sessions.create(name='1900', identifier='1900') org = Organization.objects.create(id='org-id', name='House', classification='lower') bill = Bill.objects.create(id='bill-1', identifier='HB 1', legislative_session=session) VoteEvent.objects.create(id='vote-1', identifier="Roll no. 12", bill=bill, legislative_session=session, organization=org) event1 = ge() event2 = ge() for event in [event1, event2]: item = event.add_agenda_item("Cookies will be served") item.add_vote_event(vote_event="Roll no. 12") result = EventImporter('jid', oi, pi, bi, vei).import_data([event1.as_dict()]) assert result['event']['insert'] == 1 result = EventImporter('jid', oi, pi, bi, vei).import_data([event2.as_dict()]) assert result['event']['noop'] == 1 assert Event.objects.get(name="America's Birthday" ).agenda.first().related_entities.first().vote_event_id == 'vote-1'
def do_import(self, juris, args): # import inside here because to avoid loading Django code unnecessarily from pupa.importers import (JurisdictionImporter, OrganizationImporter, PersonImporter, PostImporter, MembershipImporter, BillImporter, VoteEventImporter, EventImporter) from pupa.reports import generate_session_report from pupa.models import SessionDataQualityReport datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module) juris_importer = JurisdictionImporter(juris.jurisdiction_id) org_importer = OrganizationImporter(juris.jurisdiction_id) person_importer = PersonImporter(juris.jurisdiction_id) post_importer = PostImporter(juris.jurisdiction_id, org_importer) membership_importer = MembershipImporter(juris.jurisdiction_id, person_importer, org_importer, post_importer) bill_importer = BillImporter(juris.jurisdiction_id, org_importer, person_importer) vote_event_importer = VoteEventImporter(juris.jurisdiction_id, person_importer, org_importer, bill_importer) event_importer = EventImporter(juris.jurisdiction_id, org_importer, person_importer, bill_importer, vote_event_importer) report = {} with transaction.atomic(): print('import jurisdictions...') report.update(juris_importer.import_directory(datadir)) if settings.ENABLE_PEOPLE_AND_ORGS: print('import organizations...') report.update(org_importer.import_directory(datadir)) print('import people...') report.update(person_importer.import_directory(datadir)) print('import posts...') report.update(post_importer.import_directory(datadir)) print('import memberships...') report.update(membership_importer.import_directory(datadir)) if settings.ENABLE_BILLS: print('import bills...') report.update(bill_importer.import_directory(datadir)) if settings.ENABLE_EVENTS: print('import events...') report.update(event_importer.import_directory(datadir)) if settings.ENABLE_VOTES: print('import vote events...') report.update(vote_event_importer.import_directory(datadir)) # compile info on all sessions that were updated in this run seen_sessions = set() seen_sessions.update(bill_importer.get_seen_sessions()) seen_sessions.update(vote_event_importer.get_seen_sessions()) for session in seen_sessions: new_report = generate_session_report(session) with transaction.atomic(): SessionDataQualityReport.objects.filter(legislative_session=session).delete() new_report.save() return report
def do_import(self, juris, args): # import inside here because to avoid loading Django code unnecessarily from pupa.importers import (JurisdictionImporter, OrganizationImporter, PersonImporter, PostImporter, MembershipImporter, BillImporter, VoteImporter, EventImporter, DisclosureImporter) datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module) juris_importer = JurisdictionImporter(juris.jurisdiction_id) org_importer = OrganizationImporter(juris.jurisdiction_id) person_importer = PersonImporter(juris.jurisdiction_id) post_importer = PostImporter(juris.jurisdiction_id, org_importer) membership_importer = MembershipImporter(juris.jurisdiction_id, person_importer, org_importer, post_importer) bill_importer = BillImporter(juris.jurisdiction_id, org_importer, person_importer) vote_importer = VoteImporter(juris.jurisdiction_id, person_importer, org_importer, bill_importer) event_importer = EventImporter(juris.jurisdiction_id, org_importer, person_importer) disclosure_importer = DisclosureImporter(juris.jurisdiction_id, org_importer, person_importer, event_importer) report = {} with transaction.atomic(): print('import jurisdictions...') report.update(juris_importer.import_directory(datadir)) print('import organizations...') report.update(org_importer.import_directory(datadir)) print('import people...') report.update(person_importer.import_directory(datadir)) print('import posts...') report.update(post_importer.import_directory(datadir)) print('import memberships...') report.update(membership_importer.import_directory(datadir)) print('import bills...') report.update(bill_importer.import_directory(datadir)) print('import events...') report.update(event_importer.import_directory(datadir)) print('import disclosures...') report.update(disclosure_importer.import_directory(datadir)) print('import votes...') report.update(vote_importer.import_directory(datadir)) return report
def test_top_level_media_event(): create_jurisdiction() event1, event2 = ge(), ge() event1.add_media_link("fireworks", "http://example.com/fireworks.mov", media_type='application/octet-stream') event2.add_media_link("fireworks", "http://example.com/fireworks.mov", media_type='application/octet-stream') result = EventImporter('jid', oi, pi, bi, vei).import_data([event1.as_dict()]) assert result['event']['insert'] == 1 result = EventImporter('jid', oi, pi, bi, vei).import_data([event2.as_dict()]) assert result['event']['noop'] == 1
def test_media_document(): create_jurisdiction() event1 = ge() event2 = ge() for event in [event1, event2]: event.add_document(note="Presentation", url="http://example.com/presentation.pdf") result = EventImporter('jid', oi, pi, bi, vei).import_data([event1.as_dict()]) assert result['event']['insert'] == 1 result = EventImporter('jid', oi, pi, bi, vei).import_data([event2.as_dict()]) assert result['event']['noop'] == 1
def test_related_committee_event(): Jurisdiction.objects.create(id='jid', division_id='did') event1 = ge() event2 = ge() for event in [event1, event2]: item = event.add_agenda_item("Cookies will be served") item.add_committee(committee="Fiscal Committee") result = EventImporter('jid').import_data([event1.as_dict()]) assert result['event']['insert'] == 1 result = EventImporter('jid').import_data([event2.as_dict()]) assert result['event']['noop'] == 1
def test_related_people_event(): Jurisdiction.objects.create(id='jid', division_id='did') event1 = ge() event2 = ge() for event in [event1, event2]: item = event.add_agenda_item("Cookies will be served") item.add_person(person="John Q. Public") result = EventImporter('jid').import_data([event1.as_dict()]) assert result['event']['insert'] == 1 result = EventImporter('jid').import_data([event2.as_dict()]) assert result['event']['noop'] == 1
def test_full_event(): Jurisdiction.objects.create(id='jid', division_id='did') event = ge() result = EventImporter('jid').import_data([event.as_dict()]) assert result['event']['insert'] == 1 result = EventImporter('jid').import_data([event.as_dict()]) assert result['event']['noop'] == 1 event = ge() event.location['name'] = "United States of America" result = EventImporter('jid').import_data([event.as_dict()]) assert result['event']['update'] == 1
def test_media_document(): Jurisdiction.objects.create(id='jid', division_id='did') event1 = ge() event2 = ge() for event in [event1, event2]: event.add_document(note="Presentation", url="http://example.com/presentation.pdf") result = EventImporter('jid').import_data([event1.as_dict()]) assert result['event']['insert'] == 1 result = EventImporter('jid').import_data([event2.as_dict()]) assert result['event']['noop'] == 1
def test_bad_event_time(): create_jurisdiction() event = ge() event.start_time = "2014-07-04T05:00" pytest.raises(ValueError, EventImporter('jid', oi, pi, bi, vei).import_item, event.as_dict())
def test_bad_event_time(): Jurisdiction.objects.create(id='jid', division_id='did') event = ge() event.start_time = "2014-07-04T05:00" pytest.raises(ValueError, EventImporter('jid', oi, pi, bi, vei).import_item, event.as_dict())
def test_media_event(): Jurisdiction.objects.create(id='jid', division_id='did') event1 = ge() event2 = ge() for event in [event1, event2]: item = event.add_agenda_item("Cookies will be served") item.add_media_link(note="Hello, World", media_type='application/octet-stream', url="http://hello.world/foo") result = EventImporter('jid').import_data([event1.as_dict()]) assert result['event']['insert'] == 1 result = EventImporter('jid').import_data([event2.as_dict()]) assert result['event']['noop'] == 1
def test_top_level_media_event(): Jurisdiction.objects.create(id='jid', division_id='did') event1, event2 = ge(), ge() event1.add_media_link("fireworks", "http://example.com/fireworks.mov", media_type='application/octet-stream') event2.add_media_link("fireworks", "http://example.com/fireworks.mov", media_type='application/octet-stream') result = EventImporter('jid').import_data([event1.as_dict()]) assert result['event']['insert'] == 1 result = EventImporter('jid').import_data([event2.as_dict()]) assert result['event']['noop'] == 1
def do_import(self, juris, args): # import inside here because to avoid loading Django code unnecessarily from pupa.importers import (JurisdictionImporter, OrganizationImporter, PersonImporter, PostImporter, MembershipImporter, BillImporter, VoteEventImporter, EventImporter) datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module) juris_importer = JurisdictionImporter(juris.jurisdiction_id) org_importer = OrganizationImporter(juris.jurisdiction_id) person_importer = PersonImporter(juris.jurisdiction_id) post_importer = PostImporter(juris.jurisdiction_id, org_importer) membership_importer = MembershipImporter(juris.jurisdiction_id, person_importer, org_importer, post_importer) bill_importer = BillImporter(juris.jurisdiction_id, org_importer, person_importer) vote_event_importer = VoteEventImporter(juris.jurisdiction_id, person_importer, org_importer, bill_importer) event_importer = EventImporter(juris.jurisdiction_id, org_importer, person_importer, bill_importer, vote_event_importer) report = {} with transaction.atomic(): print('import jurisdictions...') report.update(juris_importer.import_directory(datadir)) print('import organizations...') report.update(org_importer.import_directory(datadir)) print('import people...') report.update(person_importer.import_directory(datadir)) print('import posts...') report.update(post_importer.import_directory(datadir)) print('import memberships...') report.update(membership_importer.import_directory(datadir)) print('import bills...') report.update(bill_importer.import_directory(datadir)) print('import events...') report.update(event_importer.import_directory(datadir)) print('import vote events...') report.update(vote_event_importer.import_directory(datadir)) return report
def test_event_agenda_item(): create_jurisdiction() event1 = ge() agenda = event1.add_agenda_item("first item") agenda['extras'] = {'one': 1, 'two': [2]} result = EventImporter('jid', oi, pi, bi, vei).import_data([event1.as_dict()]) assert result['event']['insert'] == 1 e = Event.objects.get() a = e.agenda.all()[0] assert a.extras == {'one': 1, 'two': [2]}
def do_import(self, juris, args): # import inside here because to avoid loading Django code unnecessarily from pupa.importers import (JurisdictionImporter, OrganizationImporter, PersonImporter, PostImporter, MembershipImporter, BillImporter, VoteEventImporter, EventImporter) from pupa.reports import generate_session_report from pupa.models import SessionDataQualityReport datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module) juris_importer = JurisdictionImporter(juris.jurisdiction_id) org_importer = OrganizationImporter(juris.jurisdiction_id) person_importer = PersonImporter(juris.jurisdiction_id) post_importer = PostImporter(juris.jurisdiction_id, org_importer) membership_importer = MembershipImporter(juris.jurisdiction_id, person_importer, org_importer, post_importer) bill_importer = BillImporter(juris.jurisdiction_id, org_importer, person_importer) vote_event_importer = VoteEventImporter(juris.jurisdiction_id, person_importer, org_importer, bill_importer) event_importer = EventImporter(juris.jurisdiction_id, org_importer, person_importer, bill_importer, vote_event_importer) report = {} with transaction.atomic(): print('import jurisdictions...') report.update(juris_importer.import_directory(datadir)) if settings.ENABLE_PEOPLE_AND_ORGS: print('import organizations...') report.update(org_importer.import_directory(datadir)) print('import people...') report.update(person_importer.import_directory(datadir)) print('import posts...') report.update(post_importer.import_directory(datadir)) print('import memberships...') report.update(membership_importer.import_directory(datadir)) if settings.ENABLE_BILLS: print('import bills...') report.update(bill_importer.import_directory(datadir)) if settings.ENABLE_EVENTS: print('import events...') report.update(event_importer.import_directory(datadir)) if settings.ENABLE_VOTES: print('import vote events...') report.update(vote_event_importer.import_directory(datadir)) # compile info on all sessions that were updated in this run seen_sessions = set() seen_sessions.update(bill_importer.get_seen_sessions()) seen_sessions.update(vote_event_importer.get_seen_sessions()) for session in seen_sessions: new_report = generate_session_report(session) with transaction.atomic(): SessionDataQualityReport.objects.filter( legislative_session=session).delete() new_report.save() return report