def test_related_people_event():
create_jurisdiction()
george = Person.objects.create(id='gw', name='George Washington')
john = Person.objects.create(id='jqp', name='John Q. Public')
o = Organization.objects.create(name='Merica', jurisdiction_id='jid')
Membership.objects.create(person=george, organization=o)
Membership.objects.create(person=john, organization=o)
event1 = ge()
event2 = ge()
for event in [event1, event2]:
item = event.add_agenda_item("Cookies will be served")
item.add_person(person="John Q. Public")
event.add_person("George Washington")
result = EventImporter('jid', oi, pi, bi, vei).import_data([event1.as_dict()])
assert result['event']['insert'] == 1
result = EventImporter('jid', oi, pi, bi, vei).import_data([event2.as_dict()])
assert result['event']['noop'] == 1
assert Event.objects.get(name="America's Birthday").participants.first().person_id == 'gw'
assert Event.objects.get(name="America's Birthday"
).agenda.first().related_entities.first().person_id == 'jqp'
def test_full_event():
create_jurisdiction()
george = Person.objects.create(id='gw', name='George Washington')
o = Organization.objects.create(name='Merica', jurisdiction_id='jid')
Membership.objects.create(person=george, organization=o)
event = ge()
result = EventImporter('jid', oi, pi, bi,
vei).import_data([event.as_dict()])
assert result['event']['insert'] == 1
event = ge()
result = EventImporter('jid', oi, pi, bi,
vei).import_data([event.as_dict()])
assert result['event']['noop'] == 1
event = ge()
event.location['name'] = "United States of America"
result = EventImporter('jid', oi, pi, bi,
vei).import_data([event.as_dict()])
assert result['event']['update'] == 1
event.location = None
result = EventImporter('jid', oi, pi, bi,
vei).import_data([event.as_dict()])
assert result['event']['update'] == 1
def test_pupa_identifier_event():
create_jurisdiction()
create_other_jurisdiction()
george = Person.objects.create(id='gw', name='George Washington')
o = Organization.objects.create(name='Merica', jurisdiction_id='jid')
Membership.objects.create(person=george, organization=o)
event = ge()
event.pupa_id = 'foo'
result = EventImporter('jid', oi, pi, bi, vei).import_data([event.as_dict()])
assert result['event']['insert'] == 1
result = EventImporter('jid', oi, pi, bi, vei).import_data([event.as_dict()])
assert result['event']['noop'] == 1
event.name = "America's Anniversary",
event.location['name'] = "United States of America"
result = EventImporter('jid', oi, pi, bi, vei).import_data([event.as_dict()])
assert result['event']['update'] == 1
event.pupa_id = 'bar'
result = EventImporter('jid', oi, pi, bi, vei).import_data([event.as_dict()])
assert result['event']['insert'] == 1
result = EventImporter('ojid', oi, pi, bi, vei).import_data([event.as_dict()])
assert result['event']['insert'] == 1
def test_related_committee_event():
j = create_jurisdiction()
j.legislative_sessions.create(name='1900', identifier='1900')
org = Organization.objects.create(id='org-id', name='House',
classification='lower',
jurisdiction=j)
Organization.objects.create(id='fiscal', name="Fiscal Committee",
classification='committee',
parent=org,
jurisdiction=j)
event1 = ge()
event2 = ge()
for event in [event1, event2]:
item = event.add_agenda_item("Cookies will be served")
item.add_committee(committee="Fiscal Committee")
result = EventImporter('jid', oi, pi, bi, vei).import_data([event1.as_dict()])
assert result['event']['insert'] == 1
result = EventImporter('jid', oi, pi, bi, vei).import_data([event2.as_dict()])
assert result['event']['noop'] == 1
assert Event.objects.get(name="America's Birthday"
).agenda.first().related_entities.first().organization_id == 'fiscal'
def test_related_vote_event():
j = create_jurisdiction()
session = j.legislative_sessions.create(name='1900', identifier='1900')
org = Organization.objects.create(id='org-id', name='House', classification='lower')
bill = Bill.objects.create(id='bill-1', identifier='HB 1',
legislative_session=session)
VoteEvent.objects.create(id='vote-1',
identifier="Roll no. 12",
bill=bill,
legislative_session=session,
organization=org)
event1 = ge()
event2 = ge()
for event in [event1, event2]:
item = event.add_agenda_item("Cookies will be served")
item.add_vote_event(vote_event="Roll no. 12")
result = EventImporter('jid', oi, pi, bi, vei).import_data([event1.as_dict()])
assert result['event']['insert'] == 1
result = EventImporter('jid', oi, pi, bi, vei).import_data([event2.as_dict()])
assert result['event']['noop'] == 1
assert Event.objects.get(name="America's Birthday"
).agenda.first().related_entities.first().vote_event_id == 'vote-1'
def do_import(self, juris, args):
# import inside here because to avoid loading Django code unnecessarily
from pupa.importers import (JurisdictionImporter, OrganizationImporter, PersonImporter,
PostImporter, MembershipImporter, BillImporter,
VoteEventImporter, EventImporter)
from pupa.reports import generate_session_report
from pupa.models import SessionDataQualityReport
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id, person_importer,
org_importer, post_importer)
bill_importer = BillImporter(juris.jurisdiction_id, org_importer, person_importer)
vote_event_importer = VoteEventImporter(juris.jurisdiction_id, person_importer,
org_importer, bill_importer)
event_importer = EventImporter(juris.jurisdiction_id,
org_importer,
person_importer,
bill_importer,
vote_event_importer)
report = {}
with transaction.atomic():
print('import jurisdictions...')
report.update(juris_importer.import_directory(datadir))
if settings.ENABLE_PEOPLE_AND_ORGS:
print('import organizations...')
report.update(org_importer.import_directory(datadir))
print('import people...')
report.update(person_importer.import_directory(datadir))
print('import posts...')
report.update(post_importer.import_directory(datadir))
print('import memberships...')
report.update(membership_importer.import_directory(datadir))
if settings.ENABLE_BILLS:
print('import bills...')
report.update(bill_importer.import_directory(datadir))
if settings.ENABLE_EVENTS:
print('import events...')
report.update(event_importer.import_directory(datadir))
if settings.ENABLE_VOTES:
print('import vote events...')
report.update(vote_event_importer.import_directory(datadir))
# compile info on all sessions that were updated in this run
seen_sessions = set()
seen_sessions.update(bill_importer.get_seen_sessions())
seen_sessions.update(vote_event_importer.get_seen_sessions())
for session in seen_sessions:
new_report = generate_session_report(session)
with transaction.atomic():
SessionDataQualityReport.objects.filter(legislative_session=session).delete()
new_report.save()
return report
def do_import(self, juris, args):
# import inside here because to avoid loading Django code unnecessarily
from pupa.importers import (JurisdictionImporter, OrganizationImporter, PersonImporter,
PostImporter, MembershipImporter, BillImporter,
VoteImporter, EventImporter,
DisclosureImporter)
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id, person_importer,
org_importer, post_importer)
bill_importer = BillImporter(juris.jurisdiction_id, org_importer, person_importer)
vote_importer = VoteImporter(juris.jurisdiction_id, person_importer, org_importer,
bill_importer)
event_importer = EventImporter(juris.jurisdiction_id,
org_importer,
person_importer)
disclosure_importer = DisclosureImporter(juris.jurisdiction_id,
org_importer,
person_importer,
event_importer)
report = {}
with transaction.atomic():
print('import jurisdictions...')
report.update(juris_importer.import_directory(datadir))
print('import organizations...')
report.update(org_importer.import_directory(datadir))
print('import people...')
report.update(person_importer.import_directory(datadir))
print('import posts...')
report.update(post_importer.import_directory(datadir))
print('import memberships...')
report.update(membership_importer.import_directory(datadir))
print('import bills...')
report.update(bill_importer.import_directory(datadir))
print('import events...')
report.update(event_importer.import_directory(datadir))
print('import disclosures...')
report.update(disclosure_importer.import_directory(datadir))
print('import votes...')
report.update(vote_importer.import_directory(datadir))
return report
def test_top_level_media_event():
create_jurisdiction()
event1, event2 = ge(), ge()
event1.add_media_link("fireworks", "http://example.com/fireworks.mov",
media_type='application/octet-stream')
event2.add_media_link("fireworks", "http://example.com/fireworks.mov",
media_type='application/octet-stream')
result = EventImporter('jid', oi, pi, bi, vei).import_data([event1.as_dict()])
assert result['event']['insert'] == 1
result = EventImporter('jid', oi, pi, bi, vei).import_data([event2.as_dict()])
assert result['event']['noop'] == 1
def test_media_document():
create_jurisdiction()
event1 = ge()
event2 = ge()
for event in [event1, event2]:
event.add_document(note="Presentation",
url="http://example.com/presentation.pdf")
result = EventImporter('jid', oi, pi, bi, vei).import_data([event1.as_dict()])
assert result['event']['insert'] == 1
result = EventImporter('jid', oi, pi, bi, vei).import_data([event2.as_dict()])
assert result['event']['noop'] == 1
def test_related_committee_event():
Jurisdiction.objects.create(id='jid', division_id='did')
event1 = ge()
event2 = ge()
for event in [event1, event2]:
item = event.add_agenda_item("Cookies will be served")
item.add_committee(committee="Fiscal Committee")
result = EventImporter('jid').import_data([event1.as_dict()])
assert result['event']['insert'] == 1
result = EventImporter('jid').import_data([event2.as_dict()])
assert result['event']['noop'] == 1
def test_related_people_event():
Jurisdiction.objects.create(id='jid', division_id='did')
event1 = ge()
event2 = ge()
for event in [event1, event2]:
item = event.add_agenda_item("Cookies will be served")
item.add_person(person="John Q. Public")
result = EventImporter('jid').import_data([event1.as_dict()])
assert result['event']['insert'] == 1
result = EventImporter('jid').import_data([event2.as_dict()])
assert result['event']['noop'] == 1
def test_full_event():
Jurisdiction.objects.create(id='jid', division_id='did')
event = ge()
result = EventImporter('jid').import_data([event.as_dict()])
assert result['event']['insert'] == 1
result = EventImporter('jid').import_data([event.as_dict()])
assert result['event']['noop'] == 1
event = ge()
event.location['name'] = "United States of America"
result = EventImporter('jid').import_data([event.as_dict()])
assert result['event']['update'] == 1
def test_media_document():
Jurisdiction.objects.create(id='jid', division_id='did')
event1 = ge()
event2 = ge()
for event in [event1, event2]:
event.add_document(note="Presentation",
url="http://example.com/presentation.pdf")
result = EventImporter('jid').import_data([event1.as_dict()])
assert result['event']['insert'] == 1
result = EventImporter('jid').import_data([event2.as_dict()])
assert result['event']['noop'] == 1
def test_bad_event_time():
create_jurisdiction()
event = ge()
event.start_time = "2014-07-04T05:00"
pytest.raises(ValueError,
EventImporter('jid', oi, pi, bi, vei).import_item,
event.as_dict())
def test_bad_event_time():
Jurisdiction.objects.create(id='jid', division_id='did')
event = ge()
event.start_time = "2014-07-04T05:00"
pytest.raises(ValueError,
EventImporter('jid', oi, pi, bi, vei).import_item,
event.as_dict())
def test_media_event():
Jurisdiction.objects.create(id='jid', division_id='did')
event1 = ge()
event2 = ge()
for event in [event1, event2]:
item = event.add_agenda_item("Cookies will be served")
item.add_media_link(note="Hello, World",
media_type='application/octet-stream',
url="http://hello.world/foo")
result = EventImporter('jid').import_data([event1.as_dict()])
assert result['event']['insert'] == 1
result = EventImporter('jid').import_data([event2.as_dict()])
assert result['event']['noop'] == 1
def test_top_level_media_event():
Jurisdiction.objects.create(id='jid', division_id='did')
event1, event2 = ge(), ge()
event1.add_media_link("fireworks",
"http://example.com/fireworks.mov",
media_type='application/octet-stream')
event2.add_media_link("fireworks",
"http://example.com/fireworks.mov",
media_type='application/octet-stream')
result = EventImporter('jid').import_data([event1.as_dict()])
assert result['event']['insert'] == 1
result = EventImporter('jid').import_data([event2.as_dict()])
assert result['event']['noop'] == 1
def do_import(self, juris, args):
# import inside here because to avoid loading Django code unnecessarily
from pupa.importers import (JurisdictionImporter, OrganizationImporter,
PersonImporter, PostImporter,
MembershipImporter, BillImporter,
VoteEventImporter, EventImporter)
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id,
person_importer, org_importer,
post_importer)
bill_importer = BillImporter(juris.jurisdiction_id, org_importer,
person_importer)
vote_event_importer = VoteEventImporter(juris.jurisdiction_id,
person_importer, org_importer,
bill_importer)
event_importer = EventImporter(juris.jurisdiction_id, org_importer,
person_importer, bill_importer,
vote_event_importer)
report = {}
with transaction.atomic():
print('import jurisdictions...')
report.update(juris_importer.import_directory(datadir))
print('import organizations...')
report.update(org_importer.import_directory(datadir))
print('import people...')
report.update(person_importer.import_directory(datadir))
print('import posts...')
report.update(post_importer.import_directory(datadir))
print('import memberships...')
report.update(membership_importer.import_directory(datadir))
print('import bills...')
report.update(bill_importer.import_directory(datadir))
print('import events...')
report.update(event_importer.import_directory(datadir))
print('import vote events...')
report.update(vote_event_importer.import_directory(datadir))
return report
def test_event_agenda_item():
create_jurisdiction()
event1 = ge()
agenda = event1.add_agenda_item("first item")
agenda['extras'] = {'one': 1, 'two': [2]}
result = EventImporter('jid', oi, pi, bi, vei).import_data([event1.as_dict()])
assert result['event']['insert'] == 1
e = Event.objects.get()
a = e.agenda.all()[0]
assert a.extras == {'one': 1, 'two': [2]}
def do_import(self, juris, args):
# import inside here because to avoid loading Django code unnecessarily
from pupa.importers import (JurisdictionImporter, OrganizationImporter,
PersonImporter, PostImporter,
MembershipImporter, BillImporter,
VoteEventImporter, EventImporter)
from pupa.reports import generate_session_report
from pupa.models import SessionDataQualityReport
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id,
person_importer, org_importer,
post_importer)
bill_importer = BillImporter(juris.jurisdiction_id, org_importer,
person_importer)
vote_event_importer = VoteEventImporter(juris.jurisdiction_id,
person_importer, org_importer,
bill_importer)
event_importer = EventImporter(juris.jurisdiction_id, org_importer,
person_importer, bill_importer,
vote_event_importer)
report = {}
with transaction.atomic():
print('import jurisdictions...')
report.update(juris_importer.import_directory(datadir))
if settings.ENABLE_PEOPLE_AND_ORGS:
print('import organizations...')
report.update(org_importer.import_directory(datadir))
print('import people...')
report.update(person_importer.import_directory(datadir))
print('import posts...')
report.update(post_importer.import_directory(datadir))
print('import memberships...')
report.update(membership_importer.import_directory(datadir))
if settings.ENABLE_BILLS:
print('import bills...')
report.update(bill_importer.import_directory(datadir))
if settings.ENABLE_EVENTS:
print('import events...')
report.update(event_importer.import_directory(datadir))
if settings.ENABLE_VOTES:
print('import vote events...')
report.update(vote_event_importer.import_directory(datadir))
# compile info on all sessions that were updated in this run
seen_sessions = set()
seen_sessions.update(bill_importer.get_seen_sessions())
seen_sessions.update(vote_event_importer.get_seen_sessions())
for session in seen_sessions:
new_report = generate_session_report(session)
with transaction.atomic():
SessionDataQualityReport.objects.filter(
legislative_session=session).delete()
new_report.save()
return report
