def test_fix_bill_id():
j = create_jurisdiction()
j.legislative_sessions.create(name='1900', identifier='1900')
org1 = ScrapeOrganization(name='House', classification='lower')
bill = ScrapeBill('HB 1', '1900', 'Test Bill ID',
classification='bill', chamber='lower')
oi = OrganizationImporter('jid')
oi.import_data([org1.as_dict()])
from pupa.settings import IMPORT_TRANSFORMERS
IMPORT_TRANSFORMERS['bill'] = {
'identifier': lambda x: re.sub(r'([A-Z]*)\s*0*([-\d]+)', r'\1 \2', x, 1)
}
bi = BillImporter('jid', oi, DumbMockImporter())
bi.import_data([bill.as_dict()])
ve = ScrapeVoteEvent(legislative_session='1900', motion_text='passage',
start_date='1900-04-02', classification='passage:bill',
result='fail', bill_chamber='lower', bill='HB1',
identifier='4',
bill_action='passage',
organization=org1._id)
VoteEventImporter('jid', DumbMockImporter(), oi, bi).import_data([
ve.as_dict(),
])
IMPORT_TRANSFORMERS['bill'] = {}
ve = VoteEvent.objects.get()
ve.bill.identifier == 'HB 1'
def do_import(self, juris, args):
# import inside here because to avoid loading Django code unnecessarily
from pupa.importers import (JurisdictionImporter, OrganizationImporter, PersonImporter,
PostImporter, MembershipImporter, BillImporter,
VoteEventImporter, EventImporter)
from pupa.reports import generate_session_report
from pupa.models import SessionDataQualityReport
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id, person_importer,
org_importer, post_importer)
bill_importer = BillImporter(juris.jurisdiction_id, org_importer, person_importer)
vote_event_importer = VoteEventImporter(juris.jurisdiction_id, person_importer,
org_importer, bill_importer)
event_importer = EventImporter(juris.jurisdiction_id,
org_importer,
person_importer,
bill_importer,
vote_event_importer)
report = {}
with transaction.atomic():
print('import jurisdictions...')
report.update(juris_importer.import_directory(datadir))
if settings.ENABLE_PEOPLE_AND_ORGS:
print('import organizations...')
report.update(org_importer.import_directory(datadir))
print('import people...')
report.update(person_importer.import_directory(datadir))
print('import posts...')
report.update(post_importer.import_directory(datadir))
print('import memberships...')
report.update(membership_importer.import_directory(datadir))
if settings.ENABLE_BILLS:
print('import bills...')
report.update(bill_importer.import_directory(datadir))
if settings.ENABLE_EVENTS:
print('import events...')
report.update(event_importer.import_directory(datadir))
if settings.ENABLE_VOTES:
print('import vote events...')
report.update(vote_event_importer.import_directory(datadir))
# compile info on all sessions that were updated in this run
seen_sessions = set()
seen_sessions.update(bill_importer.get_seen_sessions())
seen_sessions.update(vote_event_importer.get_seen_sessions())
for session in seen_sessions:
new_report = generate_session_report(session)
with transaction.atomic():
SessionDataQualityReport.objects.filter(legislative_session=session).delete()
new_report.save()
return report
def test_full_vote_event():
j = Jurisdiction.objects.create(id='jid', division_id='did')
j.legislative_sessions.create(name='1900', identifier='1900')
sp1 = ScrapePerson('John Smith', primary_org='lower')
sp2 = ScrapePerson('Adam Smith', primary_org='lower')
org = ScrapeOrganization(name='House', classification='lower')
bill = ScrapeBill('HB 1',
'1900',
'Axe & Tack Tax Act',
from_organization=org._id)
vote_event = ScrapeVoteEvent(legislative_session='1900',
motion_text='passage',
start_date='1900-04-01',
classification='passage:bill',
result='pass',
bill_chamber='lower',
bill='HB 1',
organization=org._id)
vote_event.set_count('yes', 20)
vote_event.yes('John Smith')
vote_event.no('Adam Smith')
oi = OrganizationImporter('jid')
oi.import_data([org.as_dict()])
pi = PersonImporter('jid')
pi.import_data([sp1.as_dict(), sp2.as_dict()])
mi = MembershipImporter('jid', pi, oi, DumbMockImporter())
mi.import_data([sp1._related[0].as_dict(), sp2._related[0].as_dict()])
bi = BillImporter('jid', oi, pi)
bi.import_data([bill.as_dict()])
VoteEventImporter('jid', pi, oi, bi).import_data([vote_event.as_dict()])
assert VoteEvent.objects.count() == 1
ve = VoteEvent.objects.get()
assert ve.legislative_session == LegislativeSession.objects.get()
assert ve.motion_classification == ['passage:bill']
assert ve.bill == Bill.objects.get()
count = ve.counts.get()
assert count.option == 'yes'
assert count.value == 20
votes = list(ve.votes.all())
assert len(votes) == 2
for v in ve.votes.all():
if v.voter_name == 'John Smith':
assert v.option == 'yes'
assert v.voter == Person.objects.get(name='John Smith')
else:
assert v.option == 'no'
assert v.voter == Person.objects.get(name='Adam Smith')
def do_import(self, juris, args):
# import inside here because to avoid loading Django code unnecessarily
from pupa.importers import (JurisdictionImporter, OrganizationImporter, PersonImporter,
PostImporter, MembershipImporter, BillImporter,
VoteImporter, EventImporter,
DisclosureImporter)
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id, person_importer,
org_importer, post_importer)
bill_importer = BillImporter(juris.jurisdiction_id, org_importer, person_importer)
vote_importer = VoteImporter(juris.jurisdiction_id, person_importer, org_importer,
bill_importer)
event_importer = EventImporter(juris.jurisdiction_id,
org_importer,
person_importer)
disclosure_importer = DisclosureImporter(juris.jurisdiction_id,
org_importer,
person_importer,
event_importer)
report = {}
with transaction.atomic():
print('import jurisdictions...')
report.update(juris_importer.import_directory(datadir))
print('import organizations...')
report.update(org_importer.import_directory(datadir))
print('import people...')
report.update(person_importer.import_directory(datadir))
print('import posts...')
report.update(post_importer.import_directory(datadir))
print('import memberships...')
report.update(membership_importer.import_directory(datadir))
print('import bills...')
report.update(bill_importer.import_directory(datadir))
print('import events...')
report.update(event_importer.import_directory(datadir))
print('import disclosures...')
report.update(disclosure_importer.import_directory(datadir))
print('import votes...')
report.update(vote_importer.import_directory(datadir))
return report
def test_bill_update_subsubitem():
create_jurisdiction()
create_org()
oi = OrganizationImporter('jid')
pi = PersonImporter('jid')
# initial sub-subitem
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
bill.add_version_link('printing', 'http://example.com/test.pdf', media_type='application/pdf')
result = BillImporter('jid', oi, pi).import_data([bill.as_dict()])
assert result['bill']['insert'] == 1
obj = Bill.objects.get()
assert obj.versions.count() == 1
assert obj.versions.get().links.count() == 1
# a second subsubitem, update
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
bill.add_version_link('printing', 'http://example.com/test.pdf', media_type='application/pdf')
bill.add_version_link('printing', 'http://example.com/test.text', media_type='text/plain')
result = BillImporter('jid', oi, pi).import_data([bill.as_dict()])
assert result['bill']['update'] == 1
obj = Bill.objects.get()
assert obj.versions.count() == 1
assert obj.versions.get().links.count() == 2
# same thing, noop
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
bill.add_version_link('printing', 'http://example.com/test.pdf', media_type='application/pdf')
bill.add_version_link('printing', 'http://example.com/test.text', media_type='text/plain')
result = BillImporter('jid', oi, pi).import_data([bill.as_dict()])
assert result['bill']['noop'] == 1
obj = Bill.objects.get()
assert obj.versions.count() == 1
assert obj.versions.get().links.count() == 2
# different link for second one, update
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
bill.add_version_link('printing', 'http://example.com/test.pdf', media_type='application/pdf')
bill.add_version_link('printing', 'http://example.com/diff-link.txt', media_type='text/plain')
result = BillImporter('jid', oi, pi).import_data([bill.as_dict()])
assert result['bill']['update'] == 1
obj = Bill.objects.get()
assert obj.versions.count() == 1
assert obj.versions.get().links.count() == 2
# delete one, update
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
bill.add_version_link('printing', 'http://example.com/test.pdf', media_type='application/pdf')
result = BillImporter('jid', oi, pi).import_data([bill.as_dict()])
assert result['bill']['update'] == 1
obj = Bill.objects.get()
assert obj.versions.count() == 1
assert obj.versions.get().links.count() == 1
def test_vote_event_bill_actions_two_stage():
# this test is very similar to what we're testing in test_vote_event_bill_actions w/
# ve3 and ve4, that two bills that reference the same action won't conflict w/ the
# OneToOneField, but in this case we do it in two stages so that the conflict is found
# even if the votes weren't in the same scrape
j = create_jurisdiction()
j.legislative_sessions.create(name='1900', identifier='1900')
org1 = ScrapeOrganization(name='House', classification='lower')
bill = ScrapeBill('HB 1', '1900', 'Axe & Tack Tax Act', from_organization=org1._id)
bill.add_action(description='passage', date='1900-04-02', chamber='lower')
ve1 = ScrapeVoteEvent(legislative_session='1900', motion_text='passage',
start_date='1900-04-02', classification='passage:bill',
result='pass', bill_chamber='lower', bill='HB 1',
bill_action='passage',
organization=org1._id)
ve2 = ScrapeVoteEvent(legislative_session='1900', motion_text='passage',
start_date='1900-04-02', classification='passage:bill',
result='pass', bill_chamber='lower', bill='HB 1',
bill_action='passage',
organization=org1._id)
# disambiguate them
ve1.pupa_id = 'one'
ve2.pupa_id = 'two'
oi = OrganizationImporter('jid')
oi.import_data([org1.as_dict()])
bi = BillImporter('jid', oi, DumbMockImporter())
bi.import_data([bill.as_dict()])
# first imports just fine
VoteEventImporter('jid', DumbMockImporter(), oi, bi).import_data([
ve1.as_dict(),
])
votes = list(VoteEvent.objects.all())
assert len(votes) == 1
assert votes[0].bill_action is not None
# when second is imported, ensure that action stays pinned to first just as it would
# have if they were both in same import
VoteEventImporter('jid', DumbMockImporter(), oi, bi).import_data([
ve1.as_dict(),
ve2.as_dict(),
])
votes = list(VoteEvent.objects.all())
assert len(votes) == 2
assert votes[0].bill_action is not None
assert votes[1].bill_action is None
def test_vote_event_identifier_dedupe():
j = create_jurisdiction()
j.legislative_sessions.create(name='1900', identifier='1900')
Organization.objects.create(id='org-id', name='Legislature',
classification='legislature',
jurisdiction=j)
vote_event = ScrapeVoteEvent(legislative_session='1900', start_date='2013',
classification='anything', result='passed',
motion_text='a vote on something',
identifier='Roll Call No. 1')
dmi = DumbMockImporter()
oi = OrganizationImporter('jid')
bi = BillImporter('jid', dmi, oi)
_, what = VoteEventImporter('jid', dmi, oi, bi).import_item(vote_event.as_dict())
assert what == 'insert'
assert VoteEvent.objects.count() == 1
# same exact vote event, no changes
_, what = VoteEventImporter('jid', dmi, oi, bi).import_item(vote_event.as_dict())
assert what == 'noop'
assert VoteEvent.objects.count() == 1
# new info, update
vote_event.result = 'failed'
_, what = VoteEventImporter('jid', dmi, oi, bi).import_item(vote_event.as_dict())
assert what == 'update'
assert VoteEvent.objects.count() == 1
# new bill, insert
vote_event.identifier = 'Roll Call 2'
_, what = VoteEventImporter('jid', dmi, oi, bi).import_item(vote_event.as_dict())
assert what == 'insert'
assert VoteEvent.objects.count() == 2
def test_bill_sponsor_by_identifier():
create_jurisdiction()
org = create_org()
bill = ScrapeBill('HB 1',
'1900',
'Axe & Tack Tax Act',
classification='tax bill',
chamber='lower')
bill.add_sponsorship_by_identifier(name="SNODGRASS",
classification='sponsor',
entity_type='person',
primary=True,
identifier="TOTALLY_REAL_ID",
scheme="TOTALLY_REAL_SCHEME")
oi = OrganizationImporter('jid')
pi = PersonImporter('jid')
zs = ScrapePerson(name='Zadock Snodgrass')
zs.add_identifier(identifier='TOTALLY_REAL_ID',
scheme='TOTALLY_REAL_SCHEME')
pi.import_data([zs.as_dict()])
za_db = Person.objects.get()
Membership.objects.create(person_id=za_db.id, organization_id=org.id)
BillImporter('jid', oi, pi).import_data([bill.as_dict()])
obj = Bill.objects.get()
(entry, ) = obj.sponsorships.all()
assert entry.person.name == "Zadock Snodgrass"
def test_vote_identifier_dedupe():
j = Jurisdiction.objects.create(id='jid', division_id='did')
session = j.legislative_sessions.create(name='1900', identifier='1900')
vote = ScrapeVote(
legislative_session='1900',
start_date='2013',
classification='anything',
result='passed',
motion_text='a vote on something',
identifier='Roll Call No. 1',
)
dmi = DumbMockImporter()
bi = BillImporter('jid', dmi, dmi)
_, what = VoteImporter('jid', dmi, dmi, bi).import_item(vote.as_dict())
assert what == 'insert'
assert VoteEvent.objects.count() == 1
# same exact vote, no changes
_, what = VoteImporter('jid', dmi, dmi, bi).import_item(vote.as_dict())
assert what == 'noop'
assert VoteEvent.objects.count() == 1
# new info, update
vote.result = 'failed'
_, what = VoteImporter('jid', dmi, dmi, bi).import_item(vote.as_dict())
assert what == 'update'
assert VoteEvent.objects.count() == 1
# new bill, insert
vote.identifier = 'Roll Call 2'
_, what = VoteImporter('jid', dmi, dmi, bi).import_item(vote.as_dict())
assert what == 'insert'
assert VoteEvent.objects.count() == 2
def test_vote_event_bill_id_dedupe():
j = create_jurisdiction()
session = j.legislative_sessions.create(name='1900', identifier='1900')
org = Organization.objects.create(id='org-id',
name='House',
classification='lower',
jurisdiction=j)
bill = Bill.objects.create(id='bill-1',
identifier='HB 1',
legislative_session=session,
from_organization=org)
bill2 = Bill.objects.create(id='bill-2',
identifier='HB 2',
legislative_session=session,
from_organization=org)
vote_event = ScrapeVoteEvent(legislative_session='1900',
start_date='2013',
classification='anything',
result='passed',
motion_text='a vote on something',
bill=bill.identifier,
bill_chamber='lower',
chamber='lower')
dmi = DumbMockImporter()
oi = OrganizationImporter('jid')
bi = BillImporter('jid', dmi, oi)
_, what = VoteEventImporter('jid', dmi, oi,
bi).import_item(vote_event.as_dict())
assert what == 'insert'
assert VoteEvent.objects.count() == 1
# same exact vote event, no changes
_, what = VoteEventImporter('jid', dmi, oi,
bi).import_item(vote_event.as_dict())
assert what == 'noop'
assert VoteEvent.objects.count() == 1
# new info, update
vote_event.result = 'failed'
_, what = VoteEventImporter('jid', dmi, oi,
bi).import_item(vote_event.as_dict())
assert what == 'update'
assert VoteEvent.objects.count() == 1
# new vote event, insert
vote_event = ScrapeVoteEvent(legislative_session='1900',
start_date='2013',
classification='anything',
result='passed',
motion_text='a vote on something',
bill=bill2.identifier,
bill_chamber='lower',
chamber='lower')
_, what = VoteEventImporter('jid', dmi, oi,
bi).import_item(vote_event.as_dict())
assert what == 'insert'
assert VoteEvent.objects.count() == 2
def test_full_vote_event():
j = Jurisdiction.objects.create(id='jid', division_id='did')
j.legislative_sessions.create(name='1900', identifier='1900')
sp1 = ScrapePerson('John Smith', primary_org='lower')
sp2 = ScrapePerson('Adam Smith', primary_org='lower')
org = ScrapeOrganization(name='House', classification='lower')
bill = ScrapeBill('HB 1', '1900', 'Axe & Tack Tax Act', from_organization=org._id)
vote_event = ScrapeVoteEvent(legislative_session='1900', motion_text='passage',
start_date='1900-04-01', classification='passage:bill',
result='pass', bill_chamber='lower', bill='HB 1',
organization=org._id)
vote_event.set_count('yes', 20)
vote_event.yes('John Smith')
vote_event.no('Adam Smith')
oi = OrganizationImporter('jid')
oi.import_data([org.as_dict()])
pi = PersonImporter('jid')
pi.import_data([sp1.as_dict(), sp2.as_dict()])
mi = MembershipImporter('jid', pi, oi, DumbMockImporter())
mi.import_data([sp1._related[0].as_dict(), sp2._related[0].as_dict()])
bi = BillImporter('jid', oi, pi)
bi.import_data([bill.as_dict()])
VoteEventImporter('jid', pi, oi, bi).import_data([vote_event.as_dict()])
assert VoteEvent.objects.count() == 1
ve = VoteEvent.objects.get()
assert ve.legislative_session == LegislativeSession.objects.get()
assert ve.motion_classification == ['passage:bill']
assert ve.bill == Bill.objects.get()
count = ve.counts.get()
assert count.option == 'yes'
assert count.value == 20
votes = list(ve.votes.all())
assert len(votes) == 2
for v in ve.votes.all():
if v.voter_name == 'John Smith':
assert v.option == 'yes'
assert v.voter == Person.objects.get(name='John Smith')
else:
assert v.option == 'no'
assert v.voter == Person.objects.get(name='Adam Smith')
def do_import(self, juris, args):
# import inside here because to avoid loading Django code unnecessarily
from pupa.importers import (JurisdictionImporter, OrganizationImporter,
PersonImporter, PostImporter,
MembershipImporter, BillImporter,
VoteEventImporter, EventImporter)
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id,
person_importer, org_importer,
post_importer)
bill_importer = BillImporter(juris.jurisdiction_id, org_importer,
person_importer)
vote_event_importer = VoteEventImporter(juris.jurisdiction_id,
person_importer, org_importer,
bill_importer)
event_importer = EventImporter(juris.jurisdiction_id, org_importer,
person_importer, bill_importer,
vote_event_importer)
report = {}
with transaction.atomic():
print('import jurisdictions...')
report.update(juris_importer.import_directory(datadir))
print('import organizations...')
report.update(org_importer.import_directory(datadir))
print('import people...')
report.update(person_importer.import_directory(datadir))
print('import posts...')
report.update(post_importer.import_directory(datadir))
print('import memberships...')
report.update(membership_importer.import_directory(datadir))
print('import bills...')
report.update(bill_importer.import_directory(datadir))
print('import events...')
report.update(event_importer.import_directory(datadir))
print('import vote events...')
report.update(vote_event_importer.import_directory(datadir))
return report
def test_bill_chamber_param():
create_jurisdiction()
org = create_org()
bill = ScrapeBill('HB 1', '1900', 'Axe & Tack Tax Act',
classification='tax bill', chamber='lower')
oi = OrganizationImporter('jid')
BillImporter('jid', oi).import_data([bill.as_dict()])
assert Bill.objects.get().from_organization_id == org.id
def test_bill_update():
create_jurisdiction()
create_org()
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
oi = OrganizationImporter('jid')
_, what = BillImporter('jid', oi).import_item(bill.as_dict())
assert what == 'insert'
_, what = BillImporter('jid', oi).import_item(bill.as_dict())
assert what == 'noop'
# ensure no new object was created
assert Bill.objects.count() == 1
# test basic update
bill = ScrapeBill('HB 1', '1900', '1st Bill', chamber='lower')
_, what = BillImporter('jid', oi).import_item(bill.as_dict())
assert what == 'update'
assert Bill.objects.get().title == '1st Bill'
def test_vote_event_bill_clearing():
# ensure that we don't wind up with vote events sitting around forever on bills as
# changes make it look like there are multiple vote events
j = create_jurisdiction()
session = j.legislative_sessions.create(name='1900', identifier='1900')
org = Organization.objects.create(id='org-id',
name='House',
classification='lower',
jurisdiction=j)
bill = Bill.objects.create(id='bill-1',
identifier='HB 1',
legislative_session=session,
from_organization=org)
Bill.objects.create(id='bill-2',
identifier='HB 2',
legislative_session=session,
from_organization=org)
oi = OrganizationImporter('jid')
dmi = DumbMockImporter()
bi = BillImporter('jid', dmi, oi)
vote_event1 = ScrapeVoteEvent(
legislative_session='1900',
start_date='2013',
classification='anything',
result='passed',
motion_text='a vote on somthing', # typo intentional
bill=bill.identifier,
bill_chamber='lower',
chamber='lower')
vote_event2 = ScrapeVoteEvent(legislative_session='1900',
start_date='2013',
classification='anything',
result='passed',
motion_text='a vote on something else',
bill=bill.identifier,
bill_chamber='lower',
chamber='lower')
# have to use import_data so postimport is called
VoteEventImporter('jid', dmi, oi, bi).import_data(
[vote_event1.as_dict(), vote_event2.as_dict()])
assert VoteEvent.objects.count() == 2
# a typo is fixed, we don't want 3 vote events now
vote_event1.motion_text = 'a vote on something'
VoteEventImporter('jid', dmi, oi, bi).import_data(
[vote_event1.as_dict(), vote_event2.as_dict()])
assert VoteEvent.objects.count() == 2
def test_fix_bill_id():
create_jurisdiction()
create_org()
bill = ScrapeBill('HB1',
'1900',
'Test Bill ID',
classification='bill',
chamber='lower')
oi = OrganizationImporter('jid')
pi = PersonImporter('jid')
from pupa.settings import IMPORT_TRANSFORMERS
IMPORT_TRANSFORMERS['bill'] = {
'identifier':
lambda x: re.sub(r'([A-Z]*)\s*0*([-\d]+)', r'\1 \2', x, 1)
}
bi = BillImporter('jid', oi, pi)
bi.import_data([bill.as_dict()])
IMPORT_TRANSFORMERS['bill'] = {}
b = Bill.objects.get()
assert b.identifier == 'HB 1'
def test_full_vote():
j = Jurisdiction.objects.create(id='jid', division_id='did')
session = j.legislative_sessions.create(name='1900', identifier='1900')
person = Person.objects.create(id='person-id', name='Adam Smith')
org = Organization.objects.create(id='org-id',
name='House',
classification='lower')
bill = Bill.objects.create(id='bill-id',
identifier='HB 1',
legislative_session=session,
from_organization=org)
com = Organization.objects.create(id='com-id',
name='Arbitrary Committee',
parent=org)
vote = ScrapeVote(legislative_session='1900',
motion_text='passage',
start_date='1900-04-01',
classification='passage:bill',
result='pass',
bill_chamber='lower',
bill=bill.identifier)
vote.set_count('yes', 20)
vote.yes('John Smith')
vote.no('Adam Smith')
dmi = DumbMockImporter()
bi = BillImporter('jid', dmi, dmi)
VoteImporter('jid', dmi, dmi, bi).import_data([vote.as_dict()])
assert VoteEvent.objects.count() == 1
ve = VoteEvent.objects.get()
assert ve.legislative_session_id == session.id
assert ve.motion_classification == ['passage:bill']
assert ve.bill_id == bill.id
count = ve.counts.get()
assert count.option == 'yes'
assert count.value == 20
votes = list(ve.votes.all())
assert len(votes) == 2
for v in ve.votes.all():
if v.voter_name == 'John Smith':
assert v.option == 'yes'
else:
assert v.option == 'no'
def test_bill_sponsor_limit_lookup():
create_jurisdiction()
org = create_org()
bill = ScrapeBill('HB 1',
'1900',
'Axe & Tack Tax Act',
classification='tax bill',
chamber='lower')
bill.add_sponsorship_by_identifier(name="SNODGRASS",
classification='sponsor',
entity_type='person',
primary=True,
identifier="TOTALLY_REAL_ID",
scheme="TOTALLY_REAL_SCHEME")
oi = OrganizationImporter('jid')
pi = PersonImporter('jid')
zs = ScrapePerson(name='Zadock Snodgrass', birth_date="1800-01-01")
zs.add_identifier(identifier='TOTALLY_REAL_ID',
scheme='TOTALLY_REAL_SCHEME')
pi.import_data([zs.as_dict()])
za_db = Person.objects.get()
Membership.objects.create(person_id=za_db.id, organization_id=org.id)
zs2 = ScrapePerson(name='Zadock Snodgrass', birth_date="1900-01-01")
zs2.add_identifier(identifier='TOTALLY_REAL_ID',
scheme='TOTALLY_REAL_SCHEME')
# This is contrived and perhaps broken, but we're going to check this.
# We *really* don't want to *ever* cross jurisdiction bounds.
PersonImporter('another-jurisdiction').import_data([zs.as_dict()])
BillImporter('jid', oi, pi).import_data([bill.as_dict()])
obj = Bill.objects.get()
(entry, ) = obj.sponsorships.all()
assert entry.person.name == "Zadock Snodgrass"
assert entry.person.birth_date == "1800-01-01"
def test_bill_action_extras():
create_jurisdiction()
create_org()
bill = ScrapeBill('HB 1',
'1900',
'Axe & Tack Tax Act',
classification='tax bill',
chamber='lower')
bill.add_action('sample',
'1900-01-01',
chamber='lower',
extras={'test': 3})
oi = OrganizationImporter('jid')
pi = PersonImporter('jid')
BillImporter('jid', oi, pi).import_data([bill.as_dict()])
b = Bill.objects.get()
assert b.actions.all()[0].extras == {'test': 3}
def test_vote_event_bill_actions():
j = create_jurisdiction()
j.legislative_sessions.create(name='1900', identifier='1900')
org1 = ScrapeOrganization(name='House', classification='lower')
org2 = ScrapeOrganization(name='Senate', classification='upper')
bill = ScrapeBill('HB 1', '1900', 'Axe & Tack Tax Act', from_organization=org1._id)
# add actions, passage of upper & lower on same day, something else,
# then passage in upper again on a different day
bill.add_action(description='passage', date='1900-04-01', chamber='upper')
bill.add_action(description='passage', date='1900-04-01', chamber='lower')
bill.add_action(description='other event', date='1900-04-01', chamber='lower')
bill.add_action(description='passage', date='1900-04-02', chamber='upper')
# four passage votes, one per chamber, one on 04-01, and one on 04-02
ve1 = ScrapeVoteEvent(legislative_session='1900', motion_text='passage',
start_date='1900-04-01', classification='passage:bill',
result='pass', bill_chamber='lower', bill='HB 1',
bill_action='passage',
organization=org1._id)
ve2 = ScrapeVoteEvent(legislative_session='1900', motion_text='passage',
start_date='1900-04-01', classification='passage:bill',
result='pass', bill_chamber='lower', bill='HB 1',
bill_action='passage',
organization=org2._id)
ve3 = ScrapeVoteEvent(legislative_session='1900', motion_text='passage',
start_date='1900-04-02', classification='passage:bill',
result='pass', bill_chamber='lower', bill='HB 1',
bill_action='passage',
organization=org1._id)
ve4 = ScrapeVoteEvent(legislative_session='1900', motion_text='passage',
start_date='1900-04-02', classification='passage:bill',
result='pass', bill_chamber='lower', bill='HB 1',
bill_action='passage',
organization=org2._id)
oi = OrganizationImporter('jid')
oi.import_data([org1.as_dict(), org2.as_dict()])
bi = BillImporter('jid', oi, DumbMockImporter())
bi.import_data([bill.as_dict()])
VoteEventImporter('jid', DumbMockImporter(), oi, bi).import_data([
ve1.as_dict(),
ve2.as_dict(),
ve3.as_dict(),
ve4.as_dict(),
])
bill = Bill.objects.get()
votes = list(VoteEvent.objects.all())
actions = list(bill.actions.all())
assert len(actions) == 4
assert len(votes) == 4
votes = {(v.organization.classification, v.start_date): v.bill_action
for v in votes}
# ensure that votes are matched using action, chamber, and date
assert votes[('upper', '1900-04-01')] == actions[0]
assert votes[('lower', '1900-04-01')] == actions[1]
assert votes[('upper', '1900-04-02')] == actions[3]
assert votes[('lower', '1900-04-02')] is None
def test_full_bill():
create_jurisdiction()
sp = ScrapePerson('Adam Smith')
org = ScrapeOrganization(name='House', classification='lower')
com = ScrapeOrganization(name='Arbitrary Committee',
classification='committee',
parent_id=org._id)
oldbill = ScrapeBill('HB 99',
'1899',
'Axe & Tack Tax Act',
classification='tax bill',
from_organization=org._id)
bill = ScrapeBill('HB 1',
'1900',
'Axe & Tack Tax Act',
classification='tax bill',
from_organization=org._id)
bill.subject = ['taxes', 'axes']
bill.add_identifier('SB 9')
bill.add_title('Tack & Axe Tax Act')
bill.add_action('introduced in house', '1900-04-01', chamber='lower')
act = bill.add_action('sent to arbitrary committee',
'1900-04-04',
chamber='lower')
act.add_related_entity('arbitrary committee', 'organization', com._id)
bill.add_related_bill("HB 99",
legislative_session="1899",
relation_type="prior-session")
bill.add_sponsorship('Adam Smith',
classification='extra sponsor',
entity_type='person',
primary=False,
entity_id=sp._id)
bill.add_sponsorship('Jane Smith',
classification='lead sponsor',
entity_type='person',
primary=True)
bill.add_abstract('This is an act about axes and taxes and tacks.',
note="official",
date='1969-10-20')
bill.add_document_link('Fiscal Note',
'http://example.com/fn.pdf',
media_type='application/pdf')
bill.add_document_link('Fiscal Note',
'http://example.com/fn.html',
media_type='text/html')
bill.add_version_link('Fiscal Note',
'http://example.com/v/1',
media_type='text/html')
bill.add_source('http://example.com/source')
# import bill
oi = OrganizationImporter('jid')
oi.import_data([org.as_dict(), com.as_dict()])
pi = PersonImporter('jid')
pi.import_data([sp.as_dict()])
BillImporter('jid', oi,
pi).import_data([oldbill.as_dict(),
bill.as_dict()])
# get bill from db and assert it imported correctly
b = Bill.objects.get(identifier='HB 1')
assert b.from_organization.classification == 'lower'
assert b.identifier == bill.identifier
assert b.title == bill.title
assert b.classification == bill.classification
assert b.subject == ['taxes', 'axes']
assert b.abstracts.get().note == 'official'
assert b.abstracts.get().date == '1969-10-20'
# other_title, other_identifier added
assert b.other_titles.get().title == 'Tack & Axe Tax Act'
assert b.other_identifiers.get().identifier == 'SB 9'
# actions
actions = list(b.actions.all())
assert len(actions) == 2
# ensure order was preserved (if this breaks it'll be intermittent)
assert actions[0].organization == Organization.objects.get(
classification='lower')
assert actions[0].description == "introduced in house"
assert actions[1].description == "sent to arbitrary committee"
assert (actions[1].related_entities.get().organization ==
Organization.objects.get(classification='committee'))
# related_bills were added
rb = b.related_bills.get()
assert rb.identifier == 'HB 99'
# and bill got resolved
assert rb.related_bill.identifier == 'HB 99'
# sponsors added, linked & unlinked
sponsorships = b.sponsorships.all()
assert len(sponsorships) == 2
person = Person.objects.get(name='Adam Smith')
for ss in sponsorships:
if ss.primary:
assert ss.person is None
assert ss.organization is None
else:
assert ss.person == person
# versions & documents with their links
versions = b.versions.all()
assert len(versions) == 1
assert versions[0].links.count() == 1
documents = b.documents.all()
assert len(documents) == 1
assert documents[0].links.count() == 2
# sources
assert b.sources.count() == 1
def test_bill_update_because_of_subitem():
create_jurisdiction()
create_org()
oi = OrganizationImporter('jid')
pi = PersonImporter('jid')
# initial bill
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
bill.add_action('this is an action', chamber='lower', date='1900-01-01')
result = BillImporter('jid', oi, pi).import_data([bill.as_dict()])
assert result['bill']['insert'] == 1
obj = Bill.objects.get()
assert obj.actions.count() == 1
last_updated = obj.updated_at
# now let's make sure we get updated when there are second actions
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
bill.add_action('this is an action', chamber='lower', date='1900-01-01')
bill.add_action('this is a second action',
chamber='lower',
date='1900-01-02')
result = BillImporter('jid', oi, pi).import_data([bill.as_dict()])
assert result['bill']['update'] == 1
obj = Bill.objects.get()
assert obj.actions.count() == 2
assert obj.updated_at > last_updated
# same 2 actions, noop
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
bill.add_action('this is an action', chamber='lower', date='1900-01-01')
bill.add_action('this is a second action',
chamber='lower',
date='1900-01-02')
result = BillImporter('jid', oi, pi).import_data([bill.as_dict()])
assert result['bill']['noop'] == 1
obj = Bill.objects.get()
assert obj.actions.count() == 2
# same 2 actions, different order, update
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
bill.add_action('this is a second action',
chamber='lower',
date='1900-01-02')
bill.add_action('this is an action', chamber='lower', date='1900-01-01')
result = BillImporter('jid', oi, pi).import_data([bill.as_dict()])
assert result['bill']['update'] == 1
obj = Bill.objects.get()
assert obj.actions.count() == 2
# different 2 actions, update
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
bill.add_action('this is an action', chamber='lower', date='1900-01-01')
bill.add_action('this is a different second action',
chamber='lower',
date='1900-01-02')
result = BillImporter('jid', oi, pi).import_data([bill.as_dict()])
assert result['bill']['update'] == 1
obj = Bill.objects.get()
assert obj.actions.count() == 2
# delete an action, update
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
bill.add_action('this is a second action',
chamber='lower',
date='1900-01-02')
result = BillImporter('jid', oi, pi).import_data([bill.as_dict()])
assert result['bill']['update'] == 1
obj = Bill.objects.get()
assert obj.actions.count() == 1
# delete all actions, update
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
result = BillImporter('jid', oi, pi).import_data([bill.as_dict()])
assert result['bill']['update'] == 1
obj = Bill.objects.get()
assert obj.actions.count() == 0
# and back to initial status, update
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
bill.add_action('this is an action', chamber='lower', date='1900-01-01')
result = BillImporter('jid', oi, pi).import_data([bill.as_dict()])
assert result['bill']['update'] == 1
obj = Bill.objects.get()
assert obj.actions.count() == 1
def test_vote_event_bill_actions_errors():
j = create_jurisdiction()
j.legislative_sessions.create(name='1900', identifier='1900')
org1 = ScrapeOrganization(name='House', classification='lower')
org2 = ScrapeOrganization(name='Senate', classification='upper')
bill = ScrapeBill('HB 1', '1900', 'Axe & Tack Tax Act', from_organization=org1._id)
# for this bill, two identical actions, so vote matching will fail
bill.add_action(description='passage', date='1900-04-01', chamber='lower')
bill.add_action(description='passage', date='1900-04-01', chamber='lower')
# this action is good, but two votes will try to match it
bill.add_action(description='passage', date='1900-04-02', chamber='lower')
# will match two actions
ve1 = ScrapeVoteEvent(legislative_session='1900', motion_text='passage',
start_date='1900-04-01', classification='passage:bill',
result='pass', bill_chamber='lower', bill='HB 1',
identifier='1',
bill_action='passage',
organization=org1._id)
# will match no actions
ve2 = ScrapeVoteEvent(legislative_session='1900', motion_text='passage',
start_date='1900-04-01', classification='passage:bill',
result='pass', bill_chamber='lower', bill='HB 1',
identifier='2',
bill_action='committee result',
organization=org1._id)
# these two votes will both match the same action
ve3 = ScrapeVoteEvent(legislative_session='1900', motion_text='passage',
start_date='1900-04-02', classification='passage:bill',
result='pass', bill_chamber='lower', bill='HB 1',
identifier='3',
bill_action='passage',
organization=org1._id)
ve4 = ScrapeVoteEvent(legislative_session='1900', motion_text='passage-syz',
start_date='1900-04-02', classification='passage:bill',
result='fail', bill_chamber='lower', bill='HB 1',
identifier='4',
bill_action='passage',
organization=org1._id)
oi = OrganizationImporter('jid')
oi.import_data([org1.as_dict(), org2.as_dict()])
bi = BillImporter('jid', oi, DumbMockImporter())
bi.import_data([bill.as_dict()])
VoteEventImporter('jid', DumbMockImporter(), oi, bi).import_data([
ve1.as_dict(),
ve2.as_dict(),
ve3.as_dict(),
ve4.as_dict(),
])
bill = Bill.objects.get()
votes = list(VoteEvent.objects.all())
# isn't matched, was ambiguous across two actions
assert votes[0].bill_action is None
# isn't matched, no match in actions
assert votes[1].bill_action is None
# these both try to match the same action, only first will succeed
assert votes[2].bill_action is not None
assert votes[3].bill_action is None
j = Jurisdiction.objects.create(id='ojid', division_id='ocd-division/country:ca')
return j
def ge():
event = ScrapeEvent(
name="America's Birthday",
start_date="2014-07-04T05:00Z",
location_name="America",
all_day=True)
return event
oi = OrganizationImporter('jid')
pi = PersonImporter('jid')
bi = BillImporter('jid', oi, pi)
vei = VoteEventImporter('jid', pi, oi, bi)
@pytest.mark.django_db
def test_related_people_event():
create_jurisdiction()
george = Person.objects.create(id='gw', name='George Washington')
john = Person.objects.create(id='jqp', name='John Q. Public')
o = Organization.objects.create(name='Merica', jurisdiction_id='jid')
Membership.objects.create(person=george, organization=o)
Membership.objects.create(person=john, organization=o)
event1 = ge()
event2 = ge()
def do_import(self, juris, args):
# import inside here because to avoid loading Django code unnecessarily
from pupa.importers import (JurisdictionImporter, OrganizationImporter,
PersonImporter, PostImporter,
MembershipImporter, BillImporter,
VoteEventImporter, EventImporter)
from pupa.reports import generate_session_report
from pupa.models import SessionDataQualityReport
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id,
person_importer, org_importer,
post_importer)
bill_importer = BillImporter(juris.jurisdiction_id, org_importer,
person_importer)
vote_event_importer = VoteEventImporter(juris.jurisdiction_id,
person_importer, org_importer,
bill_importer)
event_importer = EventImporter(juris.jurisdiction_id, org_importer,
person_importer, bill_importer,
vote_event_importer)
report = {}
with transaction.atomic():
print('import jurisdictions...')
report.update(juris_importer.import_directory(datadir))
if settings.ENABLE_PEOPLE_AND_ORGS:
print('import organizations...')
report.update(org_importer.import_directory(datadir))
print('import people...')
report.update(person_importer.import_directory(datadir))
print('import posts...')
report.update(post_importer.import_directory(datadir))
print('import memberships...')
report.update(membership_importer.import_directory(datadir))
if settings.ENABLE_BILLS:
print('import bills...')
report.update(bill_importer.import_directory(datadir))
if settings.ENABLE_EVENTS:
print('import events...')
report.update(event_importer.import_directory(datadir))
if settings.ENABLE_VOTES:
print('import vote events...')
report.update(vote_event_importer.import_directory(datadir))
# compile info on all sessions that were updated in this run
seen_sessions = set()
seen_sessions.update(bill_importer.get_seen_sessions())
seen_sessions.update(vote_event_importer.get_seen_sessions())
for session in seen_sessions:
new_report = generate_session_report(session)
with transaction.atomic():
SessionDataQualityReport.objects.filter(
legislative_session=session).delete()
new_report.save()
return report
