def test_vote_identifier_dedupe():
j = Jurisdiction.objects.create(id='jid', division_id='did')
session = j.legislative_sessions.create(name='1900', identifier='1900')
vote = ScrapeVote(
legislative_session='1900',
start_date='2013',
classification='anything',
result='passed',
motion_text='a vote on something',
identifier='Roll Call No. 1',
)
dmi = DumbMockImporter()
bi = BillImporter('jid', dmi, dmi)
_, what = VoteImporter('jid', dmi, dmi, bi).import_item(vote.as_dict())
assert what == 'insert'
assert VoteEvent.objects.count() == 1
# same exact vote, no changes
_, what = VoteImporter('jid', dmi, dmi, bi).import_item(vote.as_dict())
assert what == 'noop'
assert VoteEvent.objects.count() == 1
# new info, update
vote.result = 'failed'
_, what = VoteImporter('jid', dmi, dmi, bi).import_item(vote.as_dict())
assert what == 'update'
assert VoteEvent.objects.count() == 1
# new bill, insert
vote.identifier = 'Roll Call 2'
_, what = VoteImporter('jid', dmi, dmi, bi).import_item(vote.as_dict())
assert what == 'insert'
assert VoteEvent.objects.count() == 2
def do_import(self, juris, args):
# import inside here because to avoid loading Django code unnecessarily
from pupa.importers import (JurisdictionImporter, OrganizationImporter, PersonImporter,
PostImporter, MembershipImporter, BillImporter,
VoteImporter, EventImporter,
DisclosureImporter)
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id, person_importer,
org_importer, post_importer)
bill_importer = BillImporter(juris.jurisdiction_id, org_importer, person_importer)
vote_importer = VoteImporter(juris.jurisdiction_id, person_importer, org_importer,
bill_importer)
event_importer = EventImporter(juris.jurisdiction_id,
org_importer,
person_importer)
disclosure_importer = DisclosureImporter(juris.jurisdiction_id,
org_importer,
person_importer,
event_importer)
report = {}
with transaction.atomic():
print('import jurisdictions...')
report.update(juris_importer.import_directory(datadir))
print('import organizations...')
report.update(org_importer.import_directory(datadir))
print('import people...')
report.update(person_importer.import_directory(datadir))
print('import posts...')
report.update(post_importer.import_directory(datadir))
print('import memberships...')
report.update(membership_importer.import_directory(datadir))
print('import bills...')
report.update(bill_importer.import_directory(datadir))
print('import events...')
report.update(event_importer.import_directory(datadir))
print('import disclosures...')
report.update(disclosure_importer.import_directory(datadir))
print('import votes...')
report.update(vote_importer.import_directory(datadir))
return report
def test_vote_bill_id_dedupe():
j = Jurisdiction.objects.create(id='jid', division_id='did')
session = j.legislative_sessions.create(name='1900', identifier='1900')
org = Organization.objects.create(id='org-id',
name='House',
classification='lower')
bill = Bill.objects.create(id='bill-1',
identifier='HB 1',
legislative_session=session,
from_organization=org)
bill2 = Bill.objects.create(id='bill-2',
identifier='HB 2',
legislative_session=session,
from_organization=org)
vote = ScrapeVote(legislative_session='1900',
start_date='2013',
classification='anything',
result='passed',
motion_text='a vote on something',
bill=bill.identifier,
bill_chamber='lower')
dmi = DumbMockImporter()
bi = BillImporter('jid', dmi, dmi)
_, what = VoteImporter('jid', dmi, dmi, bi).import_item(vote.as_dict())
assert what == 'insert'
assert VoteEvent.objects.count() == 1
# same exact vote, no changes
_, what = VoteImporter('jid', dmi, dmi, bi).import_item(vote.as_dict())
assert what == 'noop'
assert VoteEvent.objects.count() == 1
# new info, update
vote.result = 'failed'
_, what = VoteImporter('jid', dmi, dmi, bi).import_item(vote.as_dict())
assert what == 'update'
assert VoteEvent.objects.count() == 1
# new vote, insert
vote = ScrapeVote(legislative_session='1900',
start_date='2013',
classification='anything',
result='passed',
motion_text='a vote on something',
bill=bill2.identifier,
bill_chamber='lower')
_, what = VoteImporter('jid', dmi, dmi, bi).import_item(vote.as_dict())
assert what == 'insert'
assert VoteEvent.objects.count() == 2
def test_vote_bill_clearing():
# ensure that we don't wind up with votes sitting around forever on bills as changes
# make it look like there are multiple votes
j = Jurisdiction.objects.create(id='jid', division_id='did')
session = j.legislative_sessions.create(name='1900', identifier='1900')
org = Organization.objects.create(id='org-id',
name='House',
classification='lower')
bill = Bill.objects.create(id='bill-1',
identifier='HB 1',
legislative_session=session,
from_organization=org)
bill2 = Bill.objects.create(id='bill-2',
identifier='HB 2',
legislative_session=session,
from_organization=org)
dmi = DumbMockImporter()
bi = BillImporter('jid', dmi, dmi)
vote1 = ScrapeVote(
legislative_session='1900',
start_date='2013',
classification='anything',
result='passed',
motion_text='a vote on somthing', # typo intentional
bill=bill.identifier,
bill_chamber='lower')
vote2 = ScrapeVote(legislative_session='1900',
start_date='2013',
classification='anything',
result='passed',
motion_text='a vote on something else',
bill=bill.identifier,
bill_chamber='lower')
# have to use import_data so postimport is called
VoteImporter('jid', dmi, dmi,
bi).import_data([vote1.as_dict(),
vote2.as_dict()])
assert VoteEvent.objects.count() == 2
# a typo is fixed, we don't want 3 votes now
vote1.motion_text = 'a vote on something'
VoteImporter('jid', dmi, dmi,
bi).import_data([vote1.as_dict(),
vote2.as_dict()])
assert VoteEvent.objects.count() == 2
def do_import(self, juris, args):
# import inside here because to avoid loading Django code unnecessarily
from pupa.importers import (JurisdictionImporter, OrganizationImporter,
PersonImporter, PostImporter,
MembershipImporter, BillImporter,
VoteImporter, EventImporter)
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id,
person_importer, org_importer,
post_importer)
bill_importer = BillImporter(juris.jurisdiction_id, org_importer,
person_importer)
vote_importer = VoteImporter(juris.jurisdiction_id, person_importer,
org_importer, bill_importer)
event_importer = EventImporter(juris.jurisdiction_id)
report = {}
with transaction.atomic():
print('import jurisdictions...')
report.update(juris_importer.import_directory(datadir))
print('import organizations...')
report.update(org_importer.import_directory(datadir))
print('import people...')
report.update(person_importer.import_directory(datadir))
print('import posts...')
report.update(post_importer.import_directory(datadir))
print('import memberships...')
report.update(membership_importer.import_directory(datadir))
print('import bills...')
report.update(bill_importer.import_directory(datadir))
print('import events...')
report.update(event_importer.import_directory(datadir))
print('import votes...')
report.update(vote_importer.import_directory(datadir))
return report
def test_full_vote():
j = Jurisdiction.objects.create(id='jid', division_id='did')
session = j.legislative_sessions.create(name='1900', identifier='1900')
person = Person.objects.create(id='person-id', name='Adam Smith')
org = Organization.objects.create(id='org-id',
name='House',
classification='lower')
bill = Bill.objects.create(id='bill-id',
identifier='HB 1',
legislative_session=session,
from_organization=org)
com = Organization.objects.create(id='com-id',
name='Arbitrary Committee',
parent=org)
vote = ScrapeVote(legislative_session='1900',
motion_text='passage',
start_date='1900-04-01',
classification='passage:bill',
result='pass',
bill_chamber='lower',
bill=bill.identifier)
vote.set_count('yes', 20)
vote.yes('John Smith')
vote.no('Adam Smith')
dmi = DumbMockImporter()
bi = BillImporter('jid', dmi, dmi)
VoteImporter('jid', dmi, dmi, bi).import_data([vote.as_dict()])
assert VoteEvent.objects.count() == 1
ve = VoteEvent.objects.get()
assert ve.legislative_session_id == session.id
assert ve.motion_classification == ['passage:bill']
assert ve.bill_id == bill.id
count = ve.counts.get()
assert count.option == 'yes'
assert count.value == 20
votes = list(ve.votes.all())
assert len(votes) == 2
for v in ve.votes.all():
if v.voter_name == 'John Smith':
assert v.option == 'yes'
else:
assert v.option == 'no'