def test_same_name_people():
# ensure two people with the same name don't import without birthdays
o = Organization.objects.create(name='WWE',
jurisdiction_id='jurisdiction-id')
p1 = ScrapePerson('Dwayne Johnson', image='http://example.com/1')
p2 = ScrapePerson('Dwayne Johnson', image='http://example.com/2')
# the people have the same name but are apparently different
with pytest.raises(SameNameError):
PersonImporter('jurisdiction-id').import_data(
[p1.as_dict(), p2.as_dict()])
# when we give them birth dates all is well though
p1.birth_date = '1970'
p2.birth_date = '1930'
resp = PersonImporter('jurisdiction-id').import_data(
[p1.as_dict(), p2.as_dict()])
assert resp['person']['insert'] == 2
assert resp['person']['noop'] == 0
assert resp['person']['update'] == 0
assert Person.objects.count() == 2
# fake some memberships so future lookups work on these people
for p in Person.objects.all():
Membership.objects.create(person=p, organization=o)
# and now test that an update works and we can insert a new one with the same name
p1.image = 'http://example.com/1.jpg'
p2.birth_date = '1931' # change birth_date, means a new insert
resp = PersonImporter('jurisdiction-id').import_data(
[p1.as_dict(), p2.as_dict()])
assert Person.objects.count() == 3
assert resp['person']['insert'] == 1
assert resp['person']['noop'] == 0
assert resp['person']['update'] == 1
def test_multiple_orgs_of_same_class():
"""
We should be able to set memberships on organizations with the
same classification within the same jurisdictions
"""
create_jurisdiction()
Organization.objects.create(id="fnd", name="Foundation", classification="foundation",
jurisdiction_id="fnd-jid")
Organization.objects.create(id="fdr", name="Federation", classification="foundation",
jurisdiction_id="fnd-jid")
hari = ScrapePerson('Hari Seldon',
primary_org='foundation',
role='founder',
primary_org_name='Foundation')
picard = ScrapePerson('Jean Luc Picard',
primary_org='foundation',
role='founder',
primary_org_name='Federation')
person_imp = PersonImporter('fnd-jid')
person_imp.import_data([hari.as_dict()])
person_imp.import_data([picard.as_dict()])
# try to import a membership
org_imp = OrganizationImporter('fnd-jid')
dumb_imp = DumbMockImporter()
memimp = MembershipImporter('fnd-jid', person_imp, org_imp, dumb_imp)
memimp.import_data([hari._related[0].as_dict(), picard._related[0].as_dict()])
assert Person.objects.get(name='Hari Seldon').memberships.get().organization.name == 'Foundation'
assert Person.objects.get(name='Jean Luc Picard').memberships.get().organization.name == 'Federation'
def test_no_membership_for_person_including_party():
"""
even though party is specified we should still get a no memberships error because it doesn't
bind the person to a jurisdiction, thus causing duplication
"""
create_jurisdiction()
Organization.objects.create(id="fnd",
name="Foundation",
classification="foundation",
jurisdiction_id="fnd-jid")
Organization.objects.create(id="dem",
name="Democratic",
classification="party")
# import a person with no memberships
p = ScrapePerson('a man without a country', party='Democratic')
person_imp = PersonImporter('fnd-jid')
org_imp = OrganizationImporter('fnd-jid')
person_imp.import_data([p.as_dict()])
# try to import a membership
dumb_imp = DumbMockImporter()
memimp = MembershipImporter('fnd-jid', person_imp, org_imp, dumb_imp)
with pytest.raises(NoMembershipsError):
memimp.import_data([p._related[0].as_dict()])
def test_deduplication_no_jurisdiction_overlap():
create_person()
# make sure we get a new person if we're in a different org
person = ScrapePerson('Dwayne Johnson')
pd = person.as_dict()
PersonImporter('new-jurisdiction-id').import_data([pd])
assert Person.objects.all().count() == 2
def do_import(self, juris, args):
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id,
person_importer, org_importer,
post_importer)
#bill_importer = BillImporter(juris.jurisdiction_id, org_importer)
#vote_importer = VoteImporter(juris.jurisdiction_id, person_importer, org_importer,
# bill_importer)
#event_importer = EventImporter(juris.jurisdiction_id)
report = {}
# TODO: wrap in a transaction
report.update(juris_importer.import_directory(datadir))
report.update(org_importer.import_directory(datadir))
report.update(person_importer.import_directory(datadir))
report.update(post_importer.import_directory(datadir))
report.update(membership_importer.import_directory(datadir))
#report.update(bill_importer.import_from_json(datadir))
#report.update(event_importer.import_from_json(datadir))
#report.update(vote_importer.import_from_json(datadir))
return report
def do_import(stream, transaction):
stream = list(stream)
jurisdiction_id = transaction.jurisdiction.id
juris_importer = JurisdictionImporter(jurisdiction_id)
org_importer = OrganizationImporter(jurisdiction_id)
person_importer = PersonImporter(jurisdiction_id)
post_importer = PostImporter(jurisdiction_id, org_importer)
membership_importer = MembershipImporter(jurisdiction_id, person_importer,
org_importer, post_importer)
report = {}
# This basically relates to Pupa's pupa.clu.commands.update:113
# (From there - wrap this in a transaction.)
def tfilter(otype, stream):
for el in filter(lambda x: isinstance(x, otype), stream):
yield el.as_dict()
report.update(juris_importer.import_data(tfilter(Jurisdiction, stream)))
report.update(org_importer.import_data(tfilter(Organization, stream)))
report.update(person_importer.import_data(tfilter(Person, stream)))
report.update(post_importer.import_data(tfilter(Post, stream)))
report.update(membership_importer.import_data(tfilter(Membership, stream)))
return report
def test_deduplication_same_name():
create_person()
# simplest case- just the same name
person = ScrapePerson('Dwayne Johnson')
pd = person.as_dict()
PersonImporter('jurisdiction-id').import_data([pd])
assert Person.objects.all().count() == 1
def test_bill_sponsor_by_identifier():
create_jurisdiction()
org = create_org()
bill = ScrapeBill('HB 1',
'1900',
'Axe & Tack Tax Act',
classification='tax bill',
chamber='lower')
bill.add_sponsorship_by_identifier(name="SNODGRASS",
classification='sponsor',
entity_type='person',
primary=True,
identifier="TOTALLY_REAL_ID",
scheme="TOTALLY_REAL_SCHEME")
oi = OrganizationImporter('jid')
pi = PersonImporter('jid')
zs = ScrapePerson(name='Zadock Snodgrass')
zs.add_identifier(identifier='TOTALLY_REAL_ID',
scheme='TOTALLY_REAL_SCHEME')
pi.import_data([zs.as_dict()])
za_db = Person.objects.get()
Membership.objects.create(person_id=za_db.id, organization_id=org.id)
BillImporter('jid', oi, pi).import_data([bill.as_dict()])
obj = Bill.objects.get()
(entry, ) = obj.sponsorships.all()
assert entry.person.name == "Zadock Snodgrass"
def test_deduplication_other_name_exists():
create_person()
# Rocky is already saved in other_names
person = ScrapePerson('Rocky')
pd = person.as_dict()
PersonImporter('jurisdiction-id').import_data([pd])
assert Person.objects.all().count() == 1
def test_full_person():
person = ScrapePerson('Tom Sawyer')
person.add_identifier('1')
person.add_name('Tommy', start_date='1880')
person.add_contact_detail(type='phone',
value='555-555-1234',
note='this is fake')
person.add_link('http://example.com/link')
person.add_source('http://example.com/source')
# import person
pd = person.as_dict()
PersonImporter('jurisdiction-id').import_data([pd])
# get person from db and assert it imported correctly
p = Person.objects.get()
assert 'ocd-person' in p.id
assert p.name == person.name
assert p.identifiers.all()[0].identifier == '1'
assert p.identifiers.all()[0].scheme == ''
assert p.other_names.all()[0].name == 'Tommy'
assert p.other_names.all()[0].start_date == '1880'
assert p.contact_details.all()[0].type == 'phone'
assert p.contact_details.all()[0].value == '555-555-1234'
assert p.contact_details.all()[0].note == 'this is fake'
assert p.links.all()[0].url == 'http://example.com/link'
assert p.sources.all()[0].url == 'http://example.com/source'
def test_deduplication_no_name_overlap():
create_person()
# make sure we're not just being ridiculous and avoiding importing anything in the same org
person = ScrapePerson('CM Punk')
pd = person.as_dict()
PersonImporter('jurisdiction-id').import_data([pd])
assert Person.objects.all().count() == 2
def test_invalid_fields_related_item():
p1 = ScrapePerson('Dwayne')
p1.add_link('http://example.com')
p1 = p1.as_dict()
p1['links'][0]['test'] = 3
with pytest.raises(DataImportError):
PersonImporter('jid').import_data([p1])
def test_deduplication_other_name_overlaps():
create_person()
# Person has other_name that overlaps w/ existing name
person = ScrapePerson('The Rock')
person.add_name('Dwayne Johnson')
pd = person.as_dict()
PersonImporter('jurisdiction-id').import_data([pd])
assert Person.objects.all().count() == 1
def test_resolve_json_id():
create_jurisdiction()
o = Organization.objects.create(name='WWE', jurisdiction_id='jid')
p = Person.objects.create(name='Dwayne Johnson')
p.other_names.create(name='Rock')
p.memberships.create(organization=o)
pi = PersonImporter('jid')
assert pi.resolve_json_id('~{"name": "Dwayne Johnson"}') == p.id
assert pi.resolve_json_id('~{"name": "Rock"}') == p.id
def test_same_name_second_import():
create_jurisdiction()
# ensure two people with the same name don't import without birthdays
o = Organization.objects.create(name='WWE', jurisdiction_id='jid')
p1 = ScrapePerson('Dwayne Johnson', image='http://example.com/1')
p2 = ScrapePerson('Dwayne Johnson', image='http://example.com/2')
p1.birth_date = '1970'
p2.birth_date = '1930'
# when we give them birth dates all is well though
PersonImporter('jid').import_data([p1.as_dict(), p2.as_dict()])
# fake some memberships so future lookups work on these people
for p in Person.objects.all():
Membership.objects.create(person=p, organization=o)
p3 = ScrapePerson('Dwayne Johnson', image='http://example.com/3')
with pytest.raises(SameNameError):
PersonImporter('jid').import_data([p3.as_dict()])
def test_same_name_people_other_name():
create_jurisdiction()
# ensure we're taking other_names into account for the name collision code
Organization.objects.create(name='WWE', jurisdiction_id='jid')
p1 = ScrapePerson('Dwayne Johnson', image='http://example.com/1')
p2 = ScrapePerson('Rock', image='http://example.com/2')
p2.add_name('Dwayne Johnson')
# the people have the same name but are apparently different
with pytest.raises(SameNameError):
PersonImporter('jid').import_data([p1.as_dict(), p2.as_dict()])
def test_bill_sponsor_limit_lookup():
create_jurisdiction()
org = create_org()
bill = ScrapeBill('HB 1',
'1900',
'Axe & Tack Tax Act',
classification='tax bill',
chamber='lower')
bill.add_sponsorship_by_identifier(name="SNODGRASS",
classification='sponsor',
entity_type='person',
primary=True,
identifier="TOTALLY_REAL_ID",
scheme="TOTALLY_REAL_SCHEME")
oi = OrganizationImporter('jid')
pi = PersonImporter('jid')
zs = ScrapePerson(name='Zadock Snodgrass', birth_date="1800-01-01")
zs.add_identifier(identifier='TOTALLY_REAL_ID',
scheme='TOTALLY_REAL_SCHEME')
pi.import_data([zs.as_dict()])
za_db = Person.objects.get()
Membership.objects.create(person_id=za_db.id, organization_id=org.id)
zs2 = ScrapePerson(name='Zadock Snodgrass', birth_date="1900-01-01")
zs2.add_identifier(identifier='TOTALLY_REAL_ID',
scheme='TOTALLY_REAL_SCHEME')
# This is contrived and perhaps broken, but we're going to check this.
# We *really* don't want to *ever* cross jurisdiction bounds.
PersonImporter('another-jurisdiction').import_data([zs.as_dict()])
BillImporter('jid', oi, pi).import_data([bill.as_dict()])
obj = Bill.objects.get()
(entry, ) = obj.sponsorships.all()
assert entry.person.name == "Zadock Snodgrass"
assert entry.person.birth_date == "1800-01-01"
def test_bill_update_subsubitem():
create_jurisdiction()
create_org()
oi = OrganizationImporter('jid')
pi = PersonImporter('jid')
# initial sub-subitem
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
bill.add_version_link('printing', 'http://example.com/test.pdf', media_type='application/pdf')
result = BillImporter('jid', oi, pi).import_data([bill.as_dict()])
assert result['bill']['insert'] == 1
obj = Bill.objects.get()
assert obj.versions.count() == 1
assert obj.versions.get().links.count() == 1
# a second subsubitem, update
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
bill.add_version_link('printing', 'http://example.com/test.pdf', media_type='application/pdf')
bill.add_version_link('printing', 'http://example.com/test.text', media_type='text/plain')
result = BillImporter('jid', oi, pi).import_data([bill.as_dict()])
assert result['bill']['update'] == 1
obj = Bill.objects.get()
assert obj.versions.count() == 1
assert obj.versions.get().links.count() == 2
# same thing, noop
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
bill.add_version_link('printing', 'http://example.com/test.pdf', media_type='application/pdf')
bill.add_version_link('printing', 'http://example.com/test.text', media_type='text/plain')
result = BillImporter('jid', oi, pi).import_data([bill.as_dict()])
assert result['bill']['noop'] == 1
obj = Bill.objects.get()
assert obj.versions.count() == 1
assert obj.versions.get().links.count() == 2
# different link for second one, update
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
bill.add_version_link('printing', 'http://example.com/test.pdf', media_type='application/pdf')
bill.add_version_link('printing', 'http://example.com/diff-link.txt', media_type='text/plain')
result = BillImporter('jid', oi, pi).import_data([bill.as_dict()])
assert result['bill']['update'] == 1
obj = Bill.objects.get()
assert obj.versions.count() == 1
assert obj.versions.get().links.count() == 2
# delete one, update
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
bill.add_version_link('printing', 'http://example.com/test.pdf', media_type='application/pdf')
result = BillImporter('jid', oi, pi).import_data([bill.as_dict()])
assert result['bill']['update'] == 1
obj = Bill.objects.get()
assert obj.versions.count() == 1
assert obj.versions.get().links.count() == 1
def test_bill_chamber_param():
create_jurisdiction()
org = create_org()
bill = ScrapeBill('HB 1', '1900', 'Axe & Tack Tax Act',
classification='tax bill', chamber='lower')
oi = OrganizationImporter('jid')
pi = PersonImporter('jid')
BillImporter('jid', oi, pi).import_data([bill.as_dict()])
assert Bill.objects.get().from_organization_id == org.id
def test_full_vote_event():
j = Jurisdiction.objects.create(id='jid', division_id='did')
j.legislative_sessions.create(name='1900', identifier='1900')
sp1 = ScrapePerson('John Smith', primary_org='lower')
sp2 = ScrapePerson('Adam Smith', primary_org='lower')
org = ScrapeOrganization(name='House', classification='lower')
bill = ScrapeBill('HB 1',
'1900',
'Axe & Tack Tax Act',
from_organization=org._id)
vote_event = ScrapeVoteEvent(legislative_session='1900',
motion_text='passage',
start_date='1900-04-01',
classification='passage:bill',
result='pass',
bill_chamber='lower',
bill='HB 1',
organization=org._id)
vote_event.set_count('yes', 20)
vote_event.yes('John Smith')
vote_event.no('Adam Smith')
oi = OrganizationImporter('jid')
oi.import_data([org.as_dict()])
pi = PersonImporter('jid')
pi.import_data([sp1.as_dict(), sp2.as_dict()])
mi = MembershipImporter('jid', pi, oi, DumbMockImporter())
mi.import_data([sp1._related[0].as_dict(), sp2._related[0].as_dict()])
bi = BillImporter('jid', oi, pi)
bi.import_data([bill.as_dict()])
VoteEventImporter('jid', pi, oi, bi).import_data([vote_event.as_dict()])
assert VoteEvent.objects.count() == 1
ve = VoteEvent.objects.get()
assert ve.legislative_session == LegislativeSession.objects.get()
assert ve.motion_classification == ['passage:bill']
assert ve.bill == Bill.objects.get()
count = ve.counts.get()
assert count.option == 'yes'
assert count.value == 20
votes = list(ve.votes.all())
assert len(votes) == 2
for v in ve.votes.all():
if v.voter_name == 'John Smith':
assert v.option == 'yes'
assert v.voter == Person.objects.get(name='John Smith')
else:
assert v.option == 'no'
assert v.voter == Person.objects.get(name='Adam Smith')
def test_multiple_memberships():
create_jurisdiction()
# there was a bug where two or more memberships to the same jurisdiction
# would cause an ORM error, this test ensures that it is fixed
p = Person.objects.create(name='Dwayne Johnson')
o = Organization.objects.create(name='WWE', jurisdiction_id='jid')
Membership.objects.create(person=p, organization=o)
o = Organization.objects.create(name='WWF', jurisdiction_id='jid')
Membership.objects.create(person=p, organization=o)
person = ScrapePerson('Dwayne Johnson')
pd = person.as_dict()
PersonImporter('jid').import_data([pd])
# deduplication should still work
assert Person.objects.all().count() == 1
def test_no_membership_for_person():
create_jurisdiction()
Organization.objects.create(id="fnd", name="Foundation", classification="foundation",
jurisdiction_id="fnd-jid")
# import a person with no memberships
p = ScrapePerson('a man without a country')
person_imp = PersonImporter('fnd-jid')
person_imp.import_data([p.as_dict()])
# try to import a membership
dumb_imp = DumbMockImporter()
memimp = MembershipImporter('fnd-jid', person_imp, dumb_imp, dumb_imp)
with pytest.raises(NoMembershipsError):
memimp.import_data([])
def test_resolve_json_id_multiple_family_name():
create_jurisdiction()
o = Organization.objects.create(name='WWE', jurisdiction_id='jid')
p1 = Person.objects.create(name='Dwayne Johnson', family_name='Johnson')
p1.other_names.create(name='Rock')
p2 = Person.objects.create(name='Adam Johnson', family_name='Johnson')
for p in Person.objects.all():
Membership.objects.create(person=p, organization=o)
# If there are multiple people with a family name, full name/other name
# lookups should work but family name lookups should fail.
pi = PersonImporter('jid')
assert pi.resolve_json_id('~{"name": "Dwayne Johnson"}') == p1.id
assert pi.resolve_json_id('~{"name": "Adam Johnson"}') == p2.id
with pytest.raises(UnresolvedIdError):
pi.resolve_json_id('~{"name": "Johnson"}')
def do_import(self, juris, args):
# import inside here because to avoid loading Django code unnecessarily
from pupa.importers import (JurisdictionImporter, OrganizationImporter,
PersonImporter, PostImporter,
MembershipImporter, BillImporter,
VoteEventImporter, EventImporter)
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id,
person_importer, org_importer,
post_importer)
bill_importer = BillImporter(juris.jurisdiction_id, org_importer,
person_importer)
vote_event_importer = VoteEventImporter(juris.jurisdiction_id,
person_importer, org_importer,
bill_importer)
event_importer = EventImporter(juris.jurisdiction_id, org_importer,
person_importer, bill_importer,
vote_event_importer)
report = {}
with transaction.atomic():
print('import jurisdictions...')
report.update(juris_importer.import_directory(datadir))
print('import organizations...')
report.update(org_importer.import_directory(datadir))
print('import people...')
report.update(person_importer.import_directory(datadir))
print('import posts...')
report.update(post_importer.import_directory(datadir))
print('import memberships...')
report.update(membership_importer.import_directory(datadir))
print('import bills...')
report.update(bill_importer.import_directory(datadir))
print('import events...')
report.update(event_importer.import_directory(datadir))
print('import vote events...')
report.update(vote_event_importer.import_directory(datadir))
return report
def test_same_name_people():
o = Organization.objects.create(name='WWE',
jurisdiction_id='jurisdiction-id')
# importing two people with the same name to a pristine database should error
p1 = ScrapePerson('Dwayne Johnson', image='http://example.com/1')
p2 = ScrapePerson('Dwayne Johnson', image='http://example.com/2')
with pytest.raises(SameNameError):
PersonImporter('jurisdiction-id').import_data(
[p1.as_dict(), p2.as_dict()])
# importing one person should pass
PersonImporter('jurisdiction-id').import_data([p1.as_dict()])
# create fake memberships so that future lookups work on the imported people
for p in Person.objects.all():
Membership.objects.create(person=p, organization=o)
# importing another person with the same name should fail
with pytest.raises(SameNameError):
PersonImporter('jurisdiction-id').import_data(
[p1.as_dict(), p2.as_dict()])
# adding birth dates should pass
p1.birth_date = '1970'
p2.birth_date = '1930'
resp = PersonImporter('jurisdiction-id').import_data(
[p1.as_dict(), p2.as_dict()])
assert resp['person']['insert'] == 1
assert resp['person']['noop'] == 0
assert resp['person']['update'] == 1
assert Person.objects.count() == 2
# create fake memberships so that future lookups work on the imported people
for p in Person.objects.all():
Membership.objects.create(person=p, organization=o)
# adding a third person with the same name but without a birthday should error
p3 = ScrapePerson('Dwayne Johnson', image='http://example.com/3')
with pytest.raises(SameNameError):
PersonImporter('jurisdiction-id').import_data([p3.as_dict()])
# and now test that an update works and we can insert a new one with the same name
p1.image = 'http://example.com/1.jpg'
p2.birth_date = '1931' # change birth_date, means a new insert
resp = PersonImporter('jurisdiction-id').import_data(
[p1.as_dict(), p2.as_dict()])
assert Person.objects.count() == 3
assert resp['person']['insert'] == 1
assert resp['person']['noop'] == 0
assert resp['person']['update'] == 1
def test_resolve_json_id():
p1 = ScrapePerson('Dwayne').as_dict()
p2 = ScrapePerson('Dwayne').as_dict()
pi = PersonImporter('jid')
# do import and get database id
p1_id = p1['_id']
p2_id = p2['_id']
pi.import_data([p1, p2])
db_id = Person.objects.get().id
# simplest case
assert pi.resolve_json_id(p1_id) == db_id
# duplicate should resolve to same id
assert pi.resolve_json_id(p2_id) == db_id
# a null id should map to None
assert pi.resolve_json_id(None) is None
# no such id
with pytest.raises(UnresolvedIdError):
pi.resolve_json_id('this-is-invalid')
def test_bill_update():
create_jurisdiction()
create_org()
bill = ScrapeBill('HB 1', '1900', 'First Bill', chamber='lower')
oi = OrganizationImporter('jid')
pi = PersonImporter('jid')
_, what = BillImporter('jid', oi, pi).import_item(bill.as_dict())
assert what == 'insert'
_, what = BillImporter('jid', oi, pi).import_item(bill.as_dict())
assert what == 'noop'
# ensure no new object was created
assert Bill.objects.count() == 1
# test basic update
bill = ScrapeBill('HB 1', '1900', '1st Bill', chamber='lower')
_, what = BillImporter('jid', oi, pi).import_item(bill.as_dict())
assert what == 'update'
assert Bill.objects.get().title == '1st Bill'
def test_bill_action_extras():
create_jurisdiction()
create_org()
bill = ScrapeBill('HB 1',
'1900',
'Axe & Tack Tax Act',
classification='tax bill',
chamber='lower')
bill.add_action('sample',
'1900-01-01',
chamber='lower',
extras={'test': 3})
oi = OrganizationImporter('jid')
pi = PersonImporter('jid')
BillImporter('jid', oi, pi).import_data([bill.as_dict()])
b = Bill.objects.get()
assert b.actions.all()[0].extras == {'test': 3}
def test_fix_bill_id():
create_jurisdiction()
create_org()
bill = ScrapeBill('HB1',
'1900',
'Test Bill ID',
classification='bill',
chamber='lower')
oi = OrganizationImporter('jid')
pi = PersonImporter('jid')
from pupa.settings import IMPORT_TRANSFORMERS
IMPORT_TRANSFORMERS['bill'] = {
'identifier':
lambda x: re.sub(r'([A-Z]*)\s*0*([-\d]+)', r'\1 \2', x, 1)
}
bi = BillImporter('jid', oi, pi)
bi.import_data([bill.as_dict()])
IMPORT_TRANSFORMERS['bill'] = {}
b = Bill.objects.get()
assert b.identifier == 'HB 1'
def do_import(stream, stransaction):
stream = list(stream)
jurisdiction_id = stransaction.jurisdiction.id
org_importer = OrganizationImporter(jurisdiction_id)
person_importer = PersonImporter(jurisdiction_id)
post_importer = PostImporter(jurisdiction_id, org_importer)
membership_importer = MembershipImporter(jurisdiction_id, person_importer,
org_importer, post_importer)
report = {}
def tfilter(otype, stream):
for el in filter(lambda x: isinstance(x, otype), stream):
yield el.as_dict()
with transaction.atomic():
report.update(org_importer.import_data(tfilter(Organization, stream)))
report.update(person_importer.import_data(tfilter(Person, stream)))
report.update(post_importer.import_data(tfilter(Post, stream)))
report.update(
membership_importer.import_data(tfilter(Membership, stream)))
return report
