def do_import(self, juris, args):
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id,
person_importer, org_importer,
post_importer)
#bill_importer = BillImporter(juris.jurisdiction_id, org_importer)
#vote_importer = VoteImporter(juris.jurisdiction_id, person_importer, org_importer,
# bill_importer)
#event_importer = EventImporter(juris.jurisdiction_id)
report = {}
# TODO: wrap in a transaction
report.update(juris_importer.import_directory(datadir))
report.update(org_importer.import_directory(datadir))
report.update(person_importer.import_directory(datadir))
report.update(post_importer.import_directory(datadir))
report.update(membership_importer.import_directory(datadir))
#report.update(bill_importer.import_from_json(datadir))
#report.update(event_importer.import_from_json(datadir))
#report.update(vote_importer.import_from_json(datadir))
return report
def do_import(self, juris, args):
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id, person_importer,
org_importer, post_importer)
#bill_importer = BillImporter(juris.jurisdiction_id, org_importer)
#vote_importer = VoteImporter(juris.jurisdiction_id, person_importer, org_importer,
# bill_importer)
#event_importer = EventImporter(juris.jurisdiction_id)
report = {}
# TODO: wrap in a transaction
report.update(juris_importer.import_directory(datadir))
report.update(org_importer.import_directory(datadir))
report.update(person_importer.import_directory(datadir))
report.update(post_importer.import_directory(datadir))
report.update(membership_importer.import_directory(datadir))
#report.update(bill_importer.import_from_json(datadir))
#report.update(event_importer.import_from_json(datadir))
#report.update(vote_importer.import_from_json(datadir))
return report
def test_multiple_orgs_of_same_class():
"""
We should be able to set memberships on organizations with the
same classification within the same jurisdictions
"""
Organization.objects.create(id="fnd", name="Foundation", classification="foundation",
jurisdiction_id="fnd-jid")
Organization.objects.create(id="fdr", name="Federation", classification="foundation",
jurisdiction_id="fnd-jid")
hari = ScrapePerson('Hari Seldon',
primary_org='foundation',
role='founder',
primary_org_name='Foundation')
picard = ScrapePerson('Jean Luc Picard',
primary_org='foundation',
role='founder',
primary_org_name='Federation')
person_imp = PersonImporter('fnd-jid')
person_imp.import_data([hari.as_dict()])
person_imp.import_data([picard.as_dict()])
# try to import a membership
org_imp = OrganizationImporter('fnd-jid')
dumb_imp = DumbMockImporter()
memimp = MembershipImporter('fnd-jid', person_imp, org_imp, dumb_imp)
memimp.import_data([hari._related[0].as_dict(),
picard._related[0].as_dict()])
assert Person.objects.get(name='Hari Seldon').memberships.get().organization.name == 'Foundation'
assert Person.objects.get(name='Jean Luc Picard').memberships.get().organization.name == 'Federation'
def do_import(stream, transaction):
stream = list(stream)
jurisdiction_id = transaction.jurisdiction.id
juris_importer = JurisdictionImporter(jurisdiction_id)
org_importer = OrganizationImporter(jurisdiction_id)
person_importer = PersonImporter(jurisdiction_id)
post_importer = PostImporter(jurisdiction_id, org_importer)
membership_importer = MembershipImporter(
jurisdiction_id,
person_importer,
org_importer,
post_importer
)
report = {}
# This basically relates to Pupa's pupa.clu.commands.update:113
# (From there - wrap this in a transaction.)
def tfilter(otype, stream):
for el in filter(lambda x: isinstance(x, otype), stream):
yield el.as_dict()
report.update(juris_importer.import_data(tfilter(Jurisdiction, stream)))
report.update(org_importer.import_data(tfilter(Organization, stream)))
report.update(person_importer.import_data(tfilter(Person, stream)))
report.update(post_importer.import_data(tfilter(Post, stream)))
report.update(membership_importer.import_data(
tfilter(Membership, stream)))
return report
def test_full_membership():
org = Organization.objects.create(id="fnd", name="Foundation", classification="foundation",
jurisdiction_id="fnd-jid")
hari = Person.objects.create(id="hs", name="Hari Seldon")
robot = Person.objects.create(id="robot", name="R. Daneel Olivaw")
post = Post.objects.create(id='f', label="founder", role="Founder", organization=org)
# add a membership through a post
m1 = ScrapeMembership(person_id=hari.id, organization_id=org.id, post_id=post.id)
m1.add_contact_detail(type='phone', value='555-555-1234', note='this is fake')
m1.add_link('http://example.com/link')
# add a membership direct to an organization
m2 = ScrapeMembership(person_id=robot.id, organization_id=org.id, label='member',
role='member')
dumb_imp = DumbMockImporter()
memimp = MembershipImporter('fnd-jid', dumb_imp, dumb_imp, dumb_imp)
memimp.import_data([m1.as_dict(), m2.as_dict()])
# ensure that the memberships attached in the right places
assert org.memberships.count() == 2
assert hari.memberships.count() == 1
assert robot.memberships.count() == 1
assert post.memberships.count() == 1
# ensure that the first membership has contact details and links
m = hari.memberships.get()
cd = m.contact_details.get()
assert cd.type == 'phone'
assert cd.value == '555-555-1234'
assert cd.note == 'this is fake'
assert m.links.all()[0].url == 'http://example.com/link'
def do_import(stream, stransaction):
stream = list(stream)
jurisdiction_id = stransaction.jurisdiction.id
org_importer = OrganizationImporter(jurisdiction_id)
person_importer = PersonImporter(jurisdiction_id)
post_importer = PostImporter(jurisdiction_id, org_importer)
membership_importer = MembershipImporter(
jurisdiction_id,
person_importer,
org_importer,
post_importer
)
report = {}
def tfilter(otype, stream):
for el in filter(lambda x: isinstance(x, otype), stream):
yield el.as_dict()
with transaction.atomic():
report.update(org_importer.import_data(tfilter(Organization, stream)))
report.update(person_importer.import_data(tfilter(Person, stream)))
report.update(post_importer.import_data(tfilter(Post, stream)))
report.update(membership_importer.import_data(
tfilter(Membership, stream)))
return report
def test_multiple_posts_class():
create_jurisdiction()
org = Organization.objects.create(id="fnd",
name="Foundation",
classification="foundation",
jurisdiction_id="fnd-jid")
hari = Person.objects.create(id="hs", name="Hari Seldon")
founder = Post.objects.create(id='f',
label="founder",
role="Founder",
organization=org)
chair = Post.objects.create(id='c',
label="chair",
role="Chair",
organization=org)
m1 = ScrapeMembership(person_id=hari.id,
organization_id=org.id,
post_id=founder.id)
m2 = ScrapeMembership(person_id=hari.id,
organization_id=org.id,
post_id=chair.id)
dumb_imp = DumbMockImporter()
memimp = MembershipImporter('fnd-jid', dumb_imp, dumb_imp, dumb_imp)
memimp.import_data([m1.as_dict(), m2.as_dict()])
# ensure that the memberships attached in the right places
assert org.memberships.count() == 2
assert hari.memberships.count() == 2
assert founder.memberships.count() == 1
assert chair.memberships.count() == 1
def do_import(stream, transaction):
stream = list(stream)
jurisdiction_id = transaction.jurisdiction.id
juris_importer = JurisdictionImporter(jurisdiction_id)
org_importer = OrganizationImporter(jurisdiction_id)
person_importer = PersonImporter(jurisdiction_id)
post_importer = PostImporter(jurisdiction_id, org_importer)
membership_importer = MembershipImporter(jurisdiction_id, person_importer,
org_importer, post_importer)
report = {}
# This basically relates to Pupa's pupa.clu.commands.update:113
# (From there - wrap this in a transaction.)
def tfilter(otype, stream):
for el in filter(lambda x: isinstance(x, otype), stream):
yield el.as_dict()
report.update(juris_importer.import_data(tfilter(Jurisdiction, stream)))
report.update(org_importer.import_data(tfilter(Organization, stream)))
report.update(person_importer.import_data(tfilter(Person, stream)))
report.update(post_importer.import_data(tfilter(Post, stream)))
report.update(membership_importer.import_data(tfilter(Membership, stream)))
return report
def test_no_membership_for_person_including_party():
"""
even though party is specified we should still get a no memberships error because it doesn't
bind the person to a jurisdiction, thus causing duplication
"""
create_jurisdiction()
Organization.objects.create(id="fnd",
name="Foundation",
classification="foundation",
jurisdiction_id="fnd-jid")
Organization.objects.create(id="dem",
name="Democratic",
classification="party")
# import a person with no memberships
p = ScrapePerson('a man without a country', party='Democratic')
person_imp = PersonImporter('fnd-jid')
org_imp = OrganizationImporter('fnd-jid')
person_imp.import_data([p.as_dict()])
# try to import a membership
dumb_imp = DumbMockImporter()
memimp = MembershipImporter('fnd-jid', person_imp, org_imp, dumb_imp)
with pytest.raises(NoMembershipsError):
memimp.import_data([p._related[0].as_dict()])
def test_full_membership():
create_jurisdiction()
org = Organization.objects.create(id="fnd", name="Foundation", classification="foundation",
jurisdiction_id="fnd-jid")
hari = Person.objects.create(id="hs", name="Hari Seldon")
robot = Person.objects.create(id="robot", name="R. Daneel Olivaw")
post = Post.objects.create(id='f', label="founder", role="Founder", organization=org)
# add a membership through a post
m1 = ScrapeMembership(person_id=hari.id, organization_id=org.id, post_id=post.id)
m1.add_contact_detail(type='phone', value='555-555-1234', note='this is fake')
m1.add_link('http://example.com/link')
# add a membership direct to an organization
m2 = ScrapeMembership(person_id=robot.id, organization_id=org.id, label='member',
role='member')
dumb_imp = DumbMockImporter()
memimp = MembershipImporter('fnd-jid', dumb_imp, dumb_imp, dumb_imp)
memimp.import_data([m1.as_dict(), m2.as_dict()])
# ensure that the memberships attached in the right places
assert org.memberships.count() == 2
assert hari.memberships.count() == 1
assert robot.memberships.count() == 1
assert post.memberships.count() == 1
# ensure that the first membership has contact details and links
m = hari.memberships.get()
cd = m.contact_details.get()
assert cd.type == 'phone'
assert cd.value == '555-555-1234'
assert cd.note == 'this is fake'
assert m.links.all()[0].url == 'http://example.com/link'
def test_multiple_orgs_of_same_class():
"""
We should be able to set memberships on organizations with the
same classification within the same jurisdictions
"""
create_jurisdiction()
Organization.objects.create(id="fnd", name="Foundation", classification="foundation",
jurisdiction_id="fnd-jid")
Organization.objects.create(id="fdr", name="Federation", classification="foundation",
jurisdiction_id="fnd-jid")
hari = ScrapePerson('Hari Seldon',
primary_org='foundation',
role='founder',
primary_org_name='Foundation')
picard = ScrapePerson('Jean Luc Picard',
primary_org='foundation',
role='founder',
primary_org_name='Federation')
person_imp = PersonImporter('fnd-jid')
person_imp.import_data([hari.as_dict()])
person_imp.import_data([picard.as_dict()])
# try to import a membership
org_imp = OrganizationImporter('fnd-jid')
dumb_imp = DumbMockImporter()
memimp = MembershipImporter('fnd-jid', person_imp, org_imp, dumb_imp)
memimp.import_data([hari._related[0].as_dict(), picard._related[0].as_dict()])
assert Person.objects.get(name='Hari Seldon').memberships.get().organization.name == 'Foundation'
assert Person.objects.get(name='Jean Luc Picard').memberships.get().organization.name == 'Federation'
def do_import(self, juris, args):
# import inside here because to avoid loading Django code unnecessarily
from pupa.importers import (JurisdictionImporter, OrganizationImporter, PersonImporter,
PostImporter, MembershipImporter, BillImporter,
VoteEventImporter, EventImporter)
from pupa.reports import generate_session_report
from pupa.models import SessionDataQualityReport
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id, person_importer,
org_importer, post_importer)
bill_importer = BillImporter(juris.jurisdiction_id, org_importer, person_importer)
vote_event_importer = VoteEventImporter(juris.jurisdiction_id, person_importer,
org_importer, bill_importer)
event_importer = EventImporter(juris.jurisdiction_id,
org_importer,
person_importer,
bill_importer,
vote_event_importer)
report = {}
with transaction.atomic():
print('import jurisdictions...')
report.update(juris_importer.import_directory(datadir))
if settings.ENABLE_PEOPLE_AND_ORGS:
print('import organizations...')
report.update(org_importer.import_directory(datadir))
print('import people...')
report.update(person_importer.import_directory(datadir))
print('import posts...')
report.update(post_importer.import_directory(datadir))
print('import memberships...')
report.update(membership_importer.import_directory(datadir))
if settings.ENABLE_BILLS:
print('import bills...')
report.update(bill_importer.import_directory(datadir))
if settings.ENABLE_EVENTS:
print('import events...')
report.update(event_importer.import_directory(datadir))
if settings.ENABLE_VOTES:
print('import vote events...')
report.update(vote_event_importer.import_directory(datadir))
# compile info on all sessions that were updated in this run
seen_sessions = set()
seen_sessions.update(bill_importer.get_seen_sessions())
seen_sessions.update(vote_event_importer.get_seen_sessions())
for session in seen_sessions:
new_report = generate_session_report(session)
with transaction.atomic():
SessionDataQualityReport.objects.filter(legislative_session=session).delete()
new_report.save()
return report
def test_full_vote_event():
j = Jurisdiction.objects.create(id='jid', division_id='did')
j.legislative_sessions.create(name='1900', identifier='1900')
sp1 = ScrapePerson('John Smith', primary_org='lower')
sp2 = ScrapePerson('Adam Smith', primary_org='lower')
org = ScrapeOrganization(name='House', classification='lower')
bill = ScrapeBill('HB 1',
'1900',
'Axe & Tack Tax Act',
from_organization=org._id)
vote_event = ScrapeVoteEvent(legislative_session='1900',
motion_text='passage',
start_date='1900-04-01',
classification='passage:bill',
result='pass',
bill_chamber='lower',
bill='HB 1',
organization=org._id)
vote_event.set_count('yes', 20)
vote_event.yes('John Smith')
vote_event.no('Adam Smith')
oi = OrganizationImporter('jid')
oi.import_data([org.as_dict()])
pi = PersonImporter('jid')
pi.import_data([sp1.as_dict(), sp2.as_dict()])
mi = MembershipImporter('jid', pi, oi, DumbMockImporter())
mi.import_data([sp1._related[0].as_dict(), sp2._related[0].as_dict()])
bi = BillImporter('jid', oi, pi)
bi.import_data([bill.as_dict()])
VoteEventImporter('jid', pi, oi, bi).import_data([vote_event.as_dict()])
assert VoteEvent.objects.count() == 1
ve = VoteEvent.objects.get()
assert ve.legislative_session == LegislativeSession.objects.get()
assert ve.motion_classification == ['passage:bill']
assert ve.bill == Bill.objects.get()
count = ve.counts.get()
assert count.option == 'yes'
assert count.value == 20
votes = list(ve.votes.all())
assert len(votes) == 2
for v in ve.votes.all():
if v.voter_name == 'John Smith':
assert v.option == 'yes'
assert v.voter == Person.objects.get(name='John Smith')
else:
assert v.option == 'no'
assert v.voter == Person.objects.get(name='Adam Smith')
def do_import(self, juris, args):
# import inside here because to avoid loading Django code unnecessarily
from pupa.importers import (JurisdictionImporter, OrganizationImporter, PersonImporter,
PostImporter, MembershipImporter, BillImporter,
VoteImporter, EventImporter,
DisclosureImporter)
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id, person_importer,
org_importer, post_importer)
bill_importer = BillImporter(juris.jurisdiction_id, org_importer, person_importer)
vote_importer = VoteImporter(juris.jurisdiction_id, person_importer, org_importer,
bill_importer)
event_importer = EventImporter(juris.jurisdiction_id,
org_importer,
person_importer)
disclosure_importer = DisclosureImporter(juris.jurisdiction_id,
org_importer,
person_importer,
event_importer)
report = {}
with transaction.atomic():
print('import jurisdictions...')
report.update(juris_importer.import_directory(datadir))
print('import organizations...')
report.update(org_importer.import_directory(datadir))
print('import people...')
report.update(person_importer.import_directory(datadir))
print('import posts...')
report.update(post_importer.import_directory(datadir))
print('import memberships...')
report.update(membership_importer.import_directory(datadir))
print('import bills...')
report.update(bill_importer.import_directory(datadir))
print('import events...')
report.update(event_importer.import_directory(datadir))
print('import disclosures...')
report.update(disclosure_importer.import_directory(datadir))
print('import votes...')
report.update(vote_importer.import_directory(datadir))
return report
def test_no_membership_for_person():
org = Organization.objects.create(id="fnd", name="Foundation", classification="foundation",
jurisdiction_id="fnd-jid")
# import a person with no memberships
p = ScrapePerson('a man without a country')
person_imp = PersonImporter('fnd-jid')
person_imp.import_data([p.as_dict()])
# try to import a membership
dumb_imp = DumbMockImporter()
memimp = MembershipImporter('fnd-jid', person_imp, dumb_imp, dumb_imp)
with pytest.raises(NoMembershipsError):
memimp.import_data([])
def test_no_membership_for_person():
create_jurisdiction()
Organization.objects.create(id="fnd", name="Foundation", classification="foundation",
jurisdiction_id="fnd-jid")
# import a person with no memberships
p = ScrapePerson('a man without a country')
person_imp = PersonImporter('fnd-jid')
person_imp.import_data([p.as_dict()])
# try to import a membership
dumb_imp = DumbMockImporter()
memimp = MembershipImporter('fnd-jid', person_imp, dumb_imp, dumb_imp)
with pytest.raises(NoMembershipsError):
memimp.import_data([])
def test_full_vote_event():
j = Jurisdiction.objects.create(id='jid', division_id='did')
j.legislative_sessions.create(name='1900', identifier='1900')
sp1 = ScrapePerson('John Smith', primary_org='lower')
sp2 = ScrapePerson('Adam Smith', primary_org='lower')
org = ScrapeOrganization(name='House', classification='lower')
bill = ScrapeBill('HB 1', '1900', 'Axe & Tack Tax Act', from_organization=org._id)
vote_event = ScrapeVoteEvent(legislative_session='1900', motion_text='passage',
start_date='1900-04-01', classification='passage:bill',
result='pass', bill_chamber='lower', bill='HB 1',
organization=org._id)
vote_event.set_count('yes', 20)
vote_event.yes('John Smith')
vote_event.no('Adam Smith')
oi = OrganizationImporter('jid')
oi.import_data([org.as_dict()])
pi = PersonImporter('jid')
pi.import_data([sp1.as_dict(), sp2.as_dict()])
mi = MembershipImporter('jid', pi, oi, DumbMockImporter())
mi.import_data([sp1._related[0].as_dict(), sp2._related[0].as_dict()])
bi = BillImporter('jid', oi, pi)
bi.import_data([bill.as_dict()])
VoteEventImporter('jid', pi, oi, bi).import_data([vote_event.as_dict()])
assert VoteEvent.objects.count() == 1
ve = VoteEvent.objects.get()
assert ve.legislative_session == LegislativeSession.objects.get()
assert ve.motion_classification == ['passage:bill']
assert ve.bill == Bill.objects.get()
count = ve.counts.get()
assert count.option == 'yes'
assert count.value == 20
votes = list(ve.votes.all())
assert len(votes) == 2
for v in ve.votes.all():
if v.voter_name == 'John Smith':
assert v.option == 'yes'
assert v.voter == Person.objects.get(name='John Smith')
else:
assert v.option == 'no'
assert v.voter == Person.objects.get(name='Adam Smith')
def do_import(self, juris, args):
# import inside here because to avoid loading Django code unnecessarily
from pupa.importers import (JurisdictionImporter, OrganizationImporter,
PersonImporter, PostImporter,
MembershipImporter, BillImporter,
VoteEventImporter, EventImporter)
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id,
person_importer, org_importer,
post_importer)
bill_importer = BillImporter(juris.jurisdiction_id, org_importer,
person_importer)
vote_event_importer = VoteEventImporter(juris.jurisdiction_id,
person_importer, org_importer,
bill_importer)
event_importer = EventImporter(juris.jurisdiction_id, org_importer,
person_importer, bill_importer,
vote_event_importer)
report = {}
with transaction.atomic():
print('import jurisdictions...')
report.update(juris_importer.import_directory(datadir))
print('import organizations...')
report.update(org_importer.import_directory(datadir))
print('import people...')
report.update(person_importer.import_directory(datadir))
print('import posts...')
report.update(post_importer.import_directory(datadir))
print('import memberships...')
report.update(membership_importer.import_directory(datadir))
print('import bills...')
report.update(bill_importer.import_directory(datadir))
print('import events...')
report.update(event_importer.import_directory(datadir))
print('import vote events...')
report.update(vote_event_importer.import_directory(datadir))
return report
def test_no_membership_for_person_including_party():
"""
even though party is specified we should still get a no memberships error because it doesn't
bind the person to a jurisdiction, thus causing duplication
"""
org = Organization.objects.create(id="fnd", name="Foundation", classification="foundation",
jurisdiction_id="fnd-jid")
org = Organization.objects.create(id="dem", name="Democratic", classification="party")
# import a person with no memberships
p = ScrapePerson('a man without a country', party='Democratic')
person_imp = PersonImporter('fnd-jid')
person_imp.import_data([p.as_dict()])
# try to import a membership
dumb_imp = DumbMockImporter()
memimp = MembershipImporter('fnd-jid', person_imp, dumb_imp, dumb_imp)
with pytest.raises(NoMembershipsError):
memimp.import_data([p._related[0].as_dict()])
def test_unmatched_person():
create_jurisdiction()
org = Organization.objects.create(id="fnd", name="Foundation", classification="foundation",
jurisdiction_id="fnd-jid")
# not a real person, won't have a person_id after import
m1 = ScrapeMembership(person_name='Harry Seldom', organization_id=org.id,
person_id=None
)
dumb_imp = DumbMockImporter()
memimp = MembershipImporter('fnd-jid', dumb_imp, dumb_imp, dumb_imp)
memimp.import_data([m1.as_dict()])
# ensure that the memberships attached in the right places
assert org.memberships.count() == 1
membership = org.memberships.get()
assert membership.person_id is None
assert membership.person_name == 'Harry Seldom'
def test_multiple_posts_class():
create_jurisdiction()
org = Organization.objects.create(id="fnd", name="Foundation", classification="foundation",
jurisdiction_id="fnd-jid")
hari = Person.objects.create(id="hs", name="Hari Seldon")
founder = Post.objects.create(id='f', label="founder", role="Founder", organization=org)
chair = Post.objects.create(id='c', label="chair", role="Chair", organization=org)
m1 = ScrapeMembership(person_id=hari.id, organization_id=org.id, post_id=founder.id)
m2 = ScrapeMembership(person_id=hari.id, organization_id=org.id, post_id=chair.id)
dumb_imp = DumbMockImporter()
memimp = MembershipImporter('fnd-jid', dumb_imp, dumb_imp, dumb_imp)
memimp.import_data([m1.as_dict(), m2.as_dict()])
# ensure that the memberships attached in the right places
assert org.memberships.count() == 2
assert hari.memberships.count() == 2
assert founder.memberships.count() == 1
assert chair.memberships.count() == 1
def test_unmatched_person():
create_jurisdiction()
org = Organization.objects.create(id="fnd",
name="Foundation",
classification="foundation",
jurisdiction_id="fnd-jid")
# not a real person, won't have a person_id after import
m1 = ScrapeMembership(person_name='Harry Seldom',
organization_id=org.id,
person_id=None)
dumb_imp = DumbMockImporter()
memimp = MembershipImporter('fnd-jid', dumb_imp, dumb_imp, dumb_imp)
memimp.import_data([m1.as_dict()])
# ensure that the memberships attached in the right places
assert org.memberships.count() == 1
membership = org.memberships.get()
assert membership.person_id is None
assert membership.person_name == 'Harry Seldom'
def do_import(stream, stransaction):
stream = list(stream)
jurisdiction_id = stransaction.jurisdiction.id
org_importer = OrganizationImporter(jurisdiction_id)
person_importer = PersonImporter(jurisdiction_id)
post_importer = PostImporter(jurisdiction_id, org_importer)
membership_importer = MembershipImporter(jurisdiction_id, person_importer,
org_importer, post_importer)
report = {}
def tfilter(otype, stream):
for el in filter(lambda x: isinstance(x, otype), stream):
yield el.as_dict()
with transaction.atomic():
report.update(org_importer.import_data(tfilter(Organization, stream)))
report.update(person_importer.import_data(tfilter(Person, stream)))
report.update(post_importer.import_data(tfilter(Post, stream)))
report.update(
membership_importer.import_data(tfilter(Membership, stream)))
return report
def do_import(self, juris, args):
# import inside here because to avoid loading Django code unnecessarily
from pupa.importers import (JurisdictionImporter, OrganizationImporter,
PersonImporter, PostImporter,
MembershipImporter, BillImporter,
VoteEventImporter, EventImporter)
from pupa.reports import generate_session_report
from pupa.models import SessionDataQualityReport
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(juris.jurisdiction_id,
person_importer, org_importer,
post_importer)
bill_importer = BillImporter(juris.jurisdiction_id, org_importer,
person_importer)
vote_event_importer = VoteEventImporter(juris.jurisdiction_id,
person_importer, org_importer,
bill_importer)
event_importer = EventImporter(juris.jurisdiction_id, org_importer,
person_importer, bill_importer,
vote_event_importer)
report = {}
with transaction.atomic():
print('import jurisdictions...')
report.update(juris_importer.import_directory(datadir))
if settings.ENABLE_PEOPLE_AND_ORGS:
print('import organizations...')
report.update(org_importer.import_directory(datadir))
print('import people...')
report.update(person_importer.import_directory(datadir))
print('import posts...')
report.update(post_importer.import_directory(datadir))
print('import memberships...')
report.update(membership_importer.import_directory(datadir))
if settings.ENABLE_BILLS:
print('import bills...')
report.update(bill_importer.import_directory(datadir))
if settings.ENABLE_EVENTS:
print('import events...')
report.update(event_importer.import_directory(datadir))
if settings.ENABLE_VOTES:
print('import vote events...')
report.update(vote_event_importer.import_directory(datadir))
# compile info on all sessions that were updated in this run
seen_sessions = set()
seen_sessions.update(bill_importer.get_seen_sessions())
seen_sessions.update(vote_event_importer.get_seen_sessions())
for session in seen_sessions:
new_report = generate_session_report(session)
with transaction.atomic():
SessionDataQualityReport.objects.filter(
legislative_session=session).delete()
new_report.save()
return report
def test_full_membership():
create_jurisdiction()
org = Organization.objects.create(id="fnd",
name="Foundation",
classification="foundation",
jurisdiction_id="fnd-jid")
hari = Person.objects.create(id="hs", name="Hari Seldon")
robot = Person.objects.create(id="robot", name="R. Daneel Olivaw")
post = Post.objects.create(id='f',
label="founder",
role="Founder",
organization=org)
# add a membership through a post, with a start date
m1 = ScrapeMembership(person_id=hari.id,
organization_id=org.id,
post_id=post.id,
start_date='2020-03-10',
end_date='2021-06-30')
m1.add_contact_detail(type='phone',
value='555-555-1234',
note='this is fake')
m1.add_link('http://example.com/link')
# add a membership direct to an organization, with an end date
m2 = ScrapeMembership(person_id=robot.id,
organization_id=org.id,
label='member',
role='member',
end_date='2019-11-09')
dumb_imp = DumbMockImporter()
memimp = MembershipImporter('fnd-jid', dumb_imp, dumb_imp, dumb_imp)
memimp.import_data([m1.as_dict(), m2.as_dict()])
# ensure that the memberships attached in the right places
assert org.memberships.count() == 2
assert hari.memberships.count() == 1
assert robot.memberships.count() == 1
assert post.memberships.count() == 1
# ensure that the first membership has contact details and links
m = hari.memberships.get()
cd = m.contact_details.get()
assert cd.type == 'phone'
assert cd.value == '555-555-1234'
assert cd.note == 'this is fake'
assert m.links.all()[0].url == 'http://example.com/link'
# update the imported memberships (i.e., change attributes that are not
# in the spec) and confirm they resolve correctly
memimp2 = MembershipImporter('fnd-jid', dumb_imp, dumb_imp, dumb_imp)
m1.end_date = '2022-03-10'
m2.extras = {'note': 'bleep blorp'}
import_log = memimp2.import_data([m1.as_dict(), m2.as_dict()])
assert import_log['membership']['insert'] == 0
assert import_log['membership']['update'] == 2
# confirm the membership resolved based on start date and its end date was updated
assert hari.memberships.count() == 1
assert hari.memberships.get().end_date == '2022-03-10'
# confirm the membership resolved based on end date and its extras were updated
assert robot.memberships.count() == 1
assert robot.memberships.get().extras == {'note': 'bleep blorp'}
