def test_related_vote_event():
j = create_jurisdiction()
session = j.legislative_sessions.create(name="1900", identifier="1900")
org = Organization.objects.create(id="org-id",
name="House",
classification="lower")
bill = Bill.objects.create(id="bill-1",
identifier="HB 1",
legislative_session=session)
VoteEvent.objects.create(
id="vote-1",
identifier="Roll no. 12",
bill=bill,
legislative_session=session,
organization=org,
)
event1 = ge()
event2 = ge()
for event in [event1, event2]:
item = event.add_agenda_item("Cookies will be served")
item.add_vote_event(vote_event="Roll no. 12")
result = EventImporter("jid", oi, pi, bi,
vei).import_data([event1.as_dict()])
assert result["event"]["insert"] == 1
result = EventImporter("jid", oi, pi, bi,
vei).import_data([event2.as_dict()])
assert result["event"]["noop"] == 1
assert (Event.objects.get(name="America's Birthday").agenda.first().
related_entities.first().vote_event_id == "vote-1")
def test_pupa_identifier_event():
create_jurisdiction()
create_other_jurisdiction()
george = Person.objects.create(id="gw", name="George Washington")
o = Organization.objects.create(name="Merica", jurisdiction_id="jid")
Membership.objects.create(person=george, organization=o)
event = ge()
event.pupa_id = "foo"
result = EventImporter("jid", oi, pi, bi,
vei).import_data([event.as_dict()])
assert result["event"]["insert"] == 1
result = EventImporter("jid", oi, pi, bi,
vei).import_data([event.as_dict()])
assert result["event"]["noop"] == 1
event.name = ("America's Anniversary", )
event.location["name"] = "United States of America"
result = EventImporter("jid", oi, pi, bi,
vei).import_data([event.as_dict()])
assert result["event"]["update"] == 1
event.pupa_id = "bar"
result = EventImporter("jid", oi, pi, bi,
vei).import_data([event.as_dict()])
assert result["event"]["insert"] == 1
result = EventImporter("ojid", oi, pi, bi,
vei).import_data([event.as_dict()])
assert result["event"]["insert"] == 1
def test_related_people_event():
create_jurisdiction()
george = Person.objects.create(id="gw", name="George Washington")
john = Person.objects.create(id="jqp", name="John Q. Public")
o = Organization.objects.create(name="Merica", jurisdiction_id="jid")
Membership.objects.create(person=george, organization=o)
Membership.objects.create(person=john, organization=o)
event1 = ge()
event2 = ge()
for event in [event1, event2]:
item = event.add_agenda_item("Cookies will be served")
item.add_person(person="John Q. Public")
event.add_person("George Washington")
result = EventImporter("jid", oi, pi, bi,
vei).import_data([event1.as_dict()])
assert result["event"]["insert"] == 1
result = EventImporter("jid", oi, pi, bi,
vei).import_data([event2.as_dict()])
assert result["event"]["noop"] == 1
assert (Event.objects.get(
name="America's Birthday").participants.first().person_id == "gw")
assert (Event.objects.get(name="America's Birthday").agenda.first().
related_entities.first().person_id == "jqp")
def test_related_bill_event():
j = create_jurisdiction()
session = j.legislative_sessions.create(name="2014",
identifier="2014",
start_date="2014-01-01",
end_date="2015-01-01")
Organization.objects.create(id="org-id",
name="House",
classification="lower")
Bill.objects.create(id="bill-1",
identifier="HB 101",
legislative_session=session)
event1 = ge()
event2 = ge()
for event in [event1, event2]:
item = event.add_agenda_item("Cookies will be served")
item.add_bill(bill="HB 101")
result = EventImporter("jid", vei).import_data([event1.as_dict()])
assert result["event"]["insert"] == 1
result = EventImporter("jid", vei).import_data([event2.as_dict()])
assert result["event"]["noop"] == 1
assert (Event.objects.get(name="America's Birthday").agenda.first().
related_entities.first().bill_id == "bill-1")
def test_related_committee_event():
j = create_jurisdiction()
j.legislative_sessions.create(name="1900", identifier="1900")
org = Organization.objects.create(id="org-id",
name="House",
classification="lower",
jurisdiction=j)
Organization.objects.create(
id="fiscal",
name="Fiscal Committee",
classification="committee",
parent=org,
jurisdiction=j,
)
event1 = ge()
event2 = ge()
for event in [event1, event2]:
item = event.add_agenda_item("Cookies will be served")
item.add_committee(committee="Fiscal Committee")
result = EventImporter("jid", oi, pi, bi,
vei).import_data([event1.as_dict()])
assert result["event"]["insert"] == 1
result = EventImporter("jid", oi, pi, bi,
vei).import_data([event2.as_dict()])
assert result["event"]["noop"] == 1
assert (Event.objects.get(name="America's Birthday").agenda.first().
related_entities.first().organization_id == "fiscal")
def test_media_document():
create_jurisdiction()
event1 = ge()
event2 = ge()
for event in [event1, event2]:
event.add_document(note="Presentation",
url="http://example.com/presentation.pdf")
result = EventImporter("jid", vei).import_data([event1.as_dict()])
assert result["event"]["insert"] == 1
result = EventImporter("jid", vei).import_data([event2.as_dict()])
assert result["event"]["noop"] == 1
def test_related_bill_event_duplicate_id_use_session():
# need a fresh BillImporter here to avoid caching from other tests
j = create_jurisdiction()
session = j.legislative_sessions.create(name="2010",
identifier="2010",
start_date="2010-01-01",
end_date="2010-01-01")
session2 = j.legislative_sessions.create(name="2014",
identifier="2014",
start_date="2014-01-01",
end_date="2015-01-01")
Organization.objects.create(id="org-id",
name="House",
classification="lower")
Bill.objects.create(id="bill-1",
identifier="HB 101",
legislative_session=session)
Bill.objects.create(id="bill-2",
identifier="HB 101",
legislative_session=session2)
event = ge()
item = event.add_agenda_item("Cookies will be served")
item.add_bill(bill="HB 101")
result = EventImporter("jid", vei).import_data([event.as_dict()])
assert result["event"]["insert"] == 1
assert (Event.objects.get(name="America's Birthday").agenda.first().
related_entities.first().bill_id == "bill-2")
def test_media_event():
create_jurisdiction()
event1 = ge()
event2 = ge()
for event in [event1, event2]:
item = event.add_agenda_item("Cookies will be served")
item.add_media_link(
note="Hello, World",
media_type="application/octet-stream",
url="http://hello.world/foo",
)
result = EventImporter("jid", vei).import_data([event1.as_dict()])
assert result["event"]["insert"] == 1
result = EventImporter("jid", vei).import_data([event2.as_dict()])
assert result["event"]["noop"] == 1
def test_top_level_media_event():
create_jurisdiction()
event1, event2 = ge(), ge()
event1.add_media_link(
"fireworks",
"http://example.com/fireworks.mov",
media_type="application/octet-stream",
)
event2.add_media_link(
"fireworks",
"http://example.com/fireworks.mov",
media_type="application/octet-stream",
)
result = EventImporter("jid", vei).import_data([event1.as_dict()])
assert result["event"]["insert"] == 1
result = EventImporter("jid", vei).import_data([event2.as_dict()])
assert result["event"]["noop"] == 1
def test_full_event():
create_jurisdiction()
george = Person.objects.create(id="gw", name="George Washington")
o = Organization.objects.create(name="Merica", jurisdiction_id="jid")
Membership.objects.create(person=george, organization=o)
event = ge()
result = EventImporter("jid", vei).import_data([event.as_dict()])
assert result["event"]["insert"] == 1
event = ge()
result = EventImporter("jid", vei).import_data([event.as_dict()])
assert result["event"]["noop"] == 1
event = ge()
event.location["name"] = "United States of America"
result = EventImporter("jid", vei).import_data([event.as_dict()])
assert result["event"]["update"] == 1
def test_event_agenda_item():
create_jurisdiction()
event1 = ge()
agenda = event1.add_agenda_item("first item")
agenda["extras"] = {"one": 1, "two": [2]}
result = EventImporter("jid", vei).import_data([event1.as_dict()])
assert result["event"]["insert"] == 1
e = Event.objects.get()
a = e.agenda.all()[0]
assert a.extras == {"one": 1, "two": [2]}
def do_import(juris: State, args: argparse.Namespace) -> dict[str, typing.Any]:
# import inside here because to avoid loading Django code unnecessarily
from openstates.importers import (
JurisdictionImporter,
BillImporter,
VoteEventImporter,
EventImporter,
)
from openstates.data.models import Jurisdiction as DatabaseJurisdiction
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
bill_importer = BillImporter(juris.jurisdiction_id)
vote_event_importer = VoteEventImporter(juris.jurisdiction_id,
bill_importer)
event_importer = EventImporter(juris.jurisdiction_id, vote_event_importer)
report = {}
with transaction.atomic():
print("import jurisdictions...")
report.update(juris_importer.import_directory(datadir))
print("import bills...")
report.update(bill_importer.import_directory(datadir))
print("import vote events...")
report.update(vote_event_importer.import_directory(datadir))
print("import events...")
report.update(event_importer.import_directory(datadir))
DatabaseJurisdiction.objects.filter(id=juris.jurisdiction_id).update(
latest_bill_update=datetime.datetime.utcnow())
# compile info on all sessions that were updated in this run
seen_sessions = set()
seen_sessions.update(bill_importer.get_seen_sessions())
seen_sessions.update(vote_event_importer.get_seen_sessions())
for session in seen_sessions:
generate_session_report(session)
return report
def test_event_soft_deletion():
create_jurisdiction()
event1 = ge()
event2 = ge()
event2.name = "Other Event"
result = EventImporter("jid", vei).import_data(
[event1.as_dict(), event2.as_dict()])
assert result["event"]["insert"] == 2
assert Event.objects.count() == 2
# delete
result = EventImporter("jid", vei).import_data([event1.as_dict()])
assert result["event"]["noop"] == 1
# TODO: assert result["event"]["deleted"] == 1
assert Event.objects.count() == 2
assert Event.objects.get(deleted=True).name == "Other Event"
# undelete
result = EventImporter("jid", vei).import_data(
[event1.as_dict(), event2.as_dict()])
assert result["event"]["update"] == 1
assert result["event"]["noop"] == 1
assert Event.objects.count() == 2
assert Event.objects.filter(deleted=True).count() == 0
def do_import(juris, args):
# import inside here because to avoid loading Django code unnecessarily
from openstates.importers import (
JurisdictionImporter,
OrganizationImporter,
PersonImporter,
PostImporter,
MembershipImporter,
BillImporter,
VoteEventImporter,
EventImporter,
)
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(
juris.jurisdiction_id, person_importer, org_importer, post_importer
)
bill_importer = BillImporter(juris.jurisdiction_id, org_importer, person_importer)
vote_event_importer = VoteEventImporter(
juris.jurisdiction_id, person_importer, org_importer, bill_importer
)
event_importer = EventImporter(
juris.jurisdiction_id,
org_importer,
person_importer,
bill_importer,
vote_event_importer,
)
report = {}
with transaction.atomic():
print("import jurisdictions...")
report.update(juris_importer.import_directory(datadir))
if settings.ENABLE_PEOPLE_AND_ORGS:
print("import organizations...")
report.update(org_importer.import_directory(datadir))
print("import people...")
report.update(person_importer.import_directory(datadir))
print("import posts...")
report.update(post_importer.import_directory(datadir))
print("import memberships...")
report.update(membership_importer.import_directory(datadir))
if settings.ENABLE_BILLS:
print("import bills...")
report.update(bill_importer.import_directory(datadir))
if settings.ENABLE_EVENTS:
print("import events...")
report.update(event_importer.import_directory(datadir))
if settings.ENABLE_VOTES:
print("import vote events...")
report.update(vote_event_importer.import_directory(datadir))
# compile info on all sessions that were updated in this run
seen_sessions = set()
seen_sessions.update(bill_importer.get_seen_sessions())
seen_sessions.update(vote_event_importer.get_seen_sessions())
for session in seen_sessions:
generate_session_report(session)
return report