def do_import(juris, args):
# import inside here because to avoid loading Django code unnecessarily
from openstates.importers import (
JurisdictionImporter,
BillImporter,
VoteEventImporter,
)
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
bill_importer = BillImporter(juris.jurisdiction_id)
vote_event_importer = VoteEventImporter(juris.jurisdiction_id,
bill_importer)
report = {}
with transaction.atomic():
print("import jurisdictions...")
report.update(juris_importer.import_directory(datadir))
if settings.ENABLE_BILLS:
print("import bills...")
report.update(bill_importer.import_directory(datadir))
if settings.ENABLE_VOTES:
print("import vote events...")
report.update(vote_event_importer.import_directory(datadir))
# compile info on all sessions that were updated in this run
seen_sessions = set()
seen_sessions.update(bill_importer.get_seen_sessions())
seen_sessions.update(vote_event_importer.get_seen_sessions())
for session in seen_sessions:
generate_session_report(session)
return report
def test_jurisdiction_merge_related():
Division.objects.create(id="ocd-division/country:us", name="USA")
# need to ensure legislative_sessions don't get deleted
ji = JurisdictionImporter("jurisdiction-id")
tj = FakeJurisdiction()
ji.import_item(tj.as_dict())
assert LegislativeSession.objects.count() == 2
# disallow deletion of legislative sessions as it can remove bills
tj.legislative_sessions.pop()
ji.import_item(tj.as_dict())
# should still have two
assert LegislativeSession.objects.count() == 2
# now will have three
tj.legislative_sessions.append({"identifier": "2017", "name": "2017 Session"})
ji.import_item(tj.as_dict())
assert LegislativeSession.objects.count() == 3
# and test that the non-identifier fields actually update
tj.legislative_sessions.append({"identifier": "2016", "name": "updated"})
ji.import_item(tj.as_dict())
assert LegislativeSession.objects.count() == 3
assert LegislativeSession.objects.get(identifier="2016").name == "updated"
def test_jurisdiction_update():
Division.objects.create(id="ocd-division/country:us", name="USA")
tj = FakeJurisdiction()
ji = JurisdictionImporter("jurisdiction-id")
_, what = ji.import_item(tj.as_dict())
assert what == "insert"
_, what = ji.import_item(tj.as_dict())
assert what == "noop"
assert Jurisdiction.objects.count() == 1
tj.name = "different name"
obj, what = ji.import_item(tj.as_dict())
assert what == "update"
assert Jurisdiction.objects.count() == 1
assert Jurisdiction.objects.get().name == "different name"
def test_jurisdiction_update():
Division.objects.create(id="ocd-division/country:us/state:nj", name="NJ")
tj = NewJersey()
ji = JurisdictionImporter("jurisdiction-id")
_, what = ji.import_item(tj.as_dict())
assert what == "insert"
_, what = ji.import_item(tj.as_dict())
assert what == "noop"
assert Jurisdiction.objects.count() == 1
tj.extras = {"something": "here"}
obj, what = ji.import_item(tj.as_dict())
assert what == "update"
assert Jurisdiction.objects.count() == 1
assert Jurisdiction.objects.get().extras == {"something": "here"}
def test_jurisdiction_import():
Division.objects.create(id="ocd-division/country:us", name="USA")
tj = FakeJurisdiction()
juris_dict = tj.as_dict()
JurisdictionImporter("jurisdiction-id").import_data([juris_dict])
dbj = Jurisdiction.objects.get()
assert dbj.id == tj.jurisdiction_id
assert dbj.division_id == tj.division_id
assert dbj.name == tj.name
assert dbj.url == tj.url
def do_import(juris: State, args: argparse.Namespace) -> dict[str, typing.Any]:
# import inside here because to avoid loading Django code unnecessarily
from openstates.importers import (
JurisdictionImporter,
BillImporter,
VoteEventImporter,
EventImporter,
)
from openstates.data.models import Jurisdiction as DatabaseJurisdiction
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
bill_importer = BillImporter(juris.jurisdiction_id)
vote_event_importer = VoteEventImporter(juris.jurisdiction_id,
bill_importer)
event_importer = EventImporter(juris.jurisdiction_id, vote_event_importer)
report = {}
with transaction.atomic():
print("import jurisdictions...")
report.update(juris_importer.import_directory(datadir))
print("import bills...")
report.update(bill_importer.import_directory(datadir))
print("import vote events...")
report.update(vote_event_importer.import_directory(datadir))
print("import events...")
report.update(event_importer.import_directory(datadir))
DatabaseJurisdiction.objects.filter(id=juris.jurisdiction_id).update(
latest_bill_update=datetime.datetime.utcnow())
# compile info on all sessions that were updated in this run
seen_sessions = set()
seen_sessions.update(bill_importer.get_seen_sessions())
seen_sessions.update(vote_event_importer.get_seen_sessions())
for session in seen_sessions:
generate_session_report(session)
return report
def do_import(juris, args):
# import inside here because to avoid loading Django code unnecessarily
from openstates.importers import (
JurisdictionImporter,
OrganizationImporter,
PersonImporter,
PostImporter,
MembershipImporter,
BillImporter,
VoteEventImporter,
EventImporter,
)
datadir = os.path.join(settings.SCRAPED_DATA_DIR, args.module)
juris_importer = JurisdictionImporter(juris.jurisdiction_id)
org_importer = OrganizationImporter(juris.jurisdiction_id)
person_importer = PersonImporter(juris.jurisdiction_id)
post_importer = PostImporter(juris.jurisdiction_id, org_importer)
membership_importer = MembershipImporter(
juris.jurisdiction_id, person_importer, org_importer, post_importer
)
bill_importer = BillImporter(juris.jurisdiction_id, org_importer, person_importer)
vote_event_importer = VoteEventImporter(
juris.jurisdiction_id, person_importer, org_importer, bill_importer
)
event_importer = EventImporter(
juris.jurisdiction_id,
org_importer,
person_importer,
bill_importer,
vote_event_importer,
)
report = {}
with transaction.atomic():
print("import jurisdictions...")
report.update(juris_importer.import_directory(datadir))
if settings.ENABLE_PEOPLE_AND_ORGS:
print("import organizations...")
report.update(org_importer.import_directory(datadir))
print("import people...")
report.update(person_importer.import_directory(datadir))
print("import posts...")
report.update(post_importer.import_directory(datadir))
print("import memberships...")
report.update(membership_importer.import_directory(datadir))
if settings.ENABLE_BILLS:
print("import bills...")
report.update(bill_importer.import_directory(datadir))
if settings.ENABLE_EVENTS:
print("import events...")
report.update(event_importer.import_directory(datadir))
if settings.ENABLE_VOTES:
print("import vote events...")
report.update(vote_event_importer.import_directory(datadir))
# compile info on all sessions that were updated in this run
seen_sessions = set()
seen_sessions.update(bill_importer.get_seen_sessions())
seen_sessions.update(vote_event_importer.get_seen_sessions())
for session in seen_sessions:
generate_session_report(session)
return report