def prepare_object_from_json(self, obj):
obj['name'] = fix_bill_id(obj['name'])
# obj['organization']
org = self.org_importer._resolve_org_by_chamber(self.jurisdiction_id,
obj['organization'])
#self.debug("Scraped chamber for %s was `%s'" % (obj['name'],
# obj['organization']))
obj['organization'] = org['_id']
if 'alternate_bill_ids' in obj:
obj['alternate_bill_ids'] = [fix_bill_id(bid) for bid in
obj['alternate_bill_ids']]
# XXX: subject categorizer
# XXX: action categorizer
for rel in obj['related_bills']:
rel['bill_id'] = fix_bill_id(rel['bill_id'])
spec = rel.copy()
spec['jurisdiction_id'] = obj['jurisdiction_id']
rel_obj = db.bills.find_one(spec)
if rel_obj:
rel['internal_id'] = rel_obj['_id']
else:
self.logger.warning('Unknown related bill: {chamber} '
'{session} {bill_id}'.format(**rel))
return obj
def prepare_for_db(self, data):
data['identifier'] = fix_bill_id(data['identifier'])
data['legislative_session_id'] = self.get_session_id(data.pop('legislative_session'))
if data['from_organization']:
data['from_organization_id'] = self.org_importer.resolve_json_id(
data.pop('from_organization'))
for action in data['actions']:
action['organization_id'] = self.org_importer.resolve_json_id(
action['organization_id'])
for entity in action['related_entities']:
if 'organization_id' in entity:
entity['organization_id'] = self.org_importer.resolve_json_id(
entity['organization_id'])
for sponsor in data['sponsorships']:
if 'person_id' in sponsor:
sponsor['person_id'] = self.person_importer.resolve_json_id(
sponsor['person_id'])
if 'organization_id' in sponsor:
sponsor['organization_id'] = self.org_importer.resolve_json_id(
sponsor['organization_id'])
return data
def prepare_for_db(self, data):
data['identifier'] = fix_bill_id(data['identifier'])
data['legislative_session_id'] = self.get_session_id(
data.pop('legislative_session'))
if data['from_organization']:
data['from_organization_id'] = self.org_importer.resolve_json_id(
data.pop('from_organization'))
for action in data['actions']:
action['organization_id'] = self.org_importer.resolve_json_id(
action['organization_id'])
for entity in action['related_entities']:
if 'organization_id' in entity:
entity[
'organization_id'] = self.org_importer.resolve_json_id(
entity['organization_id'])
for sponsor in data['sponsorships']:
if 'person_id' in sponsor:
sponsor['person_id'] = self.person_importer.resolve_json_id(
sponsor['person_id'])
if 'organization_id' in sponsor:
sponsor[
'organization_id'] = self.person_importer.resolve_json_id(
sponsor['organization_id'])
return data
def prepare_object_from_json(self, obj):
obj['bill_id'] = fix_bill_id(obj['bill_id'])
if 'alternate_bill_ids' in obj:
obj['alternate_bill_ids'] = [fix_bill_id(bid) for bid in
obj['alternate_bill_ids']]
# XXX: subject categorizer
# XXX: action categorizer
for rel in obj['related_bills']:
rel['bill_id'] = fix_bill_id(rel['bill_id'])
spec = rel.copy()
spec['jurisdiction_id'] = obj['jurisdiction_id']
rel_obj = db.bills.find_one(spec)
if rel_obj:
rel['internal_id'] = rel_obj['_id']
else:
self.logger.warning('Unknown related bill: {chamber} '
'{session} {bill_id}'.format(**rel))
return obj
def prepare_object_from_json(self, obj):
obj['name'] = fix_bill_id(obj['name'])
org = self.org_importer._resolve_org_by_chamber(self.jurisdiction_id, obj['organization'])
obj['organization'] = org['_id']
if 'alternate_bill_ids' in obj:
obj['alternate_bill_ids'] = [fix_bill_id(bid) for bid in obj['alternate_bill_ids']]
# XXX: subject categorizer
# XXX: action categorizer
for rel in obj['related_bills']:
rel['bill_id'] = fix_bill_id(rel['bill_id'])
spec = rel.copy()
spec['jurisdiction_id'] = obj['jurisdiction_id']
rel_obj = db.bills.find_one(spec)
if rel_obj:
rel['internal_id'] = rel_obj['_id']
else:
self.logger.warning('Unknown related bill: {chamber} '
'{session} {bill_id}'.format(**rel))
return obj
def prepare_for_db(self, data):
data['identifier'] = fix_bill_id(data['identifier'])
data['session'] = JurisdictionSession.objects.get(jurisdiction_id=self.jurisdiction_id,
name=data['session'])
if data['from_organization']:
data['from_organization_id'] = self.org_importer.resolve_json_id(
data.pop('from_organization'))
for action in data['actions']:
action['organization_id'] = self.org_importer.resolve_json_id(
action['organization_id'])
for entity in action['related_entities']:
entity['organization_id'] = self.org_importer.resolve_json_id(
entity['organization_id'])
return data
def prepare_for_db(self, data):
data['legislative_session_id'] = self.get_session_id(
data.pop('legislative_session'))
data['organization_id'] = self.org_importer.resolve_json_id(
data.pop('organization'))
bill = data.pop('bill')
if bill and bill.startswith('~'):
bill = get_pseudo_id(bill)
bill['identifier'] = fix_bill_id(bill['identifier'])
bill = _make_pseudo_id(**bill)
data['bill_id'] = self.bill_importer.resolve_json_id(bill)
bill_action = data.pop('bill_action')
if bill_action:
try:
action = BillAction.objects.get(
bill_id=data['bill_id'],
description=bill_action,
date=data['start_date'],
organization_id=data['organization_id'],
)
if action.id in self.seen_action_ids:
self.warning('can not match two VoteEvents to %s: %s',
action.id, bill_action)
else:
data['bill_action_id'] = action.id
self.seen_action_ids.add(action.id)
except BillAction.DoesNotExist:
self.warning('could not match VoteEvent to %s %s %s', bill,
bill_action, data['start_date'])
except BillAction.MultipleObjectsReturned as e:
self.warning('could not match VoteEvent to %s %s %s: %s', bill,
bill_action, data['start_date'], e)
for vote in data['votes']:
vote['voter_id'] = self.person_importer.resolve_json_id(
vote['voter_id'], allow_no_match=True)
return data
def prepare_for_db(self, data):
data['jurisdiction_id'] = self.jurisdiction_id
data['location'] = self.get_location(data['location'])
data['start_date'] = data['start_date']
data['end_date'] = data.get('end_date', "")
for participant in data['participants']:
if 'person_id' in participant:
participant[
'person_id'] = self.person_importer.resolve_json_id(
participant['person_id'], allow_no_match=True)
elif 'organization_id' in participant:
participant[
'organization_id'] = self.org_importer.resolve_json_id(
participant['organization_id'], allow_no_match=True)
for item in data['agenda']:
for entity in item['related_entities']:
if 'person_id' in entity:
entity['person_id'] = self.person_importer.resolve_json_id(
entity['person_id'], allow_no_match=True)
elif 'organization_id' in entity:
entity[
'organization_id'] = self.org_importer.resolve_json_id(
entity['organization_id'], allow_no_match=True)
elif 'bill_id' in entity:
bill = get_pseudo_id(entity['bill_id'])
bill['identifier'] = fix_bill_id(bill['identifier'])
bill = _make_pseudo_id(**bill)
entity['bill_id'] = self.bill_importer.resolve_json_id(
bill, allow_no_match=True)
elif 'vote_event_id' in entity:
entity[
'vote_event_id'] = self.vote_event_importer.resolve_json_id(
entity['vote_event_id'], allow_no_match=True)
return data
