def prepare_object_from_json(self, obj): obj['name'] = fix_bill_id(obj['name']) # obj['organization'] org = self.org_importer._resolve_org_by_chamber(self.jurisdiction_id, obj['organization']) #self.debug("Scraped chamber for %s was `%s'" % (obj['name'], # obj['organization'])) obj['organization'] = org['_id'] if 'alternate_bill_ids' in obj: obj['alternate_bill_ids'] = [fix_bill_id(bid) for bid in obj['alternate_bill_ids']] # XXX: subject categorizer # XXX: action categorizer for rel in obj['related_bills']: rel['bill_id'] = fix_bill_id(rel['bill_id']) spec = rel.copy() spec['jurisdiction_id'] = obj['jurisdiction_id'] rel_obj = db.bills.find_one(spec) if rel_obj: rel['internal_id'] = rel_obj['_id'] else: self.logger.warning('Unknown related bill: {chamber} ' '{session} {bill_id}'.format(**rel)) return obj
def prepare_for_db(self, data): data['identifier'] = fix_bill_id(data['identifier']) data['legislative_session_id'] = self.get_session_id(data.pop('legislative_session')) if data['from_organization']: data['from_organization_id'] = self.org_importer.resolve_json_id( data.pop('from_organization')) for action in data['actions']: action['organization_id'] = self.org_importer.resolve_json_id( action['organization_id']) for entity in action['related_entities']: if 'organization_id' in entity: entity['organization_id'] = self.org_importer.resolve_json_id( entity['organization_id']) for sponsor in data['sponsorships']: if 'person_id' in sponsor: sponsor['person_id'] = self.person_importer.resolve_json_id( sponsor['person_id']) if 'organization_id' in sponsor: sponsor['organization_id'] = self.org_importer.resolve_json_id( sponsor['organization_id']) return data
def prepare_for_db(self, data): data['identifier'] = fix_bill_id(data['identifier']) data['legislative_session_id'] = self.get_session_id( data.pop('legislative_session')) if data['from_organization']: data['from_organization_id'] = self.org_importer.resolve_json_id( data.pop('from_organization')) for action in data['actions']: action['organization_id'] = self.org_importer.resolve_json_id( action['organization_id']) for entity in action['related_entities']: if 'organization_id' in entity: entity[ 'organization_id'] = self.org_importer.resolve_json_id( entity['organization_id']) for sponsor in data['sponsorships']: if 'person_id' in sponsor: sponsor['person_id'] = self.person_importer.resolve_json_id( sponsor['person_id']) if 'organization_id' in sponsor: sponsor[ 'organization_id'] = self.person_importer.resolve_json_id( sponsor['organization_id']) return data
def prepare_object_from_json(self, obj): obj['bill_id'] = fix_bill_id(obj['bill_id']) if 'alternate_bill_ids' in obj: obj['alternate_bill_ids'] = [fix_bill_id(bid) for bid in obj['alternate_bill_ids']] # XXX: subject categorizer # XXX: action categorizer for rel in obj['related_bills']: rel['bill_id'] = fix_bill_id(rel['bill_id']) spec = rel.copy() spec['jurisdiction_id'] = obj['jurisdiction_id'] rel_obj = db.bills.find_one(spec) if rel_obj: rel['internal_id'] = rel_obj['_id'] else: self.logger.warning('Unknown related bill: {chamber} ' '{session} {bill_id}'.format(**rel)) return obj
def prepare_object_from_json(self, obj): obj['name'] = fix_bill_id(obj['name']) org = self.org_importer._resolve_org_by_chamber(self.jurisdiction_id, obj['organization']) obj['organization'] = org['_id'] if 'alternate_bill_ids' in obj: obj['alternate_bill_ids'] = [fix_bill_id(bid) for bid in obj['alternate_bill_ids']] # XXX: subject categorizer # XXX: action categorizer for rel in obj['related_bills']: rel['bill_id'] = fix_bill_id(rel['bill_id']) spec = rel.copy() spec['jurisdiction_id'] = obj['jurisdiction_id'] rel_obj = db.bills.find_one(spec) if rel_obj: rel['internal_id'] = rel_obj['_id'] else: self.logger.warning('Unknown related bill: {chamber} ' '{session} {bill_id}'.format(**rel)) return obj
def prepare_for_db(self, data): data['identifier'] = fix_bill_id(data['identifier']) data['session'] = JurisdictionSession.objects.get(jurisdiction_id=self.jurisdiction_id, name=data['session']) if data['from_organization']: data['from_organization_id'] = self.org_importer.resolve_json_id( data.pop('from_organization')) for action in data['actions']: action['organization_id'] = self.org_importer.resolve_json_id( action['organization_id']) for entity in action['related_entities']: entity['organization_id'] = self.org_importer.resolve_json_id( entity['organization_id']) return data
def prepare_for_db(self, data): data['legislative_session_id'] = self.get_session_id( data.pop('legislative_session')) data['organization_id'] = self.org_importer.resolve_json_id( data.pop('organization')) bill = data.pop('bill') if bill and bill.startswith('~'): bill = get_pseudo_id(bill) bill['identifier'] = fix_bill_id(bill['identifier']) bill = _make_pseudo_id(**bill) data['bill_id'] = self.bill_importer.resolve_json_id(bill) bill_action = data.pop('bill_action') if bill_action: try: action = BillAction.objects.get( bill_id=data['bill_id'], description=bill_action, date=data['start_date'], organization_id=data['organization_id'], ) if action.id in self.seen_action_ids: self.warning('can not match two VoteEvents to %s: %s', action.id, bill_action) else: data['bill_action_id'] = action.id self.seen_action_ids.add(action.id) except BillAction.DoesNotExist: self.warning('could not match VoteEvent to %s %s %s', bill, bill_action, data['start_date']) except BillAction.MultipleObjectsReturned as e: self.warning('could not match VoteEvent to %s %s %s: %s', bill, bill_action, data['start_date'], e) for vote in data['votes']: vote['voter_id'] = self.person_importer.resolve_json_id( vote['voter_id'], allow_no_match=True) return data
def prepare_for_db(self, data): data['jurisdiction_id'] = self.jurisdiction_id data['location'] = self.get_location(data['location']) data['start_date'] = data['start_date'] data['end_date'] = data.get('end_date', "") for participant in data['participants']: if 'person_id' in participant: participant[ 'person_id'] = self.person_importer.resolve_json_id( participant['person_id'], allow_no_match=True) elif 'organization_id' in participant: participant[ 'organization_id'] = self.org_importer.resolve_json_id( participant['organization_id'], allow_no_match=True) for item in data['agenda']: for entity in item['related_entities']: if 'person_id' in entity: entity['person_id'] = self.person_importer.resolve_json_id( entity['person_id'], allow_no_match=True) elif 'organization_id' in entity: entity[ 'organization_id'] = self.org_importer.resolve_json_id( entity['organization_id'], allow_no_match=True) elif 'bill_id' in entity: bill = get_pseudo_id(entity['bill_id']) bill['identifier'] = fix_bill_id(bill['identifier']) bill = _make_pseudo_id(**bill) entity['bill_id'] = self.bill_importer.resolve_json_id( bill, allow_no_match=True) elif 'vote_event_id' in entity: entity[ 'vote_event_id'] = self.vote_event_importer.resolve_json_id( entity['vote_event_id'], allow_no_match=True) return data