def setUp(self): super(EntitiesTestCase, self).setUp() self.rolex = self.create_user(foreign_id='user_3') self.col = Collection() self.col.label = 'Original Collection' self.col.foreign_id = 'test_coll_entities' db.session.add(self.col) self.col_other = Collection() self.col_other.label = 'Other Collection' self.col_other.foreign_id = 'test_coll_entities_other' db.session.add(self.col_other) db.session.flush() self.ent = Entity.save( { 'name': 'Winnie the Pooh', 'jurisdiction_code': 'pa', 'summary': 'a fictional teddy bear created by author A. A. Milne', 'identifiers': [{ 'scheme': 'wikipedia', 'identifier': 'en:Winnie-the-Pooh' }], 'other_names': [{ 'name': u'Puh der Bär' }, { 'name': 'Pooh Bear' }] }, [self.col]) db.session.add(self.ent) db.session.flush() self.other = Entity.save( { 'name': 'Pu der Bär', 'jurisdiction_code': 'de', 'description': 'he is a bear', 'identifiers': [{ 'scheme': 'wikipedia', 'identifier': 'en:Winnie-the-Pooh' }, { 'scheme': 'animals', 'identifier': 'bears.winnie.pooh' }], 'other_names': [{ 'name': u'Puh der Bär' }] }, [self.col_other]) db.session.add(self.other) self.alert = Alert() self.alert.entity = self.other db.session.add(self.alert) db.session.commit()
def setUp(self): super(EntitiesTestCase, self).setUp() self.rolex = self.create_user(foreign_id='user_3') self.col = Collection() self.col.label = 'Original Collection' self.col.foreign_id = 'test_coll_entities' db.session.add(self.col) self.col_other = Collection() self.col_other.label = 'Other Collection' self.col_other.foreign_id = 'test_coll_entities_other' db.session.add(self.col_other) db.session.flush() self.ent = Entity.save({ 'name': 'Winnie the Pooh', 'collections': [self.col], 'jurisdiction_code': 'pa', 'summary': 'a fictional teddy bear created by author A. A. Milne', 'identifiers': [{ 'scheme': 'wikipedia', 'identifier': 'en:Winnie-the-Pooh' }], 'other_names': [{ 'name': u'Puh der Bär' }, { 'name': 'Pooh Bear' }] }) db.session.add(self.ent) db.session.flush() self.other = Entity.save({ 'name': 'Pu der Bär', 'collections': [self.col_other], 'jurisdiction_code': 'de', 'description': 'he is a bear', 'identifiers': [{ 'scheme': 'wikipedia', 'identifier': 'en:Winnie-the-Pooh' }, { 'scheme': 'animals', 'identifier': 'bears.winnie.pooh' }], 'other_names': [{ 'name': u'Puh der Bär' }] }) db.session.add(self.other) self.alert = Alert() self.alert.entity = self.other db.session.add(self.alert) db.session.commit()
def emit_entity(self, collection, data): entity = Entity.save(data, [collection], merge=True) db.session.commit() log.info("Entity [%s]: %s", entity.id, entity.name) update_entity(entity) self.increment_count() return entity
def load_entity(self, name, schema): identifier = name.lower().strip() q = db.session.query(EntityIdentifier) q = q.order_by(EntityIdentifier.deleted_at.desc().nullsfirst()) q = q.filter(EntityIdentifier.scheme == self.origin) q = q.filter(EntityIdentifier.identifier == identifier) ident = q.first() if ident is not None: if ident.deleted_at is None: # TODO: add to collections? Security risk here. return ident.entity_id if ident.entity.deleted_at is None: return None data = { 'name': name, '$schema': schema, 'state': Entity.STATE_PENDING, 'identifiers': [{ 'scheme': self.origin, 'identifier': identifier }] } entity = Entity.save(data, self.collections) return entity.id
def load_entity(self, sa_id, name, full): identifier = sa_id log.debug("%s Loading %s %s", self, self.scheme, sa_id) q = db.session.query(EntityIdentifier) q = q.order_by(EntityIdentifier.deleted_at.desc().nullsfirst()) q = q.filter(EntityIdentifier.scheme == self.scheme) q = q.filter(EntityIdentifier.identifier == identifier) ident = q.first() if ident is not None: if ident.deleted_at is None: # TODO: add to collections? Security risk here. return ident.entity_id if ident.entity.deleted_at is None: return None data = { 'name': full, '$schema': DEFAULT_SCHEMA, 'identifiers': [{ 'scheme': self.scheme, 'identifier': identifier }], 'other_names': [ { 'name': name }, { 'name': sa_id }, ], 'company_number': sa_id } log.debug("%s Saving %s", self, full) entity = Entity.save(data, self.collections) return entity.id
def update(id): entity = obj_or_404(Entity.by_id(id)) entity = Entity.save(get_data(entity=entity), collection_id=entity.collection_id, merge=arg_bool('merge')) db.session.commit() analyze_entity.delay(entity.id) return view(entity.id)
def emit_entity(self, collection, data): data['collections'] = [collection] entity = Entity.save(data, merge=True) db.session.flush() update_entity_full.delay(entity.id) log.info("Entity [%s]: %s", entity.id, entity.name) self.entity_cache[collection.id].append(entity) return entity
def create(): data = request_data() data.pop('id', None) data['collections'] = get_collections(data) for collection in data['collections']: authz.require(authz.collection_write(collection.id)) entity = Entity.save(data) db.session.commit() update_entity(entity) return view(entity.id)
def crawl_collection(self, collection): if not len(collection.get('subjects', [])): return url = urljoin(self.URL, '/api/collections/%s' % collection.get('id')) collection = Collection.by_foreign_id(url, { 'label': collection.get('title') }) res = requests.get('%s/permissions' % url, headers=self.HEADERS) for perm in res.json().get('results', []): Permission.grant_foreign(collection, perm.get('role'), perm.get('read'), perm.get('write')) log.info(" > Spindle collection: %s", collection.label) res = requests.get('%s/entities' % url, headers=self.HEADERS) terms = set() existing_entities = [] for entity in res.json().get('results', []): if entity.get('name') is None: continue entity['$schema'] = SCHEMATA.get(entity.get('$schema'), OTHER) if 'jurisdiction_code' in entity: entity['jurisdiction_code'] = \ entity['jurisdiction_code'].lower() entity.pop('members', None) entity.pop('memberships', None) entity.pop('assets', None) entity.pop('owners', None) entity.pop('family_first', None) entity.pop('family_second', None) entity.pop('social_first', None) entity.pop('social_second', None) for date_field in ['birth_date']: if date_field in entity and 'T' in entity[date_field]: entity[date_field], _ = entity[date_field].split('T', 1) for on in entity.get('other_names', []): name = on.pop('alias', None) if name is not None: on['name'] = name entity['identifiers'] = [{ 'scheme': 'spindle', 'identifier': entity.pop('id', None) }] ent = Entity.save(entity, collection_id=collection.id, merge=True) db.session.flush() terms.update(ent.terms) existing_entities.append(ent.id) log.info(" # %s", ent.name) for entity in collection.entities: if entity.id not in existing_entities: entity.delete() self.emit_collection(collection, terms)
def load_entity(self, fk, name, schema): entity = Entity.by_foreign_id(fk, self.collection.id, deleted=True) if entity is not None: return entity return Entity.save({ 'name': name, 'schema': schema, 'foreign_ids': [fk], 'state': Entity.STATE_PENDING, 'data': {} }, self.collection)
def update(id): entity = obj_or_404(Entity.by_id(id)) check_authz(entity, authz.WRITE) data = request_data() data['id'] = entity.id possible_collections = authz.collections(authz.WRITE) possible_collections.extend([c.id for c in entity.collections]) data['collections'] = [c for c in get_collections(data) if c.id in possible_collections] entity = Entity.save(data, merge=arg_bool('merge')) db.session.commit() update_entity(entity) return view(entity.id)
def setUp(self): super(EntitiesTestCase, self).setUp() self.rolex = self.create_user(foreign_id='user_3') self.col = Collection() self.col.label = 'Original Collection' self.col.foreign_id = 'test_coll_entities' db.session.add(self.col) self.col_other = Collection() self.col_other.label = 'Other Collection' self.col_other.foreign_id = 'test_coll_entities_other' db.session.add(self.col_other) db.session.flush() self.ent = Entity.save( { 'schema': 'LegalEntity', 'name': 'Winnie the Pooh', 'data': { 'country': 'pa', 'summary': 'a fictional teddy bear created by A. A. Milne', 'alias': [u'Puh der Bär', 'Pooh Bear'] } }, self.col) self.other = Entity.save( { 'schema': 'LegalEntity', 'name': 'Pu der Bär', 'data': { 'country': 'de', 'description': 'he is a bear', 'alias': [u'Puh der Bär'] } }, self.col) self.alert = Alert() self.alert.entity = self.other db.session.add(self.alert) db.session.commit() index_entity(self.ent) index_entity(self.other)
def update(id): entity = obj_or_404(Entity.by_id(id)) check_authz(entity, authz.WRITE) data = request_data() data['id'] = entity.id possible_collections = authz.collections(authz.WRITE) possible_collections.extend([c.id for c in entity.collections]) data['collections'] = [ c for c in get_collections(data) if c.id in possible_collections ] entity = Entity.save(data, merge=arg_bool('merge')) db.session.commit() update_entity(entity) return view(entity.id)
def create(): data = request_data() data.pop('id', None) collections = get_collections(data) for collection in collections: authz.require(authz.collection_write(collection.id)) entity = Entity.save(data, collections) for collection in entity.collections: collection.touch() db.session.commit() log_event(request, entity_id=entity.id) update_entity(entity) return view(entity.id)
def update(id): _, entity = get_entity(id, request.authz.WRITE) try: entity = Entity.save(request_data(), entity.collection, merge=arg_bool('merge')) except (ValueError, TypeError) as ve: raise BadRequest(ve.message) entity.collection.touch() db.session.commit() log_event(request, entity_id=entity.id) update_entity(entity) return view(entity.id)
def setUp(self): super(CollectionsApiTestCase, self).setUp() self.rolex = self.create_user(foreign_id='user_3') self.col = Collection() self.col.label = 'Test Collection' self.col.foreign_id = 'test_coll_entities_api' self.col.category = 'leak' self.col.countries = [] db.session.add(self.col) db.session.flush() self.ent = Entity.save({ 'schema': 'Person', 'name': 'Winnie the Pooh', }, self.col) db.session.add(self.ent) db.session.commit()
def setUp(self): super(EntitiesApiTestCase, self).setUp() self.rolex = self.create_user(foreign_id='user_3') self.col = Collection() self.col.label = 'Test Collection' self.col.foreign_id = 'test_coll_entities_api' db.session.add(self.col) db.session.flush() self.ent = Entity.save({ 'schema': 'LegalEntity', 'name': 'Winnie the Pooh', 'data': { 'country': 'pa', } }, self.col) db.session.commit() index_entity(self.ent)
def create(): data = request_data() data.pop("id", None) collections = get_collections(data) for collection in collections: authz.require(authz.collection_write(collection.id)) try: entity = Entity.save(data, collections) except ValueError as ve: raise BadRequest(ve.message) for collection in entity.collections: collection.touch() db.session.commit() log_event(request, entity_id=entity.id) update_entity(entity) return view(entity.id)
def update(id): entity = obj_or_404(Entity.by_id(id)) check_authz(entity, authz.WRITE) data = request_data() data["id"] = entity.id possible_collections = authz.collections(authz.WRITE) possible_collections.extend([c.id for c in entity.collections]) collections = [c for c in get_collections(data) if c.id in possible_collections] try: entity = Entity.save(data, collections, merge=arg_bool("merge")) except ValueError as ve: raise BadRequest(ve.message) for collection in entity.collections: collection.touch() db.session.commit() log_event(request, entity_id=entity.id) update_entity(entity) return view(entity.id)
def crawl_source(self, source): if source.get('source_id') in IGNORE_SOURCES: return json_file = source.get('data', {}).get('json') url = urljoin(JSON_PATH, json_file) source_name = source.get('source') or source.get('source_id') label = '%s - %s' % (source.get('publisher'), source_name) collection = Collection.by_foreign_id(url, {'label': label}) Permission.grant_foreign(collection, Role.SYSTEM_GUEST, True, False) log.info(" > OpenNames collection: %s", collection.label) terms = set() existing_entities = [] db.session.flush() entities = requests.get(url).json().get('entities', []) for entity in entities: data = { 'identifiers': [{ 'scheme': 'opennames:%s' % source.get('source_id'), 'identifier': entity.get('uid') }], 'other_names': [], 'name': entity.get('name'), '$schema': SCHEMA.get(entity.get('type'), '/entity/entity.json#') } for on in entity.get('other_names', []): on['name'] = on.pop('other_name', None) data['other_names'].append(on) ent = Entity.save(data, collection_id=collection.id, merge=True) db.session.flush() terms.update(ent.terms) existing_entities.append(ent.id) log.info(" # %s", ent.name) for entity in collection.entities: if entity.id not in existing_entities: entity.delete() self.emit_collection(collection, terms)
def crawl_source(self, source): if source.get('source_id') in IGNORE_SOURCES: return json_file = source.get('data', {}).get('json') url = urljoin(JSON_PATH, json_file) source_name = source.get('source') or source.get('source_id') label = '%s - %s' % (source.get('publisher'), source_name) collection = Collection.by_foreign_id(url, { 'label': label }) Permission.grant_foreign(collection, Role.SYSTEM_GUEST, True, False) log.info(" > OpenNames collection: %s", collection.label) terms = set() existing_entities = [] db.session.flush() entities = requests.get(url).json().get('entities', []) for entity in entities: data = { 'identifiers': [{ 'scheme': 'opennames:%s' % source.get('source_id'), 'identifier': entity.get('uid') }], 'other_names': [], 'name': entity.get('name'), '$schema': SCHEMA.get(entity.get('type'), '/entity/entity.json#') } for on in entity.get('other_names', []): on['name'] = on.pop('other_name', None) data['other_names'].append(on) ent = Entity.save(data, collection_id=collection.id, merge=True) db.session.flush() terms.update(ent.terms) existing_entities.append(ent.id) log.info(" # %s", ent.name) for entity in collection.entities: if entity.id not in existing_entities: entity.delete() self.emit_collection(collection, terms)
def create(): data = request_data() collection_id = data.get('collection_id') try: collection_id = int(collection_id) except (ValueError, TypeError) as ve: raise BadRequest("Invalid collection_id") collection = obj_or_404(Collection.by_id(collection_id)) request.authz.require(request.authz.collection_write(collection.id)) try: entity = Entity.save(data, collection) except (ValueError, TypeError) as ve: raise BadRequest(ve.message) entity.collection.touch() db.session.commit() log_event(request, entity_id=entity.id) update_entity(entity) return view(entity.id)
def load_entity(self, name, schema): q = db.session.query(EntityIdentifier) q = q.order_by(EntityIdentifier.deleted_at.desc().nullsfirst()) q = q.filter(EntityIdentifier.scheme == self.origin) q = q.filter(EntityIdentifier.identifier == name) ident = q.first() if ident is not None: if ident.deleted_at is None: return ident.entity_id if ident.entity.deleted_at is None: return None data = { 'name': name, '$schema': schema, 'state': Entity.STATE_PENDING, 'identifiers': [{ 'scheme': self.origin, 'identifier': name }], 'collections': [self.load_collection()] } entity = Entity.save(data) return entity.id
def update_entity(self, entity, collection): category = REQUEST_TYPES.get(entity.get('ticket_type')) if category is None: return data = { 'identifiers': [ { 'schema': 'idashboard', 'identifier': entity.get('id') } ], 'other_names': [], 'name': entity.get('name'), '$schema': category } ent = Entity.by_identifier('idashboard', entity.get('id'), collection.id) if ent is not None: data['id'] = ent.id ent = Entity.save(data, merge=True) db.session.flush() return ent
def update_entity(self, entity, collection): category = REQUEST_TYPES.get(entity.get('ticket_type')) if category is None: return data = { 'identifiers': [{ 'schema': 'idashboard', 'identifier': entity.get('id') }], 'other_names': [], 'name': entity.get('name'), '$schema': category } ent = Entity.by_identifier('idashboard', entity.get('id'), collection.id) if ent is not None: data['id'] = ent.id ent = Entity.save(data, merge=True) db.session.flush() return ent
def create(): data = get_data() entity = Entity.save(data, collection_id=data.get('collection_id')) db.session.commit() analyze_entity.delay(entity.id) return view(entity.id)