def test_load_sqlite(self):
count = Collection.all().count()
assert 0 == count, count
db_uri = 'sqlite:///' + self.get_fixture_path('kek.sqlite')
os.environ['ALEPH_TEST_BULK_DATABASE_URI'] = db_uri
yml_path = self.get_fixture_path('kek.yml')
config = load_config_file(yml_path)
bulk_load(config)
count = Collection.all().count()
assert 1 == count, count
coll = Collection.by_foreign_id('kek')
assert coll.category == 'scrape', coll.category
_, headers = self.login(is_admin=True)
self.flush_index()
res = self.client.get('/api/2/entities?q=friede+springer',
headers=headers)
assert res.status_code == 200, res
assert res.json['total'] == 1, res.json
res0 = res.json['results'][0]
assert res0['id'] == '9895ccc1b3d6444ccc6371ae239a7d55c748a714', res0
def bulk_load(api, mapping_file):
data = load_config_file(mapping_file)
for foreign_id, config in data.items():
collection = api.load_collection_by_foreign_id(foreign_id, config)
collection_id = collection.get('id')
log.info("Bulk mapping collection ID: %s", collection_id)
api.map_collection(collection_id, data)
def bulk_load(api: AlephAPI, mapping_file: str):
data = load_config_file(mapping_file)
if not isinstance(data, dict):
raise AlephException('mapping_file has to be a json dictionary')
for foreign_id, config in data.items():
collection = api.load_collection_by_foreign_id(foreign_id, config)
collection_id = collection['id']
log.info(f"Bulk mapping collection ID: {collection_id}")
api.map_collection(collection_id, data)
def test_entity_references(self):
db_uri = 'file://' + self.get_fixture_path('experts.csv')
os.environ['ALEPH_TEST_BULK_CSV'] = db_uri
yml_path = self.get_fixture_path('experts.yml')
config = load_config_file(yml_path)
bulk_load(config)
_, headers = self.login(is_admin=True)
res = self.client.get('/api/2/entities?q=Climate', headers=headers)
assert res.json['total'] == 1, res.json
grp_id = res.json['results'][0]['id']
res = self.client.get('/api/2/entities/%s/references' % grp_id,
headers=headers)
results = res.json['results']
assert len(results) == 1, results
assert results[0]['count'] == 3, results
def test_entity_references(self):
db_uri = 'file://' + self.get_fixture_path('experts.csv')
os.environ['ALEPH_TEST_BULK_CSV'] = db_uri
yml_path = self.get_fixture_path('experts.yml')
config = load_config_file(yml_path)
bulk_load(config)
_, headers = self.login(is_admin=True)
query = '/api/2/entities?filter:schemata=Thing&q=Climate'
res = self.client.get(query, headers=headers)
assert res.json['total'] == 1, res.json
grp_id = res.json['results'][0]['id']
res = self.client.get('/api/2/entities/%s/references' % grp_id,
headers=headers)
results = res.json['results']
assert len(results) == 1, results
assert results[0]['count'] == 3, results
def test_load_csv(self):
count = Collection.all().count()
assert 0 == count, count
db_uri = 'file://' + self.get_fixture_path('experts.csv')
os.environ['ALEPH_TEST_BULK_CSV'] = db_uri
yml_path = self.get_fixture_path('experts.yml')
config = load_config_file(yml_path)
bulk_load(config)
coll = Collection.by_foreign_id('experts')
assert coll.category == 'scrape', coll.category
_, headers = self.login(is_admin=True)
count = Collection.all().count()
assert 1 == count, count
res = self.client.get('/api/2/entities?q=Greenfield', headers=headers)
assert res.status_code == 200, res
assert res.json['total'] == 1, res.json
def test_load_csv(self):
count = Collection.all().count()
assert 0 == count, count
db_uri = 'file://' + self.get_fixture_path('experts.csv')
os.environ['ALEPH_TEST_BULK_CSV'] = db_uri
yml_path = self.get_fixture_path('experts.yml')
config = load_config_file(yml_path)
bulk_load(config)
coll = Collection.by_foreign_id('experts')
assert coll.category == 'scrape', coll.category
_, headers = self.login(is_admin=True)
count = Collection.all().count()
assert 1 == count, count
url = '/api/2/entities?filter:schemata=Thing&q=Greenfield'
res = self.client.get(url, headers=headers)
assert res.status_code == 200, res
assert res.json['total'] == 1, res.json
def test_load_sqlite(self):
count = Collection.all().count()
assert 0 == count, count
db_uri = 'sqlite:///' + self.get_fixture_path('kek.sqlite')
os.environ['ALEPH_TEST_BULK_DATABASE_URI'] = db_uri
yml_path = self.get_fixture_path('kek.yml')
config = load_config_file(yml_path)
bulk_load(config)
count = Collection.all().count()
assert 1 == count, count
coll = Collection.by_foreign_id('kek')
assert coll.category == 'scrape', coll.category
_, headers = self.login(is_admin=True)
url = '/api/2/entities?filter:schemata=Thing&q=friede+springer'
res = self.client.get(url, headers=headers)
assert res.status_code == 200, res
assert res.json['total'] == 1, res.json
res0 = res.json['results'][0]
key = '9895ccc1b3d6444ccc6371ae239a7d55c748a714'
assert res0['id'].startswith(key), res0
def bulkload(file_name):
"""Index all the entities in a given dataset."""
log.info("Loading bulk data from: %s", file_name)
config = load_config_file(file_name)
bulk_load(config)
