def test_load_csv(self):
count = Collection.all().count()
assert 0 == count, count
db_uri = 'file://' + self.get_fixture_path('experts.csv')
os.environ['ALEPH_TEST_BULK_CSV'] = db_uri
yml_path = self.get_fixture_path('experts.yml')
config = load_config_file(yml_path)
bulk_load(config)
coll = Collection.by_foreign_id('experts')
assert coll.category == 'scrape', coll.category
_, headers = self.login(is_admin=True)
self.flush_index()
count = Collection.all().count()
assert 1 == count, count
res = self.client.get('/api/2/entities?q=Greenfield',
headers=headers)
assert res.status_code == 200, res
assert res.json['total'] == 1, res.json
res0 = res.json['results'][0]
assert res0['id'] == '6897ef1acd633c229d812c1c495f030d212c9081', res0
def test_load_sqlite(self):
count = Collection.all().count()
assert 0 == count, count
db_uri = 'sqlite:///' + self.get_fixture_path('kek.sqlite')
os.environ['ALEPH_TEST_BULK_DATABASE_URI'] = db_uri
yml_path = self.get_fixture_path('kek.yml')
config = load_config_file(yml_path)
bulk_load(config)
count = Collection.all().count()
assert 1 == count, count
coll = Collection.by_foreign_id('kek')
assert coll.category == 'scrape', coll.category
_, headers = self.login(is_admin=True)
flush_index()
res = self.client.get('/api/2/entities?q=friede+springer',
headers=headers)
assert res.status_code == 200, res
assert res.json['total'] == 1, res.json
res0 = res.json['results'][0]
assert res0['id'] == '9895ccc1b3d6444ccc6371ae239a7d55c748a714', res0
def test_entity_references(self):
db_uri = 'file://' + self.get_fixture_path('experts.csv')
os.environ['ALEPH_TEST_BULK_CSV'] = db_uri
yml_path = self.get_fixture_path('experts.yml')
config = load_config_file(yml_path)
bulk_load(config)
flush_index()
res = self.client.get('/api/2/entities?q=Climate')
assert res.json['total'] == 1, res.json
grp_id = res.json['results'][0]['id']
res = self.client.get('/api/2/entities/%s/references' % grp_id)
results = res.json['results']
assert len(results) == 1, results
assert results[0]['count'] == 3, results
def test_load_sqlite(self):
count = Collection.all().count()
assert 0 == count, count
yml_path = self.get_fixture_path('kek.yml')
config = load_config_file(yml_path)
bulk_load(config)
flush_index()
count = Collection.all().count()
assert 1 == count, count
res = self.client.get('/api/2/entities?q=friede+springer')
assert res.status_code == 200, res
assert res.json['total'] == 1, res.json
res0 = res.json['results'][0]
assert res0['id'] == '9895ccc1b3d6444ccc6371ae239a7d55c748a714', res0
def test_entity_references(self):
db_uri = 'file://' + self.get_fixture_path('experts.csv')
os.environ['ALEPH_TEST_BULK_CSV'] = db_uri
yml_path = self.get_fixture_path('experts.yml')
config = load_config_file(yml_path)
bulk_load(config)
_, headers = self.login(is_admin=True)
query = '/api/2/entities?filter:schemata=Thing&q=Climate'
res = self.client.get(query, headers=headers)
assert res.json['total'] == 1, res.json
grp_id = res.json['results'][0]['id']
res = self.client.get('/api/2/entities/%s/references' % grp_id,
headers=headers)
results = res.json['results']
assert len(results) == 1, results
assert results[0]['count'] == 3, results
def test_entity_references(self):
db_uri = 'file://' + self.get_fixture_path('experts.csv')
os.environ['ALEPH_TEST_BULK_CSV'] = db_uri
yml_path = self.get_fixture_path('experts.yml')
config = load_config_file(yml_path)
bulk_load(config)
_, headers = self.login(is_admin=True)
query = '/api/2/entities?filter:schemata=Thing&q=Climate'
res = self.client.get(query, headers=headers)
assert res.json['total'] == 1, res.json
grp_id = res.json['results'][0]['id']
res = self.client.get('/api/2/entities/%s/references' % grp_id,
headers=headers)
results = res.json['results']
assert len(results) == 1, results
assert results[0]['count'] == 3, results
def test_load_csv(self):
count = Collection.all().count()
assert 0 == count, count
db_uri = 'file://' + self.get_fixture_path('experts.csv')
os.environ['ALEPH_TEST_BULK_CSV'] = db_uri
yml_path = self.get_fixture_path('experts.yml')
config = load_config_file(yml_path)
bulk_load(config)
coll = Collection.by_foreign_id('experts')
assert coll.category == 'scrape', coll.category
_, headers = self.login(is_admin=True)
count = Collection.all().count()
assert 1 == count, count
url = '/api/2/entities?filter:schemata=Thing&q=Greenfield'
res = self.client.get(url, headers=headers)
assert res.status_code == 200, res
assert res.json['total'] == 1, res.json
def test_load_csv(self):
count = Collection.all().count()
assert 0 == count, count
db_uri = 'file://' + self.get_fixture_path('experts.csv')
os.environ['ALEPH_TEST_BULK_CSV'] = db_uri
yml_path = self.get_fixture_path('experts.yml')
config = load_config_file(yml_path)
bulk_load(config)
coll = Collection.by_foreign_id('experts')
assert coll.category == 'scrape', coll.category
_, headers = self.login(is_admin=True)
count = Collection.all().count()
assert 1 == count, count
url = '/api/2/entities?filter:schemata=Thing&q=Greenfield'
res = self.client.get(url, headers=headers)
assert res.status_code == 200, res
assert res.json['total'] == 1, res.json
def test_load_sqlite(self):
count = Collection.all().count()
assert 0 == count, count
db_uri = 'sqlite:///' + self.get_fixture_path('kek.sqlite')
os.environ['ALEPH_TEST_BULK_DATABASE_URI'] = db_uri
yml_path = self.get_fixture_path('kek.yml')
config = load_config_file(yml_path)
bulk_load(config)
count = Collection.all().count()
assert 1 == count, count
coll = Collection.by_foreign_id('kek')
assert coll.category == 'scrape', coll.category
_, headers = self.login(is_admin=True)
url = '/api/2/entities?filter:schemata=Thing&q=friede+springer'
res = self.client.get(url, headers=headers)
assert res.status_code == 200, res
assert res.json['total'] == 1, res.json
res0 = res.json['results'][0]
key = '9895ccc1b3d6444ccc6371ae239a7d55c748a714'
assert res0['id'].startswith(key), res0
def bulkload(file_name):
"""Index all the entities in a given dataset."""
log.info("Loading bulk data from: %s", file_name)
config = load_config_file(file_name)
bulk_load(config)
