def test_load_csv(self): count = Collection.all().count() assert 0 == count, count db_uri = 'file://' + self.get_fixture_path('experts.csv') os.environ['ALEPH_TEST_BULK_CSV'] = db_uri yml_path = self.get_fixture_path('experts.yml') config = load_config_file(yml_path) bulk_load(config) coll = Collection.by_foreign_id('experts') assert coll.category == 'scrape', coll.category _, headers = self.login(is_admin=True) self.flush_index() count = Collection.all().count() assert 1 == count, count res = self.client.get('/api/2/entities?q=Greenfield', headers=headers) assert res.status_code == 200, res assert res.json['total'] == 1, res.json res0 = res.json['results'][0] assert res0['id'] == '6897ef1acd633c229d812c1c495f030d212c9081', res0
def test_load_sqlite(self): count = Collection.all().count() assert 0 == count, count db_uri = 'sqlite:///' + self.get_fixture_path('kek.sqlite') os.environ['ALEPH_TEST_BULK_DATABASE_URI'] = db_uri yml_path = self.get_fixture_path('kek.yml') config = load_config_file(yml_path) bulk_load(config) count = Collection.all().count() assert 1 == count, count coll = Collection.by_foreign_id('kek') assert coll.category == 'scrape', coll.category _, headers = self.login(is_admin=True) flush_index() res = self.client.get('/api/2/entities?q=friede+springer', headers=headers) assert res.status_code == 200, res assert res.json['total'] == 1, res.json res0 = res.json['results'][0] assert res0['id'] == '9895ccc1b3d6444ccc6371ae239a7d55c748a714', res0
def test_entity_references(self): db_uri = 'file://' + self.get_fixture_path('experts.csv') os.environ['ALEPH_TEST_BULK_CSV'] = db_uri yml_path = self.get_fixture_path('experts.yml') config = load_config_file(yml_path) bulk_load(config) flush_index() res = self.client.get('/api/2/entities?q=Climate') assert res.json['total'] == 1, res.json grp_id = res.json['results'][0]['id'] res = self.client.get('/api/2/entities/%s/references' % grp_id) results = res.json['results'] assert len(results) == 1, results assert results[0]['count'] == 3, results
def test_load_sqlite(self): count = Collection.all().count() assert 0 == count, count yml_path = self.get_fixture_path('kek.yml') config = load_config_file(yml_path) bulk_load(config) flush_index() count = Collection.all().count() assert 1 == count, count res = self.client.get('/api/2/entities?q=friede+springer') assert res.status_code == 200, res assert res.json['total'] == 1, res.json res0 = res.json['results'][0] assert res0['id'] == '9895ccc1b3d6444ccc6371ae239a7d55c748a714', res0
def test_entity_references(self): db_uri = 'file://' + self.get_fixture_path('experts.csv') os.environ['ALEPH_TEST_BULK_CSV'] = db_uri yml_path = self.get_fixture_path('experts.yml') config = load_config_file(yml_path) bulk_load(config) _, headers = self.login(is_admin=True) query = '/api/2/entities?filter:schemata=Thing&q=Climate' res = self.client.get(query, headers=headers) assert res.json['total'] == 1, res.json grp_id = res.json['results'][0]['id'] res = self.client.get('/api/2/entities/%s/references' % grp_id, headers=headers) results = res.json['results'] assert len(results) == 1, results assert results[0]['count'] == 3, results
def test_entity_references(self): db_uri = 'file://' + self.get_fixture_path('experts.csv') os.environ['ALEPH_TEST_BULK_CSV'] = db_uri yml_path = self.get_fixture_path('experts.yml') config = load_config_file(yml_path) bulk_load(config) _, headers = self.login(is_admin=True) query = '/api/2/entities?filter:schemata=Thing&q=Climate' res = self.client.get(query, headers=headers) assert res.json['total'] == 1, res.json grp_id = res.json['results'][0]['id'] res = self.client.get('/api/2/entities/%s/references' % grp_id, headers=headers) results = res.json['results'] assert len(results) == 1, results assert results[0]['count'] == 3, results
def test_load_csv(self): count = Collection.all().count() assert 0 == count, count db_uri = 'file://' + self.get_fixture_path('experts.csv') os.environ['ALEPH_TEST_BULK_CSV'] = db_uri yml_path = self.get_fixture_path('experts.yml') config = load_config_file(yml_path) bulk_load(config) coll = Collection.by_foreign_id('experts') assert coll.category == 'scrape', coll.category _, headers = self.login(is_admin=True) count = Collection.all().count() assert 1 == count, count url = '/api/2/entities?filter:schemata=Thing&q=Greenfield' res = self.client.get(url, headers=headers) assert res.status_code == 200, res assert res.json['total'] == 1, res.json
def test_load_csv(self): count = Collection.all().count() assert 0 == count, count db_uri = 'file://' + self.get_fixture_path('experts.csv') os.environ['ALEPH_TEST_BULK_CSV'] = db_uri yml_path = self.get_fixture_path('experts.yml') config = load_config_file(yml_path) bulk_load(config) coll = Collection.by_foreign_id('experts') assert coll.category == 'scrape', coll.category _, headers = self.login(is_admin=True) count = Collection.all().count() assert 1 == count, count url = '/api/2/entities?filter:schemata=Thing&q=Greenfield' res = self.client.get(url, headers=headers) assert res.status_code == 200, res assert res.json['total'] == 1, res.json
def test_load_sqlite(self): count = Collection.all().count() assert 0 == count, count db_uri = 'sqlite:///' + self.get_fixture_path('kek.sqlite') os.environ['ALEPH_TEST_BULK_DATABASE_URI'] = db_uri yml_path = self.get_fixture_path('kek.yml') config = load_config_file(yml_path) bulk_load(config) count = Collection.all().count() assert 1 == count, count coll = Collection.by_foreign_id('kek') assert coll.category == 'scrape', coll.category _, headers = self.login(is_admin=True) url = '/api/2/entities?filter:schemata=Thing&q=friede+springer' res = self.client.get(url, headers=headers) assert res.status_code == 200, res assert res.json['total'] == 1, res.json res0 = res.json['results'][0] key = '9895ccc1b3d6444ccc6371ae239a7d55c748a714' assert res0['id'].startswith(key), res0
def bulkload(file_name): """Index all the entities in a given dataset.""" log.info("Loading bulk data from: %s", file_name) config = load_config_file(file_name) bulk_load(config)