def _test_converts_schema_to_resource_dict(self):
DatasetFactory._meta.sqlalchemy_session = self.sqlite_db.session
FileFactory._meta.sqlalchemy_session = self.sqlite_db.session
ds1 = DatasetFactory()
partition1 = PartitionFactory(dataset=ds1)
self.sqlite_db.commit()
partition1._datafile = MagicMock()
ret = _convert_partition(partition1)
self.assertIn('package_id', ret)
self.assertEqual(ret['package_id'], ds1.vid)
self.assertEqual(ret['name'], partition1.name)
def test_search_by(self):
""" search by `source example.com by county` (granularity search) """
DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session
TableFactory._meta.sqlalchemy_session = self._my_library.database.session
PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session
dataset = DatasetFactory(source='example.com')
table = TableFactory(dataset=dataset,
description='table2',
name='table2')
partition = PartitionFactory(dataset=dataset,
table=table,
grain_coverage=['county'])
self._my_library.database.commit()
self._my_library.search.index_dataset(dataset)
self._my_library.search.index_partition(partition)
# find partition in the partition index.
self._assert_finds_partition(partition, 'by county')
# finds dataset extended with partition
found = list(
self._my_library.search.search('source example.com by county'))
self.assertEqual(len(found), 1)
self.assertEqual(len(found[0].partitions), 1)
self.assertIn(partition.vid, found[0].partitions)
def test_range_and_in(self):
""" search by `table2 from 1978 to 1979 in california` (geographic bounds and temporal bounds) """
DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session
TableFactory._meta.sqlalchemy_session = self._my_library.database.session
PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session
dataset = DatasetFactory()
table = TableFactory(dataset=dataset,
description='table2',
name='table2')
partition = PartitionFactory(dataset=dataset,
table=table,
time=1,
grain_coverage=['county'],
space_coverage=['california'],
time_coverage=['1978', '1979'])
self._my_library.database.commit()
self._my_library.search.index_dataset(dataset)
self._my_library.search.index_partition(partition)
# finds dataset extended with partition
found = list(
self._my_library.search.search(
'table2 from 1978 to 1979 in california'))
self.assertEqual(len(found), 1)
self.assertEqual(len(found[0].partitions), 1)
self.assertIn(partition.vid, found[0].partitions)
def test_search_with(self):
""" search by `* with cucumber` """
DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session
PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session
TableFactory._meta.sqlalchemy_session = self._my_library.database.session
dataset = DatasetFactory()
table = TableFactory(dataset=dataset,
name='table2',
description='table2')
partition = PartitionFactory(dataset=dataset, table=table, time=1)
self._my_library.database.commit()
partition.table.add_column('id')
partition.table.add_column('column1', description='cucumber')
self._my_library.database.commit()
self._my_library.search.index_dataset(dataset)
self._my_library.search.index_partition(partition)
# find partition in the partition index.
self._assert_finds_partition(partition, 'dataset with cucumber')
# finds dataset extended with partition
found = list(self._my_library.search.search('dataset with cucumber'))
self.assertEqual(len(found), 1)
self.assertEqual(len(found[0].partitions), 1)
self.assertIn(partition.vid, found[0].partitions)
def test_search_in(self):
""" search by `source example.com in California` (geographic bounds) """
DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session
PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session
TableFactory._meta.sqlalchemy_session = self._my_library.database.session
dataset = DatasetFactory()
table = TableFactory(dataset=dataset,
name='table2',
description='table2')
partition = PartitionFactory(dataset=dataset,
table=table,
time=1,
space_coverage=['california'])
self._my_library.search.index_dataset(dataset)
self._my_library.search.index_partition(partition)
# find partition in the partition index.
self._assert_finds_partition(partition, 'in California')
# finds dataset extended with partition
found = list(
self._my_library.search.search('source example.com in California'))
self.assertEqual(len(found), 1)
self.assertEqual(len(found[0].partitions), 1)
self.assertIn(partition.vid, found[0].partitions)
def test_search_years_range(self):
""" search by `source example.com from 1978 to 1979` (temporal bounds) """
DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session
PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session
TableFactory._meta.sqlalchemy_session = self._my_library.database.session
dataset = DatasetFactory()
table = TableFactory(dataset=dataset,
name='table2',
description='table2')
partition = PartitionFactory(dataset=dataset,
table=table,
time=1,
time_coverage=['1978', '1979'])
self._my_library.database.commit()
self._my_library.search.index_partition(partition)
self._my_library.search.index_dataset(dataset)
# find partition in the partition index.
self._assert_finds_partition(partition, 'from 1978 to 1979')
# find dataset extended with partition
found = list(
self._my_library.search.search(
'source example.com from 1978 to 1979'))
self.assertEqual(len(found), 1)
self.assertEqual(len(found[0].partitions), 1)
self.assertIn(partition.vid, found[0].partitions)
def test_returns_true_if_partition_is_indexed(self):
PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session
dataset = self.new_db_dataset(self._my_library.database, n=0)
partition = PartitionFactory(dataset=dataset)
self.backend.partition_index.index_one(partition)
ret = self.backend.partition_index.is_indexed(partition)
self.assertTrue(ret)
def test_creates_partition_table(self):
DatasetFactory._meta.sqlalchemy_session = self.db.session
PartitionFactory._meta.sqlalchemy_session = self.db.session
ds1 = DatasetFactory()
PartitionFactory(dataset=ds1)
self.db.commit()
def test_returns_true_if_partition_is_already_indexed(self):
PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session
partition1 = PartitionFactory()
self.my_library.database.session.commit()
self.backend.partition_index.index_one(partition1)
self.assertTrue(self.backend.partition_index.is_indexed(partition1))
def test_search_partition_by_vname(self):
DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session
PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session
dataset = DatasetFactory()
partition = PartitionFactory(dataset=dataset)
self._my_library.database.session.commit()
self._my_library.search.index_partition(partition)
self._assert_finds_partition(partition, str(partition.identity.vname))
def test_returns_found_partitions(self):
dataset = self.new_db_dataset(self._my_library.database, n=0)
PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session
partition = PartitionFactory(dataset=dataset)
self.backend.partition_index.index_one(partition)
# testing.
ret = list(self.backend.partition_index.search(partition.vid))
self.assertEqual(len(ret), 1)
self.assertListEqual([partition.vid], [x.vid for x in ret])
def test_add_partition_to_the_index(self):
DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session
PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session
dataset = DatasetFactory()
partition = PartitionFactory(dataset=dataset)
self._my_library.database.session.commit()
self._my_library.search.index_partition(partition)
partitions = self._my_library.search.backend.partition_index.all()
all_vids = [x.vid for x in partitions]
self.assertIn(partition.vid, all_vids)
def test_adds_partition_document_to_the_index(self):
PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session
partition1 = PartitionFactory()
self.my_library.database.session.commit()
self.backend.partition_index.index_one(partition1)
# search just added document.
all_docs = list(self.backend.partition_index.index.searcher().documents())
all_vids = [x['vid'] for x in all_docs]
self.assertIn(partition1.vid, all_vids)
def test_returns_found_partition(self):
# add dataset to backend.
PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session
partition1 = PartitionFactory()
self.my_library.database.session.commit()
self.backend.partition_index.index_one(partition1)
# search just added document.
found = list(self.backend.partition_index.search(partition1.vid))
all_vids = [x.vid for x in found]
self.assertIn(partition1.vid, all_vids)
self.assertIsInstance(found[0], PartitionSearchResult)
def test_adds_partition_document_to_the_index(self):
PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session
partition1 = PartitionFactory()
self.my_library.database.session.commit()
self.backend.partition_index.index_one(partition1)
# search just added document.
query = """
SELECT vid
FROM partition_index;
"""
result = self.backend.library.database.connection.execute(query).fetchall()
self.assertEqual(result[0][0], partition1.vid)
def test_deletes_partition_from_index(self):
PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session
partition1 = PartitionFactory()
self.my_library.database.session.commit()
self.backend.partition_index.index_one(partition1)
# search just added document.
all_docs = list(self.backend.partition_index.index.searcher().documents())
self.assertIn(partition1.vid, [x['vid'] for x in all_docs])
self.backend.partition_index._delete(vid=partition1.vid)
all_docs = list(self.backend.partition_index.index.searcher().documents())
self.assertNotIn(partition1.vid, [x['vid'] for x in all_docs])
def test_deletes_given_partition_from_index(self):
PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session
dataset = self.new_db_dataset(self._my_library.database, n=0)
partition = PartitionFactory(dataset=dataset)
self.backend.partition_index.index_one(partition)
# was it really added?
ret = self.backend.partition_index.all()
self.assertEqual(len(ret), 1)
# delete and test
self.backend.partition_index._delete(vid=partition.vid)
ret = self.backend.partition_index.all()
self.assertEqual(len(ret), 0)
def test_search_partition_by_name(self):
DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session
PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session
TableFactory._meta.sqlalchemy_session = self._my_library.database.session
dataset = DatasetFactory()
table = TableFactory(dataset=dataset,
name='table2',
description='table2')
partition = PartitionFactory(dataset=dataset,
table=table,
time=1,
name='Partition1')
self._my_library.database.commit()
self._my_library.search.index_partition(partition)
self._assert_finds_partition(partition, str(partition.identity.name))
def test_replaces_partition_document_in_the_index(self):
PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session
partition1 = PartitionFactory()
self.my_library.database.session.commit()
doc = self.backend.partition_index._as_document(partition1)
self.backend.partition_index._index_document(doc)
# search just added document.
all_docs = list(self.backend.partition_index.index.searcher().documents())
all_vids = [x['vid'] for x in all_docs]
self.assertIn(partition1.vid, all_vids)
# update
doc['doc'] = u('updated')
self.backend.partition_index._index_document(doc, force=True)
all_docs = list(self.backend.partition_index.index.searcher().documents(doc='updated'))
self.assertEqual(len(all_docs), 1)
self.assertEqual(all_docs[0]['vid'], partition1.vid)
def test_creates_resource_for_each_partition_of_the_bundle(self, fake_convert, fake_call):
DatasetFactory._meta.sqlalchemy_session = self.sqlite_db.session
PartitionFactory._meta.sqlalchemy_session = self.sqlite_db.session
ds1 = DatasetFactory()
ds1.config.metadata.about.access = 'public'
p1 = PartitionFactory(dataset=ds1)
bundle = _get_fake_bundle(ds1, partitions=[p1])
fake_convert.return_value = {'name': p1.name, 'package_id': ds1.vid}
export(bundle)
# assert call to service was valid.
called = False
for call in fake_call.mock_calls:
_, args, kwargs = call
if (args[0] == 'resource_create'
and kwargs['data_dict'].get('name') == p1.name
and kwargs['data_dict'].get('package_id') == ds1.vid):
called = True
self.assertTrue(called)
def test_extends_found_dataset_with_partitions(self):
# create and index some partitions
DatasetFactory._meta.sqlalchemy_session = self.my_library.database.session
PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session
dataset = DatasetFactory(source='example.com')
dataset.config.metadata.about.title = 'Test dataset'
partition1 = PartitionFactory(dataset=dataset, vname=dataset.vname)
self.backend.dataset_index.index_one(dataset)
self.my_library.database.session.commit()
self.backend.partition_index.index_one(partition1)
# look for just added document.
found = self.backend.dataset_index.search('example.com')
found_dataset = found[0]
assert found_dataset.vid == dataset.vid
self.assertEqual(len(found_dataset.partitions), 1)
self.assertIn(partition1.vid, found_dataset.partitions)
def test_returns_found_partition(self):
# create partition and add it to the index to backend.
PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session
partition1 = PartitionFactory()
self.my_library.database.session.commit()
self.backend.partition_index.index_one(partition1)
# assert partition is added to index.
query = """
SELECT vid
FROM partition_index;
"""
result = self.backend.library.database.connection.execute(query).fetchall()
self.assertEqual(result[0][0], partition1.vid)
# search and check found result.
found = list(self.backend.partition_index.search(partition1.vid))
self.assertIsInstance(found[0], PartitionSearchResult)
all_vids = [x.vid for x in found]
self.assertIn(partition1.vid, all_vids)
def test_deletes_partition_from_index(self):
PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session
partition1 = PartitionFactory()
self.my_library.database.session.commit()
self.backend.partition_index.index_one(partition1)
query = text("""
SELECT vid
FROM partition_index
WHERE vid = :vid;
""")
result = self.backend.library.database.connection.execute(query, vid=partition1.vid).fetchall()
self.assertEqual(result, [(partition1.vid,)])
# deleting
self.backend.partition_index._delete(vid=partition1.vid)
# assert document is deleted.
result = self.backend.library.database.connection.execute(query, vid=partition1.vid).fetchall()
self.assertEqual(result, [])
def test_finds_partition_by_versioned_name(self):
PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session
partition = PartitionFactory()
self._assert_finds_by_ref(self.my_library, partition, partition.vname)
def test_returns_false_if_identifier_is_not_indexed(self):
PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session
partition1 = PartitionFactory()
self.my_library.database.session.commit()
self.assertFalse(self.backend.partition_index.is_indexed(partition1))
