def _test_converts_schema_to_resource_dict(self): DatasetFactory._meta.sqlalchemy_session = self.sqlite_db.session FileFactory._meta.sqlalchemy_session = self.sqlite_db.session ds1 = DatasetFactory() partition1 = PartitionFactory(dataset=ds1) self.sqlite_db.commit() partition1._datafile = MagicMock() ret = _convert_partition(partition1) self.assertIn('package_id', ret) self.assertEqual(ret['package_id'], ds1.vid) self.assertEqual(ret['name'], partition1.name)
def test_search_by(self): """ search by `source example.com by county` (granularity search) """ DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session TableFactory._meta.sqlalchemy_session = self._my_library.database.session PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session dataset = DatasetFactory(source='example.com') table = TableFactory(dataset=dataset, description='table2', name='table2') partition = PartitionFactory(dataset=dataset, table=table, grain_coverage=['county']) self._my_library.database.commit() self._my_library.search.index_dataset(dataset) self._my_library.search.index_partition(partition) # find partition in the partition index. self._assert_finds_partition(partition, 'by county') # finds dataset extended with partition found = list( self._my_library.search.search('source example.com by county')) self.assertEqual(len(found), 1) self.assertEqual(len(found[0].partitions), 1) self.assertIn(partition.vid, found[0].partitions)
def test_range_and_in(self): """ search by `table2 from 1978 to 1979 in california` (geographic bounds and temporal bounds) """ DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session TableFactory._meta.sqlalchemy_session = self._my_library.database.session PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session dataset = DatasetFactory() table = TableFactory(dataset=dataset, description='table2', name='table2') partition = PartitionFactory(dataset=dataset, table=table, time=1, grain_coverage=['county'], space_coverage=['california'], time_coverage=['1978', '1979']) self._my_library.database.commit() self._my_library.search.index_dataset(dataset) self._my_library.search.index_partition(partition) # finds dataset extended with partition found = list( self._my_library.search.search( 'table2 from 1978 to 1979 in california')) self.assertEqual(len(found), 1) self.assertEqual(len(found[0].partitions), 1) self.assertIn(partition.vid, found[0].partitions)
def test_search_with(self): """ search by `* with cucumber` """ DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session TableFactory._meta.sqlalchemy_session = self._my_library.database.session dataset = DatasetFactory() table = TableFactory(dataset=dataset, name='table2', description='table2') partition = PartitionFactory(dataset=dataset, table=table, time=1) self._my_library.database.commit() partition.table.add_column('id') partition.table.add_column('column1', description='cucumber') self._my_library.database.commit() self._my_library.search.index_dataset(dataset) self._my_library.search.index_partition(partition) # find partition in the partition index. self._assert_finds_partition(partition, 'dataset with cucumber') # finds dataset extended with partition found = list(self._my_library.search.search('dataset with cucumber')) self.assertEqual(len(found), 1) self.assertEqual(len(found[0].partitions), 1) self.assertIn(partition.vid, found[0].partitions)
def test_search_in(self): """ search by `source example.com in California` (geographic bounds) """ DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session TableFactory._meta.sqlalchemy_session = self._my_library.database.session dataset = DatasetFactory() table = TableFactory(dataset=dataset, name='table2', description='table2') partition = PartitionFactory(dataset=dataset, table=table, time=1, space_coverage=['california']) self._my_library.search.index_dataset(dataset) self._my_library.search.index_partition(partition) # find partition in the partition index. self._assert_finds_partition(partition, 'in California') # finds dataset extended with partition found = list( self._my_library.search.search('source example.com in California')) self.assertEqual(len(found), 1) self.assertEqual(len(found[0].partitions), 1) self.assertIn(partition.vid, found[0].partitions)
def test_search_years_range(self): """ search by `source example.com from 1978 to 1979` (temporal bounds) """ DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session TableFactory._meta.sqlalchemy_session = self._my_library.database.session dataset = DatasetFactory() table = TableFactory(dataset=dataset, name='table2', description='table2') partition = PartitionFactory(dataset=dataset, table=table, time=1, time_coverage=['1978', '1979']) self._my_library.database.commit() self._my_library.search.index_partition(partition) self._my_library.search.index_dataset(dataset) # find partition in the partition index. self._assert_finds_partition(partition, 'from 1978 to 1979') # find dataset extended with partition found = list( self._my_library.search.search( 'source example.com from 1978 to 1979')) self.assertEqual(len(found), 1) self.assertEqual(len(found[0].partitions), 1) self.assertIn(partition.vid, found[0].partitions)
def test_returns_true_if_partition_is_indexed(self): PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session dataset = self.new_db_dataset(self._my_library.database, n=0) partition = PartitionFactory(dataset=dataset) self.backend.partition_index.index_one(partition) ret = self.backend.partition_index.is_indexed(partition) self.assertTrue(ret)
def test_creates_partition_table(self): DatasetFactory._meta.sqlalchemy_session = self.db.session PartitionFactory._meta.sqlalchemy_session = self.db.session ds1 = DatasetFactory() PartitionFactory(dataset=ds1) self.db.commit()
def test_returns_true_if_partition_is_already_indexed(self): PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session partition1 = PartitionFactory() self.my_library.database.session.commit() self.backend.partition_index.index_one(partition1) self.assertTrue(self.backend.partition_index.is_indexed(partition1))
def test_search_partition_by_vname(self): DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session dataset = DatasetFactory() partition = PartitionFactory(dataset=dataset) self._my_library.database.session.commit() self._my_library.search.index_partition(partition) self._assert_finds_partition(partition, str(partition.identity.vname))
def test_returns_found_partitions(self): dataset = self.new_db_dataset(self._my_library.database, n=0) PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session partition = PartitionFactory(dataset=dataset) self.backend.partition_index.index_one(partition) # testing. ret = list(self.backend.partition_index.search(partition.vid)) self.assertEqual(len(ret), 1) self.assertListEqual([partition.vid], [x.vid for x in ret])
def test_add_partition_to_the_index(self): DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session dataset = DatasetFactory() partition = PartitionFactory(dataset=dataset) self._my_library.database.session.commit() self._my_library.search.index_partition(partition) partitions = self._my_library.search.backend.partition_index.all() all_vids = [x.vid for x in partitions] self.assertIn(partition.vid, all_vids)
def test_adds_partition_document_to_the_index(self): PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session partition1 = PartitionFactory() self.my_library.database.session.commit() self.backend.partition_index.index_one(partition1) # search just added document. all_docs = list(self.backend.partition_index.index.searcher().documents()) all_vids = [x['vid'] for x in all_docs] self.assertIn(partition1.vid, all_vids)
def test_returns_found_partition(self): # add dataset to backend. PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session partition1 = PartitionFactory() self.my_library.database.session.commit() self.backend.partition_index.index_one(partition1) # search just added document. found = list(self.backend.partition_index.search(partition1.vid)) all_vids = [x.vid for x in found] self.assertIn(partition1.vid, all_vids) self.assertIsInstance(found[0], PartitionSearchResult)
def test_adds_partition_document_to_the_index(self): PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session partition1 = PartitionFactory() self.my_library.database.session.commit() self.backend.partition_index.index_one(partition1) # search just added document. query = """ SELECT vid FROM partition_index; """ result = self.backend.library.database.connection.execute(query).fetchall() self.assertEqual(result[0][0], partition1.vid)
def test_deletes_partition_from_index(self): PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session partition1 = PartitionFactory() self.my_library.database.session.commit() self.backend.partition_index.index_one(partition1) # search just added document. all_docs = list(self.backend.partition_index.index.searcher().documents()) self.assertIn(partition1.vid, [x['vid'] for x in all_docs]) self.backend.partition_index._delete(vid=partition1.vid) all_docs = list(self.backend.partition_index.index.searcher().documents()) self.assertNotIn(partition1.vid, [x['vid'] for x in all_docs])
def test_deletes_given_partition_from_index(self): PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session dataset = self.new_db_dataset(self._my_library.database, n=0) partition = PartitionFactory(dataset=dataset) self.backend.partition_index.index_one(partition) # was it really added? ret = self.backend.partition_index.all() self.assertEqual(len(ret), 1) # delete and test self.backend.partition_index._delete(vid=partition.vid) ret = self.backend.partition_index.all() self.assertEqual(len(ret), 0)
def test_search_partition_by_name(self): DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session TableFactory._meta.sqlalchemy_session = self._my_library.database.session dataset = DatasetFactory() table = TableFactory(dataset=dataset, name='table2', description='table2') partition = PartitionFactory(dataset=dataset, table=table, time=1, name='Partition1') self._my_library.database.commit() self._my_library.search.index_partition(partition) self._assert_finds_partition(partition, str(partition.identity.name))
def test_replaces_partition_document_in_the_index(self): PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session partition1 = PartitionFactory() self.my_library.database.session.commit() doc = self.backend.partition_index._as_document(partition1) self.backend.partition_index._index_document(doc) # search just added document. all_docs = list(self.backend.partition_index.index.searcher().documents()) all_vids = [x['vid'] for x in all_docs] self.assertIn(partition1.vid, all_vids) # update doc['doc'] = u('updated') self.backend.partition_index._index_document(doc, force=True) all_docs = list(self.backend.partition_index.index.searcher().documents(doc='updated')) self.assertEqual(len(all_docs), 1) self.assertEqual(all_docs[0]['vid'], partition1.vid)
def test_creates_resource_for_each_partition_of_the_bundle(self, fake_convert, fake_call): DatasetFactory._meta.sqlalchemy_session = self.sqlite_db.session PartitionFactory._meta.sqlalchemy_session = self.sqlite_db.session ds1 = DatasetFactory() ds1.config.metadata.about.access = 'public' p1 = PartitionFactory(dataset=ds1) bundle = _get_fake_bundle(ds1, partitions=[p1]) fake_convert.return_value = {'name': p1.name, 'package_id': ds1.vid} export(bundle) # assert call to service was valid. called = False for call in fake_call.mock_calls: _, args, kwargs = call if (args[0] == 'resource_create' and kwargs['data_dict'].get('name') == p1.name and kwargs['data_dict'].get('package_id') == ds1.vid): called = True self.assertTrue(called)
def test_extends_found_dataset_with_partitions(self): # create and index some partitions DatasetFactory._meta.sqlalchemy_session = self.my_library.database.session PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session dataset = DatasetFactory(source='example.com') dataset.config.metadata.about.title = 'Test dataset' partition1 = PartitionFactory(dataset=dataset, vname=dataset.vname) self.backend.dataset_index.index_one(dataset) self.my_library.database.session.commit() self.backend.partition_index.index_one(partition1) # look for just added document. found = self.backend.dataset_index.search('example.com') found_dataset = found[0] assert found_dataset.vid == dataset.vid self.assertEqual(len(found_dataset.partitions), 1) self.assertIn(partition1.vid, found_dataset.partitions)
def test_returns_found_partition(self): # create partition and add it to the index to backend. PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session partition1 = PartitionFactory() self.my_library.database.session.commit() self.backend.partition_index.index_one(partition1) # assert partition is added to index. query = """ SELECT vid FROM partition_index; """ result = self.backend.library.database.connection.execute(query).fetchall() self.assertEqual(result[0][0], partition1.vid) # search and check found result. found = list(self.backend.partition_index.search(partition1.vid)) self.assertIsInstance(found[0], PartitionSearchResult) all_vids = [x.vid for x in found] self.assertIn(partition1.vid, all_vids)
def test_deletes_partition_from_index(self): PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session partition1 = PartitionFactory() self.my_library.database.session.commit() self.backend.partition_index.index_one(partition1) query = text(""" SELECT vid FROM partition_index WHERE vid = :vid; """) result = self.backend.library.database.connection.execute(query, vid=partition1.vid).fetchall() self.assertEqual(result, [(partition1.vid,)]) # deleting self.backend.partition_index._delete(vid=partition1.vid) # assert document is deleted. result = self.backend.library.database.connection.execute(query, vid=partition1.vid).fetchall() self.assertEqual(result, [])
def test_finds_partition_by_versioned_name(self): PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session partition = PartitionFactory() self._assert_finds_by_ref(self.my_library, partition, partition.vname)
def test_returns_false_if_identifier_is_not_indexed(self): PartitionFactory._meta.sqlalchemy_session = self.my_library.database.session partition1 = PartitionFactory() self.my_library.database.session.commit() self.assertFalse(self.backend.partition_index.is_indexed(partition1))