Ejemplo n.º 1
0
    def test_returns_list_with_all_datasets(self):
        # prepare state
        DatasetFactory._meta.sqlalchemy_session = self.db.session

        ds1 = DatasetFactory()
        ds2 = DatasetFactory()
        ds3 = DatasetFactory()

        # testing
        ret = self.db.datasets
        self.assertIsInstance(ret, list)
        self.assertEqual(len(ret), 3)
        self.assertIn(ds1, ret)
        self.assertIn(ds2, ret)
        self.assertIn(ds3, ret)
Ejemplo n.º 2
0
    def test_returns_true_if_dataset_is_already_indexed(self):
        DatasetFactory._meta.sqlalchemy_session = self.my_library.database.session
        dataset = DatasetFactory()
        self.backend.dataset_index.index_one(dataset)

        # search just added document.
        self.assertTrue(self.backend.dataset_index.is_indexed(dataset))
Ejemplo n.º 3
0
    def test_identity_fields_values(self):
        """ Test contacts group fields of the metadata config. """
        # Test both - setting and saving to db.
        top = Top()
        dataset = DatasetFactory()
        top.link_config(self.my_library.database.session, dataset)

        top.identity.bspace = 'b-space'
        top.identity.btime = 'b-time'
        top.identity.dataset = dataset.vid
        top.identity.id = dataset.id
        top.identity.revision = 7
        top.identity.source = 'example.com'
        top.identity.subset = 'mortality'
        top.identity.type = '?'
        top.identity.variation = 1
        top.identity.version = '0.0.7'

        # build from db and check
        new_top = Top()
        new_top.build_from_db(dataset)
        self.assertEqual(new_top.identity.bspace, 'b-space')
        self.assertEqual(new_top.identity.btime, 'b-time')
        self.assertEqual(new_top.identity.dataset, dataset.vid)
        self.assertEqual(new_top.identity.id, dataset.id)
        self.assertEqual(new_top.identity.revision, 7)
        self.assertEqual(new_top.identity.source, 'example.com')
        self.assertEqual(new_top.identity.subset, 'mortality')
        self.assertEqual(new_top.identity.type, '?')
        self.assertEqual(new_top.identity.variation, 1)
        self.assertEqual(new_top.identity.version, '0.0.7')
Ejemplo n.º 4
0
    def test_creates_partition_table(self):
        DatasetFactory._meta.sqlalchemy_session = self.db.session
        PartitionFactory._meta.sqlalchemy_session = self.db.session

        ds1 = DatasetFactory()
        PartitionFactory(dataset=ds1)
        self.db.commit()
Ejemplo n.º 5
0
    def test_search_with(self):
        """ search by `* with cucumber` """
        DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session
        PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session
        TableFactory._meta.sqlalchemy_session = self._my_library.database.session

        dataset = DatasetFactory()
        table = TableFactory(dataset=dataset,
                             name='table2',
                             description='table2')
        partition = PartitionFactory(dataset=dataset, table=table, time=1)
        self._my_library.database.commit()
        partition.table.add_column('id')
        partition.table.add_column('column1', description='cucumber')
        self._my_library.database.commit()
        self._my_library.search.index_dataset(dataset)
        self._my_library.search.index_partition(partition)

        # find partition in the partition index.
        self._assert_finds_partition(partition, 'dataset with cucumber')

        # finds dataset extended with partition
        found = list(self._my_library.search.search('dataset with cucumber'))
        self.assertEqual(len(found), 1)
        self.assertEqual(len(found[0].partitions), 1)
        self.assertIn(partition.vid, found[0].partitions)
Ejemplo n.º 6
0
    def test_creates_resource_for_each_external_documentation(self, fake_call):
        DatasetFactory._meta.sqlalchemy_session = self.sqlite_db.session

        ds1 = DatasetFactory()
        ds1.config.metadata.about.access = 'public'

        # create two external documentations.
        #
        site1_descr = 'Descr1'
        site1_url = 'http://example.com/1'
        site2_descr = 'Descr2'
        site2_url = 'http://example.com/2'

        ds1.config.metadata.external_documentation.site1.description = site1_descr
        ds1.config.metadata.external_documentation.site1.url = site1_url

        ds1.config.metadata.external_documentation.site2.description = site2_descr
        ds1.config.metadata.external_documentation.site2.url = site2_url

        bundle = _get_fake_bundle(ds1)
        export(bundle)

        # assert call was valid
        resource_create_calls = {}
        for call in fake_call.mock_calls:
            _, args, kwargs = call
            if args[0] == 'resource_create':
                resource_create_calls[kwargs['data_dict']['name']] = kwargs['data_dict']
        self.assertIn('site1', resource_create_calls)
        self.assertEqual(resource_create_calls['site1']['url'], site1_url)
        self.assertEqual(resource_create_calls['site1']['description'], site1_descr)

        self.assertIn('site2', resource_create_calls)
        self.assertEqual(resource_create_calls['site2']['url'], site2_url)
        self.assertEqual(resource_create_calls['site2']['description'], site2_descr)
Ejemplo n.º 7
0
    def test_dataset_config_operations(self):

        dataset = DatasetFactory()
        dataset.config.metadata.identity.id = 'd02'
        dataset.config.metadata.identity.version = '0.0.1'
        self._my_library.database.commit()

        # Refresh dataset after commit
        dataset = self._my_library.dataset(dataset.vid)

        self.assertEqual(dataset.config.metadata.identity.id, 'd02')
        self.assertEqual(dataset.config.metadata.identity.version, '0.0.1')

        self.assertEqual(10, len(dataset.config.metadata.identity))

        identity_keys = [
            'subset', 'variation', 'dataset', 'btime', 'source', 'version',
            'bspace', 'type', 'id', 'revision'
        ]

        self.assertEqual(sorted(identity_keys),
                         sorted([v for v in dataset.config.metadata.identity]))

        with self.assertRaises(AttributeError):
            dataset.config.metadata = 'foo'

        try:
            dataset.config.metadata.identity = [1, 2, 3]
        except AssertionError as exc:
            self.assertIn('Dictionary is required', str(exc))
Ejemplo n.º 8
0
    def test_range_and_in(self):
        """ search by `table2 from 1978 to 1979 in california` (geographic bounds and temporal bounds) """
        DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session
        TableFactory._meta.sqlalchemy_session = self._my_library.database.session
        PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session

        dataset = DatasetFactory()
        table = TableFactory(dataset=dataset,
                             description='table2',
                             name='table2')
        partition = PartitionFactory(dataset=dataset,
                                     table=table,
                                     time=1,
                                     grain_coverage=['county'],
                                     space_coverage=['california'],
                                     time_coverage=['1978', '1979'])
        self._my_library.database.commit()
        self._my_library.search.index_dataset(dataset)
        self._my_library.search.index_partition(partition)

        # finds dataset extended with partition
        found = list(
            self._my_library.search.search(
                'table2 from 1978 to 1979 in california'))
        self.assertEqual(len(found), 1)
        self.assertEqual(len(found[0].partitions), 1)
        self.assertIn(partition.vid, found[0].partitions)
Ejemplo n.º 9
0
    def test_unique(self):
        """ d_vid, type, group and key are unique together. """
        dataset = DatasetFactory()
        conf1 = Config(sequence_id=1,
                       d_vid=dataset.vid,
                       type='metadata',
                       group='identity',
                       key='key1',
                       value='value1')
        self._my_library.database.session.add(conf1)
        self._my_library.database.session.commit()

        dupe = Config(sequence_id=2,
                      d_vid=dataset.vid,
                      type='metadata',
                      group='identity',
                      key='key1',
                      value='value1')
        self._my_library.database.session.add(dupe)
        try:
            self._my_library.database.session.commit()
            raise AssertionError(
                'Dupe unexpectedly saved. It seems unique constraint is broken.'
            )
        except IntegrityError as exc:
            self.assertIn('UNIQUE constraint failed', str(exc))
Ejemplo n.º 10
0
    def test_search_by(self):
        """ search by `source example.com by county` (granularity search) """

        DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session
        TableFactory._meta.sqlalchemy_session = self._my_library.database.session
        PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session

        dataset = DatasetFactory(source='example.com')
        table = TableFactory(dataset=dataset,
                             description='table2',
                             name='table2')
        partition = PartitionFactory(dataset=dataset,
                                     table=table,
                                     grain_coverage=['county'])

        self._my_library.database.commit()
        self._my_library.search.index_dataset(dataset)
        self._my_library.search.index_partition(partition)

        # find partition in the partition index.
        self._assert_finds_partition(partition, 'by county')

        # finds dataset extended with partition
        found = list(
            self._my_library.search.search('source example.com by county'))
        self.assertEqual(len(found), 1)
        self.assertEqual(len(found[0].partitions), 1)
        self.assertIn(partition.vid, found[0].partitions)
Ejemplo n.º 11
0
    def test_search_in(self):
        """ search by `source example.com in California` (geographic bounds) """
        DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session
        PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session
        TableFactory._meta.sqlalchemy_session = self._my_library.database.session
        dataset = DatasetFactory()
        table = TableFactory(dataset=dataset,
                             name='table2',
                             description='table2')
        partition = PartitionFactory(dataset=dataset,
                                     table=table,
                                     time=1,
                                     space_coverage=['california'])
        self._my_library.search.index_dataset(dataset)
        self._my_library.search.index_partition(partition)

        # find partition in the partition index.
        self._assert_finds_partition(partition, 'in California')

        # finds dataset extended with partition
        found = list(
            self._my_library.search.search('source example.com in California'))
        self.assertEqual(len(found), 1)
        self.assertEqual(len(found[0].partitions), 1)
        self.assertIn(partition.vid, found[0].partitions)
Ejemplo n.º 12
0
    def test_search_years_range(self):
        """ search by `source example.com from 1978 to 1979` (temporal bounds) """
        DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session
        PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session
        TableFactory._meta.sqlalchemy_session = self._my_library.database.session

        dataset = DatasetFactory()
        table = TableFactory(dataset=dataset,
                             name='table2',
                             description='table2')
        partition = PartitionFactory(dataset=dataset,
                                     table=table,
                                     time=1,
                                     time_coverage=['1978', '1979'])
        self._my_library.database.commit()
        self._my_library.search.index_partition(partition)
        self._my_library.search.index_dataset(dataset)

        # find partition in the partition index.
        self._assert_finds_partition(partition, 'from 1978 to 1979')

        # find dataset extended with partition
        found = list(
            self._my_library.search.search(
                'source example.com from 1978 to 1979'))
        self.assertEqual(len(found), 1)
        self.assertEqual(len(found[0].partitions), 1)
        self.assertIn(partition.vid, found[0].partitions)
Ejemplo n.º 13
0
    def test_adds_dataset_document_to_the_index(self):
        DatasetFactory._meta.sqlalchemy_session = self.my_library.database.session
        dataset = DatasetFactory()
        self.backend.dataset_index.index_one(dataset)

        # search just added document.
        all_docs = list(self.backend.dataset_index.index.searcher().documents())
        self.assertEqual(all_docs[0]['vid'], dataset.vid)
Ejemplo n.º 14
0
    def test_search_dataset_by_vid(self):
        DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session
        dataset = DatasetFactory()
        self._my_library.search.index_dataset(dataset)

        found = self._my_library.search.search(dataset.vid)
        all_vids = [x.vid for x in found]
        self.assertIn(dataset.vid, all_vids)
Ejemplo n.º 15
0
    def test_raises_UnpublishedAccessError_error(self, fake_call):
        DatasetFactory._meta.sqlalchemy_session = self.sqlite_db.session

        ds1 = DatasetFactory()
        ds1.config.metadata.about.access = 'restricted'
        bundle = _get_fake_bundle(ds1)
        with self.assertRaises(UnpublishedAccessError):
            export(bundle)
Ejemplo n.º 16
0
 def test_search_partition_by_vname(self):
     DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session
     PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session
     dataset = DatasetFactory()
     partition = PartitionFactory(dataset=dataset)
     self._my_library.database.session.commit()
     self._my_library.search.index_partition(partition)
     self._assert_finds_partition(partition, str(partition.identity.vname))
Ejemplo n.º 17
0
    def test_add_dataset_to_the_index(self):
        DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session
        dataset = DatasetFactory()
        self._my_library.database.session.commit()
        self._my_library.search.index_dataset(dataset)

        datasets = self._my_library.search.backend.dataset_index.all()
        all_vids = [x.vid for x in datasets]
        self.assertIn(dataset.vid, all_vids)
Ejemplo n.º 18
0
    def test_creates_table_table(self):
        DatasetFactory._meta.sqlalchemy_session = self.db.session
        TableFactory._meta.sqlalchemy_session = self.db.session

        # Now all tables are created. Can we use ORM to create datasets?
        ds1 = DatasetFactory()
        self.db.commit()
        TableFactory(dataset=ds1)
        self.db.commit()
Ejemplo n.º 19
0
 def test_add_partition_to_the_index(self):
     DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session
     PartitionFactory._meta.sqlalchemy_session = self._my_library.database.session
     dataset = DatasetFactory()
     partition = PartitionFactory(dataset=dataset)
     self._my_library.database.session.commit()
     self._my_library.search.index_partition(partition)
     partitions = self._my_library.search.backend.partition_index.all()
     all_vids = [x.vid for x in partitions]
     self.assertIn(partition.vid, all_vids)
Ejemplo n.º 20
0
 def test_returns_config_repr(self):
     ds = DatasetFactory()
     self._my_library.database.session.commit()
     config1 = ConfigFactory(d_vid=ds.vid)
     repr_str = config1.__repr__()
     self.assertIsInstance(repr_str, text_type)
     self.assertIn(config1.d_vid, repr_str)
     self.assertIn(config1.group, repr_str)
     self.assertIn(config1.key, repr_str)
     self.assertIn(config1.value, repr_str)
Ejemplo n.º 21
0
    def test_deletes_dataset_from_index(self):
        DatasetFactory._meta.sqlalchemy_session = self.my_library.database.session
        dataset = DatasetFactory()
        self.backend.dataset_index.index_one(dataset)

        # look for just added document.
        all_docs = list(self.backend.dataset_index.index.searcher().documents())
        self.assertIn(dataset.vid, [x['vid'] for x in all_docs])
        self.backend.dataset_index._delete(vid=dataset.vid)
        all_docs = list(self.backend.dataset_index.index.searcher().documents())
        self.assertNotIn(dataset.vid, [x['vid'] for x in all_docs])
Ejemplo n.º 22
0
    def test_does_not_add_dataset_twice(self):
        DatasetFactory._meta.sqlalchemy_session = self._my_library.database.session
        dataset = DatasetFactory()
        self._my_library.search.index_dataset(dataset)

        datasets = self._my_library.search.backend.dataset_index.all()
        self.assertEqual(len(datasets), 1)

        self._my_library.search.index_dataset(dataset)
        datasets = self._my_library.search.backend.dataset_index.all()
        self.assertEqual(len(datasets), 1)
Ejemplo n.º 23
0
    def test_extends_notes_with_dataset_documentation(self):
        DatasetFactory._meta.sqlalchemy_session = self.sqlite_db.session
        FileFactory._meta.sqlalchemy_session = self.sqlite_db.session

        ds1 = DatasetFactory()
        FileFactory(dataset=ds1, path='documentation.md', contents='### Dataset documentation.')
        self.sqlite_db.commit()
        bundle = _get_fake_bundle(ds1)
        ret = _convert_bundle(bundle)

        self.assertIn('### Dataset documentation.', ret['notes'])
Ejemplo n.º 24
0
    def test_adds_dataset_document_to_the_index(self):
        DatasetFactory._meta.sqlalchemy_session = self.my_library.database.session
        dataset = DatasetFactory()
        self.backend.dataset_index.index_one(dataset)

        # search just added document.
        query = """
            SELECT vid
            FROM dataset_index;
        """
        result = self.backend.library.database.connection.execute(query).fetchall()
        self.assertEqual(result[0][0], dataset.vid)
Ejemplo n.º 25
0
    def test_returns_found_dataset(self):

        # add dataset to backend.
        DatasetFactory._meta.sqlalchemy_session = self.my_library.database.session
        dataset = DatasetFactory()
        self.backend.dataset_index.index_one(dataset)

        # search just added document.
        found = list(self.backend.dataset_index.search(dataset.vid))
        all_vids = [x.vid for x in found]
        self.assertIn(dataset.vid, all_vids)
        self.assertIsInstance(found[0], DatasetSearchResult)
Ejemplo n.º 26
0
    def test_creates_column_table(self):
        DatasetFactory._meta.sqlalchemy_session = self.db.session
        TableFactory._meta.sqlalchemy_session = self.db.session

        ds1 = DatasetFactory()
        self.db.commit()
        table = TableFactory(dataset=ds1)
        ColumnFactory._meta.sqlalchemy_session = self.db.session

        # Now all tables are created. Can we use ORM to create columns?
        ColumnFactory(name='id', table=table)
        self.db.commit()
Ejemplo n.º 27
0
    def test_populates_id_field(self, fake_before_update):
        ds = DatasetFactory()
        config1 = ConfigFactory.build(d_vid=ds.vid)
        assert config1.id is None

        mapper = Mock()
        conn = Mock()
        Config.before_insert(mapper, conn, config1)

        self.assertIsNotNone(config1.id)
        self.assertTrue(config1.id.startswith('Fds'))
        self.assertEqual(len(fake_before_update.mock_calls), 1)
Ejemplo n.º 28
0
    def test_returns_dictionary_representation_of_the_config(self):
        ds = DatasetFactory()
        config1 = ConfigFactory(d_vid=ds.vid)
        fields = [
            'id', 'sequence_id', 'dataset', 'd_vid', 'type', 'group', 'key',
            'value', 'modified', 'children', 'parent_id', 'parent'
        ]

        self.assertEqual(sorted(fields), sorted(config1.dict))

        for field in fields:
            self.assertEqual(getattr(config1, field), config1.dict[field])
Ejemplo n.º 29
0
    def _test_converts_schema_to_resource_dict(self):
        DatasetFactory._meta.sqlalchemy_session = self.sqlite_db.session
        FileFactory._meta.sqlalchemy_session = self.sqlite_db.session

        ds1 = DatasetFactory()
        partition1 = PartitionFactory(dataset=ds1)
        self.sqlite_db.commit()
        partition1._datafile = MagicMock()
        ret = _convert_partition(partition1)
        self.assertIn('package_id', ret)
        self.assertEqual(ret['package_id'], ds1.vid)
        self.assertEqual(ret['name'], partition1.name)
Ejemplo n.º 30
0
    def test_removes_dataset(self):

        # prepare state.
        DatasetFactory._meta.sqlalchemy_session = self.db.session
        ds1 = DatasetFactory()
        self.db.session.commit()
        ds1_vid = ds1.vid

        # testing
        self.db.remove_dataset(ds1)
        self.assertEqual(
            self.db.session.query(Dataset).filter_by(vid=ds1_vid).all(), [],
            'Dataset was not removed.')