def test_applies_new_migration_to_postgresql_database(self, fake_get): if self._db_type != 'postgres': self.skipTest('Postgres tests are disabled.') # replace real migrations with tests migrations. test_migrations = [ (100, 'test.test_orm.functional.migrations.0100_init'), (101, 'test.test_orm.functional.migrations.0101_add_column'), (102, 'test.test_orm.functional.migrations.0102_create_table'), (103, 'test.test_orm.functional.migrations.0103_not_ready' ) # that should not apply ] fake_get.return_value = test_migrations # create postgresql db postgres_test_db_dsn = self.config.library.database # PostgreSQLTestBase._create_postgres_test_db(get_runconfig())['test_db_dsn'] # populate database with initial schema with patch.object(database, 'SCHEMA_VERSION', 100): db = Database(postgres_test_db_dsn, engine_kwargs={'poolclass': NullPool}) db.create() db.close() # switch version and reconnect. Now both migrations should apply. with patch.object(database, 'SCHEMA_VERSION', 102): db = Database(postgres_test_db_dsn, engine_kwargs={'poolclass': NullPool}) try: # check column created by migration 101. db.connection\ .execute('SELECT column1 FROM {}.datasets;'.format(POSTGRES_SCHEMA_NAME))\ .fetchall() # check table created by migration 102. db.connection\ .execute('SELECT column1 FROM {}.table1;'.format(POSTGRES_SCHEMA_NAME))\ .fetchall() # db version changed to 102 db_version = db.connection\ .execute('SELECT version FROM {}.user_version;'.format(POSTGRES_SCHEMA_NAME))\ .fetchone()[0] self.assertEqual(db_version, 102) finally: db.close()
def test_applies_new_migration_to_postgresql_database(self, fake_get): if self._db_type != 'postgres': self.skipTest('Postgres tests are disabled.') # replace real migrations with tests migrations. test_migrations = [ (100, 'test.test_orm.functional.migrations.0100_init'), (101, 'test.test_orm.functional.migrations.0101_add_column'), (102, 'test.test_orm.functional.migrations.0102_create_table'), (103, 'test.test_orm.functional.migrations.0103_not_ready') # that should not apply ] fake_get.return_value = test_migrations # create postgresql db postgres_test_db_dsn = self.config.library.database # PostgreSQLTestBase._create_postgres_test_db(get_runconfig())['test_db_dsn'] # populate database with initial schema with patch.object(database, 'SCHEMA_VERSION', 100): db = Database(postgres_test_db_dsn, engine_kwargs={'poolclass': NullPool}) db.create() db.close() # switch version and reconnect. Now both migrations should apply. with patch.object(database, 'SCHEMA_VERSION', 102): db = Database(postgres_test_db_dsn, engine_kwargs={'poolclass': NullPool}) try: # check column created by migration 101. db.connection\ .execute('SELECT column1 FROM {}.datasets;'.format(POSTGRES_SCHEMA_NAME))\ .fetchall() # check table created by migration 102. db.connection\ .execute('SELECT column1 FROM {}.table1;'.format(POSTGRES_SCHEMA_NAME))\ .fetchall() # db version changed to 102 db_version = db.connection\ .execute('SELECT version FROM {}.user_version;'.format(POSTGRES_SCHEMA_NAME))\ .fetchone()[0] self.assertEqual(db_version, 102) finally: db.close()
class ConvertSchemaTest(unittest.TestCase): """ tests _convert_schema function. """ def setUp(self): self.sqlite_db = Database('sqlite://') self.sqlite_db.create() def _test_converts_schema_to_resource_dict(self): DatasetFactory._meta.sqlalchemy_session = self.sqlite_db.session FileFactory._meta.sqlalchemy_session = self.sqlite_db.session ds1 = DatasetFactory() partition1 = PartitionFactory(dataset=ds1) self.sqlite_db.commit() partition1._datafile = MagicMock() ret = _convert_partition(partition1) self.assertIn('package_id', ret) self.assertEqual(ret['package_id'], ds1.vid) self.assertEqual(ret['name'], partition1.name)
class ConvertPartitionTest(unittest.TestCase): def setUp(self): self.sqlite_db = Database('sqlite://') self.sqlite_db.create() def test_converts_partition_to_resource_dict(self): fake_partition = MagicMock(spec=Partition) fake_partition.dataset.vid = 'ds1vid' fake_partition.name = 'partition1' ret = _convert_partition(fake_partition) self.assertIn('package_id', ret) self.assertEqual(ret['package_id'], 'ds1vid') self.assertEqual(ret['name'], fake_partition.name) def test_converts_partition_content_to_csv(self): # prepare partition mock fake_partition = MagicMock(spec=Partition) fake_partition.dataset.vid = 'ds1vid' fake_partition.datafile.headers = ['col1', 'col2'] fake_iter = lambda: iter([{'col1': '1', 'col2': '1'}, {'col1': '2', 'col2': '2'}]) fake_partition.__iter__.side_effect = fake_iter # run. ret = _convert_partition(fake_partition) # check converted partition. self.assertIn('package_id', ret) self.assertEqual(ret['package_id'], 'ds1vid') self.assertIn('upload', ret) self.assertTrue(isinstance(ret['upload'], six.StringIO)) rows = [] reader = unicodecsv.reader(ret['upload']) for row in reader: rows.append(row) self.assertEqual(rows[0], ['col1', 'col2']) self.assertEqual(rows[1], ['1', '1']) self.assertEqual(rows[2], ['2', '2'])
class ConvertDatasetTest(unittest.TestCase): def setUp(self): self.sqlite_db = Database('sqlite://') self.sqlite_db.create() def test_converts_bundle_to_dict(self): DatasetFactory._meta.sqlalchemy_session = self.sqlite_db.session ds1 = DatasetFactory() bundle = _get_fake_bundle(ds1) self.sqlite_db.commit() ret = _convert_bundle(bundle) self.assertIn('name', ret) self.assertIsNotNone(ret['name']) self.assertEqual(ret['name'], ds1.vid) self.assertIn('title', ret) self.assertIsNotNone(ret['title']) self.assertEqual(ret['title'], ds1.config.metadata.about.title) self.assertIn('author', ret) self.assertIn('author_email', ret) self.assertIn('maintainer', ret) self.assertIn('maintainer_email', ret) def test_extends_notes_with_dataset_documentation(self): DatasetFactory._meta.sqlalchemy_session = self.sqlite_db.session FileFactory._meta.sqlalchemy_session = self.sqlite_db.session ds1 = DatasetFactory() FileFactory(dataset=ds1, path='documentation.md', contents='### Dataset documentation.') self.sqlite_db.commit() bundle = _get_fake_bundle(ds1) ret = _convert_bundle(bundle) self.assertIn('### Dataset documentation.', ret['notes'])
def test_creates_new_dataset(self): db = Database('sqlite://') db.create() ds = db.new_dataset(vid='d111', source='source', dataset='dataset') self.assertTrue(ds.vid, 'd111')
class ExportTest(unittest.TestCase): """ Tests export(bundle) function. """ def setUp(self): if not CKAN_CONFIG: raise EnvironmentError(MISSING_CREDENTIALS_MSG) self.sqlite_db = Database('sqlite://') self.sqlite_db.create() @patch('ambry.exporters.ckan.core.ckanapi.RemoteCKAN.call_action') def test_creates_package_for_given_dataset(self, fake_call): DatasetFactory._meta.sqlalchemy_session = self.sqlite_db.session ds1 = DatasetFactory() ds1.config.metadata.about.access = 'public' bundle = _get_fake_bundle(ds1) export(bundle) # assert call to service was valid. called = False for call in fake_call.mock_calls: _, args, kwargs = call if (args[0] == 'package_create' and kwargs['data_dict'].get('name') == ds1.vid): called = True self.assertTrue(called) @patch('ambry.exporters.ckan.core.ckanapi.RemoteCKAN.call_action') @patch('ambry.exporters.ckan.core._convert_partition') def test_creates_resource_for_each_partition_of_the_bundle(self, fake_convert, fake_call): DatasetFactory._meta.sqlalchemy_session = self.sqlite_db.session PartitionFactory._meta.sqlalchemy_session = self.sqlite_db.session ds1 = DatasetFactory() ds1.config.metadata.about.access = 'public' p1 = PartitionFactory(dataset=ds1) bundle = _get_fake_bundle(ds1, partitions=[p1]) fake_convert.return_value = {'name': p1.name, 'package_id': ds1.vid} export(bundle) # assert call to service was valid. called = False for call in fake_call.mock_calls: _, args, kwargs = call if (args[0] == 'resource_create' and kwargs['data_dict'].get('name') == p1.name and kwargs['data_dict'].get('package_id') == ds1.vid): called = True self.assertTrue(called) @patch('ambry.exporters.ckan.core.ckanapi.RemoteCKAN.call_action') @patch('ambry.exporters.ckan.core._convert_schema') def test_creates_resource_for_schema(self, fake_convert, fake_call): DatasetFactory._meta.sqlalchemy_session = self.sqlite_db.session ds1 = DatasetFactory() ds1.config.metadata.about.access = 'public' bundle = _get_fake_bundle(ds1) fake_convert.return_value = {'name': 'schema', 'package_id': ds1.vid} export(bundle) # assert call to service was valid. called = False for call in fake_call.mock_calls: _, args, kwargs = call if (args[0] == 'resource_create' and kwargs['data_dict'].get('name') == 'schema' and kwargs['data_dict'].get('package_id') == ds1.vid): called = True self.assertTrue(called) @patch('ambry.exporters.ckan.core.ckanapi.RemoteCKAN.call_action') def test_creates_resource_for_each_external_documentation(self, fake_call): DatasetFactory._meta.sqlalchemy_session = self.sqlite_db.session ds1 = DatasetFactory() ds1.config.metadata.about.access = 'public' # create two external documentations. # site1_descr = 'Descr1' site1_url = 'http://example.com/1' site2_descr = 'Descr2' site2_url = 'http://example.com/2' ds1.config.metadata.external_documentation.site1.description = site1_descr ds1.config.metadata.external_documentation.site1.url = site1_url ds1.config.metadata.external_documentation.site2.description = site2_descr ds1.config.metadata.external_documentation.site2.url = site2_url bundle = _get_fake_bundle(ds1) export(bundle) # assert call was valid resource_create_calls = {} for call in fake_call.mock_calls: _, args, kwargs = call if args[0] == 'resource_create': resource_create_calls[kwargs['data_dict']['name']] = kwargs['data_dict'] self.assertIn('site1', resource_create_calls) self.assertEqual(resource_create_calls['site1']['url'], site1_url) self.assertEqual(resource_create_calls['site1']['description'], site1_descr) self.assertIn('site2', resource_create_calls) self.assertEqual(resource_create_calls['site2']['url'], site2_url) self.assertEqual(resource_create_calls['site2']['description'], site2_descr) @patch('ambry.exporters.ckan.core.ckanapi.RemoteCKAN.call_action') def test_raises_UnpublishedAccessError_error(self, fake_call): DatasetFactory._meta.sqlalchemy_session = self.sqlite_db.session ds1 = DatasetFactory() ds1.config.metadata.about.access = 'restricted' bundle = _get_fake_bundle(ds1) with self.assertRaises(UnpublishedAccessError): export(bundle)
def test_creates_new_dataset(self): db = Database('sqlite://') db.create() ds = db.new_dataset(vid='d111', source='source', dataset='dataset') self.assertTrue(ds.vid, 'd111')