def setUp(self): self.schema = JsonObject(TABLE_NAME, Property('id', 'VARCHAR(36)')) self.database = Database(psycopg2.connect(CONNECTION)) self.database.open() table = TargetTable(self.schema, self.database) table.create() self.database.commit()
def test_stage_update_when_column_name_defined(self): schema = JsonObject(TABLE_NAME, Property('property1', 'VARCHAR(10)', 'someColumn'), Property('property2', 'TIMESTAMP', 'anotherColumn')) table = TargetTable(schema, self.database) table.stage_update() self.assertColumns(schema.update_table, schema)
def test_create_when_column_name_not_defined(self): schema = JsonObject(TABLE_NAME, Property('property1', 'VARCHAR(10)'), Property('property2', 'TIMESTAMP')) table = TargetTable(schema, self.database) table.create() self.assertColumns(schema.table, schema)
def test_stage_update_when_column_name_defined(self): with patch.object(Database, 'execute') as execute: schema = JsonObject( TABLE_NAME, Property('property1', 'VARCHAR(10)', 'someColumn'), Property('property2', 'TIMESTAMP', 'anotherColumn')) table = TargetTable(schema, Database(Mock())) table.stage_update() expected_sql = 'CREATE TABLE {0}_update (someColumn VARCHAR(10), ' \ 'anotherColumn TIMESTAMP)'.format(TABLE_NAME) execute.assert_called_once_with(expected_sql)
def test_create_when_column_name_not_defined_for_nested_property(self): with patch.object(Database, 'execute') as execute: schema = JsonObject( TABLE_NAME, Property('property1', 'VARCHAR(10)'), Property('property2', Property('timestamp', 'TIMESTAMP'))) table = TargetTable(schema, Database(Mock())) table.create() expected_sql = 'CREATE TABLE {0} (property1 VARCHAR(10), ' \ 'property2_timestamp TIMESTAMP)'.format(TABLE_NAME) execute.assert_called_once_with(expected_sql)
def test_create_when_column_name_not_defined(self): with patch.object(Database, 'execute') as execute: schema = JsonObject(TABLE_NAME, Property('property1', 'VARCHAR(10)'), Property('property2', 'TIMESTAMP')) table = TargetTable(schema, Database(Mock())) table.create() expected_sql = 'CREATE TABLE {0} (property1 VARCHAR(10), ' \ 'property2 TIMESTAMP)'.format(TABLE_NAME) execute.assert_called_once_with(expected_sql)
def test_stage_update_when_column_name_defined(self): with patch.object(Database, 'execute') as execute: schema = JsonObject(TABLE_NAME, Property('property1', 'VARCHAR(10)', 'someColumn'), Property('property2', 'TIMESTAMP', 'anotherColumn')) table = TargetTable(schema, Database(Mock())) table.stage_update() expected_sql = 'CREATE TABLE {0}_update (someColumn VARCHAR(10), ' \ 'anotherColumn TIMESTAMP)'.format(TABLE_NAME) execute.assert_called_once_with(expected_sql)
def test_add_step(self): schema = JsonObject(TABLE_NAME, Property('id', 'VARCHAR(36)')) bucket = Mock() database = create_autospec(Database) expected = BulkCopyFromS3JsonStep( metadata='', source='', schema=schema, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, bucket=bucket, table=TargetTable(schema, database)) pipeline = S3BulkCopyPipeline(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, bucket, database) pipeline.step(metadata='', source='', schema=schema) step = pipeline.steps()[0] self.assertEqual(expected.metadata, step.metadata) self.assertEqual(expected.source, step.source) self.assertEqual(expected.schema, step.schema) self.assertEqual(expected.aws_access_key_id, step.aws_access_key_id) self.assertEqual(expected.aws_secret_access_key, step.aws_secret_access_key) self.assertEqual(expected.bucket, step.bucket) self.assertEqual(expected.table.schema, step.table.schema) self.assertEqual(expected.table.database, step.table.database)
def test_validate_with_drop_on_run(self): with patch.object(TargetTable, 'exists') as exists: exists.return_value = True target_table = TargetTable(self.schema, Mock()) target_table.stage_update = Mock() target_table.drop = Mock() target_table.promote_update = Mock() step = BulkCopyFromS3JsonStep(metadata='', source=SOURCE, schema=self.schema, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, bucket=self.bucket, table=target_table) step.sql = Mock() step.validate() self.assert_migration_with_drop(target_table) target_table.database.rollback.assert_called_once_with()
def assert_copy_schema_to_redshift_with_drop(self, sql, execute): with patch.object(TargetTable, 'exists') as exists: exists.return_value = True target_table = TargetTable(self.schema, Mock()) target_table.stage_update = Mock() target_table.drop = Mock() target_table.create = Mock() target_table.insert_update = Mock() step = ManifestCopyFromS3JsonStep(metadata='', source=SOURCE, schema=self.schema, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, bucket=self.bucket, table=target_table) step.manifest.save = Mock(return_value=self.updated_journal) step.manifest.journal_exists = Mock(return_value=False) step.sql = Mock() execute(step) self.assert_migration_with_drop(step) step.sql.execute.assert_called_once_with((sql, self.schema.table, step.manifest.manifest_url, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, step.schema_url, step.max_error_count)) return target_table.database
def setUp(self): self.bucket = Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME, Mock()) self.bucket.save = Mock() self.bucket.delete = Mock() self.schema = JsonObject(TABLE_NAME).property('eventId', 'VARCHAR(36)') self.table = TargetTable(self.schema, Mock()) self.step = BulkCopyFromS3JsonStep(metadata='', source=SOURCE, schema=self.schema, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, bucket=self.bucket, table=self.table) self.step.sql = Mock()
def setUp(self): self.bucket = Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME, Mock()) self.bucket.save = Mock() self.bucket.delete = Mock() self.schema = JsonObject(TABLE_NAME).property('eventId', 'VARCHAR(36)') self.table = TargetTable(self.schema, Mock()) self.step = ManifestCopyFromS3JsonStep(metadata='', source=SOURCE, schema=self.schema, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, bucket=self.bucket, table=self.table) self.step.manifest = Mock() self.updated_journal = [ 'object_path/00c68a1e-85f2-49e5-9d07-6922046dbc5a', 'object_path/19440481-7766-4061-bd42-4a54fa0aac7c', 'object_path/2014-09-02/19440481-7766-4061-bd42-4a54fa0aac7c'] self.step.manifest.save = Mock(return_value=self.updated_journal) self.step.sql = Mock()
def test_insert_update(self): table = TargetTable(self.schema, self.database) table.create() table.stage_update() self.database.execute("INSERT INTO %s VALUES('%s')", (AsIs(TABLE_NAME), AsIs(1),)) self.database.execute("INSERT INTO %s VALUES('%s')", (AsIs(self.schema.update_table), AsIs(2),)) self.database.execute("INSERT INTO %s VALUES('%s')", (AsIs(self.schema.update_table), AsIs(3),)) table.insert_update() self.database.execute( "SELECT * FROM %s", (AsIs(TABLE_NAME),)) values = list(self.database.cursor.fetchall()) self.assertEqual([('1',), ('2',), ('3',)], values)
def test_promote_update(self): table = TargetTable(self.schema, self.database) table.stage_update() table.promote_update() self.assertColumns(self.schema.table, self.schema)
def test_drop(self): table = TargetTable(self.schema, self.database) table.create() table.drop() self.assertEqual(False, table.exists())
def test_not_exist(self): table = TargetTable(self.schema, self.database) self.assertEqual(False, table.exists())
def test_exist(self): table = TargetTable(self.schema, self.database) table.create() self.assertEqual(True, table.exists())