def test_have_paths_for_nested_objects(self): schema = JsonObject(TABLE_NAME, Property('property1', 'VARCHAR(10)'), Property('property2', 'TIMESTAMP'), Property('property3.dottedName', 'DOUBLE PRECISION'), Property('property4', Property('child', Property( 'subchild', 'BOOLEAN')))) self.assertEqual({'jsonpaths': ["$['property1']", "$['property2']", "$['property3.dottedName']", "$['property4']['child']['subchild']"]}, schema.paths())
def test_have_paths_for_nested_objects(self): schema = JsonObject( TABLE_NAME, Property('property1', 'VARCHAR(10)'), Property('property2', 'TIMESTAMP'), Property('property3.dottedName', 'DOUBLE PRECISION'), Property('property4', Property('child', Property('subchild', 'BOOLEAN')))) self.assertEqual( { 'jsonpaths': [ "$['property1']", "$['property2']", "$['property3.dottedName']", "$['property4']['child']['subchild']" ] }, schema.paths())
class BulkCopyFromS3JsonStepShould(unittest.TestCase): def setUp(self): self.bucket = Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME, Mock()) self.bucket.save = Mock() self.bucket.delete = Mock() self.schema = JsonObject(TABLE_NAME).property('eventId', 'VARCHAR(36)') self.table = TargetTable(self.schema, Mock()) self.step = BulkCopyFromS3JsonStep(metadata='', source=SOURCE, schema=self.schema, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, bucket=self.bucket, table=self.table) self.step.sql = Mock() def assert_migration_with_drop(self, target_table): target_table.database.open.assert_called_once_with() target_table.stage_update.assert_called_once_with() target_table.drop.assert_called_once_with() target_table.promote_update.assert_called_once_with() def assert_migration_without_drop(self, target_table): target_table.database.open.assert_called_once_with() target_table.stage_update.assert_called_once_with() self.assertEqual(False, target_table.drop.called) target_table.promote_update.assert_called_once_with() def test_save_schema_to_s3_bucket_on_run(self): self.step.run() self.bucket.save.assert_called_once_with(self.step.schema_key, json.dumps( self.schema.paths())) def test_copy_schema_to_redshift_with_drop_on_run(self): with patch.object(TargetTable, 'exists') as exists: exists.return_value = True target_table = TargetTable(self.schema, Mock()) target_table.stage_update = Mock() target_table.drop = Mock() target_table.promote_update = Mock() step = BulkCopyFromS3JsonStep(metadata='', source=SOURCE, schema=self.schema, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, bucket=self.bucket, table=target_table) step.sql = Mock() step.run() self.assert_migration_with_drop(target_table) target_table.database.commit.assert_called_once_with() def test_copy_schema_to_redshift_without_drop_on_run(self): with patch.object(TargetTable, 'exists') as exists: exists.return_value = False target_table = TargetTable(self.schema, Mock()) target_table.stage_update = Mock() target_table.drop = Mock() target_table.promote_update = Mock() step = BulkCopyFromS3JsonStep(metadata='', source=SOURCE, schema=self.schema, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, bucket=self.bucket, table=target_table) step.sql = Mock() step.run() self.assert_migration_without_drop(target_table) target_table.database.commit.assert_called_once_with() def test_validate_with_drop_on_run(self): with patch.object(TargetTable, 'exists') as exists: exists.return_value = True target_table = TargetTable(self.schema, Mock()) target_table.stage_update = Mock() target_table.drop = Mock() target_table.promote_update = Mock() step = BulkCopyFromS3JsonStep(metadata='', source=SOURCE, schema=self.schema, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, bucket=self.bucket, table=target_table) step.sql = Mock() step.validate() self.assert_migration_with_drop(target_table) target_table.database.rollback.assert_called_once_with() def test_validate_without_drop_on_run(self): with patch.object(TargetTable, 'exists') as exists: exists.return_value = False target_table = TargetTable(self.schema, Mock()) target_table.stage_update = Mock() target_table.drop = Mock() target_table.promote_update = Mock() step = BulkCopyFromS3JsonStep(metadata='', source=SOURCE, schema=self.schema, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, bucket=self.bucket, table=target_table) step.sql = Mock() step.validate() self.assert_migration_without_drop(target_table) target_table.database.rollback.assert_called_once_with() def test_source_url(self): self.assertEqual('s3://bucket/arbalest_test.event.created', self.step.source_url) def test_schema_url(self): self.assertEqual('s3://bucket/event_created_jsonpath.json', self.step.schema_url) def test_copy_sql(self): expected_copy_table_sql = "COPY %s FROM '%s' CREDENTIALS " \ "'aws_access_key_id=%s;" \ "aws_secret_access_key=%s' " \ "JSON '%s' TIMEFORMAT 'auto' " \ "MAXERROR %s" self.assertEqual(expected_copy_table_sql, self.step.copy_sql) def test_validate_sql(self): expected_validate_sql = "COPY %s FROM '%s' CREDENTIALS " \ "'aws_access_key_id=%s;" \ "aws_secret_access_key=%s' " \ "JSON '%s' TIMEFORMAT 'auto' " \ "MAXERROR %s " \ "NOLOAD" self.assertEqual(expected_validate_sql, self.step.validate_sql) def test_delete_schema_from_s3_bucket_on_run(self): self.step.run() self.bucket.delete.assert_called_once_with(self.step.schema_key)
class ManifestCopyFromS3JsonStepShould(unittest.TestCase): def setUp(self): self.bucket = Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME, Mock()) self.bucket.save = Mock() self.bucket.delete = Mock() self.schema = JsonObject(TABLE_NAME).property('eventId', 'VARCHAR(36)') self.table = TargetTable(self.schema, Mock()) self.step = ManifestCopyFromS3JsonStep(metadata='', source=SOURCE, schema=self.schema, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, bucket=self.bucket, table=self.table) self.step.manifest = Mock() self.updated_journal = [ 'object_path/00c68a1e-85f2-49e5-9d07-6922046dbc5a', 'object_path/19440481-7766-4061-bd42-4a54fa0aac7c', 'object_path/2014-09-02/19440481-7766-4061-bd42-4a54fa0aac7c'] self.step.manifest.save = Mock(return_value=self.updated_journal) self.step.sql = Mock() def assert_migration_with_drop(self, step): step.table.database.open.assert_called_once_with() step.manifest.save.assert_called_once_with() step.table.drop.assert_called_once_with() step.table.create.assert_called_once_with() def assert_migration_without_drop(self, step): step.table.database.open.assert_called_once_with() step.manifest.save.assert_called_once_with() self.assertEqual(False, step.table.drop.called) step.table.create.assert_called_once_with() def assert_migration_without_drop_and_create(self, step): step.table.database.open.assert_called_once_with() step.manifest.save.assert_called_once_with() self.assertEqual(False, step.table.drop.called) self.assertEqual(False, step.table.create.called) def assert_copy_schema_to_redshift_with_drop(self, sql, execute): with patch.object(TargetTable, 'exists') as exists: exists.return_value = True target_table = TargetTable(self.schema, Mock()) target_table.stage_update = Mock() target_table.drop = Mock() target_table.create = Mock() target_table.insert_update = Mock() step = ManifestCopyFromS3JsonStep(metadata='', source=SOURCE, schema=self.schema, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, bucket=self.bucket, table=target_table) step.manifest.save = Mock(return_value=self.updated_journal) step.manifest.journal_exists = Mock(return_value=False) step.sql = Mock() execute(step) self.assert_migration_with_drop(step) step.sql.execute.assert_called_once_with((sql, self.schema.table, step.manifest.manifest_url, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, step.schema_url, step.max_error_count)) return target_table.database def assert_copy_schema_to_redshift_without_drop(self, sql, execute): with patch.object(TargetTable, 'exists') as exists: exists.return_value = False target_table = TargetTable(self.schema, Mock()) target_table.stage_update = Mock() target_table.drop = Mock() target_table.create = Mock() target_table.insert_update = Mock() step = ManifestCopyFromS3JsonStep(metadata='', source=SOURCE, schema=self.schema, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, bucket=self.bucket, table=target_table) step.manifest.save = Mock(return_value=self.updated_journal) step.manifest.journal_exists = Mock(return_value=False) step.sql = Mock() execute(step) self.assert_migration_without_drop(step) step.sql.execute.assert_called_once_with((sql, self.schema.table, step.manifest.manifest_url, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, step.schema_url, step.max_error_count)) return target_table.database def assert_copy_schema_to_redshift_without_drop_and_create(self, sql, execute): with patch.object(TargetTable, 'exists') as exists: exists.return_value = True target_table = TargetTable(self.schema, Mock()) target_table.stage_update = Mock() target_table.drop = Mock() target_table.create = Mock() target_table.insert_update = Mock() step = ManifestCopyFromS3JsonStep(metadata='', source=SOURCE, schema=self.schema, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, bucket=self.bucket, table=target_table) step.manifest.save = Mock(return_value=self.updated_journal) step.manifest.journal_exists = Mock(return_value=True) step.sql = Mock() execute(step) self.assert_migration_without_drop_and_create(step) step.sql.execute.assert_called_once_with((sql, self.schema.table, step.manifest.manifest_url, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, step.schema_url, step.max_error_count)) return target_table.database def test_save_schema_to_s3_bucket_on_run(self): self.step.run() self.bucket.save.assert_called_once_with(self.step.schema_key, json.dumps( self.schema.paths())) def test_copy_schema_to_redshift_with_drop_when_no_journal_on_run(self): database = self.assert_copy_schema_to_redshift_with_drop( EXPECTED_COPY_SQL, lambda step: step.run()) database.commit.assert_called_once_with() def test_copy_schema_to_redshift_without_drop_when_no_journal_on_run(self): database = self.assert_copy_schema_to_redshift_without_drop( EXPECTED_COPY_SQL, lambda step: step.run()) database.commit.assert_called_once_with() def test_copy_schema_to_redshift_without_drop_and_create_when_journal_on_run( self): database = self.assert_copy_schema_to_redshift_without_drop_and_create( EXPECTED_COPY_SQL, lambda step: step.run()) database.commit.assert_called_once_with() def test_copy_schema_to_redshift_with_drop_when_no_journal_on_validate( self): database = self.assert_copy_schema_to_redshift_with_drop( EXPECTED_VALIDATE_SQL, lambda step: step.validate()) self.assertEqual(False, database.commit.called) database.rollback.assert_called_once_with() def test_copy_schema_to_redshift_without_drop_when_no_journal_on_validate( self): database = self.assert_copy_schema_to_redshift_without_drop( EXPECTED_VALIDATE_SQL, lambda step: step.validate()) self.assertEqual(False, database.commit.called) database.rollback.assert_called_once_with() def test_copy_schema_to_redshift_without_drop_and_create_when_journal_on_validate( self): database = self.assert_copy_schema_to_redshift_without_drop_and_create( EXPECTED_VALIDATE_SQL, lambda step: step.validate()) self.assertEqual(False, database.commit.called) database.rollback.assert_called_once_with() def test_schema_url(self): self.assertEqual('s3://bucket/event_created_jsonpath.json', self.step.schema_url) def test_commit_manifest_on_run(self): self.step.run() self.step.manifest.commit.assert_called_once_with( self.step.manifest.save()) def test_not_commit_manifest_on_validate(self): self.step.validate() self.assertEqual(False, self.step.manifest.commit.called) def test_delete_schema_from_s3_bucket_on_run(self): self.step.run() self.bucket.delete.assert_called_once_with(self.step.schema_key)