def setUp(self):
self.schema = JsonObject(TABLE_NAME, Property('id', 'VARCHAR(36)'))
self.database = Database(psycopg2.connect(CONNECTION))
self.database.open()
table = TargetTable(self.schema, self.database)
table.create()
self.database.commit()
def setUp(self):
self.schema = JsonObject(TABLE_NAME, Property('id', 'VARCHAR(36)'))
self.database = Database(psycopg2.connect(CONNECTION))
self.database.open()
table = TargetTable(self.schema, self.database)
table.create()
self.database.commit()
def test_stage_update_when_column_name_defined(self):
schema = JsonObject(TABLE_NAME,
Property('property1', 'VARCHAR(10)', 'someColumn'),
Property('property2', 'TIMESTAMP', 'anotherColumn'))
table = TargetTable(schema, self.database)
table.stage_update()
self.assertColumns(schema.update_table, schema)
def test_create_when_column_name_not_defined(self):
schema = JsonObject(TABLE_NAME,
Property('property1', 'VARCHAR(10)'),
Property('property2', 'TIMESTAMP'))
table = TargetTable(schema, self.database)
table.create()
self.assertColumns(schema.table, schema)
def test_stage_update_when_column_name_defined(self):
with patch.object(Database, 'execute') as execute:
schema = JsonObject(
TABLE_NAME, Property('property1', 'VARCHAR(10)', 'someColumn'),
Property('property2', 'TIMESTAMP', 'anotherColumn'))
table = TargetTable(schema, Database(Mock()))
table.stage_update()
expected_sql = 'CREATE TABLE {0}_update (someColumn VARCHAR(10), ' \
'anotherColumn TIMESTAMP)'.format(TABLE_NAME)
execute.assert_called_once_with(expected_sql)
def test_create_when_column_name_not_defined_for_nested_property(self):
with patch.object(Database, 'execute') as execute:
schema = JsonObject(
TABLE_NAME, Property('property1', 'VARCHAR(10)'),
Property('property2', Property('timestamp', 'TIMESTAMP')))
table = TargetTable(schema, Database(Mock()))
table.create()
expected_sql = 'CREATE TABLE {0} (property1 VARCHAR(10), ' \
'property2_timestamp TIMESTAMP)'.format(TABLE_NAME)
execute.assert_called_once_with(expected_sql)
def test_create_when_column_name_not_defined(self):
with patch.object(Database, 'execute') as execute:
schema = JsonObject(TABLE_NAME,
Property('property1', 'VARCHAR(10)'),
Property('property2', 'TIMESTAMP'))
table = TargetTable(schema, Database(Mock()))
table.create()
expected_sql = 'CREATE TABLE {0} (property1 VARCHAR(10), ' \
'property2 TIMESTAMP)'.format(TABLE_NAME)
execute.assert_called_once_with(expected_sql)
def test_stage_update_when_column_name_defined(self):
with patch.object(Database, 'execute') as execute:
schema = JsonObject(TABLE_NAME,
Property('property1', 'VARCHAR(10)',
'someColumn'),
Property('property2', 'TIMESTAMP',
'anotherColumn'))
table = TargetTable(schema, Database(Mock()))
table.stage_update()
expected_sql = 'CREATE TABLE {0}_update (someColumn VARCHAR(10), ' \
'anotherColumn TIMESTAMP)'.format(TABLE_NAME)
execute.assert_called_once_with(expected_sql)
def test_add_step(self):
schema = JsonObject(TABLE_NAME, Property('id', 'VARCHAR(36)'))
bucket = Mock()
database = create_autospec(Database)
expected = BulkCopyFromS3JsonStep(
metadata='',
source='',
schema=schema,
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
bucket=bucket,
table=TargetTable(schema, database))
pipeline = S3BulkCopyPipeline(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
bucket, database)
pipeline.step(metadata='', source='', schema=schema)
step = pipeline.steps()[0]
self.assertEqual(expected.metadata, step.metadata)
self.assertEqual(expected.source, step.source)
self.assertEqual(expected.schema, step.schema)
self.assertEqual(expected.aws_access_key_id, step.aws_access_key_id)
self.assertEqual(expected.aws_secret_access_key,
step.aws_secret_access_key)
self.assertEqual(expected.bucket, step.bucket)
self.assertEqual(expected.table.schema, step.table.schema)
self.assertEqual(expected.table.database, step.table.database)
def test_validate_with_drop_on_run(self):
with patch.object(TargetTable, 'exists') as exists:
exists.return_value = True
target_table = TargetTable(self.schema, Mock())
target_table.stage_update = Mock()
target_table.drop = Mock()
target_table.promote_update = Mock()
step = BulkCopyFromS3JsonStep(metadata='',
source=SOURCE,
schema=self.schema,
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
bucket=self.bucket,
table=target_table)
step.sql = Mock()
step.validate()
self.assert_migration_with_drop(target_table)
target_table.database.rollback.assert_called_once_with()
def assert_copy_schema_to_redshift_with_drop(self, sql, execute):
with patch.object(TargetTable, 'exists') as exists:
exists.return_value = True
target_table = TargetTable(self.schema, Mock())
target_table.stage_update = Mock()
target_table.drop = Mock()
target_table.create = Mock()
target_table.insert_update = Mock()
step = ManifestCopyFromS3JsonStep(metadata='',
source=SOURCE,
schema=self.schema,
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
bucket=self.bucket,
table=target_table)
step.manifest.save = Mock(return_value=self.updated_journal)
step.manifest.journal_exists = Mock(return_value=False)
step.sql = Mock()
execute(step)
self.assert_migration_with_drop(step)
step.sql.execute.assert_called_once_with((sql,
self.schema.table,
step.manifest.manifest_url,
AWS_ACCESS_KEY_ID,
AWS_SECRET_ACCESS_KEY,
step.schema_url,
step.max_error_count))
return target_table.database
def setUp(self):
self.bucket = Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
BUCKET_NAME, Mock())
self.bucket.save = Mock()
self.bucket.delete = Mock()
self.schema = JsonObject(TABLE_NAME).property('eventId', 'VARCHAR(36)')
self.table = TargetTable(self.schema, Mock())
self.step = BulkCopyFromS3JsonStep(metadata='', source=SOURCE,
schema=self.schema,
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
bucket=self.bucket, table=self.table)
self.step.sql = Mock()
def setUp(self):
self.bucket = Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
BUCKET_NAME, Mock())
self.bucket.save = Mock()
self.bucket.delete = Mock()
self.schema = JsonObject(TABLE_NAME).property('eventId', 'VARCHAR(36)')
self.table = TargetTable(self.schema, Mock())
self.step = ManifestCopyFromS3JsonStep(metadata='', source=SOURCE,
schema=self.schema,
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
bucket=self.bucket,
table=self.table)
self.step.manifest = Mock()
self.updated_journal = [
'object_path/00c68a1e-85f2-49e5-9d07-6922046dbc5a',
'object_path/19440481-7766-4061-bd42-4a54fa0aac7c',
'object_path/2014-09-02/19440481-7766-4061-bd42-4a54fa0aac7c']
self.step.manifest.save = Mock(return_value=self.updated_journal)
self.step.sql = Mock()
def test_insert_update(self):
table = TargetTable(self.schema, self.database)
table.create()
table.stage_update()
self.database.execute("INSERT INTO %s VALUES('%s')",
(AsIs(TABLE_NAME), AsIs(1),))
self.database.execute("INSERT INTO %s VALUES('%s')",
(AsIs(self.schema.update_table), AsIs(2),))
self.database.execute("INSERT INTO %s VALUES('%s')",
(AsIs(self.schema.update_table), AsIs(3),))
table.insert_update()
self.database.execute(
"SELECT * FROM %s",
(AsIs(TABLE_NAME),))
values = list(self.database.cursor.fetchall())
self.assertEqual([('1',), ('2',), ('3',)], values)
def test_promote_update(self):
table = TargetTable(self.schema, self.database)
table.stage_update()
table.promote_update()
self.assertColumns(self.schema.table, self.schema)
def test_drop(self):
table = TargetTable(self.schema, self.database)
table.create()
table.drop()
self.assertEqual(False, table.exists())
def test_not_exist(self):
table = TargetTable(self.schema, self.database)
self.assertEqual(False, table.exists())
def test_exist(self):
table = TargetTable(self.schema, self.database)
table.create()
self.assertEqual(True, table.exists())
