def setUp(self):
self.schema = JsonObject(TABLE_NAME, Property('id', 'VARCHAR(36)'))
self.database = Database(psycopg2.connect(CONNECTION))
self.database.open()
table = TargetTable(self.schema, self.database)
table.create()
self.database.commit()
def test_stage_update_when_column_name_defined(self):
with patch.object(Database, 'execute') as execute:
schema = JsonObject(
TABLE_NAME, Property('property1', 'VARCHAR(10)', 'someColumn'),
Property('property2', 'TIMESTAMP', 'anotherColumn'))
table = TargetTable(schema, Database(Mock()))
table.stage_update()
expected_sql = 'CREATE TABLE {0}_update (someColumn VARCHAR(10), ' \
'anotherColumn TIMESTAMP)'.format(TABLE_NAME)
execute.assert_called_once_with(expected_sql)
def test_create_when_column_name_not_defined_for_nested_property(self):
with patch.object(Database, 'execute') as execute:
schema = JsonObject(
TABLE_NAME, Property('property1', 'VARCHAR(10)'),
Property('property2', Property('timestamp', 'TIMESTAMP')))
table = TargetTable(schema, Database(Mock()))
table.create()
expected_sql = 'CREATE TABLE {0} (property1 VARCHAR(10), ' \
'property2_timestamp TIMESTAMP)'.format(TABLE_NAME)
execute.assert_called_once_with(expected_sql)
def __init__(self, metadata, source, schema, bucket, db_connection):
self.metadata = metadata
self.source = source
self.schema = schema
self.bucket = bucket
if isinstance(db_connection, Database):
self.database = db_connection
else:
self.database = Database(db_connection)
self.file_name = '{0}_manifest.json'.format(schema.table)
self.journal_file_name = '{0}_journal.db'.format(schema.table)
def __init__(self, aws_access_key_id, aws_secret_access_key, bucket,
db_connection):
super(S3CopyPipeline, self).__init__()
self.aws_access_key_id = aws_access_key_id
self.aws_secret_access_key = aws_secret_access_key
if isinstance(bucket, basestring):
self.bucket = Bucket(aws_access_key_id, aws_secret_access_key,
bucket)
else:
self.bucket = bucket
if isinstance(db_connection, Database):
self.database = db_connection
else:
self.database = Database(db_connection)
def sql_manifest_copy(self, metadata, source, schema, max_error_count=1):
sql_manifest_copy_step = ManifestCopyFromS3JsonStep(metadata=metadata,
source=source,
schema=schema,
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
bucket=self.bucket,
table=TargetTable(
schema,
self.database))
sql_manifest = SqlManifest(metadata, source, schema, self.bucket,
self.database)
sql_manifest.database = Database(
sqlite3.connect(sql_manifest.journal_file_name))
sql_manifest_copy_step.manifest = sql_manifest
self.__add_copy_step(sql_manifest_copy_step, max_error_count)
return self