Example #1
0
 def setUp(self):
     self.schema = JsonObject(TABLE_NAME, Property('id', 'VARCHAR(36)'))
     self.database = Database(psycopg2.connect(CONNECTION))
     self.database.open()
     table = TargetTable(self.schema, self.database)
     table.create()
     self.database.commit()
Example #2
0
    def test_stage_update_when_column_name_defined(self):
        with patch.object(Database, 'execute') as execute:
            schema = JsonObject(
                TABLE_NAME, Property('property1', 'VARCHAR(10)', 'someColumn'),
                Property('property2', 'TIMESTAMP', 'anotherColumn'))
            table = TargetTable(schema, Database(Mock()))

            table.stage_update()

            expected_sql = 'CREATE TABLE {0}_update (someColumn VARCHAR(10), ' \
                           'anotherColumn TIMESTAMP)'.format(TABLE_NAME)

            execute.assert_called_once_with(expected_sql)
Example #3
0
    def test_create_when_column_name_not_defined_for_nested_property(self):
        with patch.object(Database, 'execute') as execute:
            schema = JsonObject(
                TABLE_NAME, Property('property1', 'VARCHAR(10)'),
                Property('property2', Property('timestamp', 'TIMESTAMP')))
            table = TargetTable(schema, Database(Mock()))

            table.create()

            expected_sql = 'CREATE TABLE {0} (property1 VARCHAR(10), ' \
                           'property2_timestamp TIMESTAMP)'.format(TABLE_NAME)

            execute.assert_called_once_with(expected_sql)
Example #4
0
    def __init__(self, metadata, source, schema, bucket, db_connection):
        self.metadata = metadata
        self.source = source
        self.schema = schema
        self.bucket = bucket

        if isinstance(db_connection, Database):
            self.database = db_connection
        else:
            self.database = Database(db_connection)

        self.file_name = '{0}_manifest.json'.format(schema.table)
        self.journal_file_name = '{0}_journal.db'.format(schema.table)
Example #5
0
    def __init__(self, aws_access_key_id, aws_secret_access_key, bucket,
                 db_connection):
        super(S3CopyPipeline, self).__init__()

        self.aws_access_key_id = aws_access_key_id
        self.aws_secret_access_key = aws_secret_access_key

        if isinstance(bucket, basestring):
            self.bucket = Bucket(aws_access_key_id, aws_secret_access_key,
                                 bucket)
        else:
            self.bucket = bucket

        if isinstance(db_connection, Database):
            self.database = db_connection
        else:
            self.database = Database(db_connection)
Example #6
0
    def sql_manifest_copy(self, metadata, source, schema, max_error_count=1):
        sql_manifest_copy_step = ManifestCopyFromS3JsonStep(metadata=metadata,
                                                            source=source,
                                                            schema=schema,
                                                            aws_access_key_id=self.aws_access_key_id,
                                                            aws_secret_access_key=self.aws_secret_access_key,
                                                            bucket=self.bucket,
                                                            table=TargetTable(
                                                                schema,
                                                                self.database))

        sql_manifest = SqlManifest(metadata, source, schema, self.bucket,
                                   self.database)
        sql_manifest.database = Database(
            sqlite3.connect(sql_manifest.journal_file_name))
        sql_manifest_copy_step.manifest = sql_manifest
        self.__add_copy_step(sql_manifest_copy_step, max_error_count)
        return self