def test_list(self):
connection = Mock()
bucket = Mock()
connection.get_bucket.return_value = bucket
Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME,
connection).list('/')
bucket.list.assert_called_once_with('/', '', '', None, None)
def test_delete_key(self):
with patch.object(Key, 'delete') as delete:
connection = Mock()
key = Key()
Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME,
connection).delete(key)
delete.assert_called_once()
def test_save_key_as_string(self):
with patch.object(Key,
'set_contents_from_string') as \
set_contents_from_string:
connection = Mock()
key = Key()
contents = 'contents'
Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME,
connection).save(key, contents)
set_contents_from_string.assert_called_once_with(contents)
def setUp(self):
self.bucket = Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
BUCKET_NAME, Mock())
self.bucket.save = Mock()
self.bucket.delete = Mock()
self.schema = JsonObject(TABLE_NAME).property('eventId', 'VARCHAR(36)')
self.table = TargetTable(self.schema, Mock())
self.step = BulkCopyFromS3JsonStep(metadata='', source=SOURCE,
schema=self.schema,
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
bucket=self.bucket, table=self.table)
self.step.sql = Mock()
def __init__(self, aws_access_key_id, aws_secret_access_key, bucket,
db_connection):
super(S3CopyPipeline, self).__init__()
self.aws_access_key_id = aws_access_key_id
self.aws_secret_access_key = aws_secret_access_key
if isinstance(bucket, basestring):
self.bucket = Bucket(aws_access_key_id, aws_secret_access_key,
bucket)
else:
self.bucket = bucket
if isinstance(db_connection, Database):
self.database = db_connection
else:
self.database = Database(db_connection)
def setUp(self):
self.bucket = Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
BUCKET_NAME, Mock())
self.bucket.save = Mock()
self.bucket.delete = Mock()
self.schema = JsonObject(TABLE_NAME).property('eventId', 'VARCHAR(36)')
self.table = TargetTable(self.schema, Mock())
self.step = ManifestCopyFromS3JsonStep(metadata='', source=SOURCE,
schema=self.schema,
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
bucket=self.bucket,
table=self.table)
self.step.manifest = Mock()
self.updated_journal = [
'object_path/00c68a1e-85f2-49e5-9d07-6922046dbc5a',
'object_path/19440481-7766-4061-bd42-4a54fa0aac7c',
'object_path/2014-09-02/19440481-7766-4061-bd42-4a54fa0aac7c']
self.step.manifest.save = Mock(return_value=self.updated_journal)
self.step.sql = Mock()
def test_get_key(self):
self.assertNotEqual(
None,
Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME,
Mock()).get('key'))
def setUp(self):
self.schema = JsonObject(TABLE_NAME, Property('id', 'VARCHAR(36)'))
self.bucket = Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
BUCKET_NAME, Mock())
self.bucket.save = Mock()
self.database = create_autospec(Database)
self.key_names = [
'object_path/00c68a1e-85f2-49e5-9d07-6922046dbc5a',
'object_path/19440481-7766-4061-bd42-4a54fa0aac7c',
'object_path/2014-09-02/19440481-7766-4061-bd42-4a54fa0aac7c',
'object_path/282e6063-ecef-4e45-bdfb-9fdfb39840cd',
'object_path/35cbf09a-b2dc-43f2-96f6-7d7573906268',
'object_path/80536e83-6bbe-4a42-ade1-533d99321a6c',
'object_path/cf00b394-3ff3-4418-b244-2ccf104fcc40',
'object_path/e822e2ae-61f5-4be0-aacd-ca6de70faad1'
]
self.bucket.list = Mock(
return_value=[self.mock_key(key) for key in self.key_names])
self.manifest = SqlManifest(metadata='',
source='',
schema=self.schema,
bucket=self.bucket,
db_connection=self.database)
self.expected_manifest = {
'entries': [{
'url':
's3://{0}/object_path/00c68a1e-85f2-49e5-9d07-6922046dbc5a'.
format(BUCKET_NAME),
'mandatory':
True
}, {
'url':
's3://{0}/object_path/19440481-7766-4061-bd42-4a54fa0aac7c'.
format(BUCKET_NAME),
'mandatory':
True
}, {
'url':
's3://{0}/object_path/2014-09-02/19440481-7766-4061-bd42-4a54fa0aac7c'
.format(BUCKET_NAME),
'mandatory':
True
}, {
'url':
's3://{0}/object_path/282e6063-ecef-4e45-bdfb-9fdfb39840cd'.
format(BUCKET_NAME),
'mandatory':
True
}, {
'url':
's3://{0}/object_path/35cbf09a-b2dc-43f2-96f6-7d7573906268'.
format(BUCKET_NAME),
'mandatory':
True
}, {
'url':
's3://{0}/object_path/80536e83-6bbe-4a42-ade1-533d99321a6c'.
format(BUCKET_NAME),
'mandatory':
True
}, {
'url':
's3://{0}/object_path/cf00b394-3ff3-4418-b244-2ccf104fcc40'.
format(BUCKET_NAME),
'mandatory':
True
}, {
'url':
's3://{0}/object_path/e822e2ae-61f5-4be0-aacd-ca6de70faad1'.
format(BUCKET_NAME),
'mandatory':
True
}]
}