def test_list(self): connection = Mock() bucket = Mock() connection.get_bucket.return_value = bucket Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME, connection).list('/') bucket.list.assert_called_once_with('/', '', '', None, None)
def test_delete_key(self): with patch.object(Key, 'delete') as delete: connection = Mock() key = Key() Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME, connection).delete(key) delete.assert_called_once()
def test_save_key_as_string(self): with patch.object(Key, 'set_contents_from_string') as \ set_contents_from_string: connection = Mock() key = Key() contents = 'contents' Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME, connection).save(key, contents) set_contents_from_string.assert_called_once_with(contents)
def setUp(self): self.bucket = Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME, Mock()) self.bucket.save = Mock() self.bucket.delete = Mock() self.schema = JsonObject(TABLE_NAME).property('eventId', 'VARCHAR(36)') self.table = TargetTable(self.schema, Mock()) self.step = BulkCopyFromS3JsonStep(metadata='', source=SOURCE, schema=self.schema, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, bucket=self.bucket, table=self.table) self.step.sql = Mock()
def __init__(self, aws_access_key_id, aws_secret_access_key, bucket, db_connection): super(S3CopyPipeline, self).__init__() self.aws_access_key_id = aws_access_key_id self.aws_secret_access_key = aws_secret_access_key if isinstance(bucket, basestring): self.bucket = Bucket(aws_access_key_id, aws_secret_access_key, bucket) else: self.bucket = bucket if isinstance(db_connection, Database): self.database = db_connection else: self.database = Database(db_connection)
def setUp(self): self.bucket = Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME, Mock()) self.bucket.save = Mock() self.bucket.delete = Mock() self.schema = JsonObject(TABLE_NAME).property('eventId', 'VARCHAR(36)') self.table = TargetTable(self.schema, Mock()) self.step = ManifestCopyFromS3JsonStep(metadata='', source=SOURCE, schema=self.schema, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, bucket=self.bucket, table=self.table) self.step.manifest = Mock() self.updated_journal = [ 'object_path/00c68a1e-85f2-49e5-9d07-6922046dbc5a', 'object_path/19440481-7766-4061-bd42-4a54fa0aac7c', 'object_path/2014-09-02/19440481-7766-4061-bd42-4a54fa0aac7c'] self.step.manifest.save = Mock(return_value=self.updated_journal) self.step.sql = Mock()
def test_get_key(self): self.assertNotEqual( None, Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME, Mock()).get('key'))
def setUp(self): self.schema = JsonObject(TABLE_NAME, Property('id', 'VARCHAR(36)')) self.bucket = Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME, Mock()) self.bucket.save = Mock() self.database = create_autospec(Database) self.key_names = [ 'object_path/00c68a1e-85f2-49e5-9d07-6922046dbc5a', 'object_path/19440481-7766-4061-bd42-4a54fa0aac7c', 'object_path/2014-09-02/19440481-7766-4061-bd42-4a54fa0aac7c', 'object_path/282e6063-ecef-4e45-bdfb-9fdfb39840cd', 'object_path/35cbf09a-b2dc-43f2-96f6-7d7573906268', 'object_path/80536e83-6bbe-4a42-ade1-533d99321a6c', 'object_path/cf00b394-3ff3-4418-b244-2ccf104fcc40', 'object_path/e822e2ae-61f5-4be0-aacd-ca6de70faad1' ] self.bucket.list = Mock( return_value=[self.mock_key(key) for key in self.key_names]) self.manifest = SqlManifest(metadata='', source='', schema=self.schema, bucket=self.bucket, db_connection=self.database) self.expected_manifest = { 'entries': [{ 'url': 's3://{0}/object_path/00c68a1e-85f2-49e5-9d07-6922046dbc5a'. format(BUCKET_NAME), 'mandatory': True }, { 'url': 's3://{0}/object_path/19440481-7766-4061-bd42-4a54fa0aac7c'. format(BUCKET_NAME), 'mandatory': True }, { 'url': 's3://{0}/object_path/2014-09-02/19440481-7766-4061-bd42-4a54fa0aac7c' .format(BUCKET_NAME), 'mandatory': True }, { 'url': 's3://{0}/object_path/282e6063-ecef-4e45-bdfb-9fdfb39840cd'. format(BUCKET_NAME), 'mandatory': True }, { 'url': 's3://{0}/object_path/35cbf09a-b2dc-43f2-96f6-7d7573906268'. format(BUCKET_NAME), 'mandatory': True }, { 'url': 's3://{0}/object_path/80536e83-6bbe-4a42-ade1-533d99321a6c'. format(BUCKET_NAME), 'mandatory': True }, { 'url': 's3://{0}/object_path/cf00b394-3ff3-4418-b244-2ccf104fcc40'. format(BUCKET_NAME), 'mandatory': True }, { 'url': 's3://{0}/object_path/e822e2ae-61f5-4be0-aacd-ca6de70faad1'. format(BUCKET_NAME), 'mandatory': True }] }