def test_get_route_with_s3_credentials(self):
# Allow the user to store data in Amazon S3 if they specify all the
# correct credentials.
server = RESTServer()
server.request = flexmock()
# Presume that the user has only specified S3 credentials.
server.request.should_receive('get').with_args('name').and_return('s3')
server.request.should_receive('get').with_args('AWS_ACCESS_KEY').and_return(
'access')
server.request.should_receive('get').with_args('AWS_SECRET_KEY').and_return(
'secret')
server.request.should_receive('get').with_args('GCS_ACCESS_KEY').and_return(
'')
server.request.should_receive('get').with_args('GCS_SECRET_KEY').and_return(
'')
server.request.should_receive('get').with_args('S3_URL').and_return('')
server.request.should_receive('get').with_args('AZURE_ACCOUNT_NAME') \
.and_return('')
server.request.should_receive('get').with_args('AZURE_ACCOUNT_KEY') \
.and_return('')
# Mock out writing the file contents that were sent over.
flexmock(uuid)
uuid.should_receive('uuid4').and_return('123')
fake_file = flexmock(name='fake_file')
fake_file.should_receive('read').and_return('file contents')
fake_builtins = flexmock(sys.modules['__builtin__'])
fake_builtins.should_call('open')
fake_builtins.should_receive('open').with_args('/tmp/magik-temp-123', 'r') \
.and_return(fake_file)
# Mock out interacting with S3.
fake_storage = flexmock(name='fake_storage')
fake_storage.should_receive('download_files').with_args([{
'source' : '/baz/gbaz.txt',
'destination' : '/tmp/magik-temp-123'
}]).and_return([{
'success' : True
}])
flexmock(StorageFactory)
StorageFactory.should_receive('get_storage').with_args(dict).and_return(
fake_storage)
# Mock out writing the response.
server.response = flexmock()
server.response.should_receive('write').and_return()
# Finally, mock out removing the tempfile we created.
flexmock(os)
os.should_receive('remove').with_args('/tmp/magik-temp-123')
self.assertEquals(None, server.get('/baz/gbaz.txt'))
def test_walrus_storage_creation_without_necessary_parameters(self):
# Trying to create a WalrusStorage without the AWS_ACCESS_KEY should fail.
self.assertRaises(BadConfigurationException, StorageFactory.get_storage, {
"name" : "walrus"
})
# Similarly, creating a WalrusStorage object without the AWS_SECRET_KEY
# should fail.
self.assertRaises(BadConfigurationException, StorageFactory.get_storage, {
"name" : "walrus",
"AWS_ACCESS_KEY" : "access"
})
# If S3_URL isn't a URL, an Exception should be thrown.
self.assertRaises(BadConfigurationException, StorageFactory.get_storage, {
"name" : "walrus",
"AWS_ACCESS_KEY" : "access",
"AWS_SECRET_KEY" : "secret",
"S3_URL" : "1.2.3.4:8773/services/Walrus"
})
# If S3_URL is a URL, that should be fine.
flexmock(boto.s3.connection)
boto.s3.connection.should_receive('S3Connection')
another_walrus = StorageFactory.get_storage({
"name" : "walrus",
"AWS_ACCESS_KEY" : "access",
"AWS_SECRET_KEY" : "secret",
"S3_URL" : "http://1.2.3.4:8773/services/Walrus"
})
self.assertEquals("access", another_walrus.aws_access_key)
self.assertEquals("secret", another_walrus.aws_secret_key)
self.assertEquals("http://1.2.3.4:8773/services/Walrus",
another_walrus.s3_url)
def put(self, path):
""" Uploads a file to a cloud storage platform.
In addition to the arguments below, this method also expects the following
parameters to be posted to it:
body: The contents of the file to store.
name: The name of the cloud storage platform to interact with (e.g.,
's3').
credentials: Any AWS, GCS, Walrus, or Azure credential, that should be
used to authenticate this user.
Args:
path: A str that represents the name of the file to upload in the cloud
storage platform. The name of the bucket should be the first item, so
a path of '/mybucket/file/name.txt' indicates that the file
'file/name.txt' should be uploaded to the bucket 'mybucket'.
"""
file_contents = self.request.body
if file_contents == '':
self.response.write(
json.dumps([{
'success': False,
'failure_reason': 'no request body specified'
}]))
return
args = self.get_args_from_request_params(self.request)
storage = StorageFactory.get_storage(args)
source = self.write_temporary_file(file_contents)
source_to_dest_list = [{'source': source, 'destination': path}]
self.response.write(storage.upload_files(source_to_dest_list))
os.remove(source)
return
def setUp(self):
# Set up a mock for when we interact with S3
self.fake_gcs = flexmock(name="fake_gcs")
flexmock(boto.gs.connection)
boto.gs.connection.should_receive("GSConnection").with_args(
gs_access_key_id="access", gs_secret_access_key="secret"
).and_return(self.fake_gcs)
self.gcs = StorageFactory.get_storage({"name": "gcs", "GCS_ACCESS_KEY": "access", "GCS_SECRET_KEY": "secret"})
def setUp(self):
# Set up a mock for when we interact with S3
self.fake_azure = flexmock(name='fake_azure')
flexmock(azure.storage)
azure.storage.should_receive('BlobService').with_args('access', 'secret') \
.and_return(self.fake_azure)
self.azure = StorageFactory.get_storage({
"name" : "azure",
"AZURE_ACCOUNT_NAME" : "access",
"AZURE_ACCOUNT_KEY" : "secret"
})
def setUp(self):
# Set up a mock for when we interact with S3
self.fake_azure = flexmock(name='fake_azure')
flexmock(azure.storage)
azure.storage.should_receive('BlobService').with_args('access', 'secret') \
.and_return(self.fake_azure)
self.azure = StorageFactory.get_storage({
"name": "azure",
"AZURE_ACCOUNT_NAME": "access",
"AZURE_ACCOUNT_KEY": "secret"
})
def setUp(self):
# Set up a mock for when we interact with S3
self.fake_s3 = flexmock(name='fake_s3')
flexmock(boto.s3.connection)
boto.s3.connection.should_receive('S3Connection').with_args(
aws_access_key_id='access', aws_secret_access_key='secret') \
.and_return(self.fake_s3)
self.s3 = StorageFactory.get_storage({
"name" : "s3",
"AWS_ACCESS_KEY" : "access",
"AWS_SECRET_KEY" : "secret"
})
def setUp(self):
# Set up a mock for when we interact with Walrus
self.fake_walrus = flexmock(name='fake_walrus')
flexmock(boto.s3.connection)
boto.s3.connection.should_receive('S3Connection').and_return(
self.fake_walrus)
self.walrus = StorageFactory.get_storage({
"name" : "walrus",
"AWS_ACCESS_KEY" : "access",
"AWS_SECRET_KEY" : "secret",
"S3_URL" : "http://1.2.3.4:8773/services/Walrus"
})
def setUp(self):
# Set up a mock for when we interact with S3
self.fake_gcs = flexmock(name='fake_gcs')
flexmock(boto.gs.connection)
boto.gs.connection.should_receive('GSConnection').with_args(
gs_access_key_id='access', gs_secret_access_key='secret') \
.and_return(self.fake_gcs)
self.gcs = StorageFactory.get_storage({
"name" : "gcs",
"GCS_ACCESS_KEY" : "access",
"GCS_SECRET_KEY" : "secret"
})
def setUp(self):
# Set up a mock for when we interact with S3
self.fake_s3 = flexmock(name='fake_s3')
flexmock(boto.s3.connection)
boto.s3.connection.should_receive('S3Connection').with_args(
aws_access_key_id='access', aws_secret_access_key='secret') \
.and_return(self.fake_s3)
self.s3 = StorageFactory.get_storage({
"name": "s3",
"AWS_ACCESS_KEY": "access",
"AWS_SECRET_KEY": "secret"
})
def test_gc_storage_creation_without_necessary_parameters(self):
# Trying to create a GCStorage without the GCS_ACCESS_KEY should fail.
self.assertRaises(BadConfigurationException, StorageFactory.get_storage, {"name": "gcs"})
# Similarly, creating a GCStorage object without the GCS_SECRET_KEY
# should fail.
self.assertRaises(
BadConfigurationException, StorageFactory.get_storage, {"name": "gcs", "GCS_ACCESS_KEY": "access"}
)
# Specifying both should result in the GCStorage object being created,
# and instance variables set with those values.
gcs = StorageFactory.get_storage({"name": "gcs", "GCS_ACCESS_KEY": "access", "GCS_SECRET_KEY": "secret"})
self.assertEquals("access", gcs.gcs_access_key)
self.assertEquals("secret", gcs.gcs_secret_key)
def setUp(self):
# Set up a mock for when we interact with Walrus
self.fake_walrus = flexmock(name='fake_walrus')
flexmock(boto.s3.connection)
boto.s3.connection.should_receive('S3Connection').and_return(
self.fake_walrus)
self.walrus = StorageFactory.get_storage({
"name":
"walrus",
"AWS_ACCESS_KEY":
"access",
"AWS_SECRET_KEY":
"secret",
"S3_URL":
"http://1.2.3.4:8773/services/Walrus"
})
def delete(self, path):
""" Deletes a file from a cloud storage platform.
In addition to the arguments below, this method also expects the following
parameters to be posted to it:
name: The name of the cloud storage platform to interact with (e.g.,
's3').
credentials: Any AWS, GCS, Walrus, or Azure credential, that should be
used to authenticate this user.
Args:
path: A str that represents the name of the file to delete from the cloud
storage platform. The name of the bucket should be the first item, so
a path of '/mybucket/file/name.txt' indicates that the file
'file/name.txt' should be uploaded to the bucket 'mybucket'.
"""
args = self.get_args_from_request_params(self.request)
storage = StorageFactory.get_storage(args)
files_to_delete = [{'source': path}]
self.response.write(storage.delete_files(files_to_delete))
def get(self, path):
""" Downloads a file from a cloud storage platform.
In addition to the arguments below, this method also expects the following
parameters to be posted to it:
name: The name of the cloud storage platform to interact with (e.g.,
's3').
credentials: Any AWS, GCS, Walrus, or Azure credential, that should be
used to authenticate this user.
Args:
path: A str that represents the name of the file to download in the cloud
storage platform. The name of the bucket should be the first item, so
a path of '/mybucket/file/name.txt' indicates that the file
'file/name.txt' should be downloaded from the bucket 'mybucket'.
"""
args = self.get_args_from_request_params(self.request)
if args['name'] == '':
self.response.write(json.dumps([{
'success' : False,
'failure_reason' : 'no storage specified'
}]))
return
storage = StorageFactory.get_storage(args)
random_suffix = str(uuid.uuid4()).replace('-', '')[:10]
destination = '/tmp/magik-temp-{0}'.format(random_suffix)
source_to_dest_list = [{
'source' : path,
'destination' : destination
}]
result = storage.download_files(source_to_dest_list)
if result[0]['success'] == True:
with open(destination, 'r') as file_handle:
self.response.write(file_handle.read())
os.remove(destination)
else:
self.response.write(json.dumps(result))
return
def test_azure_storage_creation_without_necessary_parameters(self):
# Trying to create an AzureStorage without the account name should fail.
self.assertRaises(BadConfigurationException,
StorageFactory.get_storage, {"name": "azure"})
# Similarly, creating a AzureStorage object without the account key
# should fail.
self.assertRaises(BadConfigurationException,
StorageFactory.get_storage, {
"name": "azure",
"AZURE_ACCOUNT_NAME": "access"
})
# Specifying both should result in the AzureStorage object being created,
# and instance variables set with those values.
azure = StorageFactory.get_storage({
"name": "azure",
"AZURE_ACCOUNT_NAME": "access",
"AZURE_ACCOUNT_KEY": "secret"
})
self.assertEquals("access", azure.azure_account_name)
self.assertEquals("secret", azure.azure_account_key)
def delete(self, path):
""" Deletes a file from a cloud storage platform.
In addition to the arguments below, this method also expects the following
parameters to be posted to it:
name: The name of the cloud storage platform to interact with (e.g.,
's3').
credentials: Any AWS, GCS, Walrus, or Azure credential, that should be
used to authenticate this user.
Args:
path: A str that represents the name of the file to delete from the cloud
storage platform. The name of the bucket should be the first item, so
a path of '/mybucket/file/name.txt' indicates that the file
'file/name.txt' should be uploaded to the bucket 'mybucket'.
"""
args = self.get_args_from_request_params(self.request)
storage = StorageFactory.get_storage(args)
files_to_delete = [{
'source' : path
}]
self.response.write(storage.delete_files(files_to_delete))
def test_walrus_storage_creation_without_necessary_parameters(self):
# Trying to create a WalrusStorage without the AWS_ACCESS_KEY should fail.
self.assertRaises(BadConfigurationException,
StorageFactory.get_storage, {"name": "walrus"})
# Similarly, creating a WalrusStorage object without the AWS_SECRET_KEY
# should fail.
self.assertRaises(BadConfigurationException,
StorageFactory.get_storage, {
"name": "walrus",
"AWS_ACCESS_KEY": "access"
})
# If S3_URL isn't a URL, an Exception should be thrown.
self.assertRaises(
BadConfigurationException, StorageFactory.get_storage, {
"name": "walrus",
"AWS_ACCESS_KEY": "access",
"AWS_SECRET_KEY": "secret",
"S3_URL": "1.2.3.4:8773/services/Walrus"
})
# If S3_URL is a URL, that should be fine.
flexmock(boto.s3.connection)
boto.s3.connection.should_receive('S3Connection')
another_walrus = StorageFactory.get_storage({
"name":
"walrus",
"AWS_ACCESS_KEY":
"access",
"AWS_SECRET_KEY":
"secret",
"S3_URL":
"http://1.2.3.4:8773/services/Walrus"
})
self.assertEquals("access", another_walrus.aws_access_key)
self.assertEquals("secret", another_walrus.aws_secret_key)
self.assertEquals("http://1.2.3.4:8773/services/Walrus",
another_walrus.s3_url)
def test_s3_storage_creation_without_necessary_parameters(self):
# Trying to create a S3Storage without the AWS_ACCESS_KEY should fail.
self.assertRaises(BadConfigurationException,
StorageFactory.get_storage, {"name": "s3"})
# Similarly, creating a S3Storage object without the AWS_SECRET_KEY
# should fail.
self.assertRaises(BadConfigurationException,
StorageFactory.get_storage, {
"name": "s3",
"AWS_ACCESS_KEY": "access"
})
# Specifying both should result in the S3Storage object being created,
# and instance variables set with those values.
s3 = StorageFactory.get_storage({
"name": "s3",
"AWS_ACCESS_KEY": "access",
"AWS_SECRET_KEY": "secret"
})
self.assertEquals("access", s3.aws_access_key)
self.assertEquals("secret", s3.aws_secret_key)
def test_azure_storage_creation_without_necessary_parameters(self):
# Trying to create an AzureStorage without the account name should fail.
self.assertRaises(BadConfigurationException, StorageFactory.get_storage, {
"name" : "azure"
})
# Similarly, creating a AzureStorage object without the account key
# should fail.
self.assertRaises(BadConfigurationException, StorageFactory.get_storage, {
"name" : "azure",
"AZURE_ACCOUNT_NAME" : "access"
})
# Specifying both should result in the AzureStorage object being created,
# and instance variables set with those values.
azure = StorageFactory.get_storage({
"name" : "azure",
"AZURE_ACCOUNT_NAME" : "access",
"AZURE_ACCOUNT_KEY" : "secret"
})
self.assertEquals("access", azure.azure_account_name)
self.assertEquals("secret", azure.azure_account_key)
def get(self, path):
""" Downloads a file from a cloud storage platform.
In addition to the arguments below, this method also expects the following
parameters to be posted to it:
name: The name of the cloud storage platform to interact with (e.g.,
's3').
credentials: Any AWS, GCS, Walrus, or Azure credential, that should be
used to authenticate this user.
Args:
path: A str that represents the name of the file to download in the cloud
storage platform. The name of the bucket should be the first item, so
a path of '/mybucket/file/name.txt' indicates that the file
'file/name.txt' should be downloaded from the bucket 'mybucket'.
"""
args = self.get_args_from_request_params(self.request)
if args['name'] == '':
self.response.write(
json.dumps([{
'success': False,
'failure_reason': 'no storage specified'
}]))
return
storage = StorageFactory.get_storage(args)
random_suffix = str(uuid.uuid4()).replace('-', '')[:10]
destination = '/tmp/magik-temp-{0}'.format(random_suffix)
source_to_dest_list = [{'source': path, 'destination': destination}]
result = storage.download_files(source_to_dest_list)
if result[0]['success'] == True:
with open(destination, 'r') as file_handle:
self.response.write(file_handle.read())
os.remove(destination)
else:
self.response.write(json.dumps(result))
return
def put(self, path):
""" Uploads a file to a cloud storage platform.
In addition to the arguments below, this method also expects the following
parameters to be posted to it:
body: The contents of the file to store.
name: The name of the cloud storage platform to interact with (e.g.,
's3').
credentials: Any AWS, GCS, Walrus, or Azure credential, that should be
used to authenticate this user.
Args:
path: A str that represents the name of the file to upload in the cloud
storage platform. The name of the bucket should be the first item, so
a path of '/mybucket/file/name.txt' indicates that the file
'file/name.txt' should be uploaded to the bucket 'mybucket'.
"""
file_contents = self.request.body
if file_contents == '':
self.response.write(json.dumps([{
'success' : False,
'failure_reason' : 'no request body specified'
}]))
return
args = self.get_args_from_request_params(self.request)
storage = StorageFactory.get_storage(args)
source = self.write_temporary_file(file_contents)
source_to_dest_list = [{
'source' : source,
'destination' : path
}]
self.response.write(storage.upload_files(source_to_dest_list))
os.remove(source)
return