def test_get_route_with_s3_credentials(self): # Allow the user to store data in Amazon S3 if they specify all the # correct credentials. server = RESTServer() server.request = flexmock() # Presume that the user has only specified S3 credentials. server.request.should_receive('get').with_args('name').and_return('s3') server.request.should_receive('get').with_args('AWS_ACCESS_KEY').and_return( 'access') server.request.should_receive('get').with_args('AWS_SECRET_KEY').and_return( 'secret') server.request.should_receive('get').with_args('GCS_ACCESS_KEY').and_return( '') server.request.should_receive('get').with_args('GCS_SECRET_KEY').and_return( '') server.request.should_receive('get').with_args('S3_URL').and_return('') server.request.should_receive('get').with_args('AZURE_ACCOUNT_NAME') \ .and_return('') server.request.should_receive('get').with_args('AZURE_ACCOUNT_KEY') \ .and_return('') # Mock out writing the file contents that were sent over. flexmock(uuid) uuid.should_receive('uuid4').and_return('123') fake_file = flexmock(name='fake_file') fake_file.should_receive('read').and_return('file contents') fake_builtins = flexmock(sys.modules['__builtin__']) fake_builtins.should_call('open') fake_builtins.should_receive('open').with_args('/tmp/magik-temp-123', 'r') \ .and_return(fake_file) # Mock out interacting with S3. fake_storage = flexmock(name='fake_storage') fake_storage.should_receive('download_files').with_args([{ 'source' : '/baz/gbaz.txt', 'destination' : '/tmp/magik-temp-123' }]).and_return([{ 'success' : True }]) flexmock(StorageFactory) StorageFactory.should_receive('get_storage').with_args(dict).and_return( fake_storage) # Mock out writing the response. server.response = flexmock() server.response.should_receive('write').and_return() # Finally, mock out removing the tempfile we created. flexmock(os) os.should_receive('remove').with_args('/tmp/magik-temp-123') self.assertEquals(None, server.get('/baz/gbaz.txt'))
def test_walrus_storage_creation_without_necessary_parameters(self): # Trying to create a WalrusStorage without the AWS_ACCESS_KEY should fail. self.assertRaises(BadConfigurationException, StorageFactory.get_storage, { "name" : "walrus" }) # Similarly, creating a WalrusStorage object without the AWS_SECRET_KEY # should fail. self.assertRaises(BadConfigurationException, StorageFactory.get_storage, { "name" : "walrus", "AWS_ACCESS_KEY" : "access" }) # If S3_URL isn't a URL, an Exception should be thrown. self.assertRaises(BadConfigurationException, StorageFactory.get_storage, { "name" : "walrus", "AWS_ACCESS_KEY" : "access", "AWS_SECRET_KEY" : "secret", "S3_URL" : "1.2.3.4:8773/services/Walrus" }) # If S3_URL is a URL, that should be fine. flexmock(boto.s3.connection) boto.s3.connection.should_receive('S3Connection') another_walrus = StorageFactory.get_storage({ "name" : "walrus", "AWS_ACCESS_KEY" : "access", "AWS_SECRET_KEY" : "secret", "S3_URL" : "http://1.2.3.4:8773/services/Walrus" }) self.assertEquals("access", another_walrus.aws_access_key) self.assertEquals("secret", another_walrus.aws_secret_key) self.assertEquals("http://1.2.3.4:8773/services/Walrus", another_walrus.s3_url)
def put(self, path): """ Uploads a file to a cloud storage platform. In addition to the arguments below, this method also expects the following parameters to be posted to it: body: The contents of the file to store. name: The name of the cloud storage platform to interact with (e.g., 's3'). credentials: Any AWS, GCS, Walrus, or Azure credential, that should be used to authenticate this user. Args: path: A str that represents the name of the file to upload in the cloud storage platform. The name of the bucket should be the first item, so a path of '/mybucket/file/name.txt' indicates that the file 'file/name.txt' should be uploaded to the bucket 'mybucket'. """ file_contents = self.request.body if file_contents == '': self.response.write( json.dumps([{ 'success': False, 'failure_reason': 'no request body specified' }])) return args = self.get_args_from_request_params(self.request) storage = StorageFactory.get_storage(args) source = self.write_temporary_file(file_contents) source_to_dest_list = [{'source': source, 'destination': path}] self.response.write(storage.upload_files(source_to_dest_list)) os.remove(source) return
def setUp(self): # Set up a mock for when we interact with S3 self.fake_gcs = flexmock(name="fake_gcs") flexmock(boto.gs.connection) boto.gs.connection.should_receive("GSConnection").with_args( gs_access_key_id="access", gs_secret_access_key="secret" ).and_return(self.fake_gcs) self.gcs = StorageFactory.get_storage({"name": "gcs", "GCS_ACCESS_KEY": "access", "GCS_SECRET_KEY": "secret"})
def setUp(self): # Set up a mock for when we interact with S3 self.fake_azure = flexmock(name='fake_azure') flexmock(azure.storage) azure.storage.should_receive('BlobService').with_args('access', 'secret') \ .and_return(self.fake_azure) self.azure = StorageFactory.get_storage({ "name" : "azure", "AZURE_ACCOUNT_NAME" : "access", "AZURE_ACCOUNT_KEY" : "secret" })
def setUp(self): # Set up a mock for when we interact with S3 self.fake_azure = flexmock(name='fake_azure') flexmock(azure.storage) azure.storage.should_receive('BlobService').with_args('access', 'secret') \ .and_return(self.fake_azure) self.azure = StorageFactory.get_storage({ "name": "azure", "AZURE_ACCOUNT_NAME": "access", "AZURE_ACCOUNT_KEY": "secret" })
def setUp(self): # Set up a mock for when we interact with S3 self.fake_s3 = flexmock(name='fake_s3') flexmock(boto.s3.connection) boto.s3.connection.should_receive('S3Connection').with_args( aws_access_key_id='access', aws_secret_access_key='secret') \ .and_return(self.fake_s3) self.s3 = StorageFactory.get_storage({ "name" : "s3", "AWS_ACCESS_KEY" : "access", "AWS_SECRET_KEY" : "secret" })
def setUp(self): # Set up a mock for when we interact with Walrus self.fake_walrus = flexmock(name='fake_walrus') flexmock(boto.s3.connection) boto.s3.connection.should_receive('S3Connection').and_return( self.fake_walrus) self.walrus = StorageFactory.get_storage({ "name" : "walrus", "AWS_ACCESS_KEY" : "access", "AWS_SECRET_KEY" : "secret", "S3_URL" : "http://1.2.3.4:8773/services/Walrus" })
def setUp(self): # Set up a mock for when we interact with S3 self.fake_gcs = flexmock(name='fake_gcs') flexmock(boto.gs.connection) boto.gs.connection.should_receive('GSConnection').with_args( gs_access_key_id='access', gs_secret_access_key='secret') \ .and_return(self.fake_gcs) self.gcs = StorageFactory.get_storage({ "name" : "gcs", "GCS_ACCESS_KEY" : "access", "GCS_SECRET_KEY" : "secret" })
def setUp(self): # Set up a mock for when we interact with S3 self.fake_s3 = flexmock(name='fake_s3') flexmock(boto.s3.connection) boto.s3.connection.should_receive('S3Connection').with_args( aws_access_key_id='access', aws_secret_access_key='secret') \ .and_return(self.fake_s3) self.s3 = StorageFactory.get_storage({ "name": "s3", "AWS_ACCESS_KEY": "access", "AWS_SECRET_KEY": "secret" })
def test_gc_storage_creation_without_necessary_parameters(self): # Trying to create a GCStorage without the GCS_ACCESS_KEY should fail. self.assertRaises(BadConfigurationException, StorageFactory.get_storage, {"name": "gcs"}) # Similarly, creating a GCStorage object without the GCS_SECRET_KEY # should fail. self.assertRaises( BadConfigurationException, StorageFactory.get_storage, {"name": "gcs", "GCS_ACCESS_KEY": "access"} ) # Specifying both should result in the GCStorage object being created, # and instance variables set with those values. gcs = StorageFactory.get_storage({"name": "gcs", "GCS_ACCESS_KEY": "access", "GCS_SECRET_KEY": "secret"}) self.assertEquals("access", gcs.gcs_access_key) self.assertEquals("secret", gcs.gcs_secret_key)
def setUp(self): # Set up a mock for when we interact with Walrus self.fake_walrus = flexmock(name='fake_walrus') flexmock(boto.s3.connection) boto.s3.connection.should_receive('S3Connection').and_return( self.fake_walrus) self.walrus = StorageFactory.get_storage({ "name": "walrus", "AWS_ACCESS_KEY": "access", "AWS_SECRET_KEY": "secret", "S3_URL": "http://1.2.3.4:8773/services/Walrus" })
def delete(self, path): """ Deletes a file from a cloud storage platform. In addition to the arguments below, this method also expects the following parameters to be posted to it: name: The name of the cloud storage platform to interact with (e.g., 's3'). credentials: Any AWS, GCS, Walrus, or Azure credential, that should be used to authenticate this user. Args: path: A str that represents the name of the file to delete from the cloud storage platform. The name of the bucket should be the first item, so a path of '/mybucket/file/name.txt' indicates that the file 'file/name.txt' should be uploaded to the bucket 'mybucket'. """ args = self.get_args_from_request_params(self.request) storage = StorageFactory.get_storage(args) files_to_delete = [{'source': path}] self.response.write(storage.delete_files(files_to_delete))
def get(self, path): """ Downloads a file from a cloud storage platform. In addition to the arguments below, this method also expects the following parameters to be posted to it: name: The name of the cloud storage platform to interact with (e.g., 's3'). credentials: Any AWS, GCS, Walrus, or Azure credential, that should be used to authenticate this user. Args: path: A str that represents the name of the file to download in the cloud storage platform. The name of the bucket should be the first item, so a path of '/mybucket/file/name.txt' indicates that the file 'file/name.txt' should be downloaded from the bucket 'mybucket'. """ args = self.get_args_from_request_params(self.request) if args['name'] == '': self.response.write(json.dumps([{ 'success' : False, 'failure_reason' : 'no storage specified' }])) return storage = StorageFactory.get_storage(args) random_suffix = str(uuid.uuid4()).replace('-', '')[:10] destination = '/tmp/magik-temp-{0}'.format(random_suffix) source_to_dest_list = [{ 'source' : path, 'destination' : destination }] result = storage.download_files(source_to_dest_list) if result[0]['success'] == True: with open(destination, 'r') as file_handle: self.response.write(file_handle.read()) os.remove(destination) else: self.response.write(json.dumps(result)) return
def test_azure_storage_creation_without_necessary_parameters(self): # Trying to create an AzureStorage without the account name should fail. self.assertRaises(BadConfigurationException, StorageFactory.get_storage, {"name": "azure"}) # Similarly, creating a AzureStorage object without the account key # should fail. self.assertRaises(BadConfigurationException, StorageFactory.get_storage, { "name": "azure", "AZURE_ACCOUNT_NAME": "access" }) # Specifying both should result in the AzureStorage object being created, # and instance variables set with those values. azure = StorageFactory.get_storage({ "name": "azure", "AZURE_ACCOUNT_NAME": "access", "AZURE_ACCOUNT_KEY": "secret" }) self.assertEquals("access", azure.azure_account_name) self.assertEquals("secret", azure.azure_account_key)
def delete(self, path): """ Deletes a file from a cloud storage platform. In addition to the arguments below, this method also expects the following parameters to be posted to it: name: The name of the cloud storage platform to interact with (e.g., 's3'). credentials: Any AWS, GCS, Walrus, or Azure credential, that should be used to authenticate this user. Args: path: A str that represents the name of the file to delete from the cloud storage platform. The name of the bucket should be the first item, so a path of '/mybucket/file/name.txt' indicates that the file 'file/name.txt' should be uploaded to the bucket 'mybucket'. """ args = self.get_args_from_request_params(self.request) storage = StorageFactory.get_storage(args) files_to_delete = [{ 'source' : path }] self.response.write(storage.delete_files(files_to_delete))
def test_walrus_storage_creation_without_necessary_parameters(self): # Trying to create a WalrusStorage without the AWS_ACCESS_KEY should fail. self.assertRaises(BadConfigurationException, StorageFactory.get_storage, {"name": "walrus"}) # Similarly, creating a WalrusStorage object without the AWS_SECRET_KEY # should fail. self.assertRaises(BadConfigurationException, StorageFactory.get_storage, { "name": "walrus", "AWS_ACCESS_KEY": "access" }) # If S3_URL isn't a URL, an Exception should be thrown. self.assertRaises( BadConfigurationException, StorageFactory.get_storage, { "name": "walrus", "AWS_ACCESS_KEY": "access", "AWS_SECRET_KEY": "secret", "S3_URL": "1.2.3.4:8773/services/Walrus" }) # If S3_URL is a URL, that should be fine. flexmock(boto.s3.connection) boto.s3.connection.should_receive('S3Connection') another_walrus = StorageFactory.get_storage({ "name": "walrus", "AWS_ACCESS_KEY": "access", "AWS_SECRET_KEY": "secret", "S3_URL": "http://1.2.3.4:8773/services/Walrus" }) self.assertEquals("access", another_walrus.aws_access_key) self.assertEquals("secret", another_walrus.aws_secret_key) self.assertEquals("http://1.2.3.4:8773/services/Walrus", another_walrus.s3_url)
def test_s3_storage_creation_without_necessary_parameters(self): # Trying to create a S3Storage without the AWS_ACCESS_KEY should fail. self.assertRaises(BadConfigurationException, StorageFactory.get_storage, {"name": "s3"}) # Similarly, creating a S3Storage object without the AWS_SECRET_KEY # should fail. self.assertRaises(BadConfigurationException, StorageFactory.get_storage, { "name": "s3", "AWS_ACCESS_KEY": "access" }) # Specifying both should result in the S3Storage object being created, # and instance variables set with those values. s3 = StorageFactory.get_storage({ "name": "s3", "AWS_ACCESS_KEY": "access", "AWS_SECRET_KEY": "secret" }) self.assertEquals("access", s3.aws_access_key) self.assertEquals("secret", s3.aws_secret_key)
def test_azure_storage_creation_without_necessary_parameters(self): # Trying to create an AzureStorage without the account name should fail. self.assertRaises(BadConfigurationException, StorageFactory.get_storage, { "name" : "azure" }) # Similarly, creating a AzureStorage object without the account key # should fail. self.assertRaises(BadConfigurationException, StorageFactory.get_storage, { "name" : "azure", "AZURE_ACCOUNT_NAME" : "access" }) # Specifying both should result in the AzureStorage object being created, # and instance variables set with those values. azure = StorageFactory.get_storage({ "name" : "azure", "AZURE_ACCOUNT_NAME" : "access", "AZURE_ACCOUNT_KEY" : "secret" }) self.assertEquals("access", azure.azure_account_name) self.assertEquals("secret", azure.azure_account_key)
def get(self, path): """ Downloads a file from a cloud storage platform. In addition to the arguments below, this method also expects the following parameters to be posted to it: name: The name of the cloud storage platform to interact with (e.g., 's3'). credentials: Any AWS, GCS, Walrus, or Azure credential, that should be used to authenticate this user. Args: path: A str that represents the name of the file to download in the cloud storage platform. The name of the bucket should be the first item, so a path of '/mybucket/file/name.txt' indicates that the file 'file/name.txt' should be downloaded from the bucket 'mybucket'. """ args = self.get_args_from_request_params(self.request) if args['name'] == '': self.response.write( json.dumps([{ 'success': False, 'failure_reason': 'no storage specified' }])) return storage = StorageFactory.get_storage(args) random_suffix = str(uuid.uuid4()).replace('-', '')[:10] destination = '/tmp/magik-temp-{0}'.format(random_suffix) source_to_dest_list = [{'source': path, 'destination': destination}] result = storage.download_files(source_to_dest_list) if result[0]['success'] == True: with open(destination, 'r') as file_handle: self.response.write(file_handle.read()) os.remove(destination) else: self.response.write(json.dumps(result)) return
def put(self, path): """ Uploads a file to a cloud storage platform. In addition to the arguments below, this method also expects the following parameters to be posted to it: body: The contents of the file to store. name: The name of the cloud storage platform to interact with (e.g., 's3'). credentials: Any AWS, GCS, Walrus, or Azure credential, that should be used to authenticate this user. Args: path: A str that represents the name of the file to upload in the cloud storage platform. The name of the bucket should be the first item, so a path of '/mybucket/file/name.txt' indicates that the file 'file/name.txt' should be uploaded to the bucket 'mybucket'. """ file_contents = self.request.body if file_contents == '': self.response.write(json.dumps([{ 'success' : False, 'failure_reason' : 'no request body specified' }])) return args = self.get_args_from_request_params(self.request) storage = StorageFactory.get_storage(args) source = self.write_temporary_file(file_contents) source_to_dest_list = [{ 'source' : source, 'destination' : path }] self.response.write(storage.upload_files(source_to_dest_list)) os.remove(source) return