def put_file(self, local_file, remote_path):
base_path = os.path.dirname(remote_path)
filename = os.path.basename(remote_path)
create_file_body = json.dumps({
"action": "mkfile",
"name": filename,
"permissions": FILE_PERMISSIONS
})
response = self.make_call(self.client.post,
base_path,
self.metadata,
data=create_file_body)
if response.status_code != 201:
# TODO: Add better error messages
error_messages = {
404:
'The parent directory does not exist',
409:
'There is a directory entry with the given file name',
422:
'The parent path where the new file or directory is to be created is not a directory'
}
error_message = error_messages.get(response.status_code,
'Unspecified error')
raise ResponseError(
'Cannot create file %s: %s' % (remote_path, error_message),
response)
content_url = response.json()["content"]
response = self.client.put(content_url,
data=local_file,
auth=(self.api_key, self.api_secret))
if response.status_code != 204:
raise ResponseError('Cannot upload file to %s' % content_url,
response)
def get_user_handle(config):
"""Looks up the user handle or raises exception"""
response = CosmosRequests(config.credentials).get(
Routes(config.api_url).profile)
if response.status_code != 200:
raise ResponseError('Cannot access user profile details', response)
return response.json()['handle']
def delete_path(self, path, recursive=False):
r = self.make_call(self.client.delete,
path,
self.metadata,
params=dict(recursive=str(recursive).lower()))
if r.status_code != 204:
raise ResponseError('Cannot delete path %s' % path, r)
def info_command(args, config):
response = CosmosRequests(config.credentials).get(
Routes(config.api_url).info)
if response.status_code != 200:
raise ResponseError("Cannot get general information", response)
print json.dumps(response.json(), sort_keys=True, indent=4)
return 0
def terminate_cluster(self, cluster_id):
"""Request the termination of a cluster"""
r = self.__client.post(
self.__routes.cluster(cluster_id, action="terminate"))
if r.status_code != 200:
raise ResponseError("Cannot terminate cluster %s" % cluster_id, r)
return r.json()
def get_cluster_details(cluster_id, config):
"""Gets the JSON cluster description or raises an exception"""
response = CosmosRequests(config.credentials).get(
Routes(config.api_url).cluster(cluster_id))
if response.status_code == 404:
raise ExitWithError(404, "Cluster %s does not exist" % cluster_id)
if response.status_code != 200:
raise ResponseError("Cannot get cluster details for %s" % cluster_id,
response)
return response.json()
def get_file(self, remote_path, out_file):
listing = self.list_path(remote_path)
if listing is None:
raise OperationError('File %s does not exist' % remote_path)
if listing.path_type != PathTypes.FILE:
raise OperationError("Path %s is not a file" % remote_path)
content_url = listing.metadata['content']
response = self.client.get(content_url,
auth=(self.api_key, self.api_secret),
stream=True)
if response.status_code == 404:
raise ResponseError('File %s does not exist' % remote_path,
response)
if response.status_code != 200:
raise ResponseError('Cannot download file %s' % remote_path,
response)
written = 0
for chunk in response.iter_content(CHUNK_SIZE):
out_file.write(chunk)
written += len(chunk)
return written
def list_path(self, path):
"""Lists a directory or a file. Returns an instance
of DirectoryListing for existing directories,
FileMetadata for existing files or
None when path does not exist."""
r = self.make_call(self.client.get, path, self.metadata)
if r.status_code == 200:
return InfinityClient._file_or_dir(r.json())
elif r.status_code == 404:
return None
else:
raise ResponseError('Cannot list path %s' % path, r)
def test_format_webhdfs_error_messages(self):
ex = ResponseError(
"message",
mock_response(
json={
'RemoteException': {
'message': 'long Java stacktrace'
},
'IrrelevantStuff': 'skipped'
}))
self.assertIn('long Java stacktrace', str(ex))
self.assertNotIn('skipped', str(ex))
def create_cluster(self, cluster_name, cluster_size, services, shared):
"""Send a request to create a new cluster and return the response
in JSON format"""
body = json.dumps({
"name": cluster_name,
"size": cluster_size,
"optionalServices": services,
"shared": shared
})
r = self.__client.post(self.__routes.clusters(), body)
if r.status_code != 201:
raise ResponseError("Cluster creation problem", r)
return r.json()
def chmod(self, remote_path, permissions):
if not re.match('^[0-7]{3}$', permissions):
raise OperationError('%s are not valid permissions' % permissions)
chmod_body = json.dumps({
"action": "chmod",
"permissions": permissions
})
r = self.make_call(self.client.post,
remote_path,
self.metadata,
data=chmod_body)
if r.status_code != 204:
raise ResponseError(
'Cannot change permissions on path %s' % remote_path, r)
def test_format_simple_error_messages(self):
ex = ResponseError("message",
mock_response(json={'error': 'simple error'}))
self.assertIn('simple error', str(ex))
def test_format_non_authorization_errors(self):
ex = ResponseError("message", mock_response(status_code=415))
self.assertIn("HTTP error (415)", str(ex))
def test_format_authorization_errors(self):
ex = ResponseError("message", mock_response(status_code=401))
self.assertIn("Unauthorized request", str(ex))
def test_keep_attributes(self):
message = "trying to foo"
response = mock_response()
ex = ResponseError(message, response)
self.assertEquals(message, ex.message)
self.assertEquals(response, ex.response)
def get_cluster_details(self, cluster_id):
"""Obtain the details of a cluster and return them in JSON format"""
r = self.__client.get(self.__routes.cluster(cluster_id))
if r.status_code != 200:
raise ResponseError("Cannot get details for %s" % cluster_id, r)
return r.json()
def test_format_plain_text_error_messages(self):
ex = ResponseError('message', mock_response(text='plain error'))
self.assertIn('plain error', str(ex))
def get_clusters(self):
"""Obtain the list of clusters and return it in JSON format"""
r = self.__client.get(self.__routes.clusters())
if r.status_code != 200:
raise ResponseError("Cannot list clusters", r)
return r.json()