def update(cls, cluster_id, cluster_info):
"""
Update the cluster with id `cluster_id` using information provided in
`cluster_info`.
"""
conn = Qubole.agent()
return conn.put(cls.element_path(cluster_id), data=cluster_info)
def terminate(cls, cluster_id_label):
"""
Terminate the cluster with id/label `cluster_id_label`.
"""
conn = Qubole.agent()
data = {"state": "terminate"}
return conn.put(cls.element_path(cluster_id_label) + "/state", data)
def get_results(self, fp=sys.stdout, inline=True, delim=None):
"""
Fetches the result for the command represented by this object
Args:
`fp`: a file object to write the results to directly
"""
result_path = self.meta_data['results_resource']
conn = Qubole.agent()
r = conn.get(result_path, {'inline': inline})
if r.get('inline'):
fp.write(r['results'].encode('utf8'))
else:
acc = Account.find()
boto_conn = boto.connect_s3(
aws_access_key_id=acc.storage_access_key,
aws_secret_access_key=acc.storage_secret_key)
log.info("Starting download from result locations: [%s]" %
",".join(r['result_location']))
#fetch latest value of num_result_dir
num_result_dir = Command.find(self.id).num_result_dir
for s3_path in r['result_location']:
_download_to_local(boto_conn,
s3_path,
fp,
num_result_dir,
delim=delim)
def list(cls, label=None, state=None):
"""
List existing clusters present in your account.
Kwargs:
`label`: list cluster with this label
`state`: list only those clusters which are in this state
Returns:
List of clusters satisfying the given criteria
Raises:
Exception if both label and state options are provided
"""
conn = Qubole.agent()
if label is None and state is None:
return conn.get(cls.rest_entity_path)
elif label is not None and state is None:
cluster_list = conn.get(cls.rest_entity_path)
result = []
for cluster in cluster_list:
if label in cluster['cluster']['label']:
result.append(cluster)
return result
elif label is None and state is not None:
cluster_list = conn.get(cls.rest_entity_path)
result = []
for cluster in cluster_list:
if state.lower() == cluster['cluster']['state'].lower():
result.append(cluster)
return result
else:
raise Exception("Can filter either by label or" +
" by state but not both")
def terminate(cls, cluster_id):
"""
Terminate the cluster with id `cluster_id`.
"""
conn = Qubole.agent()
data = {"state": "terminate"}
return conn.put(cls.element_path(cluster_id) + "/state", data)
def start(cls, cluster_id):
"""
Start the cluster with id `cluster_id`.
"""
conn = Qubole.agent()
data = {"state": "start"}
return conn.put(cls.element_path(cluster_id) + "/state", data)
def start(cls, cluster_id_label):
"""
Start the cluster with id/label `cluster_id_label`.
"""
conn = Qubole.agent()
data = {"state": "start"}
return conn.put(cls.element_path(cluster_id_label) + "/state", data)
def list(cls, label=None, state=None):
"""
List existing clusters present in your account.
Kwargs:
`label`: list cluster with this label
`state`: list only those clusters which are in this state
Returns:
List of clusters satisfying the given criteria
Raises:
Exception if both label and state options are provided
"""
conn = Qubole.agent()
if label is None and state is None:
return conn.get(cls.rest_entity_path)
elif label is not None and state is None:
cluster_list = conn.get(cls.rest_entity_path)
result = []
for cluster in cluster_list:
if label in cluster['cluster']['label']:
result.append(cluster)
return result
elif label is None and state is not None:
cluster_list = conn.get(cls.rest_entity_path)
result = []
for cluster in cluster_list:
if state.lower() == cluster['cluster']['state'].lower():
result.append(cluster)
return result
else:
raise Exception("Can filter either by label or" +
" by state but not both")
def update(cls, cluster_id_label, cluster_info):
"""
Update the cluster with id/label `cluster_id_label` using information provided in
`cluster_info`.
"""
conn = Qubole.agent()
return conn.put(cls.element_path(cluster_id_label), data=cluster_info)
def find(cls, name="default", **kwargs):
if ((name is None) or (name == "default")):
conn = Qubole.agent()
return cls(conn.get(cls.rest_entity_path))
else:
raise ParseError(
"Bad name 'default'",
"Hadoop Clusters can only be named 'default' currently")
def get_log_id(cls, id):
"""
Fetches log for the command represented by this id
Args:
`id`: command id
"""
conn = Qubole.agent()
r = conn.get_raw(cls.element_path(id) + "/logs")
return r.text
def get_log_id(cls, id):
"""
Fetches log for the command represented by this id
Args:
`id`: command id
"""
conn = Qubole.agent()
r = conn.get_raw(cls.element_path(id) + "/logs")
return r.text
def cancel_id(cls, id):
"""
Cancels command denoted by this id
Args:
`id` - command id
"""
conn=Qubole.agent()
data={"status":"kill"}
return conn.put(cls.element_path(id), data)
def cancel_id(cls, id):
"""
Cancels command denoted by this id
Args:
`id` - command id
"""
conn=Qubole.agent()
data={"status":"kill"}
conn.put(cls.element_path(id), data)
def get_log(self):
"""
Fetches log for the command represented by this object
Returns:
The log as a string
"""
log_path = self.meta_data['logs_resource']
conn=Qubole.agent()
r=conn.get_raw(log_path)
return r.text
def get_log(self):
"""
Fetches log for the command represented by this object
Returns:
The log as a string
"""
log_path = self.meta_data['logs_resource']
conn=Qubole.agent()
r=conn.get_raw(log_path)
return r.text
def reassign_label(cls, destination_cluster, label):
"""
Reassign a label from one cluster to another.
Args:
`destination_cluster`: id/label of the cluster to move the label to
`label`: label to be moved from the source cluster
"""
conn = Qubole.agent()
data = {"destination_cluster": destination_cluster, "label": label}
return conn.put(cls.rest_entity_path + "/reassign-label", data)
def save_results_locally(self, local_file_location, boto_client=None):
"""
Saves the results to the passed in file_location
Args:
local_file_location: the place locally where you want to store the file
boto_client: the boto client to use if we're fetching results from s3.
Returns:
dictionary of:
'file_location' : <file path + file name>
'size' : <file size>
'success' : True or False
'error' : error message if there was any
"""
result_path = self.meta_data['results_resource']
conn = Qubole.agent()
r = conn.get(result_path)
if r.get('inline'):
error = ''
success = True
try:
f = open(local_file_location, 'w')
f.write(r['results'])
f.close()
except Exception as e:
error = e
success = False
result = {
'file_location': local_file_location,
'size': os.path.getsize(local_file_location),
'success': success,
'error': error
}
return result
else:
# TODO:finish
s3_locations = r.get('result_location')
if not boto_client:
log.error(
"Unable to download results, no boto client provided.\
Please fetch from S3: %s" % s3_locations)
return {'success': False, 'error': 'Not boto client'}
print s3_locations
result = {
'file_location': local_file_location,
'size': 0, #os.path.getsize(local_file_location),
'success': False,
'error': 'Not implemented'
}
return result
def save_results_locally(self, local_file_location, boto_client=None):
"""
Saves the results to the passed in file_location
Args:
local_file_location: the place locally where you want to store the file
boto_client: the boto client to use if we're fetching results from s3.
Returns:
dictionary of:
'file_location' : <file path + file name>
'size' : <file size>
'success' : True or False
'error' : error message if there was any
"""
result_path = self.meta_data['results_resource']
conn=Qubole.agent()
r = conn.get(result_path)
if r.get('inline'):
error = ''
success = True
try:
f = open(local_file_location, 'w')
f.write(r['results'])
f.close()
except Exception as e:
error = e
success = False
result = {
'file_location': local_file_location,
'size': os.path.getsize(local_file_location),
'success': success,
'error': error
}
return result
else:
# TODO:finish
s3_locations = r.get('result_location')
if not boto_client:
log.error("Unable to download results, no boto client provided.\
Please fetch from S3: %s" % s3_locations)
return {'success':False, 'error': 'Not boto client'}
print s3_locations
result = {
'file_location': local_file_location,
'size': 0, #os.path.getsize(local_file_location),
'success': False,
'error': 'Not implemented'
}
return result
def reassign_label(cls, destination_cluster, label):
"""
Reassign a label from one cluster to another.
Args:
`destination_cluster`: id of the cluster to move the label to
`label`: label to be moved from the source cluster
"""
conn = Qubole.agent()
data = {
"destination_cluster": destination_cluster,
"label": label
}
return conn.put(cls.rest_entity_path + "/reassign-label", data)
def get_results(self):
"""
Fetches the result for the command represented by this object
Returns:
The result as a string
"""
result_path = self.meta_data['results_resource']
conn=Qubole.agent()
r = conn.get(result_path)
if r.get('inline'):
return r['results']
else:
# TODO - this will be implemented in future
log.error("Unable to download results, please fetch from S3")
def list(page=None, per_page=None):
conn = Qubole.agent()
url_path = Action.rest_entity_path
params = {}
if page is not None:
params['page'] = page
if per_page is not None:
params['per_page'] = per_page
#Todo Page numbers are thrown away right now
actjson = conn.get(url_path, params)
actlist = []
for a in actjson["actions"]:
actlist.append(Action(a))
return actlist
def list(page = None, per_page = None):
conn = Qubole.agent()
url_path = Action.rest_entity_path
params = {}
if page is not None:
params['page'] = page
if per_page is not None:
params['per_page'] = per_page
#Todo Page numbers are thrown away right now
actjson = conn.get(url_path, params)
actlist = []
for a in actjson["actions"]:
actlist.append(Action(a))
return actlist
def create(cls, **kwargs):
"""
Create a command object by issuing a POST request to the /command endpoint
Note - this does not wait for the command to complete
Args:
`\**kwargs` - keyword arguments specific to command type
Returns:
Command object
"""
conn=Qubole.agent()
if kwargs.get('command_type') is None:
kwargs['command_type'] = cls.__name__
return cls(conn.post(cls.rest_entity_path, data=kwargs))
def create(cls, **kwargs):
"""
Create a command object by issuing a POST request to the /command endpoint
Note - this does not wait for the command to complete
Args:
`\**kwargs` - keyword arguments specific to command type
Returns:
Command object
"""
conn = Qubole.agent()
if kwargs.get('command_type') is None:
kwargs['command_type'] = cls.__name__
return cls(conn.post(cls.rest_entity_path, data=kwargs))
def get_results(self):
"""
Fetches the result for the command represented by this object
Returns:
The result as a string
"""
result_path = self.meta_data['results_resource']
conn = Qubole.agent()
r = conn.get(result_path)
if r.get('inline'):
return r['results']
else:
# TODO - this will be implemented in future
print r.get('result_location')
log.error("Unable to download results, please fetch from S3")
def list(cls, state=None):
"""
List existing clusters present in your account.
Kwargs:
`state`: list only those clusters which are in this state
Returns:
List of clusters satisfying the given criteria
"""
conn = Qubole.agent()
if state is None:
return conn.get(cls.rest_entity_path)
elif state is not None:
cluster_list = conn.get(cls.rest_entity_path)
result = []
for cluster in cluster_list:
if state.lower() == cluster['cluster']['state'].lower():
result.append(cluster)
return result
def list(cls, state=None):
"""
List existing clusters present in your account.
Kwargs:
`state`: list only those clusters which are in this state
Returns:
List of clusters satisfying the given criteria
"""
conn = Qubole.agent()
if state is None:
return conn.get(cls.rest_entity_path)
elif state is not None:
cluster_list = conn.get(cls.rest_entity_path)
result = []
for cluster in cluster_list:
if state.lower() == cluster['cluster']['state'].lower():
result.append(cluster)
return result
def get_results(self, fp=sys.stdout, inline=True):
"""
Fetches the result for the command represented by this object
@param fp: a file object to write the results to directly
"""
result_path = self.meta_data['results_resource']
conn=Qubole.agent()
r = conn.get(result_path , {'inline': inline})
if r.get('inline'):
fp.write(r['results'].encode('utf8'))
else:
acc = Account.find()
boto_conn = boto.connect_s3(aws_access_key_id=acc.storage_access_key,
aws_secret_access_key=acc.storage_secret_key)
log.info("Starting download from result locations: [%s]" % ",".join(r['result_location']))
for s3_path in r['result_location']:
_download_to_local(boto_conn, s3_path, fp)
def rerun(self):
conn = Qubole.agent()
return conn.post(self.element_path(self.id) + "/rerun", data=None)
def find(cls, name="default", **kwargs):
if (name is None) or (name == "default"):
conn = Qubole.agent()
return cls(conn.get(cls.rest_entity_path))
else:
raise ParseError("Bad name 'default'", "Hadoop Clusters can only be named 'default' currently")
def find(cls, id, **kwargs):
conn = Qubole.agent()
if id is not None:
return cls(conn.get(cls.element_path(id)))
def kill(self):
conn = Qubole.agent()
return conn.put(self.element_path(self.id) + "/kill", data=None)
def create(cls, cluster_info):
"""
Create a new cluster using information provided in `cluster_info`.
"""
conn = Qubole.agent()
return conn.post(cls.rest_entity_path, data=cluster_info)
def rerun(args):
conn = Qubole.agent()
ret_val = conn.post(Action.element_path(args.id) + "/rerun", data=None)
return json.dumps(ret_val, sort_keys=True, indent=4)
def kill(self):
conn = Qubole.agent()
return conn.put(self.element_path(self.id) + "/kill", data=None)
def create(cls, **kwargs):
conn=Qubole.agent()
return cls(conn.post(cls.rest_entity_path, data=kwargs))
def find(cls, **kwargs):
if cls.cached_resource is None:
conn=Qubole.agent()
cls.cached_resource = cls(conn.get(cls.rest_entity_path))
return cls.cached_resource
def delete(cls, cluster_id_label):
"""
Delete the cluster with id/label `cluster_id_label`.
"""
conn = Qubole.agent()
return conn.delete(cls.element_path(cluster_id_label))
def create(cls, cluster_info):
"""
Create a new cluster using information provided in `cluster_info`.
"""
conn = Qubole.agent()
return conn.post(cls.rest_entity_path, data=cluster_info)
def find(cls, id, **kwargs):
conn=Qubole.agent()
if id is not None:
return cls(conn.get(cls.element_path(id)))
def show(cls, cluster_id_label):
"""
Show information about the cluster with id/label `cluster_id_label`.
"""
conn = Qubole.agent()
return conn.get(cls.element_path(cluster_id_label))
def rerun(self):
conn = Qubole.agent()
return conn.post(self.element_path(self.id) + "/rerun", data=None)
def status(cls, cluster_id_label):
"""
Show the status of the cluster with id/label `cluster_id_label`.
"""
conn = Qubole.agent()
return conn.get(cls.element_path(cluster_id_label) + "/state")
def show(cls, cluster_id):
"""
Show information about the cluster with id `cluster_id`.
"""
conn = Qubole.agent()
return conn.get(cls.element_path(cluster_id))
def delete(cls, cluster_id):
"""
Delete the cluster with id `cluster_id`.
"""
conn = Qubole.agent()
return conn.delete(cls.element_path(cluster_id))
def find(cls, **kwargs):
if cls.cached_resource is None:
conn = Qubole.agent()
cls.cached_resource = cls(conn.get(cls.rest_entity_path))
return cls.cached_resource
def status(cls, cluster_id):
"""
Show the status of the cluster with id `cluster_id`.
"""
conn = Qubole.agent()
return conn.get(cls.element_path(cluster_id) + "/state")
def create(cls, **kwargs):
conn = Qubole.agent()
return cls(conn.post(cls.rest_entity_path, data=kwargs))
def rerun(args):
conn = Qubole.agent()
ret_val = conn.post(Action.element_path(args.id) + "/rerun", data=None)
return json.dumps(ret_val, sort_keys=True, indent=4)