def hadoop_action(self):
""" Method for applying an action to a Hadoop cluster"""
action = str.lower(self.opts['hadoop_status'])
clusters = get_user_clusters(self.opts['token'], self.opts['server_url'])
active_cluster = None
for cluster in clusters:
if (cluster['id'] == self.opts['cluster_id']):
active_cluster = cluster
if cluster['cluster_status'] == const_cluster_status_active:
break
else:
logging.error('Hadoop can only be managed for an active cluster.')
exit(error_fatal)
if active_cluster:
if (active_cluster['hadoop_status'] == const_hadoop_status_started and action == "start"):
logging.error('Hadoop already started.')
exit(error_fatal)
elif (active_cluster['hadoop_status'] == const_hadoop_status_stopped and action == "stop"):
logging.error('Hadoop already stopped.')
exit(error_fatal)
try:
payload = {"clusterchoice":{"id": self.opts['cluster_id'], "hadoop_status": action}}
yarn_cluster_req = ClusterRequest(self.escience_token, self.server_url, payload, action='cluster')
response = yarn_cluster_req.create_cluster()
task_id = response['clusterchoice']['task_id']
result = task_message(task_id, self.escience_token, self.server_url, wait_timer_delete)
logging.log(SUMMARY, result)
exit(SUCCESS)
except Exception, e:
stderr.write('{0}'.format('\r'))
logging.error(str(e.args[0]))
exit(error_fatal)
def file_action(self):
""" Method for taking actions to and from Hadoop filesystem """
# safe getters, defaults to False if the option is not set
opt_filelist = self.opts.get('filelist', False)
opt_fileput = self.opts.get('fileput', False)
opt_fileget = self.opts.get('fileget', False)
if opt_filelist == True:
self.list_pithos_files()
else:
clusters = get_user_clusters(self.opts['token'])
active_cluster = None
for cluster in clusters:
if (cluster['id'] == self.opts['cluster_id']):
if cluster['hadoop_status'] == const_hadoop_status_started:
active_cluster = cluster
break
else:
logging.error(' You can take file actions on active clusters with started hadoop only.')
exit(error_fatal)
source_path = self.opts['source'].split("/")
self.source_filename = source_path[len(source_path)-1]
if opt_fileput == True:
try:
if is_period(self.opts['destination']) or is_default_dir(self.opts['destination']):
self.opts['destination'] = self.source_filename
file_protocol, remain = get_file_protocol(self.opts['source'], 'fileput', 'source')
self.check_hdfs_destination(active_cluster)
if file_protocol == 'http-ftp':
self.put_from_server()
elif file_protocol == 'file':
self.put_from_local(active_cluster)
elif file_protocol == 'pithos':
kamaki_filespec = remain
self.put_from_pithos(active_cluster,kamaki_filespec)
else:
logging.error(' Error: Unrecognized source filespec.')
exit(error_fatal)
except Exception, e:
logging.error(str(e.args[0]))
exit(error_fatal)
elif opt_fileget == True:
try:
if is_period(self.opts['destination']):
self.opts['destination'] = os.getcwd()
file_protocol, remain = get_file_protocol(self.opts['destination'], 'fileget', 'destination')
if file_protocol == 'pithos':
self.get_from_hadoop_to_pithos(active_cluster, remain)
elif file_protocol == 'file' or file_protocol == "folder":
self.get_from_hadoop_to_local(active_cluster)
else:
logging.error(' Error: Unrecognized destination filespec.')
exit(error_fatal)
except Exception, e:
logging.error(str(e.args[0]))
def destroy(self):
""" Method for deleting Hadoop clusters in~okeanos."""
clusters = get_user_clusters(self.opts['token'])
for cluster in clusters:
if (cluster['id'] == self.opts['cluster_id']) and cluster['cluster_status'] == const_cluster_status_active:
break
else:
logging.error(' Only active clusters can be destroyed.')
exit(error_fatal)
try:
payload = {"clusterchoice":{"id": self.opts['cluster_id']}}
yarn_cluster_req = ClusterRequest(self.escience_token, payload, action='cluster')
response = yarn_cluster_req.delete_cluster()
task_id = response['clusterchoice']['task_id']
result = task_message(task_id, self.escience_token, wait_timer_delete)
logging.log(SUMMARY, ' Cluster with name "%s" and all its resources deleted' %(result))
except Exception, e:
logging.error(str(e.args[0]))
exit(error_fatal)
def destroy(self):
""" Method for deleting Hadoop clusters in~okeanos."""
clusters = get_user_clusters(self.opts['token'], self.opts['server_url'])
for cluster in clusters:
if (cluster['id'] == self.opts['cluster_id']) and cluster['cluster_status'] == const_cluster_status_active:
break
else:
logging.error('Only active clusters can be destroyed.')
exit(error_fatal)
try:
payload = {"clusterchoice":{"id": self.opts['cluster_id']}}
yarn_cluster_req = ClusterRequest(self.escience_token, self.server_url, payload, action='cluster')
response = yarn_cluster_req.delete_cluster()
task_id = response['clusterchoice']['task_id']
result = task_message(task_id, self.escience_token, self.server_url, wait_timer_delete)
logging.log(SUMMARY, 'Cluster with name "{0}" and all its resources deleted'.format(result))
exit(SUCCESS)
except Exception, e:
stderr.write('{0}'.format('\r'))
logging.error(str(e.args[0]))
exit(error_fatal)
def destroy_vre_machine(self):
""" Method for deleting VRE servers in~okeanos."""
vre_servers = get_user_clusters(self.opts['token'], self.opts['server_url'], choice='vreservers')
for server in vre_servers:
if (server['id'] == self.opts['server_id']) and server['server_status'] == const_cluster_status_active:
break
else:
logging.error('Only active VRE servers can be destroyed.')
exit(error_fatal)
try:
payload = {"vreserver":{"id": self.opts['server_id']}}
yarn_cluster_req = ClusterRequest(self.escience_token, self.server_url, payload, action='vre')
response = yarn_cluster_req.delete_cluster()
task_id = response['vreserver']['task_id']
result = task_message(task_id, self.escience_token, self.server_url, wait_timer_delete)
logging.log(SUMMARY, 'VRE server with name "{0}" and its IP were deleted'.format(result))
exit(SUCCESS)
except Exception, e:
stderr.write('{0}'.format('\r'))
logging.error(str(e.args[0]))
exit(error_fatal)
def list(self):
try:
self.data.extend(get_user_clusters(self.opts['token'], self.opts['server_url']))
except ClientError, e:
logging.error(e.message)
exit(error_fatal)
def file_action(self):
""" Method for taking actions to and from Hadoop filesystem """
# safe getters, defaults to False if the option is not set
opt_filelist = self.opts.get('filelist', False)
opt_fileput = self.opts.get('fileput', False)
opt_fileget = self.opts.get('fileget', False)
opt_filemkdir = self.opts.get('filemkdir', False)
if opt_filelist == True:
self.list_pithos_files()
else:
clusters = get_user_clusters(self.opts['token'], self.opts['server_url'])
active_cluster = None
for cluster in clusters:
if (cluster['id'] == self.opts['cluster_id']):
if cluster['hadoop_status'] == const_hadoop_status_started:
active_cluster = cluster
break
else:
logging.error('You can take file actions on active clusters with started hadoop only.')
exit(error_fatal)
if opt_fileput == True:
try:
sourcesLength = len(self.opts['destination'])
sources = [self.opts['source']]
destination = self.opts['destination'][-1]
if sourcesLength > 1:
if not destination.endswith("/"):
destination += '/'
for source in self.opts['destination'][:-1]:
sources.append(source)
for self.opts['source'] in sources:
self.opts['destination'] = destination
source_path = self.opts['source'].split("/")
self.source_filename = source_path[len(source_path)-1]
if is_period(self.opts['destination']) or is_default_dir(self.opts['destination']):
self.opts['destination'] = self.source_filename
file_protocol, remain = get_file_protocol(self.opts['source'], 'fileput', 'source')
self.check_hdfs_destination(active_cluster)
if file_protocol == 'http-ftp':
self.put_from_server()
elif file_protocol == 'file':
self.put_from_local(active_cluster)
elif file_protocol == 'pithos':
kamaki_filespec = remain
self.put_from_pithos(active_cluster, kamaki_filespec)
else:
logging.error('Unrecognized source filespec.')
exit(error_fatal)
except Exception, e:
stderr.write('{0}'.format('\r'))
logging.error(str(e.args[0]))
exit(error_fatal)
elif opt_fileget == True:
try:
if is_period(self.opts['destination']):
self.opts['destination'] = os.getcwd()
file_protocol, remain = get_file_protocol(self.opts['destination'], 'fileget', 'destination')
if file_protocol == 'pithos':
self.get_from_hadoop_to_pithos(active_cluster, remain)
elif file_protocol == 'file' or file_protocol == "folder":
self.get_from_hadoop_to_local(active_cluster)
else:
logging.error('Unrecognized destination filespec.')
exit(error_fatal)
except Exception, e:
stderr.write('{0}'.format('\r'))
logging.error(str(e.args[0]))
exit(error_fatal)
