def file_action(self):
""" Method for taking actions to and from Hadoop filesystem """
# safe getters, defaults to False if the option is not set
opt_filelist = self.opts.get('filelist', False)
opt_fileput = self.opts.get('fileput', False)
opt_fileget = self.opts.get('fileget', False)
if opt_filelist == True:
self.list_pithos_files()
else:
clusters = get_user_clusters(self.opts['token'])
active_cluster = None
for cluster in clusters:
if (cluster['id'] == self.opts['cluster_id']):
if cluster['hadoop_status'] == const_hadoop_status_started:
active_cluster = cluster
break
else:
logging.error(' You can take file actions on active clusters with started hadoop only.')
exit(error_fatal)
source_path = self.opts['source'].split("/")
self.source_filename = source_path[len(source_path)-1]
if opt_fileput == True:
try:
if is_period(self.opts['destination']) or is_default_dir(self.opts['destination']):
self.opts['destination'] = self.source_filename
file_protocol, remain = get_file_protocol(self.opts['source'], 'fileput', 'source')
self.check_hdfs_destination(active_cluster)
if file_protocol == 'http-ftp':
self.put_from_server()
elif file_protocol == 'file':
self.put_from_local(active_cluster)
elif file_protocol == 'pithos':
kamaki_filespec = remain
self.put_from_pithos(active_cluster,kamaki_filespec)
else:
logging.error(' Error: Unrecognized source filespec.')
exit(error_fatal)
except Exception, e:
logging.error(str(e.args[0]))
exit(error_fatal)
elif opt_fileget == True:
try:
if is_period(self.opts['destination']):
self.opts['destination'] = os.getcwd()
file_protocol, remain = get_file_protocol(self.opts['destination'], 'fileget', 'destination')
if file_protocol == 'pithos':
self.get_from_hadoop_to_pithos(active_cluster, remain)
elif file_protocol == 'file' or file_protocol == "folder":
self.get_from_hadoop_to_local(active_cluster)
else:
logging.error(' Error: Unrecognized destination filespec.')
exit(error_fatal)
except Exception, e:
logging.error(str(e.args[0]))
def replay(argv, token):
# get user's uuid
uuid = get_user_id(token)
# check files's protocol
file_protocol, remain = get_file_protocol(argv, 'fileput', 'source')
try:
if file_protocol == 'pithos':
url = pithos_url + "/" + uuid + remain
headers = {'X-Auth-Token':'{0}'.format(token)}
r=requests.get(url, headers=headers)
if r.status_code == 200:
# load the experiment from pithos file
script = yaml.load(r.text)
else:
print 'File not found on Pithos'
exit(error_fatal)
else:
# load the experiment from local file
with open(argv, 'r') as f:
script = yaml.load(f)
except Exception, e:
print e.strerror
exit(error_fatal)
def file_action(self):
""" Method for taking actions to and from Hadoop filesystem """
# safe getters, defaults to False if the option is not set
opt_filelist = self.opts.get('filelist', False)
opt_fileput = self.opts.get('fileput', False)
opt_fileget = self.opts.get('fileget', False)
opt_filemkdir = self.opts.get('filemkdir', False)
if opt_filelist == True:
self.list_pithos_files()
else:
clusters = get_user_clusters(self.opts['token'], self.opts['server_url'])
active_cluster = None
for cluster in clusters:
if (cluster['id'] == self.opts['cluster_id']):
if cluster['hadoop_status'] == const_hadoop_status_started:
active_cluster = cluster
break
else:
logging.error('You can take file actions on active clusters with started hadoop only.')
exit(error_fatal)
if opt_fileput == True:
try:
sourcesLength = len(self.opts['destination'])
sources = [self.opts['source']]
destination = self.opts['destination'][-1]
if sourcesLength > 1:
if not destination.endswith("/"):
destination += '/'
for source in self.opts['destination'][:-1]:
sources.append(source)
for self.opts['source'] in sources:
self.opts['destination'] = destination
source_path = self.opts['source'].split("/")
self.source_filename = source_path[len(source_path)-1]
if is_period(self.opts['destination']) or is_default_dir(self.opts['destination']):
self.opts['destination'] = self.source_filename
file_protocol, remain = get_file_protocol(self.opts['source'], 'fileput', 'source')
self.check_hdfs_destination(active_cluster)
if file_protocol == 'http-ftp':
self.put_from_server()
elif file_protocol == 'file':
self.put_from_local(active_cluster)
elif file_protocol == 'pithos':
kamaki_filespec = remain
self.put_from_pithos(active_cluster, kamaki_filespec)
else:
logging.error('Unrecognized source filespec.')
exit(error_fatal)
except Exception, e:
stderr.write('{0}'.format('\r'))
logging.error(str(e.args[0]))
exit(error_fatal)
elif opt_fileget == True:
try:
if is_period(self.opts['destination']):
self.opts['destination'] = os.getcwd()
file_protocol, remain = get_file_protocol(self.opts['destination'], 'fileget', 'destination')
if file_protocol == 'pithos':
self.get_from_hadoop_to_pithos(active_cluster, remain)
elif file_protocol == 'file' or file_protocol == "folder":
self.get_from_hadoop_to_local(active_cluster)
else:
logging.error('Unrecognized destination filespec.')
exit(error_fatal)
except Exception, e:
stderr.write('{0}'.format('\r'))
logging.error(str(e.args[0]))
exit(error_fatal)
self.opts['destination'] = os.getcwd()
file_protocol, remain = get_file_protocol(self.opts['destination'], 'fileget', 'destination')
if file_protocol == 'pithos':
self.get_from_hadoop_to_pithos(active_cluster, remain)
elif file_protocol == 'file' or file_protocol == "folder":
self.get_from_hadoop_to_local(active_cluster)
else:
logging.error('Unrecognized destination filespec.')
exit(error_fatal)
except Exception, e:
stderr.write('{0}'.format('\r'))
logging.error(str(e.args[0]))
exit(error_fatal)
elif opt_filemkdir == True:
try:
file_protocol, remain = get_file_protocol(self.opts['directory'], 'filemkdir', 'destination')
if file_protocol == "hdfs":
if self.opts['recursive'] == True:
str_command = " dfs -mkdir -p \"{0}\"".format(remain)
else:
str_command = " dfs -mkdir \"{0}\"".format(remain)
retcode = ssh_call_hadoop("hduser", active_cluster['master_IP'], str_command)
if str(retcode) == str(SUCCESS):
logging.log(SUMMARY, "\"{0}\" created.".format(remain))
exit(SUCCESS)
else:
logging.log(SUMMARY, "\"{0}\" not created. Use -p for a nested destination.".format(remain))
else:
logging.error('Invalid destination filesystem.')
exit(error_fatal)
except Exception, e: