Beispiel #1
0
 def file_action(self):
     """ Method for taking actions to and from Hadoop filesystem """
     # safe getters, defaults to False if the option is not set
     opt_filelist = self.opts.get('filelist', False)
     opt_fileput = self.opts.get('fileput', False)
     opt_fileget = self.opts.get('fileget', False)
     if opt_filelist == True:
         self.list_pithos_files()
     else:
         clusters = get_user_clusters(self.opts['token'])
         active_cluster = None
         for cluster in clusters:
             if (cluster['id'] == self.opts['cluster_id']):
                 if cluster['hadoop_status'] == const_hadoop_status_started:
                     active_cluster = cluster
                     break
         else:
             logging.error(' You can take file actions on active clusters with started hadoop only.')
             exit(error_fatal)
         source_path = self.opts['source'].split("/")
         self.source_filename = source_path[len(source_path)-1]
         if opt_fileput == True:
             try:
                 if is_period(self.opts['destination']) or is_default_dir(self.opts['destination']):
                     self.opts['destination'] = self.source_filename
                 file_protocol, remain = get_file_protocol(self.opts['source'], 'fileput', 'source')
                 self.check_hdfs_destination(active_cluster)
                 if file_protocol == 'http-ftp':
                     self.put_from_server()
                 elif file_protocol == 'file':
                     self.put_from_local(active_cluster)
                 elif file_protocol == 'pithos':
                     kamaki_filespec = remain
                     self.put_from_pithos(active_cluster,kamaki_filespec)
                 else:
                     logging.error(' Error: Unrecognized source filespec.')
                     exit(error_fatal)
             except Exception, e:
                 logging.error(str(e.args[0]))
                 exit(error_fatal)
         elif opt_fileget == True:
             try:
                 if is_period(self.opts['destination']):
                     self.opts['destination'] = os.getcwd()
                 file_protocol, remain = get_file_protocol(self.opts['destination'], 'fileget', 'destination')
                 if file_protocol == 'pithos':
                     self.get_from_hadoop_to_pithos(active_cluster, remain)
                 elif file_protocol == 'file' or file_protocol == "folder":
                     self.get_from_hadoop_to_local(active_cluster)
                 else:
                     logging.error(' Error: Unrecognized destination filespec.')
                     exit(error_fatal)
             except Exception, e:
                 logging.error(str(e.args[0]))
def replay(argv, token):
    
    # get user's uuid
    uuid = get_user_id(token)
    
    # check files's protocol
    file_protocol, remain = get_file_protocol(argv, 'fileput', 'source')

    try:
        if file_protocol == 'pithos':
            url = pithos_url + "/" + uuid + remain
            headers = {'X-Auth-Token':'{0}'.format(token)}
            r=requests.get(url, headers=headers)
            if r.status_code == 200:
                # load the experiment from pithos file
                script = yaml.load(r.text)
            else:
                print 'File not found on Pithos'
                exit(error_fatal)
        else:
            # load the experiment from local file
            with open(argv, 'r') as f:
                script = yaml.load(f)
    except Exception, e:
        print e.strerror
        exit(error_fatal)
Beispiel #3
0
 def file_action(self):
     """ Method for taking actions to and from Hadoop filesystem """
     # safe getters, defaults to False if the option is not set
     opt_filelist = self.opts.get('filelist', False)
     opt_fileput = self.opts.get('fileput', False)
     opt_fileget = self.opts.get('fileget', False)
     opt_filemkdir = self.opts.get('filemkdir', False)
     if opt_filelist == True:
         self.list_pithos_files()
     else:
         clusters = get_user_clusters(self.opts['token'], self.opts['server_url'])
         active_cluster = None
         for cluster in clusters:
             if (cluster['id'] == self.opts['cluster_id']):
                 if cluster['hadoop_status'] == const_hadoop_status_started:
                     active_cluster = cluster
                     break
         else:
             logging.error('You can take file actions on active clusters with started hadoop only.')
             exit(error_fatal)
         if opt_fileput == True:
             try:
                 sourcesLength = len(self.opts['destination'])
                 sources = [self.opts['source']]
                 destination = self.opts['destination'][-1]
                 if sourcesLength > 1:
                     if not destination.endswith("/"):
                         destination += '/'
                     for source in self.opts['destination'][:-1]:
                         sources.append(source)
                 for self.opts['source'] in sources:
                     self.opts['destination'] = destination
                     source_path = self.opts['source'].split("/")
                     self.source_filename = source_path[len(source_path)-1]
                     if is_period(self.opts['destination']) or is_default_dir(self.opts['destination']):
                         self.opts['destination'] = self.source_filename
                     file_protocol, remain = get_file_protocol(self.opts['source'], 'fileput', 'source')
                     self.check_hdfs_destination(active_cluster)
                     if file_protocol == 'http-ftp':
                         self.put_from_server()
                     elif file_protocol == 'file':
                         self.put_from_local(active_cluster)
                     elif file_protocol == 'pithos':
                         kamaki_filespec = remain
                         self.put_from_pithos(active_cluster, kamaki_filespec)
                     else:
                         logging.error('Unrecognized source filespec.')
                         exit(error_fatal)
                     
             except Exception, e:
                 stderr.write('{0}'.format('\r'))
                 logging.error(str(e.args[0]))
                 exit(error_fatal)
         elif opt_fileget == True:
             try:
                 if is_period(self.opts['destination']):
                     self.opts['destination'] = os.getcwd()
                 file_protocol, remain = get_file_protocol(self.opts['destination'], 'fileget', 'destination')
                 if file_protocol == 'pithos':
                     self.get_from_hadoop_to_pithos(active_cluster, remain)
                 elif file_protocol == 'file' or file_protocol == "folder":
                     self.get_from_hadoop_to_local(active_cluster)
                 else:
                     logging.error('Unrecognized destination filespec.')
                     exit(error_fatal)
             except Exception, e:
                 stderr.write('{0}'.format('\r'))
                 logging.error(str(e.args[0]))
                 exit(error_fatal)
Beispiel #4
0
             self.opts['destination'] = os.getcwd()
         file_protocol, remain = get_file_protocol(self.opts['destination'], 'fileget', 'destination')
         if file_protocol == 'pithos':
             self.get_from_hadoop_to_pithos(active_cluster, remain)
         elif file_protocol == 'file' or file_protocol == "folder":
             self.get_from_hadoop_to_local(active_cluster)
         else:
             logging.error('Unrecognized destination filespec.')
             exit(error_fatal)
     except Exception, e:
         stderr.write('{0}'.format('\r'))
         logging.error(str(e.args[0]))
         exit(error_fatal)
 elif opt_filemkdir == True:
     try:
         file_protocol, remain = get_file_protocol(self.opts['directory'], 'filemkdir', 'destination')
         if file_protocol == "hdfs":
             if self.opts['recursive'] == True:
                 str_command = " dfs -mkdir -p \"{0}\"".format(remain)
             else:
                 str_command = " dfs -mkdir \"{0}\"".format(remain)
             retcode = ssh_call_hadoop("hduser", active_cluster['master_IP'], str_command)
             if str(retcode) == str(SUCCESS):
                 logging.log(SUMMARY, "\"{0}\" created.".format(remain))
                 exit(SUCCESS)
             else:
                 logging.log(SUMMARY, "\"{0}\" not created. Use -p for a nested destination.".format(remain))
         else:
             logging.error('Invalid destination filesystem.')
             exit(error_fatal)
     except Exception, e: