def kill_batch_job(cluster_name, job_id, log, function=None): ''' Kill a batch job in the cluster. ''' # initialize the control variable OK = True # create the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Establishing connection with cluster {0} ...\n'.format( cluster_name)) (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, node_name='master') if OK: log.write('The connection is established.\n') else: for error in error_list: log.write(f'{error}\n') # kill the job in the cluster if OK: log.write(f'{xlib.get_separator()}\n') sge_env = get_sge_env() command = '{0}; qdel {1}'.format(sge_env, job_id) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: if len(stdout) > 0: for line in stdout: log.write('{0}\n'.format(line)) else: log.write('The job is been killed.\n') if len(stderr) > 0: for line in stderr: log.write('{0}\n'.format(line)) else: log.write('*** ERROR: Wrong command ---> {0}.\n'.format(command)) # close the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write( 'Closing connection with cluster {0} ...\n'.format(cluster_name)) xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write(f'{xlib.get_separator()}\n') log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def combobox_cluster_name_selected_item(self, event=None): ''' Process the event when an item of "combobox_cluster_name" has been selected ''' # set cursor to show busy status self.main.config(cursor='watch') self.main.update() # verify if the cluster name selected is different to the previous cluster name if self.wrapper_cluster_name.get() != self.cluster_name_ant: # close SSH client connection if self.cluster_name_ant is not None: xssh.close_ssh_client_connection(self.ssh_client) # create the SSH client connection (OK, error_list, self.ssh_client) = xssh.create_ssh_client_connection(self.wrapper_cluster_name.get(), 'master') if not OK: message = '' for error in error_list: message = '{0}{1}\n'.format(message, error) tkinter.messagebox.showerror('{0} - {1}'.format(xlib.get_project_name(), self.head), message) self.close() # save current cluster name as previous cluster name self.cluster_name_ant = self.wrapper_cluster_name.get() # load data in "combobox_experiment_id" self.populate_combobox_experiment_id() # set cursor to show normal status self.main.config(cursor='') self.main.update()
def get_database_file_name_list(cluster_name, database_dataset_id, file_type, passed_connection=False, ssh_client=None): ''' Get a list of the database file names in a database dataset of the cluster. ''' # initialize the control variable and the error list OK = True error_list = [] # initialize the dictionary of the database datasets database_file_name_list = [] # create the SSH client connection if not passed_connection: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name) # check the app directory is created if OK: command = '[ -d {0} ] && echo RC=0 || echo RC=1'.format( xlib.get_cluster_database_dir()) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if stdout[len(stdout) - 1] != 'RC=0': error_list.append( '*** ERROR: There is not any volume mounted in the database directory.\n' ) error_list.append( 'You have to link a volume in the mounting point {0} for the cluster {1}.\n' .format(xlib.get_cluster_database_dir(), cluster_name)) OK = False # build the list of the database file name of the database dataset if OK: if file_type == 'all': command = 'ls {0}'.format( xlib.get_cluster_database_dataset_dir(database_dataset_id)) else: command = 'ls {0}/*{1}'.format( xlib.get_cluster_database_dataset_dir(database_dataset_id), file_type) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: for line in stdout: line = line.rstrip('\n') if line != 'lost+found': database_file_name_list.append(os.path.basename(line)) # close the SSH client connection if OK and not passed_connection: xssh.close_ssh_client_connection(ssh_client) # return the control variable, error list and list of the database file names return (OK, error_list, database_file_name_list)
def get_reference_dataset_dict(cluster_name, passed_connection=False, ssh_client=None): ''' Get a dictionary with the reference datasets in the cluster. ''' # initialize the control variable and the error list OK = True error_list = [] # get the reference directory in the cluster cluster_reference_dir = xlib.get_cluster_reference_dir() # initialize the dictionary of the reference datasets reference_dataset_dict = {} # create the SSH client connection if not passed_connection: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name) # check the app directory is created if OK: command = '[ -d {0} ] && echo RC=0 || echo RC=1'.format( cluster_reference_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if stdout[len(stdout) - 1] != 'RC=0': error_list.append( '*** ERROR: There is not any volume mounted in the reference directory.\n' ) error_list.append( 'You have to link a volume in the mounting point {0} for the cluster {1}.\n' .format(cluster_reference_dir, cluster_name)) OK = False # build the dictionary of the reference datasets if OK: command = 'ls {0}'.format(cluster_reference_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: for line in stdout: line = line.rstrip('\n') if line != 'lost+found': reference_dataset_id = line reference_dataset_name = reference_dataset_id reference_dataset_dict[reference_dataset_id] = { 'reference_dataset_id': reference_dataset_id, 'reference_dataset_name': reference_dataset_name } # close the SSH client connection if OK and not passed_connection: xssh.close_ssh_client_connection(ssh_client) # return the control variable, error list and dictionary of the reference datasets return (OK, error_list, reference_dataset_dict)
def get_reference_file_name_list(cluster_name, reference_dataset_id, passed_connection=False, ssh_client=None): ''' Get a list of the reference file names in a reference dataset of the cluster. ''' # initialize the control variable and the error list OK = True error_list = [] # get the reference directory in the cluster cluster_reference_dir = xlib.get_cluster_reference_dir() # initialize the dictionary of the reference datasets reference_file_name_list = [] # create the SSH client connection if not passed_connection: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name) # check the reference directory is created if OK: command = '[ -d {0} ] && echo RC=0 || echo RC=1'.format( cluster_reference_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if stdout[len(stdout) - 1] != 'RC=0': error_list.append( '*** ERROR: There is not any volume mounted in the reference directory.\n' ) error_list.append( 'You have to link a volume in the mounting point {0} for the cluster {1}.\n' .format(cluster_reference_dir, cluster_name)) OK = False # get the reference dataset directory reference_dataset_dir = xlib.get_cluster_reference_dataset_dir( reference_dataset_id) # build the list of the reference file name of the reference dataset if OK: command = 'find {0} -maxdepth 1 -type f'.format(reference_dataset_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: for line in stdout: line = line.rstrip('\n') if line != 'lost+found': reference_file_name_list.append(os.path.basename(line)) # close the SSH client connection if OK and not passed_connection: xssh.close_ssh_client_connection(ssh_client) # return the control variable, error list and list of the reference file names return (OK, error_list, reference_file_name_list)
def show_cluster_composition(cluster_name, log, function=None): ''' Show cluster information of every node: OS, CPU number and memory. ''' # initialize the control variable OK = True # create the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Establishing connection with cluster {0} ...\n'.format( cluster_name)) (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, node_name='master') if OK: log.write('The connection is established.\n') else: for error in error_list: log.write(f'{error}\n') # show the cluster composition if OK: log.write(f'{xlib.get_separator()}\n') sge_env = get_sge_env() command = '{0}; qhost'.format(sge_env) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: if len(stdout) > 0: for line in stdout: log.write('{0}\n'.format(line)) else: log.write('Cluster composition not found.\n') else: log.write('*** ERROR: Wrong command ---> {0}.\n'.format(command)) # close the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write( 'Closing connection with cluster {0} ...\n'.format(cluster_name)) xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write(f'{xlib.get_separator()}\n') log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def form_kill_batch_job(): ''' Kill a batch job in the cluster. ''' # initialize the control variable OK = True # print the header clib.clear_screen() clib.print_headers_with_environment('Cluster operation - Kill batch job') # get the cluster name, experiment identification, read dataset identification and the file pattern print(xlib.get_separator()) if xec2.get_running_cluster_list(volume_creator_included=False) == []: print('WARNING: There is not any running cluster.') OK = False else: cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True) # create the SSH client connection if OK: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master') for error in error_list: print(error) # get the batch job identificaction if OK: batch_job_id = cinputs.input_batch_job_id(ssh_client, help=True) if batch_job_id == '': print('WARNING: There is not any batch job.') OK = False # confirm the kill of the batch job if OK: print(xlib.get_separator()) OK = clib.confirm_action('The batch job {0} is going to be killed.'.format(batch_job_id)) # kill the batch job if OK: devstdout = xlib.DevStdOut(xcluster.kill_batch_job.__name__) xcluster.kill_batch_job(cluster_name, batch_job_id, devstdout, function=None) # close the SSH client connection if OK: xssh.close_ssh_client_connection(ssh_client) # show continuation message print(xlib.get_separator()) input('Press [Intro] to continue ...')
def form_view_cluster_experiment_process_log(): ''' View the log of an experiment process in the cluster. ''' # initialize the control variable OK = True # print the header clib.clear_screen() clib.print_headers_with_environment('Logs - View an experiment process log in the cluster') # get the clustner name if OK: print(xlib.get_separator()) if xec2.get_running_cluster_list(volume_creator_included=False) != []: cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True) else: print('WARNING: There is not any running cluster.') OK = False # create the SSH client connection if OK: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master') for error in error_list: log.write('{0}\n'.format(error)) # create the SSH transport connection if OK: (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name, 'master') for error in error_list: log.write('{0}\n'.format(error)) # create the SFTP client if OK: sftp_client = xssh.create_sftp_client(ssh_transport) # get the experiment identification if OK: experiment_id = cinputs.input_experiment_id(ssh_client, help=True) if experiment_id == '': print('WARNING: The cluster has not experiment data.') OK = False # get the result_dataset identification if OK: result_dataset_id = cinputs.input_result_dataset_id('uncompressed', ssh_client, experiment_id, help=True) if result_dataset_id == '': print('WARNING: The experiment {0} has not result datasets.'.format(experiment_id)) OK = False # create the local path if not os.path.exists(xlib.get_temp_dir()): os.makedirs(xlib.get_temp_dir()) # get the log file name and build local and cluster paths if OK: log_file = xlib.get_cluster_log_file() local_path = '{0}/{1}'.format(xlib.get_temp_dir(), log_file) cluster_path = '{0}/{1}/{2}'.format(xlib.get_cluster_experiment_result_dir(experiment_id), result_dataset_id, log_file) # download the log file from the cluster if OK: print(xlib.get_separator()) print('The file {0} is being downloaded from {1} ...'.format(log_file, cluster_path)) OK = xssh.get_file(sftp_client, cluster_path, local_path) if OK: print('The file has been uploaded.') # close the SSH transport connection if OK: xssh.close_ssh_transport_connection(ssh_transport) # close the SSH client connection if OK: xssh.close_ssh_client_connection(ssh_client) # view the log file if OK: text = 'Logs - View an experiment process log in the cluster' OK = clib.view_file(local_path, text) # show continuation message print(xlib.get_separator()) input('Press [Intro] to continue ...')
def upload_database_dataset(cluster_name, log, function=None): ''' Upload the database dataset to the cluster. ''' # initialize the control variable OK = True # warn that the log window does not have to be closed if not isinstance(log, xlib.DevStdOut): log.write( 'This process might take several minutes. Do not close this window, please wait!\n' ) # check the database transfer config file log.write(f'{xlib.get_separator()}\n') log.write('Checking the database transfer config file ...\n') if check_database_transfer_config_file(strict=True): log.write('The file is OK.\n') else: log.write( '*** ERROR: The database transfer config file is not valid.\n') log.write('Please correct this file or recreate the config files.\n') OK = False # create the SSH client connection if OK: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name) for error in error_list: log.write(f'{error}\n') # create the SSH transport connection if OK: (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name) for error in error_list: log.write(f'{error}\n') # create the SFTP client if OK: sftp_client = xssh.create_sftp_client(ssh_transport) # upload the database dataset if OK: # get the option dictionary database_transfer_options_dict = xlib.get_option_dict( get_database_transfer_config_file()) # get the database dataset identification and the local directory of the database files database_dataset_id = database_transfer_options_dict['identification'][ 'database_dataset_id'] local_dir = database_transfer_options_dict['identification'][ 'local_dir'] # set the cluster database directory cluster_database_dir = '{0}/{1}'.format( xlib.get_cluster_database_dir(), database_dataset_id) # create the data directory in the cluster log.write(f'{xlib.get_separator()}\n') log.write( 'The database directory {0} in the cluster is being created ...\n'. format(cluster_database_dir)) command = 'mkdir --parents {0}'.format(cluster_database_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The directory is created.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # get the sections list sections_list = [] for section in database_transfer_options_dict.keys(): sections_list.append(section) sections_list.sort() # for each section "file-n" for section in sections_list: # check than the section identification is like file-n if re.match('^file-[0-9]+$', section): # get the file name file_name = database_transfer_options_dict[section][ 'file_name'] # set the local path and cluster path local_path = '{0}/{1}'.format(local_dir, file_name) cluster_path = '{0}/{1}'.format(cluster_database_dir, file_name) # upload the database file to the cluster log.write(f'{xlib.get_separator()}\n') log.write('The file {0} is being uploaded to {1} ...\n'.format( file_name, cluster_database_dir)) (OK, error_list) = xssh.put_file(sftp_client, local_path, cluster_path) if OK: log.write('The file has been uploaded.\n') else: for error in error_list: log.write(f'{error}\n') break # close the SSH transport connection if OK: xssh.close_ssh_transport_connection(ssh_transport) # close the SSH client connection if OK: xssh.close_ssh_client_connection(ssh_client) # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write(f'{xlib.get_separator()}\n') log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def install_node_infrastructure_software_ant(cluster_name, node_name, log): ''' Install infrastructure software in a node. ''' # initialize the control variable OK = True # create the SSH client connection log.write('{0}\n'.format(xlib.get_separator())) log.write('Connecting the SSH client in node {0} ...\n'.format(node_name)) (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, node_name) if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # update file /etc/apt/sources.list if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Updating the file /etc/apt/sources.list in node {0} ...\n'.format( node_name)) command = 'sed -i "s/us-east-1\.ec2\.archive\.ubuntu\.com/old-releases\.ubuntu\.com/g" /etc/apt/sources.list' (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) log.write('command: {0}\n'.format(command)) log.write('OK: {0}\n'.format(OK)) log.write('stdout: {0}\n'.format(stdout)) log.write('stderr: {0}\n'.format(stderr)) OK = True if OK: command = 'sed -i "s/security\.ubuntu\.com/old-releases\.ubuntu\.com/g" /etc/apt/sources.list' (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) log.write('command: {0}\n'.format(command)) log.write('OK: {0}\n'.format(OK)) log.write('stdout: {0}\n'.format(stdout)) log.write('stderr: {0}\n'.format(stderr)) OK = True if OK: command = 'export DEBIAN_FRONTEND=noninteractive; apt-get update' (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) log.write('command: {0}\n'.format(command)) log.write('OK: {0}\n'.format(OK)) log.write('stdout: {0}\n'.format(stdout)) log.write('stderr: {0}\n'.format(stderr)) OK = True if OK: log.write('The file is updated.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # install libtbb2 if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Installing the package libtbb2 in node {0} (it can be a very slow process, please be patient) ...\n' .format(node_name)) command = 'export DEBIAN_FRONTEND=noninteractive; apt-get --assume-yes --force-yes install libtbb2' (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) log.write('command: {0}\n'.format(command)) log.write('OK: {0}\n'.format(OK)) log.write('stdout: {0}\n'.format(stdout)) log.write('stderr: {0}\n'.format(stderr)) OK = True if OK: log.write('The package is installed.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) ## install OpenJDK8 #if OK: # log.write('{0}\n'.format(xlib.get_separator())) # log.write('Installing the package OpenJDK8 in node {0} (it can be a very slow process, please be patient) ...\n'.format(node_name)) # command = 'echo "deb http://ppa.launchpad.net/openjdk-r/ppa/ubuntu trusty main" | tee /etc/apt/sources.list.d/openjdk-r.list' # (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # log.write('command: {0}\n'.format(command)) # log.write('OK: {0}\n'.format(OK)) # log.write('stdout: {0}\n'.format(stdout)) # log.write('stderr: {0}\n'.format(stderr)) # OK = True #if OK: # command = 'echo "deb-src http://ppa.launchpad.net/openjdk-r/ppa/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/openjdk-r.list' # (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # log.write('command: {0}\n'.format(command)) # log.write('OK: {0}\n'.format(OK)) # log.write('stdout: {0}\n'.format(stdout)) # log.write('stderr: {0}\n'.format(stderr)) # OK = True #if OK: # #command = 'export DEBIAN_FRONTEND=noninteractive; apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886' # command = 'export DEBIAN_FRONTEND=noninteractive; apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EB9B1D8886F44E2A' # (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # log.write('command: {0}\n'.format(command)) # log.write('OK: {0}\n'.format(OK)) # log.write('stdout: {0}\n'.format(stdout)) # log.write('stderr: {0}\n'.format(stderr)) # OK = True #if OK: # command = 'export DEBIAN_FRONTEND=noninteractive; apt-get update' # (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # log.write('command: {0}\n'.format(command)) # log.write('OK: {0}\n'.format(OK)) # log.write('stdout: {0}\n'.format(stdout)) # log.write('stderr: {0}\n'.format(stderr)) # OK = True #if OK: # command = 'export DEBIAN_FRONTEND=noninteractive; apt-get --assume-yes --force-yes install openjdk-8-jdk' # (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # log.write('command: {0}\n'.format(command)) # log.write('OK: {0}\n'.format(OK)) # log.write('stdout: {0}\n'.format(stdout)) # log.write('stderr: {0}\n'.format(stderr)) # OK = True #if OK: # command = 'export DEBIAN_FRONTEND=noninteractive; update-java-alternatives --set java-1.8.0-openjdk-amd64' # (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # log.write('command: {0}\n'.format(command)) # log.write('OK: {0}\n'.format(OK)) # log.write('stdout: {0}\n'.format(stdout)) # log.write('stderr: {0}\n'.format(stderr)) # OK = True #if OK: # log.write('OpenJDK8 is installed.\n') #else: # log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # install Oracle Java 8 # -- if OK: # -- log.write('{0}\n'.format(xlib.get_separator())) # -- log.write('Installing Oracle Java 8 in node {0} (it can be a very slow process, please be patient) ...\n'.format(node_name)) # -- command = 'echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list' # -- (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # -- log.write('command: {0}\n'.format(command)) # -- log.write('OK: {0}\n'.format(OK)) # -- log.write('stdout: {0}\n'.format(stdout)) # -- log.write('stderr: {0}\n'.format(stderr)) # -- OK = True # -- if OK: # -- command = 'echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list' # -- (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # -- log.write('command: {0}\n'.format(command)) # -- log.write('OK: {0}\n'.format(OK)) # -- log.write('stdout: {0}\n'.format(stdout)) # -- log.write('stderr: {0}\n'.format(stderr)) # -- OK = True # -- if OK: # -- command = 'export DEBIAN_FRONTEND=noninteractive; apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886' # -- (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # -- log.write('command: {0}\n'.format(command)) # -- log.write('OK: {0}\n'.format(OK)) # -- log.write('stdout: {0}\n'.format(stdout)) # -- log.write('stderr: {0}\n'.format(stderr)) # -- OK = True # -- if OK: # -- command = 'export DEBIAN_FRONTEND=noninteractive; apt-get update' # -- (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # -- log.write('command: {0}\n'.format(command)) # -- log.write('OK: {0}\n'.format(OK)) # -- log.write('stdout: {0}\n'.format(stdout)) # -- log.write('stderr: {0}\n'.format(stderr)) # -- OK = True # -- if OK: # -- command = 'echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections' # -- (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # -- log.write('command: {0}\n'.format(command)) # -- log.write('OK: {0}\n'.format(OK)) # -- log.write('stdout: {0}\n'.format(stdout)) # -- log.write('stderr: {0}\n'.format(stderr)) # -- OK = True # -- if OK: # -- command = 'export DEBIAN_FRONTEND=noninteractive; apt-get --assume-yes --force-yes install openjdk-8-jdk' # -- (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # -- log.write('command: {0}\n'.format(command)) # -- log.write('OK: {0}\n'.format(OK)) # -- log.write('stdout: {0}\n'.format(stdout)) # -- log.write('stderr: {0}\n'.format(stderr)) # -- OK = True # -- if OK: # -- log.write('Java 8 is installed.\n') # -- else: # -- log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # close the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # return the control variable return OK
def install_node_infrastructure_software(cluster_name, node_name, log): ''' Install infraestructure software in a node. ''' # initialize the control variable OK = True # get the infrastructure software installation script path in local compute local_script_path = get_infrastructure_software_installation_script() # set the infrastructure software installation script path in node node_script_path = './{0}'.format(os.path.basename(local_script_path)) # set the infrastructure software installation log path in node node_log_path = node_script_path[:node_script_path.find('.sh')] + '.log' # build the infrastructure software installation script if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Building the infrastructure software installation script {0} ...\n' .format(local_script_path)) (OK, error_list ) = build_infrastructure_software_installation_script(cluster_name) if OK: log.write('The file is built.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # create the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Connecting the SSH client to node {0} ...\n'.format(node_name)) (OK, error_list, ssh_client) = xssh.create_ssh_client_connection( cluster_name, node_name) if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # create the SSH transport connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Connecting the SSH transport to node {0} ...\n'.format(node_name)) (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection( cluster_name, node_name) if OK: log.write('The SSH transport is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # create the SFTP client if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Connecting the SFTP client to node {0} ...\n'.format(node_name)) sftp_client = xssh.create_sftp_client(ssh_transport) log.write('The SFTP client is connected.\n') # upload the infraestructe software installation script to the node if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Uploading the infraestructe software installation script to the node {0} ...\n' .format(node_name)) (OK, error_list) = xssh.put_file(sftp_client, local_script_path, node_script_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # set run permision to the infraestructe software installation script in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Setting on the run permision ...\n') command = 'chmod u+x {0}'.format(node_script_path) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # submit the infraestructe software installation script if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Submitting the infraestructe software installation script in the node {0} ...\n' .format(node_name)) command = '{0} &>{1} &'.format(node_script_path, node_log_path) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The script is submitted.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # close the SSH transport connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH transport connection ...\n') xssh.close_ssh_transport_connection(ssh_transport) log.write('The connection is closed.\n') # close the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # return the control variable return OK
def form_show_status_batch_jobs(): ''' Show the status of batch jobs in the cluster. ''' # initialize the control variable OK = True # initialize the list of identification of the batch jobs batch_job_id_list = [] # print the header clib.clear_screen() clib.print_headers_with_environment('Cluster operation - Show status batch jobs') # get the cluster name print(xlib.get_separator()) if xec2.get_running_cluster_list(volume_creator_included=False) != []: cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True) else: print('WARNING: There is not any running cluster.') OK = False # create the SSH client connection if OK: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master') for error in error_list: print(error) # get the batch job dictionary if help: (OK, error_list, batch_job_dict) = xcluster.get_batch_job_dict(ssh_client) # build the list of identifications of the batch jobs if OK and help: for job_id in batch_job_dict.keys(): batch_job_id_list.append(job_id) if batch_job_id_list != []: batch_job_id_list.sort() else: print('WARNING: There is not any batch job.') OK = False # print the batch jobs if OK and help: print(xlib.get_separator()) # set data width job_id_width = 6 job_name_width = 10 state_width = 15 start_date_width = 10 start_time_width = 10 # set line template line_template = '{0:' + str(job_id_width) + '} {1:' + str(job_name_width) + '} {2:' + str(state_width) + '} {3:' + str(start_date_width) + '} {4:' + str(start_time_width) + '}' # print header print(line_template.format('Job id', 'Job name', 'State', 'Start date', 'Start time')) print(line_template.format('=' * job_id_width, '=' * job_name_width, '=' * state_width, '=' * start_date_width, '=' * start_time_width)) # print detail lines for job_id in batch_job_id_list: job_name = batch_job_dict[job_id]['job_name'] state = batch_job_dict[job_id]['state'] start_date = batch_job_dict[job_id]['start_date'] start_time = batch_job_dict[job_id]['start_time'] print(line_template.format(job_id, job_name, state, start_date, start_time)) # close the SSH client connection if OK: xssh.close_ssh_client_connection(ssh_client) # show continuation message print(xlib.get_separator()) input('Press [Intro] to continue ...')
def form_list_cluster_experiment_processes(): ''' List the processes of an experiment in the cluster. ''' # initialize the control variable OK = True # print the header clib.clear_screen() clib.print_headers_with_environment('Logs - List experiment processes in the cluster') # get the cluster name print(xlib.get_separator()) if xec2.get_running_cluster_list(volume_creator_included=False) != []: cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True) else: print('WARNING: There is not any running cluster.') OK = False # create the SSH client connection if OK: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master') for error in error_list: log.write('{0}\n'.format(error)) # get experiment identification if OK: experiment_id = cinputs.input_experiment_id(ssh_client, help=True) if experiment_id == '': print('WARNING: The cluster {0} has not experiment data.'.format(cluster_name)) OK = False # get the result dataset list of the experiment if OK: command = 'cd {0}/{1}; for list in `ls`; do ls -ld $list | grep -v ^- > /dev/null && echo $list; done;'.format(xlib.get_cluster_result_dir(), experiment_id) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: result_dataset_id_list = [] for line in stdout: line = line.rstrip('\n') if line != 'lost+found': result_dataset_id_list.append(line) # print the result dataset identification list of the experiment if OK: print(xlib.get_separator()) if result_dataset_id_list == []: print('*** WARNING: There is not any result dataset of the experiment {0}.'.format(experiment_id)) else: result_dataset_id_list.sort() # set data width result_dataset_width = 25 bioinfo_app_width = 25 # set line template line_template = '{0:' + str(result_dataset_width) + '} {1:' + str(bioinfo_app_width) + '}' # print header print(line_template.format('Result dataset', 'Bioinfo app / Utility')) print(line_template.format('=' * result_dataset_width, '=' * bioinfo_app_width)) # print detail lines for result_dataset_id in result_dataset_id_list: if result_dataset_id.startswith(xlib.get_bedtools_code()+'-'): bioinfo_app_name = xlib.get_bedtools_name() elif result_dataset_id.startswith(xlib.get_blastplus_code()+'-'): bioinfo_app_name = xlib.get_blastplus_name() elif result_dataset_id.startswith(xlib.get_bowtie2_code()+'-'): bioinfo_app_name = xlib.get_bowtie2_name() elif result_dataset_id.startswith(xlib.get_busco_code()+'-'): bioinfo_app_name = xlib.get_busco_name() elif result_dataset_id.startswith(xlib.get_cd_hit_code()+'-'): bioinfo_app_name = xlib.get_cd_hit_est_name() elif result_dataset_id.startswith(xlib.get_cd_hit_code()+'-'): bioinfo_app_name = xlib.get_cd_hit_est_name() elif result_dataset_id.startswith(xlib.get_detonate_code()+'-'): bioinfo_app_name = xlib.get_detonate_name() elif result_dataset_id.startswith(xlib.get_emboss_code()+'-'): bioinfo_app_name = xlib.get_emboss_name() elif result_dataset_id.startswith(xlib.get_fastqc_code()+'-'): bioinfo_app_name = xlib.get_fastqc_name() elif result_dataset_id.startswith(xlib.get_gmap_code()+'-'): bioinfo_app_name = xlib.get_gmap_name() elif result_dataset_id.startswith(xlib.get_gmap_gsnap_code()+'-'): bioinfo_app_name = xlib.get_gmap_gsnap_name() elif result_dataset_id.startswith(xlib.get_gzip_code()+'-'): bioinfo_app_name = xlib.get_gzip_name() elif result_dataset_id.startswith(xlib.get_insilico_read_normalization_code()+'-'): bioinfo_app_name = xlib.get_insilico_read_normalization_name() elif result_dataset_id.startswith(xlib.get_miniconda3_code()+'-'): bioinfo_app_name = xlib.get_miniconda3_name() elif result_dataset_id.startswith(xlib.get_ngshelper_code()+'-'): bioinfo_app_name = xlib.get_ngshelper_name() elif result_dataset_id.startswith(xlib.get_quast_code()+'-'): bioinfo_app_name = xlib.get_quast_name() elif result_dataset_id.startswith(xlib.get_r_code()+'-'): bioinfo_app_name = xlib.get_r_name() elif result_dataset_id.startswith(xlib.get_ref_eval_code()+'-'): bioinfo_app_name = xlib.get_ref_eval_name() elif result_dataset_id.startswith(xlib.get_rnaquast_code()+'-'): bioinfo_app_name = xlib.get_rnaquast_name() elif result_dataset_id.startswith(xlib.get_rsem_code()+'-'): bioinfo_app_name = xlib.get_rsem_name() elif result_dataset_id.startswith(xlib.get_rsem_eval_code()+'-'): bioinfo_app_name = xlib.get_rsem_eval_name() elif result_dataset_id.startswith(xlib.get_samtools_code()+'-'): bioinfo_app_name = xlib.get_samtools_name() elif result_dataset_id.startswith(xlib.get_soapdenovotrans_code()+'-'): bioinfo_app_name = xlib.get_soapdenovotrans_name() elif result_dataset_id.startswith(xlib.get_star_code()+'-'): bioinfo_app_name = xlib.get_star_name() elif result_dataset_id.startswith(xlib.get_transabyss_code()+'-'): bioinfo_app_name = xlib.get_transabyss_name() elif result_dataset_id.startswith(xlib.get_transcript_filter_code()+'-'): bioinfo_app_name = xlib.get_transcript_filter_name() elif result_dataset_id.startswith(xlib.get_transcriptome_blastx_code()+'-'): bioinfo_app_name = xlib.get_transcriptome_blastx_name() elif result_dataset_id.startswith(xlib.get_transrate_code()+'-'): bioinfo_app_name = xlib.get_transrate_name() elif result_dataset_id.startswith(xlib.get_trimmomatic_code()+'-'): bioinfo_app_name = xlib.get_trimmomatic_name() elif result_dataset_id.startswith(xlib.get_trinity_code()+'-'): bioinfo_app_name = xlib.get_trinity_name() else: bioinfo_app_name = 'xxx' print(line_template.format(result_dataset_id, bioinfo_app_name)) # close the SSH client connection if OK: xssh.close_ssh_client_connection(ssh_client) # show continuation message print(xlib.get_separator()) input('Press [Intro] to continue ...')
def create_volume(volume_name, volume_type, volume_size, terminate_indicator, log, function=None): ''' Create a volume in the current zone. ''' # initialize the control variable OK = True # get the volume creator name volume_creator_name = xlib.get_volume_creator_name() # warn that the log window must not be closed if not isinstance(log, xlib.DevStdOut): log.write( 'This process might take a few minutes. Do not close this window, please wait!\n' ) # verify the volume creator is running if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Verifing if volume creator is running ...\n') (master_state_code, master_state_name) = xec2.get_node_state(volume_creator_name, 'master') if master_state_code == 16: log.write('The volume creator is running.\n') else: log.write( '*** WARNING: The volume creator is not running. It will be created.\n' ) (OK, master_state_code, master_state_name) = xcluster.create_cluster(volume_creator_name, volume_creator_name, log, function=None, is_menu_call=False) # get the master node identification if OK: node_id = xec2.get_node_id(volume_creator_name, 'master') if node_id == '': log.write( '*** ERROR: The master identification of the volume creator not has been got.\n' ) OK = False # create the SSH client connection log.write('{0}\n'.format(xlib.get_separator())) log.write('Connecting SSH client ...\n') (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(volume_creator_name, 'master') if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # warn that the requirements are being verified log.write('{0}\n'.format(xlib.get_separator())) log.write('Verifying process requirements ...\n') # get current region and zone names region_name = xconfiguration.get_current_region_name() zone_name = xconfiguration.get_current_zone_name() # verify that the zone is available if OK: if not xec2.is_zone_available(region_name, zone_name): log.write( '*** ERROR: The zone {0} is not available in the region {1}.\n' .format(zone_name, region_name)) OK = False # verify that the volume is not created if OK: if xec2.is_volume_created(volume_name, zone_name): log.write( '*** WARNING: The volume {0} is already created.\n'.format( volume_name)) OK = False # warn that the requirements are OK if OK: log.write('Process requirements are OK.\n') # create the volume if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Creating volume {0} ...\n'.format(volume_name)) (OK, volume_id) = xec2.create_volume(volume_name, volume_type, volume_size) if OK: log.write( 'The volume is created with the identification {0}.\n'.format( volume_id)) else: log.write('*** ERROR: The volume is not created.\n') # wait for the volume status to be available if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Waiting for the volume state to be available ...\n') i = 0 available_state_timeout = 120 while True: i += 1 volume_status = xec2.get_volume_state(volume_name, zone_name) time.sleep(1) if volume_status == 'available': log.write('The volume is now available.\n') break elif i > available_state_timeout: log.write( '*** The volume is not available after {0} s.\n'.format( available_state_timeout)) Ok = False break # set the aws device and get de machine device if OK: aws_device = '/dev/sdp' machine_device = xlib.get_machine_device_file(aws_device) # attach the volume to the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Attaching volume {0} to node master of volume creator ...\n'. format(volume_name)) OK = xec2.attach_volume(node_id, volume_id, aws_device) if OK: log.write('The volume is attached.\n') else: log.write('*** ERROR: The volume is not attached.\n') # wait for the volume attachment to be available if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Waiting for the volume attachment to be available ...\n') i = 0 attachment_timeout = 120 while True: i += 1 volume_attachments = xec2.get_volume_attachments( volume_name, zone_name) if volume_attachments != []: log.write('The volume attachment is now available.\n') break elif i > attachment_timeout: log.write( '*** ERROR: The volume attachment is not available after {0} s.\n' .format(attachment_timeout)) Ok = False break time.sleep(1) # wait for the device availability if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Waiting for the availabity of the device {0} ...\n'.format( machine_device)) i = 0 format_timeout = 120 try: while True: i += 1 command = 'hdparm -z {0}'.format(machine_device) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) command = 'lsblk --list --noheadings --output NAME' (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) for line in stdout: if line == os.path.basename(machine_device): log.write('The device is available.\n') raise xlib.BreakAllLoops if i > format_timeout: log.write( '*** ERROR: The device is not available after {0} s.\n' .format(format_timeout)) OK = False break time.sleep(1) except xlib.BreakAllLoops: pass # format the volume if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Formating volume {0} to ext4 file system type ...\n'.format( volume_name)) command = 'mkfs -t ext4 {0} 2>&1; RC=$?; echo "RC=$RC"'.format( machine_device) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if stdout[len(stdout) - 1] == 'RC=0': log.write('The volume is formatted.\n') OK = True else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) OK = False # detach the volume to the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Detaching volume {0} from node master of volume creator ...\n'. format(volume_name)) OK = xec2.detach_volume(node_id, volume_id, aws_device) if OK: log.write('The volume is detached.\n') else: log.write('*** ERROR: The volume is not detached.\n') # terminate volume creator if OK: if terminate_indicator: OK = xcluster.terminate_cluster(volume_creator_name, True, log, function=None, is_menu_call=False) else: log.write('{0}\n'.format(xlib.get_separator())) log.write('You do not indicate to terminate the volume creator.\n') log.write( 'Remember to terminate it when you finish to create the volumes!!!\n' ) # close the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write('{0}\n'.format(xlib.get_separator())) log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def get_read_dataset_dict(cluster_name, experiment_id, passed_connection=False, ssh_client=None): ''' Get a dictoinary with the read datasets of an experiment in the cluster. ''' # initialize the control variable and the error list OK = True error_list = [] # get the read directory in the cluster cluster_read_dir = xlib.get_cluster_read_dir() # initialize the dictionary of the read datasets read_dataset_dict = {} # create the SSH client connection if not passed_connection: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection( cluster_name, 'master') # verify the read directory is created if OK: command = '[ -d {0} ] && echo RC=0 || echo RC=1'.format( cluster_read_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if stdout[len(stdout) - 1] != 'RC=0': error_list.append( '*** ERROR: There is not any volume mounted in the read directory.\n' ) error_list.append( 'You must link a volume in the mounting point {0} for the template {1}.\n' .format(cluster_read_dir, cluster_name)) OK = False # get the dictionary of the read datasets if OK: command = 'ls {0}/{1}'.format(cluster_read_dir, experiment_id) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: for line in stdout: line = line.rstrip('\n') if line != 'lost+found': read_dataset_id = line if read_dataset_id == xlib.get_uploaded_read_dataset_name( ): read_dataset_name = ' uploaded reads' elif read_dataset_id.startswith( xlib.get_insilico_read_normalization_code()): mo = re.match( '{0}-(.+)-(.+)'.format( xlib.get_insilico_read_normalization_code()), read_dataset_id) date = mo.group(1) time = mo.group(2) read_dataset_name = '{0} ({1} {2})'.format( xlib.get_insilico_read_normalization_name(), date, time) elif read_dataset_id.startswith( xlib.get_trimmomatic_code()): mo = re.match( '{0}-(.+)-(.+)'.format( xlib.get_trimmomatic_code()), read_dataset_id) date = mo.group(1) time = mo.group(2) read_dataset_name = '{0} ({1} {2})'.format( xlib.get_trimmomatic_name(), date, time) else: read_dataset_name = 'xxx' read_dataset_dict[read_dataset_id] = { 'read_dataset_id': read_dataset_id, 'read_dataset_name': read_dataset_name } # close the SSH client connection if OK and not passed_connection: xssh.close_ssh_client_connection(ssh_client) # return the control variable, error list and dictionary of the read datasets return (OK, error_list, read_dataset_dict)
def form_view_cluster_start_log(): ''' View the cluster start log. ''' # initialize the control variable OK = True # print the header clib.clear_screen() clib.print_headers_with_environment('Logs - View the cluster start log') # get the clustner name if OK: print(xlib.get_separator()) if xec2.get_running_cluster_list(only_environment_cluster=True, volume_creator_included=False) != []: cluster_name = cinputs.input_cluster_name( volume_creator_included=False, help=True) else: print('WARNING: There is not any running cluster.') OK = False # create the SSH client connection if OK: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name) for error in error_list: print(error) # create the SSH transport connection if OK: (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name) for error in error_list: print(error) # create the SFTP client if OK: sftp_client = xssh.create_sftp_client(ssh_transport) # create the local path if not os.path.exists(xlib.get_temp_dir()): os.makedirs(xlib.get_temp_dir()) # get the log file name and build local and cluster paths if OK: local_path = f'{xlib.get_temp_dir()}/{os.path.basename(xinstance.get_infrastructure_software_installation_log())}' cluster_path = f'/home/ubuntu/{os.path.basename(xinstance.get_infrastructure_software_installation_log())}' # download the log file from the cluster if OK: print(xlib.get_separator()) print( f'The file {os.path.basename(xinstance.get_infrastructure_software_installation_log())} is being downloaded from {cluster_path} ...' ) OK = xssh.get_file(sftp_client, cluster_path, local_path) if OK: print('The file has been uploaded.') # close the SSH transport connection if OK: xssh.close_ssh_transport_connection(ssh_transport) # close the SSH client connection if OK: xssh.close_ssh_client_connection(ssh_client) # view the log file if OK: text = 'Logs - View the cluster start log' OK = clib.view_file(local_path, text) # show continuation message input('Press [Intro] to continue ...')
def get_result_dataset_dict(cluster_name, experiment_id, status, passed_connection, ssh_client): ''' Get a dictionary with the result datasets of an experiment in the cluster. ''' # initialize the control variable and the error list OK = True error_list = [] # get the result directory in the cluster cluster_result_dir = xlib.get_cluster_result_dir() # initialize the dictionary of the result datasets result_dataset_dict = {} # create the SSH client connection if not passed_connection: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master') # verify the result directory is created if OK: command = '[ -d {0} ] && echo RC=0 || echo RC=1'.format(cluster_result_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if stdout[len(stdout) - 1] != 'RC=0': error_list.append('*** ERROR: There is not any volume mounted in the result directory.\n') error_list.append('You must link a volume in the mounting point {0} for the template {1}.\n'.format(cluster_result_dir, cluster_name)) OK = False # get the dictionary of the result datasets if OK: if status == 'uncompressed': command = 'cd {0}/{1}; for list in `ls`; do ls -ld $list | grep -v ^- > /dev/null && echo $list; done;'.format(cluster_result_dir, experiment_id) elif status == 'compressed': command = 'cd {0}/{1}; for list in `ls`; do ls -ld $list | grep -v ^d > /dev/null && echo $list; done;'.format(cluster_result_dir, experiment_id) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: if status == 'uncompressed': input_pattern = '{0}-(.+)-(.+)' output_pattern = '{0} ({1} {2})' elif status == 'compressed': input_pattern = '{0}-(.+)-(.+).tar.gz' output_pattern = '{0} ({1} {2}) [compressed]' for line in stdout: line = line.rstrip('\n') if line != 'lost+found': result_dataset_id = line if result_dataset_id.startswith(xlib.get_cd_hit_est_code()+'-'): mo = re.match(input_pattern.format(xlib.get_cd_hit_est_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_cd_hit_est_name(), date, time) elif result_dataset_id.startswith(xlib.get_fastqc_code()+'-'): mo = re.match(input_pattern.format(xlib.get_fastqc_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_fastqc_name(), date, time) elif result_dataset_id.startswith(xlib.get_gzip_code()+'-'): mo = re.match(input_pattern.format(xlib.get_gzip_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_gzip_name(), date, time) elif result_dataset_id.startswith(xlib.get_insilico_read_normalization_code()+'-'): mo = re.match(input_pattern.format(xlib.get_insilico_read_normalization_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_insilico_read_normalization_name(), date, time) elif result_dataset_id.startswith(xlib.get_quast_code()+'-'): mo = re.match(input_pattern.format(xlib.get_quast_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_quast_name(), date, time) elif result_dataset_id.startswith(xlib.get_ref_eval_code()+'-'): mo = re.match(input_pattern.format(xlib.get_ref_eval_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_ref_eval_name(), date, time) elif result_dataset_id.startswith(xlib.get_rnaquast_code()+'-'): mo = re.match(input_pattern.format(xlib.get_rnaquast_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_rnaquast_name(), date, time) elif result_dataset_id.startswith(xlib.get_rsem_eval_code()+'-'): mo = re.match(input_pattern.format(xlib.get_rsem_eval_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_rsem_eval_name(), date, time) elif result_dataset_id.startswith(xlib.get_soapdenovotrans_code()+'-'): mo = re.match(input_pattern.format(xlib.get_soapdenovotrans_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_soapdenovotrans_name(), date, time) elif result_dataset_id.startswith(xlib.get_star_code()+'-'): mo = re.match(input_pattern.format(xlib.get_star_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_star_name(), date, time) elif result_dataset_id.startswith(xlib.get_transabyss_code()+'-'): mo = re.match(input_pattern.format(xlib.get_transabyss_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_transabyss_name(), date, time) elif result_dataset_id.startswith(xlib.get_transcript_filter_code()+'-'): mo = re.match(input_pattern.format(xlib.get_transcript_filter_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_transcript_filter_name(), date, time) elif result_dataset_id.startswith(xlib.get_transcriptome_blastx_code()+'-'): mo = re.match(input_pattern.format(xlib.get_transcriptome_blastx_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_transcriptome_blastx_name(), date, time) elif result_dataset_id.startswith(xlib.get_transrate_code()+'-'): mo = re.match(input_pattern.format(xlib.get_transrate_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_transrate_name(), date, time) elif result_dataset_id.startswith(xlib.get_trimmomatic_code()+'-'): mo = re.match(input_pattern.format(xlib.get_trimmomatic_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_trimmomatic_name(), date, time) elif result_dataset_id.startswith(xlib.get_trinity_code()+'-'): mo = re.match(input_pattern.format(xlib.get_trinity_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_trinity_name(), date, time) else: result_dataset_name = result_dataset_id result_dataset_dict[result_dataset_id] = {'result_dataset_id': result_dataset_id, 'result_dataset_name': result_dataset_name} # close the SSH client connection if OK and not passed_connection: xssh.close_ssh_client_connection(ssh_client) # return the control variable, error list and dictionary of the result datasets return (OK, error_list, result_dataset_dict)
def install_node_infrastructure_software(cluster_name, node_name, log): ''' Install infrastructure software in a node. ''' # initialize the control variable OK = True # get the infrastructure software installation script path in local compute local_script_path = get_infrastructure_software_installation_script() # set the infrastructure software installation script path in node node_script_path = f'./{os.path.basename(local_script_path)}' # set the infrastructure software installation log path in node node_log_path = node_script_path[:node_script_path.find('.sh')] + '.log' # build the infrastructure software installation script if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Building the infrastructure software installation script {local_script_path} ...\n' ) (OK, error_list ) = build_infrastructure_software_installation_script(cluster_name) if OK: log.write('The file is built.\n') else: for error in error_list: log.write(f'{error}\n') # create the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write(f'Connecting the SSH client to node {node_name} ...\n') (OK, error_list, ssh_client) = xssh.create_ssh_client_connection( cluster_name, node_name) if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write(f'{error}\n') # create the SSH transport connection if OK: log.write(f'{xlib.get_separator()}\n') log.write(f'Connecting the SSH transport to node {node_name} ...\n') (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection( cluster_name, node_name) if OK: log.write('The SSH transport is connected.\n') else: for error in error_list: log.write(f'{error}\n') # create the SFTP client if OK: log.write(f'{xlib.get_separator()}\n') log.write(f'Connecting the SFTP client to node {node_name} ...\n') sftp_client = xssh.create_sftp_client(ssh_transport) log.write('The SFTP client is connected.\n') # download the AWSCLI2 compressed file to local computer if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Downloading the {xlib.get_awscli_name()} compressed file to local computer ...\n' ) local_path = f'{xlib.get_temp_dir()}/{xlib.get_awscli_name()}.zip' if not os.path.exists(os.path.dirname(local_path)): os.makedirs(os.path.dirname(local_path)) try: urllib.request.urlretrieve(xlib.get_awscli_url(), local_path) except Exception as e: log.write(f'*** EXCEPTION: "{e}".') log.write( f'*** ERROR: The file {xlib.get_awscli_url()} can not be downloaded.\n' ) OK = False else: log.write('The file is downloaded.\n') # upload the AWSCLI2 compressed file to cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Uploading the {xlib.get_awscli_name()} compressed file to the cluster ...\n' ) cluster_path = f'./{os.path.basename(local_path)}' (OK, error_list) = xssh.put_file(sftp_client, local_path, cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write(f'{error}\n') # upload the infraestructe software installation script to the node if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Uploading the infraestructe software installation script to the node {node_name} ...\n' ) (OK, error_list) = xssh.put_file(sftp_client, local_script_path, node_script_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write(f'{error}\n') # set run permision to the infraestructe software installation script in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Setting on the run permision ...\n') command = f'chmod u+x {node_script_path}' (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # submit the infraestructe software installation script if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Submitting the infraestructe software installation script in the node {node_name} ...\n' ) command = f'{node_script_path} &>{node_log_path} &' (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The script is submitted.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # close the SSH transport connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Closing the SSH transport connection ...\n') xssh.close_ssh_transport_connection(ssh_transport) log.write('The connection is closed.\n') # close the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # return the control variable return OK
def unmount_volume(cluster_name, node_name, volume_name, log, function=None, is_menu_call=True): ''' Unmount a volume in a node. ''' # initialize the control variable OK = True # warn that the log window must not be closed if not isinstance(log, xlib.DevStdOut) and is_menu_call: log.write( 'This process might take a few minutes. Do not close this window, please wait!\n' ) # create the SSH client connection log.write('{0}\n'.format(xlib.get_separator())) log.write('Connecting SSH client ...\n') (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master') if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # warn that the requirements are being verified if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Verifying process requirements ...\n') # verify the master is running if OK: (master_state_code, master_state_name) = xec2.get_node_state(cluster_name, 'master') if master_state_code != 16: log.write( '*** ERROR: The cluster {0} is not running. Its state is {1} ({2}).\n' .format(cluster_name, master_state_code, master_state_name)) OK = False # get the zone name of the node if OK: zone_name = xec2.get_node_zone_name(cluster_name, node_name) # get the node identification if OK: node_id = xec2.get_node_id(cluster_name, node_name) if node_id == '': log.write( '*** ERROR: The {0} identification of the cluster {1} not has been got.\n' .format(node_name, cluster_name)) OK = False # verify the volume is created if OK: if not xec2.is_volume_created(volume_name, zone_name): log.write('*** ERROR: The volume {0} is not created.\n'.format( volume_name)) OK = False # get AWS file device if OK: aws_device_file = xec2.get_volume_device_file(cluster_name, node_name, volume_name) if aws_device_file == '': log.write( '*** ERROR: the file device of the volume is not found.\n') OK = False # get the volume identification if OK: volume_id = xec2.get_volume_id(volume_name, zone_name) if volume_id == '': log.write( '*** ERROR: The identificaction of volume {0} not has been got.\n' .format(volume_name)) OK = False # get de file device file if OK: machine_device_file = xlib.get_machine_device_file(aws_device_file) # warn that the requirements are OK if OK: log.write('Process requirements are OK.\n') # unmount the volume to the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Unmounting volume {0} from the device file {1} ...\n'.format( volume_name, machine_device_file)) command = 'umount {0}'.format(machine_device_file) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The volume is unmounted.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # detach the volume to the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Detaching volume {0} from node {1} of cluster {2} ...\n'.format( volume_name, node_name, cluster_name)) OK = xec2.detach_volume(node_id, volume_id, aws_device_file) if OK: log.write('The volume is detached.\n') else: log.write('*** ERROR: The volume is not detached.\n') # close the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write('{0}\n'.format(xlib.get_separator())) log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def run_gzip_process(cluster_name, dataset_type, log, function=None): ''' Run a gzip process. ''' # initialize the control variable OK = True # get the gzip code and name gzip_code = xlib.get_gzip_code() gzip_name = xlib.get_gzip_name() # get the gzip option dictionary gzip_option_dict = xlib.get_option_dict(get_gzip_config_file(dataset_type)) # get the experiment identification experiment_id = gzip_option_dict['identification']['experiment_id'] # get the gzip process script path in the local computer gzip_process_script = get_gzip_process_script(dataset_type) # get the gzip process starter path in the local computer gzip_process_starter = get_gzip_process_starter(dataset_type) # warn that the log window does not have to be closed if not isinstance(log, xlib.DevStdOut): log.write('This process might take several minutes. Do not close this window, please wait!\n') # check the gzip config file log.write(f'{xlib.get_separator()}\n') log.write('Checking the {0} config file ...\n'.format(gzip_name)) (OK, error_list) = check_gzip_config_file(dataset_type, strict=True) if OK: log.write('The file is OK.\n') else: log.write('*** ERROR: The config file is not valid.\n') log.write('Please correct this file or recreate the config files.\n') # create the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Connecting the SSH client ...\n') (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name) if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write(f'{error}\n') # create the SSH transport connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Connecting the SSH transport ...\n') (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name) if OK: log.write('The SSH transport is connected.\n') else: for error in error_list: log.write(f'{error}\n') # create the SFTP client if OK: log.write(f'{xlib.get_separator()}\n') log.write('Connecting the SFTP client ...\n') sftp_client = xssh.create_sftp_client(ssh_transport) log.write('The SFTP client is connected.\n') # warn that the requirements are being verified if OK: log.write(f'{xlib.get_separator()}\n') log.write('Checking process requirements ...\n') # check the master is running if OK: (master_state_code, master_state_name) = xec2.get_node_state(cluster_name) if master_state_code != 16: log.write(f'*** ERROR: The cluster {cluster_name} is not running. Its state is {master_state_code} ({master_state_name}).\n') OK = False # warn that the requirements are OK if OK: log.write('Process requirements are OK.\n') # determine the run directory in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Determining the run directory in the cluster ...\n') if dataset_type == 'reference': current_run_dir = xlib.get_cluster_current_run_dir('reference', gzip_code) elif dataset_type == 'database': current_run_dir = xlib.get_cluster_current_run_dir('database', gzip_code) else: current_run_dir = xlib.get_cluster_current_run_dir(experiment_id, gzip_code) command = f'mkdir --parents {current_run_dir}' (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write(f'The directory path is {current_run_dir}.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # build the gzip process script if OK: log.write(f'{xlib.get_separator()}\n') log.write('Building the process script {0} ...\n'.format(gzip_process_script)) (OK, error_list) = build_gzip_process_script(cluster_name, dataset_type, current_run_dir) if OK: log.write('The file is built.\n') else: log.write('*** ERROR: The file could not be built.\n') # upload the gzip process script to the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Uploading the process script {0} to the directory {1} ...\n'.format(gzip_process_script, current_run_dir)) cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_script)) (OK, error_list) = xssh.put_file(sftp_client, gzip_process_script, cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write(f'{error}\n') # set run permision to the gzip process script in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(gzip_process_script))) command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_script)) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # build the gzip process starter if OK: log.write(f'{xlib.get_separator()}\n') log.write('Building the process starter {0} ...\n'.format(gzip_process_starter)) (OK, error_list) = build_gzip_process_starter(dataset_type, current_run_dir) if OK: log.write('The file is built.\n') else: log.write('***ERROR: The file could not be built.\n') # upload the gzip process starter to the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Uploading the process starter {0} to the directory {1} ...\n'.format(gzip_process_starter, current_run_dir)) cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_starter)) (OK, error_list) = xssh.put_file(sftp_client, gzip_process_starter, cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write(f'{error}\n') # set run permision to the gzip process starter in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(gzip_process_starter))) command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_starter)) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # submit the gzip process if OK: log.write(f'{xlib.get_separator()}\n') log.write('Submitting the process script {0}/{1} ...\n'.format(current_run_dir, os.path.basename(gzip_process_starter))) OK = xssh.submit_script(cluster_name, ssh_client, current_run_dir, os.path.basename(gzip_process_starter), log) # close the SSH transport connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Closing the SSH transport connection ...\n') xssh.close_ssh_transport_connection(ssh_transport) log.write('The connection is closed.\n') # close the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write(f'{xlib.get_separator()}\n') log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def run_gmap_process(cluster_name, log, function=None): ''' Run a GMAP process. ''' # initialize the control variable OK = True # get the GMAP option dictionary gmap_option_dict = xlib.get_option_dict(get_gmap_config_file()) # get the experiment identification experiment_id = gmap_option_dict['identification']['experiment_id'] # warn that the log window must not be closed if not isinstance(log, xlib.DevStdOut): log.write('This process might take several minutes. Do not close this window, please wait!\n') # validate the GMAP config file log.write('{0}\n'.format(xlib.get_separator())) log.write('Validating the {0} config file ...\n'.format(xlib.get_gmap_name())) (OK, error_list) = validate_gmap_config_file(strict=True) if OK: log.write('The config file is OK.\n') else: log.write('*** ERROR: The config file is not valid.\n') log.write('Please correct this file or recreate the config files.\n') # create the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Connecting the SSH client ...\n') (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master') if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # create the SSH transport connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Connecting the SSH transport ...\n') (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name, 'master') if OK: log.write('The SSH transport is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # create the SFTP client if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Connecting the SFTP client ...\n') sftp_client = xssh.create_sftp_client(ssh_transport) log.write('The SFTP client is connected.\n') # warn that the requirements are being verified if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Verifying process requirements ...\n') # verify the master is running if OK: (master_state_code, master_state_name) = xec2.get_node_state(cluster_name, 'master') if master_state_code != 16: log.write('*** ERROR: The cluster {0} is not running. Its state is {1} ({2}).\n'.format(cluster_name, master_state_code, master_state_name)) OK = False # verify the GMAP-GSNAP is setup if OK: (OK, error_list, is_setup) = xbioinfoapp.is_setup_bioconda_package(xlib.get_gmap_gsnap_bioconda_code(), cluster_name, True, ssh_client) if OK: if not is_setup: log.write('*** ERROR: {0} is not setup.\n'.format(xlib.get_gmap_name())) OK = False else: log.write('*** ERROR: The verification of {0} setup could not be performed.\n'.format(xlib.get_gmap_name())) # warn that the requirements are OK if OK: log.write('Process requirements are OK.\n') # determine the run directory in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Determining the run directory in the cluster ...\n') current_run_dir = xlib.get_cluster_current_run_dir(experiment_id, xlib.get_gmap_code()) command = 'mkdir --parents {0}'.format(current_run_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The directory path is {0}.\n'.format(current_run_dir)) else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # build the GMAP process script if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Building the process script {0} ...\n'.format(get_gmap_process_script())) (OK, error_list) = build_gmap_process_script(cluster_name, current_run_dir) if OK: log.write('The file is built.\n') if not OK: log.write('*** ERROR: The file could not be built.\n') # upload the GMAP process script in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Uploading the process script {0} in the directory {1} of the master ...\n'.format(get_gmap_process_script(), current_run_dir)) cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_script())) (OK, error_list) = xssh.put_file(sftp_client, get_gmap_process_script(), cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # set run permision to the GMAP process script in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(get_gmap_process_script()))) command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_script())) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # build the GMAP process starter if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Building the process starter {0} ...\n'.format(get_gmap_process_starter())) (OK, error_list) = build_gmap_process_starter(current_run_dir) if OK: log.write('The file is built.\n') if not OK: log.write('***ERROR: The file could not be built.\n') # upload the GMAP process starter in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Uploading the process starter {0} in the directory {1} of the master ...\n'.format(get_gmap_process_starter(), current_run_dir)) cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_starter())) (OK, error_list) = xssh.put_file(sftp_client, get_gmap_process_starter(), cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # set run permision to the GMAP process starter in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(get_gmap_process_starter()))) command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_starter())) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # submit the GMAP process if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Submitting the process script {0}/{1} ...\n'.format(current_run_dir, os.path.basename(get_gmap_process_starter()))) sge_env = xcluster.get_sge_env() command = '{0}; qsub -V -b n -cwd {1}/{2}'.format(sge_env, current_run_dir, os.path.basename(get_gmap_process_starter())) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: for line in stdout: log.write('{0}\n'.format(line)) else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # close the SSH transport connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH transport connection ...\n') xssh.close_ssh_transport_connection(ssh_transport) log.write('The connection is closed.\n') # close the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write('{0}\n'.format(xlib.get_separator())) log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def upload_read_dataset(cluster_name, log, function=None): ''' Upload the read dataset to the cluster. ''' # initialize the control variable OK = True # get the read transfer config file read_transfer_config_file = get_read_transfer_config_file() # warn that the log window must not be closed if not isinstance(log, xlib.DevStdOut): log.write( 'This process might take several minutes. Do not close this window, please wait!\n' ) # get and validate the read transfer config file log.write('{0}\n'.format(xlib.get_separator())) log.write('The read transfer config file is been validating ...\n') if validate_read_transfer_config_file(strict=True): log.write('The config file is OK.\n') else: log.write('*** ERROR: The read transfer config file is not valid.\n') log.write('Please correct this file or recreate the config files.\n') OK = False # create the SSH client connection if OK: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection( cluster_name, 'master') for error in error_list: log.write('{0}\n'.format(error)) # create the SSH transport connection if OK: (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection( cluster_name, 'master') for error in error_list: log.write('{0}\n'.format(error)) # create the SFTP client if OK: sftp_client = xssh.create_sftp_client(ssh_transport) # get the options dictionary if OK: read_transfer_options_dict = xlib.get_option_dict( read_transfer_config_file) # get the experiment identification and create the experiment reads directory if OK: # get the experiment identification experiment_id = read_transfer_options_dict['identification'][ 'experiment_id'] # Get the directory of read and results datasets of the experiment cluster_experiment_reads_dir = xlib.get_cluster_experiment_read_dataset_dir( experiment_id, xlib.get_uploaded_read_dataset_name()) cluster_experiment_result_dir = xlib.get_cluster_experiment_result_dir( experiment_id) # create the experiment reads directory log.write('{0}\n'.format(xlib.get_separator())) log.write( 'The reads directory {0} in the cluster is being created ...\n'. format(cluster_experiment_reads_dir)) command = 'mkdir --parents {0}'.format(cluster_experiment_reads_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The directory is created.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # create the experiment run result directory log.write('{0}\n'.format(xlib.get_separator())) log.write( 'The run result directory {0} in the cluster is being created ...\n' .format(cluster_experiment_result_dir)) command = 'mkdir --parents {0}'.format(cluster_experiment_result_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The directory is created.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # upload the read dataset if OK: # get the sections list sections_list = [] for section in read_transfer_options_dict.keys(): sections_list.append(section) sections_list.sort() # for each section "file-n" for section in sections_list: # verify than the section identification is like file-n if re.match('^file-[0-9]+$', section): # get local path and cluster directory local_path = read_transfer_options_dict[section]['local_path'] # upload the reference file in the cluster log.write('{0}\n'.format(xlib.get_separator())) log.write('The file {0} is being uploaded to {1} ...\n'.format( local_path, cluster_experiment_reads_dir)) cluster_path = '{0}/{1}'.format(cluster_experiment_reads_dir, os.path.basename(local_path)) (OK, error_list) = xssh.put_file(sftp_client, local_path, cluster_path) if OK: log.write('The file has been uploaded.\n') else: for error in error_list: log.write('{0}\n'.format(error)) break # close the SSH transport connection if OK: xssh.close_ssh_transport_connection(ssh_transport) # close the SSH client connection if OK: xssh.close_ssh_client_connection(ssh_client) # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write('{0}\n'.format(xlib.get_separator())) log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def download_result_dataset(cluster_name, log, function=None): ''' Download the result dataset of a run from the cluster. ''' # initialize the control variable OK = True # get the read transfer config file result_transfer_config_file = get_result_transfer_config_file() # warn that the log window must not be closed if not isinstance(log, xlib.DevStdOut): log.write('This process might take several minutes. Do not close this window, please wait!\n') # get and validate the result transfer config file log.write('{0}\n'.format(xlib.get_separator())) log.write('The result transfer config file is been validating ...\n') if validate_result_transfer_config_file(strict=True): log.write('The config file is OK.\n') else: log.write('*** ERROR: The result transfer config file is not valid.\n') log.write('Please correct this file or recreate the config files.\n') OK = False # create the SSH client connection if OK: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master') for error in error_list: log.write('{0}\n'.format(error)) # create the SSH transport connection if OK: (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name, 'master') for error in error_list: log.write('{0}\n'.format(error)) # create the SFTP client if OK: sftp_client = xssh.create_sftp_client(ssh_transport) # get the options dictionary if OK: result_transfer_options_dict = xlib.get_option_dict(result_transfer_config_file) # download the result dataset if OK: # get the sections list sections_list = [] for section in result_transfer_options_dict.keys(): sections_list.append(section) sections_list.sort() # get the experiment identification, run identification and local directory from the section "identification" experiment_id = result_transfer_options_dict['identification']['experiment_id'] result_dataset_id = result_transfer_options_dict['identification']['result_dataset_id'] status = result_transfer_options_dict['identification']['status'].lower() local_dir = result_transfer_options_dict['identification']['local_dir'] # download files when the status is uncompressed if status == 'uncompressed': # for each section "file-n" for section in sections_list: # verify than the section identification is like file-n if re.match('^file-[0-9]+$', section): # get the dataset subdirectory and file name dataset_subdirectory = result_transfer_options_dict[section]['dataset_subdirectory'] file_name = result_transfer_options_dict[section]['file_name'] # verify if the dataset subdirectory is created pathlib.Path(os.path.normpath('{0}/{1}'.format(local_dir, dataset_subdirectory))).mkdir(parents=True, exist_ok=True) # assign the cluster path and local path cluster_path = '{0}/{1}/{2}/{3}/{4}'.format(xlib.get_cluster_result_dir(), experiment_id, result_dataset_id, dataset_subdirectory, file_name) local_path = os.path.normpath('{0}/{1}/{2}'.format(local_dir, dataset_subdirectory, file_name)) # download the result file from the cluster log.write('{0}\n'.format(xlib.get_separator())) log.write('Downloading the file {0} to {1} ...\n'.format(cluster_path, local_dir)) (OK, error_list) = xssh.get_file(sftp_client, cluster_path, local_path) if OK: log.write('The file has been downloaded.\n') else: for error in error_list: log.write('{0}\n'.format(error)) break # download files when the status is compressed elif status == 'compressed': # assign the cluster path and local path cluster_path = '{0}/{1}/{2}'.format(xlib.get_cluster_result_dir(), experiment_id, result_dataset_id) local_path = '{0}/{1}'.format(local_dir, result_dataset_id) # download the result file from the cluster log.write('{0}\n'.format(xlib.get_separator())) log.write('Downloading the file {0} to {1} ...\n'.format(cluster_path, local_dir)) (OK, error_list) = xssh.get_file(sftp_client, cluster_path, local_path) if OK: log.write('The file has been downloaded.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # close the SSH transport connection if OK: xssh.close_ssh_transport_connection(ssh_transport) # close the SSH client connection if OK: xssh.close_ssh_client_connection(ssh_client) # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write('{0}\n'.format(xlib.get_separator())) log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def run_htseq_count_process(cluster_name, log, function=None): ''' Run a htseq-count process. ''' # initialize the control variable OK = True # get the htseq-count option dictionary htseq_count_option_dict = xlib.get_option_dict( get_htseq_count_config_file()) # get the experiment identification experiment_id = htseq_count_option_dict['identification']['experiment_id'] # warn that the log window does not have to be closed if not isinstance(log, xlib.DevStdOut): log.write( 'This process might take several minutes. Do not close this window, please wait!\n' ) # check the htseq-count config file log.write(f'{xlib.get_separator()}\n') log.write(f'Checking the {xlib.get_htseq_count_name()} config file ...\n') (OK, error_list) = check_htseq_count_config_file(strict=True) if OK: log.write('The file is OK.\n') else: log.write('*** ERROR: The config file is not valid.\n') log.write('Please correct this file or recreate the config files.\n') # create the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Connecting the SSH client ...\n') (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name) if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write(f'{error}\n') # create the SSH transport connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Connecting the SSH transport ...\n') (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name) if OK: log.write('The SSH transport is connected.\n') else: for error in error_list: log.write(f'{error}\n') # create the SFTP client if OK: log.write(f'{xlib.get_separator()}\n') log.write('Connecting the SFTP client ...\n') sftp_client = xssh.create_sftp_client(ssh_transport) log.write('The SFTP client is connected.\n') # warn that the requirements are being verified if OK: log.write(f'{xlib.get_separator()}\n') log.write('Checking process requirements ...\n') # check the master is running if OK: (master_state_code, master_state_name) = xec2.get_node_state(cluster_name) if master_state_code != 16: log.write( f'*** ERROR: The cluster {cluster_name} is not running. Its state is {master_state_code} ({master_state_name}).\n' ) OK = False # check HTSeq is installed if OK: (OK, error_list, is_installed) = xbioinfoapp.is_installed_anaconda_package( xlib.get_htseq_anaconda_code(), cluster_name, True, ssh_client) if OK: if not is_installed: log.write( f'*** ERROR: {xlib.get_htseq_name()} is not installed.\n') OK = False else: log.write( f'*** ERROR: The verification of {xlib.get_htseq_name()} installation could not be performed.\n' ) # warn that the requirements are OK if OK: log.write('Process requirements are OK.\n') # determine the run directory in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Determining the run directory in the cluster ...\n') current_run_dir = xlib.get_cluster_current_run_dir( experiment_id, xlib.get_htseq_count_code()) command = f'mkdir --parents {current_run_dir}' (OK, _, _) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write(f'The directory path is {current_run_dir}.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # build the htseq-count process script if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Building the process script {get_htseq_count_process_script()} ...\n' ) (OK, error_list) = build_htseq_count_process_script( cluster_name, current_run_dir) if OK: log.write('The file is built.\n') if not OK: log.write('*** ERROR: The file could not be built.\n') # upload the htseq-count process script to the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Uploading the process script {get_htseq_count_process_script()} to the directory {current_run_dir} ...\n' ) cluster_path = f'{current_run_dir}/{os.path.basename(get_htseq_count_process_script())}' (OK, error_list) = xssh.put_file(sftp_client, get_htseq_count_process_script(), cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write(f'{error}\n') # set run permision to the htseq-count process script in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Setting on the run permision of {current_run_dir}/{os.path.basename(get_htseq_count_process_script())} ...\n' ) command = f'chmod u+x {current_run_dir}/{os.path.basename(get_htseq_count_process_script())}' (OK, _, _) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # build the htseq-count process starter if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Building the process starter {get_htseq_count_process_starter()} ...\n' ) (OK, error_list) = build_htseq_count_process_starter(current_run_dir) if OK: log.write('The file is built.\n') if not OK: log.write('***ERROR: The file could not be built.\n') # upload the htseq-count process starter to the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Uploading the process starter {get_htseq_count_process_starter()} to the directory {current_run_dir} ...\n' ) cluster_path = f'{current_run_dir}/{os.path.basename(get_htseq_count_process_starter())}' (OK, error_list) = xssh.put_file(sftp_client, get_htseq_count_process_starter(), cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write(f'{error}\n') # set run permision to the htseq-count process starter in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Setting on the run permision of {current_run_dir}/{os.path.basename(get_htseq_count_process_starter())} ...\n' ) command = f'chmod u+x {current_run_dir}/{os.path.basename(get_htseq_count_process_starter())}' (OK, _, _) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # submit the htseq-count process if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Submitting the process script {current_run_dir}/{os.path.basename(get_htseq_count_process_starter())} ...\n' ) OK = xssh.submit_script( cluster_name, ssh_client, current_run_dir, os.path.basename(get_htseq_count_process_starter()), log) # close the SSH transport connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Closing the SSH transport connection ...\n') xssh.close_ssh_transport_connection(ssh_transport) log.write('The connection is closed.\n') # close the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write(f'{xlib.get_separator()}\n') log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def form_list_cluster_experiment_processes(): ''' List the processes of an experiment in the cluster. ''' # initialize the control variable OK = True # print the header clib.clear_screen() clib.print_headers_with_environment( 'Logs - List experiment processes in the cluster') # get the cluster name print(xlib.get_separator()) if xec2.get_running_cluster_list(only_environment_cluster=True, volume_creator_included=False) != []: cluster_name = cinputs.input_cluster_name( volume_creator_included=False, help=True) else: print('WARNING: There is not any running cluster.') OK = False # create the SSH client connection if OK: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name) for error in error_list: print(error) # get experiment identification if OK: experiment_id = cinputs.input_experiment_id(ssh_client, help=True) if experiment_id == '': print( f'WARNING: The cluster {cluster_name} does not have experiment data.' ) OK = False # get the result dataset list of the experiment if OK: command = f'cd {xlib.get_cluster_result_dir()}/{experiment_id}; for list in `ls`; do ls -ld $list | grep -v ^- > /dev/null && echo $list; done;' (OK, stdout, _) = xssh.execute_cluster_command(ssh_client, command) if OK: result_dataset_id_list = [] for line in stdout: line = line.rstrip('\n') if line != 'lost+found': result_dataset_id_list.append(line) # print the result dataset identification list of the experiment if OK: print(xlib.get_separator()) if result_dataset_id_list == []: print( f'*** WARNING: There is not any result dataset of the experiment {experiment_id}.' ) else: result_dataset_id_list.sort() # set data width result_dataset_width = 30 bioinfo_app_width = 25 # set line line = '{0:' + str(result_dataset_width) + '} {1:' + str( bioinfo_app_width) + '}' # print header print(line.format('Result dataset', 'Bioinfo app / Utility')) print( line.format('=' * result_dataset_width, '=' * bioinfo_app_width)) # print detail lines for result_dataset_id in result_dataset_id_list: if result_dataset_id.startswith(xlib.get_bedtools_code() + '-'): bioinfo_app_name = xlib.get_bedtools_name() elif result_dataset_id.startswith(xlib.get_blastplus_code() + '-'): bioinfo_app_name = xlib.get_blastplus_name() elif result_dataset_id.startswith(xlib.get_bcftools_code() + '-'): bioinfo_app_name = xlib.get_bcftools_name() elif result_dataset_id.startswith(xlib.get_bowtie2_code() + '-'): bioinfo_app_name = xlib.get_bowtie2_name() elif result_dataset_id.startswith(xlib.get_busco_code() + '-'): bioinfo_app_name = xlib.get_busco_name() elif result_dataset_id.startswith(xlib.get_cd_hit_code() + '-'): bioinfo_app_name = xlib.get_cd_hit_name() elif result_dataset_id.startswith(xlib.get_cd_hit_est_code() + '-'): bioinfo_app_name = xlib.get_cd_hit_est_name() elif result_dataset_id.startswith(xlib.get_cuffdiff_code() + '-'): bioinfo_app_name = xlib.get_cuffdiff_name() elif result_dataset_id.startswith(xlib.get_cufflinks_code() + '-'): bioinfo_app_name = xlib.get_cufflinks_name() elif result_dataset_id.startswith( xlib.get_cufflinks_cuffmerge_code() + '-'): bioinfo_app_name = xlib.get_cufflinks_cuffmerge_name() elif result_dataset_id.startswith(xlib.get_cuffnorm_code() + '-'): bioinfo_app_name = xlib.get_cuffnorm_name() elif result_dataset_id.startswith(xlib.get_cuffquant_code() + '-'): bioinfo_app_name = xlib.get_cuffquant_name() elif result_dataset_id.startswith(xlib.get_cutadapt_code() + '-'): bioinfo_app_name = xlib.get_cutadapt_name() elif result_dataset_id.startswith( xlib.get_ddradseq_simulation_code() + '-'): bioinfo_app_name = xlib.get_ddradseq_simulation_name() elif result_dataset_id.startswith( xlib.get_ddradseqtools_code() + '-'): bioinfo_app_name = xlib.get_ddradseqtools_name() elif result_dataset_id.startswith(xlib.get_detonate_code() + '-'): bioinfo_app_name = xlib.get_detonate_name() elif result_dataset_id.startswith(xlib.get_diamond_code() + '-'): bioinfo_app_name = xlib.get_diamond_name() elif result_dataset_id.startswith(xlib.get_emboss_code() + '-'): bioinfo_app_name = xlib.get_emboss_name() elif result_dataset_id.startswith( xlib.get_entrez_direct_code() + '-'): bioinfo_app_name = xlib.get_entrez_direct_name() elif result_dataset_id.startswith(xlib.get_express_code() + '-'): bioinfo_app_name = xlib.get_express_name() elif result_dataset_id.startswith(xlib.get_fastqc_code() + '-'): bioinfo_app_name = xlib.get_fastqc_name() elif result_dataset_id.startswith(xlib.get_ggtrinity_code() + '-'): bioinfo_app_name = xlib.get_ggtrinity_name() elif result_dataset_id.startswith(xlib.get_gmap_gsnap_code() + '-'): bioinfo_app_name = xlib.get_gmap_gsnap_name() elif result_dataset_id.startswith(xlib.get_gmap_code() + '-'): bioinfo_app_name = xlib.get_gmap_name() elif result_dataset_id.startswith(xlib.get_gsnap_code() + '-'): bioinfo_app_name = xlib.get_gsnap_name() elif result_dataset_id.startswith(xlib.get_gzip_code() + '-'): bioinfo_app_name = xlib.get_gzip_name() elif result_dataset_id.startswith(xlib.get_hisat2_code() + '-'): bioinfo_app_name = xlib.get_hisat2_name() elif result_dataset_id.startswith(xlib.get_htseq_code() + '-'): bioinfo_app_name = xlib.get_htseq_name() elif result_dataset_id.startswith(xlib.get_htseq_count_code() + '-'): bioinfo_app_name = xlib.get_htseq_count_name() elif result_dataset_id.startswith( xlib.get_insilico_read_normalization_code() + '-'): bioinfo_app_name = xlib.get_insilico_read_normalization_name( ) elif result_dataset_id.startswith(xlib.get_ipyrad_code() + '-'): bioinfo_app_name = xlib.get_ipyrad_name() elif result_dataset_id.startswith(xlib.get_kallisto_code() + '-'): bioinfo_app_name = xlib.get_kallisto_name() elif result_dataset_id.startswith(xlib.get_miniconda3_code() + '-'): bioinfo_app_name = xlib.get_miniconda3_name() elif result_dataset_id.startswith(xlib.get_ngshelper_code() + '-'): bioinfo_app_name = xlib.get_ngshelper_name() elif result_dataset_id.startswith(xlib.get_quast_code() + '-'): bioinfo_app_name = xlib.get_quast_name() elif result_dataset_id.startswith(xlib.get_r_code() + '-'): bioinfo_app_name = xlib.get_r_name() elif result_dataset_id.startswith(xlib.get_raddesigner_code() + '-'): bioinfo_app_name = xlib.get_raddesigner_name() elif result_dataset_id.startswith(xlib.get_ref_eval_code() + '-'): bioinfo_app_name = xlib.get_ref_eval_name() elif result_dataset_id.startswith(xlib.get_rnaquast_code() + '-'): bioinfo_app_name = xlib.get_rnaquast_name() elif result_dataset_id.startswith(xlib.get_rsem_code() + '-'): bioinfo_app_name = xlib.get_rsem_name() elif result_dataset_id.startswith(xlib.get_rsem_eval_code() + '-'): bioinfo_app_name = xlib.get_rsem_eval_name() elif result_dataset_id.startswith(xlib.get_rsitesearch_code() + '-'): bioinfo_app_name = xlib.get_rsitesearch_name() elif result_dataset_id.startswith(xlib.get_samtools_code() + '-'): bioinfo_app_name = xlib.get_samtools_name() elif result_dataset_id.startswith(xlib.get_soapdenovo2_code() + '-'): bioinfo_app_name = xlib.get_soapdenovo2_name() elif result_dataset_id.startswith( xlib.get_soapdenovotrans_code() + '-'): bioinfo_app_name = xlib.get_soapdenovotrans_name() elif result_dataset_id.startswith(xlib.get_star_code() + '-'): bioinfo_app_name = xlib.get_star_name() elif result_dataset_id.startswith(xlib.get_starcode_code() + '-'): bioinfo_app_name = xlib.get_starcode_name() elif result_dataset_id.startswith(xlib.get_toa_code() + '-'): bioinfo_app_name = xlib.get_toa_name() elif result_dataset_id.startswith( xlib.get_toa_process_download_basic_data_code() + '-'): bioinfo_app_name = xlib.get_toa_process_download_basic_data_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_download_dicots_04_code() + '-'): bioinfo_app_name = xlib.get_toa_process_download_dicots_04_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_download_gene_code() + '-'): bioinfo_app_name = xlib.get_toa_process_download_gene_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_download_go_code() + '-'): bioinfo_app_name = xlib.get_toa_process_download_go_name() elif result_dataset_id.startswith( xlib.get_toa_process_download_gymno_01_code() + '-'): bioinfo_app_name = xlib.get_toa_process_download_gymno_01_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_download_interpro_code() + '-'): bioinfo_app_name = xlib.get_toa_process_download_interpro_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_download_monocots_04_code() + '-'): bioinfo_app_name = xlib.get_toa_process_download_monocots_04_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_download_taxonomy_code() + '-'): bioinfo_app_name = xlib.get_toa_process_download_taxonomy_name( ) elif result_dataset_id.startswith( xlib. get_toa_process_gilist_viridiplantae_nucleotide_gi_code( ) + '-'): bioinfo_app_name = xlib.get_toa_process_gilist_viridiplantae_nucleotide_gi_name( ) elif result_dataset_id.startswith( xlib. get_toa_process_gilist_viridiplantae_protein_gi_code() + '-'): bioinfo_app_name = xlib.get_toa_process_gilist_viridiplantae_protein_gi_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_load_basic_data_code() + '-'): bioinfo_app_name = xlib.get_toa_process_load_basic_data_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_load_dicots_04_code() + '-'): bioinfo_app_name = xlib.get_toa_process_load_dicots_04_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_load_gene_code() + '-'): bioinfo_app_name = xlib.get_toa_process_load_gene_name() elif result_dataset_id.startswith( xlib.get_toa_process_load_go_code() + '-'): bioinfo_app_name = xlib.get_toa_process_load_go_name() elif result_dataset_id.startswith( xlib.get_toa_process_load_gymno_01_code() + '-'): bioinfo_app_name = xlib.get_toa_process_load_gymno_01_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_load_interpro_code() + '-'): bioinfo_app_name = xlib.get_toa_process_load_interpro_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_load_monocots_04_code() + '-'): bioinfo_app_name = xlib.get_toa_process_load_monocots_04_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_merge_annotations_code() + '-'): bioinfo_app_name = xlib.get_toa_process_merge_annotations_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_nr_blastplus_db_code() + '-'): bioinfo_app_name = xlib.get_toa_process_nr_blastplus_db_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_nr_diamond_db_code() + '-'): bioinfo_app_name = xlib.get_toa_process_nr_diamond_db_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_nt_blastplus_db_code() + '-'): bioinfo_app_name = xlib.get_toa_process_nt_blastplus_db_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_pipeline_aminoacid_code() + '-'): bioinfo_app_name = xlib.get_toa_process_pipeline_aminoacid_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_pipeline_nucleotide_code() + '-'): bioinfo_app_name = xlib.get_toa_process_pipeline_nucleotide_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_proteome_dicots_04_code() + '-'): bioinfo_app_name = xlib.get_toa_process_proteome_dicots_04_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_proteome_gymno_01_code() + '-'): bioinfo_app_name = xlib.get_toa_process_proteome_gymno_01_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_proteome_monocots_04_code() + '-'): bioinfo_app_name = xlib.get_toa_process_proteome_monocots_04_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_proteome_refseq_plant_code() + '-'): bioinfo_app_name = xlib.get_toa_process_proteome_refseq_plant_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_rebuild_toa_database_code() + '-'): bioinfo_app_name = xlib.get_get_toa_process_rebuild_toa_database_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_recreate_toa_database_code() + '-'): bioinfo_app_name = xlib.get_get_toa_process_recreate_toa_database_name( ) elif result_dataset_id.startswith(xlib.get_tophat_code() + '-'): bioinfo_app_name = xlib.get_tophat_name() elif result_dataset_id.startswith(xlib.get_transabyss_code() + '-'): bioinfo_app_name = xlib.get_transabyss_name() elif result_dataset_id.startswith( xlib.get_transcript_filter_code() + '-'): bioinfo_app_name = xlib.get_transcript_filter_name() elif result_dataset_id.startswith( xlib.get_transcriptome_blastx_code() + '-'): bioinfo_app_name = xlib.get_transcriptome_blastx_name() elif result_dataset_id.startswith( xlib.get_transdecoder_code() + '-'): bioinfo_app_name = xlib.get_transdecoder_name() elif result_dataset_id.startswith(xlib.get_transrate_code() + '-'): bioinfo_app_name = xlib.get_transrate_name() elif result_dataset_id.startswith(xlib.get_trimmomatic_code() + '-'): bioinfo_app_name = xlib.get_trimmomatic_name() elif result_dataset_id.startswith(xlib.get_trinity_code() + '-'): bioinfo_app_name = xlib.get_trinity_name() elif result_dataset_id.startswith( xlib.get_variant_calling_code() + '-'): bioinfo_app_name = xlib.get_variant_calling_name() elif result_dataset_id.startswith(xlib.get_vcftools_code() + '-'): bioinfo_app_name = xlib.get_vcftools_name() elif result_dataset_id.startswith( xlib.get_vcftools_perl_libraries_code() + '-'): bioinfo_app_name = xlib.get_vcftools_perl_libraries_name() elif result_dataset_id.startswith(xlib.get_vsearch_code() + '-'): bioinfo_app_name = xlib.get_vsearch_name() else: bioinfo_app_name = 'xxx' print(line.format(result_dataset_id, bioinfo_app_name)) # close the SSH client connection if OK: xssh.close_ssh_client_connection(ssh_client) # show continuation message print(xlib.get_separator()) input('Press [Intro] to continue ...')