def get_reference_file_name_list(cluster_name, reference_dataset_id, passed_connection=False, ssh_client=None): ''' Get a list of the reference file names in a reference dataset of the cluster. ''' # initialize the control variable and the error list OK = True error_list = [] # get the reference directory in the cluster cluster_reference_dir = xlib.get_cluster_reference_dir() # initialize the dictionary of the reference datasets reference_file_name_list = [] # create the SSH client connection if not passed_connection: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name) # check the reference directory is created if OK: command = '[ -d {0} ] && echo RC=0 || echo RC=1'.format( cluster_reference_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if stdout[len(stdout) - 1] != 'RC=0': error_list.append( '*** ERROR: There is not any volume mounted in the reference directory.\n' ) error_list.append( 'You have to link a volume in the mounting point {0} for the cluster {1}.\n' .format(cluster_reference_dir, cluster_name)) OK = False # get the reference dataset directory reference_dataset_dir = xlib.get_cluster_reference_dataset_dir( reference_dataset_id) # build the list of the reference file name of the reference dataset if OK: command = 'find {0} -maxdepth 1 -type f'.format(reference_dataset_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: for line in stdout: line = line.rstrip('\n') if line != 'lost+found': reference_file_name_list.append(os.path.basename(line)) # close the SSH client connection if OK and not passed_connection: xssh.close_ssh_client_connection(ssh_client) # return the control variable, error list and list of the reference file names return (OK, error_list, reference_file_name_list)
def get_database_file_name_list(cluster_name, database_dataset_id, file_type, passed_connection=False, ssh_client=None): ''' Get a list of the database file names in a database dataset of the cluster. ''' # initialize the control variable and the error list OK = True error_list = [] # initialize the dictionary of the database datasets database_file_name_list = [] # create the SSH client connection if not passed_connection: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name) # check the app directory is created if OK: command = '[ -d {0} ] && echo RC=0 || echo RC=1'.format( xlib.get_cluster_database_dir()) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if stdout[len(stdout) - 1] != 'RC=0': error_list.append( '*** ERROR: There is not any volume mounted in the database directory.\n' ) error_list.append( 'You have to link a volume in the mounting point {0} for the cluster {1}.\n' .format(xlib.get_cluster_database_dir(), cluster_name)) OK = False # build the list of the database file name of the database dataset if OK: if file_type == 'all': command = 'ls {0}'.format( xlib.get_cluster_database_dataset_dir(database_dataset_id)) else: command = 'ls {0}/*{1}'.format( xlib.get_cluster_database_dataset_dir(database_dataset_id), file_type) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: for line in stdout: line = line.rstrip('\n') if line != 'lost+found': database_file_name_list.append(os.path.basename(line)) # close the SSH client connection if OK and not passed_connection: xssh.close_ssh_client_connection(ssh_client) # return the control variable, error list and list of the database file names return (OK, error_list, database_file_name_list)
def get_reference_dataset_dict(cluster_name, passed_connection=False, ssh_client=None): ''' Get a dictionary with the reference datasets in the cluster. ''' # initialize the control variable and the error list OK = True error_list = [] # get the reference directory in the cluster cluster_reference_dir = xlib.get_cluster_reference_dir() # initialize the dictionary of the reference datasets reference_dataset_dict = {} # create the SSH client connection if not passed_connection: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name) # check the app directory is created if OK: command = '[ -d {0} ] && echo RC=0 || echo RC=1'.format( cluster_reference_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if stdout[len(stdout) - 1] != 'RC=0': error_list.append( '*** ERROR: There is not any volume mounted in the reference directory.\n' ) error_list.append( 'You have to link a volume in the mounting point {0} for the cluster {1}.\n' .format(cluster_reference_dir, cluster_name)) OK = False # build the dictionary of the reference datasets if OK: command = 'ls {0}'.format(cluster_reference_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: for line in stdout: line = line.rstrip('\n') if line != 'lost+found': reference_dataset_id = line reference_dataset_name = reference_dataset_id reference_dataset_dict[reference_dataset_id] = { 'reference_dataset_id': reference_dataset_id, 'reference_dataset_name': reference_dataset_name } # close the SSH client connection if OK and not passed_connection: xssh.close_ssh_client_connection(ssh_client) # return the control variable, error list and dictionary of the reference datasets return (OK, error_list, reference_dataset_dict)
def populate_combobox_experiment_id(self): ''' Populate data in "combobox_experiment_id". ''' # clear the value selected in the combobox self.wrapper_experiment_id.set('') # initialize the experiment identification list experiment_id_list = [] # get the experiment identifications command = 'ls {0}'.format(xlib.get_cluster_result_dir()) (OK, stdout, stderr) = xssh.execute_cluster_command(self.ssh_client, command) if OK: for line in stdout: line = line.rstrip('\n') if line != 'lost+found': experiment_id_list.append(line) # verify if there are any experimment identifications if experiment_id_list == []: message = 'The cluster has not experiment data.' tkinter.messagebox.showwarning('{0} - {1}'.format(xlib.get_project_name(), self.head), message) return # load the names of clusters which are running in the combobox self.combobox_experiment_id['values'] = experiment_id_list
def kill_batch_job(cluster_name, job_id, log, function=None): ''' Kill a batch job in the cluster. ''' # initialize the control variable OK = True # create the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Establishing connection with cluster {0} ...\n'.format( cluster_name)) (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, node_name='master') if OK: log.write('The connection is established.\n') else: for error in error_list: log.write(f'{error}\n') # kill the job in the cluster if OK: log.write(f'{xlib.get_separator()}\n') sge_env = get_sge_env() command = '{0}; qdel {1}'.format(sge_env, job_id) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: if len(stdout) > 0: for line in stdout: log.write('{0}\n'.format(line)) else: log.write('The job is been killed.\n') if len(stderr) > 0: for line in stderr: log.write('{0}\n'.format(line)) else: log.write('*** ERROR: Wrong command ---> {0}.\n'.format(command)) # close the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write( 'Closing connection with cluster {0} ...\n'.format(cluster_name)) xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write(f'{xlib.get_separator()}\n') log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def show_cluster_composition(cluster_name, log, function=None): ''' Show cluster information of every node: OS, CPU number and memory. ''' # initialize the control variable OK = True # create the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Establishing connection with cluster {0} ...\n'.format( cluster_name)) (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, node_name='master') if OK: log.write('The connection is established.\n') else: for error in error_list: log.write(f'{error}\n') # show the cluster composition if OK: log.write(f'{xlib.get_separator()}\n') sge_env = get_sge_env() command = '{0}; qhost'.format(sge_env) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: if len(stdout) > 0: for line in stdout: log.write('{0}\n'.format(line)) else: log.write('Cluster composition not found.\n') else: log.write('*** ERROR: Wrong command ---> {0}.\n'.format(command)) # close the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write( 'Closing connection with cluster {0} ...\n'.format(cluster_name)) xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write(f'{xlib.get_separator()}\n') log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def upload_read_dataset(cluster_name, log, function=None): ''' Upload the read dataset to the cluster. ''' # initialize the control variable OK = True # get the read transfer config file read_transfer_config_file = get_read_transfer_config_file() # warn that the log window must not be closed if not isinstance(log, xlib.DevStdOut): log.write( 'This process might take several minutes. Do not close this window, please wait!\n' ) # get and validate the read transfer config file log.write('{0}\n'.format(xlib.get_separator())) log.write('The read transfer config file is been validating ...\n') if validate_read_transfer_config_file(strict=True): log.write('The config file is OK.\n') else: log.write('*** ERROR: The read transfer config file is not valid.\n') log.write('Please correct this file or recreate the config files.\n') OK = False # create the SSH client connection if OK: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection( cluster_name, 'master') for error in error_list: log.write('{0}\n'.format(error)) # create the SSH transport connection if OK: (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection( cluster_name, 'master') for error in error_list: log.write('{0}\n'.format(error)) # create the SFTP client if OK: sftp_client = xssh.create_sftp_client(ssh_transport) # get the options dictionary if OK: read_transfer_options_dict = xlib.get_option_dict( read_transfer_config_file) # get the experiment identification and create the experiment reads directory if OK: # get the experiment identification experiment_id = read_transfer_options_dict['identification'][ 'experiment_id'] # Get the directory of read and results datasets of the experiment cluster_experiment_reads_dir = xlib.get_cluster_experiment_read_dataset_dir( experiment_id, xlib.get_uploaded_read_dataset_name()) cluster_experiment_result_dir = xlib.get_cluster_experiment_result_dir( experiment_id) # create the experiment reads directory log.write('{0}\n'.format(xlib.get_separator())) log.write( 'The reads directory {0} in the cluster is being created ...\n'. format(cluster_experiment_reads_dir)) command = 'mkdir --parents {0}'.format(cluster_experiment_reads_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The directory is created.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # create the experiment run result directory log.write('{0}\n'.format(xlib.get_separator())) log.write( 'The run result directory {0} in the cluster is being created ...\n' .format(cluster_experiment_result_dir)) command = 'mkdir --parents {0}'.format(cluster_experiment_result_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The directory is created.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # upload the read dataset if OK: # get the sections list sections_list = [] for section in read_transfer_options_dict.keys(): sections_list.append(section) sections_list.sort() # for each section "file-n" for section in sections_list: # verify than the section identification is like file-n if re.match('^file-[0-9]+$', section): # get local path and cluster directory local_path = read_transfer_options_dict[section]['local_path'] # upload the reference file in the cluster log.write('{0}\n'.format(xlib.get_separator())) log.write('The file {0} is being uploaded to {1} ...\n'.format( local_path, cluster_experiment_reads_dir)) cluster_path = '{0}/{1}'.format(cluster_experiment_reads_dir, os.path.basename(local_path)) (OK, error_list) = xssh.put_file(sftp_client, local_path, cluster_path) if OK: log.write('The file has been uploaded.\n') else: for error in error_list: log.write('{0}\n'.format(error)) break # close the SSH transport connection if OK: xssh.close_ssh_transport_connection(ssh_transport) # close the SSH client connection if OK: xssh.close_ssh_client_connection(ssh_client) # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write('{0}\n'.format(xlib.get_separator())) log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def form_list_cluster_experiment_processes(): ''' List the processes of an experiment in the cluster. ''' # initialize the control variable OK = True # print the header clib.clear_screen() clib.print_headers_with_environment('Logs - List experiment processes in the cluster') # get the cluster name print(xlib.get_separator()) if xec2.get_running_cluster_list(volume_creator_included=False) != []: cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True) else: print('WARNING: There is not any running cluster.') OK = False # create the SSH client connection if OK: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master') for error in error_list: log.write('{0}\n'.format(error)) # get experiment identification if OK: experiment_id = cinputs.input_experiment_id(ssh_client, help=True) if experiment_id == '': print('WARNING: The cluster {0} has not experiment data.'.format(cluster_name)) OK = False # get the result dataset list of the experiment if OK: command = 'cd {0}/{1}; for list in `ls`; do ls -ld $list | grep -v ^- > /dev/null && echo $list; done;'.format(xlib.get_cluster_result_dir(), experiment_id) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: result_dataset_id_list = [] for line in stdout: line = line.rstrip('\n') if line != 'lost+found': result_dataset_id_list.append(line) # print the result dataset identification list of the experiment if OK: print(xlib.get_separator()) if result_dataset_id_list == []: print('*** WARNING: There is not any result dataset of the experiment {0}.'.format(experiment_id)) else: result_dataset_id_list.sort() # set data width result_dataset_width = 25 bioinfo_app_width = 25 # set line template line_template = '{0:' + str(result_dataset_width) + '} {1:' + str(bioinfo_app_width) + '}' # print header print(line_template.format('Result dataset', 'Bioinfo app / Utility')) print(line_template.format('=' * result_dataset_width, '=' * bioinfo_app_width)) # print detail lines for result_dataset_id in result_dataset_id_list: if result_dataset_id.startswith(xlib.get_bedtools_code()+'-'): bioinfo_app_name = xlib.get_bedtools_name() elif result_dataset_id.startswith(xlib.get_blastplus_code()+'-'): bioinfo_app_name = xlib.get_blastplus_name() elif result_dataset_id.startswith(xlib.get_bowtie2_code()+'-'): bioinfo_app_name = xlib.get_bowtie2_name() elif result_dataset_id.startswith(xlib.get_busco_code()+'-'): bioinfo_app_name = xlib.get_busco_name() elif result_dataset_id.startswith(xlib.get_cd_hit_code()+'-'): bioinfo_app_name = xlib.get_cd_hit_est_name() elif result_dataset_id.startswith(xlib.get_cd_hit_code()+'-'): bioinfo_app_name = xlib.get_cd_hit_est_name() elif result_dataset_id.startswith(xlib.get_detonate_code()+'-'): bioinfo_app_name = xlib.get_detonate_name() elif result_dataset_id.startswith(xlib.get_emboss_code()+'-'): bioinfo_app_name = xlib.get_emboss_name() elif result_dataset_id.startswith(xlib.get_fastqc_code()+'-'): bioinfo_app_name = xlib.get_fastqc_name() elif result_dataset_id.startswith(xlib.get_gmap_code()+'-'): bioinfo_app_name = xlib.get_gmap_name() elif result_dataset_id.startswith(xlib.get_gmap_gsnap_code()+'-'): bioinfo_app_name = xlib.get_gmap_gsnap_name() elif result_dataset_id.startswith(xlib.get_gzip_code()+'-'): bioinfo_app_name = xlib.get_gzip_name() elif result_dataset_id.startswith(xlib.get_insilico_read_normalization_code()+'-'): bioinfo_app_name = xlib.get_insilico_read_normalization_name() elif result_dataset_id.startswith(xlib.get_miniconda3_code()+'-'): bioinfo_app_name = xlib.get_miniconda3_name() elif result_dataset_id.startswith(xlib.get_ngshelper_code()+'-'): bioinfo_app_name = xlib.get_ngshelper_name() elif result_dataset_id.startswith(xlib.get_quast_code()+'-'): bioinfo_app_name = xlib.get_quast_name() elif result_dataset_id.startswith(xlib.get_r_code()+'-'): bioinfo_app_name = xlib.get_r_name() elif result_dataset_id.startswith(xlib.get_ref_eval_code()+'-'): bioinfo_app_name = xlib.get_ref_eval_name() elif result_dataset_id.startswith(xlib.get_rnaquast_code()+'-'): bioinfo_app_name = xlib.get_rnaquast_name() elif result_dataset_id.startswith(xlib.get_rsem_code()+'-'): bioinfo_app_name = xlib.get_rsem_name() elif result_dataset_id.startswith(xlib.get_rsem_eval_code()+'-'): bioinfo_app_name = xlib.get_rsem_eval_name() elif result_dataset_id.startswith(xlib.get_samtools_code()+'-'): bioinfo_app_name = xlib.get_samtools_name() elif result_dataset_id.startswith(xlib.get_soapdenovotrans_code()+'-'): bioinfo_app_name = xlib.get_soapdenovotrans_name() elif result_dataset_id.startswith(xlib.get_star_code()+'-'): bioinfo_app_name = xlib.get_star_name() elif result_dataset_id.startswith(xlib.get_transabyss_code()+'-'): bioinfo_app_name = xlib.get_transabyss_name() elif result_dataset_id.startswith(xlib.get_transcript_filter_code()+'-'): bioinfo_app_name = xlib.get_transcript_filter_name() elif result_dataset_id.startswith(xlib.get_transcriptome_blastx_code()+'-'): bioinfo_app_name = xlib.get_transcriptome_blastx_name() elif result_dataset_id.startswith(xlib.get_transrate_code()+'-'): bioinfo_app_name = xlib.get_transrate_name() elif result_dataset_id.startswith(xlib.get_trimmomatic_code()+'-'): bioinfo_app_name = xlib.get_trimmomatic_name() elif result_dataset_id.startswith(xlib.get_trinity_code()+'-'): bioinfo_app_name = xlib.get_trinity_name() else: bioinfo_app_name = 'xxx' print(line_template.format(result_dataset_id, bioinfo_app_name)) # close the SSH client connection if OK: xssh.close_ssh_client_connection(ssh_client) # show continuation message print(xlib.get_separator()) input('Press [Intro] to continue ...')
def run_gzip_process(cluster_name, dataset_type, log, function=None): ''' Run a gzip process. ''' # initialize the control variable OK = True # get the gzip code and name gzip_code = xlib.get_gzip_code() gzip_name = xlib.get_gzip_name() # get the gzip option dictionary gzip_option_dict = xlib.get_option_dict(get_gzip_config_file(dataset_type)) # get the experiment identification experiment_id = gzip_option_dict['identification']['experiment_id'] # get the gzip process script path in the local computer gzip_process_script = get_gzip_process_script(dataset_type) # get the gzip process starter path in the local computer gzip_process_starter = get_gzip_process_starter(dataset_type) # warn that the log window does not have to be closed if not isinstance(log, xlib.DevStdOut): log.write('This process might take several minutes. Do not close this window, please wait!\n') # check the gzip config file log.write(f'{xlib.get_separator()}\n') log.write('Checking the {0} config file ...\n'.format(gzip_name)) (OK, error_list) = check_gzip_config_file(dataset_type, strict=True) if OK: log.write('The file is OK.\n') else: log.write('*** ERROR: The config file is not valid.\n') log.write('Please correct this file or recreate the config files.\n') # create the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Connecting the SSH client ...\n') (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name) if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write(f'{error}\n') # create the SSH transport connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Connecting the SSH transport ...\n') (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name) if OK: log.write('The SSH transport is connected.\n') else: for error in error_list: log.write(f'{error}\n') # create the SFTP client if OK: log.write(f'{xlib.get_separator()}\n') log.write('Connecting the SFTP client ...\n') sftp_client = xssh.create_sftp_client(ssh_transport) log.write('The SFTP client is connected.\n') # warn that the requirements are being verified if OK: log.write(f'{xlib.get_separator()}\n') log.write('Checking process requirements ...\n') # check the master is running if OK: (master_state_code, master_state_name) = xec2.get_node_state(cluster_name) if master_state_code != 16: log.write(f'*** ERROR: The cluster {cluster_name} is not running. Its state is {master_state_code} ({master_state_name}).\n') OK = False # warn that the requirements are OK if OK: log.write('Process requirements are OK.\n') # determine the run directory in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Determining the run directory in the cluster ...\n') if dataset_type == 'reference': current_run_dir = xlib.get_cluster_current_run_dir('reference', gzip_code) elif dataset_type == 'database': current_run_dir = xlib.get_cluster_current_run_dir('database', gzip_code) else: current_run_dir = xlib.get_cluster_current_run_dir(experiment_id, gzip_code) command = f'mkdir --parents {current_run_dir}' (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write(f'The directory path is {current_run_dir}.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # build the gzip process script if OK: log.write(f'{xlib.get_separator()}\n') log.write('Building the process script {0} ...\n'.format(gzip_process_script)) (OK, error_list) = build_gzip_process_script(cluster_name, dataset_type, current_run_dir) if OK: log.write('The file is built.\n') else: log.write('*** ERROR: The file could not be built.\n') # upload the gzip process script to the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Uploading the process script {0} to the directory {1} ...\n'.format(gzip_process_script, current_run_dir)) cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_script)) (OK, error_list) = xssh.put_file(sftp_client, gzip_process_script, cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write(f'{error}\n') # set run permision to the gzip process script in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(gzip_process_script))) command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_script)) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # build the gzip process starter if OK: log.write(f'{xlib.get_separator()}\n') log.write('Building the process starter {0} ...\n'.format(gzip_process_starter)) (OK, error_list) = build_gzip_process_starter(dataset_type, current_run_dir) if OK: log.write('The file is built.\n') else: log.write('***ERROR: The file could not be built.\n') # upload the gzip process starter to the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Uploading the process starter {0} to the directory {1} ...\n'.format(gzip_process_starter, current_run_dir)) cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_starter)) (OK, error_list) = xssh.put_file(sftp_client, gzip_process_starter, cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write(f'{error}\n') # set run permision to the gzip process starter in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(gzip_process_starter))) command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_starter)) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # submit the gzip process if OK: log.write(f'{xlib.get_separator()}\n') log.write('Submitting the process script {0}/{1} ...\n'.format(current_run_dir, os.path.basename(gzip_process_starter))) OK = xssh.submit_script(cluster_name, ssh_client, current_run_dir, os.path.basename(gzip_process_starter), log) # close the SSH transport connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Closing the SSH transport connection ...\n') xssh.close_ssh_transport_connection(ssh_transport) log.write('The connection is closed.\n') # close the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write(f'{xlib.get_separator()}\n') log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def install_node_infrastructure_software_ant(cluster_name, node_name, log): ''' Install infrastructure software in a node. ''' # initialize the control variable OK = True # create the SSH client connection log.write('{0}\n'.format(xlib.get_separator())) log.write('Connecting the SSH client in node {0} ...\n'.format(node_name)) (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, node_name) if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # update file /etc/apt/sources.list if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Updating the file /etc/apt/sources.list in node {0} ...\n'.format( node_name)) command = 'sed -i "s/us-east-1\.ec2\.archive\.ubuntu\.com/old-releases\.ubuntu\.com/g" /etc/apt/sources.list' (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) log.write('command: {0}\n'.format(command)) log.write('OK: {0}\n'.format(OK)) log.write('stdout: {0}\n'.format(stdout)) log.write('stderr: {0}\n'.format(stderr)) OK = True if OK: command = 'sed -i "s/security\.ubuntu\.com/old-releases\.ubuntu\.com/g" /etc/apt/sources.list' (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) log.write('command: {0}\n'.format(command)) log.write('OK: {0}\n'.format(OK)) log.write('stdout: {0}\n'.format(stdout)) log.write('stderr: {0}\n'.format(stderr)) OK = True if OK: command = 'export DEBIAN_FRONTEND=noninteractive; apt-get update' (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) log.write('command: {0}\n'.format(command)) log.write('OK: {0}\n'.format(OK)) log.write('stdout: {0}\n'.format(stdout)) log.write('stderr: {0}\n'.format(stderr)) OK = True if OK: log.write('The file is updated.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # install libtbb2 if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Installing the package libtbb2 in node {0} (it can be a very slow process, please be patient) ...\n' .format(node_name)) command = 'export DEBIAN_FRONTEND=noninteractive; apt-get --assume-yes --force-yes install libtbb2' (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) log.write('command: {0}\n'.format(command)) log.write('OK: {0}\n'.format(OK)) log.write('stdout: {0}\n'.format(stdout)) log.write('stderr: {0}\n'.format(stderr)) OK = True if OK: log.write('The package is installed.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) ## install OpenJDK8 #if OK: # log.write('{0}\n'.format(xlib.get_separator())) # log.write('Installing the package OpenJDK8 in node {0} (it can be a very slow process, please be patient) ...\n'.format(node_name)) # command = 'echo "deb http://ppa.launchpad.net/openjdk-r/ppa/ubuntu trusty main" | tee /etc/apt/sources.list.d/openjdk-r.list' # (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # log.write('command: {0}\n'.format(command)) # log.write('OK: {0}\n'.format(OK)) # log.write('stdout: {0}\n'.format(stdout)) # log.write('stderr: {0}\n'.format(stderr)) # OK = True #if OK: # command = 'echo "deb-src http://ppa.launchpad.net/openjdk-r/ppa/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/openjdk-r.list' # (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # log.write('command: {0}\n'.format(command)) # log.write('OK: {0}\n'.format(OK)) # log.write('stdout: {0}\n'.format(stdout)) # log.write('stderr: {0}\n'.format(stderr)) # OK = True #if OK: # #command = 'export DEBIAN_FRONTEND=noninteractive; apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886' # command = 'export DEBIAN_FRONTEND=noninteractive; apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EB9B1D8886F44E2A' # (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # log.write('command: {0}\n'.format(command)) # log.write('OK: {0}\n'.format(OK)) # log.write('stdout: {0}\n'.format(stdout)) # log.write('stderr: {0}\n'.format(stderr)) # OK = True #if OK: # command = 'export DEBIAN_FRONTEND=noninteractive; apt-get update' # (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # log.write('command: {0}\n'.format(command)) # log.write('OK: {0}\n'.format(OK)) # log.write('stdout: {0}\n'.format(stdout)) # log.write('stderr: {0}\n'.format(stderr)) # OK = True #if OK: # command = 'export DEBIAN_FRONTEND=noninteractive; apt-get --assume-yes --force-yes install openjdk-8-jdk' # (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # log.write('command: {0}\n'.format(command)) # log.write('OK: {0}\n'.format(OK)) # log.write('stdout: {0}\n'.format(stdout)) # log.write('stderr: {0}\n'.format(stderr)) # OK = True #if OK: # command = 'export DEBIAN_FRONTEND=noninteractive; update-java-alternatives --set java-1.8.0-openjdk-amd64' # (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # log.write('command: {0}\n'.format(command)) # log.write('OK: {0}\n'.format(OK)) # log.write('stdout: {0}\n'.format(stdout)) # log.write('stderr: {0}\n'.format(stderr)) # OK = True #if OK: # log.write('OpenJDK8 is installed.\n') #else: # log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # install Oracle Java 8 # -- if OK: # -- log.write('{0}\n'.format(xlib.get_separator())) # -- log.write('Installing Oracle Java 8 in node {0} (it can be a very slow process, please be patient) ...\n'.format(node_name)) # -- command = 'echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list' # -- (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # -- log.write('command: {0}\n'.format(command)) # -- log.write('OK: {0}\n'.format(OK)) # -- log.write('stdout: {0}\n'.format(stdout)) # -- log.write('stderr: {0}\n'.format(stderr)) # -- OK = True # -- if OK: # -- command = 'echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list' # -- (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # -- log.write('command: {0}\n'.format(command)) # -- log.write('OK: {0}\n'.format(OK)) # -- log.write('stdout: {0}\n'.format(stdout)) # -- log.write('stderr: {0}\n'.format(stderr)) # -- OK = True # -- if OK: # -- command = 'export DEBIAN_FRONTEND=noninteractive; apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886' # -- (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # -- log.write('command: {0}\n'.format(command)) # -- log.write('OK: {0}\n'.format(OK)) # -- log.write('stdout: {0}\n'.format(stdout)) # -- log.write('stderr: {0}\n'.format(stderr)) # -- OK = True # -- if OK: # -- command = 'export DEBIAN_FRONTEND=noninteractive; apt-get update' # -- (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # -- log.write('command: {0}\n'.format(command)) # -- log.write('OK: {0}\n'.format(OK)) # -- log.write('stdout: {0}\n'.format(stdout)) # -- log.write('stderr: {0}\n'.format(stderr)) # -- OK = True # -- if OK: # -- command = 'echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections' # -- (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # -- log.write('command: {0}\n'.format(command)) # -- log.write('OK: {0}\n'.format(OK)) # -- log.write('stdout: {0}\n'.format(stdout)) # -- log.write('stderr: {0}\n'.format(stderr)) # -- OK = True # -- if OK: # -- command = 'export DEBIAN_FRONTEND=noninteractive; apt-get --assume-yes --force-yes install openjdk-8-jdk' # -- (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) # -- log.write('command: {0}\n'.format(command)) # -- log.write('OK: {0}\n'.format(OK)) # -- log.write('stdout: {0}\n'.format(stdout)) # -- log.write('stderr: {0}\n'.format(stderr)) # -- OK = True # -- if OK: # -- log.write('Java 8 is installed.\n') # -- else: # -- log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # close the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # return the control variable return OK
def upload_database_dataset(cluster_name, log, function=None): ''' Upload the database dataset to the cluster. ''' # initialize the control variable OK = True # warn that the log window does not have to be closed if not isinstance(log, xlib.DevStdOut): log.write( 'This process might take several minutes. Do not close this window, please wait!\n' ) # check the database transfer config file log.write(f'{xlib.get_separator()}\n') log.write('Checking the database transfer config file ...\n') if check_database_transfer_config_file(strict=True): log.write('The file is OK.\n') else: log.write( '*** ERROR: The database transfer config file is not valid.\n') log.write('Please correct this file or recreate the config files.\n') OK = False # create the SSH client connection if OK: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name) for error in error_list: log.write(f'{error}\n') # create the SSH transport connection if OK: (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name) for error in error_list: log.write(f'{error}\n') # create the SFTP client if OK: sftp_client = xssh.create_sftp_client(ssh_transport) # upload the database dataset if OK: # get the option dictionary database_transfer_options_dict = xlib.get_option_dict( get_database_transfer_config_file()) # get the database dataset identification and the local directory of the database files database_dataset_id = database_transfer_options_dict['identification'][ 'database_dataset_id'] local_dir = database_transfer_options_dict['identification'][ 'local_dir'] # set the cluster database directory cluster_database_dir = '{0}/{1}'.format( xlib.get_cluster_database_dir(), database_dataset_id) # create the data directory in the cluster log.write(f'{xlib.get_separator()}\n') log.write( 'The database directory {0} in the cluster is being created ...\n'. format(cluster_database_dir)) command = 'mkdir --parents {0}'.format(cluster_database_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The directory is created.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # get the sections list sections_list = [] for section in database_transfer_options_dict.keys(): sections_list.append(section) sections_list.sort() # for each section "file-n" for section in sections_list: # check than the section identification is like file-n if re.match('^file-[0-9]+$', section): # get the file name file_name = database_transfer_options_dict[section][ 'file_name'] # set the local path and cluster path local_path = '{0}/{1}'.format(local_dir, file_name) cluster_path = '{0}/{1}'.format(cluster_database_dir, file_name) # upload the database file to the cluster log.write(f'{xlib.get_separator()}\n') log.write('The file {0} is being uploaded to {1} ...\n'.format( file_name, cluster_database_dir)) (OK, error_list) = xssh.put_file(sftp_client, local_path, cluster_path) if OK: log.write('The file has been uploaded.\n') else: for error in error_list: log.write(f'{error}\n') break # close the SSH transport connection if OK: xssh.close_ssh_transport_connection(ssh_transport) # close the SSH client connection if OK: xssh.close_ssh_client_connection(ssh_client) # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write(f'{xlib.get_separator()}\n') log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def run_gmap_process(cluster_name, log, function=None): ''' Run a GMAP process. ''' # initialize the control variable OK = True # get the GMAP option dictionary gmap_option_dict = xlib.get_option_dict(get_gmap_config_file()) # get the experiment identification experiment_id = gmap_option_dict['identification']['experiment_id'] # warn that the log window must not be closed if not isinstance(log, xlib.DevStdOut): log.write('This process might take several minutes. Do not close this window, please wait!\n') # validate the GMAP config file log.write('{0}\n'.format(xlib.get_separator())) log.write('Validating the {0} config file ...\n'.format(xlib.get_gmap_name())) (OK, error_list) = validate_gmap_config_file(strict=True) if OK: log.write('The config file is OK.\n') else: log.write('*** ERROR: The config file is not valid.\n') log.write('Please correct this file or recreate the config files.\n') # create the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Connecting the SSH client ...\n') (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master') if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # create the SSH transport connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Connecting the SSH transport ...\n') (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name, 'master') if OK: log.write('The SSH transport is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # create the SFTP client if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Connecting the SFTP client ...\n') sftp_client = xssh.create_sftp_client(ssh_transport) log.write('The SFTP client is connected.\n') # warn that the requirements are being verified if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Verifying process requirements ...\n') # verify the master is running if OK: (master_state_code, master_state_name) = xec2.get_node_state(cluster_name, 'master') if master_state_code != 16: log.write('*** ERROR: The cluster {0} is not running. Its state is {1} ({2}).\n'.format(cluster_name, master_state_code, master_state_name)) OK = False # verify the GMAP-GSNAP is setup if OK: (OK, error_list, is_setup) = xbioinfoapp.is_setup_bioconda_package(xlib.get_gmap_gsnap_bioconda_code(), cluster_name, True, ssh_client) if OK: if not is_setup: log.write('*** ERROR: {0} is not setup.\n'.format(xlib.get_gmap_name())) OK = False else: log.write('*** ERROR: The verification of {0} setup could not be performed.\n'.format(xlib.get_gmap_name())) # warn that the requirements are OK if OK: log.write('Process requirements are OK.\n') # determine the run directory in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Determining the run directory in the cluster ...\n') current_run_dir = xlib.get_cluster_current_run_dir(experiment_id, xlib.get_gmap_code()) command = 'mkdir --parents {0}'.format(current_run_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The directory path is {0}.\n'.format(current_run_dir)) else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # build the GMAP process script if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Building the process script {0} ...\n'.format(get_gmap_process_script())) (OK, error_list) = build_gmap_process_script(cluster_name, current_run_dir) if OK: log.write('The file is built.\n') if not OK: log.write('*** ERROR: The file could not be built.\n') # upload the GMAP process script in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Uploading the process script {0} in the directory {1} of the master ...\n'.format(get_gmap_process_script(), current_run_dir)) cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_script())) (OK, error_list) = xssh.put_file(sftp_client, get_gmap_process_script(), cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # set run permision to the GMAP process script in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(get_gmap_process_script()))) command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_script())) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # build the GMAP process starter if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Building the process starter {0} ...\n'.format(get_gmap_process_starter())) (OK, error_list) = build_gmap_process_starter(current_run_dir) if OK: log.write('The file is built.\n') if not OK: log.write('***ERROR: The file could not be built.\n') # upload the GMAP process starter in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Uploading the process starter {0} in the directory {1} of the master ...\n'.format(get_gmap_process_starter(), current_run_dir)) cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_starter())) (OK, error_list) = xssh.put_file(sftp_client, get_gmap_process_starter(), cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # set run permision to the GMAP process starter in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(get_gmap_process_starter()))) command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_starter())) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # submit the GMAP process if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Submitting the process script {0}/{1} ...\n'.format(current_run_dir, os.path.basename(get_gmap_process_starter()))) sge_env = xcluster.get_sge_env() command = '{0}; qsub -V -b n -cwd {1}/{2}'.format(sge_env, current_run_dir, os.path.basename(get_gmap_process_starter())) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: for line in stdout: log.write('{0}\n'.format(line)) else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # close the SSH transport connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH transport connection ...\n') xssh.close_ssh_transport_connection(ssh_transport) log.write('The connection is closed.\n') # close the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write('{0}\n'.format(xlib.get_separator())) log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def install_node_infrastructure_software(cluster_name, node_name, log): ''' Install infraestructure software in a node. ''' # initialize the control variable OK = True # get the infrastructure software installation script path in local compute local_script_path = get_infrastructure_software_installation_script() # set the infrastructure software installation script path in node node_script_path = './{0}'.format(os.path.basename(local_script_path)) # set the infrastructure software installation log path in node node_log_path = node_script_path[:node_script_path.find('.sh')] + '.log' # build the infrastructure software installation script if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Building the infrastructure software installation script {0} ...\n' .format(local_script_path)) (OK, error_list ) = build_infrastructure_software_installation_script(cluster_name) if OK: log.write('The file is built.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # create the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Connecting the SSH client to node {0} ...\n'.format(node_name)) (OK, error_list, ssh_client) = xssh.create_ssh_client_connection( cluster_name, node_name) if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # create the SSH transport connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Connecting the SSH transport to node {0} ...\n'.format(node_name)) (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection( cluster_name, node_name) if OK: log.write('The SSH transport is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # create the SFTP client if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Connecting the SFTP client to node {0} ...\n'.format(node_name)) sftp_client = xssh.create_sftp_client(ssh_transport) log.write('The SFTP client is connected.\n') # upload the infraestructe software installation script to the node if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Uploading the infraestructe software installation script to the node {0} ...\n' .format(node_name)) (OK, error_list) = xssh.put_file(sftp_client, local_script_path, node_script_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # set run permision to the infraestructe software installation script in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Setting on the run permision ...\n') command = 'chmod u+x {0}'.format(node_script_path) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # submit the infraestructe software installation script if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Submitting the infraestructe software installation script in the node {0} ...\n' .format(node_name)) command = '{0} &>{1} &'.format(node_script_path, node_log_path) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The script is submitted.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # close the SSH transport connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH transport connection ...\n') xssh.close_ssh_transport_connection(ssh_transport) log.write('The connection is closed.\n') # close the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # return the control variable return OK
def execute(self, event=None): ''' Execute the list the result logs in the cluster. ''' # validate inputs OK = self.validate_inputs() if not OK: message = 'Some input values are not OK.' tkinter.messagebox.showerror('{0} - {1}'.format(xlib.get_project_name(), self.head), message) # get the run dictionary of the experiment if OK: # -- command = 'ls {0}/{1}'.format(xlib.get_cluster_result_dir(), self.wrapper_experiment_id.get()) command = 'cd {0}/{1}; for list in `ls`; do ls -ld $list | grep -v ^- > /dev/null && echo $list; done;'.format(xlib.get_cluster_result_dir(), self.wrapper_experiment_id.get()) (OK, stdout, stderr) = xssh.execute_cluster_command(self.ssh_client, command) if OK: result_dataset_dict = {} for line in stdout: line = line.rstrip('\n') if line != 'lost+found': result_dataset_id = line try: pattern = r'^(.+)\-(.+)\-(.+)$' mo = re.search(pattern, result_dataset_id) bioinfo_app_code = mo.group(1).strip() yymmdd = mo.group(2) hhmmss = mo.group(3) date = '20{0}-{1}-{2}'.format(yymmdd[:2], yymmdd[2:4], yymmdd[4:]) time = '{0}:{1}:{2}'.format(hhmmss[:2], hhmmss[2:4], hhmmss[4:]) except: bioinfo_app_code = 'xxx' date = '0000-00-00' time = '00:00:00' if result_dataset_id.startswith(xlib.get_bedtools_code()+'-'): bioinfo_app_name = xlib.get_bedtools_name() elif result_dataset_id.startswith(xlib.get_blastplus_code()+'-'): bioinfo_app_name = xlib.get_blastplus_name() elif result_dataset_id.startswith(xlib.get_bowtie2_code()+'-'): bioinfo_app_name = xlib.get_bowtie2_name() elif result_dataset_id.startswith(xlib.get_busco_code()+'-'): bioinfo_app_name = xlib.get_busco_name() elif result_dataset_id.startswith(xlib.get_cd_hit_code()+'-'): bioinfo_app_name = xlib.get_cd_hit_name() elif result_dataset_id.startswith(xlib.get_cd_hit_est_code()+'-'): bioinfo_app_name = xlib.get_cd_hit_est_name() elif result_dataset_id.startswith(xlib.get_detonate_code()+'-'): bioinfo_app_name = xlib.get_detonate_name() elif result_dataset_id.startswith(xlib.get_emboss_code()+'-'): bioinfo_app_name = xlib.get_emboss_name() elif result_dataset_id.startswith(xlib.get_fastqc_code()+'-'): bioinfo_app_name = xlib.get_fastqc_name() elif result_dataset_id.startswith(xlib.get_gmap_code()+'-'): bioinfo_app_name = xlib.get_gmap_name() elif result_dataset_id.startswith(xlib.get_gmap_gsnap_code()+'-'): bioinfo_app_name = xlib.get_gmap_gsnap_name() elif result_dataset_id.startswith(xlib.get_gzip_code()+'-'): bioinfo_app_name = xlib.get_gzip_name() elif result_dataset_id.startswith(xlib.get_insilico_read_normalization_code()+'-'): bioinfo_app_name = xlib.get_insilico_read_normalization_name() elif result_dataset_id.startswith(xlib.get_miniconda3_code()+'-'): bioinfo_app_name = xlib.get_miniconda3_name() elif result_dataset_id.startswith(xlib.get_ngshelper_code()+'-'): bioinfo_app_name = xlib.get_ngshelper_name() elif result_dataset_id.startswith(xlib.get_quast_code()+'-'): bioinfo_app_name = xlib.get_quast_name() elif result_dataset_id.startswith(xlib.get_r_code()+'-'): bioinfo_app_name = xlib.get_r_name() elif result_dataset_id.startswith(xlib.get_ref_eval_code()+'-'): bioinfo_app_name = xlib.get_ref_eval_name() elif result_dataset_id.startswith(xlib.get_rnaquast_code()+'-'): bioinfo_app_name = xlib.get_rnaquast_name() elif result_dataset_id.startswith(xlib.get_rsem_code()+'-'): bioinfo_app_name = xlib.get_rsem_name() elif result_dataset_id.startswith(xlib.get_rsem_eval_code()+'-'): bioinfo_app_name = xlib.get_rsem_eval_name() elif result_dataset_id.startswith(xlib.get_samtools_code()+'-'): bioinfo_app_name = xlib.get_samtools_name() elif result_dataset_id.startswith(xlib.get_soapdenovotrans_code()+'-'): bioinfo_app_name = xlib.get_soapdenovotrans_name() elif result_dataset_id.startswith(xlib.get_star_code()+'-'): bioinfo_app_name = xlib.get_star_name() elif result_dataset_id.startswith(xlib.get_transabyss_code()+'-'): bioinfo_app_name = xlib.get_transabyss_name() elif result_dataset_id.startswith(xlib.get_transcript_filter_code()+'-'): bioinfo_app_name = xlib.get_transcript_filter_name() elif result_dataset_id.startswith(xlib.get_transcriptome_blastx_code()+'-'): bioinfo_app_name = xlib.get_transcriptome_blastx_name() elif result_dataset_id.startswith(xlib.get_transrate_code()+'-'): bioinfo_app_name = xlib.get_transrate_name() elif result_dataset_id.startswith(xlib.get_trimmomatic_code()+'-'): bioinfo_app_name = xlib.get_trimmomatic_name() elif result_dataset_id.startswith(xlib.get_trinity_code()+'-'): bioinfo_app_name = xlib.get_trinity_name() else: bioinfo_app_name = 'xxx' result_dataset_dict[result_dataset_id] = {'experiment_id': self.wrapper_experiment_id.get(), 'result_dataset_id': result_dataset_id, 'bioinfo_app': bioinfo_app_name, 'date': date, 'time': time} # verify if there are any nodes running if OK: if result_dataset_dict == {}: message = 'There is not any run.' tkinter.messagebox.showwarning('{0} - {1}'.format(xlib.get_project_name(), self.head), message) # build the data list if OK: data_list = ['experiment_id', 'result_dataset_id', 'bioinfo_app', 'date', 'time'] # build the data dictionary if OK: data_dict = {} data_dict['experiment_id']= {'text': 'Experiment id. / Process', 'width': 200, 'aligment': 'left'} data_dict['result_dataset_id'] = {'text': 'Result dataset', 'width': 200, 'aligment': 'left'} data_dict['bioinfo_app'] = {'text': 'Bioinfo app / Utility', 'width': 200, 'aligment': 'left'} data_dict['date'] = {'text': 'Date', 'width': 80, 'aligment': 'right'} data_dict['time'] = {'text': 'Time', 'width': 80, 'aligment': 'right'} # create the dialog Table to show the nodes running if OK: dialog_table = gdialogs.DialogTable(self, 'Experiment runs in {0}/{1}'.format(xlib.get_cluster_result_dir(), self.wrapper_experiment_id.get()), 400, 900, data_list, data_dict, result_dataset_dict, 'view_result_logs', [self.wrapper_cluster_name.get()]) self.wait_window(dialog_table) # close the form if OK: self.close()
def get_batch_job_dict(ssh_client): ''' Get a dictionary with the batch jobs in the cluster. ''' # initialize the control variable and the error list OK = True error_list = [] # initialize the dictionary of the batch jobs batch_job_dict = {} # list the jobs running in the cluster and load the dictionary if OK: sge_env = get_sge_env() command = '{0}; qstat'.format(sge_env) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: if len(stdout) > 0: i = 0 for line in stdout: if i >= 2: try: line_pattern = r'^(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(.+)$' mo = re.search(line_pattern, line.strip()) job_id = mo.group(1).strip() priority = mo.group(2).strip() job_name = mo.group(3).strip() user = mo.group(4).strip() state_id = mo.group(5).strip() if state_id == 'd': state_name = 'deletion' elif state_id == 'E': state_name = 'error' elif state_id == 'h': state_name = 'hold' elif state_id == 'r': state_name = 'running' elif state_id == 'R': state_name = 'restarted' elif state_id in ['s', 'S']: state_name = 'suspended' elif state_id == 't': state_name = 'transfering' elif state_id == 'T': state_name = 'thresold' elif state_id in ['w', 'qw']: state_name = 'waiting' else: state_name = 'xxx' state = '{0} ({1})'.format(state_id, state_name) mm_dd_yyyy = mo.group(6).strip() start_date = '{0}-{1}-{2}'.format( mm_dd_yyyy[6:], mm_dd_yyyy[:2], mm_dd_yyyy[3:5]) start_time = mo.group(7).strip() batch_job_dict[job_id] = { 'job_id': job_id, 'priority': priority, 'job_name': job_name, 'user': user, 'state_id': state_id, 'state_name': state_name, 'state': state, 'start_date': start_date, 'start_time': start_time } except Exception as e: pass i += 1 else: error_list.append( '*** ERROR: Wrong command ---> {0}.\n'.format(command)) # return the control variable, error list and dictionary of the batch jobs return (OK, error_list, batch_job_dict)
def get_read_dataset_dict(cluster_name, experiment_id, passed_connection=False, ssh_client=None): ''' Get a dictoinary with the read datasets of an experiment in the cluster. ''' # initialize the control variable and the error list OK = True error_list = [] # get the read directory in the cluster cluster_read_dir = xlib.get_cluster_read_dir() # initialize the dictionary of the read datasets read_dataset_dict = {} # create the SSH client connection if not passed_connection: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection( cluster_name, 'master') # verify the read directory is created if OK: command = '[ -d {0} ] && echo RC=0 || echo RC=1'.format( cluster_read_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if stdout[len(stdout) - 1] != 'RC=0': error_list.append( '*** ERROR: There is not any volume mounted in the read directory.\n' ) error_list.append( 'You must link a volume in the mounting point {0} for the template {1}.\n' .format(cluster_read_dir, cluster_name)) OK = False # get the dictionary of the read datasets if OK: command = 'ls {0}/{1}'.format(cluster_read_dir, experiment_id) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: for line in stdout: line = line.rstrip('\n') if line != 'lost+found': read_dataset_id = line if read_dataset_id == xlib.get_uploaded_read_dataset_name( ): read_dataset_name = ' uploaded reads' elif read_dataset_id.startswith( xlib.get_insilico_read_normalization_code()): mo = re.match( '{0}-(.+)-(.+)'.format( xlib.get_insilico_read_normalization_code()), read_dataset_id) date = mo.group(1) time = mo.group(2) read_dataset_name = '{0} ({1} {2})'.format( xlib.get_insilico_read_normalization_name(), date, time) elif read_dataset_id.startswith( xlib.get_trimmomatic_code()): mo = re.match( '{0}-(.+)-(.+)'.format( xlib.get_trimmomatic_code()), read_dataset_id) date = mo.group(1) time = mo.group(2) read_dataset_name = '{0} ({1} {2})'.format( xlib.get_trimmomatic_name(), date, time) else: read_dataset_name = 'xxx' read_dataset_dict[read_dataset_id] = { 'read_dataset_id': read_dataset_id, 'read_dataset_name': read_dataset_name } # close the SSH client connection if OK and not passed_connection: xssh.close_ssh_client_connection(ssh_client) # return the control variable, error list and dictionary of the read datasets return (OK, error_list, read_dataset_dict)
def get_result_dataset_dict(cluster_name, experiment_id, status, passed_connection, ssh_client): ''' Get a dictionary with the result datasets of an experiment in the cluster. ''' # initialize the control variable and the error list OK = True error_list = [] # get the result directory in the cluster cluster_result_dir = xlib.get_cluster_result_dir() # initialize the dictionary of the result datasets result_dataset_dict = {} # create the SSH client connection if not passed_connection: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master') # verify the result directory is created if OK: command = '[ -d {0} ] && echo RC=0 || echo RC=1'.format(cluster_result_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if stdout[len(stdout) - 1] != 'RC=0': error_list.append('*** ERROR: There is not any volume mounted in the result directory.\n') error_list.append('You must link a volume in the mounting point {0} for the template {1}.\n'.format(cluster_result_dir, cluster_name)) OK = False # get the dictionary of the result datasets if OK: if status == 'uncompressed': command = 'cd {0}/{1}; for list in `ls`; do ls -ld $list | grep -v ^- > /dev/null && echo $list; done;'.format(cluster_result_dir, experiment_id) elif status == 'compressed': command = 'cd {0}/{1}; for list in `ls`; do ls -ld $list | grep -v ^d > /dev/null && echo $list; done;'.format(cluster_result_dir, experiment_id) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: if status == 'uncompressed': input_pattern = '{0}-(.+)-(.+)' output_pattern = '{0} ({1} {2})' elif status == 'compressed': input_pattern = '{0}-(.+)-(.+).tar.gz' output_pattern = '{0} ({1} {2}) [compressed]' for line in stdout: line = line.rstrip('\n') if line != 'lost+found': result_dataset_id = line if result_dataset_id.startswith(xlib.get_cd_hit_est_code()+'-'): mo = re.match(input_pattern.format(xlib.get_cd_hit_est_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_cd_hit_est_name(), date, time) elif result_dataset_id.startswith(xlib.get_fastqc_code()+'-'): mo = re.match(input_pattern.format(xlib.get_fastqc_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_fastqc_name(), date, time) elif result_dataset_id.startswith(xlib.get_gzip_code()+'-'): mo = re.match(input_pattern.format(xlib.get_gzip_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_gzip_name(), date, time) elif result_dataset_id.startswith(xlib.get_insilico_read_normalization_code()+'-'): mo = re.match(input_pattern.format(xlib.get_insilico_read_normalization_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_insilico_read_normalization_name(), date, time) elif result_dataset_id.startswith(xlib.get_quast_code()+'-'): mo = re.match(input_pattern.format(xlib.get_quast_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_quast_name(), date, time) elif result_dataset_id.startswith(xlib.get_ref_eval_code()+'-'): mo = re.match(input_pattern.format(xlib.get_ref_eval_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_ref_eval_name(), date, time) elif result_dataset_id.startswith(xlib.get_rnaquast_code()+'-'): mo = re.match(input_pattern.format(xlib.get_rnaquast_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_rnaquast_name(), date, time) elif result_dataset_id.startswith(xlib.get_rsem_eval_code()+'-'): mo = re.match(input_pattern.format(xlib.get_rsem_eval_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_rsem_eval_name(), date, time) elif result_dataset_id.startswith(xlib.get_soapdenovotrans_code()+'-'): mo = re.match(input_pattern.format(xlib.get_soapdenovotrans_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_soapdenovotrans_name(), date, time) elif result_dataset_id.startswith(xlib.get_star_code()+'-'): mo = re.match(input_pattern.format(xlib.get_star_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_star_name(), date, time) elif result_dataset_id.startswith(xlib.get_transabyss_code()+'-'): mo = re.match(input_pattern.format(xlib.get_transabyss_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_transabyss_name(), date, time) elif result_dataset_id.startswith(xlib.get_transcript_filter_code()+'-'): mo = re.match(input_pattern.format(xlib.get_transcript_filter_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_transcript_filter_name(), date, time) elif result_dataset_id.startswith(xlib.get_transcriptome_blastx_code()+'-'): mo = re.match(input_pattern.format(xlib.get_transcriptome_blastx_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_transcriptome_blastx_name(), date, time) elif result_dataset_id.startswith(xlib.get_transrate_code()+'-'): mo = re.match(input_pattern.format(xlib.get_transrate_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_transrate_name(), date, time) elif result_dataset_id.startswith(xlib.get_trimmomatic_code()+'-'): mo = re.match(input_pattern.format(xlib.get_trimmomatic_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_trimmomatic_name(), date, time) elif result_dataset_id.startswith(xlib.get_trinity_code()+'-'): mo = re.match(input_pattern.format(xlib.get_trinity_code()), result_dataset_id) date = mo.group(1) time = mo.group(2) result_dataset_name = output_pattern.format(xlib.get_trinity_name(), date, time) else: result_dataset_name = result_dataset_id result_dataset_dict[result_dataset_id] = {'result_dataset_id': result_dataset_id, 'result_dataset_name': result_dataset_name} # close the SSH client connection if OK and not passed_connection: xssh.close_ssh_client_connection(ssh_client) # return the control variable, error list and dictionary of the result datasets return (OK, error_list, result_dataset_dict)
def create_volume(volume_name, volume_type, volume_size, terminate_indicator, log, function=None): ''' Create a volume in the current zone. ''' # initialize the control variable OK = True # get the volume creator name volume_creator_name = xlib.get_volume_creator_name() # warn that the log window must not be closed if not isinstance(log, xlib.DevStdOut): log.write( 'This process might take a few minutes. Do not close this window, please wait!\n' ) # verify the volume creator is running if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Verifing if volume creator is running ...\n') (master_state_code, master_state_name) = xec2.get_node_state(volume_creator_name, 'master') if master_state_code == 16: log.write('The volume creator is running.\n') else: log.write( '*** WARNING: The volume creator is not running. It will be created.\n' ) (OK, master_state_code, master_state_name) = xcluster.create_cluster(volume_creator_name, volume_creator_name, log, function=None, is_menu_call=False) # get the master node identification if OK: node_id = xec2.get_node_id(volume_creator_name, 'master') if node_id == '': log.write( '*** ERROR: The master identification of the volume creator not has been got.\n' ) OK = False # create the SSH client connection log.write('{0}\n'.format(xlib.get_separator())) log.write('Connecting SSH client ...\n') (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(volume_creator_name, 'master') if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # warn that the requirements are being verified log.write('{0}\n'.format(xlib.get_separator())) log.write('Verifying process requirements ...\n') # get current region and zone names region_name = xconfiguration.get_current_region_name() zone_name = xconfiguration.get_current_zone_name() # verify that the zone is available if OK: if not xec2.is_zone_available(region_name, zone_name): log.write( '*** ERROR: The zone {0} is not available in the region {1}.\n' .format(zone_name, region_name)) OK = False # verify that the volume is not created if OK: if xec2.is_volume_created(volume_name, zone_name): log.write( '*** WARNING: The volume {0} is already created.\n'.format( volume_name)) OK = False # warn that the requirements are OK if OK: log.write('Process requirements are OK.\n') # create the volume if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Creating volume {0} ...\n'.format(volume_name)) (OK, volume_id) = xec2.create_volume(volume_name, volume_type, volume_size) if OK: log.write( 'The volume is created with the identification {0}.\n'.format( volume_id)) else: log.write('*** ERROR: The volume is not created.\n') # wait for the volume status to be available if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Waiting for the volume state to be available ...\n') i = 0 available_state_timeout = 120 while True: i += 1 volume_status = xec2.get_volume_state(volume_name, zone_name) time.sleep(1) if volume_status == 'available': log.write('The volume is now available.\n') break elif i > available_state_timeout: log.write( '*** The volume is not available after {0} s.\n'.format( available_state_timeout)) Ok = False break # set the aws device and get de machine device if OK: aws_device = '/dev/sdp' machine_device = xlib.get_machine_device_file(aws_device) # attach the volume to the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Attaching volume {0} to node master of volume creator ...\n'. format(volume_name)) OK = xec2.attach_volume(node_id, volume_id, aws_device) if OK: log.write('The volume is attached.\n') else: log.write('*** ERROR: The volume is not attached.\n') # wait for the volume attachment to be available if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Waiting for the volume attachment to be available ...\n') i = 0 attachment_timeout = 120 while True: i += 1 volume_attachments = xec2.get_volume_attachments( volume_name, zone_name) if volume_attachments != []: log.write('The volume attachment is now available.\n') break elif i > attachment_timeout: log.write( '*** ERROR: The volume attachment is not available after {0} s.\n' .format(attachment_timeout)) Ok = False break time.sleep(1) # wait for the device availability if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Waiting for the availabity of the device {0} ...\n'.format( machine_device)) i = 0 format_timeout = 120 try: while True: i += 1 command = 'hdparm -z {0}'.format(machine_device) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) command = 'lsblk --list --noheadings --output NAME' (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) for line in stdout: if line == os.path.basename(machine_device): log.write('The device is available.\n') raise xlib.BreakAllLoops if i > format_timeout: log.write( '*** ERROR: The device is not available after {0} s.\n' .format(format_timeout)) OK = False break time.sleep(1) except xlib.BreakAllLoops: pass # format the volume if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Formating volume {0} to ext4 file system type ...\n'.format( volume_name)) command = 'mkfs -t ext4 {0} 2>&1; RC=$?; echo "RC=$RC"'.format( machine_device) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if stdout[len(stdout) - 1] == 'RC=0': log.write('The volume is formatted.\n') OK = True else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) OK = False # detach the volume to the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Detaching volume {0} from node master of volume creator ...\n'. format(volume_name)) OK = xec2.detach_volume(node_id, volume_id, aws_device) if OK: log.write('The volume is detached.\n') else: log.write('*** ERROR: The volume is not detached.\n') # terminate volume creator if OK: if terminate_indicator: OK = xcluster.terminate_cluster(volume_creator_name, True, log, function=None, is_menu_call=False) else: log.write('{0}\n'.format(xlib.get_separator())) log.write('You do not indicate to terminate the volume creator.\n') log.write( 'Remember to terminate it when you finish to create the volumes!!!\n' ) # close the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write('{0}\n'.format(xlib.get_separator())) log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def install_node_infrastructure_software(cluster_name, node_name, log): ''' Install infrastructure software in a node. ''' # initialize the control variable OK = True # get the infrastructure software installation script path in local compute local_script_path = get_infrastructure_software_installation_script() # set the infrastructure software installation script path in node node_script_path = f'./{os.path.basename(local_script_path)}' # set the infrastructure software installation log path in node node_log_path = node_script_path[:node_script_path.find('.sh')] + '.log' # build the infrastructure software installation script if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Building the infrastructure software installation script {local_script_path} ...\n' ) (OK, error_list ) = build_infrastructure_software_installation_script(cluster_name) if OK: log.write('The file is built.\n') else: for error in error_list: log.write(f'{error}\n') # create the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write(f'Connecting the SSH client to node {node_name} ...\n') (OK, error_list, ssh_client) = xssh.create_ssh_client_connection( cluster_name, node_name) if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write(f'{error}\n') # create the SSH transport connection if OK: log.write(f'{xlib.get_separator()}\n') log.write(f'Connecting the SSH transport to node {node_name} ...\n') (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection( cluster_name, node_name) if OK: log.write('The SSH transport is connected.\n') else: for error in error_list: log.write(f'{error}\n') # create the SFTP client if OK: log.write(f'{xlib.get_separator()}\n') log.write(f'Connecting the SFTP client to node {node_name} ...\n') sftp_client = xssh.create_sftp_client(ssh_transport) log.write('The SFTP client is connected.\n') # download the AWSCLI2 compressed file to local computer if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Downloading the {xlib.get_awscli_name()} compressed file to local computer ...\n' ) local_path = f'{xlib.get_temp_dir()}/{xlib.get_awscli_name()}.zip' if not os.path.exists(os.path.dirname(local_path)): os.makedirs(os.path.dirname(local_path)) try: urllib.request.urlretrieve(xlib.get_awscli_url(), local_path) except Exception as e: log.write(f'*** EXCEPTION: "{e}".') log.write( f'*** ERROR: The file {xlib.get_awscli_url()} can not be downloaded.\n' ) OK = False else: log.write('The file is downloaded.\n') # upload the AWSCLI2 compressed file to cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Uploading the {xlib.get_awscli_name()} compressed file to the cluster ...\n' ) cluster_path = f'./{os.path.basename(local_path)}' (OK, error_list) = xssh.put_file(sftp_client, local_path, cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write(f'{error}\n') # upload the infraestructe software installation script to the node if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Uploading the infraestructe software installation script to the node {node_name} ...\n' ) (OK, error_list) = xssh.put_file(sftp_client, local_script_path, node_script_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write(f'{error}\n') # set run permision to the infraestructe software installation script in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Setting on the run permision ...\n') command = f'chmod u+x {node_script_path}' (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # submit the infraestructe software installation script if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Submitting the infraestructe software installation script in the node {node_name} ...\n' ) command = f'{node_script_path} &>{node_log_path} &' (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The script is submitted.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # close the SSH transport connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Closing the SSH transport connection ...\n') xssh.close_ssh_transport_connection(ssh_transport) log.write('The connection is closed.\n') # close the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # return the control variable return OK
def unmount_volume(cluster_name, node_name, volume_name, log, function=None, is_menu_call=True): ''' Unmount a volume in a node. ''' # initialize the control variable OK = True # warn that the log window must not be closed if not isinstance(log, xlib.DevStdOut) and is_menu_call: log.write( 'This process might take a few minutes. Do not close this window, please wait!\n' ) # create the SSH client connection log.write('{0}\n'.format(xlib.get_separator())) log.write('Connecting SSH client ...\n') (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master') if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # warn that the requirements are being verified if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Verifying process requirements ...\n') # verify the master is running if OK: (master_state_code, master_state_name) = xec2.get_node_state(cluster_name, 'master') if master_state_code != 16: log.write( '*** ERROR: The cluster {0} is not running. Its state is {1} ({2}).\n' .format(cluster_name, master_state_code, master_state_name)) OK = False # get the zone name of the node if OK: zone_name = xec2.get_node_zone_name(cluster_name, node_name) # get the node identification if OK: node_id = xec2.get_node_id(cluster_name, node_name) if node_id == '': log.write( '*** ERROR: The {0} identification of the cluster {1} not has been got.\n' .format(node_name, cluster_name)) OK = False # verify the volume is created if OK: if not xec2.is_volume_created(volume_name, zone_name): log.write('*** ERROR: The volume {0} is not created.\n'.format( volume_name)) OK = False # get AWS file device if OK: aws_device_file = xec2.get_volume_device_file(cluster_name, node_name, volume_name) if aws_device_file == '': log.write( '*** ERROR: the file device of the volume is not found.\n') OK = False # get the volume identification if OK: volume_id = xec2.get_volume_id(volume_name, zone_name) if volume_id == '': log.write( '*** ERROR: The identificaction of volume {0} not has been got.\n' .format(volume_name)) OK = False # get de file device file if OK: machine_device_file = xlib.get_machine_device_file(aws_device_file) # warn that the requirements are OK if OK: log.write('Process requirements are OK.\n') # unmount the volume to the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Unmounting volume {0} from the device file {1} ...\n'.format( volume_name, machine_device_file)) command = 'umount {0}'.format(machine_device_file) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The volume is unmounted.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # detach the volume to the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Detaching volume {0} from node {1} of cluster {2} ...\n'.format( volume_name, node_name, cluster_name)) OK = xec2.detach_volume(node_id, volume_id, aws_device_file) if OK: log.write('The volume is detached.\n') else: log.write('*** ERROR: The volume is not detached.\n') # close the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write('{0}\n'.format(xlib.get_separator())) log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def run_htseq_count_process(cluster_name, log, function=None): ''' Run a htseq-count process. ''' # initialize the control variable OK = True # get the htseq-count option dictionary htseq_count_option_dict = xlib.get_option_dict( get_htseq_count_config_file()) # get the experiment identification experiment_id = htseq_count_option_dict['identification']['experiment_id'] # warn that the log window does not have to be closed if not isinstance(log, xlib.DevStdOut): log.write( 'This process might take several minutes. Do not close this window, please wait!\n' ) # check the htseq-count config file log.write(f'{xlib.get_separator()}\n') log.write(f'Checking the {xlib.get_htseq_count_name()} config file ...\n') (OK, error_list) = check_htseq_count_config_file(strict=True) if OK: log.write('The file is OK.\n') else: log.write('*** ERROR: The config file is not valid.\n') log.write('Please correct this file or recreate the config files.\n') # create the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Connecting the SSH client ...\n') (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name) if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write(f'{error}\n') # create the SSH transport connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Connecting the SSH transport ...\n') (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name) if OK: log.write('The SSH transport is connected.\n') else: for error in error_list: log.write(f'{error}\n') # create the SFTP client if OK: log.write(f'{xlib.get_separator()}\n') log.write('Connecting the SFTP client ...\n') sftp_client = xssh.create_sftp_client(ssh_transport) log.write('The SFTP client is connected.\n') # warn that the requirements are being verified if OK: log.write(f'{xlib.get_separator()}\n') log.write('Checking process requirements ...\n') # check the master is running if OK: (master_state_code, master_state_name) = xec2.get_node_state(cluster_name) if master_state_code != 16: log.write( f'*** ERROR: The cluster {cluster_name} is not running. Its state is {master_state_code} ({master_state_name}).\n' ) OK = False # check HTSeq is installed if OK: (OK, error_list, is_installed) = xbioinfoapp.is_installed_anaconda_package( xlib.get_htseq_anaconda_code(), cluster_name, True, ssh_client) if OK: if not is_installed: log.write( f'*** ERROR: {xlib.get_htseq_name()} is not installed.\n') OK = False else: log.write( f'*** ERROR: The verification of {xlib.get_htseq_name()} installation could not be performed.\n' ) # warn that the requirements are OK if OK: log.write('Process requirements are OK.\n') # determine the run directory in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Determining the run directory in the cluster ...\n') current_run_dir = xlib.get_cluster_current_run_dir( experiment_id, xlib.get_htseq_count_code()) command = f'mkdir --parents {current_run_dir}' (OK, _, _) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write(f'The directory path is {current_run_dir}.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # build the htseq-count process script if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Building the process script {get_htseq_count_process_script()} ...\n' ) (OK, error_list) = build_htseq_count_process_script( cluster_name, current_run_dir) if OK: log.write('The file is built.\n') if not OK: log.write('*** ERROR: The file could not be built.\n') # upload the htseq-count process script to the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Uploading the process script {get_htseq_count_process_script()} to the directory {current_run_dir} ...\n' ) cluster_path = f'{current_run_dir}/{os.path.basename(get_htseq_count_process_script())}' (OK, error_list) = xssh.put_file(sftp_client, get_htseq_count_process_script(), cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write(f'{error}\n') # set run permision to the htseq-count process script in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Setting on the run permision of {current_run_dir}/{os.path.basename(get_htseq_count_process_script())} ...\n' ) command = f'chmod u+x {current_run_dir}/{os.path.basename(get_htseq_count_process_script())}' (OK, _, _) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # build the htseq-count process starter if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Building the process starter {get_htseq_count_process_starter()} ...\n' ) (OK, error_list) = build_htseq_count_process_starter(current_run_dir) if OK: log.write('The file is built.\n') if not OK: log.write('***ERROR: The file could not be built.\n') # upload the htseq-count process starter to the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Uploading the process starter {get_htseq_count_process_starter()} to the directory {current_run_dir} ...\n' ) cluster_path = f'{current_run_dir}/{os.path.basename(get_htseq_count_process_starter())}' (OK, error_list) = xssh.put_file(sftp_client, get_htseq_count_process_starter(), cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write(f'{error}\n') # set run permision to the htseq-count process starter in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Setting on the run permision of {current_run_dir}/{os.path.basename(get_htseq_count_process_starter())} ...\n' ) command = f'chmod u+x {current_run_dir}/{os.path.basename(get_htseq_count_process_starter())}' (OK, _, _) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # submit the htseq-count process if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Submitting the process script {current_run_dir}/{os.path.basename(get_htseq_count_process_starter())} ...\n' ) OK = xssh.submit_script( cluster_name, ssh_client, current_run_dir, os.path.basename(get_htseq_count_process_starter()), log) # close the SSH transport connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Closing the SSH transport connection ...\n') xssh.close_ssh_transport_connection(ssh_transport) log.write('The connection is closed.\n') # close the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write(f'{xlib.get_separator()}\n') log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def form_list_cluster_experiment_processes(): ''' List the processes of an experiment in the cluster. ''' # initialize the control variable OK = True # print the header clib.clear_screen() clib.print_headers_with_environment( 'Logs - List experiment processes in the cluster') # get the cluster name print(xlib.get_separator()) if xec2.get_running_cluster_list(only_environment_cluster=True, volume_creator_included=False) != []: cluster_name = cinputs.input_cluster_name( volume_creator_included=False, help=True) else: print('WARNING: There is not any running cluster.') OK = False # create the SSH client connection if OK: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name) for error in error_list: print(error) # get experiment identification if OK: experiment_id = cinputs.input_experiment_id(ssh_client, help=True) if experiment_id == '': print( f'WARNING: The cluster {cluster_name} does not have experiment data.' ) OK = False # get the result dataset list of the experiment if OK: command = f'cd {xlib.get_cluster_result_dir()}/{experiment_id}; for list in `ls`; do ls -ld $list | grep -v ^- > /dev/null && echo $list; done;' (OK, stdout, _) = xssh.execute_cluster_command(ssh_client, command) if OK: result_dataset_id_list = [] for line in stdout: line = line.rstrip('\n') if line != 'lost+found': result_dataset_id_list.append(line) # print the result dataset identification list of the experiment if OK: print(xlib.get_separator()) if result_dataset_id_list == []: print( f'*** WARNING: There is not any result dataset of the experiment {experiment_id}.' ) else: result_dataset_id_list.sort() # set data width result_dataset_width = 30 bioinfo_app_width = 25 # set line line = '{0:' + str(result_dataset_width) + '} {1:' + str( bioinfo_app_width) + '}' # print header print(line.format('Result dataset', 'Bioinfo app / Utility')) print( line.format('=' * result_dataset_width, '=' * bioinfo_app_width)) # print detail lines for result_dataset_id in result_dataset_id_list: if result_dataset_id.startswith(xlib.get_bedtools_code() + '-'): bioinfo_app_name = xlib.get_bedtools_name() elif result_dataset_id.startswith(xlib.get_blastplus_code() + '-'): bioinfo_app_name = xlib.get_blastplus_name() elif result_dataset_id.startswith(xlib.get_bcftools_code() + '-'): bioinfo_app_name = xlib.get_bcftools_name() elif result_dataset_id.startswith(xlib.get_bowtie2_code() + '-'): bioinfo_app_name = xlib.get_bowtie2_name() elif result_dataset_id.startswith(xlib.get_busco_code() + '-'): bioinfo_app_name = xlib.get_busco_name() elif result_dataset_id.startswith(xlib.get_cd_hit_code() + '-'): bioinfo_app_name = xlib.get_cd_hit_name() elif result_dataset_id.startswith(xlib.get_cd_hit_est_code() + '-'): bioinfo_app_name = xlib.get_cd_hit_est_name() elif result_dataset_id.startswith(xlib.get_cuffdiff_code() + '-'): bioinfo_app_name = xlib.get_cuffdiff_name() elif result_dataset_id.startswith(xlib.get_cufflinks_code() + '-'): bioinfo_app_name = xlib.get_cufflinks_name() elif result_dataset_id.startswith( xlib.get_cufflinks_cuffmerge_code() + '-'): bioinfo_app_name = xlib.get_cufflinks_cuffmerge_name() elif result_dataset_id.startswith(xlib.get_cuffnorm_code() + '-'): bioinfo_app_name = xlib.get_cuffnorm_name() elif result_dataset_id.startswith(xlib.get_cuffquant_code() + '-'): bioinfo_app_name = xlib.get_cuffquant_name() elif result_dataset_id.startswith(xlib.get_cutadapt_code() + '-'): bioinfo_app_name = xlib.get_cutadapt_name() elif result_dataset_id.startswith( xlib.get_ddradseq_simulation_code() + '-'): bioinfo_app_name = xlib.get_ddradseq_simulation_name() elif result_dataset_id.startswith( xlib.get_ddradseqtools_code() + '-'): bioinfo_app_name = xlib.get_ddradseqtools_name() elif result_dataset_id.startswith(xlib.get_detonate_code() + '-'): bioinfo_app_name = xlib.get_detonate_name() elif result_dataset_id.startswith(xlib.get_diamond_code() + '-'): bioinfo_app_name = xlib.get_diamond_name() elif result_dataset_id.startswith(xlib.get_emboss_code() + '-'): bioinfo_app_name = xlib.get_emboss_name() elif result_dataset_id.startswith( xlib.get_entrez_direct_code() + '-'): bioinfo_app_name = xlib.get_entrez_direct_name() elif result_dataset_id.startswith(xlib.get_express_code() + '-'): bioinfo_app_name = xlib.get_express_name() elif result_dataset_id.startswith(xlib.get_fastqc_code() + '-'): bioinfo_app_name = xlib.get_fastqc_name() elif result_dataset_id.startswith(xlib.get_ggtrinity_code() + '-'): bioinfo_app_name = xlib.get_ggtrinity_name() elif result_dataset_id.startswith(xlib.get_gmap_gsnap_code() + '-'): bioinfo_app_name = xlib.get_gmap_gsnap_name() elif result_dataset_id.startswith(xlib.get_gmap_code() + '-'): bioinfo_app_name = xlib.get_gmap_name() elif result_dataset_id.startswith(xlib.get_gsnap_code() + '-'): bioinfo_app_name = xlib.get_gsnap_name() elif result_dataset_id.startswith(xlib.get_gzip_code() + '-'): bioinfo_app_name = xlib.get_gzip_name() elif result_dataset_id.startswith(xlib.get_hisat2_code() + '-'): bioinfo_app_name = xlib.get_hisat2_name() elif result_dataset_id.startswith(xlib.get_htseq_code() + '-'): bioinfo_app_name = xlib.get_htseq_name() elif result_dataset_id.startswith(xlib.get_htseq_count_code() + '-'): bioinfo_app_name = xlib.get_htseq_count_name() elif result_dataset_id.startswith( xlib.get_insilico_read_normalization_code() + '-'): bioinfo_app_name = xlib.get_insilico_read_normalization_name( ) elif result_dataset_id.startswith(xlib.get_ipyrad_code() + '-'): bioinfo_app_name = xlib.get_ipyrad_name() elif result_dataset_id.startswith(xlib.get_kallisto_code() + '-'): bioinfo_app_name = xlib.get_kallisto_name() elif result_dataset_id.startswith(xlib.get_miniconda3_code() + '-'): bioinfo_app_name = xlib.get_miniconda3_name() elif result_dataset_id.startswith(xlib.get_ngshelper_code() + '-'): bioinfo_app_name = xlib.get_ngshelper_name() elif result_dataset_id.startswith(xlib.get_quast_code() + '-'): bioinfo_app_name = xlib.get_quast_name() elif result_dataset_id.startswith(xlib.get_r_code() + '-'): bioinfo_app_name = xlib.get_r_name() elif result_dataset_id.startswith(xlib.get_raddesigner_code() + '-'): bioinfo_app_name = xlib.get_raddesigner_name() elif result_dataset_id.startswith(xlib.get_ref_eval_code() + '-'): bioinfo_app_name = xlib.get_ref_eval_name() elif result_dataset_id.startswith(xlib.get_rnaquast_code() + '-'): bioinfo_app_name = xlib.get_rnaquast_name() elif result_dataset_id.startswith(xlib.get_rsem_code() + '-'): bioinfo_app_name = xlib.get_rsem_name() elif result_dataset_id.startswith(xlib.get_rsem_eval_code() + '-'): bioinfo_app_name = xlib.get_rsem_eval_name() elif result_dataset_id.startswith(xlib.get_rsitesearch_code() + '-'): bioinfo_app_name = xlib.get_rsitesearch_name() elif result_dataset_id.startswith(xlib.get_samtools_code() + '-'): bioinfo_app_name = xlib.get_samtools_name() elif result_dataset_id.startswith(xlib.get_soapdenovo2_code() + '-'): bioinfo_app_name = xlib.get_soapdenovo2_name() elif result_dataset_id.startswith( xlib.get_soapdenovotrans_code() + '-'): bioinfo_app_name = xlib.get_soapdenovotrans_name() elif result_dataset_id.startswith(xlib.get_star_code() + '-'): bioinfo_app_name = xlib.get_star_name() elif result_dataset_id.startswith(xlib.get_starcode_code() + '-'): bioinfo_app_name = xlib.get_starcode_name() elif result_dataset_id.startswith(xlib.get_toa_code() + '-'): bioinfo_app_name = xlib.get_toa_name() elif result_dataset_id.startswith( xlib.get_toa_process_download_basic_data_code() + '-'): bioinfo_app_name = xlib.get_toa_process_download_basic_data_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_download_dicots_04_code() + '-'): bioinfo_app_name = xlib.get_toa_process_download_dicots_04_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_download_gene_code() + '-'): bioinfo_app_name = xlib.get_toa_process_download_gene_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_download_go_code() + '-'): bioinfo_app_name = xlib.get_toa_process_download_go_name() elif result_dataset_id.startswith( xlib.get_toa_process_download_gymno_01_code() + '-'): bioinfo_app_name = xlib.get_toa_process_download_gymno_01_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_download_interpro_code() + '-'): bioinfo_app_name = xlib.get_toa_process_download_interpro_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_download_monocots_04_code() + '-'): bioinfo_app_name = xlib.get_toa_process_download_monocots_04_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_download_taxonomy_code() + '-'): bioinfo_app_name = xlib.get_toa_process_download_taxonomy_name( ) elif result_dataset_id.startswith( xlib. get_toa_process_gilist_viridiplantae_nucleotide_gi_code( ) + '-'): bioinfo_app_name = xlib.get_toa_process_gilist_viridiplantae_nucleotide_gi_name( ) elif result_dataset_id.startswith( xlib. get_toa_process_gilist_viridiplantae_protein_gi_code() + '-'): bioinfo_app_name = xlib.get_toa_process_gilist_viridiplantae_protein_gi_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_load_basic_data_code() + '-'): bioinfo_app_name = xlib.get_toa_process_load_basic_data_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_load_dicots_04_code() + '-'): bioinfo_app_name = xlib.get_toa_process_load_dicots_04_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_load_gene_code() + '-'): bioinfo_app_name = xlib.get_toa_process_load_gene_name() elif result_dataset_id.startswith( xlib.get_toa_process_load_go_code() + '-'): bioinfo_app_name = xlib.get_toa_process_load_go_name() elif result_dataset_id.startswith( xlib.get_toa_process_load_gymno_01_code() + '-'): bioinfo_app_name = xlib.get_toa_process_load_gymno_01_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_load_interpro_code() + '-'): bioinfo_app_name = xlib.get_toa_process_load_interpro_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_load_monocots_04_code() + '-'): bioinfo_app_name = xlib.get_toa_process_load_monocots_04_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_merge_annotations_code() + '-'): bioinfo_app_name = xlib.get_toa_process_merge_annotations_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_nr_blastplus_db_code() + '-'): bioinfo_app_name = xlib.get_toa_process_nr_blastplus_db_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_nr_diamond_db_code() + '-'): bioinfo_app_name = xlib.get_toa_process_nr_diamond_db_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_nt_blastplus_db_code() + '-'): bioinfo_app_name = xlib.get_toa_process_nt_blastplus_db_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_pipeline_aminoacid_code() + '-'): bioinfo_app_name = xlib.get_toa_process_pipeline_aminoacid_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_pipeline_nucleotide_code() + '-'): bioinfo_app_name = xlib.get_toa_process_pipeline_nucleotide_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_proteome_dicots_04_code() + '-'): bioinfo_app_name = xlib.get_toa_process_proteome_dicots_04_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_proteome_gymno_01_code() + '-'): bioinfo_app_name = xlib.get_toa_process_proteome_gymno_01_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_proteome_monocots_04_code() + '-'): bioinfo_app_name = xlib.get_toa_process_proteome_monocots_04_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_proteome_refseq_plant_code() + '-'): bioinfo_app_name = xlib.get_toa_process_proteome_refseq_plant_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_rebuild_toa_database_code() + '-'): bioinfo_app_name = xlib.get_get_toa_process_rebuild_toa_database_name( ) elif result_dataset_id.startswith( xlib.get_toa_process_recreate_toa_database_code() + '-'): bioinfo_app_name = xlib.get_get_toa_process_recreate_toa_database_name( ) elif result_dataset_id.startswith(xlib.get_tophat_code() + '-'): bioinfo_app_name = xlib.get_tophat_name() elif result_dataset_id.startswith(xlib.get_transabyss_code() + '-'): bioinfo_app_name = xlib.get_transabyss_name() elif result_dataset_id.startswith( xlib.get_transcript_filter_code() + '-'): bioinfo_app_name = xlib.get_transcript_filter_name() elif result_dataset_id.startswith( xlib.get_transcriptome_blastx_code() + '-'): bioinfo_app_name = xlib.get_transcriptome_blastx_name() elif result_dataset_id.startswith( xlib.get_transdecoder_code() + '-'): bioinfo_app_name = xlib.get_transdecoder_name() elif result_dataset_id.startswith(xlib.get_transrate_code() + '-'): bioinfo_app_name = xlib.get_transrate_name() elif result_dataset_id.startswith(xlib.get_trimmomatic_code() + '-'): bioinfo_app_name = xlib.get_trimmomatic_name() elif result_dataset_id.startswith(xlib.get_trinity_code() + '-'): bioinfo_app_name = xlib.get_trinity_name() elif result_dataset_id.startswith( xlib.get_variant_calling_code() + '-'): bioinfo_app_name = xlib.get_variant_calling_name() elif result_dataset_id.startswith(xlib.get_vcftools_code() + '-'): bioinfo_app_name = xlib.get_vcftools_name() elif result_dataset_id.startswith( xlib.get_vcftools_perl_libraries_code() + '-'): bioinfo_app_name = xlib.get_vcftools_perl_libraries_name() elif result_dataset_id.startswith(xlib.get_vsearch_code() + '-'): bioinfo_app_name = xlib.get_vsearch_name() else: bioinfo_app_name = 'xxx' print(line.format(result_dataset_id, bioinfo_app_name)) # close the SSH client connection if OK: xssh.close_ssh_client_connection(ssh_client) # show continuation message print(xlib.get_separator()) input('Press [Intro] to continue ...')