def upload_read_dataset(cluster_name, log, function=None): ''' Upload the read dataset to the cluster. ''' # initialize the control variable OK = True # get the read transfer config file read_transfer_config_file = get_read_transfer_config_file() # warn that the log window must not be closed if not isinstance(log, xlib.DevStdOut): log.write( 'This process might take several minutes. Do not close this window, please wait!\n' ) # get and validate the read transfer config file log.write('{0}\n'.format(xlib.get_separator())) log.write('The read transfer config file is been validating ...\n') if validate_read_transfer_config_file(strict=True): log.write('The config file is OK.\n') else: log.write('*** ERROR: The read transfer config file is not valid.\n') log.write('Please correct this file or recreate the config files.\n') OK = False # create the SSH client connection if OK: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection( cluster_name, 'master') for error in error_list: log.write('{0}\n'.format(error)) # create the SSH transport connection if OK: (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection( cluster_name, 'master') for error in error_list: log.write('{0}\n'.format(error)) # create the SFTP client if OK: sftp_client = xssh.create_sftp_client(ssh_transport) # get the options dictionary if OK: read_transfer_options_dict = xlib.get_option_dict( read_transfer_config_file) # get the experiment identification and create the experiment reads directory if OK: # get the experiment identification experiment_id = read_transfer_options_dict['identification'][ 'experiment_id'] # Get the directory of read and results datasets of the experiment cluster_experiment_reads_dir = xlib.get_cluster_experiment_read_dataset_dir( experiment_id, xlib.get_uploaded_read_dataset_name()) cluster_experiment_result_dir = xlib.get_cluster_experiment_result_dir( experiment_id) # create the experiment reads directory log.write('{0}\n'.format(xlib.get_separator())) log.write( 'The reads directory {0} in the cluster is being created ...\n'. format(cluster_experiment_reads_dir)) command = 'mkdir --parents {0}'.format(cluster_experiment_reads_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The directory is created.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # create the experiment run result directory log.write('{0}\n'.format(xlib.get_separator())) log.write( 'The run result directory {0} in the cluster is being created ...\n' .format(cluster_experiment_result_dir)) command = 'mkdir --parents {0}'.format(cluster_experiment_result_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The directory is created.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # upload the read dataset if OK: # get the sections list sections_list = [] for section in read_transfer_options_dict.keys(): sections_list.append(section) sections_list.sort() # for each section "file-n" for section in sections_list: # verify than the section identification is like file-n if re.match('^file-[0-9]+$', section): # get local path and cluster directory local_path = read_transfer_options_dict[section]['local_path'] # upload the reference file in the cluster log.write('{0}\n'.format(xlib.get_separator())) log.write('The file {0} is being uploaded to {1} ...\n'.format( local_path, cluster_experiment_reads_dir)) cluster_path = '{0}/{1}'.format(cluster_experiment_reads_dir, os.path.basename(local_path)) (OK, error_list) = xssh.put_file(sftp_client, local_path, cluster_path) if OK: log.write('The file has been uploaded.\n') else: for error in error_list: log.write('{0}\n'.format(error)) break # close the SSH transport connection if OK: xssh.close_ssh_transport_connection(ssh_transport) # close the SSH client connection if OK: xssh.close_ssh_client_connection(ssh_client) # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write('{0}\n'.format(xlib.get_separator())) log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def run_htseq_count_process(cluster_name, log, function=None): ''' Run a htseq-count process. ''' # initialize the control variable OK = True # get the htseq-count option dictionary htseq_count_option_dict = xlib.get_option_dict( get_htseq_count_config_file()) # get the experiment identification experiment_id = htseq_count_option_dict['identification']['experiment_id'] # warn that the log window does not have to be closed if not isinstance(log, xlib.DevStdOut): log.write( 'This process might take several minutes. Do not close this window, please wait!\n' ) # check the htseq-count config file log.write(f'{xlib.get_separator()}\n') log.write(f'Checking the {xlib.get_htseq_count_name()} config file ...\n') (OK, error_list) = check_htseq_count_config_file(strict=True) if OK: log.write('The file is OK.\n') else: log.write('*** ERROR: The config file is not valid.\n') log.write('Please correct this file or recreate the config files.\n') # create the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Connecting the SSH client ...\n') (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name) if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write(f'{error}\n') # create the SSH transport connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Connecting the SSH transport ...\n') (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name) if OK: log.write('The SSH transport is connected.\n') else: for error in error_list: log.write(f'{error}\n') # create the SFTP client if OK: log.write(f'{xlib.get_separator()}\n') log.write('Connecting the SFTP client ...\n') sftp_client = xssh.create_sftp_client(ssh_transport) log.write('The SFTP client is connected.\n') # warn that the requirements are being verified if OK: log.write(f'{xlib.get_separator()}\n') log.write('Checking process requirements ...\n') # check the master is running if OK: (master_state_code, master_state_name) = xec2.get_node_state(cluster_name) if master_state_code != 16: log.write( f'*** ERROR: The cluster {cluster_name} is not running. Its state is {master_state_code} ({master_state_name}).\n' ) OK = False # check HTSeq is installed if OK: (OK, error_list, is_installed) = xbioinfoapp.is_installed_anaconda_package( xlib.get_htseq_anaconda_code(), cluster_name, True, ssh_client) if OK: if not is_installed: log.write( f'*** ERROR: {xlib.get_htseq_name()} is not installed.\n') OK = False else: log.write( f'*** ERROR: The verification of {xlib.get_htseq_name()} installation could not be performed.\n' ) # warn that the requirements are OK if OK: log.write('Process requirements are OK.\n') # determine the run directory in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Determining the run directory in the cluster ...\n') current_run_dir = xlib.get_cluster_current_run_dir( experiment_id, xlib.get_htseq_count_code()) command = f'mkdir --parents {current_run_dir}' (OK, _, _) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write(f'The directory path is {current_run_dir}.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # build the htseq-count process script if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Building the process script {get_htseq_count_process_script()} ...\n' ) (OK, error_list) = build_htseq_count_process_script( cluster_name, current_run_dir) if OK: log.write('The file is built.\n') if not OK: log.write('*** ERROR: The file could not be built.\n') # upload the htseq-count process script to the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Uploading the process script {get_htseq_count_process_script()} to the directory {current_run_dir} ...\n' ) cluster_path = f'{current_run_dir}/{os.path.basename(get_htseq_count_process_script())}' (OK, error_list) = xssh.put_file(sftp_client, get_htseq_count_process_script(), cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write(f'{error}\n') # set run permision to the htseq-count process script in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Setting on the run permision of {current_run_dir}/{os.path.basename(get_htseq_count_process_script())} ...\n' ) command = f'chmod u+x {current_run_dir}/{os.path.basename(get_htseq_count_process_script())}' (OK, _, _) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # build the htseq-count process starter if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Building the process starter {get_htseq_count_process_starter()} ...\n' ) (OK, error_list) = build_htseq_count_process_starter(current_run_dir) if OK: log.write('The file is built.\n') if not OK: log.write('***ERROR: The file could not be built.\n') # upload the htseq-count process starter to the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Uploading the process starter {get_htseq_count_process_starter()} to the directory {current_run_dir} ...\n' ) cluster_path = f'{current_run_dir}/{os.path.basename(get_htseq_count_process_starter())}' (OK, error_list) = xssh.put_file(sftp_client, get_htseq_count_process_starter(), cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write(f'{error}\n') # set run permision to the htseq-count process starter in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Setting on the run permision of {current_run_dir}/{os.path.basename(get_htseq_count_process_starter())} ...\n' ) command = f'chmod u+x {current_run_dir}/{os.path.basename(get_htseq_count_process_starter())}' (OK, _, _) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # submit the htseq-count process if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Submitting the process script {current_run_dir}/{os.path.basename(get_htseq_count_process_starter())} ...\n' ) OK = xssh.submit_script( cluster_name, ssh_client, current_run_dir, os.path.basename(get_htseq_count_process_starter()), log) # close the SSH transport connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Closing the SSH transport connection ...\n') xssh.close_ssh_transport_connection(ssh_transport) log.write('The connection is closed.\n') # close the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write(f'{xlib.get_separator()}\n') log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def run_gzip_process(cluster_name, dataset_type, log, function=None): ''' Run a gzip process. ''' # initialize the control variable OK = True # get the gzip code and name gzip_code = xlib.get_gzip_code() gzip_name = xlib.get_gzip_name() # get the gzip option dictionary gzip_option_dict = xlib.get_option_dict(get_gzip_config_file(dataset_type)) # get the experiment identification experiment_id = gzip_option_dict['identification']['experiment_id'] # get the gzip process script path in the local computer gzip_process_script = get_gzip_process_script(dataset_type) # get the gzip process starter path in the local computer gzip_process_starter = get_gzip_process_starter(dataset_type) # warn that the log window does not have to be closed if not isinstance(log, xlib.DevStdOut): log.write('This process might take several minutes. Do not close this window, please wait!\n') # check the gzip config file log.write(f'{xlib.get_separator()}\n') log.write('Checking the {0} config file ...\n'.format(gzip_name)) (OK, error_list) = check_gzip_config_file(dataset_type, strict=True) if OK: log.write('The file is OK.\n') else: log.write('*** ERROR: The config file is not valid.\n') log.write('Please correct this file or recreate the config files.\n') # create the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Connecting the SSH client ...\n') (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name) if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write(f'{error}\n') # create the SSH transport connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Connecting the SSH transport ...\n') (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name) if OK: log.write('The SSH transport is connected.\n') else: for error in error_list: log.write(f'{error}\n') # create the SFTP client if OK: log.write(f'{xlib.get_separator()}\n') log.write('Connecting the SFTP client ...\n') sftp_client = xssh.create_sftp_client(ssh_transport) log.write('The SFTP client is connected.\n') # warn that the requirements are being verified if OK: log.write(f'{xlib.get_separator()}\n') log.write('Checking process requirements ...\n') # check the master is running if OK: (master_state_code, master_state_name) = xec2.get_node_state(cluster_name) if master_state_code != 16: log.write(f'*** ERROR: The cluster {cluster_name} is not running. Its state is {master_state_code} ({master_state_name}).\n') OK = False # warn that the requirements are OK if OK: log.write('Process requirements are OK.\n') # determine the run directory in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Determining the run directory in the cluster ...\n') if dataset_type == 'reference': current_run_dir = xlib.get_cluster_current_run_dir('reference', gzip_code) elif dataset_type == 'database': current_run_dir = xlib.get_cluster_current_run_dir('database', gzip_code) else: current_run_dir = xlib.get_cluster_current_run_dir(experiment_id, gzip_code) command = f'mkdir --parents {current_run_dir}' (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write(f'The directory path is {current_run_dir}.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # build the gzip process script if OK: log.write(f'{xlib.get_separator()}\n') log.write('Building the process script {0} ...\n'.format(gzip_process_script)) (OK, error_list) = build_gzip_process_script(cluster_name, dataset_type, current_run_dir) if OK: log.write('The file is built.\n') else: log.write('*** ERROR: The file could not be built.\n') # upload the gzip process script to the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Uploading the process script {0} to the directory {1} ...\n'.format(gzip_process_script, current_run_dir)) cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_script)) (OK, error_list) = xssh.put_file(sftp_client, gzip_process_script, cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write(f'{error}\n') # set run permision to the gzip process script in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(gzip_process_script))) command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_script)) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # build the gzip process starter if OK: log.write(f'{xlib.get_separator()}\n') log.write('Building the process starter {0} ...\n'.format(gzip_process_starter)) (OK, error_list) = build_gzip_process_starter(dataset_type, current_run_dir) if OK: log.write('The file is built.\n') else: log.write('***ERROR: The file could not be built.\n') # upload the gzip process starter to the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Uploading the process starter {0} to the directory {1} ...\n'.format(gzip_process_starter, current_run_dir)) cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_starter)) (OK, error_list) = xssh.put_file(sftp_client, gzip_process_starter, cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write(f'{error}\n') # set run permision to the gzip process starter in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(gzip_process_starter))) command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_starter)) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # submit the gzip process if OK: log.write(f'{xlib.get_separator()}\n') log.write('Submitting the process script {0}/{1} ...\n'.format(current_run_dir, os.path.basename(gzip_process_starter))) OK = xssh.submit_script(cluster_name, ssh_client, current_run_dir, os.path.basename(gzip_process_starter), log) # close the SSH transport connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Closing the SSH transport connection ...\n') xssh.close_ssh_transport_connection(ssh_transport) log.write('The connection is closed.\n') # close the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write(f'{xlib.get_separator()}\n') log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def upload_database_dataset(cluster_name, log, function=None): ''' Upload the database dataset to the cluster. ''' # initialize the control variable OK = True # warn that the log window does not have to be closed if not isinstance(log, xlib.DevStdOut): log.write( 'This process might take several minutes. Do not close this window, please wait!\n' ) # check the database transfer config file log.write(f'{xlib.get_separator()}\n') log.write('Checking the database transfer config file ...\n') if check_database_transfer_config_file(strict=True): log.write('The file is OK.\n') else: log.write( '*** ERROR: The database transfer config file is not valid.\n') log.write('Please correct this file or recreate the config files.\n') OK = False # create the SSH client connection if OK: (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name) for error in error_list: log.write(f'{error}\n') # create the SSH transport connection if OK: (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name) for error in error_list: log.write(f'{error}\n') # create the SFTP client if OK: sftp_client = xssh.create_sftp_client(ssh_transport) # upload the database dataset if OK: # get the option dictionary database_transfer_options_dict = xlib.get_option_dict( get_database_transfer_config_file()) # get the database dataset identification and the local directory of the database files database_dataset_id = database_transfer_options_dict['identification'][ 'database_dataset_id'] local_dir = database_transfer_options_dict['identification'][ 'local_dir'] # set the cluster database directory cluster_database_dir = '{0}/{1}'.format( xlib.get_cluster_database_dir(), database_dataset_id) # create the data directory in the cluster log.write(f'{xlib.get_separator()}\n') log.write( 'The database directory {0} in the cluster is being created ...\n'. format(cluster_database_dir)) command = 'mkdir --parents {0}'.format(cluster_database_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The directory is created.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # get the sections list sections_list = [] for section in database_transfer_options_dict.keys(): sections_list.append(section) sections_list.sort() # for each section "file-n" for section in sections_list: # check than the section identification is like file-n if re.match('^file-[0-9]+$', section): # get the file name file_name = database_transfer_options_dict[section][ 'file_name'] # set the local path and cluster path local_path = '{0}/{1}'.format(local_dir, file_name) cluster_path = '{0}/{1}'.format(cluster_database_dir, file_name) # upload the database file to the cluster log.write(f'{xlib.get_separator()}\n') log.write('The file {0} is being uploaded to {1} ...\n'.format( file_name, cluster_database_dir)) (OK, error_list) = xssh.put_file(sftp_client, local_path, cluster_path) if OK: log.write('The file has been uploaded.\n') else: for error in error_list: log.write(f'{error}\n') break # close the SSH transport connection if OK: xssh.close_ssh_transport_connection(ssh_transport) # close the SSH client connection if OK: xssh.close_ssh_client_connection(ssh_client) # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write(f'{xlib.get_separator()}\n') log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def install_node_infrastructure_software(cluster_name, node_name, log): ''' Install infraestructure software in a node. ''' # initialize the control variable OK = True # get the infrastructure software installation script path in local compute local_script_path = get_infrastructure_software_installation_script() # set the infrastructure software installation script path in node node_script_path = './{0}'.format(os.path.basename(local_script_path)) # set the infrastructure software installation log path in node node_log_path = node_script_path[:node_script_path.find('.sh')] + '.log' # build the infrastructure software installation script if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Building the infrastructure software installation script {0} ...\n' .format(local_script_path)) (OK, error_list ) = build_infrastructure_software_installation_script(cluster_name) if OK: log.write('The file is built.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # create the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Connecting the SSH client to node {0} ...\n'.format(node_name)) (OK, error_list, ssh_client) = xssh.create_ssh_client_connection( cluster_name, node_name) if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # create the SSH transport connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Connecting the SSH transport to node {0} ...\n'.format(node_name)) (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection( cluster_name, node_name) if OK: log.write('The SSH transport is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # create the SFTP client if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Connecting the SFTP client to node {0} ...\n'.format(node_name)) sftp_client = xssh.create_sftp_client(ssh_transport) log.write('The SFTP client is connected.\n') # upload the infraestructe software installation script to the node if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Uploading the infraestructe software installation script to the node {0} ...\n' .format(node_name)) (OK, error_list) = xssh.put_file(sftp_client, local_script_path, node_script_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # set run permision to the infraestructe software installation script in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Setting on the run permision ...\n') command = 'chmod u+x {0}'.format(node_script_path) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # submit the infraestructe software installation script if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write( 'Submitting the infraestructe software installation script in the node {0} ...\n' .format(node_name)) command = '{0} &>{1} &'.format(node_script_path, node_log_path) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The script is submitted.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # close the SSH transport connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH transport connection ...\n') xssh.close_ssh_transport_connection(ssh_transport) log.write('The connection is closed.\n') # close the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # return the control variable return OK
def run_gmap_process(cluster_name, log, function=None): ''' Run a GMAP process. ''' # initialize the control variable OK = True # get the GMAP option dictionary gmap_option_dict = xlib.get_option_dict(get_gmap_config_file()) # get the experiment identification experiment_id = gmap_option_dict['identification']['experiment_id'] # warn that the log window must not be closed if not isinstance(log, xlib.DevStdOut): log.write('This process might take several minutes. Do not close this window, please wait!\n') # validate the GMAP config file log.write('{0}\n'.format(xlib.get_separator())) log.write('Validating the {0} config file ...\n'.format(xlib.get_gmap_name())) (OK, error_list) = validate_gmap_config_file(strict=True) if OK: log.write('The config file is OK.\n') else: log.write('*** ERROR: The config file is not valid.\n') log.write('Please correct this file or recreate the config files.\n') # create the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Connecting the SSH client ...\n') (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master') if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # create the SSH transport connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Connecting the SSH transport ...\n') (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name, 'master') if OK: log.write('The SSH transport is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # create the SFTP client if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Connecting the SFTP client ...\n') sftp_client = xssh.create_sftp_client(ssh_transport) log.write('The SFTP client is connected.\n') # warn that the requirements are being verified if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Verifying process requirements ...\n') # verify the master is running if OK: (master_state_code, master_state_name) = xec2.get_node_state(cluster_name, 'master') if master_state_code != 16: log.write('*** ERROR: The cluster {0} is not running. Its state is {1} ({2}).\n'.format(cluster_name, master_state_code, master_state_name)) OK = False # verify the GMAP-GSNAP is setup if OK: (OK, error_list, is_setup) = xbioinfoapp.is_setup_bioconda_package(xlib.get_gmap_gsnap_bioconda_code(), cluster_name, True, ssh_client) if OK: if not is_setup: log.write('*** ERROR: {0} is not setup.\n'.format(xlib.get_gmap_name())) OK = False else: log.write('*** ERROR: The verification of {0} setup could not be performed.\n'.format(xlib.get_gmap_name())) # warn that the requirements are OK if OK: log.write('Process requirements are OK.\n') # determine the run directory in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Determining the run directory in the cluster ...\n') current_run_dir = xlib.get_cluster_current_run_dir(experiment_id, xlib.get_gmap_code()) command = 'mkdir --parents {0}'.format(current_run_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The directory path is {0}.\n'.format(current_run_dir)) else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # build the GMAP process script if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Building the process script {0} ...\n'.format(get_gmap_process_script())) (OK, error_list) = build_gmap_process_script(cluster_name, current_run_dir) if OK: log.write('The file is built.\n') if not OK: log.write('*** ERROR: The file could not be built.\n') # upload the GMAP process script in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Uploading the process script {0} in the directory {1} of the master ...\n'.format(get_gmap_process_script(), current_run_dir)) cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_script())) (OK, error_list) = xssh.put_file(sftp_client, get_gmap_process_script(), cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # set run permision to the GMAP process script in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(get_gmap_process_script()))) command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_script())) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # build the GMAP process starter if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Building the process starter {0} ...\n'.format(get_gmap_process_starter())) (OK, error_list) = build_gmap_process_starter(current_run_dir) if OK: log.write('The file is built.\n') if not OK: log.write('***ERROR: The file could not be built.\n') # upload the GMAP process starter in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Uploading the process starter {0} in the directory {1} of the master ...\n'.format(get_gmap_process_starter(), current_run_dir)) cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_starter())) (OK, error_list) = xssh.put_file(sftp_client, get_gmap_process_starter(), cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # set run permision to the GMAP process starter in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(get_gmap_process_starter()))) command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_starter())) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # submit the GMAP process if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Submitting the process script {0}/{1} ...\n'.format(current_run_dir, os.path.basename(get_gmap_process_starter()))) sge_env = xcluster.get_sge_env() command = '{0}; qsub -V -b n -cwd {1}/{2}'.format(sge_env, current_run_dir, os.path.basename(get_gmap_process_starter())) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: for line in stdout: log.write('{0}\n'.format(line)) else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # close the SSH transport connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH transport connection ...\n') xssh.close_ssh_transport_connection(ssh_transport) log.write('The connection is closed.\n') # close the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write('{0}\n'.format(xlib.get_separator())) log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK
def install_node_infrastructure_software(cluster_name, node_name, log): ''' Install infrastructure software in a node. ''' # initialize the control variable OK = True # get the infrastructure software installation script path in local compute local_script_path = get_infrastructure_software_installation_script() # set the infrastructure software installation script path in node node_script_path = f'./{os.path.basename(local_script_path)}' # set the infrastructure software installation log path in node node_log_path = node_script_path[:node_script_path.find('.sh')] + '.log' # build the infrastructure software installation script if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Building the infrastructure software installation script {local_script_path} ...\n' ) (OK, error_list ) = build_infrastructure_software_installation_script(cluster_name) if OK: log.write('The file is built.\n') else: for error in error_list: log.write(f'{error}\n') # create the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write(f'Connecting the SSH client to node {node_name} ...\n') (OK, error_list, ssh_client) = xssh.create_ssh_client_connection( cluster_name, node_name) if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write(f'{error}\n') # create the SSH transport connection if OK: log.write(f'{xlib.get_separator()}\n') log.write(f'Connecting the SSH transport to node {node_name} ...\n') (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection( cluster_name, node_name) if OK: log.write('The SSH transport is connected.\n') else: for error in error_list: log.write(f'{error}\n') # create the SFTP client if OK: log.write(f'{xlib.get_separator()}\n') log.write(f'Connecting the SFTP client to node {node_name} ...\n') sftp_client = xssh.create_sftp_client(ssh_transport) log.write('The SFTP client is connected.\n') # download the AWSCLI2 compressed file to local computer if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Downloading the {xlib.get_awscli_name()} compressed file to local computer ...\n' ) local_path = f'{xlib.get_temp_dir()}/{xlib.get_awscli_name()}.zip' if not os.path.exists(os.path.dirname(local_path)): os.makedirs(os.path.dirname(local_path)) try: urllib.request.urlretrieve(xlib.get_awscli_url(), local_path) except Exception as e: log.write(f'*** EXCEPTION: "{e}".') log.write( f'*** ERROR: The file {xlib.get_awscli_url()} can not be downloaded.\n' ) OK = False else: log.write('The file is downloaded.\n') # upload the AWSCLI2 compressed file to cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Uploading the {xlib.get_awscli_name()} compressed file to the cluster ...\n' ) cluster_path = f'./{os.path.basename(local_path)}' (OK, error_list) = xssh.put_file(sftp_client, local_path, cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write(f'{error}\n') # upload the infraestructe software installation script to the node if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Uploading the infraestructe software installation script to the node {node_name} ...\n' ) (OK, error_list) = xssh.put_file(sftp_client, local_script_path, node_script_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write(f'{error}\n') # set run permision to the infraestructe software installation script in the cluster if OK: log.write(f'{xlib.get_separator()}\n') log.write('Setting on the run permision ...\n') command = f'chmod u+x {node_script_path}' (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # submit the infraestructe software installation script if OK: log.write(f'{xlib.get_separator()}\n') log.write( f'Submitting the infraestructe software installation script in the node {node_name} ...\n' ) command = f'{node_script_path} &>{node_log_path} &' (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The script is submitted.\n') else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # close the SSH transport connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Closing the SSH transport connection ...\n') xssh.close_ssh_transport_connection(ssh_transport) log.write('The connection is closed.\n') # close the SSH client connection if OK: log.write(f'{xlib.get_separator()}\n') log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # return the control variable return OK