def submit_script(cluster_name, ssh_client, current_run_dir, script, log): ''' ''' # initialize the control variable OK = True # submit the script starter if xec2.get_cluster_mode( cluster_name) == xconfiguration.get_cluster_mode_native(): command = f'nohup {current_run_dir}/{script} &>/dev/null &' elif xec2.get_cluster_mode( cluster_name) == xconfiguration.get_cluster_mode_starcluster(): sge_env = xcluster.get_sge_env() command = f'{sge_env}; qsub -V -b n -cwd {current_run_dir}/{script}' (OK, stdout, stderr) = execute_cluster_command(ssh_client, command) if OK: log.write('The script is submitted.\n') for line in stdout: log.write('{0}\n'.format(line)) else: log.write(f'*** ERROR: Wrong command ---> {command}\n') # return the control variable return OK
def run_gmap_process(cluster_name, log, function=None): ''' Run a GMAP process. ''' # initialize the control variable OK = True # get the GMAP option dictionary gmap_option_dict = xlib.get_option_dict(get_gmap_config_file()) # get the experiment identification experiment_id = gmap_option_dict['identification']['experiment_id'] # warn that the log window must not be closed if not isinstance(log, xlib.DevStdOut): log.write('This process might take several minutes. Do not close this window, please wait!\n') # validate the GMAP config file log.write('{0}\n'.format(xlib.get_separator())) log.write('Validating the {0} config file ...\n'.format(xlib.get_gmap_name())) (OK, error_list) = validate_gmap_config_file(strict=True) if OK: log.write('The config file is OK.\n') else: log.write('*** ERROR: The config file is not valid.\n') log.write('Please correct this file or recreate the config files.\n') # create the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Connecting the SSH client ...\n') (OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master') if OK: log.write('The SSH client is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # create the SSH transport connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Connecting the SSH transport ...\n') (OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name, 'master') if OK: log.write('The SSH transport is connected.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # create the SFTP client if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Connecting the SFTP client ...\n') sftp_client = xssh.create_sftp_client(ssh_transport) log.write('The SFTP client is connected.\n') # warn that the requirements are being verified if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Verifying process requirements ...\n') # verify the master is running if OK: (master_state_code, master_state_name) = xec2.get_node_state(cluster_name, 'master') if master_state_code != 16: log.write('*** ERROR: The cluster {0} is not running. Its state is {1} ({2}).\n'.format(cluster_name, master_state_code, master_state_name)) OK = False # verify the GMAP-GSNAP is setup if OK: (OK, error_list, is_setup) = xbioinfoapp.is_setup_bioconda_package(xlib.get_gmap_gsnap_bioconda_code(), cluster_name, True, ssh_client) if OK: if not is_setup: log.write('*** ERROR: {0} is not setup.\n'.format(xlib.get_gmap_name())) OK = False else: log.write('*** ERROR: The verification of {0} setup could not be performed.\n'.format(xlib.get_gmap_name())) # warn that the requirements are OK if OK: log.write('Process requirements are OK.\n') # determine the run directory in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Determining the run directory in the cluster ...\n') current_run_dir = xlib.get_cluster_current_run_dir(experiment_id, xlib.get_gmap_code()) command = 'mkdir --parents {0}'.format(current_run_dir) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The directory path is {0}.\n'.format(current_run_dir)) else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # build the GMAP process script if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Building the process script {0} ...\n'.format(get_gmap_process_script())) (OK, error_list) = build_gmap_process_script(cluster_name, current_run_dir) if OK: log.write('The file is built.\n') if not OK: log.write('*** ERROR: The file could not be built.\n') # upload the GMAP process script in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Uploading the process script {0} in the directory {1} of the master ...\n'.format(get_gmap_process_script(), current_run_dir)) cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_script())) (OK, error_list) = xssh.put_file(sftp_client, get_gmap_process_script(), cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # set run permision to the GMAP process script in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(get_gmap_process_script()))) command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_script())) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # build the GMAP process starter if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Building the process starter {0} ...\n'.format(get_gmap_process_starter())) (OK, error_list) = build_gmap_process_starter(current_run_dir) if OK: log.write('The file is built.\n') if not OK: log.write('***ERROR: The file could not be built.\n') # upload the GMAP process starter in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Uploading the process starter {0} in the directory {1} of the master ...\n'.format(get_gmap_process_starter(), current_run_dir)) cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_starter())) (OK, error_list) = xssh.put_file(sftp_client, get_gmap_process_starter(), cluster_path) if OK: log.write('The file is uploaded.\n') else: for error in error_list: log.write('{0}\n'.format(error)) # set run permision to the GMAP process starter in the cluster if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(get_gmap_process_starter()))) command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_starter())) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: log.write('The run permision is set.\n') else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # submit the GMAP process if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Submitting the process script {0}/{1} ...\n'.format(current_run_dir, os.path.basename(get_gmap_process_starter()))) sge_env = xcluster.get_sge_env() command = '{0}; qsub -V -b n -cwd {1}/{2}'.format(sge_env, current_run_dir, os.path.basename(get_gmap_process_starter())) (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command) if OK: for line in stdout: log.write('{0}\n'.format(line)) else: log.write('*** ERROR: Wrong command ---> {0}\n'.format(command)) # close the SSH transport connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH transport connection ...\n') xssh.close_ssh_transport_connection(ssh_transport) log.write('The connection is closed.\n') # close the SSH client connection if OK: log.write('{0}\n'.format(xlib.get_separator())) log.write('Closing the SSH client connection ...\n') xssh.close_ssh_client_connection(ssh_client) log.write('The connection is closed.\n') # warn that the log window can be closed if not isinstance(log, xlib.DevStdOut): log.write('{0}\n'.format(xlib.get_separator())) log.write('You can close this window now.\n') # execute final function if function is not None: function() # return the control variable return OK