def run_gzip_process(cluster_name, dataset_type, log, function=None):
'''
Run a gzip process.
'''
# initialize the control variable
OK = True
# get the gzip code and name
gzip_code = xlib.get_gzip_code()
gzip_name = xlib.get_gzip_name()
# get the gzip option dictionary
gzip_option_dict = xlib.get_option_dict(get_gzip_config_file(dataset_type))
# get the experiment identification
experiment_id = gzip_option_dict['identification']['experiment_id']
# get the gzip process script path in the local computer
gzip_process_script = get_gzip_process_script(dataset_type)
# get the gzip process starter path in the local computer
gzip_process_starter = get_gzip_process_starter(dataset_type)
# warn that the log window does not have to be closed
if not isinstance(log, xlib.DevStdOut):
log.write('This process might take several minutes. Do not close this window, please wait!\n')
# check the gzip config file
log.write(f'{xlib.get_separator()}\n')
log.write('Checking the {0} config file ...\n'.format(gzip_name))
(OK, error_list) = check_gzip_config_file(dataset_type, strict=True)
if OK:
log.write('The file is OK.\n')
else:
log.write('*** ERROR: The config file is not valid.\n')
log.write('Please correct this file or recreate the config files.\n')
# create the SSH client connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Connecting the SSH client ...\n')
(OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name)
if OK:
log.write('The SSH client is connected.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# create the SSH transport connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Connecting the SSH transport ...\n')
(OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name)
if OK:
log.write('The SSH transport is connected.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# create the SFTP client
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Connecting the SFTP client ...\n')
sftp_client = xssh.create_sftp_client(ssh_transport)
log.write('The SFTP client is connected.\n')
# warn that the requirements are being verified
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Checking process requirements ...\n')
# check the master is running
if OK:
(master_state_code, master_state_name) = xec2.get_node_state(cluster_name)
if master_state_code != 16:
log.write(f'*** ERROR: The cluster {cluster_name} is not running. Its state is {master_state_code} ({master_state_name}).\n')
OK = False
# warn that the requirements are OK
if OK:
log.write('Process requirements are OK.\n')
# determine the run directory in the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Determining the run directory in the cluster ...\n')
if dataset_type == 'reference':
current_run_dir = xlib.get_cluster_current_run_dir('reference', gzip_code)
elif dataset_type == 'database':
current_run_dir = xlib.get_cluster_current_run_dir('database', gzip_code)
else:
current_run_dir = xlib.get_cluster_current_run_dir(experiment_id, gzip_code)
command = f'mkdir --parents {current_run_dir}'
(OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write(f'The directory path is {current_run_dir}.\n')
else:
log.write(f'*** ERROR: Wrong command ---> {command}\n')
# build the gzip process script
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Building the process script {0} ...\n'.format(gzip_process_script))
(OK, error_list) = build_gzip_process_script(cluster_name, dataset_type, current_run_dir)
if OK:
log.write('The file is built.\n')
else:
log.write('*** ERROR: The file could not be built.\n')
# upload the gzip process script to the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Uploading the process script {0} to the directory {1} ...\n'.format(gzip_process_script, current_run_dir))
cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_script))
(OK, error_list) = xssh.put_file(sftp_client, gzip_process_script, cluster_path)
if OK:
log.write('The file is uploaded.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# set run permision to the gzip process script in the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(gzip_process_script)))
command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_script))
(OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The run permision is set.\n')
else:
log.write(f'*** ERROR: Wrong command ---> {command}\n')
# build the gzip process starter
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Building the process starter {0} ...\n'.format(gzip_process_starter))
(OK, error_list) = build_gzip_process_starter(dataset_type, current_run_dir)
if OK:
log.write('The file is built.\n')
else:
log.write('***ERROR: The file could not be built.\n')
# upload the gzip process starter to the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Uploading the process starter {0} to the directory {1} ...\n'.format(gzip_process_starter, current_run_dir))
cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_starter))
(OK, error_list) = xssh.put_file(sftp_client, gzip_process_starter, cluster_path)
if OK:
log.write('The file is uploaded.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# set run permision to the gzip process starter in the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(gzip_process_starter)))
command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_starter))
(OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The run permision is set.\n')
else:
log.write(f'*** ERROR: Wrong command ---> {command}\n')
# submit the gzip process
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Submitting the process script {0}/{1} ...\n'.format(current_run_dir, os.path.basename(gzip_process_starter)))
OK = xssh.submit_script(cluster_name, ssh_client, current_run_dir, os.path.basename(gzip_process_starter), log)
# close the SSH transport connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Closing the SSH transport connection ...\n')
xssh.close_ssh_transport_connection(ssh_transport)
log.write('The connection is closed.\n')
# close the SSH client connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Closing the SSH client connection ...\n')
xssh.close_ssh_client_connection(ssh_client)
log.write('The connection is closed.\n')
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut):
log.write(f'{xlib.get_separator()}\n')
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK
def run_htseq_count_process(cluster_name, log, function=None):
'''
Run a htseq-count process.
'''
# initialize the control variable
OK = True
# get the htseq-count option dictionary
htseq_count_option_dict = xlib.get_option_dict(
get_htseq_count_config_file())
# get the experiment identification
experiment_id = htseq_count_option_dict['identification']['experiment_id']
# warn that the log window does not have to be closed
if not isinstance(log, xlib.DevStdOut):
log.write(
'This process might take several minutes. Do not close this window, please wait!\n'
)
# check the htseq-count config file
log.write(f'{xlib.get_separator()}\n')
log.write(f'Checking the {xlib.get_htseq_count_name()} config file ...\n')
(OK, error_list) = check_htseq_count_config_file(strict=True)
if OK:
log.write('The file is OK.\n')
else:
log.write('*** ERROR: The config file is not valid.\n')
log.write('Please correct this file or recreate the config files.\n')
# create the SSH client connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Connecting the SSH client ...\n')
(OK, error_list,
ssh_client) = xssh.create_ssh_client_connection(cluster_name)
if OK:
log.write('The SSH client is connected.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# create the SSH transport connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Connecting the SSH transport ...\n')
(OK, error_list,
ssh_transport) = xssh.create_ssh_transport_connection(cluster_name)
if OK:
log.write('The SSH transport is connected.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# create the SFTP client
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Connecting the SFTP client ...\n')
sftp_client = xssh.create_sftp_client(ssh_transport)
log.write('The SFTP client is connected.\n')
# warn that the requirements are being verified
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Checking process requirements ...\n')
# check the master is running
if OK:
(master_state_code,
master_state_name) = xec2.get_node_state(cluster_name)
if master_state_code != 16:
log.write(
f'*** ERROR: The cluster {cluster_name} is not running. Its state is {master_state_code} ({master_state_name}).\n'
)
OK = False
# check HTSeq is installed
if OK:
(OK, error_list,
is_installed) = xbioinfoapp.is_installed_anaconda_package(
xlib.get_htseq_anaconda_code(), cluster_name, True, ssh_client)
if OK:
if not is_installed:
log.write(
f'*** ERROR: {xlib.get_htseq_name()} is not installed.\n')
OK = False
else:
log.write(
f'*** ERROR: The verification of {xlib.get_htseq_name()} installation could not be performed.\n'
)
# warn that the requirements are OK
if OK:
log.write('Process requirements are OK.\n')
# determine the run directory in the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Determining the run directory in the cluster ...\n')
current_run_dir = xlib.get_cluster_current_run_dir(
experiment_id, xlib.get_htseq_count_code())
command = f'mkdir --parents {current_run_dir}'
(OK, _, _) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write(f'The directory path is {current_run_dir}.\n')
else:
log.write(f'*** ERROR: Wrong command ---> {command}\n')
# build the htseq-count process script
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
f'Building the process script {get_htseq_count_process_script()} ...\n'
)
(OK, error_list) = build_htseq_count_process_script(
cluster_name, current_run_dir)
if OK:
log.write('The file is built.\n')
if not OK:
log.write('*** ERROR: The file could not be built.\n')
# upload the htseq-count process script to the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
f'Uploading the process script {get_htseq_count_process_script()} to the directory {current_run_dir} ...\n'
)
cluster_path = f'{current_run_dir}/{os.path.basename(get_htseq_count_process_script())}'
(OK, error_list) = xssh.put_file(sftp_client,
get_htseq_count_process_script(),
cluster_path)
if OK:
log.write('The file is uploaded.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# set run permision to the htseq-count process script in the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
f'Setting on the run permision of {current_run_dir}/{os.path.basename(get_htseq_count_process_script())} ...\n'
)
command = f'chmod u+x {current_run_dir}/{os.path.basename(get_htseq_count_process_script())}'
(OK, _, _) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The run permision is set.\n')
else:
log.write(f'*** ERROR: Wrong command ---> {command}\n')
# build the htseq-count process starter
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
f'Building the process starter {get_htseq_count_process_starter()} ...\n'
)
(OK, error_list) = build_htseq_count_process_starter(current_run_dir)
if OK:
log.write('The file is built.\n')
if not OK:
log.write('***ERROR: The file could not be built.\n')
# upload the htseq-count process starter to the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
f'Uploading the process starter {get_htseq_count_process_starter()} to the directory {current_run_dir} ...\n'
)
cluster_path = f'{current_run_dir}/{os.path.basename(get_htseq_count_process_starter())}'
(OK, error_list) = xssh.put_file(sftp_client,
get_htseq_count_process_starter(),
cluster_path)
if OK:
log.write('The file is uploaded.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# set run permision to the htseq-count process starter in the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
f'Setting on the run permision of {current_run_dir}/{os.path.basename(get_htseq_count_process_starter())} ...\n'
)
command = f'chmod u+x {current_run_dir}/{os.path.basename(get_htseq_count_process_starter())}'
(OK, _, _) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The run permision is set.\n')
else:
log.write(f'*** ERROR: Wrong command ---> {command}\n')
# submit the htseq-count process
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
f'Submitting the process script {current_run_dir}/{os.path.basename(get_htseq_count_process_starter())} ...\n'
)
OK = xssh.submit_script(
cluster_name, ssh_client, current_run_dir,
os.path.basename(get_htseq_count_process_starter()), log)
# close the SSH transport connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Closing the SSH transport connection ...\n')
xssh.close_ssh_transport_connection(ssh_transport)
log.write('The connection is closed.\n')
# close the SSH client connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Closing the SSH client connection ...\n')
xssh.close_ssh_client_connection(ssh_client)
log.write('The connection is closed.\n')
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut):
log.write(f'{xlib.get_separator()}\n')
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK
def run_gmap_process(cluster_name, log, function=None):
'''
Run a GMAP process.
'''
# initialize the control variable
OK = True
# get the GMAP option dictionary
gmap_option_dict = xlib.get_option_dict(get_gmap_config_file())
# get the experiment identification
experiment_id = gmap_option_dict['identification']['experiment_id']
# warn that the log window must not be closed
if not isinstance(log, xlib.DevStdOut):
log.write('This process might take several minutes. Do not close this window, please wait!\n')
# validate the GMAP config file
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Validating the {0} config file ...\n'.format(xlib.get_gmap_name()))
(OK, error_list) = validate_gmap_config_file(strict=True)
if OK:
log.write('The config file is OK.\n')
else:
log.write('*** ERROR: The config file is not valid.\n')
log.write('Please correct this file or recreate the config files.\n')
# create the SSH client connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Connecting the SSH client ...\n')
(OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master')
if OK:
log.write('The SSH client is connected.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# create the SSH transport connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Connecting the SSH transport ...\n')
(OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name, 'master')
if OK:
log.write('The SSH transport is connected.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# create the SFTP client
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Connecting the SFTP client ...\n')
sftp_client = xssh.create_sftp_client(ssh_transport)
log.write('The SFTP client is connected.\n')
# warn that the requirements are being verified
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Verifying process requirements ...\n')
# verify the master is running
if OK:
(master_state_code, master_state_name) = xec2.get_node_state(cluster_name, 'master')
if master_state_code != 16:
log.write('*** ERROR: The cluster {0} is not running. Its state is {1} ({2}).\n'.format(cluster_name, master_state_code, master_state_name))
OK = False
# verify the GMAP-GSNAP is setup
if OK:
(OK, error_list, is_setup) = xbioinfoapp.is_setup_bioconda_package(xlib.get_gmap_gsnap_bioconda_code(), cluster_name, True, ssh_client)
if OK:
if not is_setup:
log.write('*** ERROR: {0} is not setup.\n'.format(xlib.get_gmap_name()))
OK = False
else:
log.write('*** ERROR: The verification of {0} setup could not be performed.\n'.format(xlib.get_gmap_name()))
# warn that the requirements are OK
if OK:
log.write('Process requirements are OK.\n')
# determine the run directory in the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Determining the run directory in the cluster ...\n')
current_run_dir = xlib.get_cluster_current_run_dir(experiment_id, xlib.get_gmap_code())
command = 'mkdir --parents {0}'.format(current_run_dir)
(OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The directory path is {0}.\n'.format(current_run_dir))
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# build the GMAP process script
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Building the process script {0} ...\n'.format(get_gmap_process_script()))
(OK, error_list) = build_gmap_process_script(cluster_name, current_run_dir)
if OK:
log.write('The file is built.\n')
if not OK:
log.write('*** ERROR: The file could not be built.\n')
# upload the GMAP process script in the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Uploading the process script {0} in the directory {1} of the master ...\n'.format(get_gmap_process_script(), current_run_dir))
cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_script()))
(OK, error_list) = xssh.put_file(sftp_client, get_gmap_process_script(), cluster_path)
if OK:
log.write('The file is uploaded.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# set run permision to the GMAP process script in the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(get_gmap_process_script())))
command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_script()))
(OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The run permision is set.\n')
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# build the GMAP process starter
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Building the process starter {0} ...\n'.format(get_gmap_process_starter()))
(OK, error_list) = build_gmap_process_starter(current_run_dir)
if OK:
log.write('The file is built.\n')
if not OK:
log.write('***ERROR: The file could not be built.\n')
# upload the GMAP process starter in the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Uploading the process starter {0} in the directory {1} of the master ...\n'.format(get_gmap_process_starter(), current_run_dir))
cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_starter()))
(OK, error_list) = xssh.put_file(sftp_client, get_gmap_process_starter(), cluster_path)
if OK:
log.write('The file is uploaded.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# set run permision to the GMAP process starter in the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(get_gmap_process_starter())))
command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_starter()))
(OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The run permision is set.\n')
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# submit the GMAP process
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Submitting the process script {0}/{1} ...\n'.format(current_run_dir, os.path.basename(get_gmap_process_starter())))
sge_env = xcluster.get_sge_env()
command = '{0}; qsub -V -b n -cwd {1}/{2}'.format(sge_env, current_run_dir, os.path.basename(get_gmap_process_starter()))
(OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
for line in stdout:
log.write('{0}\n'.format(line))
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# close the SSH transport connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Closing the SSH transport connection ...\n')
xssh.close_ssh_transport_connection(ssh_transport)
log.write('The connection is closed.\n')
# close the SSH client connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Closing the SSH client connection ...\n')
xssh.close_ssh_client_connection(ssh_client)
log.write('The connection is closed.\n')
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut):
log.write('{0}\n'.format(xlib.get_separator()))
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK