def build_gmap_process_script(cluster_name, current_run_dir):
'''
Build the current GMAP process script.
'''
# initialize the control variable and the error list
OK = True
error_list = []
# get the GMAP option dictionary
gmap_option_dict = xlib.get_option_dict(get_gmap_config_file())
# get the options
experiment_id = gmap_option_dict['identification']['experiment_id']
reference_dataset_id = gmap_option_dict['identification']['reference_dataset_id']
reference_file = gmap_option_dict['identification']['reference_file']
assembly_software = gmap_option_dict['identification']['assembly_software']
assembly_dataset_id = gmap_option_dict['identification']['assembly_dataset_id']
assembly_type = gmap_option_dict['identification']['assembly_type']
threads = gmap_option_dict['GMAP parameters']['threads']
kmer = gmap_option_dict['GMAP parameters']['kmer']
sampling = gmap_option_dict['GMAP parameters']['sampling']
input_buffer_size = gmap_option_dict['GMAP parameters']['input-buffer-size']
output_buffer_size = gmap_option_dict['GMAP parameters']['output-buffer-size']
prunelevel = gmap_option_dict['GMAP parameters']['prunelevel']
format = gmap_option_dict['GMAP parameters']['format']
other_parameters = gmap_option_dict['GMAP parameters']['other_parameters']
# set the cluster reference dataset directory
cluster_reference_dataset_dir = xlib.get_cluster_reference_dataset_dir(reference_dataset_id)
# set the cluster reference file
cluster_reference_file = xlib.get_cluster_reference_file(reference_dataset_id, reference_file)
# set the GMAP database name
reference_file_name, reference_file_extension = os.path.splitext(reference_file)
gmap_database = '{0}-gmap_database'.format(reference_file_name)
# set the transcriptome file path
if assembly_software == xlib.get_soapdenovotrans_code():
if assembly_type.upper() == 'CONTIGS':
transcriptome_file = '{0}/{1}-{2}.contig'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id), experiment_id, assembly_dataset_id)
elif assembly_type.upper() == 'SCAFFOLDS':
transcriptome_file = '{0}/{1}-{2}.scafSeq'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id), experiment_id, assembly_dataset_id)
elif assembly_software == xlib.get_transabyss_code():
transcriptome_file = '{0}/transabyss-final.fa'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id))
elif assembly_software == xlib.get_trinity_code():
transcriptome_file = '{0}/Trinity.fasta'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id))
elif assembly_software == xlib.get_star_code():
transcriptome_file = '{0}/Trinity-GG.fasta'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id))
elif assembly_software == xlib.get_cd_hit_est_code():
transcriptome_file = '{0}/clustered-transcriptome.fasta'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id))
elif assembly_software == xlib.get_transcript_filter_code():
transcriptome_file = '{0}/filtered-transcriptome.fasta'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id))
# set the output file path
output_file = 'gmap_output_{0}.txt'.format(format.lower())
# get the GMAP process script name
gmap_process_script = get_gmap_process_script()
# write the GMAP process script
try:
if not os.path.exists(os.path.dirname(gmap_process_script)):
os.makedirs(os.path.dirname(gmap_process_script))
with open(gmap_process_script, mode='w', encoding='utf8', newline='\n') as file_id:
file_id.write('{0}\n'.format('#!/bin/bash'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('GMAP_GSNAP_PATH={0}/{1}/envs/{2}/bin'.format(xlib.get_cluster_app_dir(), xlib.get_miniconda3_name(), xlib.get_gmap_gsnap_bioconda_code())))
file_id.write('{0}\n'.format('PATH=$GMAP_GSNAP_PATH:$PATH'))
file_id.write('{0}\n'.format('SEP="#########################################"'))
file_id.write('{0}\n'.format('cd {0}/{1}/bin'.format(xlib.get_cluster_app_dir(), xlib.get_miniconda3_name())))
file_id.write('{0}\n'.format('source activate {0}'.format(xlib.get_gmap_gsnap_bioconda_code())))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('function init'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' INIT_DATETIME=`date --utc +%s`'))
file_id.write('{0}\n'.format(' FORMATTED_INIT_DATETIME=`date --date="@$INIT_DATETIME" "+%Y-%m-%d %H:%M:%S"`'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' echo "Script started in node $HOSTNAME of cluster {0} at $FORMATTED_INIT_DATETIME UTC."'.format(cluster_name)))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('function build_gmap_database'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' cd {0}'.format(current_run_dir)))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' /usr/bin/time \\'))
file_id.write('{0}\n'.format(' --format="$SEP\\nElapsed real time (s): %e\\nCPU time in kernel mode (s): %S\\nCPU time in user mode (s): %U\\nPercentage of CPU: %P\\nMaximum resident set size(Kb): %M\\nAverage total memory use (Kb):%K" \\'))
file_id.write('{0}\n'.format(' gmap_build \\'))
file_id.write('{0}\n'.format(' --dir={0}\\'.format(cluster_reference_dataset_dir)))
file_id.write('{0}\n'.format(' --db={0}\\'.format(gmap_database)))
if kmer.upper() != 'NONE':
file_id.write('{0}\n'.format(' --kmer={0} \\'.format(kmer)))
file_id.write('{0}\n'.format(' {0}'.format(cluster_reference_file)))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('function run_gmap_process'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' cd {0}'.format(current_run_dir)))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' gmap --version'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' /usr/bin/time \\'))
file_id.write('{0}\n'.format(' --format="$SEP\\nElapsed real time (s): %e\\nCPU time in kernel mode (s): %S\\nCPU time in user mode (s): %U\\nPercentage of CPU: %P\\nMaximum resident set size(Kb): %M\\nAverage total memory use (Kb):%K" \\'))
file_id.write('{0}\n'.format(' gmap \\'))
file_id.write('{0}\n'.format(' --nthreads={0} \\'.format(threads)))
file_id.write('{0}\n'.format(' --dir={0} \\'.format(cluster_reference_dataset_dir)))
file_id.write('{0}\n'.format(' --db={0} \\'.format(gmap_database)))
if kmer.upper() != 'NONE':
file_id.write('{0}\n'.format(' --kmer={0} \\'.format(kmer)))
if sampling.upper() != 'NONE':
file_id.write('{0}\n'.format(' --sampling={0} \\'.format(sampling)))
file_id.write('{0}\n'.format(' --input-buffer-size={0} \\'.format(input_buffer_size)))
file_id.write('{0}\n'.format(' --output-buffer-size={0} \\'.format(output_buffer_size)))
file_id.write('{0}\n'.format(' --prunelevel={0} \\'.format(prunelevel)))
if format.upper() == 'COMPRESS':
file_id.write('{0}\n'.format(' --compress \\'))
elif format.upper() == 'SUMMARY':
file_id.write('{0}\n'.format(' --summary \\'))
elif format.upper() == 'ALIGN':
file_id.write('{0}\n'.format(' --align \\'))
else:
file_id.write('{0}\n'.format(' --format={0} \\'.format(format.lower())))
file_id.write('{0}\n'.format(' --ordered \\'))
file_id.write('{0}\n'.format(' --nofails \\'))
if other_parameters.upper() != 'NONE':
parameter_list = [x.strip() for x in other_parameters.split(';')]
for i in range(len(parameter_list)):
if parameter_list[i].find('=') > 0:
pattern = r'^--(.+)=(.+)$'
mo = re.search(pattern, parameter_list[i])
parameter_name = mo.group(1).strip()
parameter_value = mo.group(2).strip()
file_id.write('{0}\n'.format(' --{0}={1} \\'.format(parameter_name, parameter_value)))
else:
pattern = r'^--(.+)$'
mo = re.search(pattern, parameter_list[i])
parameter_name = mo.group(1).strip()
file_id.write('{0}\n'.format(' --{0} \\'.format(parameter_name)))
file_id.write('{0}\n'.format(' {0} \\'.format(transcriptome_file)))
file_id.write('{0}\n'.format(' > {0}'.format(output_file)))
file_id.write('{0}\n'.format(' RC=$?'))
file_id.write('{0}\n'.format(' if [ $RC -ne 0 ]; then manage_error gmap $RC; fi'))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('function end'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' END_DATETIME=`date --utc +%s`'))
file_id.write('{0}\n'.format(' FORMATTED_END_DATETIME=`date --date="@$END_DATETIME" "+%Y-%m-%d %H:%M:%S"`'))
file_id.write('{0}\n'.format(' calculate_duration'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' echo "Script ended OK at $FORMATTED_END_DATETIME UTC with a run duration of $DURATION s ($FORMATTED_DURATION)."'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' RECIPIENT={0}'.format(xconfiguration.get_contact_data())))
file_id.write('{0}\n'.format(' SUBJECT="{0}: {1} process"'.format(xlib.get_project_name(), xlib.get_gmap_name())))
file_id.write('{0}\n'.format(' MESSAGE="The {0} process in node $HOSTNAME of cluster {1} ended OK at $FORMATTED_END_DATETIME with a run duration of $DURATION s ($FORMATTED_DURATION). Please review its log.<br/><br/>Regards,<br/>GI Genetica, Fisiologia e Historia Forestal<br/>Dpto. Sistemas y Recursos Naturales<br/>ETSI Montes, Forestal y del Medio Natural<br/>Universidad Politecnica de Madrid<br/>https://github.com/ggfhf/"'.format(xlib.get_gmap_name(), cluster_name)))
file_id.write('{0}\n'.format(' mail --append "Content-type: text/html;" --subject "$SUBJECT" "$RECIPIENT" <<< "$MESSAGE"'))
file_id.write('{0}\n'.format(' exit 0'))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('function manage_error'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' END_DATETIME=`date --utc +%s`'))
file_id.write('{0}\n'.format(' FORMATTED_END_DATETIME=`date --date="@$END_DATETIME" "+%Y-%m-%d %H:%M:%S"`'))
file_id.write('{0}\n'.format(' calculate_duration'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' echo "ERROR: $1 returned error $2"'))
file_id.write('{0}\n'.format(' echo "Script ended WRONG at $FORMATTED_END_DATETIME UTC with a run duration of $DURATION s ($FORMATTED_DURATION)."'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' RECIPIENT={0}'.format(xconfiguration.get_contact_data())))
file_id.write('{0}\n'.format(' SUBJECT="{0}: {1} process"'.format(xlib.get_project_name(), xlib.get_gmap_name())))
file_id.write('{0}\n'.format(' MESSAGE="The {0} process in node $HOSTNAME of cluster {1} ended WRONG at $FORMATTED_END_DATETIME with a run duration of $DURATION s ($FORMATTED_DURATION). Please review its log.<br/><br/>Regards,<br/>GI Genetica, Fisiologia e Historia Forestal<br/>Dpto. Sistemas y Recursos Naturales<br/>ETSI Montes, Forestal y del Medio Natural<br/>Universidad Politecnica de Madrid<br/>https://github.com/ggfhf/"'.format(xlib.get_gmap_name(), cluster_name)))
file_id.write('{0}\n'.format(' mail --append "Content-type: text/html;" --subject "$SUBJECT" "$RECIPIENT" <<< "$MESSAGE"'))
file_id.write('{0}\n'.format(' exit 3'))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('function calculate_duration'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' DURATION=`expr $END_DATETIME - $INIT_DATETIME`'))
file_id.write('{0}\n'.format(' HH=`expr $DURATION / 3600`'))
file_id.write('{0}\n'.format(' MM=`expr $DURATION % 3600 / 60`'))
file_id.write('{0}\n'.format(' SS=`expr $DURATION % 60`'))
file_id.write('{0}\n'.format(' FORMATTED_DURATION=`printf "%03d:%02d:%02d\\n" $HH $MM $SS`'))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('init'))
file_id.write('{0}\n'.format('build_gmap_database'))
file_id.write('{0}\n'.format('run_gmap_process'))
file_id.write('{0}\n'.format('end'))
except:
error_list.append('*** ERROR: The file {0} can not be created'.format(gmap_process_script))
OK = False
# return the control variable and the error list
return (OK, error_list)
def build_fastqc_process_script(cluster_name, current_run_dir):
'''
Build the current FastQC process script.
'''
# initialize the control variable and the error list
OK = True
error_list = []
# get the FastQC option dictionary
fastqc_option_dict = xlib.get_option_dict(get_fastqc_config_file())
# get the options
experiment_id = fastqc_option_dict['identification']['experiment_id']
read_dataset_id = fastqc_option_dict['identification']['read_dataset_id']
threads = fastqc_option_dict['FastQC parameters']['threads']
# get the sections list
sections_list = []
for section in fastqc_option_dict.keys():
sections_list.append(section)
sections_list.sort()
# build the file name list
file_name_list = []
for section in sections_list:
# if the section identification is like library-n
if re.match('^file-[0-9]+$', section):
file_name = fastqc_option_dict[section]['file_name']
file_name_list.append(file_name)
# write the FastQC process script
try:
if not os.path.exists(os.path.dirname(get_fastqc_process_script())):
os.makedirs(os.path.dirname(get_fastqc_process_script()))
with open(get_fastqc_process_script(),
mode='w',
encoding='utf8',
newline='\n') as file_id:
file_id.write('{0}\n'.format('#!/bin/bash'))
file_id.write('{0}\n'.format(
'#-------------------------------------------------------------------------------'
))
file_id.write('{0}\n'.format(
'FASTQC_PATH={0}/{1}/envs/{2}/bin'.format(
xlib.get_cluster_app_dir(), xlib.get_miniconda3_name(),
xlib.get_fastqc_bioconda_code())))
file_id.write('{0}\n'.format('PATH=$FASTQC_PATH:$PATH'))
file_id.write('{0}\n'.format(
'SEP="#########################################"'))
file_id.write('{0}\n'.format('cd {0}/{1}/bin'.format(
xlib.get_cluster_app_dir(), xlib.get_miniconda3_name())))
file_id.write('{0}\n'.format('source activate {0}'.format(
xlib.get_fastqc_bioconda_code())))
file_id.write('{0}\n'.format(
'#-------------------------------------------------------------------------------'
))
file_id.write('{0}\n'.format('function init'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' INIT_DATETIME=`date --utc +%s`'))
file_id.write('{0}\n'.format(
' FORMATTED_INIT_DATETIME=`date --date="@$INIT_DATETIME" "+%Y-%m-%d %H:%M:%S"`'
))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(
' echo "Script started in node $HOSTNAME of cluster {0} at $FORMATTED_INIT_DATETIME UTC."'
.format(cluster_name)))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format(
'#-------------------------------------------------------------------------------'
))
file_id.write('{0}\n'.format('function run_fastqc_process'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' cd {0}'.format(current_run_dir)))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' fastqc --version'))
for file_name in file_name_list:
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' /usr/bin/time \\'))
file_id.write('{0}\n'.format(
' --format="$SEP\\nElapsed real time (s): %e\\nCPU time in kernel mode (s): %S\\nCPU time in user mode (s): %U\\nPercentage of CPU: %P\\nMaximum resident set size(Kb): %M\\nAverage total memory use (Kb):%K" \\'
))
file_id.write('{0}\n'.format(' fastqc \\'))
file_id.write('{0}\n'.format(' {0} \\'.format(
xlib.get_cluster_read_file(experiment_id, read_dataset_id,
file_name))))
file_id.write('{0}\n'.format(
' --threads={0} \\'.format(threads)))
file_id.write('{0}\n'.format(
' --outdir={0}'.format(current_run_dir)))
file_id.write('{0}\n'.format(' RC=$?'))
file_id.write('{0}\n'.format(
' if [ $RC -ne 0 ]; then manage_error fastqc $RC; fi'))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format(
'#-------------------------------------------------------------------------------'
))
file_id.write('{0}\n'.format('function end'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' END_DATETIME=`date --utc +%s`'))
file_id.write('{0}\n'.format(
' FORMATTED_END_DATETIME=`date --date="@$END_DATETIME" "+%Y-%m-%d %H:%M:%S"`'
))
file_id.write('{0}\n'.format(' calculate_duration'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(
' echo "Script ended OK at $FORMATTED_END_DATETIME UTC with a run duration of $DURATION s ($FORMATTED_DURATION)."'
))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' RECIPIENT={0}'.format(
xconfiguration.get_contact_data())))
file_id.write('{0}\n'.format(
' SUBJECT="{0}: {1} process"'.format(
xlib.get_project_name(), xlib.get_fastqc_name())))
file_id.write('{0}\n'.format(
' MESSAGE="The {0} process in node $HOSTNAME of cluster {1} ended OK at $FORMATTED_END_DATETIME with a run duration of $DURATION s ($FORMATTED_DURATION). Please review its log.<br/><br/>Regards,<br/>GI Genetica, Fisiologia e Historia Forestal<br/>Dpto. Sistemas y Recursos Naturales<br/>ETSI Montes, Forestal y del Medio Natural<br/>Universidad Politecnica de Madrid<br/>https://github.com/ggfhf/"'
.format(xlib.get_fastqc_name(), cluster_name)))
file_id.write('{0}\n'.format(
' mail --append "Content-type: text/html;" --subject "$SUBJECT" "$RECIPIENT" <<< "$MESSAGE"'
))
file_id.write('{0}\n'.format(' exit 0'))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format(
'#-------------------------------------------------------------------------------'
))
file_id.write('{0}\n'.format('function manage_error'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' END_DATETIME=`date --utc +%s`'))
file_id.write('{0}\n'.format(
' FORMATTED_END_DATETIME=`date --date="@$END_DATETIME" "+%Y-%m-%d %H:%M:%S"`'
))
file_id.write('{0}\n'.format(' calculate_duration'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write(
'{0}\n'.format(' echo "ERROR: $1 returned error $2"'))
file_id.write('{0}\n'.format(
' echo "Script ended WRONG at $FORMATTED_END_DATETIME UTC with a run duration of $DURATION s ($FORMATTED_DURATION)."'
))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' RECIPIENT={0}'.format(
xconfiguration.get_contact_data())))
file_id.write('{0}\n'.format(
' SUBJECT="{0}: {1} process"'.format(
xlib.get_project_name(), xlib.get_fastqc_name())))
file_id.write('{0}\n'.format(
' MESSAGE="The {0} process in node $HOSTNAME of cluster {1} ended WRONG at $FORMATTED_END_DATETIME with a run duration of $DURATION s ($FORMATTED_DURATION). Please review its log.<br/><br/>Regards,<br/>GI Genetica, Fisiologia e Historia Forestal<br/>Dpto. Sistemas y Recursos Naturales<br/>ETSI Montes, Forestal y del Medio Natural<br/>Universidad Politecnica de Madrid<br/>https://github.com/ggfhf/"'
.format(xlib.get_fastqc_name(), cluster_name)))
file_id.write('{0}\n'.format(
' mail --append "Content-type: text/html;" --subject "$SUBJECT" "$RECIPIENT" <<< "$MESSAGE"'
))
file_id.write('{0}\n'.format(' exit 3'))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format(
'#-------------------------------------------------------------------------------'
))
file_id.write('{0}\n'.format('function calculate_duration'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(
' DURATION=`expr $END_DATETIME - $INIT_DATETIME`'))
file_id.write('{0}\n'.format(' HH=`expr $DURATION / 3600`'))
file_id.write(
'{0}\n'.format(' MM=`expr $DURATION % 3600 / 60`'))
file_id.write('{0}\n'.format(' SS=`expr $DURATION % 60`'))
file_id.write('{0}\n'.format(
' FORMATTED_DURATION=`printf "%03d:%02d:%02d\\n" $HH $MM $SS`'
))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format(
'#-------------------------------------------------------------------------------'
))
file_id.write('{0}\n'.format('init'))
file_id.write('{0}\n'.format('run_fastqc_process'))
file_id.write('{0}\n'.format('end'))
except:
error_list.append('*** ERROR: The file {0} can not be created'.format(
get_fastqc_process_script()))
OK = False
# return the control variable and the error list
return (OK, error_list)
def build_cd_hit_est_process_script(cluster_name, current_run_dir):
'''
Build the current CD-HIT-EST process script.
'''
# initialize the control variable and the error list
OK = True
error_list = []
# get the option dictionary
cd_hit_est_option_dict = xlib.get_option_dict(get_cd_hit_est_config_file())
# get the options
experiment_id = cd_hit_est_option_dict['identification']['experiment_id']
assembly_software = cd_hit_est_option_dict['identification'][
'assembly_software']
assembly_dataset_id = cd_hit_est_option_dict['identification'][
'assembly_dataset_id']
assembly_type = cd_hit_est_option_dict['identification']['assembly_type']
threads = cd_hit_est_option_dict['CD-HIT-EST parameters']['threads']
memory_limit = cd_hit_est_option_dict['CD-HIT-EST parameters'][
'memory_limit']
seq_identity_threshold = cd_hit_est_option_dict['CD-HIT-EST parameters'][
'seq_identity_threshold']
word_length = cd_hit_est_option_dict['CD-HIT-EST parameters'][
'word_length']
mask = cd_hit_est_option_dict['CD-HIT-EST parameters']['mask']
match = cd_hit_est_option_dict['CD-HIT-EST parameters']['match']
mismatch = cd_hit_est_option_dict['CD-HIT-EST parameters']['mismatch']
other_parameters = cd_hit_est_option_dict['CD-HIT-EST parameters'][
'other_parameters']
# set the transcriptome file path
if assembly_software == xlib.get_soapdenovotrans_code():
if assembly_type == 'CONTIGS':
transcriptome_file = '{0}/{1}-{2}.contig'.format(
xlib.get_cluster_experiment_result_dataset_dir(
experiment_id, assembly_dataset_id), experiment_id,
assembly_dataset_id)
elif assembly_type == 'SCAFFOLDS':
transcriptome_file = '{0}/{1}-{2}.scafSeq'.format(
xlib.get_cluster_experiment_result_dataset_dir(
experiment_id, assembly_dataset_id), experiment_id,
assembly_dataset_id)
elif assembly_software == xlib.get_transabyss_code():
transcriptome_file = '{0}/transabyss-final.fa'.format(
xlib.get_cluster_experiment_result_dataset_dir(
experiment_id, assembly_dataset_id))
elif assembly_software == xlib.get_trinity_code():
transcriptome_file = '{0}/Trinity.fasta'.format(
xlib.get_cluster_experiment_result_dataset_dir(
experiment_id, assembly_dataset_id))
elif assembly_software == xlib.get_star_code():
transcriptome_file = '{0}/Trinity-GG.fasta'.format(
xlib.get_cluster_experiment_result_dataset_dir(
experiment_id, assembly_dataset_id))
elif assembly_software == xlib.get_cd_hit_est_code():
transcriptome_file = '{0}/clustered-transcriptome.fasta'.format(
xlib.get_cluster_experiment_result_dataset_dir(
experiment_id, assembly_dataset_id))
elif assembly_software == xlib.get_transcript_filter_code():
transcriptome_file = '{0}/filtered-transcriptome.fasta'.format(
xlib.get_cluster_experiment_result_dataset_dir(
experiment_id, assembly_dataset_id))
# set the output file path
if OK:
output_file = '{0}/clustered-transcriptome.fasta'.format(
current_run_dir)
# write the CD-HIT-EST process script
try:
if not os.path.exists(os.path.dirname(
get_cd_hit_est_process_script())):
os.makedirs(os.path.dirname(get_cd_hit_est_process_script()))
with open(get_cd_hit_est_process_script(),
mode='w',
encoding='utf8',
newline='\n') as file_id:
file_id.write('{0}\n'.format('#!/bin/bash'))
file_id.write('{0}\n'.format(
'#-------------------------------------------------------------------------------'
))
file_id.write('{0}\n'.format(
'CDHIT_PATH={0}/{1}/envs/{2}/bin'.format(
xlib.get_cluster_app_dir(), xlib.get_miniconda3_name(),
xlib.get_cd_hit_bioconda_code())))
file_id.write('{0}\n'.format('PATH=$CDHIT_PATH:$PATH'))
file_id.write('{0}\n'.format(
'SEP="#########################################"'))
file_id.write('{0}\n'.format('cd {0}/{1}/bin'.format(
xlib.get_cluster_app_dir(), xlib.get_miniconda3_name())))
file_id.write('{0}\n'.format('source activate {0}'.format(
xlib.get_cd_hit_bioconda_code())))
file_id.write('{0}\n'.format(
'#-------------------------------------------------------------------------------'
))
file_id.write('{0}\n'.format('function init'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' INIT_DATETIME=`date --utc +%s`'))
file_id.write('{0}\n'.format(
' FORMATTED_INIT_DATETIME=`date --date="@$INIT_DATETIME" "+%Y-%m-%d %H:%M:%S"`'
))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(
' echo "Script started in node $HOSTNAME of cluster {0} at $FORMATTED_INIT_DATETIME UTC."'
.format(cluster_name)))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format(
'#-------------------------------------------------------------------------------'
))
file_id.write('{0}\n'.format('function run_cd_hit_est_process'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' cd {0}'.format(current_run_dir)))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(
' echo "Running {0} process ..."'.format(
xlib.get_cd_hit_est_name())))
file_id.write('{0}\n'.format(' /usr/bin/time \\'))
file_id.write('{0}\n'.format(
' --format="$SEP\\nElapsed real time (s): %e\\nCPU time in kernel mode (s): %S\\nCPU time in user mode (s): %U\\nPercentage of CPU: %P\\nMaximum resident set size(Kb): %M\\nAverage total memory use (Kb):%K" \\'
))
file_id.write('{0}\n'.format(' cd-hit-est \\'))
file_id.write('{0}\n'.format(
' -T {0} \\'.format(threads)))
file_id.write('{0}\n'.format(
' -M {0} \\'.format(memory_limit)))
file_id.write('{0}\n'.format(
' -i {0} \\'.format(transcriptome_file)))
file_id.write('{0}\n'.format(
' -c {0} \\'.format(seq_identity_threshold)))
file_id.write('{0}\n'.format(
' -n {0} \\'.format(word_length)))
file_id.write('{0}\n'.format(
' -mask {0} \\'.format(mask)))
file_id.write('{0}\n'.format(
' -match {0} \\'.format(match)))
file_id.write('{0}\n'.format(
' -mismatch {0} \\'.format(mismatch)))
if other_parameters.upper() == 'NONE':
file_id.write('{0}\n'.format(
' -o {0}'.format(output_file)))
else:
file_id.write('{0}\n'.format(
' -o {0} \\'.format(output_file)))
parameter_list = [
x.strip() for x in other_parameters.split(';')
]
for i in range(len(parameter_list)):
if parameter_list[i].find('=') > 0:
pattern = r'^--(.+)=(.+)$'
mo = re.search(pattern, parameter_list[i])
parameter_name = mo.group(1).strip()
parameter_value = mo.group(2).strip()
if i < len(parameter_list) - 1:
file_id.write('{0}\n'.format(
' -{0} {1} \\'.format(
parameter_name, parameter_value)))
else:
file_id.write('{0}\n'.format(
' -{0} {1}'.format(
parameter_name, parameter_value)))
else:
pattern = r'^--(.+)$'
mo = re.search(pattern, parameter_list[i])
parameter_name = mo.group(1).strip()
if i < len(parameter_list):
file_id.write('{0}\n'.format(
' -{0} \\'.format(parameter_name)))
else:
file_id.write('{0}\n'.format(
' -{0}'.format(parameter_name)))
i += 1
file_id.write('{0}\n'.format(' RC=$?'))
file_id.write('{0}\n'.format(
' if [ $RC -ne 0 ]; then manage_error cd-hit-est $RC; fi'))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format(
'#-------------------------------------------------------------------------------'
))
file_id.write('{0}\n'.format('function end'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' END_DATETIME=`date --utc +%s`'))
file_id.write('{0}\n'.format(
' FORMATTED_END_DATETIME=`date --date="@$END_DATETIME" "+%Y-%m-%d %H:%M:%S"`'
))
file_id.write('{0}\n'.format(' calculate_duration'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(
' echo "Script ended OK at $FORMATTED_END_DATETIME UTC with a run duration of $DURATION s ($FORMATTED_DURATION)."'
))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' RECIPIENT={0}'.format(
xconfiguration.get_contact_data())))
file_id.write('{0}\n'.format(
' SUBJECT="{0}: {1} process"'.format(
xlib.get_project_name(), xlib.get_cd_hit_est_name())))
file_id.write('{0}\n'.format(
' MESSAGE="The {0} process in node $HOSTNAME of cluster {0} ended OK at $FORMATTED_END_DATETIME with a run duration of $DURATION s ($FORMATTED_DURATION). Please review its log.<br/><br/>Regards,<br/>GI Genetica, Fisiologia e Historia Forestal<br/>Dpto. Sistemas y Recursos Naturales<br/>ETSI Montes, Forestal y del Medio Natural<br/>Universidad Politecnica de Madrid<br/>https://github.com/ggfhf/"'
.format(xlib.get_rsem_eval_name(), cluster_name)))
file_id.write('{0}\n'.format(
' mail --append "Content-type: text/html;" --subject "$SUBJECT" "$RECIPIENT" <<< "$MESSAGE"'
))
file_id.write('{0}\n'.format(' exit 0'))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format(
'#-------------------------------------------------------------------------------'
))
file_id.write('{0}\n'.format('function manage_error'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' END_DATETIME=`date --utc +%s`'))
file_id.write('{0}\n'.format(
' FORMATTED_END_DATETIME=`date --date="@$END_DATETIME" "+%Y-%m-%d %H:%M:%S"`'
))
file_id.write('{0}\n'.format(' calculate_duration'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write(
'{0}\n'.format(' echo "ERROR: $1 returned error $2"'))
file_id.write('{0}\n'.format(
' echo "Script ended WRONG at $FORMATTED_END_DATETIME UTC with a run duration of $DURATION s ($FORMATTED_DURATION)."'
))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' RECIPIENT={0}'.format(
xconfiguration.get_contact_data())))
file_id.write('{0}\n'.format(
' SUBJECT="{0}: {1} process"'.format(
xlib.get_project_name(), xlib.get_cd_hit_est_name())))
file_id.write('{0}\n'.format(
' MESSAGE="The {0} process in node $HOSTNAME of cluster {0} ended WRONG at $FORMATTED_END_DATETIME with a run duration of $DURATION s ($FORMATTED_DURATION). Please review its log.<br/><br/>Regards,<br/>GI Genetica, Fisiologia e Historia Forestal<br/>Dpto. Sistemas y Recursos Naturales<br/>ETSI Montes, Forestal y del Medio Natural<br/>Universidad Politecnica de Madrid<br/>https://github.com/ggfhf/"'
.format(xlib.get_rsem_eval_name(), cluster_name)))
file_id.write('{0}\n'.format(
' mail --append "Content-type: text/html;" --subject "$SUBJECT" "$RECIPIENT" <<< "$MESSAGE"'
))
file_id.write('{0}\n'.format(' exit 3'))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format(
'#-------------------------------------------------------------------------------'
))
file_id.write('{0}\n'.format('function calculate_duration'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(
' DURATION=`expr $END_DATETIME - $INIT_DATETIME`'))
file_id.write('{0}\n'.format(' HH=`expr $DURATION / 3600`'))
file_id.write(
'{0}\n'.format(' MM=`expr $DURATION % 3600 / 60`'))
file_id.write('{0}\n'.format(' SS=`expr $DURATION % 60`'))
file_id.write('{0}\n'.format(
' FORMATTED_DURATION=`printf "%03d:%02d:%02d\\n" $HH $MM $SS`'
))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format(
'#-------------------------------------------------------------------------------'
))
file_id.write('{0}\n'.format('init'))
file_id.write('{0}\n'.format('run_cd_hit_est_process'))
file_id.write('{0}\n'.format('end'))
except:
error_list.append('*** ERROR: The file {0} can not be created'.format(
get_cd_hit_est_process_script()))
OK = False
# return the control variable and the error list
return (OK, error_list)
def build_quast_process_script(cluster_name, current_run_dir):
'''
Build the current QUAST process script.
'''
# initialize the control variable and the error list
OK = True
error_list = []
# get the QUAST option dictionary
quast_option_dict = xlib.get_option_dict(get_quast_config_file())
# get the options
experiment_id = quast_option_dict['identification']['experiment_id']
reference_dataset_id = quast_option_dict['identification']['reference_dataset_id']
reference_file = quast_option_dict['identification']['reference_file']
assembly_software = quast_option_dict['identification']['assembly_software']
assembly_dataset_id = quast_option_dict['identification']['assembly_dataset_id']
assembly_type = quast_option_dict['identification']['assembly_type']
threads = quast_option_dict['QUAST parameters']['threads']
# set the reference file path
if reference_dataset_id.upper() != 'NONE':
reference_file_path = xlib.get_cluster_reference_file(reference_dataset_id, reference_file)
# set the transcriptome file path
if assembly_software == xlib.get_soapdenovotrans_code():
if assembly_type.upper() == 'CONTIGS':
transcriptome_file = '{0}/{1}-{2}.contig'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id), experiment_id, assembly_dataset_id)
elif assembly_type.upper() == 'SCAFFOLDS':
transcriptome_file = '{0}/{1}-{2}.scafSeq'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id), experiment_id, assembly_dataset_id)
elif assembly_software == xlib.get_transabyss_code():
transcriptome_file = '{0}/transabyss-final.fa'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id))
elif assembly_software == xlib.get_trinity_code():
transcriptome_file = '{0}/Trinity.fasta'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id))
elif assembly_software == xlib.get_star_code():
transcriptome_file = '{0}/Trinity-GG.fasta'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id))
elif assembly_software == xlib.get_cd_hit_est_code():
transcriptome_file = '{0}/clustered-transcriptome.fasta'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id))
elif assembly_software == xlib.get_transcript_filter_code():
transcriptome_file = '{0}/filtered-transcriptome.fasta'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id))
# get the QUAST process script name
quast_process_script = get_quast_process_script()
# write the QUAST process script
try:
if not os.path.exists(os.path.dirname(quast_process_script)):
os.makedirs(os.path.dirname(quast_process_script))
with open(quast_process_script, mode='w', encoding='utf8', newline='\n') as file_id:
file_id.write('{0}\n'.format('#!/bin/bash'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('QUAST_PATH={0}/{1}/envs/{2}/bin'.format(xlib.get_cluster_app_dir(), xlib.get_miniconda3_name(), xlib.get_quast_bioconda_code())))
file_id.write('{0}\n'.format('PATH=$QUAST_PATH:$PATH'))
file_id.write('{0}\n'.format('SEP="#########################################"'))
file_id.write('{0}\n'.format('cd {0}/{1}/bin'.format(xlib.get_cluster_app_dir(), xlib.get_miniconda3_name())))
file_id.write('{0}\n'.format('source activate {0}'.format(xlib.get_quast_bioconda_code())))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('function init'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' INIT_DATETIME=`date --utc +%s`'))
file_id.write('{0}\n'.format(' FORMATTED_INIT_DATETIME=`date --date="@$INIT_DATETIME" "+%Y-%m-%d %H:%M:%S"`'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' echo "Script started in node $HOSTNAME of cluster {0} at $FORMATTED_INIT_DATETIME UTC."'.format(cluster_name)))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('function run_quast_process'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' cd {0}'.format(current_run_dir)))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' quast.py --version'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' /usr/bin/time \\'))
file_id.write('{0}\n'.format(' --format="$SEP\\nElapsed real time (s): %e\\nCPU time in kernel mode (s): %S\\nCPU time in user mode (s): %U\\nPercentage of CPU: %P\\nMaximum resident set size(Kb): %M\\nAverage total memory use (Kb):%K" \\'))
file_id.write('{0}\n'.format(' quast.py \\'))
file_id.write('{0}\n'.format(' --threads {0} \\'.format(threads)))
file_id.write('{0}\n'.format(' --output-dir {0} \\'.format(current_run_dir)))
if reference_dataset_id.upper() != 'NONE':
file_id.write('{0}\n'.format(' -R {0} \\'.format(reference_file_path)))
if assembly_type.upper() == 'SCAFFOLDS':
file_id.write('{0}\n'.format(' --scaffolds \\'))
file_id.write('{0}\n'.format(' {0}'.format(transcriptome_file)))
file_id.write('{0}\n'.format(' RC=$?'))
file_id.write('{0}\n'.format(' if [ $RC -ne 0 ]; then manage_error quast.py $RC; fi'))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('function end'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' END_DATETIME=`date --utc +%s`'))
file_id.write('{0}\n'.format(' FORMATTED_END_DATETIME=`date --date="@$END_DATETIME" "+%Y-%m-%d %H:%M:%S"`'))
file_id.write('{0}\n'.format(' calculate_duration'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' echo "Script ended OK at $FORMATTED_END_DATETIME UTC with a run duration of $DURATION s ($FORMATTED_DURATION)."'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' RECIPIENT={0}'.format(xconfiguration.get_contact_data())))
file_id.write('{0}\n'.format(' SUBJECT="{0}: {1} process"'.format(xlib.get_project_name(), xlib.get_quast_name())))
file_id.write('{0}\n'.format(' MESSAGE="The {0} process in node $HOSTNAME of cluster {1} ended OK at $FORMATTED_END_DATETIME with a run duration of $DURATION s ($FORMATTED_DURATION). Please review its log.<br/><br/>Regards,<br/>GI Genetica, Fisiologia e Historia Forestal<br/>Dpto. Sistemas y Recursos Naturales<br/>ETSI Montes, Forestal y del Medio Natural<br/>Universidad Politecnica de Madrid<br/>https://github.com/ggfhf/"'.format(xlib.get_quast_name(), cluster_name)))
file_id.write('{0}\n'.format(' mail --append "Content-type: text/html;" --subject "$SUBJECT" "$RECIPIENT" <<< "$MESSAGE"'))
file_id.write('{0}\n'.format(' exit 0'))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('function manage_error'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' END_DATETIME=`date --utc +%s`'))
file_id.write('{0}\n'.format(' FORMATTED_END_DATETIME=`date --date="@$END_DATETIME" "+%Y-%m-%d %H:%M:%S"`'))
file_id.write('{0}\n'.format(' calculate_duration'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' echo "ERROR: $1 returned error $2"'))
file_id.write('{0}\n'.format(' echo "Script ended WRONG at $FORMATTED_END_DATETIME UTC with a run duration of $DURATION s ($FORMATTED_DURATION)."'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' RECIPIENT={0}'.format(xconfiguration.get_contact_data())))
file_id.write('{0}\n'.format(' SUBJECT="{0}: {1} process"'.format(xlib.get_project_name(), xlib.get_quast_name())))
file_id.write('{0}\n'.format(' MESSAGE="The {0} process in node $HOSTNAME of cluster {1} ended WRONG at $FORMATTED_END_DATETIME with a run duration of $DURATION s ($FORMATTED_DURATION). Please review its log.<br/><br/>Regards,<br/>GI Genetica, Fisiologia e Historia Forestal<br/>Dpto. Sistemas y Recursos Naturales<br/>ETSI Montes, Forestal y del Medio Natural<br/>Universidad Politecnica de Madrid<br/>https://github.com/ggfhf/"'.format(xlib.get_quast_name(), cluster_name)))
file_id.write('{0}\n'.format(' mail --append "Content-type: text/html;" --subject "$SUBJECT" "$RECIPIENT" <<< "$MESSAGE"'))
file_id.write('{0}\n'.format(' exit 3'))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('function calculate_duration'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' DURATION=`expr $END_DATETIME - $INIT_DATETIME`'))
file_id.write('{0}\n'.format(' HH=`expr $DURATION / 3600`'))
file_id.write('{0}\n'.format(' MM=`expr $DURATION % 3600 / 60`'))
file_id.write('{0}\n'.format(' SS=`expr $DURATION % 60`'))
file_id.write('{0}\n'.format(' FORMATTED_DURATION=`printf "%03d:%02d:%02d\\n" $HH $MM $SS`'))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('init'))
file_id.write('{0}\n'.format('run_quast_process'))
file_id.write('{0}\n'.format('end'))
except:
error_list.append('*** ERROR: The file {0} can not be created'.format(quast_process_script))
OK = False
# return the control variable and the error list
return (OK, error_list)
def build_busco_process_script(cluster_name, current_run_dir):
'''
Build the current BUSCO process script.
'''
# initialize the control variable and the error list
OK = True
error_list = []
# get the BUSCO option dictionary
busco_option_dict = xlib.get_option_dict(get_busco_config_file())
# get the options
experiment_id = busco_option_dict['identification']['experiment_id']
assembly_software = busco_option_dict['identification']['assembly_software']
assembly_dataset_id = busco_option_dict['identification']['assembly_dataset_id']
assembly_type = busco_option_dict['identification']['assembly_type']
ncpu = busco_option_dict['BUSCO parameters']['ncpu']
lineage_data = busco_option_dict['BUSCO parameters']['lineage_data']
lineage_data_file = '{0}.tar.gz'.format(lineage_data)
lineage_data_url = 'http://busco.ezlab.org/v2/datasets/{0}'.format(lineage_data_file)
mode = busco_option_dict['BUSCO parameters']['mode'].lower()
evalue = busco_option_dict['BUSCO parameters']['evalue']
limit = busco_option_dict['BUSCO parameters']['limit']
species = busco_option_dict['BUSCO parameters']['species']
long = busco_option_dict['BUSCO parameters']['long'].upper()
augustus_options = busco_option_dict['BUSCO parameters']['augustus_options'].upper()
# set the transcriptome file path
if assembly_software == xlib.get_soapdenovotrans_code():
if assembly_type == 'CONTIGS':
transcriptome_file = '{0}/{1}-{2}.contig'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id), experiment_id, assembly_dataset_id)
elif assembly_type == 'SCAFFOLDS':
transcriptome_file = '{0}/{1}-{2}.scafSeq'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id), experiment_id, assembly_dataset_id)
elif assembly_software == xlib.get_transabyss_code():
transcriptome_file = '{0}/transabyss-final.fa'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id))
elif assembly_software == xlib.get_trinity_code():
transcriptome_file = '{0}/Trinity.fasta'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id))
elif assembly_software == xlib.get_star_code():
transcriptome_file = '{0}/Trinity-GG.fasta'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id))
elif assembly_software == xlib.get_cd_hit_est_code():
transcriptome_file = '{0}/clustered-transcriptome.fasta'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id))
elif assembly_software == xlib.get_transcript_filter_code():
transcriptome_file = '{0}/filtered-transcriptome.fasta'.format(xlib.get_cluster_experiment_result_dataset_dir(experiment_id, assembly_dataset_id))
# write the BUSCO process script
try:
if not os.path.exists(os.path.dirname(get_busco_process_script())):
os.makedirs(os.path.dirname(get_busco_process_script()))
with open(get_busco_process_script(), mode='w', encoding='utf8', newline='\n') as file_id:
file_id.write('{0}\n'.format('#!/bin/bash'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('BUSCO_PATH={0}/{1}/envs/{2}/bin'.format(xlib.get_cluster_app_dir(), xlib.get_miniconda3_name(), xlib.get_busco_bioconda_code())))
file_id.write('{0}\n'.format('export PATH=$BUSCO_PATH:$PATH'))
file_id.write('{0}\n'.format('SEP="#########################################"'))
file_id.write('{0}\n'.format('cd {0}/{1}/bin'.format(xlib.get_cluster_app_dir(), xlib.get_miniconda3_name())))
file_id.write('{0}\n'.format('source activate {0}'.format(xlib.get_busco_bioconda_code())))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('function init'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' INIT_DATETIME=`date --utc +%s`'))
file_id.write('{0}\n'.format(' FORMATTED_INIT_DATETIME=`date --date="@$INIT_DATETIME" "+%Y-%m-%d %H:%M:%S"`'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' echo "Script started in node $HOSTNAME of cluster {0} at $FORMATTED_INIT_DATETIME UTC."'.format(cluster_name)))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('function download_lineage_data'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' cd {0}'.format(current_run_dir)))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' echo "Downloading lineage data ..."'))
file_id.write('{0}\n'.format(' wget --quiet --output-document ./{0} {1}'.format(lineage_data_file, lineage_data_url)))
file_id.write('{0}\n'.format(' tar -xzvf ./{0}'.format(lineage_data_file)))
file_id.write('{0}\n'.format(' rm ./{0}'.format(lineage_data_file)))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('function run_busco_process'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' cd {0}'.format(current_run_dir)))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' run_BUSCO.py --version'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' /usr/bin/time \\'))
file_id.write('{0}\n'.format(' --format="$SEP\\nElapsed real time (s): %e\\nCPU time in kernel mode (s): %S\\nCPU time in user mode (s): %U\\nPercentage of CPU: %P\\nMaximum resident set size(Kb): %M\\nAverage total memory use (Kb):%K" \\'))
file_id.write('{0}\n'.format(' run_BUSCO.py \\'))
file_id.write('{0}\n'.format(' --cpu={0} \\'.format(ncpu)))
file_id.write('{0}\n'.format(' --lineage_path=./{0} \\'.format(lineage_data)))
file_id.write('{0}\n'.format(' --mode={0} \\'.format(mode)))
file_id.write('{0}\n'.format(' --evalue={0} \\'.format(evalue)))
file_id.write('{0}\n'.format(' --limit={0} \\'.format(limit)))
if species.upper() != 'NONE':
file_id.write('{0}\n'.format(' --species={0} \\'.format(species)))
if long == 'YES':
file_id.write('{0}\n'.format(' --long \\'))
if augustus_options.upper() != 'NONE':
file_id.write('{0}\n'.format(" --august_options='{0}' \\".format(augustus_options)))
file_id.write('{0}\n'.format(' --in={0} \\'.format(transcriptome_file)))
file_id.write('{0}\n'.format(' --out={0}'.format(os.path.basename(current_run_dir))))
file_id.write('{0}\n'.format(' RC=$?'))
file_id.write('{0}\n'.format(' if [ $RC -ne 0 ]; then manage_error run_BUSCO.py $RC; fi'))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('function end'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' END_DATETIME=`date --utc +%s`'))
file_id.write('{0}\n'.format(' FORMATTED_END_DATETIME=`date --date="@$END_DATETIME" "+%Y-%m-%d %H:%M:%S"`'))
file_id.write('{0}\n'.format(' calculate_duration'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' echo "Script ended OK at $FORMATTED_END_DATETIME UTC with a run duration of $DURATION s ($FORMATTED_DURATION)."'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' RECIPIENT={0}'.format(xconfiguration.get_contact_data())))
file_id.write('{0}\n'.format(' SUBJECT="{0}: {1} process"'.format(xlib.get_project_name(), xlib.get_busco_name())))
file_id.write('{0}\n'.format(' MESSAGE="The {0} process in node $HOSTNAME of cluster {1} ended OK at $FORMATTED_END_DATETIME with a run duration of $DURATION s ($FORMATTED_DURATION). Please review its log.<br/><br/>Regards,<br/>GI Genetica, Fisiologia e Historia Forestal<br/>Dpto. Sistemas y Recursos Naturales<br/>ETSI Montes, Forestal y del Medio Natural<br/>Universidad Politecnica de Madrid<br/>https://github.com/ggfhf/"'.format(xlib.get_busco_name(), cluster_name)))
file_id.write('{0}\n'.format(' mail --append "Content-type: text/html;" --subject "$SUBJECT" "$RECIPIENT" <<< "$MESSAGE"'))
file_id.write('{0}\n'.format(' exit 0'))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('function manage_error'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' END_DATETIME=`date --utc +%s`'))
file_id.write('{0}\n'.format(' FORMATTED_END_DATETIME=`date --date="@$END_DATETIME" "+%Y-%m-%d %H:%M:%S"`'))
file_id.write('{0}\n'.format(' calculate_duration'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' echo "ERROR: $1 returned error $2"'))
file_id.write('{0}\n'.format(' echo "Script ended WRONG at $FORMATTED_END_DATETIME UTC with a run duration of $DURATION s ($FORMATTED_DURATION)."'))
file_id.write('{0}\n'.format(' echo "$SEP"'))
file_id.write('{0}\n'.format(' RECIPIENT={0}'.format(xconfiguration.get_contact_data())))
file_id.write('{0}\n'.format(' SUBJECT="{0}: {1} process"'.format(xlib.get_project_name(), xlib.get_busco_name())))
file_id.write('{0}\n'.format(' MESSAGE="The {0} process in node $HOSTNAME of cluster {1} ended WRONG at $FORMATTED_END_DATETIME with a run duration of $DURATION s ($FORMATTED_DURATION). Please review its log.<br/><br/>Regards,<br/>GI Genetica, Fisiologia e Historia Forestal<br/>Dpto. Sistemas y Recursos Naturales<br/>ETSI Montes, Forestal y del Medio Natural<br/>Universidad Politecnica de Madrid<br/>https://github.com/ggfhf/"'.format(xlib.get_busco_name(), cluster_name)))
file_id.write('{0}\n'.format(' mail --append "Content-type: text/html;" --subject "$SUBJECT" "$RECIPIENT" <<< "$MESSAGE"'))
file_id.write('{0}\n'.format(' exit 3'))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('function calculate_duration'))
file_id.write('{0}\n'.format('{'))
file_id.write('{0}\n'.format(' DURATION=`expr $END_DATETIME - $INIT_DATETIME`'))
file_id.write('{0}\n'.format(' HH=`expr $DURATION / 3600`'))
file_id.write('{0}\n'.format(' MM=`expr $DURATION % 3600 / 60`'))
file_id.write('{0}\n'.format(' SS=`expr $DURATION % 60`'))
file_id.write('{0}\n'.format(' FORMATTED_DURATION=`printf "%03d:%02d:%02d\\n" $HH $MM $SS`'))
file_id.write('{0}\n'.format('}'))
file_id.write('{0}\n'.format('#-------------------------------------------------------------------------------'))
file_id.write('{0}\n'.format('init'))
file_id.write('{0}\n'.format('download_lineage_data'))
file_id.write('{0}\n'.format('run_busco_process'))
file_id.write('{0}\n'.format('end'))
except:
error_list.append('*** ERROR: The file {0} can not be created'.format(get_busco_process_script()))
OK = False
# return the control variable and the error list
return (OK, error_list)
def run_gzip_process(cluster_name, dataset_type, log, function=None):
'''
Run a gzip process.
'''
# initialize the control variable
OK = True
# get the gzip code and name
gzip_code = xlib.get_gzip_code()
gzip_name = xlib.get_gzip_name()
# get the gzip option dictionary
gzip_option_dict = xlib.get_option_dict(get_gzip_config_file(dataset_type))
# get the experiment identification
experiment_id = gzip_option_dict['identification']['experiment_id']
# get the gzip process script path in the local computer
gzip_process_script = get_gzip_process_script(dataset_type)
# get the gzip process starter path in the local computer
gzip_process_starter = get_gzip_process_starter(dataset_type)
# get the aplication directory in the cluster
cluster_app_dir = xlib.get_cluster_app_dir()
# warn that the log window must not be closed
if not isinstance(log, xlib.DevStdOut):
log.write(
'This process might take several minutes. Do not close this window, please wait!\n'
)
# validate the gzip config file
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Validating the {0} config file ...\n'.format(gzip_name))
(OK, error_list) = validate_gzip_config_file(dataset_type, strict=True)
if OK:
log.write('The config file is OK.\n')
else:
log.write('*** ERROR: The config file is not valid.\n')
log.write('Please correct this file or recreate the config files.\n')
# create the SSH client connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Connecting the SSH client ...\n')
(OK, error_list, ssh_client) = xssh.create_ssh_client_connection(
cluster_name, 'master')
if OK:
log.write('The SSH client is connected.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# create the SSH transport connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Connecting the SSH transport ...\n')
(OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(
cluster_name, 'master')
if OK:
log.write('The SSH transport is connected.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# create the SFTP client
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Connecting the SFTP client ...\n')
sftp_client = xssh.create_sftp_client(ssh_transport)
log.write('The SFTP client is connected.\n')
# warn that the requirements are being verified
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Verifying process requirements ...\n')
# verify the master is running
if OK:
(master_state_code,
master_state_name) = xec2.get_node_state(cluster_name, 'master')
if master_state_code != 16:
log.write(
'*** ERROR: The cluster {0} is not running. Its state is {1} ({2}).\n'
.format(cluster_name, master_state_code, master_state_name))
OK = False
# warn that the requirements are OK
if OK:
log.write('Process requirements are OK.\n')
# determine the run directory in the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Determining the run directory in the cluster ...\n')
if dataset_type == 'reference':
current_run_dir = xlib.get_cluster_current_run_dir(
'reference', gzip_code)
elif dataset_type == 'database':
current_run_dir = xlib.get_cluster_current_run_dir(
'database', gzip_code)
else:
current_run_dir = xlib.get_cluster_current_run_dir(
experiment_id, gzip_code)
command = 'mkdir --parents {0}'.format(current_run_dir)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The directory path is {0}.\n'.format(current_run_dir))
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# build the gzip process script
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Building the process script {0} ...\n'.format(
gzip_process_script))
(OK, error_list) = build_gzip_process_script(cluster_name,
dataset_type,
current_run_dir)
if OK:
log.write('The file is built.\n')
else:
log.write('*** ERROR: The file could not be built.\n')
# upload the gzip process script to the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write(
'Uploading the process script {0} to the directory {1} of the master ...\n'
.format(gzip_process_script, current_run_dir))
cluster_path = '{0}/{1}'.format(current_run_dir,
os.path.basename(gzip_process_script))
(OK, error_list) = xssh.put_file(sftp_client, gzip_process_script,
cluster_path)
if OK:
log.write('The file is uploaded.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# set run permision to the gzip process script in the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Setting on the run permision of {0}/{1} ...\n'.format(
current_run_dir, os.path.basename(gzip_process_script)))
command = 'chmod u+x {0}/{1}'.format(
current_run_dir, os.path.basename(gzip_process_script))
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The run permision is set.\n')
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# build the gzip process starter
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Building the process starter {0} ...\n'.format(
gzip_process_starter))
(OK, error_list) = build_gzip_process_starter(dataset_type,
current_run_dir)
if OK:
log.write('The file is built.\n')
else:
log.write('***ERROR: The file could not be built.\n')
# upload the gzip process starter to the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write(
'Uploading the process starter {0} to the directory {1} of the master ...\n'
.format(gzip_process_starter, current_run_dir))
cluster_path = '{0}/{1}'.format(current_run_dir,
os.path.basename(gzip_process_starter))
(OK, error_list) = xssh.put_file(sftp_client, gzip_process_starter,
cluster_path)
if OK:
log.write('The file is uploaded.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# set run permision to the gzip process starter in the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Setting on the run permision of {0}/{1} ...\n'.format(
current_run_dir, os.path.basename(gzip_process_starter)))
command = 'chmod u+x {0}/{1}'.format(
current_run_dir, os.path.basename(gzip_process_starter))
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The run permision is set.\n')
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# submit the gzip process
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Submitting the process script {0}/{1} ...\n'.format(
current_run_dir, os.path.basename(gzip_process_starter)))
sge_env = xcluster.get_sge_env()
command = '{0}; qsub -V -b n -cwd {1}/{2}'.format(
sge_env, current_run_dir, os.path.basename(gzip_process_starter))
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
for line in stdout:
log.write('{0}\n'.format(line))
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# close the SSH transport connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Closing the SSH transport connection ...\n')
xssh.close_ssh_transport_connection(ssh_transport)
log.write('The connection is closed.\n')
# close the SSH client connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Closing the SSH client connection ...\n')
xssh.close_ssh_client_connection(ssh_client)
log.write('The connection is closed.\n')
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut):
log.write('{0}\n'.format(xlib.get_separator()))
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK