def form_show_cluster_composing():
'''
Show cluster information of every node: OS, CPU number and memory.
'''
# initialize the control variable
OK = True
# print the header
clib.clear_screen()
clib.print_headers_with_environment('Cluster operation - Show cluster composing')
# get the cluster name
print(xlib.get_separator())
if xec2.get_running_cluster_list(volume_creator_included=False) == []:
print('WARNING: There is not any running cluster.')
OK = False
else:
cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True)
# show the status of batch jobs
if OK:
devstdout = xlib.DevStdOut(xcluster.show_cluster_composing.__name__)
xcluster.show_cluster_composing(cluster_name, devstdout, function=None)
# show continuation message
print(xlib.get_separator())
input('Press [Intro] to continue ...')
def form_restart_cluster():
'''
Restart a cluster.
'''
# initialize the control variable
OK = True
# print the header
clib.clear_screen()
clib.print_headers_with_environment('Cluster operation - Restart cluster')
# get the cluster name
print(xlib.get_separator())
if xec2.get_running_cluster_list(volume_creator_included=False) == []:
print('WARNING: There is not any running cluster.')
OK = False
else:
cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True)
# confirm the restarting of the cluster
if OK:
print(xlib.get_separator())
OK = clib.confirm_action('The cluster is going to be restarted.')
# stop the cluster
if OK:
devstdout = xlib.DevStdOut(xcluster.restart_cluster.__name__)
OK = xcluster.restart_cluster(cluster_name, devstdout, function=None)
# show continuation message
print(xlib.get_separator())
input('Press [Intro] to continue ...')
def form_open_terminal():
'''
Open a terminal windows of a cluster node.
'''
# initialize the control variable
OK = True
# print the header
clib.clear_screen()
clib.print_headers_with_environment('Cloud control - Open a terminal')
# get the cluster name and node name
print(xlib.get_separator())
if xec2.get_running_cluster_list(volume_creator_included=False) != []:
cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True)
node_name = cinputs.input_node_name(cluster_name, new=False, is_master_valid=True, help=True)
else:
print('WARNING: There is not any running cluster.')
OK = False
# confirm the terminal opening
if OK:
print(xlib.get_separator())
OK = clib.confirm_action('The terminal is going to be opened using StarCluster.')
# open de terminal
if OK:
xcluster.open_terminal(cluster_name, node_name)
# show continuation message
print(xlib.get_separator())
input('Press [Intro] to continue ...')
def list_clusters(log, function=None):
'''
List clusters.
'''
# initialize the control variable
OK = True
# get current region
region_name = xconfiguration.get_current_region_name()
# warn that the log window must not be closed
if not isinstance(log, xlib.DevStdOut):
log.write(
'This process might take a few minutes. Do not close this window, please wait!\n'
)
# warn that the requirements are being verified
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Verifying process requirements ...\n')
# verify if there are some running clusters
if xec2.get_running_cluster_list(volume_creator_included=True) == []:
log.write('WARNING: There is not any running cluster.\n')
OK = False
# warn that the requirements are OK
if OK:
log.write('Process requirements are OK.\n')
# list clusters
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Listing clusters using StarCluster ...\n')
log.write('\n')
command = '{0} --region={1} listclusters'.format(
xlib.get_starcluster(), region_name)
rc = xlib.run_command(command, log)
if rc != 0:
log.write('*** ERROR: Return code {0} in command -> {1}\n'.format(
rc, command))
OK = False
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut):
log.write('{0}\n'.format(xlib.get_separator()))
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK
def form_kill_batch_job():
'''
Kill a batch job in the cluster.
'''
# initialize the control variable
OK = True
# print the header
clib.clear_screen()
clib.print_headers_with_environment('Cluster operation - Kill batch job')
# get the cluster name, experiment identification, read dataset identification and the file pattern
print(xlib.get_separator())
if xec2.get_running_cluster_list(volume_creator_included=False) == []:
print('WARNING: There is not any running cluster.')
OK = False
else:
cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True)
# create the SSH client connection
if OK:
(OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master')
for error in error_list:
print(error)
# get the batch job identificaction
if OK:
batch_job_id = cinputs.input_batch_job_id(ssh_client, help=True)
if batch_job_id == '':
print('WARNING: There is not any batch job.')
OK = False
# confirm the kill of the batch job
if OK:
print(xlib.get_separator())
OK = clib.confirm_action('The batch job {0} is going to be killed.'.format(batch_job_id))
# kill the batch job
if OK:
devstdout = xlib.DevStdOut(xcluster.kill_batch_job.__name__)
xcluster.kill_batch_job(cluster_name, batch_job_id, devstdout, function=None)
# close the SSH client connection
if OK:
xssh.close_ssh_client_connection(ssh_client)
# show continuation message
print(xlib.get_separator())
input('Press [Intro] to continue ...')
def populate_combobox_cluster_name(self):
'''
Populate data in "combobox_cluster_name".
'''
# clear the value selected in the combobox
self.wrapper_cluster_name.set('')
# verify if there are some running clusters
running_cluster_list = xec2.get_running_cluster_list(volume_creator_included=False)
if running_cluster_list == []:
message = 'There is not any running cluster.'
tkinter.messagebox.showwarning('{0} - {1}'.format(xlib.get_project_name(), self.head), message)
return
# load the names of clusters which are running in the combobox
self.combobox_cluster_name['values'] = running_cluster_list
def form_mount_volume():
'''
Mount a volume in a node.
'''
# initialize the control variable
OK = True
# print the header
clib.clear_screen()
clib.print_headers_with_environment('Volume operation - Mount volume in a node')
# get current zone name
zone_name = xconfiguration.get_current_zone_name()
# get the cluster name and node name
print(xlib.get_separator())
if xec2.get_running_cluster_list(volume_creator_included=False) != []:
cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True)
node_name = cinputs.input_node_name(cluster_name, new=False, is_master_valid=True, help=True)
else:
print('WARNING: There is not any running cluster.')
OK = False
# get the volume name, AWS device file and directory path
if OK:
volume_name = cinputs.input_volume_name(zone_name, template_name='', help=True, help_type='created')
aws_device_file = cinputs.input_device_file(node_name, volume_name)
mounting_path = cinputs.input_mounting_path(node_name, aws_device_file)
# confirm the mounting of the volume
if OK:
print(xlib.get_separator())
OK = clib.confirm_action('The volume is going to be mounted.')
# mount the volume in the node
if OK:
devstdout = xlib.DevStdOut(xvolume.mount_volume.__name__)
xvolume.mount_volume(cluster_name, node_name, volume_name, aws_device_file, mounting_path, devstdout, function=None, is_menu_call=True)
# show continuation message
print(xlib.get_separator())
input('Press [Intro] to continue ...')
def form_terminate_cluster(force):
'''
Terminate a cluster.
'''
# initialize the control variable
OK = True
# print the header
clib.clear_screen()
if not force:
clib.print_headers_with_environment('Cluster operation - Terminate cluster')
else:
clib.print_headers_with_environment('Cluster operation - Force termination of a cluster')
# get the cluster name that must be terminated
print(xlib.get_separator())
if not force:
if xec2.get_running_cluster_list(volume_creator_included=False) == []:
print('WARNING: There is not any running cluster.')
OK = False
else:
cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True)
else:
cluster_name = cinputs.input_template_name(volume_creator_included=False, help=False, is_all_possible=False)
# confirm the termination of the cluster
if OK:
print(xlib.get_separator())
if not force:
OK = clib.confirm_action('The cluster is going to be terminated.')
else:
OK = clib.confirm_action('The cluster is going to be forced to terminate.')
# terminate the cluster
if OK:
devstdout = xlib.DevStdOut(xcluster.terminate_cluster.__name__)
OK = xcluster.terminate_cluster(cluster_name, force, devstdout, function=None)
# show continuation message
print(xlib.get_separator())
input('Press [Intro] to continue ...')
def form_add_node():
'''
Add a node in a cluster.
'''
# initialize the control variable
OK = True
# print the header
clib.clear_screen()
clib.print_headers_with_environment('Node operation - Add node in a cluster')
# get the cluster name and node name
print(xlib.get_separator())
if xec2.get_running_cluster_list(volume_creator_included=False) != []:
cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True)
if len(xec2.get_cluster_node_list(cluster_name)) >= xec2.get_max_node_number():
print('WARNING: The maximum number ({0}) of instances is already running.'.format(xec2.get_max_node_number()))
OK = False
else:
node_name = cinputs.input_node_name(cluster_name, new=True, is_master_valid=False, help=True)
else:
print('WARNING: There is not any running cluster.')
OK = False
# confirm the addition of the node in the cluster
if OK:
print(xlib.get_separator())
OK = clib.confirm_action('The node is going to be added.')
# add node in cluster
if OK:
devstdout = xlib.DevStdOut(xnode.add_node.__name__)
xnode.add_node(cluster_name, node_name, devstdout, function=None)
# show continuation message
print(xlib.get_separator())
input('Press [Intro] to continue ...')
def form_remove_node():
'''
Remove a node in a cluster.
'''
# initialize the control variable
OK = True
# print the header
clib.clear_screen()
clib.print_headers_with_environment('Node operation - Remove node in a cluster')
# get the cluster name and node name
print(xlib.get_separator())
if xec2.get_running_cluster_list(volume_creator_included=False) != []:
cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True)
node_name = cinputs.input_node_name(cluster_name, new=False, is_master_valid=False, help=True)
if node_name == []:
print('WARNING: There is not any running node besides the master.')
OK = False
else:
print('WARNING: There is not any running cluster.')
OK = False
# confirm the removal of the node in the cluster
if OK:
print(xlib.get_separator())
OK = clib.confirm_action('The node is going to be removed.')
# remove node
if OK:
devstdout = xlib.DevStdOut(xnode.remove_node.__name__)
xnode.remove_node(cluster_name, node_name, devstdout, function=None)
# show continuation message
print(xlib.get_separator())
input('Press [Intro] to continue ...')
def form_show_status_batch_jobs():
'''
Show the status of batch jobs in the cluster.
'''
# initialize the control variable
OK = True
# initialize the list of identification of the batch jobs
batch_job_id_list = []
# print the header
clib.clear_screen()
clib.print_headers_with_environment('Cluster operation - Show status batch jobs')
# get the cluster name
print(xlib.get_separator())
if xec2.get_running_cluster_list(volume_creator_included=False) != []:
cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True)
else:
print('WARNING: There is not any running cluster.')
OK = False
# create the SSH client connection
if OK:
(OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master')
for error in error_list:
print(error)
# get the batch job dictionary
if help:
(OK, error_list, batch_job_dict) = xcluster.get_batch_job_dict(ssh_client)
# build the list of identifications of the batch jobs
if OK and help:
for job_id in batch_job_dict.keys():
batch_job_id_list.append(job_id)
if batch_job_id_list != []:
batch_job_id_list.sort()
else:
print('WARNING: There is not any batch job.')
OK = False
# print the batch jobs
if OK and help:
print(xlib.get_separator())
# set data width
job_id_width = 6
job_name_width = 10
state_width = 15
start_date_width = 10
start_time_width = 10
# set line template
line_template = '{0:' + str(job_id_width) + '} {1:' + str(job_name_width) + '} {2:' + str(state_width) + '} {3:' + str(start_date_width) + '} {4:' + str(start_time_width) + '}'
# print header
print(line_template.format('Job id', 'Job name', 'State', 'Start date', 'Start time'))
print(line_template.format('=' * job_id_width, '=' * job_name_width, '=' * state_width, '=' * start_date_width, '=' * start_time_width))
# print detail lines
for job_id in batch_job_id_list:
job_name = batch_job_dict[job_id]['job_name']
state = batch_job_dict[job_id]['state']
start_date = batch_job_dict[job_id]['start_date']
start_time = batch_job_dict[job_id]['start_time']
print(line_template.format(job_id, job_name, state, start_date, start_time))
# close the SSH client connection
if OK:
xssh.close_ssh_client_connection(ssh_client)
# show continuation message
print(xlib.get_separator())
input('Press [Intro] to continue ...')
def list_clusters(log, function=None):
'''
List clusters.
'''
# initialize the control variable
OK = True
# warn that the log window does not have to be closed
if not isinstance(log, xlib.DevStdOut):
log.write(
'This process might take a few minutes. Do not close this window, please wait!\n'
)
# warn that the requirements are being verified
log.write(f'{xlib.get_separator()}\n')
log.write('Checking process requirements ...\n')
# check if there are some running clusters
if xec2.get_running_cluster_list(only_environment_cluster=False,
volume_creator_included=True) == []:
log.write('WARNING: There is not any running cluster.\n')
OK = False
# warn that the requirements are OK
if OK:
log.write('Process requirements are OK.\n')
# list clusters
if OK:
# get the running cluster list
running_cluster_list = xec2.get_running_cluster_list(
only_environment_cluster=False, volume_creator_included=True)
# for each cluster
for cluster_name in running_cluster_list:
# set the master name
if xec2.get_cluster_mode(
cluster_name) == xconfiguration.get_cluster_mode_native():
master_name = 'instance'
elif xec2.get_cluster_mode(
cluster_name
) == xconfiguration.get_cluster_mode_starcluster():
master_name = 'master'
# get the cluster data of the master or instance node
master_data = xec2.get_node_data_dict(cluster_name, master_name)
# get the dictionary of volumes attached to the master or instance node
volume_attached_to_node_dict = xec2.get_volume_attached_to_node_dict(
cluster_name, master_name)
# print the cluster data
log.write(f'{xlib.get_separator()}\n')
log.write('\n')
log.write(f'Cluster: {cluster_name}\n')
# -- log.write(f'========={"="*len(cluster_name)}\n')
log.write('\n')
log.write(
f' Cluster mode.....: {xec2.get_cluster_mode(cluster_name)}\n'
)
log.write(f' AMI id...........: {master_data["image_id"]}\n')
log.write(
f' Instance type....: {master_data["instance_type"]}\n')
log.write(
f' Security group...: {master_data["security_group_name"]}\n'
)
log.write('\n')
# print the master node data
log.write(f' Node: {master_name}\n')
log.write(
f' State...............: {master_data["state"]}\n')
log.write(
f' Launch time.........: {master_data["launch_time"]}\n'
)
log.write(
f' Public IP address...: {master_data["public_ip_address"]}\n'
)
log.write(
f' Public DNS name.....: {master_data["public_dns_name"]}\n'
)
log.write(' Attached volumes....:\n')
for volume_id in sorted(volume_attached_to_node_dict):
log.write(
f' Id: {volume_id} - Name: {volume_attached_to_node_dict[volume_id]["volume_name"]}\n'
)
log.write('\n')
# set the list of additional nodes to the master
cluster_node_list = xec2.get_cluster_node_list(cluster_name)
cluster_node_list.remove(master_name)
# for each additional node
if cluster_node_list is not None:
for node_name in cluster_node_list:
# get node data
node_data = xec2.get_node_data_dict(
cluster_name, node_name)
# get the dictionary of volumes attached to the node
volume_attached_to_node_dict = xec2.get_volume_attached_to_node_dict(
cluster_name, node_name)
# print the additional node data
log.write(f' Node: {node_name}\n')
log.write(
f' State...............: {node_data["state"]}\n'
)
log.write(
f' Launch time.........: {node_data["launch_time"]} UTC\n'
)
log.write(
f' Public IP address...: {node_data["public_ip_address"]}\n'
)
log.write(
f' Public DNS name.....: {node_data["public_dns_name"]}\n'
)
log.write(' Attached volumes....:\n')
for volume_id in sorted(volume_attached_to_node_dict):
log.write(
f' Id: {volume_id} - Name: {volume_attached_to_node_dict[volume_id]["volume_name"]}\n'
)
log.write('\n')
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut):
log.write(f'{xlib.get_separator()}\n')
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK
def form_list_cluster_experiment_processes():
'''
List the processes of an experiment in the cluster.
'''
# initialize the control variable
OK = True
# print the header
clib.clear_screen()
clib.print_headers_with_environment('Logs - List experiment processes in the cluster')
# get the cluster name
print(xlib.get_separator())
if xec2.get_running_cluster_list(volume_creator_included=False) != []:
cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True)
else:
print('WARNING: There is not any running cluster.')
OK = False
# create the SSH client connection
if OK:
(OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master')
for error in error_list:
log.write('{0}\n'.format(error))
# get experiment identification
if OK:
experiment_id = cinputs.input_experiment_id(ssh_client, help=True)
if experiment_id == '':
print('WARNING: The cluster {0} has not experiment data.'.format(cluster_name))
OK = False
# get the result dataset list of the experiment
if OK:
command = 'cd {0}/{1}; for list in `ls`; do ls -ld $list | grep -v ^- > /dev/null && echo $list; done;'.format(xlib.get_cluster_result_dir(), experiment_id)
(OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
result_dataset_id_list = []
for line in stdout:
line = line.rstrip('\n')
if line != 'lost+found':
result_dataset_id_list.append(line)
# print the result dataset identification list of the experiment
if OK:
print(xlib.get_separator())
if result_dataset_id_list == []:
print('*** WARNING: There is not any result dataset of the experiment {0}.'.format(experiment_id))
else:
result_dataset_id_list.sort()
# set data width
result_dataset_width = 25
bioinfo_app_width = 25
# set line template
line_template = '{0:' + str(result_dataset_width) + '} {1:' + str(bioinfo_app_width) + '}'
# print header
print(line_template.format('Result dataset', 'Bioinfo app / Utility'))
print(line_template.format('=' * result_dataset_width, '=' * bioinfo_app_width))
# print detail lines
for result_dataset_id in result_dataset_id_list:
if result_dataset_id.startswith(xlib.get_bedtools_code()+'-'):
bioinfo_app_name = xlib.get_bedtools_name()
elif result_dataset_id.startswith(xlib.get_blastplus_code()+'-'):
bioinfo_app_name = xlib.get_blastplus_name()
elif result_dataset_id.startswith(xlib.get_bowtie2_code()+'-'):
bioinfo_app_name = xlib.get_bowtie2_name()
elif result_dataset_id.startswith(xlib.get_busco_code()+'-'):
bioinfo_app_name = xlib.get_busco_name()
elif result_dataset_id.startswith(xlib.get_cd_hit_code()+'-'):
bioinfo_app_name = xlib.get_cd_hit_est_name()
elif result_dataset_id.startswith(xlib.get_cd_hit_code()+'-'):
bioinfo_app_name = xlib.get_cd_hit_est_name()
elif result_dataset_id.startswith(xlib.get_detonate_code()+'-'):
bioinfo_app_name = xlib.get_detonate_name()
elif result_dataset_id.startswith(xlib.get_emboss_code()+'-'):
bioinfo_app_name = xlib.get_emboss_name()
elif result_dataset_id.startswith(xlib.get_fastqc_code()+'-'):
bioinfo_app_name = xlib.get_fastqc_name()
elif result_dataset_id.startswith(xlib.get_gmap_code()+'-'):
bioinfo_app_name = xlib.get_gmap_name()
elif result_dataset_id.startswith(xlib.get_gmap_gsnap_code()+'-'):
bioinfo_app_name = xlib.get_gmap_gsnap_name()
elif result_dataset_id.startswith(xlib.get_gzip_code()+'-'):
bioinfo_app_name = xlib.get_gzip_name()
elif result_dataset_id.startswith(xlib.get_insilico_read_normalization_code()+'-'):
bioinfo_app_name = xlib.get_insilico_read_normalization_name()
elif result_dataset_id.startswith(xlib.get_miniconda3_code()+'-'):
bioinfo_app_name = xlib.get_miniconda3_name()
elif result_dataset_id.startswith(xlib.get_ngshelper_code()+'-'):
bioinfo_app_name = xlib.get_ngshelper_name()
elif result_dataset_id.startswith(xlib.get_quast_code()+'-'):
bioinfo_app_name = xlib.get_quast_name()
elif result_dataset_id.startswith(xlib.get_r_code()+'-'):
bioinfo_app_name = xlib.get_r_name()
elif result_dataset_id.startswith(xlib.get_ref_eval_code()+'-'):
bioinfo_app_name = xlib.get_ref_eval_name()
elif result_dataset_id.startswith(xlib.get_rnaquast_code()+'-'):
bioinfo_app_name = xlib.get_rnaquast_name()
elif result_dataset_id.startswith(xlib.get_rsem_code()+'-'):
bioinfo_app_name = xlib.get_rsem_name()
elif result_dataset_id.startswith(xlib.get_rsem_eval_code()+'-'):
bioinfo_app_name = xlib.get_rsem_eval_name()
elif result_dataset_id.startswith(xlib.get_samtools_code()+'-'):
bioinfo_app_name = xlib.get_samtools_name()
elif result_dataset_id.startswith(xlib.get_soapdenovotrans_code()+'-'):
bioinfo_app_name = xlib.get_soapdenovotrans_name()
elif result_dataset_id.startswith(xlib.get_star_code()+'-'):
bioinfo_app_name = xlib.get_star_name()
elif result_dataset_id.startswith(xlib.get_transabyss_code()+'-'):
bioinfo_app_name = xlib.get_transabyss_name()
elif result_dataset_id.startswith(xlib.get_transcript_filter_code()+'-'):
bioinfo_app_name = xlib.get_transcript_filter_name()
elif result_dataset_id.startswith(xlib.get_transcriptome_blastx_code()+'-'):
bioinfo_app_name = xlib.get_transcriptome_blastx_name()
elif result_dataset_id.startswith(xlib.get_transrate_code()+'-'):
bioinfo_app_name = xlib.get_transrate_name()
elif result_dataset_id.startswith(xlib.get_trimmomatic_code()+'-'):
bioinfo_app_name = xlib.get_trimmomatic_name()
elif result_dataset_id.startswith(xlib.get_trinity_code()+'-'):
bioinfo_app_name = xlib.get_trinity_name()
else:
bioinfo_app_name = 'xxx'
print(line_template.format(result_dataset_id, bioinfo_app_name))
# close the SSH client connection
if OK:
xssh.close_ssh_client_connection(ssh_client)
# show continuation message
print(xlib.get_separator())
input('Press [Intro] to continue ...')
def form_view_cluster_experiment_process_log():
'''
View the log of an experiment process in the cluster.
'''
# initialize the control variable
OK = True
# print the header
clib.clear_screen()
clib.print_headers_with_environment('Logs - View an experiment process log in the cluster')
# get the clustner name
if OK:
print(xlib.get_separator())
if xec2.get_running_cluster_list(volume_creator_included=False) != []:
cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True)
else:
print('WARNING: There is not any running cluster.')
OK = False
# create the SSH client connection
if OK:
(OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master')
for error in error_list:
log.write('{0}\n'.format(error))
# create the SSH transport connection
if OK:
(OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name, 'master')
for error in error_list:
log.write('{0}\n'.format(error))
# create the SFTP client
if OK:
sftp_client = xssh.create_sftp_client(ssh_transport)
# get the experiment identification
if OK:
experiment_id = cinputs.input_experiment_id(ssh_client, help=True)
if experiment_id == '':
print('WARNING: The cluster has not experiment data.')
OK = False
# get the result_dataset identification
if OK:
result_dataset_id = cinputs.input_result_dataset_id('uncompressed', ssh_client, experiment_id, help=True)
if result_dataset_id == '':
print('WARNING: The experiment {0} has not result datasets.'.format(experiment_id))
OK = False
# create the local path
if not os.path.exists(xlib.get_temp_dir()):
os.makedirs(xlib.get_temp_dir())
# get the log file name and build local and cluster paths
if OK:
log_file = xlib.get_cluster_log_file()
local_path = '{0}/{1}'.format(xlib.get_temp_dir(), log_file)
cluster_path = '{0}/{1}/{2}'.format(xlib.get_cluster_experiment_result_dir(experiment_id), result_dataset_id, log_file)
# download the log file from the cluster
if OK:
print(xlib.get_separator())
print('The file {0} is being downloaded from {1} ...'.format(log_file, cluster_path))
OK = xssh.get_file(sftp_client, cluster_path, local_path)
if OK:
print('The file has been uploaded.')
# close the SSH transport connection
if OK:
xssh.close_ssh_transport_connection(ssh_transport)
# close the SSH client connection
if OK:
xssh.close_ssh_client_connection(ssh_client)
# view the log file
if OK:
text = 'Logs - View an experiment process log in the cluster'
OK = clib.view_file(local_path, text)
# show continuation message
print(xlib.get_separator())
input('Press [Intro] to continue ...')
def form_view_cluster_start_log():
'''
View the cluster start log.
'''
# initialize the control variable
OK = True
# print the header
clib.clear_screen()
clib.print_headers_with_environment('Logs - View the cluster start log')
# get the clustner name
if OK:
print(xlib.get_separator())
if xec2.get_running_cluster_list(only_environment_cluster=True,
volume_creator_included=False) != []:
cluster_name = cinputs.input_cluster_name(
volume_creator_included=False, help=True)
else:
print('WARNING: There is not any running cluster.')
OK = False
# create the SSH client connection
if OK:
(OK, error_list,
ssh_client) = xssh.create_ssh_client_connection(cluster_name)
for error in error_list:
print(error)
# create the SSH transport connection
if OK:
(OK, error_list,
ssh_transport) = xssh.create_ssh_transport_connection(cluster_name)
for error in error_list:
print(error)
# create the SFTP client
if OK:
sftp_client = xssh.create_sftp_client(ssh_transport)
# create the local path
if not os.path.exists(xlib.get_temp_dir()):
os.makedirs(xlib.get_temp_dir())
# get the log file name and build local and cluster paths
if OK:
local_path = f'{xlib.get_temp_dir()}/{os.path.basename(xinstance.get_infrastructure_software_installation_log())}'
cluster_path = f'/home/ubuntu/{os.path.basename(xinstance.get_infrastructure_software_installation_log())}'
# download the log file from the cluster
if OK:
print(xlib.get_separator())
print(
f'The file {os.path.basename(xinstance.get_infrastructure_software_installation_log())} is being downloaded from {cluster_path} ...'
)
OK = xssh.get_file(sftp_client, cluster_path, local_path)
if OK:
print('The file has been uploaded.')
# close the SSH transport connection
if OK:
xssh.close_ssh_transport_connection(ssh_transport)
# close the SSH client connection
if OK:
xssh.close_ssh_client_connection(ssh_client)
# view the log file
if OK:
text = 'Logs - View the cluster start log'
OK = clib.view_file(local_path, text)
# show continuation message
input('Press [Intro] to continue ...')
def form_list_cluster_experiment_processes():
'''
List the processes of an experiment in the cluster.
'''
# initialize the control variable
OK = True
# print the header
clib.clear_screen()
clib.print_headers_with_environment(
'Logs - List experiment processes in the cluster')
# get the cluster name
print(xlib.get_separator())
if xec2.get_running_cluster_list(only_environment_cluster=True,
volume_creator_included=False) != []:
cluster_name = cinputs.input_cluster_name(
volume_creator_included=False, help=True)
else:
print('WARNING: There is not any running cluster.')
OK = False
# create the SSH client connection
if OK:
(OK, error_list,
ssh_client) = xssh.create_ssh_client_connection(cluster_name)
for error in error_list:
print(error)
# get experiment identification
if OK:
experiment_id = cinputs.input_experiment_id(ssh_client, help=True)
if experiment_id == '':
print(
f'WARNING: The cluster {cluster_name} does not have experiment data.'
)
OK = False
# get the result dataset list of the experiment
if OK:
command = f'cd {xlib.get_cluster_result_dir()}/{experiment_id}; for list in `ls`; do ls -ld $list | grep -v ^- > /dev/null && echo $list; done;'
(OK, stdout, _) = xssh.execute_cluster_command(ssh_client, command)
if OK:
result_dataset_id_list = []
for line in stdout:
line = line.rstrip('\n')
if line != 'lost+found':
result_dataset_id_list.append(line)
# print the result dataset identification list of the experiment
if OK:
print(xlib.get_separator())
if result_dataset_id_list == []:
print(
f'*** WARNING: There is not any result dataset of the experiment {experiment_id}.'
)
else:
result_dataset_id_list.sort()
# set data width
result_dataset_width = 30
bioinfo_app_width = 25
# set line
line = '{0:' + str(result_dataset_width) + '} {1:' + str(
bioinfo_app_width) + '}'
# print header
print(line.format('Result dataset', 'Bioinfo app / Utility'))
print(
line.format('=' * result_dataset_width,
'=' * bioinfo_app_width))
# print detail lines
for result_dataset_id in result_dataset_id_list:
if result_dataset_id.startswith(xlib.get_bedtools_code() +
'-'):
bioinfo_app_name = xlib.get_bedtools_name()
elif result_dataset_id.startswith(xlib.get_blastplus_code() +
'-'):
bioinfo_app_name = xlib.get_blastplus_name()
elif result_dataset_id.startswith(xlib.get_bcftools_code() +
'-'):
bioinfo_app_name = xlib.get_bcftools_name()
elif result_dataset_id.startswith(xlib.get_bowtie2_code() +
'-'):
bioinfo_app_name = xlib.get_bowtie2_name()
elif result_dataset_id.startswith(xlib.get_busco_code() + '-'):
bioinfo_app_name = xlib.get_busco_name()
elif result_dataset_id.startswith(xlib.get_cd_hit_code() +
'-'):
bioinfo_app_name = xlib.get_cd_hit_name()
elif result_dataset_id.startswith(xlib.get_cd_hit_est_code() +
'-'):
bioinfo_app_name = xlib.get_cd_hit_est_name()
elif result_dataset_id.startswith(xlib.get_cuffdiff_code() +
'-'):
bioinfo_app_name = xlib.get_cuffdiff_name()
elif result_dataset_id.startswith(xlib.get_cufflinks_code() +
'-'):
bioinfo_app_name = xlib.get_cufflinks_name()
elif result_dataset_id.startswith(
xlib.get_cufflinks_cuffmerge_code() + '-'):
bioinfo_app_name = xlib.get_cufflinks_cuffmerge_name()
elif result_dataset_id.startswith(xlib.get_cuffnorm_code() +
'-'):
bioinfo_app_name = xlib.get_cuffnorm_name()
elif result_dataset_id.startswith(xlib.get_cuffquant_code() +
'-'):
bioinfo_app_name = xlib.get_cuffquant_name()
elif result_dataset_id.startswith(xlib.get_cutadapt_code() +
'-'):
bioinfo_app_name = xlib.get_cutadapt_name()
elif result_dataset_id.startswith(
xlib.get_ddradseq_simulation_code() + '-'):
bioinfo_app_name = xlib.get_ddradseq_simulation_name()
elif result_dataset_id.startswith(
xlib.get_ddradseqtools_code() + '-'):
bioinfo_app_name = xlib.get_ddradseqtools_name()
elif result_dataset_id.startswith(xlib.get_detonate_code() +
'-'):
bioinfo_app_name = xlib.get_detonate_name()
elif result_dataset_id.startswith(xlib.get_diamond_code() +
'-'):
bioinfo_app_name = xlib.get_diamond_name()
elif result_dataset_id.startswith(xlib.get_emboss_code() +
'-'):
bioinfo_app_name = xlib.get_emboss_name()
elif result_dataset_id.startswith(
xlib.get_entrez_direct_code() + '-'):
bioinfo_app_name = xlib.get_entrez_direct_name()
elif result_dataset_id.startswith(xlib.get_express_code() +
'-'):
bioinfo_app_name = xlib.get_express_name()
elif result_dataset_id.startswith(xlib.get_fastqc_code() +
'-'):
bioinfo_app_name = xlib.get_fastqc_name()
elif result_dataset_id.startswith(xlib.get_ggtrinity_code() +
'-'):
bioinfo_app_name = xlib.get_ggtrinity_name()
elif result_dataset_id.startswith(xlib.get_gmap_gsnap_code() +
'-'):
bioinfo_app_name = xlib.get_gmap_gsnap_name()
elif result_dataset_id.startswith(xlib.get_gmap_code() + '-'):
bioinfo_app_name = xlib.get_gmap_name()
elif result_dataset_id.startswith(xlib.get_gsnap_code() + '-'):
bioinfo_app_name = xlib.get_gsnap_name()
elif result_dataset_id.startswith(xlib.get_gzip_code() + '-'):
bioinfo_app_name = xlib.get_gzip_name()
elif result_dataset_id.startswith(xlib.get_hisat2_code() +
'-'):
bioinfo_app_name = xlib.get_hisat2_name()
elif result_dataset_id.startswith(xlib.get_htseq_code() + '-'):
bioinfo_app_name = xlib.get_htseq_name()
elif result_dataset_id.startswith(xlib.get_htseq_count_code() +
'-'):
bioinfo_app_name = xlib.get_htseq_count_name()
elif result_dataset_id.startswith(
xlib.get_insilico_read_normalization_code() + '-'):
bioinfo_app_name = xlib.get_insilico_read_normalization_name(
)
elif result_dataset_id.startswith(xlib.get_ipyrad_code() +
'-'):
bioinfo_app_name = xlib.get_ipyrad_name()
elif result_dataset_id.startswith(xlib.get_kallisto_code() +
'-'):
bioinfo_app_name = xlib.get_kallisto_name()
elif result_dataset_id.startswith(xlib.get_miniconda3_code() +
'-'):
bioinfo_app_name = xlib.get_miniconda3_name()
elif result_dataset_id.startswith(xlib.get_ngshelper_code() +
'-'):
bioinfo_app_name = xlib.get_ngshelper_name()
elif result_dataset_id.startswith(xlib.get_quast_code() + '-'):
bioinfo_app_name = xlib.get_quast_name()
elif result_dataset_id.startswith(xlib.get_r_code() + '-'):
bioinfo_app_name = xlib.get_r_name()
elif result_dataset_id.startswith(xlib.get_raddesigner_code() +
'-'):
bioinfo_app_name = xlib.get_raddesigner_name()
elif result_dataset_id.startswith(xlib.get_ref_eval_code() +
'-'):
bioinfo_app_name = xlib.get_ref_eval_name()
elif result_dataset_id.startswith(xlib.get_rnaquast_code() +
'-'):
bioinfo_app_name = xlib.get_rnaquast_name()
elif result_dataset_id.startswith(xlib.get_rsem_code() + '-'):
bioinfo_app_name = xlib.get_rsem_name()
elif result_dataset_id.startswith(xlib.get_rsem_eval_code() +
'-'):
bioinfo_app_name = xlib.get_rsem_eval_name()
elif result_dataset_id.startswith(xlib.get_rsitesearch_code() +
'-'):
bioinfo_app_name = xlib.get_rsitesearch_name()
elif result_dataset_id.startswith(xlib.get_samtools_code() +
'-'):
bioinfo_app_name = xlib.get_samtools_name()
elif result_dataset_id.startswith(xlib.get_soapdenovo2_code() +
'-'):
bioinfo_app_name = xlib.get_soapdenovo2_name()
elif result_dataset_id.startswith(
xlib.get_soapdenovotrans_code() + '-'):
bioinfo_app_name = xlib.get_soapdenovotrans_name()
elif result_dataset_id.startswith(xlib.get_star_code() + '-'):
bioinfo_app_name = xlib.get_star_name()
elif result_dataset_id.startswith(xlib.get_starcode_code() +
'-'):
bioinfo_app_name = xlib.get_starcode_name()
elif result_dataset_id.startswith(xlib.get_toa_code() + '-'):
bioinfo_app_name = xlib.get_toa_name()
elif result_dataset_id.startswith(
xlib.get_toa_process_download_basic_data_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_download_basic_data_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_download_dicots_04_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_download_dicots_04_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_download_gene_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_download_gene_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_download_go_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_download_go_name()
elif result_dataset_id.startswith(
xlib.get_toa_process_download_gymno_01_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_download_gymno_01_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_download_interpro_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_download_interpro_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_download_monocots_04_code() +
'-'):
bioinfo_app_name = xlib.get_toa_process_download_monocots_04_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_download_taxonomy_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_download_taxonomy_name(
)
elif result_dataset_id.startswith(
xlib.
get_toa_process_gilist_viridiplantae_nucleotide_gi_code(
) + '-'):
bioinfo_app_name = xlib.get_toa_process_gilist_viridiplantae_nucleotide_gi_name(
)
elif result_dataset_id.startswith(
xlib.
get_toa_process_gilist_viridiplantae_protein_gi_code()
+ '-'):
bioinfo_app_name = xlib.get_toa_process_gilist_viridiplantae_protein_gi_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_load_basic_data_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_load_basic_data_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_load_dicots_04_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_load_dicots_04_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_load_gene_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_load_gene_name()
elif result_dataset_id.startswith(
xlib.get_toa_process_load_go_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_load_go_name()
elif result_dataset_id.startswith(
xlib.get_toa_process_load_gymno_01_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_load_gymno_01_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_load_interpro_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_load_interpro_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_load_monocots_04_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_load_monocots_04_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_merge_annotations_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_merge_annotations_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_nr_blastplus_db_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_nr_blastplus_db_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_nr_diamond_db_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_nr_diamond_db_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_nt_blastplus_db_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_nt_blastplus_db_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_pipeline_aminoacid_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_pipeline_aminoacid_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_pipeline_nucleotide_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_pipeline_nucleotide_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_proteome_dicots_04_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_proteome_dicots_04_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_proteome_gymno_01_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_proteome_gymno_01_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_proteome_monocots_04_code() +
'-'):
bioinfo_app_name = xlib.get_toa_process_proteome_monocots_04_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_proteome_refseq_plant_code() +
'-'):
bioinfo_app_name = xlib.get_toa_process_proteome_refseq_plant_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_rebuild_toa_database_code() +
'-'):
bioinfo_app_name = xlib.get_get_toa_process_rebuild_toa_database_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_recreate_toa_database_code() +
'-'):
bioinfo_app_name = xlib.get_get_toa_process_recreate_toa_database_name(
)
elif result_dataset_id.startswith(xlib.get_tophat_code() +
'-'):
bioinfo_app_name = xlib.get_tophat_name()
elif result_dataset_id.startswith(xlib.get_transabyss_code() +
'-'):
bioinfo_app_name = xlib.get_transabyss_name()
elif result_dataset_id.startswith(
xlib.get_transcript_filter_code() + '-'):
bioinfo_app_name = xlib.get_transcript_filter_name()
elif result_dataset_id.startswith(
xlib.get_transcriptome_blastx_code() + '-'):
bioinfo_app_name = xlib.get_transcriptome_blastx_name()
elif result_dataset_id.startswith(
xlib.get_transdecoder_code() + '-'):
bioinfo_app_name = xlib.get_transdecoder_name()
elif result_dataset_id.startswith(xlib.get_transrate_code() +
'-'):
bioinfo_app_name = xlib.get_transrate_name()
elif result_dataset_id.startswith(xlib.get_trimmomatic_code() +
'-'):
bioinfo_app_name = xlib.get_trimmomatic_name()
elif result_dataset_id.startswith(xlib.get_trinity_code() +
'-'):
bioinfo_app_name = xlib.get_trinity_name()
elif result_dataset_id.startswith(
xlib.get_variant_calling_code() + '-'):
bioinfo_app_name = xlib.get_variant_calling_name()
elif result_dataset_id.startswith(xlib.get_vcftools_code() +
'-'):
bioinfo_app_name = xlib.get_vcftools_name()
elif result_dataset_id.startswith(
xlib.get_vcftools_perl_libraries_code() + '-'):
bioinfo_app_name = xlib.get_vcftools_perl_libraries_name()
elif result_dataset_id.startswith(xlib.get_vsearch_code() +
'-'):
bioinfo_app_name = xlib.get_vsearch_name()
else:
bioinfo_app_name = 'xxx'
print(line.format(result_dataset_id, bioinfo_app_name))
# close the SSH client connection
if OK:
xssh.close_ssh_client_connection(ssh_client)
# show continuation message
print(xlib.get_separator())
input('Press [Intro] to continue ...')