def kill_batch_job(cluster_name, job_id, log, function=None):
'''
Kill a batch job in the cluster.
'''
# initialize the control variable
OK = True
# create the SSH client connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Establishing connection with cluster {0} ...\n'.format(
cluster_name))
(OK, error_list,
ssh_client) = xssh.create_ssh_client_connection(cluster_name,
node_name='master')
if OK:
log.write('The connection is established.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# kill the job in the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
sge_env = get_sge_env()
command = '{0}; qdel {1}'.format(sge_env, job_id)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
if len(stdout) > 0:
for line in stdout:
log.write('{0}\n'.format(line))
else:
log.write('The job is been killed.\n')
if len(stderr) > 0:
for line in stderr:
log.write('{0}\n'.format(line))
else:
log.write('*** ERROR: Wrong command ---> {0}.\n'.format(command))
# close the SSH client connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
'Closing connection with cluster {0} ...\n'.format(cluster_name))
xssh.close_ssh_client_connection(ssh_client)
log.write('The connection is closed.\n')
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut):
log.write(f'{xlib.get_separator()}\n')
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK
def combobox_cluster_name_selected_item(self, event=None):
'''
Process the event when an item of "combobox_cluster_name" has been selected
'''
# set cursor to show busy status
self.main.config(cursor='watch')
self.main.update()
# verify if the cluster name selected is different to the previous cluster name
if self.wrapper_cluster_name.get() != self.cluster_name_ant:
# close SSH client connection
if self.cluster_name_ant is not None:
xssh.close_ssh_client_connection(self.ssh_client)
# create the SSH client connection
(OK, error_list, self.ssh_client) = xssh.create_ssh_client_connection(self.wrapper_cluster_name.get(), 'master')
if not OK:
message = ''
for error in error_list:
message = '{0}{1}\n'.format(message, error)
tkinter.messagebox.showerror('{0} - {1}'.format(xlib.get_project_name(), self.head), message)
self.close()
# save current cluster name as previous cluster name
self.cluster_name_ant = self.wrapper_cluster_name.get()
# load data in "combobox_experiment_id"
self.populate_combobox_experiment_id()
# set cursor to show normal status
self.main.config(cursor='')
self.main.update()
def get_database_file_name_list(cluster_name,
database_dataset_id,
file_type,
passed_connection=False,
ssh_client=None):
'''
Get a list of the database file names in a database dataset of the cluster.
'''
# initialize the control variable and the error list
OK = True
error_list = []
# initialize the dictionary of the database datasets
database_file_name_list = []
# create the SSH client connection
if not passed_connection:
(OK, error_list,
ssh_client) = xssh.create_ssh_client_connection(cluster_name)
# check the app directory is created
if OK:
command = '[ -d {0} ] && echo RC=0 || echo RC=1'.format(
xlib.get_cluster_database_dir())
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if stdout[len(stdout) - 1] != 'RC=0':
error_list.append(
'*** ERROR: There is not any volume mounted in the database directory.\n'
)
error_list.append(
'You have to link a volume in the mounting point {0} for the cluster {1}.\n'
.format(xlib.get_cluster_database_dir(), cluster_name))
OK = False
# build the list of the database file name of the database dataset
if OK:
if file_type == 'all':
command = 'ls {0}'.format(
xlib.get_cluster_database_dataset_dir(database_dataset_id))
else:
command = 'ls {0}/*{1}'.format(
xlib.get_cluster_database_dataset_dir(database_dataset_id),
file_type)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
for line in stdout:
line = line.rstrip('\n')
if line != 'lost+found':
database_file_name_list.append(os.path.basename(line))
# close the SSH client connection
if OK and not passed_connection:
xssh.close_ssh_client_connection(ssh_client)
# return the control variable, error list and list of the database file names
return (OK, error_list, database_file_name_list)
def get_reference_dataset_dict(cluster_name,
passed_connection=False,
ssh_client=None):
'''
Get a dictionary with the reference datasets in the cluster.
'''
# initialize the control variable and the error list
OK = True
error_list = []
# get the reference directory in the cluster
cluster_reference_dir = xlib.get_cluster_reference_dir()
# initialize the dictionary of the reference datasets
reference_dataset_dict = {}
# create the SSH client connection
if not passed_connection:
(OK, error_list,
ssh_client) = xssh.create_ssh_client_connection(cluster_name)
# check the app directory is created
if OK:
command = '[ -d {0} ] && echo RC=0 || echo RC=1'.format(
cluster_reference_dir)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if stdout[len(stdout) - 1] != 'RC=0':
error_list.append(
'*** ERROR: There is not any volume mounted in the reference directory.\n'
)
error_list.append(
'You have to link a volume in the mounting point {0} for the cluster {1}.\n'
.format(cluster_reference_dir, cluster_name))
OK = False
# build the dictionary of the reference datasets
if OK:
command = 'ls {0}'.format(cluster_reference_dir)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
for line in stdout:
line = line.rstrip('\n')
if line != 'lost+found':
reference_dataset_id = line
reference_dataset_name = reference_dataset_id
reference_dataset_dict[reference_dataset_id] = {
'reference_dataset_id': reference_dataset_id,
'reference_dataset_name': reference_dataset_name
}
# close the SSH client connection
if OK and not passed_connection:
xssh.close_ssh_client_connection(ssh_client)
# return the control variable, error list and dictionary of the reference datasets
return (OK, error_list, reference_dataset_dict)
def get_reference_file_name_list(cluster_name,
reference_dataset_id,
passed_connection=False,
ssh_client=None):
'''
Get a list of the reference file names in a reference dataset of the cluster.
'''
# initialize the control variable and the error list
OK = True
error_list = []
# get the reference directory in the cluster
cluster_reference_dir = xlib.get_cluster_reference_dir()
# initialize the dictionary of the reference datasets
reference_file_name_list = []
# create the SSH client connection
if not passed_connection:
(OK, error_list,
ssh_client) = xssh.create_ssh_client_connection(cluster_name)
# check the reference directory is created
if OK:
command = '[ -d {0} ] && echo RC=0 || echo RC=1'.format(
cluster_reference_dir)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if stdout[len(stdout) - 1] != 'RC=0':
error_list.append(
'*** ERROR: There is not any volume mounted in the reference directory.\n'
)
error_list.append(
'You have to link a volume in the mounting point {0} for the cluster {1}.\n'
.format(cluster_reference_dir, cluster_name))
OK = False
# get the reference dataset directory
reference_dataset_dir = xlib.get_cluster_reference_dataset_dir(
reference_dataset_id)
# build the list of the reference file name of the reference dataset
if OK:
command = 'find {0} -maxdepth 1 -type f'.format(reference_dataset_dir)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
for line in stdout:
line = line.rstrip('\n')
if line != 'lost+found':
reference_file_name_list.append(os.path.basename(line))
# close the SSH client connection
if OK and not passed_connection:
xssh.close_ssh_client_connection(ssh_client)
# return the control variable, error list and list of the reference file names
return (OK, error_list, reference_file_name_list)
def show_cluster_composition(cluster_name, log, function=None):
'''
Show cluster information of every node: OS, CPU number and memory.
'''
# initialize the control variable
OK = True
# create the SSH client connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Establishing connection with cluster {0} ...\n'.format(
cluster_name))
(OK, error_list,
ssh_client) = xssh.create_ssh_client_connection(cluster_name,
node_name='master')
if OK:
log.write('The connection is established.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# show the cluster composition
if OK:
log.write(f'{xlib.get_separator()}\n')
sge_env = get_sge_env()
command = '{0}; qhost'.format(sge_env)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
if len(stdout) > 0:
for line in stdout:
log.write('{0}\n'.format(line))
else:
log.write('Cluster composition not found.\n')
else:
log.write('*** ERROR: Wrong command ---> {0}.\n'.format(command))
# close the SSH client connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
'Closing connection with cluster {0} ...\n'.format(cluster_name))
xssh.close_ssh_client_connection(ssh_client)
log.write('The connection is closed.\n')
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut):
log.write(f'{xlib.get_separator()}\n')
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK
def close(self, event=None):
'''
Close "FormViewResultLogs".
'''
# close SSH client connection
if self.cluster_name_ant is not None:
xssh.close_ssh_client_connection(self.ssh_client)
# clear the label of the current process name
self.main.label_process['text'] = ''
# close the current form
self.main.close_current_form()
def form_kill_batch_job():
'''
Kill a batch job in the cluster.
'''
# initialize the control variable
OK = True
# print the header
clib.clear_screen()
clib.print_headers_with_environment('Cluster operation - Kill batch job')
# get the cluster name, experiment identification, read dataset identification and the file pattern
print(xlib.get_separator())
if xec2.get_running_cluster_list(volume_creator_included=False) == []:
print('WARNING: There is not any running cluster.')
OK = False
else:
cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True)
# create the SSH client connection
if OK:
(OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master')
for error in error_list:
print(error)
# get the batch job identificaction
if OK:
batch_job_id = cinputs.input_batch_job_id(ssh_client, help=True)
if batch_job_id == '':
print('WARNING: There is not any batch job.')
OK = False
# confirm the kill of the batch job
if OK:
print(xlib.get_separator())
OK = clib.confirm_action('The batch job {0} is going to be killed.'.format(batch_job_id))
# kill the batch job
if OK:
devstdout = xlib.DevStdOut(xcluster.kill_batch_job.__name__)
xcluster.kill_batch_job(cluster_name, batch_job_id, devstdout, function=None)
# close the SSH client connection
if OK:
xssh.close_ssh_client_connection(ssh_client)
# show continuation message
print(xlib.get_separator())
input('Press [Intro] to continue ...')
def upload_database_dataset(cluster_name, log, function=None):
'''
Upload the database dataset to the cluster.
'''
# initialize the control variable
OK = True
# warn that the log window does not have to be closed
if not isinstance(log, xlib.DevStdOut):
log.write(
'This process might take several minutes. Do not close this window, please wait!\n'
)
# check the database transfer config file
log.write(f'{xlib.get_separator()}\n')
log.write('Checking the database transfer config file ...\n')
if check_database_transfer_config_file(strict=True):
log.write('The file is OK.\n')
else:
log.write(
'*** ERROR: The database transfer config file is not valid.\n')
log.write('Please correct this file or recreate the config files.\n')
OK = False
# create the SSH client connection
if OK:
(OK, error_list,
ssh_client) = xssh.create_ssh_client_connection(cluster_name)
for error in error_list:
log.write(f'{error}\n')
# create the SSH transport connection
if OK:
(OK, error_list,
ssh_transport) = xssh.create_ssh_transport_connection(cluster_name)
for error in error_list:
log.write(f'{error}\n')
# create the SFTP client
if OK:
sftp_client = xssh.create_sftp_client(ssh_transport)
# upload the database dataset
if OK:
# get the option dictionary
database_transfer_options_dict = xlib.get_option_dict(
get_database_transfer_config_file())
# get the database dataset identification and the local directory of the database files
database_dataset_id = database_transfer_options_dict['identification'][
'database_dataset_id']
local_dir = database_transfer_options_dict['identification'][
'local_dir']
# set the cluster database directory
cluster_database_dir = '{0}/{1}'.format(
xlib.get_cluster_database_dir(), database_dataset_id)
# create the data directory in the cluster
log.write(f'{xlib.get_separator()}\n')
log.write(
'The database directory {0} in the cluster is being created ...\n'.
format(cluster_database_dir))
command = 'mkdir --parents {0}'.format(cluster_database_dir)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The directory is created.\n')
else:
log.write(f'*** ERROR: Wrong command ---> {command}\n')
# get the sections list
sections_list = []
for section in database_transfer_options_dict.keys():
sections_list.append(section)
sections_list.sort()
# for each section "file-n"
for section in sections_list:
# check than the section identification is like file-n
if re.match('^file-[0-9]+$', section):
# get the file name
file_name = database_transfer_options_dict[section][
'file_name']
# set the local path and cluster path
local_path = '{0}/{1}'.format(local_dir, file_name)
cluster_path = '{0}/{1}'.format(cluster_database_dir,
file_name)
# upload the database file to the cluster
log.write(f'{xlib.get_separator()}\n')
log.write('The file {0} is being uploaded to {1} ...\n'.format(
file_name, cluster_database_dir))
(OK, error_list) = xssh.put_file(sftp_client, local_path,
cluster_path)
if OK:
log.write('The file has been uploaded.\n')
else:
for error in error_list:
log.write(f'{error}\n')
break
# close the SSH transport connection
if OK:
xssh.close_ssh_transport_connection(ssh_transport)
# close the SSH client connection
if OK:
xssh.close_ssh_client_connection(ssh_client)
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut):
log.write(f'{xlib.get_separator()}\n')
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK
def install_node_infrastructure_software_ant(cluster_name, node_name, log):
'''
Install infrastructure software in a node.
'''
# initialize the control variable
OK = True
# create the SSH client connection
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Connecting the SSH client in node {0} ...\n'.format(node_name))
(OK, error_list,
ssh_client) = xssh.create_ssh_client_connection(cluster_name, node_name)
if OK:
log.write('The SSH client is connected.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# update file /etc/apt/sources.list
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write(
'Updating the file /etc/apt/sources.list in node {0} ...\n'.format(
node_name))
command = 'sed -i "s/us-east-1\.ec2\.archive\.ubuntu\.com/old-releases\.ubuntu\.com/g" /etc/apt/sources.list'
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
log.write('command: {0}\n'.format(command))
log.write('OK: {0}\n'.format(OK))
log.write('stdout: {0}\n'.format(stdout))
log.write('stderr: {0}\n'.format(stderr))
OK = True
if OK:
command = 'sed -i "s/security\.ubuntu\.com/old-releases\.ubuntu\.com/g" /etc/apt/sources.list'
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
log.write('command: {0}\n'.format(command))
log.write('OK: {0}\n'.format(OK))
log.write('stdout: {0}\n'.format(stdout))
log.write('stderr: {0}\n'.format(stderr))
OK = True
if OK:
command = 'export DEBIAN_FRONTEND=noninteractive; apt-get update'
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
log.write('command: {0}\n'.format(command))
log.write('OK: {0}\n'.format(OK))
log.write('stdout: {0}\n'.format(stdout))
log.write('stderr: {0}\n'.format(stderr))
OK = True
if OK:
log.write('The file is updated.\n')
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# install libtbb2
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write(
'Installing the package libtbb2 in node {0} (it can be a very slow process, please be patient) ...\n'
.format(node_name))
command = 'export DEBIAN_FRONTEND=noninteractive; apt-get --assume-yes --force-yes install libtbb2'
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
log.write('command: {0}\n'.format(command))
log.write('OK: {0}\n'.format(OK))
log.write('stdout: {0}\n'.format(stdout))
log.write('stderr: {0}\n'.format(stderr))
OK = True
if OK:
log.write('The package is installed.\n')
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
## install OpenJDK8
#if OK:
# log.write('{0}\n'.format(xlib.get_separator()))
# log.write('Installing the package OpenJDK8 in node {0} (it can be a very slow process, please be patient) ...\n'.format(node_name))
# command = 'echo "deb http://ppa.launchpad.net/openjdk-r/ppa/ubuntu trusty main" | tee /etc/apt/sources.list.d/openjdk-r.list'
# (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
# log.write('command: {0}\n'.format(command))
# log.write('OK: {0}\n'.format(OK))
# log.write('stdout: {0}\n'.format(stdout))
# log.write('stderr: {0}\n'.format(stderr))
# OK = True
#if OK:
# command = 'echo "deb-src http://ppa.launchpad.net/openjdk-r/ppa/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/openjdk-r.list'
# (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
# log.write('command: {0}\n'.format(command))
# log.write('OK: {0}\n'.format(OK))
# log.write('stdout: {0}\n'.format(stdout))
# log.write('stderr: {0}\n'.format(stderr))
# OK = True
#if OK:
# #command = 'export DEBIAN_FRONTEND=noninteractive; apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886'
# command = 'export DEBIAN_FRONTEND=noninteractive; apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EB9B1D8886F44E2A'
# (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
# log.write('command: {0}\n'.format(command))
# log.write('OK: {0}\n'.format(OK))
# log.write('stdout: {0}\n'.format(stdout))
# log.write('stderr: {0}\n'.format(stderr))
# OK = True
#if OK:
# command = 'export DEBIAN_FRONTEND=noninteractive; apt-get update'
# (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
# log.write('command: {0}\n'.format(command))
# log.write('OK: {0}\n'.format(OK))
# log.write('stdout: {0}\n'.format(stdout))
# log.write('stderr: {0}\n'.format(stderr))
# OK = True
#if OK:
# command = 'export DEBIAN_FRONTEND=noninteractive; apt-get --assume-yes --force-yes install openjdk-8-jdk'
# (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
# log.write('command: {0}\n'.format(command))
# log.write('OK: {0}\n'.format(OK))
# log.write('stdout: {0}\n'.format(stdout))
# log.write('stderr: {0}\n'.format(stderr))
# OK = True
#if OK:
# command = 'export DEBIAN_FRONTEND=noninteractive; update-java-alternatives --set java-1.8.0-openjdk-amd64'
# (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
# log.write('command: {0}\n'.format(command))
# log.write('OK: {0}\n'.format(OK))
# log.write('stdout: {0}\n'.format(stdout))
# log.write('stderr: {0}\n'.format(stderr))
# OK = True
#if OK:
# log.write('OpenJDK8 is installed.\n')
#else:
# log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# install Oracle Java 8
# -- if OK:
# -- log.write('{0}\n'.format(xlib.get_separator()))
# -- log.write('Installing Oracle Java 8 in node {0} (it can be a very slow process, please be patient) ...\n'.format(node_name))
# -- command = 'echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list'
# -- (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
# -- log.write('command: {0}\n'.format(command))
# -- log.write('OK: {0}\n'.format(OK))
# -- log.write('stdout: {0}\n'.format(stdout))
# -- log.write('stderr: {0}\n'.format(stderr))
# -- OK = True
# -- if OK:
# -- command = 'echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list'
# -- (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
# -- log.write('command: {0}\n'.format(command))
# -- log.write('OK: {0}\n'.format(OK))
# -- log.write('stdout: {0}\n'.format(stdout))
# -- log.write('stderr: {0}\n'.format(stderr))
# -- OK = True
# -- if OK:
# -- command = 'export DEBIAN_FRONTEND=noninteractive; apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886'
# -- (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
# -- log.write('command: {0}\n'.format(command))
# -- log.write('OK: {0}\n'.format(OK))
# -- log.write('stdout: {0}\n'.format(stdout))
# -- log.write('stderr: {0}\n'.format(stderr))
# -- OK = True
# -- if OK:
# -- command = 'export DEBIAN_FRONTEND=noninteractive; apt-get update'
# -- (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
# -- log.write('command: {0}\n'.format(command))
# -- log.write('OK: {0}\n'.format(OK))
# -- log.write('stdout: {0}\n'.format(stdout))
# -- log.write('stderr: {0}\n'.format(stderr))
# -- OK = True
# -- if OK:
# -- command = 'echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections'
# -- (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
# -- log.write('command: {0}\n'.format(command))
# -- log.write('OK: {0}\n'.format(OK))
# -- log.write('stdout: {0}\n'.format(stdout))
# -- log.write('stderr: {0}\n'.format(stderr))
# -- OK = True
# -- if OK:
# -- command = 'export DEBIAN_FRONTEND=noninteractive; apt-get --assume-yes --force-yes install openjdk-8-jdk'
# -- (OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
# -- log.write('command: {0}\n'.format(command))
# -- log.write('OK: {0}\n'.format(OK))
# -- log.write('stdout: {0}\n'.format(stdout))
# -- log.write('stderr: {0}\n'.format(stderr))
# -- OK = True
# -- if OK:
# -- log.write('Java 8 is installed.\n')
# -- else:
# -- log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# close the SSH client connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Closing the SSH client connection ...\n')
xssh.close_ssh_client_connection(ssh_client)
log.write('The connection is closed.\n')
# return the control variable
return OK
def install_node_infrastructure_software(cluster_name, node_name, log):
'''
Install infraestructure software in a node.
'''
# initialize the control variable
OK = True
# get the infrastructure software installation script path in local compute
local_script_path = get_infrastructure_software_installation_script()
# set the infrastructure software installation script path in node
node_script_path = './{0}'.format(os.path.basename(local_script_path))
# set the infrastructure software installation log path in node
node_log_path = node_script_path[:node_script_path.find('.sh')] + '.log'
# build the infrastructure software installation script
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write(
'Building the infrastructure software installation script {0} ...\n'
.format(local_script_path))
(OK, error_list
) = build_infrastructure_software_installation_script(cluster_name)
if OK:
log.write('The file is built.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# create the SSH client connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write(
'Connecting the SSH client to node {0} ...\n'.format(node_name))
(OK, error_list, ssh_client) = xssh.create_ssh_client_connection(
cluster_name, node_name)
if OK:
log.write('The SSH client is connected.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# create the SSH transport connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write(
'Connecting the SSH transport to node {0} ...\n'.format(node_name))
(OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(
cluster_name, node_name)
if OK:
log.write('The SSH transport is connected.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# create the SFTP client
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write(
'Connecting the SFTP client to node {0} ...\n'.format(node_name))
sftp_client = xssh.create_sftp_client(ssh_transport)
log.write('The SFTP client is connected.\n')
# upload the infraestructe software installation script to the node
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write(
'Uploading the infraestructe software installation script to the node {0} ...\n'
.format(node_name))
(OK, error_list) = xssh.put_file(sftp_client, local_script_path,
node_script_path)
if OK:
log.write('The file is uploaded.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# set run permision to the infraestructe software installation script in the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Setting on the run permision ...\n')
command = 'chmod u+x {0}'.format(node_script_path)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The run permision is set.\n')
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# submit the infraestructe software installation script
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write(
'Submitting the infraestructe software installation script in the node {0} ...\n'
.format(node_name))
command = '{0} &>{1} &'.format(node_script_path, node_log_path)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The script is submitted.\n')
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# close the SSH transport connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Closing the SSH transport connection ...\n')
xssh.close_ssh_transport_connection(ssh_transport)
log.write('The connection is closed.\n')
# close the SSH client connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Closing the SSH client connection ...\n')
xssh.close_ssh_client_connection(ssh_client)
log.write('The connection is closed.\n')
# return the control variable
return OK
def download_result_dataset(cluster_name, log, function=None):
'''
Download the result dataset of a run from the cluster.
'''
# initialize the control variable
OK = True
# get the read transfer config file
result_transfer_config_file = get_result_transfer_config_file()
# warn that the log window must not be closed
if not isinstance(log, xlib.DevStdOut):
log.write('This process might take several minutes. Do not close this window, please wait!\n')
# get and validate the result transfer config file
log.write('{0}\n'.format(xlib.get_separator()))
log.write('The result transfer config file is been validating ...\n')
if validate_result_transfer_config_file(strict=True):
log.write('The config file is OK.\n')
else:
log.write('*** ERROR: The result transfer config file is not valid.\n')
log.write('Please correct this file or recreate the config files.\n')
OK = False
# create the SSH client connection
if OK:
(OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master')
for error in error_list:
log.write('{0}\n'.format(error))
# create the SSH transport connection
if OK:
(OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name, 'master')
for error in error_list:
log.write('{0}\n'.format(error))
# create the SFTP client
if OK:
sftp_client = xssh.create_sftp_client(ssh_transport)
# get the options dictionary
if OK:
result_transfer_options_dict = xlib.get_option_dict(result_transfer_config_file)
# download the result dataset
if OK:
# get the sections list
sections_list = []
for section in result_transfer_options_dict.keys():
sections_list.append(section)
sections_list.sort()
# get the experiment identification, run identification and local directory from the section "identification"
experiment_id = result_transfer_options_dict['identification']['experiment_id']
result_dataset_id = result_transfer_options_dict['identification']['result_dataset_id']
status = result_transfer_options_dict['identification']['status'].lower()
local_dir = result_transfer_options_dict['identification']['local_dir']
# download files when the status is uncompressed
if status == 'uncompressed':
# for each section "file-n"
for section in sections_list:
# verify than the section identification is like file-n
if re.match('^file-[0-9]+$', section):
# get the dataset subdirectory and file name
dataset_subdirectory = result_transfer_options_dict[section]['dataset_subdirectory']
file_name = result_transfer_options_dict[section]['file_name']
# verify if the dataset subdirectory is created
pathlib.Path(os.path.normpath('{0}/{1}'.format(local_dir, dataset_subdirectory))).mkdir(parents=True, exist_ok=True)
# assign the cluster path and local path
cluster_path = '{0}/{1}/{2}/{3}/{4}'.format(xlib.get_cluster_result_dir(), experiment_id, result_dataset_id, dataset_subdirectory, file_name)
local_path = os.path.normpath('{0}/{1}/{2}'.format(local_dir, dataset_subdirectory, file_name))
# download the result file from the cluster
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Downloading the file {0} to {1} ...\n'.format(cluster_path, local_dir))
(OK, error_list) = xssh.get_file(sftp_client, cluster_path, local_path)
if OK:
log.write('The file has been downloaded.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
break
# download files when the status is compressed
elif status == 'compressed':
# assign the cluster path and local path
cluster_path = '{0}/{1}/{2}'.format(xlib.get_cluster_result_dir(), experiment_id, result_dataset_id)
local_path = '{0}/{1}'.format(local_dir, result_dataset_id)
# download the result file from the cluster
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Downloading the file {0} to {1} ...\n'.format(cluster_path, local_dir))
(OK, error_list) = xssh.get_file(sftp_client, cluster_path, local_path)
if OK:
log.write('The file has been downloaded.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# close the SSH transport connection
if OK:
xssh.close_ssh_transport_connection(ssh_transport)
# close the SSH client connection
if OK:
xssh.close_ssh_client_connection(ssh_client)
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut):
log.write('{0}\n'.format(xlib.get_separator()))
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK
def form_show_status_batch_jobs():
'''
Show the status of batch jobs in the cluster.
'''
# initialize the control variable
OK = True
# initialize the list of identification of the batch jobs
batch_job_id_list = []
# print the header
clib.clear_screen()
clib.print_headers_with_environment('Cluster operation - Show status batch jobs')
# get the cluster name
print(xlib.get_separator())
if xec2.get_running_cluster_list(volume_creator_included=False) != []:
cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True)
else:
print('WARNING: There is not any running cluster.')
OK = False
# create the SSH client connection
if OK:
(OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master')
for error in error_list:
print(error)
# get the batch job dictionary
if help:
(OK, error_list, batch_job_dict) = xcluster.get_batch_job_dict(ssh_client)
# build the list of identifications of the batch jobs
if OK and help:
for job_id in batch_job_dict.keys():
batch_job_id_list.append(job_id)
if batch_job_id_list != []:
batch_job_id_list.sort()
else:
print('WARNING: There is not any batch job.')
OK = False
# print the batch jobs
if OK and help:
print(xlib.get_separator())
# set data width
job_id_width = 6
job_name_width = 10
state_width = 15
start_date_width = 10
start_time_width = 10
# set line template
line_template = '{0:' + str(job_id_width) + '} {1:' + str(job_name_width) + '} {2:' + str(state_width) + '} {3:' + str(start_date_width) + '} {4:' + str(start_time_width) + '}'
# print header
print(line_template.format('Job id', 'Job name', 'State', 'Start date', 'Start time'))
print(line_template.format('=' * job_id_width, '=' * job_name_width, '=' * state_width, '=' * start_date_width, '=' * start_time_width))
# print detail lines
for job_id in batch_job_id_list:
job_name = batch_job_dict[job_id]['job_name']
state = batch_job_dict[job_id]['state']
start_date = batch_job_dict[job_id]['start_date']
start_time = batch_job_dict[job_id]['start_time']
print(line_template.format(job_id, job_name, state, start_date, start_time))
# close the SSH client connection
if OK:
xssh.close_ssh_client_connection(ssh_client)
# show continuation message
print(xlib.get_separator())
input('Press [Intro] to continue ...')
def form_view_cluster_start_log():
'''
View the cluster start log.
'''
# initialize the control variable
OK = True
# print the header
clib.clear_screen()
clib.print_headers_with_environment('Logs - View the cluster start log')
# get the clustner name
if OK:
print(xlib.get_separator())
if xec2.get_running_cluster_list(only_environment_cluster=True,
volume_creator_included=False) != []:
cluster_name = cinputs.input_cluster_name(
volume_creator_included=False, help=True)
else:
print('WARNING: There is not any running cluster.')
OK = False
# create the SSH client connection
if OK:
(OK, error_list,
ssh_client) = xssh.create_ssh_client_connection(cluster_name)
for error in error_list:
print(error)
# create the SSH transport connection
if OK:
(OK, error_list,
ssh_transport) = xssh.create_ssh_transport_connection(cluster_name)
for error in error_list:
print(error)
# create the SFTP client
if OK:
sftp_client = xssh.create_sftp_client(ssh_transport)
# create the local path
if not os.path.exists(xlib.get_temp_dir()):
os.makedirs(xlib.get_temp_dir())
# get the log file name and build local and cluster paths
if OK:
local_path = f'{xlib.get_temp_dir()}/{os.path.basename(xinstance.get_infrastructure_software_installation_log())}'
cluster_path = f'/home/ubuntu/{os.path.basename(xinstance.get_infrastructure_software_installation_log())}'
# download the log file from the cluster
if OK:
print(xlib.get_separator())
print(
f'The file {os.path.basename(xinstance.get_infrastructure_software_installation_log())} is being downloaded from {cluster_path} ...'
)
OK = xssh.get_file(sftp_client, cluster_path, local_path)
if OK:
print('The file has been uploaded.')
# close the SSH transport connection
if OK:
xssh.close_ssh_transport_connection(ssh_transport)
# close the SSH client connection
if OK:
xssh.close_ssh_client_connection(ssh_client)
# view the log file
if OK:
text = 'Logs - View the cluster start log'
OK = clib.view_file(local_path, text)
# show continuation message
input('Press [Intro] to continue ...')
def form_view_cluster_experiment_process_log():
'''
View the log of an experiment process in the cluster.
'''
# initialize the control variable
OK = True
# print the header
clib.clear_screen()
clib.print_headers_with_environment('Logs - View an experiment process log in the cluster')
# get the clustner name
if OK:
print(xlib.get_separator())
if xec2.get_running_cluster_list(volume_creator_included=False) != []:
cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True)
else:
print('WARNING: There is not any running cluster.')
OK = False
# create the SSH client connection
if OK:
(OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master')
for error in error_list:
log.write('{0}\n'.format(error))
# create the SSH transport connection
if OK:
(OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name, 'master')
for error in error_list:
log.write('{0}\n'.format(error))
# create the SFTP client
if OK:
sftp_client = xssh.create_sftp_client(ssh_transport)
# get the experiment identification
if OK:
experiment_id = cinputs.input_experiment_id(ssh_client, help=True)
if experiment_id == '':
print('WARNING: The cluster has not experiment data.')
OK = False
# get the result_dataset identification
if OK:
result_dataset_id = cinputs.input_result_dataset_id('uncompressed', ssh_client, experiment_id, help=True)
if result_dataset_id == '':
print('WARNING: The experiment {0} has not result datasets.'.format(experiment_id))
OK = False
# create the local path
if not os.path.exists(xlib.get_temp_dir()):
os.makedirs(xlib.get_temp_dir())
# get the log file name and build local and cluster paths
if OK:
log_file = xlib.get_cluster_log_file()
local_path = '{0}/{1}'.format(xlib.get_temp_dir(), log_file)
cluster_path = '{0}/{1}/{2}'.format(xlib.get_cluster_experiment_result_dir(experiment_id), result_dataset_id, log_file)
# download the log file from the cluster
if OK:
print(xlib.get_separator())
print('The file {0} is being downloaded from {1} ...'.format(log_file, cluster_path))
OK = xssh.get_file(sftp_client, cluster_path, local_path)
if OK:
print('The file has been uploaded.')
# close the SSH transport connection
if OK:
xssh.close_ssh_transport_connection(ssh_transport)
# close the SSH client connection
if OK:
xssh.close_ssh_client_connection(ssh_client)
# view the log file
if OK:
text = 'Logs - View an experiment process log in the cluster'
OK = clib.view_file(local_path, text)
# show continuation message
print(xlib.get_separator())
input('Press [Intro] to continue ...')
def form_list_cluster_experiment_processes():
'''
List the processes of an experiment in the cluster.
'''
# initialize the control variable
OK = True
# print the header
clib.clear_screen()
clib.print_headers_with_environment('Logs - List experiment processes in the cluster')
# get the cluster name
print(xlib.get_separator())
if xec2.get_running_cluster_list(volume_creator_included=False) != []:
cluster_name = cinputs.input_cluster_name(volume_creator_included=False, help=True)
else:
print('WARNING: There is not any running cluster.')
OK = False
# create the SSH client connection
if OK:
(OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master')
for error in error_list:
log.write('{0}\n'.format(error))
# get experiment identification
if OK:
experiment_id = cinputs.input_experiment_id(ssh_client, help=True)
if experiment_id == '':
print('WARNING: The cluster {0} has not experiment data.'.format(cluster_name))
OK = False
# get the result dataset list of the experiment
if OK:
command = 'cd {0}/{1}; for list in `ls`; do ls -ld $list | grep -v ^- > /dev/null && echo $list; done;'.format(xlib.get_cluster_result_dir(), experiment_id)
(OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
result_dataset_id_list = []
for line in stdout:
line = line.rstrip('\n')
if line != 'lost+found':
result_dataset_id_list.append(line)
# print the result dataset identification list of the experiment
if OK:
print(xlib.get_separator())
if result_dataset_id_list == []:
print('*** WARNING: There is not any result dataset of the experiment {0}.'.format(experiment_id))
else:
result_dataset_id_list.sort()
# set data width
result_dataset_width = 25
bioinfo_app_width = 25
# set line template
line_template = '{0:' + str(result_dataset_width) + '} {1:' + str(bioinfo_app_width) + '}'
# print header
print(line_template.format('Result dataset', 'Bioinfo app / Utility'))
print(line_template.format('=' * result_dataset_width, '=' * bioinfo_app_width))
# print detail lines
for result_dataset_id in result_dataset_id_list:
if result_dataset_id.startswith(xlib.get_bedtools_code()+'-'):
bioinfo_app_name = xlib.get_bedtools_name()
elif result_dataset_id.startswith(xlib.get_blastplus_code()+'-'):
bioinfo_app_name = xlib.get_blastplus_name()
elif result_dataset_id.startswith(xlib.get_bowtie2_code()+'-'):
bioinfo_app_name = xlib.get_bowtie2_name()
elif result_dataset_id.startswith(xlib.get_busco_code()+'-'):
bioinfo_app_name = xlib.get_busco_name()
elif result_dataset_id.startswith(xlib.get_cd_hit_code()+'-'):
bioinfo_app_name = xlib.get_cd_hit_est_name()
elif result_dataset_id.startswith(xlib.get_cd_hit_code()+'-'):
bioinfo_app_name = xlib.get_cd_hit_est_name()
elif result_dataset_id.startswith(xlib.get_detonate_code()+'-'):
bioinfo_app_name = xlib.get_detonate_name()
elif result_dataset_id.startswith(xlib.get_emboss_code()+'-'):
bioinfo_app_name = xlib.get_emboss_name()
elif result_dataset_id.startswith(xlib.get_fastqc_code()+'-'):
bioinfo_app_name = xlib.get_fastqc_name()
elif result_dataset_id.startswith(xlib.get_gmap_code()+'-'):
bioinfo_app_name = xlib.get_gmap_name()
elif result_dataset_id.startswith(xlib.get_gmap_gsnap_code()+'-'):
bioinfo_app_name = xlib.get_gmap_gsnap_name()
elif result_dataset_id.startswith(xlib.get_gzip_code()+'-'):
bioinfo_app_name = xlib.get_gzip_name()
elif result_dataset_id.startswith(xlib.get_insilico_read_normalization_code()+'-'):
bioinfo_app_name = xlib.get_insilico_read_normalization_name()
elif result_dataset_id.startswith(xlib.get_miniconda3_code()+'-'):
bioinfo_app_name = xlib.get_miniconda3_name()
elif result_dataset_id.startswith(xlib.get_ngshelper_code()+'-'):
bioinfo_app_name = xlib.get_ngshelper_name()
elif result_dataset_id.startswith(xlib.get_quast_code()+'-'):
bioinfo_app_name = xlib.get_quast_name()
elif result_dataset_id.startswith(xlib.get_r_code()+'-'):
bioinfo_app_name = xlib.get_r_name()
elif result_dataset_id.startswith(xlib.get_ref_eval_code()+'-'):
bioinfo_app_name = xlib.get_ref_eval_name()
elif result_dataset_id.startswith(xlib.get_rnaquast_code()+'-'):
bioinfo_app_name = xlib.get_rnaquast_name()
elif result_dataset_id.startswith(xlib.get_rsem_code()+'-'):
bioinfo_app_name = xlib.get_rsem_name()
elif result_dataset_id.startswith(xlib.get_rsem_eval_code()+'-'):
bioinfo_app_name = xlib.get_rsem_eval_name()
elif result_dataset_id.startswith(xlib.get_samtools_code()+'-'):
bioinfo_app_name = xlib.get_samtools_name()
elif result_dataset_id.startswith(xlib.get_soapdenovotrans_code()+'-'):
bioinfo_app_name = xlib.get_soapdenovotrans_name()
elif result_dataset_id.startswith(xlib.get_star_code()+'-'):
bioinfo_app_name = xlib.get_star_name()
elif result_dataset_id.startswith(xlib.get_transabyss_code()+'-'):
bioinfo_app_name = xlib.get_transabyss_name()
elif result_dataset_id.startswith(xlib.get_transcript_filter_code()+'-'):
bioinfo_app_name = xlib.get_transcript_filter_name()
elif result_dataset_id.startswith(xlib.get_transcriptome_blastx_code()+'-'):
bioinfo_app_name = xlib.get_transcriptome_blastx_name()
elif result_dataset_id.startswith(xlib.get_transrate_code()+'-'):
bioinfo_app_name = xlib.get_transrate_name()
elif result_dataset_id.startswith(xlib.get_trimmomatic_code()+'-'):
bioinfo_app_name = xlib.get_trimmomatic_name()
elif result_dataset_id.startswith(xlib.get_trinity_code()+'-'):
bioinfo_app_name = xlib.get_trinity_name()
else:
bioinfo_app_name = 'xxx'
print(line_template.format(result_dataset_id, bioinfo_app_name))
# close the SSH client connection
if OK:
xssh.close_ssh_client_connection(ssh_client)
# show continuation message
print(xlib.get_separator())
input('Press [Intro] to continue ...')
def get_read_dataset_dict(cluster_name,
experiment_id,
passed_connection=False,
ssh_client=None):
'''
Get a dictoinary with the read datasets of an experiment in the cluster.
'''
# initialize the control variable and the error list
OK = True
error_list = []
# get the read directory in the cluster
cluster_read_dir = xlib.get_cluster_read_dir()
# initialize the dictionary of the read datasets
read_dataset_dict = {}
# create the SSH client connection
if not passed_connection:
(OK, error_list, ssh_client) = xssh.create_ssh_client_connection(
cluster_name, 'master')
# verify the read directory is created
if OK:
command = '[ -d {0} ] && echo RC=0 || echo RC=1'.format(
cluster_read_dir)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if stdout[len(stdout) - 1] != 'RC=0':
error_list.append(
'*** ERROR: There is not any volume mounted in the read directory.\n'
)
error_list.append(
'You must link a volume in the mounting point {0} for the template {1}.\n'
.format(cluster_read_dir, cluster_name))
OK = False
# get the dictionary of the read datasets
if OK:
command = 'ls {0}/{1}'.format(cluster_read_dir, experiment_id)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
for line in stdout:
line = line.rstrip('\n')
if line != 'lost+found':
read_dataset_id = line
if read_dataset_id == xlib.get_uploaded_read_dataset_name(
):
read_dataset_name = ' uploaded reads'
elif read_dataset_id.startswith(
xlib.get_insilico_read_normalization_code()):
mo = re.match(
'{0}-(.+)-(.+)'.format(
xlib.get_insilico_read_normalization_code()),
read_dataset_id)
date = mo.group(1)
time = mo.group(2)
read_dataset_name = '{0} ({1} {2})'.format(
xlib.get_insilico_read_normalization_name(), date,
time)
elif read_dataset_id.startswith(
xlib.get_trimmomatic_code()):
mo = re.match(
'{0}-(.+)-(.+)'.format(
xlib.get_trimmomatic_code()), read_dataset_id)
date = mo.group(1)
time = mo.group(2)
read_dataset_name = '{0} ({1} {2})'.format(
xlib.get_trimmomatic_name(), date, time)
else:
read_dataset_name = 'xxx'
read_dataset_dict[read_dataset_id] = {
'read_dataset_id': read_dataset_id,
'read_dataset_name': read_dataset_name
}
# close the SSH client connection
if OK and not passed_connection:
xssh.close_ssh_client_connection(ssh_client)
# return the control variable, error list and dictionary of the read datasets
return (OK, error_list, read_dataset_dict)
def install_node_infrastructure_software(cluster_name, node_name, log):
'''
Install infrastructure software in a node.
'''
# initialize the control variable
OK = True
# get the infrastructure software installation script path in local compute
local_script_path = get_infrastructure_software_installation_script()
# set the infrastructure software installation script path in node
node_script_path = f'./{os.path.basename(local_script_path)}'
# set the infrastructure software installation log path in node
node_log_path = node_script_path[:node_script_path.find('.sh')] + '.log'
# build the infrastructure software installation script
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
f'Building the infrastructure software installation script {local_script_path} ...\n'
)
(OK, error_list
) = build_infrastructure_software_installation_script(cluster_name)
if OK:
log.write('The file is built.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# create the SSH client connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(f'Connecting the SSH client to node {node_name} ...\n')
(OK, error_list, ssh_client) = xssh.create_ssh_client_connection(
cluster_name, node_name)
if OK:
log.write('The SSH client is connected.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# create the SSH transport connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(f'Connecting the SSH transport to node {node_name} ...\n')
(OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(
cluster_name, node_name)
if OK:
log.write('The SSH transport is connected.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# create the SFTP client
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(f'Connecting the SFTP client to node {node_name} ...\n')
sftp_client = xssh.create_sftp_client(ssh_transport)
log.write('The SFTP client is connected.\n')
# download the AWSCLI2 compressed file to local computer
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
f'Downloading the {xlib.get_awscli_name()} compressed file to local computer ...\n'
)
local_path = f'{xlib.get_temp_dir()}/{xlib.get_awscli_name()}.zip'
if not os.path.exists(os.path.dirname(local_path)):
os.makedirs(os.path.dirname(local_path))
try:
urllib.request.urlretrieve(xlib.get_awscli_url(), local_path)
except Exception as e:
log.write(f'*** EXCEPTION: "{e}".')
log.write(
f'*** ERROR: The file {xlib.get_awscli_url()} can not be downloaded.\n'
)
OK = False
else:
log.write('The file is downloaded.\n')
# upload the AWSCLI2 compressed file to cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
f'Uploading the {xlib.get_awscli_name()} compressed file to the cluster ...\n'
)
cluster_path = f'./{os.path.basename(local_path)}'
(OK, error_list) = xssh.put_file(sftp_client, local_path, cluster_path)
if OK:
log.write('The file is uploaded.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# upload the infraestructe software installation script to the node
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
f'Uploading the infraestructe software installation script to the node {node_name} ...\n'
)
(OK, error_list) = xssh.put_file(sftp_client, local_script_path,
node_script_path)
if OK:
log.write('The file is uploaded.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# set run permision to the infraestructe software installation script in the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Setting on the run permision ...\n')
command = f'chmod u+x {node_script_path}'
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The run permision is set.\n')
else:
log.write(f'*** ERROR: Wrong command ---> {command}\n')
# submit the infraestructe software installation script
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
f'Submitting the infraestructe software installation script in the node {node_name} ...\n'
)
command = f'{node_script_path} &>{node_log_path} &'
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The script is submitted.\n')
else:
log.write(f'*** ERROR: Wrong command ---> {command}\n')
# close the SSH transport connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Closing the SSH transport connection ...\n')
xssh.close_ssh_transport_connection(ssh_transport)
log.write('The connection is closed.\n')
# close the SSH client connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Closing the SSH client connection ...\n')
xssh.close_ssh_client_connection(ssh_client)
log.write('The connection is closed.\n')
# return the control variable
return OK
def unmount_volume(cluster_name,
node_name,
volume_name,
log,
function=None,
is_menu_call=True):
'''
Unmount a volume in a node.
'''
# initialize the control variable
OK = True
# warn that the log window must not be closed
if not isinstance(log, xlib.DevStdOut) and is_menu_call:
log.write(
'This process might take a few minutes. Do not close this window, please wait!\n'
)
# create the SSH client connection
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Connecting SSH client ...\n')
(OK, error_list,
ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master')
if OK:
log.write('The SSH client is connected.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# warn that the requirements are being verified
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Verifying process requirements ...\n')
# verify the master is running
if OK:
(master_state_code,
master_state_name) = xec2.get_node_state(cluster_name, 'master')
if master_state_code != 16:
log.write(
'*** ERROR: The cluster {0} is not running. Its state is {1} ({2}).\n'
.format(cluster_name, master_state_code, master_state_name))
OK = False
# get the zone name of the node
if OK:
zone_name = xec2.get_node_zone_name(cluster_name, node_name)
# get the node identification
if OK:
node_id = xec2.get_node_id(cluster_name, node_name)
if node_id == '':
log.write(
'*** ERROR: The {0} identification of the cluster {1} not has been got.\n'
.format(node_name, cluster_name))
OK = False
# verify the volume is created
if OK:
if not xec2.is_volume_created(volume_name, zone_name):
log.write('*** ERROR: The volume {0} is not created.\n'.format(
volume_name))
OK = False
# get AWS file device
if OK:
aws_device_file = xec2.get_volume_device_file(cluster_name, node_name,
volume_name)
if aws_device_file == '':
log.write(
'*** ERROR: the file device of the volume is not found.\n')
OK = False
# get the volume identification
if OK:
volume_id = xec2.get_volume_id(volume_name, zone_name)
if volume_id == '':
log.write(
'*** ERROR: The identificaction of volume {0} not has been got.\n'
.format(volume_name))
OK = False
# get de file device file
if OK:
machine_device_file = xlib.get_machine_device_file(aws_device_file)
# warn that the requirements are OK
if OK:
log.write('Process requirements are OK.\n')
# unmount the volume to the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write(
'Unmounting volume {0} from the device file {1} ...\n'.format(
volume_name, machine_device_file))
command = 'umount {0}'.format(machine_device_file)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The volume is unmounted.\n')
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# detach the volume to the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write(
'Detaching volume {0} from node {1} of cluster {2} ...\n'.format(
volume_name, node_name, cluster_name))
OK = xec2.detach_volume(node_id, volume_id, aws_device_file)
if OK:
log.write('The volume is detached.\n')
else:
log.write('*** ERROR: The volume is not detached.\n')
# close the SSH client connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Closing the SSH client connection ...\n')
xssh.close_ssh_client_connection(ssh_client)
log.write('The connection is closed.\n')
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut):
log.write('{0}\n'.format(xlib.get_separator()))
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK
def create_volume(volume_name,
volume_type,
volume_size,
terminate_indicator,
log,
function=None):
'''
Create a volume in the current zone.
'''
# initialize the control variable
OK = True
# get the volume creator name
volume_creator_name = xlib.get_volume_creator_name()
# warn that the log window must not be closed
if not isinstance(log, xlib.DevStdOut):
log.write(
'This process might take a few minutes. Do not close this window, please wait!\n'
)
# verify the volume creator is running
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Verifing if volume creator is running ...\n')
(master_state_code,
master_state_name) = xec2.get_node_state(volume_creator_name,
'master')
if master_state_code == 16:
log.write('The volume creator is running.\n')
else:
log.write(
'*** WARNING: The volume creator is not running. It will be created.\n'
)
(OK, master_state_code,
master_state_name) = xcluster.create_cluster(volume_creator_name,
volume_creator_name,
log,
function=None,
is_menu_call=False)
# get the master node identification
if OK:
node_id = xec2.get_node_id(volume_creator_name, 'master')
if node_id == '':
log.write(
'*** ERROR: The master identification of the volume creator not has been got.\n'
)
OK = False
# create the SSH client connection
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Connecting SSH client ...\n')
(OK, error_list,
ssh_client) = xssh.create_ssh_client_connection(volume_creator_name,
'master')
if OK:
log.write('The SSH client is connected.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# warn that the requirements are being verified
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Verifying process requirements ...\n')
# get current region and zone names
region_name = xconfiguration.get_current_region_name()
zone_name = xconfiguration.get_current_zone_name()
# verify that the zone is available
if OK:
if not xec2.is_zone_available(region_name, zone_name):
log.write(
'*** ERROR: The zone {0} is not available in the region {1}.\n'
.format(zone_name, region_name))
OK = False
# verify that the volume is not created
if OK:
if xec2.is_volume_created(volume_name, zone_name):
log.write(
'*** WARNING: The volume {0} is already created.\n'.format(
volume_name))
OK = False
# warn that the requirements are OK
if OK:
log.write('Process requirements are OK.\n')
# create the volume
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Creating volume {0} ...\n'.format(volume_name))
(OK, volume_id) = xec2.create_volume(volume_name, volume_type,
volume_size)
if OK:
log.write(
'The volume is created with the identification {0}.\n'.format(
volume_id))
else:
log.write('*** ERROR: The volume is not created.\n')
# wait for the volume status to be available
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Waiting for the volume state to be available ...\n')
i = 0
available_state_timeout = 120
while True:
i += 1
volume_status = xec2.get_volume_state(volume_name, zone_name)
time.sleep(1)
if volume_status == 'available':
log.write('The volume is now available.\n')
break
elif i > available_state_timeout:
log.write(
'*** The volume is not available after {0} s.\n'.format(
available_state_timeout))
Ok = False
break
# set the aws device and get de machine device
if OK:
aws_device = '/dev/sdp'
machine_device = xlib.get_machine_device_file(aws_device)
# attach the volume to the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write(
'Attaching volume {0} to node master of volume creator ...\n'.
format(volume_name))
OK = xec2.attach_volume(node_id, volume_id, aws_device)
if OK:
log.write('The volume is attached.\n')
else:
log.write('*** ERROR: The volume is not attached.\n')
# wait for the volume attachment to be available
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Waiting for the volume attachment to be available ...\n')
i = 0
attachment_timeout = 120
while True:
i += 1
volume_attachments = xec2.get_volume_attachments(
volume_name, zone_name)
if volume_attachments != []:
log.write('The volume attachment is now available.\n')
break
elif i > attachment_timeout:
log.write(
'*** ERROR: The volume attachment is not available after {0} s.\n'
.format(attachment_timeout))
Ok = False
break
time.sleep(1)
# wait for the device availability
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Waiting for the availabity of the device {0} ...\n'.format(
machine_device))
i = 0
format_timeout = 120
try:
while True:
i += 1
command = 'hdparm -z {0}'.format(machine_device)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
command = 'lsblk --list --noheadings --output NAME'
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
for line in stdout:
if line == os.path.basename(machine_device):
log.write('The device is available.\n')
raise xlib.BreakAllLoops
if i > format_timeout:
log.write(
'*** ERROR: The device is not available after {0} s.\n'
.format(format_timeout))
OK = False
break
time.sleep(1)
except xlib.BreakAllLoops:
pass
# format the volume
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Formating volume {0} to ext4 file system type ...\n'.format(
volume_name))
command = 'mkfs -t ext4 {0} 2>&1; RC=$?; echo "RC=$RC"'.format(
machine_device)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if stdout[len(stdout) - 1] == 'RC=0':
log.write('The volume is formatted.\n')
OK = True
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
OK = False
# detach the volume to the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write(
'Detaching volume {0} from node master of volume creator ...\n'.
format(volume_name))
OK = xec2.detach_volume(node_id, volume_id, aws_device)
if OK:
log.write('The volume is detached.\n')
else:
log.write('*** ERROR: The volume is not detached.\n')
# terminate volume creator
if OK:
if terminate_indicator:
OK = xcluster.terminate_cluster(volume_creator_name,
True,
log,
function=None,
is_menu_call=False)
else:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('You do not indicate to terminate the volume creator.\n')
log.write(
'Remember to terminate it when you finish to create the volumes!!!\n'
)
# close the SSH client connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Closing the SSH client connection ...\n')
xssh.close_ssh_client_connection(ssh_client)
log.write('The connection is closed.\n')
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut):
log.write('{0}\n'.format(xlib.get_separator()))
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK
def run_gzip_process(cluster_name, dataset_type, log, function=None):
'''
Run a gzip process.
'''
# initialize the control variable
OK = True
# get the gzip code and name
gzip_code = xlib.get_gzip_code()
gzip_name = xlib.get_gzip_name()
# get the gzip option dictionary
gzip_option_dict = xlib.get_option_dict(get_gzip_config_file(dataset_type))
# get the experiment identification
experiment_id = gzip_option_dict['identification']['experiment_id']
# get the gzip process script path in the local computer
gzip_process_script = get_gzip_process_script(dataset_type)
# get the gzip process starter path in the local computer
gzip_process_starter = get_gzip_process_starter(dataset_type)
# warn that the log window does not have to be closed
if not isinstance(log, xlib.DevStdOut):
log.write('This process might take several minutes. Do not close this window, please wait!\n')
# check the gzip config file
log.write(f'{xlib.get_separator()}\n')
log.write('Checking the {0} config file ...\n'.format(gzip_name))
(OK, error_list) = check_gzip_config_file(dataset_type, strict=True)
if OK:
log.write('The file is OK.\n')
else:
log.write('*** ERROR: The config file is not valid.\n')
log.write('Please correct this file or recreate the config files.\n')
# create the SSH client connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Connecting the SSH client ...\n')
(OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name)
if OK:
log.write('The SSH client is connected.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# create the SSH transport connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Connecting the SSH transport ...\n')
(OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name)
if OK:
log.write('The SSH transport is connected.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# create the SFTP client
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Connecting the SFTP client ...\n')
sftp_client = xssh.create_sftp_client(ssh_transport)
log.write('The SFTP client is connected.\n')
# warn that the requirements are being verified
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Checking process requirements ...\n')
# check the master is running
if OK:
(master_state_code, master_state_name) = xec2.get_node_state(cluster_name)
if master_state_code != 16:
log.write(f'*** ERROR: The cluster {cluster_name} is not running. Its state is {master_state_code} ({master_state_name}).\n')
OK = False
# warn that the requirements are OK
if OK:
log.write('Process requirements are OK.\n')
# determine the run directory in the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Determining the run directory in the cluster ...\n')
if dataset_type == 'reference':
current_run_dir = xlib.get_cluster_current_run_dir('reference', gzip_code)
elif dataset_type == 'database':
current_run_dir = xlib.get_cluster_current_run_dir('database', gzip_code)
else:
current_run_dir = xlib.get_cluster_current_run_dir(experiment_id, gzip_code)
command = f'mkdir --parents {current_run_dir}'
(OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write(f'The directory path is {current_run_dir}.\n')
else:
log.write(f'*** ERROR: Wrong command ---> {command}\n')
# build the gzip process script
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Building the process script {0} ...\n'.format(gzip_process_script))
(OK, error_list) = build_gzip_process_script(cluster_name, dataset_type, current_run_dir)
if OK:
log.write('The file is built.\n')
else:
log.write('*** ERROR: The file could not be built.\n')
# upload the gzip process script to the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Uploading the process script {0} to the directory {1} ...\n'.format(gzip_process_script, current_run_dir))
cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_script))
(OK, error_list) = xssh.put_file(sftp_client, gzip_process_script, cluster_path)
if OK:
log.write('The file is uploaded.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# set run permision to the gzip process script in the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(gzip_process_script)))
command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_script))
(OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The run permision is set.\n')
else:
log.write(f'*** ERROR: Wrong command ---> {command}\n')
# build the gzip process starter
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Building the process starter {0} ...\n'.format(gzip_process_starter))
(OK, error_list) = build_gzip_process_starter(dataset_type, current_run_dir)
if OK:
log.write('The file is built.\n')
else:
log.write('***ERROR: The file could not be built.\n')
# upload the gzip process starter to the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Uploading the process starter {0} to the directory {1} ...\n'.format(gzip_process_starter, current_run_dir))
cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_starter))
(OK, error_list) = xssh.put_file(sftp_client, gzip_process_starter, cluster_path)
if OK:
log.write('The file is uploaded.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# set run permision to the gzip process starter in the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(gzip_process_starter)))
command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(gzip_process_starter))
(OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The run permision is set.\n')
else:
log.write(f'*** ERROR: Wrong command ---> {command}\n')
# submit the gzip process
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Submitting the process script {0}/{1} ...\n'.format(current_run_dir, os.path.basename(gzip_process_starter)))
OK = xssh.submit_script(cluster_name, ssh_client, current_run_dir, os.path.basename(gzip_process_starter), log)
# close the SSH transport connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Closing the SSH transport connection ...\n')
xssh.close_ssh_transport_connection(ssh_transport)
log.write('The connection is closed.\n')
# close the SSH client connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Closing the SSH client connection ...\n')
xssh.close_ssh_client_connection(ssh_client)
log.write('The connection is closed.\n')
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut):
log.write(f'{xlib.get_separator()}\n')
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK
def run_gmap_process(cluster_name, log, function=None):
'''
Run a GMAP process.
'''
# initialize the control variable
OK = True
# get the GMAP option dictionary
gmap_option_dict = xlib.get_option_dict(get_gmap_config_file())
# get the experiment identification
experiment_id = gmap_option_dict['identification']['experiment_id']
# warn that the log window must not be closed
if not isinstance(log, xlib.DevStdOut):
log.write('This process might take several minutes. Do not close this window, please wait!\n')
# validate the GMAP config file
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Validating the {0} config file ...\n'.format(xlib.get_gmap_name()))
(OK, error_list) = validate_gmap_config_file(strict=True)
if OK:
log.write('The config file is OK.\n')
else:
log.write('*** ERROR: The config file is not valid.\n')
log.write('Please correct this file or recreate the config files.\n')
# create the SSH client connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Connecting the SSH client ...\n')
(OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master')
if OK:
log.write('The SSH client is connected.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# create the SSH transport connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Connecting the SSH transport ...\n')
(OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(cluster_name, 'master')
if OK:
log.write('The SSH transport is connected.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# create the SFTP client
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Connecting the SFTP client ...\n')
sftp_client = xssh.create_sftp_client(ssh_transport)
log.write('The SFTP client is connected.\n')
# warn that the requirements are being verified
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Verifying process requirements ...\n')
# verify the master is running
if OK:
(master_state_code, master_state_name) = xec2.get_node_state(cluster_name, 'master')
if master_state_code != 16:
log.write('*** ERROR: The cluster {0} is not running. Its state is {1} ({2}).\n'.format(cluster_name, master_state_code, master_state_name))
OK = False
# verify the GMAP-GSNAP is setup
if OK:
(OK, error_list, is_setup) = xbioinfoapp.is_setup_bioconda_package(xlib.get_gmap_gsnap_bioconda_code(), cluster_name, True, ssh_client)
if OK:
if not is_setup:
log.write('*** ERROR: {0} is not setup.\n'.format(xlib.get_gmap_name()))
OK = False
else:
log.write('*** ERROR: The verification of {0} setup could not be performed.\n'.format(xlib.get_gmap_name()))
# warn that the requirements are OK
if OK:
log.write('Process requirements are OK.\n')
# determine the run directory in the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Determining the run directory in the cluster ...\n')
current_run_dir = xlib.get_cluster_current_run_dir(experiment_id, xlib.get_gmap_code())
command = 'mkdir --parents {0}'.format(current_run_dir)
(OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The directory path is {0}.\n'.format(current_run_dir))
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# build the GMAP process script
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Building the process script {0} ...\n'.format(get_gmap_process_script()))
(OK, error_list) = build_gmap_process_script(cluster_name, current_run_dir)
if OK:
log.write('The file is built.\n')
if not OK:
log.write('*** ERROR: The file could not be built.\n')
# upload the GMAP process script in the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Uploading the process script {0} in the directory {1} of the master ...\n'.format(get_gmap_process_script(), current_run_dir))
cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_script()))
(OK, error_list) = xssh.put_file(sftp_client, get_gmap_process_script(), cluster_path)
if OK:
log.write('The file is uploaded.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# set run permision to the GMAP process script in the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(get_gmap_process_script())))
command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_script()))
(OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The run permision is set.\n')
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# build the GMAP process starter
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Building the process starter {0} ...\n'.format(get_gmap_process_starter()))
(OK, error_list) = build_gmap_process_starter(current_run_dir)
if OK:
log.write('The file is built.\n')
if not OK:
log.write('***ERROR: The file could not be built.\n')
# upload the GMAP process starter in the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Uploading the process starter {0} in the directory {1} of the master ...\n'.format(get_gmap_process_starter(), current_run_dir))
cluster_path = '{0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_starter()))
(OK, error_list) = xssh.put_file(sftp_client, get_gmap_process_starter(), cluster_path)
if OK:
log.write('The file is uploaded.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# set run permision to the GMAP process starter in the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Setting on the run permision of {0}/{1} ...\n'.format(current_run_dir, os.path.basename(get_gmap_process_starter())))
command = 'chmod u+x {0}/{1}'.format(current_run_dir, os.path.basename(get_gmap_process_starter()))
(OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The run permision is set.\n')
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# submit the GMAP process
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Submitting the process script {0}/{1} ...\n'.format(current_run_dir, os.path.basename(get_gmap_process_starter())))
sge_env = xcluster.get_sge_env()
command = '{0}; qsub -V -b n -cwd {1}/{2}'.format(sge_env, current_run_dir, os.path.basename(get_gmap_process_starter()))
(OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
for line in stdout:
log.write('{0}\n'.format(line))
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# close the SSH transport connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Closing the SSH transport connection ...\n')
xssh.close_ssh_transport_connection(ssh_transport)
log.write('The connection is closed.\n')
# close the SSH client connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Closing the SSH client connection ...\n')
xssh.close_ssh_client_connection(ssh_client)
log.write('The connection is closed.\n')
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut):
log.write('{0}\n'.format(xlib.get_separator()))
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK
def upload_read_dataset(cluster_name, log, function=None):
'''
Upload the read dataset to the cluster.
'''
# initialize the control variable
OK = True
# get the read transfer config file
read_transfer_config_file = get_read_transfer_config_file()
# warn that the log window must not be closed
if not isinstance(log, xlib.DevStdOut):
log.write(
'This process might take several minutes. Do not close this window, please wait!\n'
)
# get and validate the read transfer config file
log.write('{0}\n'.format(xlib.get_separator()))
log.write('The read transfer config file is been validating ...\n')
if validate_read_transfer_config_file(strict=True):
log.write('The config file is OK.\n')
else:
log.write('*** ERROR: The read transfer config file is not valid.\n')
log.write('Please correct this file or recreate the config files.\n')
OK = False
# create the SSH client connection
if OK:
(OK, error_list, ssh_client) = xssh.create_ssh_client_connection(
cluster_name, 'master')
for error in error_list:
log.write('{0}\n'.format(error))
# create the SSH transport connection
if OK:
(OK, error_list, ssh_transport) = xssh.create_ssh_transport_connection(
cluster_name, 'master')
for error in error_list:
log.write('{0}\n'.format(error))
# create the SFTP client
if OK:
sftp_client = xssh.create_sftp_client(ssh_transport)
# get the options dictionary
if OK:
read_transfer_options_dict = xlib.get_option_dict(
read_transfer_config_file)
# get the experiment identification and create the experiment reads directory
if OK:
# get the experiment identification
experiment_id = read_transfer_options_dict['identification'][
'experiment_id']
# Get the directory of read and results datasets of the experiment
cluster_experiment_reads_dir = xlib.get_cluster_experiment_read_dataset_dir(
experiment_id, xlib.get_uploaded_read_dataset_name())
cluster_experiment_result_dir = xlib.get_cluster_experiment_result_dir(
experiment_id)
# create the experiment reads directory
log.write('{0}\n'.format(xlib.get_separator()))
log.write(
'The reads directory {0} in the cluster is being created ...\n'.
format(cluster_experiment_reads_dir))
command = 'mkdir --parents {0}'.format(cluster_experiment_reads_dir)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The directory is created.\n')
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# create the experiment run result directory
log.write('{0}\n'.format(xlib.get_separator()))
log.write(
'The run result directory {0} in the cluster is being created ...\n'
.format(cluster_experiment_result_dir))
command = 'mkdir --parents {0}'.format(cluster_experiment_result_dir)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The directory is created.\n')
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
# upload the read dataset
if OK:
# get the sections list
sections_list = []
for section in read_transfer_options_dict.keys():
sections_list.append(section)
sections_list.sort()
# for each section "file-n"
for section in sections_list:
# verify than the section identification is like file-n
if re.match('^file-[0-9]+$', section):
# get local path and cluster directory
local_path = read_transfer_options_dict[section]['local_path']
# upload the reference file in the cluster
log.write('{0}\n'.format(xlib.get_separator()))
log.write('The file {0} is being uploaded to {1} ...\n'.format(
local_path, cluster_experiment_reads_dir))
cluster_path = '{0}/{1}'.format(cluster_experiment_reads_dir,
os.path.basename(local_path))
(OK, error_list) = xssh.put_file(sftp_client, local_path,
cluster_path)
if OK:
log.write('The file has been uploaded.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
break
# close the SSH transport connection
if OK:
xssh.close_ssh_transport_connection(ssh_transport)
# close the SSH client connection
if OK:
xssh.close_ssh_client_connection(ssh_client)
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut):
log.write('{0}\n'.format(xlib.get_separator()))
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK
def get_result_dataset_dict(cluster_name, experiment_id, status, passed_connection, ssh_client):
'''
Get a dictionary with the result datasets of an experiment in the cluster.
'''
# initialize the control variable and the error list
OK = True
error_list = []
# get the result directory in the cluster
cluster_result_dir = xlib.get_cluster_result_dir()
# initialize the dictionary of the result datasets
result_dataset_dict = {}
# create the SSH client connection
if not passed_connection:
(OK, error_list, ssh_client) = xssh.create_ssh_client_connection(cluster_name, 'master')
# verify the result directory is created
if OK:
command = '[ -d {0} ] && echo RC=0 || echo RC=1'.format(cluster_result_dir)
(OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
if stdout[len(stdout) - 1] != 'RC=0':
error_list.append('*** ERROR: There is not any volume mounted in the result directory.\n')
error_list.append('You must link a volume in the mounting point {0} for the template {1}.\n'.format(cluster_result_dir, cluster_name))
OK = False
# get the dictionary of the result datasets
if OK:
if status == 'uncompressed':
command = 'cd {0}/{1}; for list in `ls`; do ls -ld $list | grep -v ^- > /dev/null && echo $list; done;'.format(cluster_result_dir, experiment_id)
elif status == 'compressed':
command = 'cd {0}/{1}; for list in `ls`; do ls -ld $list | grep -v ^d > /dev/null && echo $list; done;'.format(cluster_result_dir, experiment_id)
(OK, stdout, stderr) = xssh.execute_cluster_command(ssh_client, command)
if OK:
if status == 'uncompressed':
input_pattern = '{0}-(.+)-(.+)'
output_pattern = '{0} ({1} {2})'
elif status == 'compressed':
input_pattern = '{0}-(.+)-(.+).tar.gz'
output_pattern = '{0} ({1} {2}) [compressed]'
for line in stdout:
line = line.rstrip('\n')
if line != 'lost+found':
result_dataset_id = line
if result_dataset_id.startswith(xlib.get_cd_hit_est_code()+'-'):
mo = re.match(input_pattern.format(xlib.get_cd_hit_est_code()), result_dataset_id)
date = mo.group(1)
time = mo.group(2)
result_dataset_name = output_pattern.format(xlib.get_cd_hit_est_name(), date, time)
elif result_dataset_id.startswith(xlib.get_fastqc_code()+'-'):
mo = re.match(input_pattern.format(xlib.get_fastqc_code()), result_dataset_id)
date = mo.group(1)
time = mo.group(2)
result_dataset_name = output_pattern.format(xlib.get_fastqc_name(), date, time)
elif result_dataset_id.startswith(xlib.get_gzip_code()+'-'):
mo = re.match(input_pattern.format(xlib.get_gzip_code()), result_dataset_id)
date = mo.group(1)
time = mo.group(2)
result_dataset_name = output_pattern.format(xlib.get_gzip_name(), date, time)
elif result_dataset_id.startswith(xlib.get_insilico_read_normalization_code()+'-'):
mo = re.match(input_pattern.format(xlib.get_insilico_read_normalization_code()), result_dataset_id)
date = mo.group(1)
time = mo.group(2)
result_dataset_name = output_pattern.format(xlib.get_insilico_read_normalization_name(), date, time)
elif result_dataset_id.startswith(xlib.get_quast_code()+'-'):
mo = re.match(input_pattern.format(xlib.get_quast_code()), result_dataset_id)
date = mo.group(1)
time = mo.group(2)
result_dataset_name = output_pattern.format(xlib.get_quast_name(), date, time)
elif result_dataset_id.startswith(xlib.get_ref_eval_code()+'-'):
mo = re.match(input_pattern.format(xlib.get_ref_eval_code()), result_dataset_id)
date = mo.group(1)
time = mo.group(2)
result_dataset_name = output_pattern.format(xlib.get_ref_eval_name(), date, time)
elif result_dataset_id.startswith(xlib.get_rnaquast_code()+'-'):
mo = re.match(input_pattern.format(xlib.get_rnaquast_code()), result_dataset_id)
date = mo.group(1)
time = mo.group(2)
result_dataset_name = output_pattern.format(xlib.get_rnaquast_name(), date, time)
elif result_dataset_id.startswith(xlib.get_rsem_eval_code()+'-'):
mo = re.match(input_pattern.format(xlib.get_rsem_eval_code()), result_dataset_id)
date = mo.group(1)
time = mo.group(2)
result_dataset_name = output_pattern.format(xlib.get_rsem_eval_name(), date, time)
elif result_dataset_id.startswith(xlib.get_soapdenovotrans_code()+'-'):
mo = re.match(input_pattern.format(xlib.get_soapdenovotrans_code()), result_dataset_id)
date = mo.group(1)
time = mo.group(2)
result_dataset_name = output_pattern.format(xlib.get_soapdenovotrans_name(), date, time)
elif result_dataset_id.startswith(xlib.get_star_code()+'-'):
mo = re.match(input_pattern.format(xlib.get_star_code()), result_dataset_id)
date = mo.group(1)
time = mo.group(2)
result_dataset_name = output_pattern.format(xlib.get_star_name(), date, time)
elif result_dataset_id.startswith(xlib.get_transabyss_code()+'-'):
mo = re.match(input_pattern.format(xlib.get_transabyss_code()), result_dataset_id)
date = mo.group(1)
time = mo.group(2)
result_dataset_name = output_pattern.format(xlib.get_transabyss_name(), date, time)
elif result_dataset_id.startswith(xlib.get_transcript_filter_code()+'-'):
mo = re.match(input_pattern.format(xlib.get_transcript_filter_code()), result_dataset_id)
date = mo.group(1)
time = mo.group(2)
result_dataset_name = output_pattern.format(xlib.get_transcript_filter_name(), date, time)
elif result_dataset_id.startswith(xlib.get_transcriptome_blastx_code()+'-'):
mo = re.match(input_pattern.format(xlib.get_transcriptome_blastx_code()), result_dataset_id)
date = mo.group(1)
time = mo.group(2)
result_dataset_name = output_pattern.format(xlib.get_transcriptome_blastx_name(), date, time)
elif result_dataset_id.startswith(xlib.get_transrate_code()+'-'):
mo = re.match(input_pattern.format(xlib.get_transrate_code()), result_dataset_id)
date = mo.group(1)
time = mo.group(2)
result_dataset_name = output_pattern.format(xlib.get_transrate_name(), date, time)
elif result_dataset_id.startswith(xlib.get_trimmomatic_code()+'-'):
mo = re.match(input_pattern.format(xlib.get_trimmomatic_code()), result_dataset_id)
date = mo.group(1)
time = mo.group(2)
result_dataset_name = output_pattern.format(xlib.get_trimmomatic_name(), date, time)
elif result_dataset_id.startswith(xlib.get_trinity_code()+'-'):
mo = re.match(input_pattern.format(xlib.get_trinity_code()), result_dataset_id)
date = mo.group(1)
time = mo.group(2)
result_dataset_name = output_pattern.format(xlib.get_trinity_name(), date, time)
else:
result_dataset_name = result_dataset_id
result_dataset_dict[result_dataset_id] = {'result_dataset_id': result_dataset_id, 'result_dataset_name': result_dataset_name}
# close the SSH client connection
if OK and not passed_connection:
xssh.close_ssh_client_connection(ssh_client)
# return the control variable, error list and dictionary of the result datasets
return (OK, error_list, result_dataset_dict)
def run_htseq_count_process(cluster_name, log, function=None):
'''
Run a htseq-count process.
'''
# initialize the control variable
OK = True
# get the htseq-count option dictionary
htseq_count_option_dict = xlib.get_option_dict(
get_htseq_count_config_file())
# get the experiment identification
experiment_id = htseq_count_option_dict['identification']['experiment_id']
# warn that the log window does not have to be closed
if not isinstance(log, xlib.DevStdOut):
log.write(
'This process might take several minutes. Do not close this window, please wait!\n'
)
# check the htseq-count config file
log.write(f'{xlib.get_separator()}\n')
log.write(f'Checking the {xlib.get_htseq_count_name()} config file ...\n')
(OK, error_list) = check_htseq_count_config_file(strict=True)
if OK:
log.write('The file is OK.\n')
else:
log.write('*** ERROR: The config file is not valid.\n')
log.write('Please correct this file or recreate the config files.\n')
# create the SSH client connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Connecting the SSH client ...\n')
(OK, error_list,
ssh_client) = xssh.create_ssh_client_connection(cluster_name)
if OK:
log.write('The SSH client is connected.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# create the SSH transport connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Connecting the SSH transport ...\n')
(OK, error_list,
ssh_transport) = xssh.create_ssh_transport_connection(cluster_name)
if OK:
log.write('The SSH transport is connected.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# create the SFTP client
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Connecting the SFTP client ...\n')
sftp_client = xssh.create_sftp_client(ssh_transport)
log.write('The SFTP client is connected.\n')
# warn that the requirements are being verified
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Checking process requirements ...\n')
# check the master is running
if OK:
(master_state_code,
master_state_name) = xec2.get_node_state(cluster_name)
if master_state_code != 16:
log.write(
f'*** ERROR: The cluster {cluster_name} is not running. Its state is {master_state_code} ({master_state_name}).\n'
)
OK = False
# check HTSeq is installed
if OK:
(OK, error_list,
is_installed) = xbioinfoapp.is_installed_anaconda_package(
xlib.get_htseq_anaconda_code(), cluster_name, True, ssh_client)
if OK:
if not is_installed:
log.write(
f'*** ERROR: {xlib.get_htseq_name()} is not installed.\n')
OK = False
else:
log.write(
f'*** ERROR: The verification of {xlib.get_htseq_name()} installation could not be performed.\n'
)
# warn that the requirements are OK
if OK:
log.write('Process requirements are OK.\n')
# determine the run directory in the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Determining the run directory in the cluster ...\n')
current_run_dir = xlib.get_cluster_current_run_dir(
experiment_id, xlib.get_htseq_count_code())
command = f'mkdir --parents {current_run_dir}'
(OK, _, _) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write(f'The directory path is {current_run_dir}.\n')
else:
log.write(f'*** ERROR: Wrong command ---> {command}\n')
# build the htseq-count process script
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
f'Building the process script {get_htseq_count_process_script()} ...\n'
)
(OK, error_list) = build_htseq_count_process_script(
cluster_name, current_run_dir)
if OK:
log.write('The file is built.\n')
if not OK:
log.write('*** ERROR: The file could not be built.\n')
# upload the htseq-count process script to the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
f'Uploading the process script {get_htseq_count_process_script()} to the directory {current_run_dir} ...\n'
)
cluster_path = f'{current_run_dir}/{os.path.basename(get_htseq_count_process_script())}'
(OK, error_list) = xssh.put_file(sftp_client,
get_htseq_count_process_script(),
cluster_path)
if OK:
log.write('The file is uploaded.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# set run permision to the htseq-count process script in the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
f'Setting on the run permision of {current_run_dir}/{os.path.basename(get_htseq_count_process_script())} ...\n'
)
command = f'chmod u+x {current_run_dir}/{os.path.basename(get_htseq_count_process_script())}'
(OK, _, _) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The run permision is set.\n')
else:
log.write(f'*** ERROR: Wrong command ---> {command}\n')
# build the htseq-count process starter
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
f'Building the process starter {get_htseq_count_process_starter()} ...\n'
)
(OK, error_list) = build_htseq_count_process_starter(current_run_dir)
if OK:
log.write('The file is built.\n')
if not OK:
log.write('***ERROR: The file could not be built.\n')
# upload the htseq-count process starter to the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
f'Uploading the process starter {get_htseq_count_process_starter()} to the directory {current_run_dir} ...\n'
)
cluster_path = f'{current_run_dir}/{os.path.basename(get_htseq_count_process_starter())}'
(OK, error_list) = xssh.put_file(sftp_client,
get_htseq_count_process_starter(),
cluster_path)
if OK:
log.write('The file is uploaded.\n')
else:
for error in error_list:
log.write(f'{error}\n')
# set run permision to the htseq-count process starter in the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
f'Setting on the run permision of {current_run_dir}/{os.path.basename(get_htseq_count_process_starter())} ...\n'
)
command = f'chmod u+x {current_run_dir}/{os.path.basename(get_htseq_count_process_starter())}'
(OK, _, _) = xssh.execute_cluster_command(ssh_client, command)
if OK:
log.write('The run permision is set.\n')
else:
log.write(f'*** ERROR: Wrong command ---> {command}\n')
# submit the htseq-count process
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write(
f'Submitting the process script {current_run_dir}/{os.path.basename(get_htseq_count_process_starter())} ...\n'
)
OK = xssh.submit_script(
cluster_name, ssh_client, current_run_dir,
os.path.basename(get_htseq_count_process_starter()), log)
# close the SSH transport connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Closing the SSH transport connection ...\n')
xssh.close_ssh_transport_connection(ssh_transport)
log.write('The connection is closed.\n')
# close the SSH client connection
if OK:
log.write(f'{xlib.get_separator()}\n')
log.write('Closing the SSH client connection ...\n')
xssh.close_ssh_client_connection(ssh_client)
log.write('The connection is closed.\n')
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut):
log.write(f'{xlib.get_separator()}\n')
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK
def form_list_cluster_experiment_processes():
'''
List the processes of an experiment in the cluster.
'''
# initialize the control variable
OK = True
# print the header
clib.clear_screen()
clib.print_headers_with_environment(
'Logs - List experiment processes in the cluster')
# get the cluster name
print(xlib.get_separator())
if xec2.get_running_cluster_list(only_environment_cluster=True,
volume_creator_included=False) != []:
cluster_name = cinputs.input_cluster_name(
volume_creator_included=False, help=True)
else:
print('WARNING: There is not any running cluster.')
OK = False
# create the SSH client connection
if OK:
(OK, error_list,
ssh_client) = xssh.create_ssh_client_connection(cluster_name)
for error in error_list:
print(error)
# get experiment identification
if OK:
experiment_id = cinputs.input_experiment_id(ssh_client, help=True)
if experiment_id == '':
print(
f'WARNING: The cluster {cluster_name} does not have experiment data.'
)
OK = False
# get the result dataset list of the experiment
if OK:
command = f'cd {xlib.get_cluster_result_dir()}/{experiment_id}; for list in `ls`; do ls -ld $list | grep -v ^- > /dev/null && echo $list; done;'
(OK, stdout, _) = xssh.execute_cluster_command(ssh_client, command)
if OK:
result_dataset_id_list = []
for line in stdout:
line = line.rstrip('\n')
if line != 'lost+found':
result_dataset_id_list.append(line)
# print the result dataset identification list of the experiment
if OK:
print(xlib.get_separator())
if result_dataset_id_list == []:
print(
f'*** WARNING: There is not any result dataset of the experiment {experiment_id}.'
)
else:
result_dataset_id_list.sort()
# set data width
result_dataset_width = 30
bioinfo_app_width = 25
# set line
line = '{0:' + str(result_dataset_width) + '} {1:' + str(
bioinfo_app_width) + '}'
# print header
print(line.format('Result dataset', 'Bioinfo app / Utility'))
print(
line.format('=' * result_dataset_width,
'=' * bioinfo_app_width))
# print detail lines
for result_dataset_id in result_dataset_id_list:
if result_dataset_id.startswith(xlib.get_bedtools_code() +
'-'):
bioinfo_app_name = xlib.get_bedtools_name()
elif result_dataset_id.startswith(xlib.get_blastplus_code() +
'-'):
bioinfo_app_name = xlib.get_blastplus_name()
elif result_dataset_id.startswith(xlib.get_bcftools_code() +
'-'):
bioinfo_app_name = xlib.get_bcftools_name()
elif result_dataset_id.startswith(xlib.get_bowtie2_code() +
'-'):
bioinfo_app_name = xlib.get_bowtie2_name()
elif result_dataset_id.startswith(xlib.get_busco_code() + '-'):
bioinfo_app_name = xlib.get_busco_name()
elif result_dataset_id.startswith(xlib.get_cd_hit_code() +
'-'):
bioinfo_app_name = xlib.get_cd_hit_name()
elif result_dataset_id.startswith(xlib.get_cd_hit_est_code() +
'-'):
bioinfo_app_name = xlib.get_cd_hit_est_name()
elif result_dataset_id.startswith(xlib.get_cuffdiff_code() +
'-'):
bioinfo_app_name = xlib.get_cuffdiff_name()
elif result_dataset_id.startswith(xlib.get_cufflinks_code() +
'-'):
bioinfo_app_name = xlib.get_cufflinks_name()
elif result_dataset_id.startswith(
xlib.get_cufflinks_cuffmerge_code() + '-'):
bioinfo_app_name = xlib.get_cufflinks_cuffmerge_name()
elif result_dataset_id.startswith(xlib.get_cuffnorm_code() +
'-'):
bioinfo_app_name = xlib.get_cuffnorm_name()
elif result_dataset_id.startswith(xlib.get_cuffquant_code() +
'-'):
bioinfo_app_name = xlib.get_cuffquant_name()
elif result_dataset_id.startswith(xlib.get_cutadapt_code() +
'-'):
bioinfo_app_name = xlib.get_cutadapt_name()
elif result_dataset_id.startswith(
xlib.get_ddradseq_simulation_code() + '-'):
bioinfo_app_name = xlib.get_ddradseq_simulation_name()
elif result_dataset_id.startswith(
xlib.get_ddradseqtools_code() + '-'):
bioinfo_app_name = xlib.get_ddradseqtools_name()
elif result_dataset_id.startswith(xlib.get_detonate_code() +
'-'):
bioinfo_app_name = xlib.get_detonate_name()
elif result_dataset_id.startswith(xlib.get_diamond_code() +
'-'):
bioinfo_app_name = xlib.get_diamond_name()
elif result_dataset_id.startswith(xlib.get_emboss_code() +
'-'):
bioinfo_app_name = xlib.get_emboss_name()
elif result_dataset_id.startswith(
xlib.get_entrez_direct_code() + '-'):
bioinfo_app_name = xlib.get_entrez_direct_name()
elif result_dataset_id.startswith(xlib.get_express_code() +
'-'):
bioinfo_app_name = xlib.get_express_name()
elif result_dataset_id.startswith(xlib.get_fastqc_code() +
'-'):
bioinfo_app_name = xlib.get_fastqc_name()
elif result_dataset_id.startswith(xlib.get_ggtrinity_code() +
'-'):
bioinfo_app_name = xlib.get_ggtrinity_name()
elif result_dataset_id.startswith(xlib.get_gmap_gsnap_code() +
'-'):
bioinfo_app_name = xlib.get_gmap_gsnap_name()
elif result_dataset_id.startswith(xlib.get_gmap_code() + '-'):
bioinfo_app_name = xlib.get_gmap_name()
elif result_dataset_id.startswith(xlib.get_gsnap_code() + '-'):
bioinfo_app_name = xlib.get_gsnap_name()
elif result_dataset_id.startswith(xlib.get_gzip_code() + '-'):
bioinfo_app_name = xlib.get_gzip_name()
elif result_dataset_id.startswith(xlib.get_hisat2_code() +
'-'):
bioinfo_app_name = xlib.get_hisat2_name()
elif result_dataset_id.startswith(xlib.get_htseq_code() + '-'):
bioinfo_app_name = xlib.get_htseq_name()
elif result_dataset_id.startswith(xlib.get_htseq_count_code() +
'-'):
bioinfo_app_name = xlib.get_htseq_count_name()
elif result_dataset_id.startswith(
xlib.get_insilico_read_normalization_code() + '-'):
bioinfo_app_name = xlib.get_insilico_read_normalization_name(
)
elif result_dataset_id.startswith(xlib.get_ipyrad_code() +
'-'):
bioinfo_app_name = xlib.get_ipyrad_name()
elif result_dataset_id.startswith(xlib.get_kallisto_code() +
'-'):
bioinfo_app_name = xlib.get_kallisto_name()
elif result_dataset_id.startswith(xlib.get_miniconda3_code() +
'-'):
bioinfo_app_name = xlib.get_miniconda3_name()
elif result_dataset_id.startswith(xlib.get_ngshelper_code() +
'-'):
bioinfo_app_name = xlib.get_ngshelper_name()
elif result_dataset_id.startswith(xlib.get_quast_code() + '-'):
bioinfo_app_name = xlib.get_quast_name()
elif result_dataset_id.startswith(xlib.get_r_code() + '-'):
bioinfo_app_name = xlib.get_r_name()
elif result_dataset_id.startswith(xlib.get_raddesigner_code() +
'-'):
bioinfo_app_name = xlib.get_raddesigner_name()
elif result_dataset_id.startswith(xlib.get_ref_eval_code() +
'-'):
bioinfo_app_name = xlib.get_ref_eval_name()
elif result_dataset_id.startswith(xlib.get_rnaquast_code() +
'-'):
bioinfo_app_name = xlib.get_rnaquast_name()
elif result_dataset_id.startswith(xlib.get_rsem_code() + '-'):
bioinfo_app_name = xlib.get_rsem_name()
elif result_dataset_id.startswith(xlib.get_rsem_eval_code() +
'-'):
bioinfo_app_name = xlib.get_rsem_eval_name()
elif result_dataset_id.startswith(xlib.get_rsitesearch_code() +
'-'):
bioinfo_app_name = xlib.get_rsitesearch_name()
elif result_dataset_id.startswith(xlib.get_samtools_code() +
'-'):
bioinfo_app_name = xlib.get_samtools_name()
elif result_dataset_id.startswith(xlib.get_soapdenovo2_code() +
'-'):
bioinfo_app_name = xlib.get_soapdenovo2_name()
elif result_dataset_id.startswith(
xlib.get_soapdenovotrans_code() + '-'):
bioinfo_app_name = xlib.get_soapdenovotrans_name()
elif result_dataset_id.startswith(xlib.get_star_code() + '-'):
bioinfo_app_name = xlib.get_star_name()
elif result_dataset_id.startswith(xlib.get_starcode_code() +
'-'):
bioinfo_app_name = xlib.get_starcode_name()
elif result_dataset_id.startswith(xlib.get_toa_code() + '-'):
bioinfo_app_name = xlib.get_toa_name()
elif result_dataset_id.startswith(
xlib.get_toa_process_download_basic_data_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_download_basic_data_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_download_dicots_04_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_download_dicots_04_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_download_gene_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_download_gene_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_download_go_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_download_go_name()
elif result_dataset_id.startswith(
xlib.get_toa_process_download_gymno_01_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_download_gymno_01_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_download_interpro_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_download_interpro_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_download_monocots_04_code() +
'-'):
bioinfo_app_name = xlib.get_toa_process_download_monocots_04_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_download_taxonomy_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_download_taxonomy_name(
)
elif result_dataset_id.startswith(
xlib.
get_toa_process_gilist_viridiplantae_nucleotide_gi_code(
) + '-'):
bioinfo_app_name = xlib.get_toa_process_gilist_viridiplantae_nucleotide_gi_name(
)
elif result_dataset_id.startswith(
xlib.
get_toa_process_gilist_viridiplantae_protein_gi_code()
+ '-'):
bioinfo_app_name = xlib.get_toa_process_gilist_viridiplantae_protein_gi_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_load_basic_data_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_load_basic_data_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_load_dicots_04_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_load_dicots_04_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_load_gene_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_load_gene_name()
elif result_dataset_id.startswith(
xlib.get_toa_process_load_go_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_load_go_name()
elif result_dataset_id.startswith(
xlib.get_toa_process_load_gymno_01_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_load_gymno_01_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_load_interpro_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_load_interpro_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_load_monocots_04_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_load_monocots_04_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_merge_annotations_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_merge_annotations_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_nr_blastplus_db_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_nr_blastplus_db_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_nr_diamond_db_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_nr_diamond_db_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_nt_blastplus_db_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_nt_blastplus_db_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_pipeline_aminoacid_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_pipeline_aminoacid_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_pipeline_nucleotide_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_pipeline_nucleotide_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_proteome_dicots_04_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_proteome_dicots_04_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_proteome_gymno_01_code() + '-'):
bioinfo_app_name = xlib.get_toa_process_proteome_gymno_01_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_proteome_monocots_04_code() +
'-'):
bioinfo_app_name = xlib.get_toa_process_proteome_monocots_04_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_proteome_refseq_plant_code() +
'-'):
bioinfo_app_name = xlib.get_toa_process_proteome_refseq_plant_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_rebuild_toa_database_code() +
'-'):
bioinfo_app_name = xlib.get_get_toa_process_rebuild_toa_database_name(
)
elif result_dataset_id.startswith(
xlib.get_toa_process_recreate_toa_database_code() +
'-'):
bioinfo_app_name = xlib.get_get_toa_process_recreate_toa_database_name(
)
elif result_dataset_id.startswith(xlib.get_tophat_code() +
'-'):
bioinfo_app_name = xlib.get_tophat_name()
elif result_dataset_id.startswith(xlib.get_transabyss_code() +
'-'):
bioinfo_app_name = xlib.get_transabyss_name()
elif result_dataset_id.startswith(
xlib.get_transcript_filter_code() + '-'):
bioinfo_app_name = xlib.get_transcript_filter_name()
elif result_dataset_id.startswith(
xlib.get_transcriptome_blastx_code() + '-'):
bioinfo_app_name = xlib.get_transcriptome_blastx_name()
elif result_dataset_id.startswith(
xlib.get_transdecoder_code() + '-'):
bioinfo_app_name = xlib.get_transdecoder_name()
elif result_dataset_id.startswith(xlib.get_transrate_code() +
'-'):
bioinfo_app_name = xlib.get_transrate_name()
elif result_dataset_id.startswith(xlib.get_trimmomatic_code() +
'-'):
bioinfo_app_name = xlib.get_trimmomatic_name()
elif result_dataset_id.startswith(xlib.get_trinity_code() +
'-'):
bioinfo_app_name = xlib.get_trinity_name()
elif result_dataset_id.startswith(
xlib.get_variant_calling_code() + '-'):
bioinfo_app_name = xlib.get_variant_calling_name()
elif result_dataset_id.startswith(xlib.get_vcftools_code() +
'-'):
bioinfo_app_name = xlib.get_vcftools_name()
elif result_dataset_id.startswith(
xlib.get_vcftools_perl_libraries_code() + '-'):
bioinfo_app_name = xlib.get_vcftools_perl_libraries_name()
elif result_dataset_id.startswith(xlib.get_vsearch_code() +
'-'):
bioinfo_app_name = xlib.get_vsearch_name()
else:
bioinfo_app_name = 'xxx'
print(line.format(result_dataset_id, bioinfo_app_name))
# close the SSH client connection
if OK:
xssh.close_ssh_client_connection(ssh_client)
# show continuation message
print(xlib.get_separator())
input('Press [Intro] to continue ...')
