def form_create_keypairs():
'''
Create the key pairs of a region.
'''
# initialize the control variable
OK = True
# print the header and get the cluster name
clib.clear_screen()
clib.print_headers_with_environment('Security - Create key pairs')
# get current region name
region_name = xconfiguration.get_current_region_name()
# confirm the creation of the key pairs
print(xlib.get_separator())
OK = clib.confirm_action('The key pairs of the region {0} are going to be created.'.format(region_name))
# create key pairs
if OK:
print(xlib.get_separator())
print('The key pairs of the region {0} are been created ...'.format(region_name))
(OK, error_list) = xec2.create_keypairs(region_name)
if OK:
print('The key pairs and their corresponding local files have been created.')
else:
for error in error_list:
print(error)
# show continuation message
print(xlib.get_separator())
input('Press [Intro] to continue ...')
def list_clusters(log, function=None):
'''
List clusters.
'''
# initialize the control variable
OK = True
# get current region
region_name = xconfiguration.get_current_region_name()
# warn that the log window must not be closed
if not isinstance(log, xlib.DevStdOut):
log.write(
'This process might take a few minutes. Do not close this window, please wait!\n'
)
# warn that the requirements are being verified
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Verifying process requirements ...\n')
# verify if there are some running clusters
if xec2.get_running_cluster_list(volume_creator_included=True) == []:
log.write('WARNING: There is not any running cluster.\n')
OK = False
# warn that the requirements are OK
if OK:
log.write('Process requirements are OK.\n')
# list clusters
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Listing clusters using StarCluster ...\n')
log.write('\n')
command = '{0} --region={1} listclusters'.format(
xlib.get_starcluster(), region_name)
rc = xlib.run_command(command, log)
if rc != 0:
log.write('*** ERROR: Return code {0} in command -> {1}\n'.format(
rc, command))
OK = False
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut):
log.write('{0}\n'.format(xlib.get_separator()))
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK
def print_headers_with_environment(process_name):
'''
Print the headers of a screen with environmen information.
'''
# print the project name, version and the process name
title = f'{xlib.get_project_name()} v {xlib.get_project_version()} - {process_name}'
line = '-' * len(title)
print(f'+-{line}-+')
print(f'| {title} |')
print(f'+-{line}-+')
print()
# get current region and zone names
region_name = xconfiguration.get_current_region_name()
zone_name = xconfiguration.get_current_zone_name()
# print the environment and the current region and zone names
print(
f'Environment: {xconfiguration.environment} - Region: {region_name} - Zone: {zone_name}'
)
print()
def print_headers_with_environment(process_name):
'''
Print the headers of a screen with environmen information.
'''
# print the project name, version and the process name
title = '{0} v {1} - {2}'.format(xlib.get_project_name(),
xlib.get_project_version(), process_name)
line = '-' * len(title)
print('+-{0}-+'.format(line))
print('| {0} |'.format(title))
print('+-{0}-+'.format(line))
print()
# get current region and zone names
region_name = xconfiguration.get_current_region_name()
zone_name = xconfiguration.get_current_zone_name()
# print the environment and the current region and zone names
print('Environment: {0} - Region: {1} - Zone: {2}'.format(
xconfiguration.environment, region_name, zone_name))
print()
def form_create_ngscloud_config_file(is_menu_call):
'''
Create the NGScloud config file corresponding to the environment.
'''
# initialize the control variable
OK = True
# print the header
if is_menu_call:
clib.clear_screen()
clib.print_headers_with_environment('Configuration - Recreate TransciptomeCloud config file')
# get current region and zone names
region_name = xconfiguration.get_current_region_name()
zone_name = xconfiguration.get_current_zone_name()
# get basic AWS data and contact e-mail address from NGScloud config file
(user_id, access_key_id, secret_access_key) = xconfiguration.get_basic_aws_data()
email = xconfiguration.get_contact_data()
# confirm or change the AWS data and contact e-mail address
print(xlib.get_separator())
user_id = cinputs.input_user_id(user_id)
access_key_id = cinputs.input_access_key_id(access_key_id)
secret_access_key = cinputs.input_secret_access_key(secret_access_key)
email = cinputs.input_email(email)
# verify the AWS access key identification and the AWS secret access key
print(xlib.get_separator())
print('Verifying the AWS access key identification and the AWS secret access key')
OK = xec2.verify_aws_credentials(access_key_id, secret_access_key)
if OK:
print('The credentials are OK.')
else:
print('ERROR: The credentials are wrong. Please review your access key identification and secret access key in the AWS web.')
if not is_menu_call:
raise xlib.ProgramException('EXIT')
# confirm the creation of the NGScloud config file
if OK:
if is_menu_call:
print(xlib.get_separator())
OK = clib.confirm_action('The {0} config file is going to be created. The previous files will be lost.'.format(xlib.get_project_name()))
# create the NGScloud config file corresponding to the environment
if OK:
print(xlib.get_separator())
print('The file {0} is being created ...'.format(xconfiguration.get_ngscloud_config_file()))
(OK, error_list) = xconfiguration.create_ngscloud_config_file(user_id, access_key_id, secret_access_key, email)
if OK:
print('The config file is created with default values.')
print()
print('You can modify the conection data and contact e-mail address in:')
print(' "Cloud control" -> "Configuration" -> "Update connection data and contact e-mail"')
print('The assigned region and zone are {0} and {1}, respectively. You can modify them in:'.format(xconfiguration.get_default_region_name(), xconfiguration.get_default_zone_name()))
print(' "Cloud control" -> "Configuration" -> "Update region and zone data"')
else:
for error in error_list:
print(error)
raise xlib.ProgramException('C001')
# show continuation message
print(xlib.get_separator())
input('Press [Intro] to continue ...')
def terminate_cluster(cluster_name,
force,
log,
function=None,
is_menu_call=True):
'''
Terminate a cluster.
'''
# initialize the control variable
OK = True
# get current region
region_name = xconfiguration.get_current_region_name()
# warn that the log window does not have to be closed
if not isinstance(log, xlib.DevStdOut) and is_menu_call:
log.write(
'This process might take a few minutes. Do not close this window, please wait!\n'
)
# warn that the requirements are being verified
log.write(f'{xlib.get_separator()}\n')
log.write('Checking process requirements ...\n')
# check the cluster is running
if OK:
(master_state_code,
master_state_name) = xec2.get_node_state(cluster_name,
node_name='master')
if not force and master_state_code != 16:
log.write(
f'*** ERROR: The cluster {cluster_name} is not running. Its state is {master_state_code} ({master_state_name}).\n'
)
OK = False
# warn that the requirements are OK
if OK:
log.write('Process requirements are OK.\n')
# terminate cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
if cluster_name == xlib.get_volume_creator_name():
log.write('Terminating the volume creator using StarCluster ...\n')
else:
log.write(
'Terminating the cluster {0} using StarCluster ...\n'.format(
cluster_name))
log.write('\n')
if not force:
command = '{0} --region={1} terminate --confirm {2}'.format(
xlib.get_starcluster(), region_name, cluster_name)
else:
command = '{0} --region={1} terminate --force --confirm {2}'.format(
xlib.get_starcluster(), region_name, cluster_name)
rc = xlib.run_command(command, log)
log.write('\n')
if rc == 0:
if cluster_name == xlib.get_volume_creator_name():
log.write('The volume creator is terminated.\n')
else:
log.write('The cluster is terminated.\n')
else:
log.write('*** ERROR: Return code {0} in command -> {1}\n'.format(
rc, command))
OK = False
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut) and is_menu_call:
log.write(f'{xlib.get_separator()}\n')
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK
def create_cluster(template_name,
cluster_name,
log,
function=None,
is_menu_call=True):
'''
Create a cluster from a template name.
'''
# initialize the control variable
OK = True
# initialize the state variables
master_state_code = ''
master_state_name = ''
# get current region and zone names
region_name = xconfiguration.get_current_region_name()
zone_name = xconfiguration.get_current_zone_name()
# warn that the log window does not have to be closed
if not isinstance(log, xlib.DevStdOut) and is_menu_call:
log.write(
'This process might take several minutes. Do not close this window, please wait!\n'
)
# warn that the requirements are being verified
log.write(f'{xlib.get_separator()}\n')
log.write('Checking process requirements ...\n')
# check that the cluster is defined in the NGScloud config file
if OK:
if not xconfiguration.is_template_defined(template_name):
log.write(
'*** ERROR: The cluster {0} is not defined in the {1} config file.\n'
.format(cluster_name, xlib.get_project_name()))
OK = False
# check that the cluster mode is None
if OK:
if xec2.get_cluster_mode(cluster_name) is not None:
log.write('*** ERROR: There is a cluster or a instance running.\n')
OK = False
# check that the zone is available
if OK:
if not xec2.is_zone_available(region_name, zone_name):
log.write(
'*** ERROR: The zone name {0} is not available.\n'.format(
zone_name))
OK = False
# warn that the requirements are OK
if OK:
log.write('Process requirements are OK.\n')
# create the cluster
if OK:
log.write(f'{xlib.get_separator()}\n')
if cluster_name == xlib.get_volume_creator_name():
log.write('Creating the volume creator using StarCluster ...\n')
else:
log.write(
'Creating the cluster {0} using StarCluster ...\n'.format(
cluster_name))
log.write('\n')
if template_name == xlib.get_volume_creator_name():
command = '{0} --region={1} start --availability-zone={2} --cluster-template={3} --disable-queue {4}'.format(
xlib.get_starcluster(), region_name, zone_name, template_name,
cluster_name)
else:
command = '{0} --region={1} start --availability-zone={2} --cluster-template={3} {4}'.format(
xlib.get_starcluster(), region_name, zone_name, template_name,
cluster_name)
rc = xlib.run_command(command, log)
log.write('\n')
if rc == 0:
(master_state_code,
master_state_name) = xec2.get_node_state(cluster_name,
node_name='master')
if cluster_name == xlib.get_volume_creator_name():
log.write('The volume creator is created.\n')
else:
log.write('The cluster is created.\n')
else:
log.write('*** ERROR: Return code {0} in command -> {1}\n'.format(
rc, command))
log.write('***')
log.write(
'*** You have to terminate {0} (option "Force termination of a cluster")\n'
.format(cluster_name))
log.write('*** and create it again.\n')
OK = False
# install infrastructure software in every node of the cluster
if OK:
if cluster_name != xlib.get_volume_creator_name():
cluster_node_list = xec2.get_cluster_node_list(cluster_name)
for node_name in cluster_node_list:
OK = xnode.install_node_infrastructure_software(
cluster_name, node_name, log)
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut) and is_menu_call:
log.write(f'{xlib.get_separator()}\n')
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable and the state
return (OK, master_state_code, master_state_name)
def build_infrastructure_software_installation_script(cluster_name):
'''
Build the infrastructure software installation script.
'''
# initialize the control variable and the error list
OK = True
error_list = []
# get the connetion data
(user_id, access_key_id,
secret_access_key) = xconfiguration.get_basic_aws_data()
# get the old region and user identification
current_region_name = xconfiguration.get_current_region_name()
# get the NGScloud config file
ngscloud_config_file = xconfiguration.get_ngscloud_config_file()
# get the option dictionary corresponding to the NGScloud config file
ngscloud_options_dict = xlib.get_option_dict(ngscloud_config_file)
# get the dataset structure and NGScloud_volume
dataset_structure = ngscloud_options_dict['dataset info'][
'dataset_structure']
# write the infrastructure software installation script
try:
if not os.path.exists(
os.path.dirname(
get_infrastructure_software_installation_script())):
os.makedirs(
os.path.dirname(
get_infrastructure_software_installation_script()))
with open(get_infrastructure_software_installation_script(),
mode='w',
encoding='iso-8859-1',
newline='\n') as script_file_id:
script_file_id.write('#!/bin/bash\n')
script_file_id.write(
'#-------------------------------------------------------------------------------\n'
)
script_file_id.write(
'SEP="#########################################"\n')
script_file_id.write(
'export HOST_IP=`curl --silent checkip.amazonaws.com`\n')
script_file_id.write(
'export HOST_ADDRESS="ec2-${HOST_IP//./-}-compute-1.amazonaws.com"\n'
)
script_file_id.write(
'export AWS_CONFIG_FILE=/home/ubuntu/.aws/config\n')
script_file_id.write(
'export AWS_SHARED_CREDENTIALS_FILE=/home/ubuntu/.aws/credentials\n'
)
script_file_id.write(
'#-------------------------------------------------------------------------------\n'
)
script_file_id.write('function init\n')
script_file_id.write('{\n')
script_file_id.write(' INIT_DATETIME=`date --utc +%s`\n')
script_file_id.write(
' FORMATTED_INIT_DATETIME=`date --date="@$INIT_DATETIME" "+%Y-%m-%d %H:%M:%S"`\n'
)
script_file_id.write(' echo "$SEP"\n')
script_file_id.write(
' echo "Script started at $FORMATTED_INIT_DATETIME+00:00."\n'
)
script_file_id.write(' echo "$SEP"\n')
script_file_id.write(f' echo "CLUSTER: {cluster_name}"\n')
script_file_id.write(' echo "HOST NAME: $HOSTNAME"\n')
script_file_id.write(' echo "HOST IP: $HOST_IP"\n')
script_file_id.write(' echo "HOST ADDRESS: $HOST_ADDRESS"\n')
script_file_id.write('}\n')
if dataset_structure in [
xconfiguration.get_dataset_structure_singlevolume(),
xconfiguration.get_dataset_structure_none()
]:
script_file_id.write(
'#-------------------------------------------------------------------------------\n'
)
script_file_id.write('function create_dataset_structure\n')
script_file_id.write('{\n')
script_file_id.write(' echo "$SEP"\n')
script_file_id.write(
' echo "Creating the dataset structure ..."\n')
script_file_id.write(
f' sudo mkdir --parents {xlib.get_cluster_app_dir()}\n')
script_file_id.write(
f' sudo mkdir --parents {xlib.get_cluster_database_dir()}\n'
)
script_file_id.write(
f' sudo mkdir --parents {xlib.get_cluster_read_dir()}\n'
)
script_file_id.write(
f' sudo mkdir --parents {xlib.get_cluster_reference_dir()}\n'
)
script_file_id.write(
f' sudo mkdir --parents {xlib.get_cluster_result_dir()}\n'
)
script_file_id.write(
' echo "The dataset structure is created."\n')
script_file_id.write('}\n')
script_file_id.write(
'#-------------------------------------------------------------------------------\n'
)
script_file_id.write('function install_awscli\n')
script_file_id.write('{\n')
script_file_id.write(' echo "$SEP"\n')
script_file_id.write(' echo "Installing the AWS CLI ..."\n')
script_file_id.write(f' unzip {xlib.get_awscli_name()}.zip\n')
script_file_id.write(' RC=$?\n')
script_file_id.write(' if [ $RC -ne 0 ]; then unzip $RC; fi\n')
script_file_id.write(' sudo ./aws/install\n')
script_file_id.write(' RC=$?\n')
script_file_id.write(
' if [ $RC -ne 0 ]; then install $RC; fi\n')
script_file_id.write(' rm -rf aws\n')
script_file_id.write(' RC=$?\n')
script_file_id.write(' if [ $RC -ne 0 ]; then rm $RC; fi\n')
script_file_id.write(f' rm {xlib.get_awscli_name()}.zip\n')
script_file_id.write(' RC=$?\n')
script_file_id.write(' if [ $RC -ne 0 ]; then rm $RC; fi\n')
script_file_id.write(' echo "The package is installed."\n')
script_file_id.write('}\n')
script_file_id.write(
'#-------------------------------------------------------------------------------\n'
)
script_file_id.write('function setup_aws\n')
script_file_id.write('{\n')
script_file_id.write(' echo "$SEP"\n')
script_file_id.write(' echo "Setting up AWS ..."\n')
script_file_id.write(' UBUNTU_AWS_DIR=/home/ubuntu/.aws\n')
script_file_id.write(' mkdir --parents $UBUNTU_AWS_DIR\n')
script_file_id.write(f' CONFIG_FILE=$UBUNTU_AWS_DIR/config\n')
script_file_id.write(' echo "[default]" > $CONFIG_FILE\n')
script_file_id.write(
f' echo "region = {current_region_name}" >> $CONFIG_FILE\n')
script_file_id.write(
' CREDENTIALS_FILE=$UBUNTU_AWS_DIR/credentials\n')
script_file_id.write(' echo "[default]" > $CREDENTIALS_FILE\n')
script_file_id.write(
f' echo "aws_access_key_id = {access_key_id}" >> $CREDENTIALS_FILE\n'
)
script_file_id.write(
f' echo "aws_secret_access_key = {secret_access_key}" >> $CREDENTIALS_FILE\n'
)
script_file_id.write(' sudo echo "AWS is set up."\n')
script_file_id.write('}\n')
script_file_id.write(
'#-------------------------------------------------------------------------------\n'
)
script_file_id.write('function fix_source_list\n')
script_file_id.write('{\n')
script_file_id.write(' echo "$SEP"\n')
script_file_id.write(
' echo "Fixing file /etc/apt/sources.list ..."\n')
script_file_id.write(
' sed -i "s/us-east-1.ec2.archive.ubuntu.com/old-releases.ubuntu.com/g" /etc/apt/sources.list\n'
)
script_file_id.write(' RC=$?\n')
script_file_id.write(
' if [ $RC -ne 0 ]; then manage_error sed $RC; fi\n')
script_file_id.write(
' sed -i "s/security.ubuntu.com/old-releases.ubuntu\.com/g" /etc/apt/sources.list\n'
)
script_file_id.write(' RC=$?\n')
script_file_id.write(
' if [ $RC -ne 0 ]; then manage_error sed $RC; fi\n')
script_file_id.write(' apt-get update\n')
script_file_id.write(' RC=$?\n')
script_file_id.write(
' if [ $RC -ne 0 ]; then manage_error apt-get $RC; fi\n')
script_file_id.write(' echo\n')
script_file_id.write(' echo "The file is fixed."\n')
script_file_id.write('}\n')
script_file_id.write(
'#-------------------------------------------------------------------------------\n'
)
script_file_id.write('function install_xorg\n')
script_file_id.write('{\n')
script_file_id.write(' echo "$SEP"\n')
script_file_id.write(
' echo "Installing the package xorg ..."\n')
script_file_id.write(
' sudo apt-get --assume-yes install xorg\n')
script_file_id.write(' RC=$?\n')
script_file_id.write(
' if [ $RC -ne 0 ]; then manage_error apt-get $RC; fi\n')
script_file_id.write(' echo "The package is installed."\n')
script_file_id.write('}\n')
script_file_id.write(
'#-------------------------------------------------------------------------------\n'
)
script_file_id.write('function install_libtbb2\n')
script_file_id.write('{\n')
script_file_id.write(' echo "$SEP"\n')
script_file_id.write(
' echo "Installing the package libtbb2 ..."\n')
script_file_id.write(' echo\n')
script_file_id.write(' apt-get --assume-yes install libtbb2\n')
script_file_id.write(' RC=$?\n')
script_file_id.write(
' if [ $RC -ne 0 ]; then manage_error apt-get $RC; fi\n')
script_file_id.write(' echo\n')
script_file_id.write(' echo "The package is installed."\n')
script_file_id.write('}\n')
script_file_id.write(
'#-------------------------------------------------------------------------------\n'
)
script_file_id.write('function install_libxt6\n')
script_file_id.write('{\n')
script_file_id.write(' echo "$SEP"\n')
script_file_id.write(
' echo "Installing the package libxt6 ..."\n')
script_file_id.write(
' sudo apt-get --assume-yes install libxt6\n')
script_file_id.write(' RC=$?\n')
script_file_id.write(
' if [ $RC -ne 0 ]; then manage_error apt-get $RC; fi\n')
script_file_id.write(' echo "The package is installed."\n')
script_file_id.write('}\n')
script_file_id.write(
'#-------------------------------------------------------------------------------\n'
)
script_file_id.write('function install_parallel\n')
script_file_id.write('{\n')
script_file_id.write(' echo "$SEP"\n')
script_file_id.write(
' echo "Installing the package parallel ..."\n')
script_file_id.write(
' sudo apt-get --assume-yes install parallel\n')
script_file_id.write(' RC=$?\n')
script_file_id.write(
' if [ $RC -ne 0 ]; then manage_error apt-get $RC; fi\n')
script_file_id.write(' echo "The package is installed."\n')
script_file_id.write('}\n')
script_file_id.write(
'#-------------------------------------------------------------------------------\n'
)
script_file_id.write('function install_texlive\n')
script_file_id.write('{\n')
script_file_id.write(' echo "$SEP"\n')
script_file_id.write(
' echo "Installing the package texlive ..."\n')
script_file_id.write(
' sudo apt-get --assume-yes install texlive-latex-base\n')
script_file_id.write(' RC=$?\n')
script_file_id.write(
' if [ $RC -ne 0 ]; then manage_error apt-get $RC; fi\n')
script_file_id.write(
' sudo apt-get --assume-yes install texlive-fonts-recommended\n'
)
script_file_id.write(' RC=$?\n')
script_file_id.write(
' if [ $RC -ne 0 ]; then manage_error apt-get $RC; fi\n')
script_file_id.write(
' sudo apt-get --assume-yes install texlive-fonts-extra\n')
script_file_id.write(' RC=$?\n')
script_file_id.write(
' if [ $RC -ne 0 ]; then manage_error apt-get $RC; fi\n')
script_file_id.write(
' sudo apt-get --assume-yes install texlive-latex-extra\n')
script_file_id.write(' RC=$?\n')
script_file_id.write(
' if [ $RC -ne 0 ]; then manage_error apt-get $RC; fi\n')
script_file_id.write(' echo "The package is installed."\n')
script_file_id.write('}\n')
script_file_id.write(
'#-------------------------------------------------------------------------------\n'
)
script_file_id.write('function uninstall_mysql\n')
script_file_id.write('{\n')
script_file_id.write(' echo "$SEP"\n')
script_file_id.write(' echo "Uninstalling MySQL ..."\n')
script_file_id.write(
' sudo apt-get purge --auto-remove --assume-yes mysql-client mysql-client-5.5 mysql-client-core-5.5 mysql-common mysql-server mysql-server-5.5 mysql-server-core-5.5\n'
)
script_file_id.write(' RC=$?\n')
script_file_id.write(
' if [ $RC -ne 0 ]; then manage_error apt-get $RC; fi\n')
script_file_id.write(' echo "MySQL is uninstalled."\n')
script_file_id.write('}\n')
script_file_id.write(
'#-------------------------------------------------------------------------------\n'
)
script_file_id.write('function create_swapfile\n')
script_file_id.write('{\n')
script_file_id.write(' echo "$SEP"\n')
script_file_id.write(
' echo "Creating a file which will be used for swap ..."\n')
script_file_id.write(
' sudo dd if=/dev/zero of=/swapfile bs=1024 count=2097152\n'
)
script_file_id.write(' RC=$?\n')
script_file_id.write(
' if [ $RC -ne 0 ]; then manage_error dd $RC; fi\n')
script_file_id.write(' sudo chmod 600 /swapfile\n')
script_file_id.write(' RC=$?\n')
script_file_id.write(
' if [ $RC -ne 0 ]; then manage_error chmod $RC; fi\n')
script_file_id.write(' sudo mkswap /swapfile\n')
script_file_id.write(' RC=$?\n')
script_file_id.write(
' if [ $RC -ne 0 ]; then manage_error mkswap $RC; fi\n')
script_file_id.write(' sudo swapon /swapfile\n')
script_file_id.write(' RC=$?\n')
script_file_id.write(
' if [ $RC -ne 0 ]; then manage_error swapon $RC; fi\n')
script_file_id.write(
' sudo echo "/swapfile swap swap defaults 0 0" >> /etc/fstab\n'
)
script_file_id.write(' RC=$?\n')
script_file_id.write(
' if [ $RC -ne 0 ]; then manage_error echo $RC; fi\n')
script_file_id.write(' echo\n')
script_file_id.write(' echo "The file is created."\n')
script_file_id.write('}\n')
script_file_id.write(
'#-------------------------------------------------------------------------------\n'
)
script_file_id.write('function end\n')
script_file_id.write('{\n')
script_file_id.write(' END_DATETIME=`date --utc +%s`\n')
script_file_id.write(
' FORMATTED_END_DATETIME=`date --date="@$END_DATETIME" "+%Y-%m-%d %H:%M:%S"`\n'
)
script_file_id.write(' calculate_duration\n')
script_file_id.write(' echo "$SEP"\n')
script_file_id.write(
' echo "Script ended OK at $FORMATTED_END_DATETIME+00:00 with a run duration of $DURATION s ($FORMATTED_DURATION)."\n'
)
script_file_id.write(' echo "$SEP"\n')
script_file_id.write(' send_mail ok\n')
script_file_id.write(' exit 0\n')
script_file_id.write('}\n')
script_file_id.write(
'#-------------------------------------------------------------------------------\n'
)
script_file_id.write('function manage_error\n')
script_file_id.write('{\n')
script_file_id.write(' END_DATETIME=`date --utc +%s`\n')
script_file_id.write(
' FORMATTED_END_DATETIME=`date --date="@$END_DATETIME" "+%Y-%m-%d %H:%M:%S"`\n'
)
script_file_id.write(' calculate_duration\n')
script_file_id.write(' echo "$SEP"\n')
script_file_id.write(' echo "ERROR: $1 returned error $2"\n')
script_file_id.write(
' echo "Script ended WRONG at $FORMATTED_END_DATETIME+00:00 with a run duration of $DURATION s ($FORMATTED_DURATION)."\n'
)
script_file_id.write(' echo "$SEP"\n')
script_file_id.write(' send_mail wrong\n')
script_file_id.write(' exit 3\n')
script_file_id.write('}\n')
script_file_id.write(
'#-------------------------------------------------------------------------------\n'
)
process_name = 'Infrastructure software installation'
mail_message_ok = xlib.get_mail_message_ok(process_name,
cluster_name)
mail_message_wrong = xlib.get_mail_message_wrong(
process_name, cluster_name)
script_file_id.write('function send_mail\n')
script_file_id.write('{\n')
script_file_id.write(
f' SUBJECT="{xlib.get_project_name()}: {process_name}"\n')
script_file_id.write(' if [ "$1" == "ok" ]; then\n')
script_file_id.write(f' MESSAGE="{mail_message_ok}"\n')
script_file_id.write(' elif [ "$1" == "wrong" ]; then\n')
script_file_id.write(f' MESSAGE="{mail_message_wrong}"\n')
script_file_id.write(' else\n')
script_file_id.write(' MESSAGE=""\n')
script_file_id.write(' fi\n')
script_file_id.write(
' DESTINATION_FILE=mail-destination.json\n')
script_file_id.write(' echo "{" > $DESTINATION_FILE\n')
script_file_id.write(
f' echo " \\\"ToAddresses\\\": [\\\"{xconfiguration.get_contact_data()}\\\"]," >> $DESTINATION_FILE\n'
)
script_file_id.write(
' echo " \\\"CcAddresses\\\": []," >> $DESTINATION_FILE\n'
)
script_file_id.write(
' echo " \\\"BccAddresses\\\": []" >> $DESTINATION_FILE\n'
)
script_file_id.write(' echo "}" >> $DESTINATION_FILE\n')
script_file_id.write(' MESSAGE_FILE=mail-message.json\n')
script_file_id.write(' echo "{" > $MESSAGE_FILE\n')
script_file_id.write(
' echo " \\\"Subject\\\": {" >> $MESSAGE_FILE\n')
script_file_id.write(
' echo " \\\"Data\\\": \\\"$SUBJECT\\\"," >> $MESSAGE_FILE\n'
)
script_file_id.write(
' echo " \\\"Charset\\\": \\\"UTF-8\\\"" >> $MESSAGE_FILE\n'
)
script_file_id.write(' echo " }," >> $MESSAGE_FILE\n')
script_file_id.write(
' echo " \\\"Body\\\": {" >> $MESSAGE_FILE\n')
script_file_id.write(
' echo " \\\"Html\\\": {" >> $MESSAGE_FILE\n')
script_file_id.write(
' echo " \\\"Data\\\": \\\"$MESSAGE\\\"," >> $MESSAGE_FILE\n'
)
script_file_id.write(
' echo " \\\"Charset\\\": \\\"UTF-8\\\"" >> $MESSAGE_FILE\n'
)
script_file_id.write(' echo " }" >> $MESSAGE_FILE\n')
script_file_id.write(' echo " }" >> $MESSAGE_FILE\n')
script_file_id.write(' echo "}" >> $MESSAGE_FILE\n')
script_file_id.write(
f' aws ses send-email --from {xconfiguration.get_contact_data()} --destination file://$DESTINATION_FILE --message file://$MESSAGE_FILE\n'
)
script_file_id.write('}\n')
script_file_id.write(
'#-------------------------------------------------------------------------------\n'
)
script_file_id.write('function calculate_duration\n')
script_file_id.write('{\n')
script_file_id.write(
' DURATION=`expr $END_DATETIME - $INIT_DATETIME`\n')
script_file_id.write(' HH=`expr $DURATION / 3600`\n')
script_file_id.write(' MM=`expr $DURATION % 3600 / 60`\n')
script_file_id.write(' SS=`expr $DURATION % 60`\n')
script_file_id.write(
' FORMATTED_DURATION=`printf "%03d:%02d:%02d\\n" $HH $MM $SS`\n'
)
script_file_id.write('}\n')
script_file_id.write(
'#-------------------------------------------------------------------------------\n'
)
script_file_id.write('init\n')
if dataset_structure in [
xconfiguration.get_dataset_structure_singlevolume(),
xconfiguration.get_dataset_structure_none()
]:
script_file_id.write('create_dataset_structure\n')
script_file_id.write('install_awscli\n')
script_file_id.write('setup_aws\n')
script_file_id.write('fix_source_list\n')
script_file_id.write('install_xorg\n')
script_file_id.write('install_libtbb2\n')
script_file_id.write('install_libxt6\n')
script_file_id.write('install_parallel\n')
script_file_id.write('install_texlive\n')
script_file_id.write('uninstall_mysql\n')
# -- script_file_id.write( 'create_swapfile\n')
script_file_id.write('end\n')
except Exception as e:
error_list.append(f'*** EXCEPTION: "{e}".')
error_list.append(
f'*** ERROR: The file {get_infrastructure_software_installation_script()} can not be created'
)
OK = False
# return the control variable and the error list
return (OK, error_list)
def create_volume(volume_name,
volume_type,
volume_size,
terminate_indicator,
log,
function=None):
'''
Create a volume in the current zone.
'''
# initialize the control variable
OK = True
# get the volume creator name
volume_creator_name = xlib.get_volume_creator_name()
# warn that the log window must not be closed
if not isinstance(log, xlib.DevStdOut):
log.write(
'This process might take a few minutes. Do not close this window, please wait!\n'
)
# verify the volume creator is running
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Verifing if volume creator is running ...\n')
(master_state_code,
master_state_name) = xec2.get_node_state(volume_creator_name,
'master')
if master_state_code == 16:
log.write('The volume creator is running.\n')
else:
log.write(
'*** WARNING: The volume creator is not running. It will be created.\n'
)
(OK, master_state_code,
master_state_name) = xcluster.create_cluster(volume_creator_name,
volume_creator_name,
log,
function=None,
is_menu_call=False)
# get the master node identification
if OK:
node_id = xec2.get_node_id(volume_creator_name, 'master')
if node_id == '':
log.write(
'*** ERROR: The master identification of the volume creator not has been got.\n'
)
OK = False
# create the SSH client connection
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Connecting SSH client ...\n')
(OK, error_list,
ssh_client) = xssh.create_ssh_client_connection(volume_creator_name,
'master')
if OK:
log.write('The SSH client is connected.\n')
else:
for error in error_list:
log.write('{0}\n'.format(error))
# warn that the requirements are being verified
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Verifying process requirements ...\n')
# get current region and zone names
region_name = xconfiguration.get_current_region_name()
zone_name = xconfiguration.get_current_zone_name()
# verify that the zone is available
if OK:
if not xec2.is_zone_available(region_name, zone_name):
log.write(
'*** ERROR: The zone {0} is not available in the region {1}.\n'
.format(zone_name, region_name))
OK = False
# verify that the volume is not created
if OK:
if xec2.is_volume_created(volume_name, zone_name):
log.write(
'*** WARNING: The volume {0} is already created.\n'.format(
volume_name))
OK = False
# warn that the requirements are OK
if OK:
log.write('Process requirements are OK.\n')
# create the volume
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Creating volume {0} ...\n'.format(volume_name))
(OK, volume_id) = xec2.create_volume(volume_name, volume_type,
volume_size)
if OK:
log.write(
'The volume is created with the identification {0}.\n'.format(
volume_id))
else:
log.write('*** ERROR: The volume is not created.\n')
# wait for the volume status to be available
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Waiting for the volume state to be available ...\n')
i = 0
available_state_timeout = 120
while True:
i += 1
volume_status = xec2.get_volume_state(volume_name, zone_name)
time.sleep(1)
if volume_status == 'available':
log.write('The volume is now available.\n')
break
elif i > available_state_timeout:
log.write(
'*** The volume is not available after {0} s.\n'.format(
available_state_timeout))
Ok = False
break
# set the aws device and get de machine device
if OK:
aws_device = '/dev/sdp'
machine_device = xlib.get_machine_device_file(aws_device)
# attach the volume to the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write(
'Attaching volume {0} to node master of volume creator ...\n'.
format(volume_name))
OK = xec2.attach_volume(node_id, volume_id, aws_device)
if OK:
log.write('The volume is attached.\n')
else:
log.write('*** ERROR: The volume is not attached.\n')
# wait for the volume attachment to be available
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Waiting for the volume attachment to be available ...\n')
i = 0
attachment_timeout = 120
while True:
i += 1
volume_attachments = xec2.get_volume_attachments(
volume_name, zone_name)
if volume_attachments != []:
log.write('The volume attachment is now available.\n')
break
elif i > attachment_timeout:
log.write(
'*** ERROR: The volume attachment is not available after {0} s.\n'
.format(attachment_timeout))
Ok = False
break
time.sleep(1)
# wait for the device availability
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Waiting for the availabity of the device {0} ...\n'.format(
machine_device))
i = 0
format_timeout = 120
try:
while True:
i += 1
command = 'hdparm -z {0}'.format(machine_device)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
command = 'lsblk --list --noheadings --output NAME'
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
for line in stdout:
if line == os.path.basename(machine_device):
log.write('The device is available.\n')
raise xlib.BreakAllLoops
if i > format_timeout:
log.write(
'*** ERROR: The device is not available after {0} s.\n'
.format(format_timeout))
OK = False
break
time.sleep(1)
except xlib.BreakAllLoops:
pass
# format the volume
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Formating volume {0} to ext4 file system type ...\n'.format(
volume_name))
command = 'mkfs -t ext4 {0} 2>&1; RC=$?; echo "RC=$RC"'.format(
machine_device)
(OK, stdout,
stderr) = xssh.execute_cluster_command(ssh_client, command)
if stdout[len(stdout) - 1] == 'RC=0':
log.write('The volume is formatted.\n')
OK = True
else:
log.write('*** ERROR: Wrong command ---> {0}\n'.format(command))
OK = False
# detach the volume to the cluster
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write(
'Detaching volume {0} from node master of volume creator ...\n'.
format(volume_name))
OK = xec2.detach_volume(node_id, volume_id, aws_device)
if OK:
log.write('The volume is detached.\n')
else:
log.write('*** ERROR: The volume is not detached.\n')
# terminate volume creator
if OK:
if terminate_indicator:
OK = xcluster.terminate_cluster(volume_creator_name,
True,
log,
function=None,
is_menu_call=False)
else:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('You do not indicate to terminate the volume creator.\n')
log.write(
'Remember to terminate it when you finish to create the volumes!!!\n'
)
# close the SSH client connection
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Closing the SSH client connection ...\n')
xssh.close_ssh_client_connection(ssh_client)
log.write('The connection is closed.\n')
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut):
log.write('{0}\n'.format(xlib.get_separator()))
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK
def remove_volume(volume_name, log, function=None):
'''
Delete a volume in the current zone.
'''
# initialize the control variable
OK = True
# warn that the log window must not be closed
if not isinstance(log, xlib.DevStdOut):
log.write(
'This process might take a few minutes. Do not close this window, please wait!\n'
)
# warn that the requirements are being verified
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Verifying process requirements ...\n')
# get current region and zone names
region_name = xconfiguration.get_current_region_name()
zone_name = xconfiguration.get_current_zone_name()
# verify that the zone is available
if OK:
if not xec2.is_zone_available(region_name, zone_name):
log.write(
'*** ERROR: The zone {0} is not available in the region {1}.\n'
.format(zone_name, region_name))
OK = False
# verify that the volume is available
if OK:
volume_status = xec2.get_volume_state(volume_name, zone_name)
if volume_status != 'available':
log.write(
'*** ERROR: The volume {0} is not available in the zone {1}.\n'
.format(volume_name, zone_name))
OK = False
# verify that the volume is not linked to any cluster templates
if OK:
if volume_name in xconfiguration.get_volume_names_list():
log.write(
'*** ERROR: The volume is linked to some cluster templates.\n')
OK = False
# warn that the requirements are OK
if OK:
log.write('Process requirements are OK.\n')
# delete the volume
if OK:
log.write('{0}\n'.format(xlib.get_separator()))
log.write('Removing volume {0} ...\n'.format(volume_name))
OK = xec2.delete_volume(volume_name)
if OK:
log.write(
'The volume is been deleted. It may remain in the deleting state for several minutes.\n'
)
else:
log.write('*** ERROR: The volume is not deleted.\n')
# warn that the log window can be closed
if not isinstance(log, xlib.DevStdOut):
log.write('{0}\n'.format(xlib.get_separator()))
log.write('You can close this window now.\n')
# execute final function
if function is not None:
function()
# return the control variable
return OK