def connect_to_resource(resource):
"""connect to resource defined in resource dictionary"""
log.info("Validating resource accessibility. Connecting to %s.",
resource['name'])
if resource['ssh_private_key_file'] is not None and os.path.isfile(
resource['ssh_private_key_file']) is False:
log.error("Can not access ssh private key (%s)"
"", resource['ssh_private_key_file'])
exit(1)
str_io = io.StringIO()
try:
sys.stdout = sys.stderr = str_io
rsh = akrr.util.ssh.ssh_resource(resource)
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
log.info("Successfully connected to %s\n", resource['name'])
log.empty_line()
return rsh
except AkrrError:
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
log.critical("Can not connect to %s\nMessage:\n%s", resource['name'],
str_io.getvalue())
exit(1)
def validate_resource_parameter_file(resource_name):
"""validate resource parameter file and return dictionary with resource configuration"""
# @todo reuse cfg.verify_resource_params
default_resource_param_filename = os.path.join(cfg.akrr_mod_dir,
"default_conf",
"default.resource.conf")
resource_param_filename = os.path.join(cfg.cfg_dir, "resources",
resource_name, "resource.conf")
log.info("Validating %s parameters from %s", resource_name,
resource_param_filename)
if not os.path.isfile(resource_param_filename):
log.error("resource parameters file (%s) does not exist!",
resource_param_filename)
exit(1)
# check syntax
try:
tmp = {}
exec(
compile(
open(default_resource_param_filename).read(),
default_resource_param_filename, 'exec'), tmp)
exec(
compile(
open(resource_param_filename).read(), resource_param_filename,
'exec'), tmp)
except Exception as e:
log.critical(
"Can not load resource from %s.\nProbably invalid syntax.",
resource_param_filename)
raise e
resource = None
try:
# now we can load akrr, parameters checking did h
resource = cfg.find_resource_by_name(resource_name)
except Exception as e:
log.error("Can not load resource config from %s!\n%s\n%s",
resource_param_filename, str(e), traceback.format_exc())
exit(1)
log.info(
"Syntax of %s is correct and all necessary parameters are present.",
resource_param_filename)
log.empty_line()
return resource
def ask(header, validate: Callable[[str], bool] = None, default: str = None):
"""
Ask user question, validate if requested and return entered value
"""
import akrr.util.log as log
from akrr.util import smart_str_merge
prompt = "[%s] " % default if default is not None else ""
while True:
log.log_input(header + ":")
user_input = input(prompt)
if default is not None and user_input == "":
user_input = default
if validate(user_input):
log.empty_line()
return user_input
else:
log.error("Incorrect value try again")
def multiple_choice(header,
choices: Sequence[str],
default: str = None) -> str:
"""
Ask user multiple choice question, check for correctness and return entered value
"""
import akrr.util.log as log
from akrr.util import smart_str_merge
prompt = "[%s] " % default if default is not None else ""
while True:
log.log_input(header + " (%s):", smart_str_merge(choices))
user_input = input(prompt)
if default is not None and user_input == "":
user_input = default
if user_input in choices:
log.empty_line()
return user_input
else:
log.error("Incorrect value try again")
def check_create_dirs(rsh, resource):
log.info("Checking directory locations\n")
d = resource['akrr_data']
log.info("Checking: %s:%s", resource['remote_access_node'], d)
status, msg = check_dir(rsh, d, exit_on_fail=True, try_to_create=True)
log.info(msg)
d = resource['appkernel_dir']
log.info("Checking: %s:%s", resource['remote_access_node'], d)
status, msg = check_dir(rsh, d, exit_on_fail=True, try_to_create=True)
log.info(msg)
d = resource['network_scratch']
log.info("Checking: %s:%s", resource['remote_access_node'], d)
status, msg = check_dir(rsh, d, exit_on_fail=False, try_to_create=True)
if status is True:
log.info(msg)
else:
log.warning_count += 1
log.warning(msg)
log.warning(
"WARNING %d: network scratch might be have a different location on head node, "
"so if it is by design it is ok", log.warning_count)
d = resource['local_scratch']
log.info("Checking: %s:%s", resource['remote_access_node'], d)
status, msg = check_dir(rsh, d, exit_on_fail=False, try_to_create=False)
if status is True:
log.info(msg)
else:
log.warning_count += 1
log.warning(msg)
log.warning(
"WARNING %d: local scratch might be have a different location on head node, "
"so if it is by design it is ok", log.warning_count)
log.empty_line()
def read_db_user_credentials(self):
"""
Get DB access user credential
"""
log.info(
"Before Installation continues we need to setup the database.")
#######################
# mod_akrr
self.akrr_db_user_name, self.akrr_db_user_password = _read_username_password(
"Please specify a database user to access mod_akrr database (Used by AKRR)"
"(This user will be created if it does not already exist):",
self.akrr_db_user_name, self.akrr_db_user_password,
self.default_akrr_user)
log.empty_line()
# check if user, db already there
user_exists, db_exists, user_rights_are_correct = AKRRSetup._check_user_db_priv_on_dbserver(
self.akrr_db_user_name, self.akrr_db_user_password,
self.akrr_db_host, self.akrr_db_port, self.akrr_db_name, "ALL")
# ask for su user on this machine if needed
if not user_exists or not db_exists or not user_rights_are_correct:
self.akrr_db_su_user_name, self.akrr_db_su_user_password = \
_check_and_read_su_credentials_for_dbserver(None, None, self.akrr_db_host, self.akrr_db_port)
log.empty_line()
#######################
# mod_appkernel
same_host_as_ak = self.ak_db_host == self.akrr_db_host and self.ak_db_port == self.akrr_db_port
self.ak_db_user_name, self.ak_db_user_password = _read_username_password(
"Please specify a database user to access mod_appkernel database "
"(Used by XDMoD appkernel module, AKRR creates and syncronize resource and appkernel description)"
"(This user will be created if it does not already exist):",
self.ak_db_user_name, self.ak_db_user_password,
self.akrr_db_user_name,
self.akrr_db_user_password if same_host_as_ak else None)
log.empty_line()
# check if user, db already there
user_exists, db_exists, user_rights_are_correct = AKRRSetup._check_user_db_priv_on_dbserver(
self.ak_db_user_name, self.ak_db_user_password, self.ak_db_host,
self.ak_db_port, self.ak_db_name, "ALL")
# ask for su user on this machine
if not user_exists or not db_exists or not user_rights_are_correct:
self.ak_db_su_user_name, self.ak_db_su_user_password = \
_check_and_read_su_credentials_for_dbserver(self.akrr_db_su_user_name, self.akrr_db_su_user_password,
self.ak_db_host, self.ak_db_port)
log.empty_line()
#######################
# modw
same_host_as_xd = self.xd_db_host == self.ak_db_host and self.xd_db_port == self.ak_db_port
self.xd_db_user_name, self.xd_db_user_password = _read_username_password(
"Please specify the user that will be connecting to the XDMoD database (modw):",
self.xd_db_user_name, self.xd_db_user_password,
self.ak_db_user_name,
self.ak_db_user_password if same_host_as_xd else None)
log.empty_line()
# check if user, db already there
user_exists, db_exists, user_rights_are_correct = AKRRSetup._check_user_db_priv_on_dbserver(
self.xd_db_user_name, self.xd_db_user_password, self.xd_db_host,
self.xd_db_port, self.xd_db_name, "ALL")
# ask for su user on this machine
if not user_exists or not db_exists or not user_rights_are_correct:
self.xd_db_su_user_name, self.xd_db_su_user_password = \
_check_and_read_su_credentials_for_dbserver(self.akrr_db_su_user_name, self.akrr_db_su_user_password,
self.ak_db_host, self.ak_db_port)
log.empty_line()
def resource_deploy(args):
global checking_frequency
resource_name = args.resource
if 'dry_run' in args:
akrr.dry_run = args.dry_run
else:
akrr.dry_run = False
if "checking_frequency" in args:
checking_frequency = args.checking_frequency
if "appkernel" in args:
app_name = args.appkernel
else:
app_name = "test"
if "nodes" in args:
nodes = int(args.nodes)
else:
nodes = 2
log.error_count = 0
log.warning_count = 0
# validate resource configuration and get config
resource = validate_resource_parameter_file(resource_name)
# connect to resource
if resource['batch_scheduler'].lower() == "openstack":
# Start instance if it is cloud
openstack_server = akrr.util.openstack.OpenStackServer(
resource=resource)
resource['openstack_server'] = openstack_server
openstack_server.create()
resource['remote_access_node'] = openstack_server.ip
rsh = connect_to_resource(resource)
# do tests
check_shell(rsh, resource)
check_create_dirs(rsh, resource)
# deploy inputs and sources
copy_exec_sources_and_inputs(rsh, resource)
# check that app.signature calculator on headnode
check_appsig(rsh, resource)
# close connection we don't need it any more
rsh.close(force=True)
del rsh
if resource['batch_scheduler'].lower() == "openstack":
# delete instance if it is cloud
akrr.util.openstack.OpenStackServer(resource=resource)
resource['openstack_server'].delete()
resource['remote_access_node'] = None
# run test job to queue
run_test_job(resource, app_name, nodes)
if resource['batch_scheduler'].lower() == "openstack":
# Start instance if it is cloud
openstack_server = akrr.util.openstack.OpenStackServer(
resource=resource)
resource['openstack_server'] = openstack_server
openstack_server.create()
resource['remote_access_node'] = openstack_server.ip
if log.error_count == 0:
append_to_bashrc(resource)
enable_resource_for_execution(resource)
if resource['batch_scheduler'].lower() == "openstack":
# delete instance if it is cloud
akrr.util.openstack.OpenStackServer(resource=resource)
resource['openstack_server'].delete()
resource['remote_access_node'] = None
log.empty_line()
log.info("Result:")
if log.error_count > 0:
log.error("There are %d errors, fix them.", log.error_count)
if log.warning_count > 0:
log.warning(
"There are %d warnings.\nif warnings have sense you can move to next step!\n",
log.warning_count)
if log.error_count == 0 and log.warning_count == 0:
log.info("\nDONE, you can move to next step!\n")
def resource_add(config):
"""add resource, config should have following members
dry_run - Dry Run No files will actually be created
minimalistic - Minimize questions number, configuration files will be edited manually
no-ping - do not run ping to test headnode name
verbose
"""
global verbose
global no_ping
global minimalistic
global resource_name
global remote_access_node
global remote_access_method
global remote_copy_method
global ssh_username
global ssh_password
global ssh_private_key_file
global ssh_private_key_password
global network_scratch
global local_scratch
global akrr_data
global appkernel_dir
global batch_scheduler
global batch_job_header_template
if config.verbose:
verbose = True
log.info("Beginning Initiation of New Resource...")
verbose = config.verbose
akrr.dry_run = config.dry_run
no_ping = config.no_ping
minimalistic = config.minimalistic
log.info("Retrieving Resources from XDMoD Database...")
# RETRIEVE: the resources from XDMoD
resources = retrieve_resources_from_xdmod()
log.info("Found following resources from XDMoD Database:\n" +
" resource_id name\n" + "\n".join([
" %11d %-40s" % (resource_id, resource_name)
for resource_name, resource_id in resources
]) + "\n")
if len(resources) > 0:
while True:
log.log_input(
'Enter resource_id for import (enter 0 for no match):')
resource_id = input()
if validate_resource_id(resource_id, resources):
break
log.warning("Incorrect resource_id try again")
log.empty_line()
resource_id = int(resource_id)
else:
resource_id = 0
if resource_id <= 0: # i.e. no match from XDMoD DB
resource_id = None
resource_name = ""
while True:
if resource_id is None:
log.log_input('Enter AKRR resource name:')
resource_name = input()
else:
resource_name2 = get_resource_name_by_id(resource_id, resources)
log.log_input(
'Enter AKRR resource name, hit enter to use same name as in XDMoD Database [%s]:'
% (resource_name2, ))
resource_name = input()
if resource_name.strip() == "":
resource_name = resource_name2
if validate_resource_name(resource_name):
break
log.empty_line()
while True:
log.log_input(
'Enter queuing system on resource (slurm, pbs or openstack): ')
queuing_system = input()
if validate_queuing_system(queuing_system):
break
else:
log.error("Incorrect queuing_system try again")
batch_scheduler = queuing_system
log.empty_line()
if minimalistic is False:
get_remote_access_method()
get_system_characteristics()
get_file_system_access_points()
log.debug(
"Summary of parameters" + "resource_name: {}".format(resource_name) +
"remote_access_node: {}".format(remote_access_node) +
"remote_access_method: {}".format(remote_access_method) +
"remote_copy_method: {}".format(remote_copy_method) +
"ssh_username: {}".format(ssh_username) +
"ssh_password: {}".format(ssh_password) +
"ssh_private_key_file: {}".format(ssh_private_key_file) +
"ssh_private_key_password: {}".format(ssh_private_key_password) +
"network_scratch: {}".format(network_scratch) +
"local_scratch: {}".format(local_scratch) +
"akrr_data: {}".format(akrr_data) +
"appkernel_dir: {}".format(appkernel_dir) +
"batch_scheduler: {}".format(batch_scheduler) +
"batch_job_header_template: {}".format(batch_job_header_template) +
"\n")
generate_resource_config(resource_id, resource_name, queuing_system)
log.info("Initiation of new resource is completed.\n"
" Edit batch_job_header_template variable in {}\n"
" and move to resource validation and deployment step.\n"
" i.e. execute:\n"
" akrr resource deploy -r {}".format(
resource_cfg_filename, resource_name))
def get_file_system_access_points():
global resource_name
global network_scratch
global local_scratch
global akrr_data
global appkernel_dir
home_dir = akrr.util.ssh.ssh_command(rsh, "echo $HOME").strip()
scratch_network_dir = akrr.util.ssh.ssh_command(rsh,
"echo $SCRATCH").strip()
# local_scratch
local_scratch_default = "/tmp"
while True:
log.log_input(
"Enter location of local scratch (visible only to single node):")
local_scratch = input("[%s]" % local_scratch_default)
if local_scratch.strip() == "":
local_scratch = local_scratch_default
status, msg = check_dir_simple(rsh, local_scratch)
if status:
log.info(msg)
log.empty_line()
break
else:
log.warning(msg)
log.warning(
'local scratch might be have a different location on head node, so if it is by design it is ok'
)
log.empty_line()
break
local_scratch = akrr.util.ssh.ssh_command(rsh, "echo %s" %
(local_scratch, )).strip()
# network_scratch
network_scratch_default = ""
if scratch_network_dir != "":
network_scratch_default = scratch_network_dir
network_scratch_visible = False
while True:
log.log_input(
"Enter location of network scratch (visible only to all nodes),"
"used for temporary storage of app kernel input/output:")
if network_scratch_default != "":
network_scratch = input("[%s]" % network_scratch_default)
if network_scratch.strip() == "":
network_scratch = network_scratch_default
else:
network_scratch = input("")
if network_scratch == "":
log.error("Incorrect value for network_scratch, try again")
continue
status, msg = check_dir(rsh,
network_scratch,
exit_on_fail=False,
try_to_create=True)
if status:
log.info(msg)
network_scratch_visible = True
log.empty_line()
break
else:
log.warning(msg)
break
network_scratch = akrr.util.ssh.ssh_command(
rsh, "echo %s" % (network_scratch, )).strip()
# appkernel_dir
appker_dir_default = os.path.join(home_dir, "appker", resource_name)
while True:
log.log_input(
"Enter future location of app kernels input and executable files:")
appkernel_dir = input("[%s]" % appker_dir_default)
if appkernel_dir.strip() == "":
appkernel_dir = appker_dir_default
status, msg = check_dir(rsh,
appkernel_dir,
exit_on_fail=False,
try_to_create=True)
if status:
log.info(msg)
log.empty_line()
break
else:
log.error(msg)
appkernel_dir = akrr.util.ssh.ssh_command(rsh, "echo %s" %
(appkernel_dir, )).strip()
# akrr_data
akrr_data_default = os.path.join(home_dir, "akrr_data", resource_name)
if network_scratch_visible:
akrr_data_default = os.path.join(network_scratch, "akrr_data",
resource_name)
while True:
log.log_input(
"Enter future locations for app kernels working directories (can or even should be on scratch space):"
)
akrr_data = input("[%s]" % akrr_data_default)
if akrr_data.strip() == "":
akrr_data = akrr_data_default
status, msg = check_dir(rsh,
akrr_data,
exit_on_fail=False,
try_to_create=True)
if status:
log.info(msg)
log.empty_line()
break
else:
log.error(msg)
akrr_data = akrr.util.ssh.ssh_command(rsh,
"echo %s" % (akrr_data, )).strip()
def get_remote_access_method():
global resource_name
global remote_access_node
global remote_access_method
global remote_copy_method
global ssh_username
global ssh_password
global ssh_password4thisSession
global ssh_private_key_file
global ssh_private_key_password
global rsh
global no_ping
# set remote_access_node
while True:
log.log_input(
"Enter Resource head node (access node) full name (e.g. headnode.somewhere.org):"
)
remote_access_node = input("[%s] " % resource_name)
if remote_access_node.strip() == "":
remote_access_node = resource_name
response = os.system("ping -c 1 -w2 " + remote_access_node +
" > /dev/null 2>&1")
if response == 0:
break
else:
if no_ping:
log.warning("Can not ping %s, but asked to ignore it.",
remote_access_node)
break
log.error("Incorrect head node name (can not ping %s), try again",
remote_access_node)
# set ssh_username
current_user = getpass.getuser()
ask_for_user_name = True
while True:
if ask_for_user_name:
log.log_input("Enter username for resource access:")
ssh_username = input("[%s] " % current_user)
if ssh_username.strip() == "":
ssh_username = current_user
current_user = ssh_username
# check password-less access
if ssh_password is None:
log.info("Checking for password-less access")
else:
log.info("Checking for resource access")
successfully_connected = check_connection_to_resource()
if successfully_connected:
if ssh_password is None:
log.info("Can access resource without password")
else:
log.info("Can access resource")
if successfully_connected is False:
log.info("Can not access resource without password")
action_list = [(
"TryAgain",
"The private and public keys was generated manually, right now. Try again."
)]
# check private keys
user_home_dir = os.path.expanduser("~")
user_ssh_dir = os.path.join(user_home_dir, '.ssh')
if os.path.isdir(user_ssh_dir):
private_keys = [
os.path.join(user_ssh_dir, f[:-4])
for f in os.listdir(user_ssh_dir)
if os.path.isfile(os.path.join(user_ssh_dir, f))
and f[-4:] == '.pub'
and os.path.isfile(os.path.join(user_ssh_dir, f[:-4]))
]
else:
private_keys = []
if len(private_keys) > 0:
action_list.append(("UseExistingPrivateKey",
"Use existing private and public key."))
default_action = len(action_list)
action_list.append(
("GenNewKey", "Generate new private and public key."))
action_list.append(("UsePassword", "Use password directly."))
log.empty_line()
log.info("Select authentication method:\n" + "\n".join([
"%3d %s" % (i, desc)
for i, (_, desc) in enumerate(action_list)
]))
while True:
log.log_input("Select option from list above:")
try:
action = input("[%s] " % default_action)
if action.strip() == "":
action = default_action
else:
action = int(action)
if action < 0 or action >= len(action_list):
raise ValueError()
break
except (ValueError, TypeError):
log.error("Incorrect entry, try again.")
# do the action
log.empty_line()
if action_list[action][0] == "TryAgain":
continue
if action_list[action][0] == "UsePassword":
log.log_input("Enter password for %s@%s:" %
(ssh_username, remote_access_node))
ssh_password = getpass.getpass("")
ask_for_user_name = not ask_for_user_name
continue
if action_list[action][0] == "UseExistingPrivateKey":
log.info("Available private keys:" + "\n".join(
["%3d %s" % (i, p) for i, p in enumerate(private_keys)]))
while True:
log.log_input("Select key number from list above:")
try:
i_key = input("")
i_key = int(i_key)
if i_key < 0 or i_key >= len(private_keys):
raise ValueError()
break
except (ValueError, TypeError):
log.error("Incorrect entry, try again.")
ssh_private_key_file = private_keys[i_key]
ask_for_user_name = not ask_for_user_name
continue
if action_list[action][0] == "GenNewKey":
count = 0
while True:
log.log_input(
"Enter password for %s@%s (will be used only during this session):"
% (ssh_username, remote_access_node))
ssh_password4thisSession = getpass.getpass("")
ssh_password = ssh_password4thisSession
if check_connection_to_resource():
break
count += 1
if count >= 3:
break
# generate keys
log.log_input("Enter private key name:")
ssh_private_key_file = input("[id_rsa_%s]" % resource_name)
if ssh_private_key_file.strip() == "":
ssh_private_key_file = "id_rsa_%s" % resource_name
ssh_private_key_file = os.path.join(user_home_dir, '.ssh',
ssh_private_key_file)
log.log_input(
"Enter passphrase for new key (leave empty for passwordless access):"
)
ssh_private_key_password = getpass.getpass("")
if akrr.dry_run:
successfully_connected = True
else:
ssh_password = None
os.system("ssh-keygen -t rsa -N \"%s\" -f %s" %
(ssh_private_key_password, ssh_private_key_file))
if ssh_private_key_password.strip() == "":
ssh_private_key_password = None
# copy keys
akrr.util.ssh.ssh_access(
remote_access_node,
ssh='ssh-copy-id',
username=ssh_username,
password=ssh_password4thisSession,
private_key_file=ssh_private_key_file,
private_key_password=None,
logfile=sys.stdout,
command='')
ask_for_user_name = not ask_for_user_name
continue
if successfully_connected:
break
else:
log.error("Incorrect resource access credential")
if successfully_connected:
log.empty_line()
log.info("Connecting to " + resource_name)
str_io = io.StringIO()
try:
sys.stdout = sys.stderr = str_io
rsh = akrr.util.ssh.ssh_access(
remote_access_node,
ssh=remote_access_method,
username=ssh_username,
password=ssh_password,
private_key_file=ssh_private_key_file,
private_key_password=ssh_private_key_password,
logfile=sys.stdout,
command=None)
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
except Exception as e:
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
log.debug(str_io.getvalue())
raise e
log.info(" Done")
log.empty_line()
return successfully_connected
def check_connection_to_resource():
"""check the connection to remote resource."""
global remote_access_node
global remote_access_method
global remote_copy_method
global ssh_username
global ssh_password
global ssh_password4thisSession
global ssh_private_key_file
global ssh_private_key_password
successfully_connected = False
passphrase_entrance_count = 0
authorize_key_count = 0
while True:
# Try to connect
str_io = io.StringIO()
try:
sys.stdout = sys.stderr = str_io
akrr.util.ssh.ssh_access(
remote_access_node,
ssh=remote_access_method,
username=ssh_username,
password=ssh_password,
private_key_file=ssh_private_key_file,
private_key_password=ssh_private_key_password,
logfile=str_io,
command='ls')
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
successfully_connected = True
break
except Exception:
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
response = str_io.getvalue()
log.debug(
"Had attempted to access resource without password and failed, below is resource response"
+ "=" * 80 + str_io.getvalue() + "=" * 80)
# check if it asking for passphrase
m = re.search(r"Enter passphrase for key '(.*)':", response)
if m:
if passphrase_entrance_count >= 3:
ssh_private_key_password = None
ssh_private_key_file = None
break
if passphrase_entrance_count > 0:
log.error("Incorrect passphrase try again")
ssh_private_key_file = m.group(1)
log.log_input("Enter passphrase for key '%s':" %
ssh_private_key_file)
ssh_private_key_password = getpass.getpass("")
passphrase_entrance_count += 1
continue
m2 = re.search(r"[pP]assword:", response)
if m is None and ssh_private_key_file is not None and m2:
log.warning(
"Can not login to head node. "
"Probably the public key of private key was not authorized on head node"
)
log.info(
"Will try to add public key to list of authorized keys on head node"
)
while True:
try:
authorize_key_count += 1
log.log_input(
"Enter password for %s@%s (will be used only during this session):"
% (ssh_username, remote_access_node))
ssh_password4thisSession = getpass.getpass("")
log.empty_line()
str_io = io.StringIO()
sys.stdout = sys.stderr = str_io
akrr.util.ssh.ssh_access(
remote_access_node,
ssh='ssh-copy-id',
username=ssh_username,
password=ssh_password4thisSession,
private_key_file=ssh_private_key_file,
private_key_password=None,
logfile=str_io,
command='')
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
log.info(response)
log.info(
"Have added public key to list of authorized keys on head node, "
"will attempt to connect again.")
log.empty_line()
break
except Exception:
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
if verbose:
log.debug(
"Had attempted to add public key to list of authorized keys on head node and failed, "
+ "below is resource response" + "=" * 80 +
str_io.getvalue() + "=" * 80)
log.error("Incorrect password try again.")
if authorize_key_count >= 3:
break
if authorize_key_count < 3:
continue
break
return successfully_connected
def resource_add(config):
"""add resource, config should have following members
dry_run - Dry Run No files will actually be created
minimalistic - Minimize questions number, configuration files will be edited manually
no-ping - do not run ping to test headnode name
verbose
"""
global verbose
global no_ping
global minimalistic
global resource_name
global remote_access_node
global remote_access_method
global remote_copy_method
global ssh_username
global ssh_password
global ssh_private_key_file
global ssh_private_key_password
global network_scratch
global local_scratch
global akrr_data
global appkernel_dir
global batch_scheduler
global batch_job_header_template
if config.verbose:
verbose = True
log.info("Beginning Initiation of New Resource...")
verbose = config.verbose
akrr.dry_run = config.dry_run
no_ping = config.no_ping
minimalistic = config.minimalistic
log.info("Retrieving Resources from XDMoD Database...")
# RETRIEVE: the resources from XDMoD
resources = retrieve_resources_from_xdmod()
log.info("Found following resources from XDMoD Database:\n" +
" resource_id name\n" + "\n".join([
" %11d %-40s" % (resource_id, resource_name)
for resource_name, resource_id in resources
]) + "\n")
resource_id = None
if len(resources) > 0:
while True:
log.log_input(
'Enter resource_id for import (enter None for no match):')
resource_id = input()
if validate_resource_id(resource_id, resources):
break
log.warning("Incorrect resource_id try again")
log.empty_line()
if resource_id != "None":
resource_id = int(resource_id)
else:
resource_id = None
resource_name = ask.ask('Enter AKRR resource name',
validate=validate_resource_name,
default=None if resource_id is None else
get_resource_name_by_id(resource_id, resources))
batch_scheduler = ask.multiple_choice_enum(
'Enter queuing system on resource', QueuingSystemType).value
if minimalistic is False:
if batch_scheduler is QueuingSystemType.openstack.value:
_get_openstack_details()
get_system_characteristics()
elif batch_scheduler is QueuingSystemType.googlecloud.value:
_get_googlecloud_details()
get_system_characteristics()
else:
get_remote_access_method()
get_system_characteristics()
get_file_system_access_points()
log.debug(
"Summary of parameters" + "resource_name: {}".format(resource_name) +
"remote_access_node: {}".format(remote_access_node) +
"remote_access_method: {}".format(remote_access_method) +
"remote_copy_method: {}".format(remote_copy_method) +
"ssh_username: {}".format(ssh_username) +
"ssh_password: {}".format(ssh_password) +
"ssh_private_key_file: {}".format(ssh_private_key_file) +
"ssh_private_key_password: {}".format(ssh_private_key_password) +
"network_scratch: {}".format(network_scratch) +
"local_scratch: {}".format(local_scratch) +
"akrr_data: {}".format(akrr_data) +
"appkernel_dir: {}".format(appkernel_dir) +
"batch_scheduler: {}".format(batch_scheduler) +
"batch_job_header_template: {}".format(batch_job_header_template) +
"\n")
generate_resource_config(resource_id, resource_name, batch_scheduler)
log.info("Initiation of new resource is completed.\n"
" Edit batch_job_header_template variable in {}\n"
" and move to resource validation and deployment step.\n"
" i.e. execute:\n"
" akrr resource deploy -r {}".format(
resource_cfg_filename, resource_name))
def read_db_user_credentials(self):
"""
Get DB access user credential
"""
# mod_akrr
log.info("Before Installation continues we need to setup the database.")
self.akrr_db_user_name, self.akrr_db_user_password = _read_username_password(
"Please specify a database user to access mod_akrr database (Used by AKRR)"
"(This user will be created if it does not already exist):",
self.akrr_db_user_name,
self.akrr_db_user_password,
self.default_akrr_user)
log.empty_line()
# check if user, db already there
db_exists = False
user_rights_are_correct = False
try:
# connect with provided user, Exception will raise if user can not connect
_, cur = get_con_to_db(self.akrr_db_user_name, self.akrr_db_user_password, self.akrr_db_host,
self.akrr_db_port)
client_host = get_db_client_host(cur)
user_exists = True
db_exists = db_exist(cur, self.akrr_db_name)
if not db_exists:
log.debug("Database {} doesn't exists on {}".format(self.akrr_db_name, self.akrr_db_host))
user_rights_are_correct = db_check_priv(cur, self.akrr_db_name, "ALL", self.akrr_db_user_name, client_host)
if not user_rights_are_correct:
log.debug(
"User {} doesn't have right privilege on {}, should be ALL".format(
self.akrr_db_user_name, self.akrr_db_name))
except MySQLdb.Error:
user_exists = False
log.debug("User ({}) does not exists on {}".format(self.akrr_db_user_name, self.akrr_db_host))
# ask for su user on this machine if needed
if not user_exists or not db_exists or not user_rights_are_correct:
self.akrr_db_su_user_name, \
self.akrr_db_su_user_password = _read_sql_su_credentials(self.akrr_db_host, self.akrr_db_port)
log.empty_line()
###
# mod_appkernel
same_host_as_ak = self.ak_db_host == self.akrr_db_host and self.ak_db_port == self.akrr_db_port
self.ak_db_user_name, self.ak_db_user_password = _read_username_password(
"Please specify a database user to access mod_appkernel database "
"(Used by XDMoD appkernel module, AKRR creates and syncronize resource and appkernel description)"
"(This user will be created if it does not already exist):",
self.ak_db_user_name,
self.ak_db_user_password,
self.akrr_db_user_name,
self.akrr_db_user_password if same_host_as_ak else None
)
log.empty_line()
# ask for su user on this machine
db_exists = False
user_rights_are_correct = False
try:
_, cur = get_con_to_db(self.ak_db_user_name, self.ak_db_user_password, self.ak_db_host, self.ak_db_port)
client_host = get_db_client_host(cur)
user_exists = True
db_exists = db_exist(cur, self.ak_db_name)
if not db_exists:
log.debug("Database {} doesn't exists on {}".format(self.ak_db_name, self.ak_db_host))
user_rights_are_correct = db_check_priv(cur, self.ak_db_name, "ALL", self.ak_db_user_name, client_host)
if not user_rights_are_correct:
log.debug(
"User {} doesn't have right privelege on {}, should be ALL".format(
self.ak_db_user_name, self.ak_db_name))
except Exception:
user_exists = False
log.debug("User ({}) does not exists on {}".format(self.akrr_db_user_name, self.akrr_db_host))
if not user_exists or not db_exists or not user_rights_are_correct:
self.ak_db_su_user_name = self.akrr_db_su_user_name
self.ak_db_su_user_password = self.akrr_db_su_user_password
try:
get_con_to_db(self.ak_db_su_user_name, self.ak_db_su_user_password, self.ak_db_host, self.ak_db_port)
except Exception:
self.ak_db_su_user_name, \
self.ak_db_su_user_password = _read_sql_su_credentials(self.ak_db_host, self.ak_db_port)
log.empty_line()
##
# modw
same_host_as_xd = self.xd_db_host == self.ak_db_host and self.xd_db_port == self.ak_db_port
self.xd_db_user_name, self.xd_db_user_password = _read_username_password(
"Please specify the user that will be connecting to the XDMoD database (modw):",
self.xd_db_user_name,
self.xd_db_user_password,
self.ak_db_user_name,
self.ak_db_user_password if same_host_as_xd else None
)
log.empty_line()
# ask for su user on this machine
db_exists = False
user_rights_are_correct = False
try:
_, cur = get_con_to_db(self.xd_db_user_name, self.xd_db_user_password, self.xd_db_host, self.xd_db_port)
client_host = get_db_client_host(cur)
user_exists = True
db_exists = db_exist(cur, "modw")
if not db_exists:
log.debug("Database {} doesn't exists on {}".format(self.xd_db_name, self.xd_db_host))
user_rights_are_correct = db_check_priv(cur, self.xd_db_name, "SELECT", self.xd_db_user_name, client_host)
if not user_rights_are_correct:
log.debug(
"User {} doesn't have right privelege on {}, should be at least SELECT".format(
self.xd_db_user_name, self.xd_db_name))
except Exception:
user_exists = False
log.debug("User ({}) does not exists on {}".format(self.xd_db_user_name, self.xd_db_host))
if not user_exists or not db_exists or not user_rights_are_correct:
self.xd_db_su_user_name = self.ak_db_su_user_name
self.xd_db_su_user_password = self.ak_db_su_user_password
try:
get_con_to_db(self.xd_db_su_user_name, self.xd_db_su_user_password, self.xd_db_host, self.xd_db_port)
except Exception:
self.ak_db_su_user_name, \
self.ak_db_su_user_password = _read_sql_su_credentials(self.xd_db_host, self.xd_db_port)
log.empty_line()
