def _send_su_user_password(bash, user, password):
if user is None:
return
# set AKRR database root user
if testInvalidAdministrativeDatabaseUser:
log.info("Entering invalid administrative database user")
bash.expectSendline(
r'.*INPUT.* Please provide an administrative database user.*\nUsername:'******'.*INPUT.* Please provide the password.*\n',
"invalid",
timeout=fast_timeout)
bash.justExpect(
r'.*ERROR.* Entered credential is not valid. Please try again.',
timeout=fast_timeout)
log.info("\nEntering valid administrative database user")
bash.expectSendline(
r'.*INPUT.* Please provide an administrative database user.*\nUsername:'******'.*INPUT.* Please provide the password.*\n',
password,
timeout=fast_timeout)
def connect_to_resource(resource):
"""connect to resource defined in resource dictionary"""
log.info("Validating resource accessibility. Connecting to %s.",
resource['name'])
if resource['sshPrivateKeyFile'] is not None and os.path.isfile(
resource['sshPrivateKeyFile']) is False:
log.error("Can not access ssh private key (%s)"
"", resource['sshPrivateKeyFile'])
exit(1)
str_io = io.StringIO()
try:
sys.stdout = sys.stderr = str_io
rsh = cfg.sshResource(resource)
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
log.info("Successfully connected to %s\n", resource['name'])
log.empty_line()
return rsh
except akrrError:
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
log.critical("Can not connect to %s\nMessage:\n%s", resource['name'],
str_io.getvalue())
exit(1)
def run(self):
# check
self.check_utils()
self.check_previous_installation()
# ask info
self.read_db_user_credentials()
if self.install_cron_scripts_flag:
self.ask_cron_email()
# if it is dry_run
# all question are asked, this is dry run, so nothing else to do")
self.init_mysql_dbs()
self.init_dir()
self.generate_self_signed_certificate()
self.generate_settings_file()
self.set_permission_on_files()
self.db_check()
self.generate_tables()
self.start_daemon()
self.check_daemon()
if self.install_cron_scripts_flag:
self.install_cron_scripts()
if in_src_install:
self.update_bashrc()
log.info("AKRR is set up and is running.")
def _remove_dir(path):
if os.path.exists(path):
log.info("Deleting " + path)
if not dry_run:
shutil.rmtree(path)
else:
log.info("Directory " + path + " doesn't exists.")
def _send_user_password(bash,
expect,
user,
password,
test_mismatching_password=True):
bash.expectSendline(expect, user, timeout=fast_timeout)
bash.justExpect("\n")
if password is None:
return
if test_mismatching_password:
log.info("Entering not matching password")
bash.expectSendline(r'.*INPUT.* Please specify a password.*\n',
'password',
timeout=fast_timeout)
bash.expectSendline(r'.*INPUT.* Please reenter the password.*\n',
'not_matching_password',
timeout=fast_timeout)
bash.justExpect(
r'.*ERROR.* Entered passwords do not match. Please try again.',
timeout=fast_timeout)
log.info("\nEntering matching password")
bash.expectSendline(r'.*INPUT.* Please specify a password.*\n',
password,
timeout=fast_timeout)
bash.expectSendline(r'.*INPUT.* Please reenter the password.*\n',
password,
timeout=fast_timeout)
def generate_tables(self):
log.info("Creating tables and populating them with initial values.")
from .generate_tables import create_and_populate_mod_akrr_tables, create_and_populate_mod_appkernel_tables
create_and_populate_mod_akrr_tables(dry_run)
create_and_populate_mod_appkernel_tables(dry_run)
def _create_db_user_gran_priv_if_needed(con_fun, user, password,
db, priv):
log.info("Creating %s and user to access it if needed" %
(db, ))
su_con, su_cur = con_fun(True, None)
client_host = get_db_client_host(su_cur)
_cursor_execute(
su_cur, "CREATE DATABASE IF NOT EXISTS %s" % (cv(db), ))
su_cur.execute(
"SELECT * FROM mysql.user WHERE User=%s AND Host=%s",
(user, client_host))
if len(su_cur.fetchall()) == 0:
# Older version of MySQL do not support CREATE USER IF NOT EXISTS
# so need to do checking
_cursor_execute(su_cur,
"CREATE USER %s@%s IDENTIFIED BY %s",
(user, client_host, password))
_cursor_execute(
su_cur,
"GRANT " + cv(priv) + " ON " + cv(db) + ".* TO %s@%s",
(user, client_host))
su_con.commit()
def _read_username_password(prompt="Enter username:"******"user",
password_on_default_user=None):
log.log_input(prompt)
if username is None:
username = input('[{0}] '.format(default_username))
if username == '':
username = default_username
else:
log.info("User, " + username + ", already entered.")
if username == default_username and password is None and password_on_default_user is not None:
password = password_on_default_user
if password is None:
while True:
log.log_input("Please specify a password:"******"Please reenter the password:"******"Entered passwords do not match. Please try again.")
else:
log.info("Password already entered.")
return username, password
def run_it(_):
from .util import print_important_env
print_important_env()
log.info("AKRR Setup")
_config_setup()
setup()
def check_daemon(self):
"""Check that the daemon is running"""
log.info("Checking that AKRR daemon is running")
if dry_run: return
akrr_cli = os.path.join(akrr_bin_dir, 'akrr')
status = subprocess.call(akrr_cli + " daemon check", shell=True)
if status != 0: exit(status)
def start_daemon(self):
"""Start the daemon"""
log.info("Starting AKRR daemon")
if dry_run: return
akrr_cli = os.path.join(akrr_bin_dir, 'akrr')
status = subprocess.call(akrr_cli + " daemon start", shell=True)
if status != 0: exit(status)
def db_check(self):
log.info("Checking acces to DBs.")
if dry_run: return
from . import db_check
if not db_check.db_check(mod_appkernel=not self.stand_alone,
modw=not self.stand_alone):
exit(1)
def check_shell(rsh, resource):
log.info("Checking if shell is BASH\n")
msg = cfg.sshCommand(rsh, "echo $BASH")
if msg.count("bash") > 0:
log.info("Shell is BASH\n")
else:
log.error(
"Shell on headnode of %s is not BASH, change it to bash and try again.\n",
resource['name'])
exit(1)
def set_permission_on_files(self):
log.info(
"Removing access for group members and everybody for all files as it might contain sensitive information."
)
if not dry_run:
subprocess.check_output("""
chmod -R g-rwx {akrr_home}
chmod -R o-rwx {akrr_home}
""".format(akrr_home=akrr_home),
shell=True)
def run(self):
"""execute what asked in command line"""
log.info("AKRR Regression Tests")
cli_args = self.root_parser.parse_args()
self.process_common_args(cli_args)
if hasattr(cli_args, "func"):
cli_args.func(cli_args)
else:
log.error("There is no command specified!")
def append_to_bashrc(resource):
# append environment variables to .bashrc
log.info("\nAdding AKRR enviroment variables to resource's .bashrc!\n")
if dry_run:
return
str_io = io.StringIO()
try:
sys.stdout = sys.stderr = str_io
rsh = cfg.sshResource(resource)
akrr_header = 'AKRR Remote Resource Environment Variables'
out = cfg.sshCommand(
rsh, '''if [ -e $HOME/.bashrc ]
then
if [[ `grep "\#''' + akrr_header + ''' \[Start\]" $HOME/.bashrc` == *"''' +
akrr_header + ''' [Start]"* ]]
then
echo "Updating AKRR record in $HOME/.bashrc, backing to $HOME/.bashrc_akrrbak"
cp $HOME/.bashrc $HOME/.bashrc_akrrbak
head -n "$(( $(grep -n '\#''' + akrr_header +
''' \[Start\]' $HOME/.bashrc_akrrbak | head -n 1 | cut -d ":" -f 1) - 1 ))" $HOME/.bashrc_akrrbak > $HOME/.bashrc
tail -n "+$(( $(grep -n '\#''' + akrr_header +
''' \[End\]' $HOME/.bashrc_akrrbak | head -n 1 | cut -d ":" -f 1) + 1 ))" $HOME/.bashrc_akrrbak >> $HOME/.bashrc
fi
fi''')
log.debug2(out)
out = cfg.sshCommand(
rsh, '''
echo "Appending AKRR records to $HOME/.bashrc"
echo "#''' + akrr_header + ''' [Start]" >> $HOME/.bashrc
echo "export AKRR_NETWORK_SCRATCH=\\"''' + resource['networkScratch'] +
'''\\"" >> $HOME/.bashrc
echo "export AKRR_LOCAL_SCRATCH=\\"''' + resource['localScratch'] +
'''\\"" >> $HOME/.bashrc
echo "export AKRR_APPKER_DIR=\\"''' + resource['appKerDir'] +
'''\\"" >> $HOME/.bashrc
echo "export AKRR_AKRR_DIR=\\"''' + resource['akrrData'] +
'''\\"" >> $HOME/.bashrc
echo "#''' + akrr_header + ''' [End]" >> $HOME/.bashrc
''')
log.debug2(out)
rsh.close(force=True)
del rsh
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
except Exception as e:
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
log.critical(
"Can not connect to %s\nProbably invalid credential, see full error report:\n%s",
resource['name'], str_io.getvalue())
raise e
def check_rw_db(connection_func, pre_msg, post_msg):
"""
Check that the user has the correct privileges to the database
at the end of the connection provided by 'connection_func'. Specifically, checking
for read / write permissions ( and create table ).
:type connection_func function
:type pre_msg str
:type post_msg str
:param connection_func: the function that will provide a (connection, cursor) tuple.
:param pre_msg: a message to be provided to the user before the checks begin.
:param post_msg: a message to be provided to the user after the checks are successful
:return: true if the database is available / the provided user has the correct privileges.
"""
success = False
log.info(pre_msg)
try:
connection, cursor = connection_func()
try:
with connection:
result = cursor.execute(
"CREATE TABLE CREATE_ME(`id` INT NOT NULL PRIMARY KEY, `name` VARCHAR(48));"
)
success = True if result == 0 else False
if success:
log.info(post_msg, success)
else:
log.error(post_msg, success)
except MySQLdb.Error as e:
log.error(
'Unable to create a table w/ the provided username. %s: %s',
e.args[0], e.args[1])
connection, cursor = connection_func()
try:
with connection:
cursor.execute("DROP TABLE CREATE_ME;")
except MySQLdb.Error as e:
log.error(
'Unable to drop the table created to check permissions. %s: %s',
e.args[0], e.args[1])
except MySQLdb.Error as e:
log.error('Unable to connect to Database. %s: %s', e.args[0],
e.args[1])
return success
def setup():
if add_fake_modw:
_add_fake_modw()
# start bash shell
bash = get_bash()
bash.output = ""
bash.timeoutMessage = 'Unexpected behavior of prep.sh (premature EOF or TIMEOUT)'
bash.runcmd('which python3', printOutput=True)
bash.runcmd('which ' + cfg.which_akrr, printOutput=True)
# start akrr setup
bash.startcmd(cfg.which_akrr + " setup " + dry_run_flag)
# set database user for AKRR
_send_user_password(
bash,
r'Please specify a database user to access mod_akrr database.*\n\[\S+\]',
akrr_db_user_name, akrr_db_user_password)
_send_su_user_password(bash, akrr_db_su_user_name,
akrr_db_su_user_password)
# bAK database:
_send_user_password(
bash,
r'Please specify a database user to access mod_appkernel database.*\n\[\S+\]',
ak_db_user_name, ak_db_user_password)
_send_su_user_password(bash, ak_db_su_user_name, ak_db_su_user_password)
# XD database:
_send_user_password(
bash,
r'Please specify the user that will be connecting to the XDMoD database.*\n\[\S+\]',
ak_db_user_name, ak_db_user_password)
_send_su_user_password(bash, ak_db_su_user_name, ak_db_su_user_password)
bash.expectSendline(
r'.*INPUT.* Please enter the e-mail where cron will send messages.*\n',
"" if cron_email is None else cron_email)
# wait for prompt
bash.justExpect(bash.prompt, timeout=60)
log.info(bash.output)
if bash.output.count("AKRR is set up and is running.") == 0:
log.critical("AKRR was not set up")
exit(1)
else:
log.info("AKRR is set up and is running.")
return
def enable_resource_for_execution(resource):
"""populate mod_appkernel database and allow execution of jobs on this resource"""
if dry_run:
return
resource_name = resource['name']
try:
con_ak, cur_ak = cfg.getAKDB(True)
cur_ak.execute('''SELECT * FROM resource WHERE nickname=%s''',
(resource_name, ))
resource_in_ak_db = cur_ak.fetchall()
if len(resource_in_ak_db) == 0:
log.warning(
"There is no record of %s in mod_appkernel.resource will add one.",
resource_name)
cur_ak.execute(
'''INSERT INTO resource (resource,nickname,description,enabled,visible)
VALUES(%s,%s,%s,0,0);''',
(resource['name'], resource['name'], resource['info']))
con_ak.commit()
cur_ak.execute('''SELECT * FROM resource WHERE nickname=%s''',
(resource_name, ))
resource_in_ak_db = cur_ak.fetchall()
resource_in_ak_db = resource_in_ak_db[0]
# enable and make visible
cur_ak.execute(
'''UPDATE resource
SET enabled=1,visible=1
WHERE resource_id=%s;''',
(resource_in_ak_db['resource_id'], ))
con_ak.commit()
log.info(
"Enabled %s in mod_appkernel.resource for tasks execution and made it visible to XDMoD UI.",
resource_name)
except MySQLdb.Error:
log.error("Can not connect to AK DB\n" "Probably invalid credential")
# enabling resource for execution
try:
r = akrrrestclient.put('/resources/' + resource_name + '/on')
if r.status_code == 200:
log.info('Successfully enabled ' + resource_name)
else:
log.error(
"Can not enable resource through AKRR REST API ( %s )\nSee server response below\n%s",
akrrrestclient.restapi_host, json.dumps(r.json(), indent=4))
except requests.RequestException:
log.error(
"Can not enable resource through AKRR REST API ( %s )\n"
"Is it still running?\n", akrrrestclient.restapi_host)
def generate_resource_config(resource_id, m_resource_name, queuing_system):
from akrr.util.sql import cursor_execute
log.info("Initiating %s at AKRR" % (m_resource_name, ))
if not dry_run:
os.mkdir(os.path.join(resources_dir, m_resource_name), 0o700)
file_path = os.path.abspath(
os.path.join(resources_dir, m_resource_name, 'resource.conf'))
global resource_cfg_filename
resource_cfg_filename = file_path
create_resource_config(file_path, queuing_system)
# add entry to mod_appkernel.resource
con_ak, cur_ak = cfg.getAKDB(True)
cur_ak.execute('''SELECT * FROM resource WHERE nickname=%s''',
(m_resource_name, ))
resource_in_ak_db = cur_ak.fetchall()
if len(resource_in_ak_db) == 0:
cursor_execute(
cur_ak,
"INSERT INTO resource (resource,nickname,description,enabled,visible,xdmod_resource_id)"
"VALUES(%s,%s,%s,0,0,%s);",
(m_resource_name, m_resource_name, m_resource_name, resource_id),
dry_run)
con_ak.commit()
cur_ak.execute('''SELECT * FROM resource WHERE nickname=%s''',
(m_resource_name, ))
if not dry_run:
resource_in_ak_db = cur_ak.fetchall()
resource_id_in_ak_db = resource_in_ak_db[0]['resource_id']
else:
resource_id_in_ak_db = 123
# add entry to mod_akrr.resource
db, cur = cfg.getDB(True)
cur.execute('''SELECT * FROM resources WHERE name=%s''',
(m_resource_name, ))
resource_in_db = cur.fetchall()
if len(resource_in_db) == 0:
cursor_execute(
cur, '''INSERT INTO resources (id,xdmod_resource_id,name,enabled)
VALUES(%s,%s,%s,%s);''',
(resource_id_in_ak_db, resource_id, m_resource_name, 0), dry_run)
db.commit()
log.info("Resource configuration is in " + file_path)
def check_appsig(rsh, resource):
log.info("Testing app.signature calculator on headnode\n")
out = cfg.sshCommand(
rsh, "%s/execs/bin/appsigcheck.sh `which md5sum`" %
(resource['appKerDir'], ))
if out.count("===ExeBinSignature===") > 0 and out.count("MD5:") > 0:
log.info("App.signature calculator is working on headnode\n")
else:
if dry_run:
log.dry_run("App.signature calculator is not working\n")
return
log.error(
"App.signature calculator is not working\n" +
"See full error report below\n%s", out)
exit(1)
def create_resource_config(file_path, queuing_system):
"""
create resource file from template
"""
with open(
os.path.join(cfg.templates_dir,
'template.%s.conf' % queuing_system)) as fin:
template = fin.read()
fin.close()
def update_template(s, variable, in_quotes=True):
pattern = '^' + variable + '\s*=\s*.*$'
replace = variable + ' = '
value = globals()[variable]
if value is None:
replace += 'None'
else:
if in_quotes:
replace += '"'
replace += str(value)
if in_quotes:
replace += '"'
out = []
lines = s.splitlines()
for line in lines:
out.append(re.sub(pattern, replace, line))
return "\n".join(out)
template = update_template(template, 'ppn', in_quotes=False)
for v in [
'remoteAccessNode', 'remoteAccessMethod', 'remoteCopyMethod',
'sshUserName', 'sshPassword', 'sshPrivateKeyFile',
'sshPrivateKeyPassword', 'networkScratch', 'localScratch',
'akrrData', 'appKerDir', 'batchScheduler'
]:
template = update_template(template, v)
template += "\n\n"
if not dry_run:
with open(file_path, 'w') as fout:
fout.write(template)
fout.close()
else:
log.info('Dry Run Mode: Would have written to: {}'
'It content would be: {}'.format(file_path, template))
def print_important_env():
msg = "Some environment values:\n"
# which akrr
try:
which_akrr = run_cmd_getoutput("which akrr").strip()
msg = msg + "\twhich akrr: " + str(which_akrr) + "\n"
except:
msg = msg + "\twhich akrr: None\n"
# which python3
which_python3 = run_cmd_getoutput("which python3").strip()
msg = msg + "\twhich python3: " + str(which_python3) + "\n"
# AKRR_CONF
akrr_cfg = os.getenv("AKRR_CONF")
msg = msg + "\tAKRR_CONF: " + str(akrr_cfg) + "\n"
log.info(msg)
def _stop_akrr():
akrr_daemon_pids = _get_akrr_pids()
if akrr_daemon_pids is None:
log.info("AKRR daemon is not running.")
return
log.info("AKRR daemon is running, trying to stop it.")
try:
if os.path.exists(os.path.join(cfg.which_akrr)):
cmd = "{} {} daemon stop".format(sys.executable, cfg.which_akrr)
if not dry_run:
subprocess.check_output(
cmd, shell=True, timeout=60)
else:
log.dry_run("should execute: " + cmd)
except subprocess.CalledProcessError:
pass
akrr_daemon_pids = _get_akrr_pids()
if akrr_daemon_pids is None:
return
try:
for pid in akrr_daemon_pids:
cmd = "kill -9 {}".format(pid)
if not dry_run:
subprocess.check_output(
cmd, shell=True, timeout=60)
else:
log.dry_run("should execute: " + cmd)
except subprocess.CalledProcessError:
pass
akrr_daemon_pids = _get_akrr_pids()
if akrr_daemon_pids is None:
return
if dry_run:
log.dry_run("at this point AKRR daemon should be stopped")
return
raise Exception("was not able to stop AKRR daemon")
def runcmd(self, cmd, clearSpecialSymbols=True, printOutput=False, addAfter=False):
self.sendline(cmd)
self.expect(self.prompt)
self.lastcmd = cmd + "\n"
output = self.getCmdOutput(clearSpecialSymbols=clearSpecialSymbols, addAfter=addAfter, replaceCMD=False)
if hasattr(self, 'output'): self.output += output
if printOutput:
if self.echo:
log.info(output)
else:
log.info("command: `{}` output: \n{}".format(cmd, output))
sys.stdout.flush()
else:
if self.echo:
log.debug2(output)
else:
log.debug2("command: `{}` output: \n{}".format(cmd, output))
sys.stdout.flush()
return output
def add_all_resources(resource=None, **_):
log.info("resource config: Initial resource configuration")
if resource is not None or resource == "":
resources_to_include = [r.stip() for r in resource.split(",")]
else:
resources_to_include = None
resources_to_add = []
if 'resource' not in cfg.yml:
return
for resource_rec in cfg.yml['resource']:
if resources_to_include is None:
resources_to_add.append(resource_rec)
elif 'resource_name' is resource_rec and resource_rec['resource_name'] in resources_to_include:
resources_to_add.append(resource_rec)
for resource_rec in resources_to_add:
resource_add(**resource_rec)
def deploy_all_resources(resource=None, **_):
log.info("Deploying AK to resources")
if resource is not None or resource == "":
resources_to_include = [r.stip() for r in resource.split(",")]
else:
resources_to_include = None
resources_to_add = []
if 'resource' not in cfg.yml:
return
for resource_rec in cfg.yml['resource']:
if resources_to_include is None:
resources_to_add.append(resource_rec)
elif 'resource_name' is resource_rec and resource_rec['resource_name'] in resources_to_include:
resources_to_add.append(resource_rec)
for resource_rec in resources_to_add:
resource_deploy(**resource_rec)
def check_r_db(connection_func, pre_msg, post_msg):
"""
Check that the user has the correct privileges to the database
at the end of the connection provided by 'connection_func'.
Specifically checking for read permissions.
:type connection_func function
:type pre_msg str
:type post_msg str
:param connection_func: the function that will provide a (connection, cursor) tuple.
:param pre_msg: a message to be provided to the user before the checks begin.
:param post_msg: a message to be provided to the user after the checks are successful
:return: true if the database is available / the provided user has the correct privileges.
"""
success = False
log.info(pre_msg)
try:
connection, cursor = connection_func()
try:
with connection:
result = cursor.execute(
"SELECT COUNT(*) FROM `modw`.`resourcefact`;")
success = True if result >= 0 else False
if success:
log.info(post_msg, success)
else:
log.error(post_msg, success)
except MySQLdb.Error as e:
log.error('Unable to select from `modw`.`resourcefact`. %s: %s',
e.args[0], e.args[1])
except MySQLdb.Error as e:
log.error('Unable to connect to Database. %s: %s', e.args[0],
e.args[1])
return success
def run_test_job(resource, app_name="test", nodes=2):
log.info(
"Will send test job to queue, wait till it executed and will analyze the output"
)
log.debug("Will use AKRR REST API at {}".format(
akrrrestclient.restapi_host))
check_connection_to_rest_api()
if dry_run:
return
task_id = check_if_test_job_already_submitted(resource, app_name)
if task_id is None:
task_id = submit_test_job(resource, app_name, nodes)
monitor_test_job(task_id)
analyse_test_job_results(task_id, resource, app_name)
os.remove(get_test_job_lock_filename(resource, app_name))
def drop_db(db, cur, db_name, dry_run=False):
"""remove dbname database from db,cur connection"""
from akrr import log
cur.execute("SHOW databases")
databases = cur.fetchall()
if db_name in {v["Database"] for v in databases}:
log.info("Removing %s database from %s ", db_name, str(db))
msg = "SQL(%s): " % (str(db), )
msg = msg + "DROP DATABASE IF EXISTS %s" % (db_name, )
if dry_run:
log.dry_run(msg)
if dry_run:
return
# remove user
cur.execute("DROP DATABASE IF EXISTS %s" % (db_name, ))
db.commit()
else:
log.info("Database %s is not present on %s ", db_name, str(db))
