def _send_su_user_password(bash, user, password): if user is None: return # set AKRR database root user if testInvalidAdministrativeDatabaseUser: log.info("Entering invalid administrative database user") bash.expectSendline( r'.*INPUT.* Please provide an administrative database user.*\nUsername:'******'.*INPUT.* Please provide the password.*\n', "invalid", timeout=fast_timeout) bash.justExpect( r'.*ERROR.* Entered credential is not valid. Please try again.', timeout=fast_timeout) log.info("\nEntering valid administrative database user") bash.expectSendline( r'.*INPUT.* Please provide an administrative database user.*\nUsername:'******'.*INPUT.* Please provide the password.*\n', password, timeout=fast_timeout)
def connect_to_resource(resource): """connect to resource defined in resource dictionary""" log.info("Validating resource accessibility. Connecting to %s.", resource['name']) if resource['sshPrivateKeyFile'] is not None and os.path.isfile( resource['sshPrivateKeyFile']) is False: log.error("Can not access ssh private key (%s)" "", resource['sshPrivateKeyFile']) exit(1) str_io = io.StringIO() try: sys.stdout = sys.stderr = str_io rsh = cfg.sshResource(resource) sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ log.info("Successfully connected to %s\n", resource['name']) log.empty_line() return rsh except akrrError: sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ log.critical("Can not connect to %s\nMessage:\n%s", resource['name'], str_io.getvalue()) exit(1)
def run(self): # check self.check_utils() self.check_previous_installation() # ask info self.read_db_user_credentials() if self.install_cron_scripts_flag: self.ask_cron_email() # if it is dry_run # all question are asked, this is dry run, so nothing else to do") self.init_mysql_dbs() self.init_dir() self.generate_self_signed_certificate() self.generate_settings_file() self.set_permission_on_files() self.db_check() self.generate_tables() self.start_daemon() self.check_daemon() if self.install_cron_scripts_flag: self.install_cron_scripts() if in_src_install: self.update_bashrc() log.info("AKRR is set up and is running.")
def _remove_dir(path): if os.path.exists(path): log.info("Deleting " + path) if not dry_run: shutil.rmtree(path) else: log.info("Directory " + path + " doesn't exists.")
def _send_user_password(bash, expect, user, password, test_mismatching_password=True): bash.expectSendline(expect, user, timeout=fast_timeout) bash.justExpect("\n") if password is None: return if test_mismatching_password: log.info("Entering not matching password") bash.expectSendline(r'.*INPUT.* Please specify a password.*\n', 'password', timeout=fast_timeout) bash.expectSendline(r'.*INPUT.* Please reenter the password.*\n', 'not_matching_password', timeout=fast_timeout) bash.justExpect( r'.*ERROR.* Entered passwords do not match. Please try again.', timeout=fast_timeout) log.info("\nEntering matching password") bash.expectSendline(r'.*INPUT.* Please specify a password.*\n', password, timeout=fast_timeout) bash.expectSendline(r'.*INPUT.* Please reenter the password.*\n', password, timeout=fast_timeout)
def generate_tables(self): log.info("Creating tables and populating them with initial values.") from .generate_tables import create_and_populate_mod_akrr_tables, create_and_populate_mod_appkernel_tables create_and_populate_mod_akrr_tables(dry_run) create_and_populate_mod_appkernel_tables(dry_run)
def _create_db_user_gran_priv_if_needed(con_fun, user, password, db, priv): log.info("Creating %s and user to access it if needed" % (db, )) su_con, su_cur = con_fun(True, None) client_host = get_db_client_host(su_cur) _cursor_execute( su_cur, "CREATE DATABASE IF NOT EXISTS %s" % (cv(db), )) su_cur.execute( "SELECT * FROM mysql.user WHERE User=%s AND Host=%s", (user, client_host)) if len(su_cur.fetchall()) == 0: # Older version of MySQL do not support CREATE USER IF NOT EXISTS # so need to do checking _cursor_execute(su_cur, "CREATE USER %s@%s IDENTIFIED BY %s", (user, client_host, password)) _cursor_execute( su_cur, "GRANT " + cv(priv) + " ON " + cv(db) + ".* TO %s@%s", (user, client_host)) su_con.commit()
def _read_username_password(prompt="Enter username:"******"user", password_on_default_user=None): log.log_input(prompt) if username is None: username = input('[{0}] '.format(default_username)) if username == '': username = default_username else: log.info("User, " + username + ", already entered.") if username == default_username and password is None and password_on_default_user is not None: password = password_on_default_user if password is None: while True: log.log_input("Please specify a password:"******"Please reenter the password:"******"Entered passwords do not match. Please try again.") else: log.info("Password already entered.") return username, password
def run_it(_): from .util import print_important_env print_important_env() log.info("AKRR Setup") _config_setup() setup()
def check_daemon(self): """Check that the daemon is running""" log.info("Checking that AKRR daemon is running") if dry_run: return akrr_cli = os.path.join(akrr_bin_dir, 'akrr') status = subprocess.call(akrr_cli + " daemon check", shell=True) if status != 0: exit(status)
def start_daemon(self): """Start the daemon""" log.info("Starting AKRR daemon") if dry_run: return akrr_cli = os.path.join(akrr_bin_dir, 'akrr') status = subprocess.call(akrr_cli + " daemon start", shell=True) if status != 0: exit(status)
def db_check(self): log.info("Checking acces to DBs.") if dry_run: return from . import db_check if not db_check.db_check(mod_appkernel=not self.stand_alone, modw=not self.stand_alone): exit(1)
def check_shell(rsh, resource): log.info("Checking if shell is BASH\n") msg = cfg.sshCommand(rsh, "echo $BASH") if msg.count("bash") > 0: log.info("Shell is BASH\n") else: log.error( "Shell on headnode of %s is not BASH, change it to bash and try again.\n", resource['name']) exit(1)
def set_permission_on_files(self): log.info( "Removing access for group members and everybody for all files as it might contain sensitive information." ) if not dry_run: subprocess.check_output(""" chmod -R g-rwx {akrr_home} chmod -R o-rwx {akrr_home} """.format(akrr_home=akrr_home), shell=True)
def run(self): """execute what asked in command line""" log.info("AKRR Regression Tests") cli_args = self.root_parser.parse_args() self.process_common_args(cli_args) if hasattr(cli_args, "func"): cli_args.func(cli_args) else: log.error("There is no command specified!")
def append_to_bashrc(resource): # append environment variables to .bashrc log.info("\nAdding AKRR enviroment variables to resource's .bashrc!\n") if dry_run: return str_io = io.StringIO() try: sys.stdout = sys.stderr = str_io rsh = cfg.sshResource(resource) akrr_header = 'AKRR Remote Resource Environment Variables' out = cfg.sshCommand( rsh, '''if [ -e $HOME/.bashrc ] then if [[ `grep "\#''' + akrr_header + ''' \[Start\]" $HOME/.bashrc` == *"''' + akrr_header + ''' [Start]"* ]] then echo "Updating AKRR record in $HOME/.bashrc, backing to $HOME/.bashrc_akrrbak" cp $HOME/.bashrc $HOME/.bashrc_akrrbak head -n "$(( $(grep -n '\#''' + akrr_header + ''' \[Start\]' $HOME/.bashrc_akrrbak | head -n 1 | cut -d ":" -f 1) - 1 ))" $HOME/.bashrc_akrrbak > $HOME/.bashrc tail -n "+$(( $(grep -n '\#''' + akrr_header + ''' \[End\]' $HOME/.bashrc_akrrbak | head -n 1 | cut -d ":" -f 1) + 1 ))" $HOME/.bashrc_akrrbak >> $HOME/.bashrc fi fi''') log.debug2(out) out = cfg.sshCommand( rsh, ''' echo "Appending AKRR records to $HOME/.bashrc" echo "#''' + akrr_header + ''' [Start]" >> $HOME/.bashrc echo "export AKRR_NETWORK_SCRATCH=\\"''' + resource['networkScratch'] + '''\\"" >> $HOME/.bashrc echo "export AKRR_LOCAL_SCRATCH=\\"''' + resource['localScratch'] + '''\\"" >> $HOME/.bashrc echo "export AKRR_APPKER_DIR=\\"''' + resource['appKerDir'] + '''\\"" >> $HOME/.bashrc echo "export AKRR_AKRR_DIR=\\"''' + resource['akrrData'] + '''\\"" >> $HOME/.bashrc echo "#''' + akrr_header + ''' [End]" >> $HOME/.bashrc ''') log.debug2(out) rsh.close(force=True) del rsh sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ except Exception as e: sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ log.critical( "Can not connect to %s\nProbably invalid credential, see full error report:\n%s", resource['name'], str_io.getvalue()) raise e
def check_rw_db(connection_func, pre_msg, post_msg): """ Check that the user has the correct privileges to the database at the end of the connection provided by 'connection_func'. Specifically, checking for read / write permissions ( and create table ). :type connection_func function :type pre_msg str :type post_msg str :param connection_func: the function that will provide a (connection, cursor) tuple. :param pre_msg: a message to be provided to the user before the checks begin. :param post_msg: a message to be provided to the user after the checks are successful :return: true if the database is available / the provided user has the correct privileges. """ success = False log.info(pre_msg) try: connection, cursor = connection_func() try: with connection: result = cursor.execute( "CREATE TABLE CREATE_ME(`id` INT NOT NULL PRIMARY KEY, `name` VARCHAR(48));" ) success = True if result == 0 else False if success: log.info(post_msg, success) else: log.error(post_msg, success) except MySQLdb.Error as e: log.error( 'Unable to create a table w/ the provided username. %s: %s', e.args[0], e.args[1]) connection, cursor = connection_func() try: with connection: cursor.execute("DROP TABLE CREATE_ME;") except MySQLdb.Error as e: log.error( 'Unable to drop the table created to check permissions. %s: %s', e.args[0], e.args[1]) except MySQLdb.Error as e: log.error('Unable to connect to Database. %s: %s', e.args[0], e.args[1]) return success
def setup(): if add_fake_modw: _add_fake_modw() # start bash shell bash = get_bash() bash.output = "" bash.timeoutMessage = 'Unexpected behavior of prep.sh (premature EOF or TIMEOUT)' bash.runcmd('which python3', printOutput=True) bash.runcmd('which ' + cfg.which_akrr, printOutput=True) # start akrr setup bash.startcmd(cfg.which_akrr + " setup " + dry_run_flag) # set database user for AKRR _send_user_password( bash, r'Please specify a database user to access mod_akrr database.*\n\[\S+\]', akrr_db_user_name, akrr_db_user_password) _send_su_user_password(bash, akrr_db_su_user_name, akrr_db_su_user_password) # bAK database: _send_user_password( bash, r'Please specify a database user to access mod_appkernel database.*\n\[\S+\]', ak_db_user_name, ak_db_user_password) _send_su_user_password(bash, ak_db_su_user_name, ak_db_su_user_password) # XD database: _send_user_password( bash, r'Please specify the user that will be connecting to the XDMoD database.*\n\[\S+\]', ak_db_user_name, ak_db_user_password) _send_su_user_password(bash, ak_db_su_user_name, ak_db_su_user_password) bash.expectSendline( r'.*INPUT.* Please enter the e-mail where cron will send messages.*\n', "" if cron_email is None else cron_email) # wait for prompt bash.justExpect(bash.prompt, timeout=60) log.info(bash.output) if bash.output.count("AKRR is set up and is running.") == 0: log.critical("AKRR was not set up") exit(1) else: log.info("AKRR is set up and is running.") return
def enable_resource_for_execution(resource): """populate mod_appkernel database and allow execution of jobs on this resource""" if dry_run: return resource_name = resource['name'] try: con_ak, cur_ak = cfg.getAKDB(True) cur_ak.execute('''SELECT * FROM resource WHERE nickname=%s''', (resource_name, )) resource_in_ak_db = cur_ak.fetchall() if len(resource_in_ak_db) == 0: log.warning( "There is no record of %s in mod_appkernel.resource will add one.", resource_name) cur_ak.execute( '''INSERT INTO resource (resource,nickname,description,enabled,visible) VALUES(%s,%s,%s,0,0);''', (resource['name'], resource['name'], resource['info'])) con_ak.commit() cur_ak.execute('''SELECT * FROM resource WHERE nickname=%s''', (resource_name, )) resource_in_ak_db = cur_ak.fetchall() resource_in_ak_db = resource_in_ak_db[0] # enable and make visible cur_ak.execute( '''UPDATE resource SET enabled=1,visible=1 WHERE resource_id=%s;''', (resource_in_ak_db['resource_id'], )) con_ak.commit() log.info( "Enabled %s in mod_appkernel.resource for tasks execution and made it visible to XDMoD UI.", resource_name) except MySQLdb.Error: log.error("Can not connect to AK DB\n" "Probably invalid credential") # enabling resource for execution try: r = akrrrestclient.put('/resources/' + resource_name + '/on') if r.status_code == 200: log.info('Successfully enabled ' + resource_name) else: log.error( "Can not enable resource through AKRR REST API ( %s )\nSee server response below\n%s", akrrrestclient.restapi_host, json.dumps(r.json(), indent=4)) except requests.RequestException: log.error( "Can not enable resource through AKRR REST API ( %s )\n" "Is it still running?\n", akrrrestclient.restapi_host)
def generate_resource_config(resource_id, m_resource_name, queuing_system): from akrr.util.sql import cursor_execute log.info("Initiating %s at AKRR" % (m_resource_name, )) if not dry_run: os.mkdir(os.path.join(resources_dir, m_resource_name), 0o700) file_path = os.path.abspath( os.path.join(resources_dir, m_resource_name, 'resource.conf')) global resource_cfg_filename resource_cfg_filename = file_path create_resource_config(file_path, queuing_system) # add entry to mod_appkernel.resource con_ak, cur_ak = cfg.getAKDB(True) cur_ak.execute('''SELECT * FROM resource WHERE nickname=%s''', (m_resource_name, )) resource_in_ak_db = cur_ak.fetchall() if len(resource_in_ak_db) == 0: cursor_execute( cur_ak, "INSERT INTO resource (resource,nickname,description,enabled,visible,xdmod_resource_id)" "VALUES(%s,%s,%s,0,0,%s);", (m_resource_name, m_resource_name, m_resource_name, resource_id), dry_run) con_ak.commit() cur_ak.execute('''SELECT * FROM resource WHERE nickname=%s''', (m_resource_name, )) if not dry_run: resource_in_ak_db = cur_ak.fetchall() resource_id_in_ak_db = resource_in_ak_db[0]['resource_id'] else: resource_id_in_ak_db = 123 # add entry to mod_akrr.resource db, cur = cfg.getDB(True) cur.execute('''SELECT * FROM resources WHERE name=%s''', (m_resource_name, )) resource_in_db = cur.fetchall() if len(resource_in_db) == 0: cursor_execute( cur, '''INSERT INTO resources (id,xdmod_resource_id,name,enabled) VALUES(%s,%s,%s,%s);''', (resource_id_in_ak_db, resource_id, m_resource_name, 0), dry_run) db.commit() log.info("Resource configuration is in " + file_path)
def check_appsig(rsh, resource): log.info("Testing app.signature calculator on headnode\n") out = cfg.sshCommand( rsh, "%s/execs/bin/appsigcheck.sh `which md5sum`" % (resource['appKerDir'], )) if out.count("===ExeBinSignature===") > 0 and out.count("MD5:") > 0: log.info("App.signature calculator is working on headnode\n") else: if dry_run: log.dry_run("App.signature calculator is not working\n") return log.error( "App.signature calculator is not working\n" + "See full error report below\n%s", out) exit(1)
def create_resource_config(file_path, queuing_system): """ create resource file from template """ with open( os.path.join(cfg.templates_dir, 'template.%s.conf' % queuing_system)) as fin: template = fin.read() fin.close() def update_template(s, variable, in_quotes=True): pattern = '^' + variable + '\s*=\s*.*$' replace = variable + ' = ' value = globals()[variable] if value is None: replace += 'None' else: if in_quotes: replace += '"' replace += str(value) if in_quotes: replace += '"' out = [] lines = s.splitlines() for line in lines: out.append(re.sub(pattern, replace, line)) return "\n".join(out) template = update_template(template, 'ppn', in_quotes=False) for v in [ 'remoteAccessNode', 'remoteAccessMethod', 'remoteCopyMethod', 'sshUserName', 'sshPassword', 'sshPrivateKeyFile', 'sshPrivateKeyPassword', 'networkScratch', 'localScratch', 'akrrData', 'appKerDir', 'batchScheduler' ]: template = update_template(template, v) template += "\n\n" if not dry_run: with open(file_path, 'w') as fout: fout.write(template) fout.close() else: log.info('Dry Run Mode: Would have written to: {}' 'It content would be: {}'.format(file_path, template))
def print_important_env(): msg = "Some environment values:\n" # which akrr try: which_akrr = run_cmd_getoutput("which akrr").strip() msg = msg + "\twhich akrr: " + str(which_akrr) + "\n" except: msg = msg + "\twhich akrr: None\n" # which python3 which_python3 = run_cmd_getoutput("which python3").strip() msg = msg + "\twhich python3: " + str(which_python3) + "\n" # AKRR_CONF akrr_cfg = os.getenv("AKRR_CONF") msg = msg + "\tAKRR_CONF: " + str(akrr_cfg) + "\n" log.info(msg)
def _stop_akrr(): akrr_daemon_pids = _get_akrr_pids() if akrr_daemon_pids is None: log.info("AKRR daemon is not running.") return log.info("AKRR daemon is running, trying to stop it.") try: if os.path.exists(os.path.join(cfg.which_akrr)): cmd = "{} {} daemon stop".format(sys.executable, cfg.which_akrr) if not dry_run: subprocess.check_output( cmd, shell=True, timeout=60) else: log.dry_run("should execute: " + cmd) except subprocess.CalledProcessError: pass akrr_daemon_pids = _get_akrr_pids() if akrr_daemon_pids is None: return try: for pid in akrr_daemon_pids: cmd = "kill -9 {}".format(pid) if not dry_run: subprocess.check_output( cmd, shell=True, timeout=60) else: log.dry_run("should execute: " + cmd) except subprocess.CalledProcessError: pass akrr_daemon_pids = _get_akrr_pids() if akrr_daemon_pids is None: return if dry_run: log.dry_run("at this point AKRR daemon should be stopped") return raise Exception("was not able to stop AKRR daemon")
def runcmd(self, cmd, clearSpecialSymbols=True, printOutput=False, addAfter=False): self.sendline(cmd) self.expect(self.prompt) self.lastcmd = cmd + "\n" output = self.getCmdOutput(clearSpecialSymbols=clearSpecialSymbols, addAfter=addAfter, replaceCMD=False) if hasattr(self, 'output'): self.output += output if printOutput: if self.echo: log.info(output) else: log.info("command: `{}` output: \n{}".format(cmd, output)) sys.stdout.flush() else: if self.echo: log.debug2(output) else: log.debug2("command: `{}` output: \n{}".format(cmd, output)) sys.stdout.flush() return output
def add_all_resources(resource=None, **_): log.info("resource config: Initial resource configuration") if resource is not None or resource == "": resources_to_include = [r.stip() for r in resource.split(",")] else: resources_to_include = None resources_to_add = [] if 'resource' not in cfg.yml: return for resource_rec in cfg.yml['resource']: if resources_to_include is None: resources_to_add.append(resource_rec) elif 'resource_name' is resource_rec and resource_rec['resource_name'] in resources_to_include: resources_to_add.append(resource_rec) for resource_rec in resources_to_add: resource_add(**resource_rec)
def deploy_all_resources(resource=None, **_): log.info("Deploying AK to resources") if resource is not None or resource == "": resources_to_include = [r.stip() for r in resource.split(",")] else: resources_to_include = None resources_to_add = [] if 'resource' not in cfg.yml: return for resource_rec in cfg.yml['resource']: if resources_to_include is None: resources_to_add.append(resource_rec) elif 'resource_name' is resource_rec and resource_rec['resource_name'] in resources_to_include: resources_to_add.append(resource_rec) for resource_rec in resources_to_add: resource_deploy(**resource_rec)
def check_r_db(connection_func, pre_msg, post_msg): """ Check that the user has the correct privileges to the database at the end of the connection provided by 'connection_func'. Specifically checking for read permissions. :type connection_func function :type pre_msg str :type post_msg str :param connection_func: the function that will provide a (connection, cursor) tuple. :param pre_msg: a message to be provided to the user before the checks begin. :param post_msg: a message to be provided to the user after the checks are successful :return: true if the database is available / the provided user has the correct privileges. """ success = False log.info(pre_msg) try: connection, cursor = connection_func() try: with connection: result = cursor.execute( "SELECT COUNT(*) FROM `modw`.`resourcefact`;") success = True if result >= 0 else False if success: log.info(post_msg, success) else: log.error(post_msg, success) except MySQLdb.Error as e: log.error('Unable to select from `modw`.`resourcefact`. %s: %s', e.args[0], e.args[1]) except MySQLdb.Error as e: log.error('Unable to connect to Database. %s: %s', e.args[0], e.args[1]) return success
def run_test_job(resource, app_name="test", nodes=2): log.info( "Will send test job to queue, wait till it executed and will analyze the output" ) log.debug("Will use AKRR REST API at {}".format( akrrrestclient.restapi_host)) check_connection_to_rest_api() if dry_run: return task_id = check_if_test_job_already_submitted(resource, app_name) if task_id is None: task_id = submit_test_job(resource, app_name, nodes) monitor_test_job(task_id) analyse_test_job_results(task_id, resource, app_name) os.remove(get_test_job_lock_filename(resource, app_name))
def drop_db(db, cur, db_name, dry_run=False): """remove dbname database from db,cur connection""" from akrr import log cur.execute("SHOW databases") databases = cur.fetchall() if db_name in {v["Database"] for v in databases}: log.info("Removing %s database from %s ", db_name, str(db)) msg = "SQL(%s): " % (str(db), ) msg = msg + "DROP DATABASE IF EXISTS %s" % (db_name, ) if dry_run: log.dry_run(msg) if dry_run: return # remove user cur.execute("DROP DATABASE IF EXISTS %s" % (db_name, )) db.commit() else: log.info("Database %s is not present on %s ", db_name, str(db))