def app_list():
from prettytable import PrettyTable
from .cfg import apps, resources
from akrr.db import get_akrr_db
from collections import OrderedDict
db, cur = get_akrr_db(dict_cursor=True)
# appkernels
cur.execute("select * from app_kernels")
app_table = OrderedDict(((a["name"], a) for a in cur.fetchall()))
pt = PrettyTable()
pt.field_names = ["App", "Enabled"]
pt.align["App"] = "l"
# Apps in db
for app_name, app_opt in app_table.items():
pt.add_row([app_name, app_opt["enabled"] > 0])
# Apps not in db
for app_name in apps.keys():
if app_name not in app_table:
pt.add_row([app_name, "None"])
log.info("All available appkernels:\n" + str(pt))
# get enable table
cur.execute("select * from resources")
resource_app_enabled = OrderedDict(((r["name"], {"apps": OrderedDict(), **r}) for r in cur.fetchall()))
cur.execute(
"select ra.id as resource_app_kernels_id,\n"
" ra.resource_id,r.name as resource, xdmod_resource_id,\n"
" r.name as resource,r.enabled as resource_enabled,\n"
" ra.app_kernel_id,a.name as app,a.enabled as app_enabled,\n"
" ra.enabled as resource_app_enabled, a.nodes_list\n"
"from resource_app_kernels ra\n"
"left join resources r on ra.resource_id=r.id\n"
"left join app_kernels a on ra.app_kernel_id=a.id")
for ra in cur.fetchall():
resource_app_enabled[ra["resource"]]["apps"][ra["app"]] = ra
pt = PrettyTable()
pt.field_names = ["Resource", "App", "Enabled"]
pt.align["Resource"] = "l"
for resource_name, resource_opt in resources.items():
for app_name, app_opt in apps.items():
if resource_name in app_opt['appkernel_on_resource']:
ra_enabled = None
if resource_name in resource_app_enabled and app_name in resource_app_enabled[resource_name]["apps"]:
ra_enabled = resource_app_enabled[resource_name]["apps"][app_name]["resource_app_enabled"] != 0
pt.add_row([resource_name, app_name, ra_enabled])
log.info("Appkernels on resources\n" + str(pt))
return
def app_list():
from prettytable import PrettyTable
from .cfg import apps, resources
from akrr.db import get_akrr_db
from collections import OrderedDict
db, cur = get_akrr_db(dict_cursor=True)
# appkernels
cur.execute("select * from app_kernels")
app_table = OrderedDict(((a["name"], a) for a in cur.fetchall()))
pt = PrettyTable()
pt.field_names = ["App", "Enabled"]
pt.align["App"] = "l"
# Apps in db
for app_name, app_opt in app_table.items():
pt.add_row([app_name, app_opt["enabled"] > 0])
# Apps not in db
for app_name in apps.keys():
if app_name not in app_table:
pt.add_row([app_name, "None"])
log.info("All available appkernels:\n" + str(pt))
# get enable table
resource_app_enabled = app_get_enabled()
pt = PrettyTable()
pt.field_names = ["Resource", "App", "Enabled"]
pt.align["Resource"] = "l"
for resource_name, resource_opt in resources.items():
for app_name, app_opt in apps.items():
if resource_name in app_opt['appkernel_on_resource']:
ra_enabled = None
if resource_name in resource_app_enabled and app_name in resource_app_enabled[
resource_name]["apps"]:
ra_enabled = resource_app_enabled[resource_name]["apps"][
app_name]["resource_app_enabled"] != 0
pt.add_row([resource_name, app_name, ra_enabled])
log.info("Appkernels on resources\n" + str(pt))
return
def app_get_enabled():
from collections import OrderedDict
from akrr.db import get_akrr_db
db, cur = get_akrr_db(dict_cursor=True)
cur.execute("select * from resources")
resource_app_enabled = OrderedDict(((r["name"], {
"apps": OrderedDict(),
**r
}) for r in cur.fetchall()))
cur.execute(
"select ra.id as resource_app_kernels_id,\n"
" ra.resource_id,r.name as resource, xdmod_resource_id,\n"
" r.name as resource,r.enabled as resource_enabled,\n"
" ra.app_kernel_id,a.name as app,a.enabled as app_enabled,\n"
" ra.enabled as resource_app_enabled, a.nodes_list\n"
"from resource_app_kernels ra\n"
"left join resources r on ra.resource_id=r.id\n"
"left join app_kernels a on ra.app_kernel_id=a.id")
for ra in cur.fetchall():
resource_app_enabled[ra["resource"]]["apps"][ra["app"]] = ra
return resource_app_enabled
def app_add(resource: str,
appkernel: str,
execution_method: str = "hpc",
dry_run: bool = False):
"""add app configuration to resource"""
log.info("Generating application kernel configuration for %s on %s",
appkernel, resource)
try:
cfg.find_resource_by_name(resource)
except Exception:
msg = "Can not find resource: %s" % resource
log.error(msg)
raise AkrrValueException(msg)
try:
appcfg = cfg.find_app_by_name(appkernel)
except Exception:
msg = "Can not find application kernel: %s" % appkernel
log.error(msg)
raise AkrrValueException(msg)
cfg_filename = os.path.join(cfg.cfg_dir, 'resources', resource,
appkernel + ".app.conf")
cfg_default_template_filename = os.path.join(cfg.templates_dir,
appkernel + ".app.conf")
cfg_template_filename = os.path.join(
cfg.templates_dir, "%s.%s.app.conf" % (appkernel, execution_method))
if (not os.path.isfile(cfg_template_filename)
) and execution_method == "hpc":
cfg_template_filename = cfg_default_template_filename
if (not os.path.isfile(cfg_template_filename)
) and os.path.isfile(cfg_default_template_filename):
msg = (
"Can not find template file for %s application kernel in %s execution mode.\n"
"Try default execution mode (hpc) and customize it for your needs."
) % (appkernel, cfg_template_filename)
log.error(msg)
raise AkrrValueException(msg)
if os.path.isfile(cfg_filename):
msg = "Configuration file for %s on %s already exist. For regeneration delete it, %s" % (
appkernel, resource, cfg_filename)
log.error(msg)
log.info("Application kernel configuration for %s on %s is in: \n\t%s",
appkernel, resource, cfg_filename)
raise AkrrValueException(msg)
if not os.path.isfile(cfg_template_filename):
msg = "Can not find template file for application kernel: %s" % cfg_template_filename
log.error(msg)
raise AkrrValueException(msg)
# check that app is in db
from akrr.cli.generate_tables import populate_mod_akrr_appkernels, populate_mod_appkernel_app_kernel_def
from akrr.db import get_akrr_db, get_ak_db
con_ak, cur_ak = get_ak_db()
con_akrr, cur_akrr = get_akrr_db()
sql = "select * from app_kernel_def where ak_base_name='%s'" % appkernel
cur_ak.execute(sql)
result = cur_ak.fetchall()
if len(result) == 0:
if "db_setup" in appcfg and "mod_appkernel_app_kernel_def" in appcfg[
'db_setup']:
populate_mod_appkernel_app_kernel_def(
con_ak,
cur_ak,
dry_run=dry_run,
mod_appkernel_app_kernel_def=(
appcfg['db_setup']["mod_appkernel_app_kernel_def"], ))
else:
log.warning(
"%s is not in database and there is no info on how to add it. XDMoD would not ingest it."
)
sql = "select * from app_kernels where name='%s'" % appkernel
cur_akrr.execute(sql)
result = cur_akrr.fetchall()
if len(result) == 0:
if "db_setup" in appcfg and "mod_akrr_appkernels" in appcfg['db_setup']:
populate_mod_akrr_appkernels(
con_akrr,
cur_akrr,
dry_run=dry_run,
mod_akrr_appkernels=(
appcfg['db_setup']["mod_akrr_appkernels"], ))
else:
log.warning(
"%s is not in database and there is no info on how to add it. XDMoD would not ingest it."
)
if dry_run:
log.dry_run(
"Initial application kernel configuration for %s on %s, should be copied \n\tfrom %s to %s"
% (appkernel, resource, cfg_template_filename, cfg_filename))
else:
shutil.copyfile(cfg_template_filename, cfg_filename)
if os.path.isfile(cfg_filename):
log.info(
"Application kernel configuration for %s on %s is in: \n\t%s",
appkernel, resource, cfg_filename)
def task_delete_selection(resource: str = None, appkernel: str = None, nodes: str = None, group_id: str = None,
active_tasks=False, scheduled_tasks=False):
"""
delete tasks from schedule
"""
from akrr import akrrrestclient
import json
from akrr.db import get_akrr_db
from akrr.daemon import delete_task
import time
if not (resource or appkernel or nodes or group_id):
raise AkrrValueException("Something out of resource/appkernel/nodes/group id should be set!")
db, cur = get_akrr_db(dict_cursor=True)
# ask scheduler not to start new tasks
if akrrrestclient.post('/scheduler/no_new_tasks').status_code != 200:
raise AkrrRestAPIException("Can not post scheduler/no_new_tasks")
if active_tasks:
# Now we need to wait till scheduler will be done checking active tasks
while True:
sql = "SELECT task_id FROM active_tasks WHERE task_lock > 0"
log.debug(sql)
cur.execute(sql)
n_active_checking_task = len(cur.fetchall())
if n_active_checking_task==0:
break
log.info("There are %d task which daemon is actively working on, waiting for it to pause.", n_active_checking_task)
time.sleep(5)
# now daemon is not working on any tasks
# now we can work with db
where = []
if resource:
where.append("resource='%s'" % resource)
if appkernel:
appkernel_list = ["'" + ak.strip() + "'" for ak in appkernel.split(',')] if ',' in appkernel else ["'" + appkernel + "'"]
where.append("app IN (" + ",".join(appkernel_list) + ")")
if group_id:
where.append("group_id='%s'" % group_id)
active_tasks_ids = []
if nodes:
node_list = [int(node.strip()) for node in nodes.split(',')] if ',' in nodes else [int(nodes)]
for node in node_list:
where_node1 = where + ["resource_param LIKE \"%'nnodes':"+str(node)+"}%\""]
where_node2 = where + ["resource_param LIKE \"%'nnodes':"+str(node)+",%\""]
for where_node in [where_node1, where_node2]:
if scheduled_tasks:
sql = "DELETE FROM scheduled_tasks WHERE " + " AND ".join(where_node)
log.debug(sql)
cur.execute(sql)
if active_tasks:
sql = "SELECT task_id FROM active_tasks WHERE " + " AND ".join(where_node)
log.debug(sql)
cur.execute(sql)
active_tasks_ids += [int(t['task_id']) for t in cur.fetchall()]
else:
if scheduled_tasks:
sql = "DELETE FROM scheduled_tasks WHERE " + " AND ".join(where)
log.debug(sql)
cur.execute(sql)
if active_tasks:
sql = "SELECT task_id FROM active_tasks WHERE " + " AND ".join(where)
log.debug(sql)
cur.execute(sql)
active_tasks_ids += [int(t['task_id']) for t in cur.fetchall()]
if active_tasks:
if len(active_tasks_ids)==0:
log.info("No active tasks to delete")
else:
for task_id in active_tasks_ids:
log.info("Deleting task_id %d", task_id)
delete_task(task_id, remove_from_scheduled_queue=False, remove_from_active_queue=True,
remove_derived_task=False)
if scheduled_tasks or active_tasks:
db.commit()
# ask scheduler can start new tasks now
if akrrrestclient.post('/scheduler/new_tasks_on').status_code != 200:
raise AkrrRestAPIException("Can not post scheduler/new_tasks_on")
log.info("Done")