def put(self, doc_name, config_dict=None):
"""
"""
if doc_name == "DEFAULT":
return ReqMgrConfigDataCache.putDefaultConfig()
else:
return ReqMgrConfigDataCache.replaceConfig(doc_name, config_dict)
def put(self, doc_name, config_dict=None):
"""
"""
if doc_name == "DEFAULT":
return ReqMgrConfigDataCache.putDefaultConfig()
else:
return ReqMgrConfigDataCache.replaceConfig(doc_name, config_dict)
def __init__(self, config):
self.config = config
cherrypy.log("Creating CouchDB connection instances to "
"CouchDB host: '%s' ..." % config.couch_host)
# each CouchDB database will be available under it name in this instance
dbs = (config.couch_reqmgr_db, config.couch_reqmgr_aux_db)
# will be necessary later:
#config.couch_config_cache_db,
#config.couch_workload_summary_db,
#config.couch_wmstats_db,
for db in dbs:
setattr(self, db, self._create_conn(db))
aux_db = self.get_db(config.couch_reqmgr_aux_db)
ReqMgrConfigDataCache.set_aux_db(aux_db)
def format_algo_web_list(task_name, task_type, split_param):
algo_config = ReqMgrConfigDataCache.getConfig("EDITABLE_SPLITTING_PARAM_CONFIG")
fdict = {"taskName": task_name}
fdict["taskType"] = task_type
default_algo = split_param["algorithm"]
algo_list = algo_config["algo_list_by_types"][task_type]
param_list = []
if default_algo in algo_list:
new_param = {"algorithm": default_algo}
for key, value in split_param.items():
if key in algo_config["algo_params"][default_algo]:
new_param[key] = value
param_list.append(new_param)
elif default_algo == "":
raise cherrypy.HTTPError(400, "Algorithm name is empty: %s" % split_param)
else:
param_list.append(split_param)
# If task type is merge don't allow change the algorithm
if fdict["taskType"] != "Merge":
for algo in algo_list:
if algo != default_algo:
param = {"algorithm": algo}
param.update(algo_config["algo_params"][algo])
param_list.append(param)
fdict["splitParamList"] = param_list
return fdict
def __init__(self, config):
self.config = config
cherrypy.log("Creating CouchDB connection instances to "
"CouchDB host: '%s' ..." % config.couch_host)
# each CouchDB database will be available under it name in this instance
dbs = (config.couch_reqmgr_db,
config.couch_reqmgr_aux_db)
# will be necessary later:
#config.couch_config_cache_db,
#config.couch_workload_summary_db,
#config.couch_wmstats_db,
for db in dbs:
setattr(self, db, self._create_conn(db))
aux_db = self.get_db(config.couch_reqmgr_aux_db)
ReqMgrConfigDataCache.set_aux_db(aux_db)
def format_algo_web_list(task_name, task_type, split_param):
algo_config = ReqMgrConfigDataCache.getConfig(
"EDITABLE_SPLITTING_PARAM_CONFIG")
fdict = {"taskName": task_name}
fdict["taskType"] = task_type
default_algo = split_param["algorithm"]
algo_list = algo_config["algo_list_by_types"][task_type]
param_list = []
if default_algo in algo_list:
new_param = {"algorithm": default_algo}
for key, value in split_param.items():
if key in algo_config["algo_params"][default_algo]:
new_param[key] = value
param_list.append(new_param)
elif default_algo == "":
raise cherrypy.HTTPError(400,
"Algorithm name is empty: %s" % split_param)
else:
param_list.append(split_param)
# If task type is merge don't allow change the algorithm
if fdict["taskType"] != "Merge":
for algo in algo_list:
if algo != default_algo:
param = {"algorithm": algo}
param.update(algo_config["algo_params"][algo])
param_list.append(param)
fdict["splitParamList"] = param_list
return fdict
def _maskResult(self, mask, result):
"""
If a mask of parameters was provided in the query string, then filter
the request key/values accordingly.
:param mask: a list of strings (keys of the request dictionary)
:param result: a dict key'ed by the request name, with the whole
request dictionary as a value
:return: updates the result object in place and returns it (dict)
"""
if len(mask) == 1 and mask[0] == "DAS":
mask = ReqMgrConfigDataCache.getConfig("DAS_RESULT_FILTER")["filter_list"]
if len(mask) > 0:
maskedResult = {}
for reqName, reqDict in result.items():
reqInfo = RequestInfo(reqDict)
maskedResult.setdefault(reqName, {})
for maskKey in mask:
foundValue = reqInfo.get(maskKey, None)
maskedResult[reqName].update({maskKey: foundValue})
return maskedResult
else:
return result
def create_web_splitting_format(split_info):
web_form = []
splitSettings = ReqMgrConfigDataCache.getConfig("EDITABLE_SPLITTING_PARAM_CONFIG")
for sp in split_info:
# skip Cleanup and LogCollect: don't allow change the param
if sp["taskType"] not in ["Cleanup", "LogCollect"]:
web_form.append(format_algo_web_list(sp["taskName"], sp["taskType"],
sp["splitParams"], splitSettings))
return web_form
def create_web_splitting_format(split_info):
web_form = []
splitSettings = ReqMgrConfigDataCache.getConfig("EDITABLE_SPLITTING_PARAM_CONFIG")
for sp in split_info:
# skip Cleanup and LogCollect: don't allow change the param
if sp["taskType"] not in ["Cleanup", "LogCollect"]:
web_form.append(format_algo_web_list(sp["taskName"], sp["taskType"],
sp["splitParams"], splitSettings))
return web_form
def getWritePermission(request_args):
requestType = request_args["RequestType"]
requestStatus = request_args.get("RequestStatus", None)
permission_config = ReqMgrConfigDataCache.getConfig("PERMISSION_BY_REQUEST_TYPE")
default_permission = permission_config["DEFAULT_STATUS"]
if requestStatus is None:
return permission_config["NOSTATUS"]
elif requestType not in permission_config:
return permission_config["DEFAULT_TYPE"].get(requestStatus, default_permission)
else:
return permission_config[requestType].get(requestStatus, default_permission)
def _validate_split_param(split_algo, split_param):
"""
validate param for editing, also returns param type
"""
algo_config = ReqMgrConfigDataCache.getConfig("EDITABLE_SPLITTING_PARAM_CONFIG")
valid_params = algo_config["algo_params"][split_algo]
if split_param in valid_params:
if isinstance(valid_params[split_param], bool):
cast_type = bool
else:
cast_type = int
return (True, cast_type)
else:
return (False, None)
def post(self, name):
"""
Parse job splitting parameters sent from the splitting parameter update
page. Pull down the request and modify the new spec applying the
updated splitting parameters.
"""
data = cherrypy.request.body.read()
splittingInfo = json.loads(data)
helper = WMWorkloadHelper()
try:
helper.loadSpecFromCouch(self.reqdb_url, name)
except Exception:
raise cherrypy.HTTPError(404,
"Cannot find workload for: %s" % name)
splitSettings = ReqMgrConfigDataCache.getConfig(
"EDITABLE_SPLITTING_PARAM_CONFIG")
for taskInfo in splittingInfo:
splittingTask = taskInfo["taskName"]
splittingAlgo = taskInfo["splitAlgo"]
submittedParams = taskInfo["splitParams"]
splitParams = {}
for param in submittedParams:
validFlag, castType = _validate_split_param(
splittingAlgo, param, splitSettings)
if validFlag:
_assign_key_value(param, submittedParams[param],
splitParams, castType)
else:
msg = "Parameter '%s' is not supported in the algorithm '%s'" % (
param, splittingAlgo)
raise cherrypy.HTTPError(400, msg)
helper.setJobSplittingParameters(splittingTask,
splittingAlgo,
splitParams,
updateOnly=True)
# Now persist all these changes in the workload
url = "%s/%s" % (self.reqdb_url, name)
result = helper.saveCouchUrl(url)
return result
def _mask_result(self, mask, result):
if len(mask) == 1 and mask[0] == "DAS":
mask = ReqMgrConfigDataCache.getConfig("DAS_RESULT_FILTER")["filter_list"]
if len(mask) > 0:
masked_result = {}
for req_name, req_info in result.items():
masked_result.setdefault(req_name, {})
for mask_key in mask:
masked_result[req_name].update({mask_key: req_info.get(mask_key, None)})
if "TaskChain" in req_info:
self._maskTaskStepChain(masked_result[req_name], req_info, "Task", mask_key)
elif "StepChain" in req_info:
self._maskTaskStepChain(masked_result[req_name], req_info,"Step", mask_key)
return masked_result
else:
return result
def _mask_result(self, mask, result):
if len(mask) == 1 and mask[0] == "DAS":
mask = ReqMgrConfigDataCache.getConfig("DAS_RESULT_FILTER")["filter_list"]
if len(mask) > 0:
masked_result = {}
for req_name, req_info in result.items():
masked_result.setdefault(req_name, {})
for mask_key in mask:
masked_result[req_name].update({mask_key: req_info.get(mask_key, None)})
if "TaskChain" in req_info:
self._maskTaskStepChain(masked_result[req_name], req_info, "Task", mask_key)
elif "StepChain" in req_info:
self._maskTaskStepChain(masked_result[req_name], req_info, "Step", mask_key)
return masked_result
else:
return result
def create_updatable_splitting_format(split_info):
"""
_create_updatable_splitting_format_
Returns the workflow job splitting without parameters that
cannot be updated in the POST call.
"""
splitInfo = []
splitSettings = ReqMgrConfigDataCache.getConfig("EDITABLE_SPLITTING_PARAM_CONFIG")
for taskInfo in split_info:
if taskInfo["taskType"] not in ["Cleanup", "LogCollect"]:
splittingAlgo = taskInfo["splitAlgo"]
submittedParams = taskInfo["splitParams"]
splitParams = {}
for param in submittedParams:
validFlag, _ = _validate_split_param(splittingAlgo, param, splitSettings)
if validFlag:
splitParams[param] = taskInfo["splitParams"][param]
taskInfo["splitParams"] = splitParams
splitInfo.append(taskInfo)
return splitInfo
def create_updatable_splitting_format(split_info):
"""
_create_updatable_splitting_format_
Returns the workflow job splitting without parameters that
cannot be updated in the POST call.
"""
splitInfo = []
splitSettings = ReqMgrConfigDataCache.getConfig("EDITABLE_SPLITTING_PARAM_CONFIG")
for taskInfo in split_info:
if taskInfo["taskType"] not in ["Cleanup", "LogCollect"]:
splittingAlgo = taskInfo["splitAlgo"]
submittedParams = taskInfo["splitParams"]
splitParams = {}
for param in submittedParams:
validFlag, _ = _validate_split_param(splittingAlgo, param, splitSettings)
if validFlag:
splitParams[param] = taskInfo["splitParams"][param]
taskInfo["splitParams"] = splitParams
splitInfo.append(taskInfo)
return splitInfo
def post(self, name):
"""
Parse job splitting parameters sent from the splitting parameter update
page. Pull down the request and modify the new spec applying the
updated splitting parameters.
"""
data = cherrypy.request.body.read()
splittingInfo = json.loads(data)
cherrypy.log("Updating job splitting for '%s' with these args: %s" % (name, splittingInfo))
helper = WMWorkloadHelper()
try:
helper.loadSpecFromCouch(self.reqdb_url, name)
except Exception:
raise cherrypy.HTTPError(404, "Cannot find workload for: %s" % name)
splitSettings = ReqMgrConfigDataCache.getConfig("EDITABLE_SPLITTING_PARAM_CONFIG")
for taskInfo in splittingInfo:
splittingTask = taskInfo["taskName"]
splittingAlgo = taskInfo["splitAlgo"]
submittedParams = taskInfo["splitParams"]
splitParams = {}
for param in submittedParams:
validFlag, castType = _validate_split_param(splittingAlgo, param, splitSettings)
if validFlag:
_assign_key_value(param, submittedParams[param], splitParams, castType)
else:
msg = "Parameter '%s' is not supported in the algorithm '%s'" % (param, splittingAlgo)
raise cherrypy.HTTPError(400, msg)
helper.setJobSplittingParameters(splittingTask, splittingAlgo, splitParams, updateOnly=True)
# Now persist all these changes in the workload
url = "%s/%s" % (self.reqdb_url, name)
result = helper.saveCouchUrl(url)
return result
def get(self, doc_name):
"""
"""
config = ReqMgrConfigDataCache.getConfig(doc_name)
return rows([config])
def get(self, doc_name):
"""
"""
config = ReqMgrConfigDataCache.getConfig(doc_name)
return rows([config])
