def put(self, doc_name, config_dict=None): """ """ if doc_name == "DEFAULT": return ReqMgrConfigDataCache.putDefaultConfig() else: return ReqMgrConfigDataCache.replaceConfig(doc_name, config_dict)
def __init__(self, config): self.config = config cherrypy.log("Creating CouchDB connection instances to " "CouchDB host: '%s' ..." % config.couch_host) # each CouchDB database will be available under it name in this instance dbs = (config.couch_reqmgr_db, config.couch_reqmgr_aux_db) # will be necessary later: #config.couch_config_cache_db, #config.couch_workload_summary_db, #config.couch_wmstats_db, for db in dbs: setattr(self, db, self._create_conn(db)) aux_db = self.get_db(config.couch_reqmgr_aux_db) ReqMgrConfigDataCache.set_aux_db(aux_db)
def format_algo_web_list(task_name, task_type, split_param): algo_config = ReqMgrConfigDataCache.getConfig("EDITABLE_SPLITTING_PARAM_CONFIG") fdict = {"taskName": task_name} fdict["taskType"] = task_type default_algo = split_param["algorithm"] algo_list = algo_config["algo_list_by_types"][task_type] param_list = [] if default_algo in algo_list: new_param = {"algorithm": default_algo} for key, value in split_param.items(): if key in algo_config["algo_params"][default_algo]: new_param[key] = value param_list.append(new_param) elif default_algo == "": raise cherrypy.HTTPError(400, "Algorithm name is empty: %s" % split_param) else: param_list.append(split_param) # If task type is merge don't allow change the algorithm if fdict["taskType"] != "Merge": for algo in algo_list: if algo != default_algo: param = {"algorithm": algo} param.update(algo_config["algo_params"][algo]) param_list.append(param) fdict["splitParamList"] = param_list return fdict
def format_algo_web_list(task_name, task_type, split_param): algo_config = ReqMgrConfigDataCache.getConfig( "EDITABLE_SPLITTING_PARAM_CONFIG") fdict = {"taskName": task_name} fdict["taskType"] = task_type default_algo = split_param["algorithm"] algo_list = algo_config["algo_list_by_types"][task_type] param_list = [] if default_algo in algo_list: new_param = {"algorithm": default_algo} for key, value in split_param.items(): if key in algo_config["algo_params"][default_algo]: new_param[key] = value param_list.append(new_param) elif default_algo == "": raise cherrypy.HTTPError(400, "Algorithm name is empty: %s" % split_param) else: param_list.append(split_param) # If task type is merge don't allow change the algorithm if fdict["taskType"] != "Merge": for algo in algo_list: if algo != default_algo: param = {"algorithm": algo} param.update(algo_config["algo_params"][algo]) param_list.append(param) fdict["splitParamList"] = param_list return fdict
def _maskResult(self, mask, result): """ If a mask of parameters was provided in the query string, then filter the request key/values accordingly. :param mask: a list of strings (keys of the request dictionary) :param result: a dict key'ed by the request name, with the whole request dictionary as a value :return: updates the result object in place and returns it (dict) """ if len(mask) == 1 and mask[0] == "DAS": mask = ReqMgrConfigDataCache.getConfig("DAS_RESULT_FILTER")["filter_list"] if len(mask) > 0: maskedResult = {} for reqName, reqDict in result.items(): reqInfo = RequestInfo(reqDict) maskedResult.setdefault(reqName, {}) for maskKey in mask: foundValue = reqInfo.get(maskKey, None) maskedResult[reqName].update({maskKey: foundValue}) return maskedResult else: return result
def create_web_splitting_format(split_info): web_form = [] splitSettings = ReqMgrConfigDataCache.getConfig("EDITABLE_SPLITTING_PARAM_CONFIG") for sp in split_info: # skip Cleanup and LogCollect: don't allow change the param if sp["taskType"] not in ["Cleanup", "LogCollect"]: web_form.append(format_algo_web_list(sp["taskName"], sp["taskType"], sp["splitParams"], splitSettings)) return web_form
def getWritePermission(request_args): requestType = request_args["RequestType"] requestStatus = request_args.get("RequestStatus", None) permission_config = ReqMgrConfigDataCache.getConfig("PERMISSION_BY_REQUEST_TYPE") default_permission = permission_config["DEFAULT_STATUS"] if requestStatus is None: return permission_config["NOSTATUS"] elif requestType not in permission_config: return permission_config["DEFAULT_TYPE"].get(requestStatus, default_permission) else: return permission_config[requestType].get(requestStatus, default_permission)
def _validate_split_param(split_algo, split_param): """ validate param for editing, also returns param type """ algo_config = ReqMgrConfigDataCache.getConfig("EDITABLE_SPLITTING_PARAM_CONFIG") valid_params = algo_config["algo_params"][split_algo] if split_param in valid_params: if isinstance(valid_params[split_param], bool): cast_type = bool else: cast_type = int return (True, cast_type) else: return (False, None)
def post(self, name): """ Parse job splitting parameters sent from the splitting parameter update page. Pull down the request and modify the new spec applying the updated splitting parameters. """ data = cherrypy.request.body.read() splittingInfo = json.loads(data) helper = WMWorkloadHelper() try: helper.loadSpecFromCouch(self.reqdb_url, name) except Exception: raise cherrypy.HTTPError(404, "Cannot find workload for: %s" % name) splitSettings = ReqMgrConfigDataCache.getConfig( "EDITABLE_SPLITTING_PARAM_CONFIG") for taskInfo in splittingInfo: splittingTask = taskInfo["taskName"] splittingAlgo = taskInfo["splitAlgo"] submittedParams = taskInfo["splitParams"] splitParams = {} for param in submittedParams: validFlag, castType = _validate_split_param( splittingAlgo, param, splitSettings) if validFlag: _assign_key_value(param, submittedParams[param], splitParams, castType) else: msg = "Parameter '%s' is not supported in the algorithm '%s'" % ( param, splittingAlgo) raise cherrypy.HTTPError(400, msg) helper.setJobSplittingParameters(splittingTask, splittingAlgo, splitParams, updateOnly=True) # Now persist all these changes in the workload url = "%s/%s" % (self.reqdb_url, name) result = helper.saveCouchUrl(url) return result
def _mask_result(self, mask, result): if len(mask) == 1 and mask[0] == "DAS": mask = ReqMgrConfigDataCache.getConfig("DAS_RESULT_FILTER")["filter_list"] if len(mask) > 0: masked_result = {} for req_name, req_info in result.items(): masked_result.setdefault(req_name, {}) for mask_key in mask: masked_result[req_name].update({mask_key: req_info.get(mask_key, None)}) if "TaskChain" in req_info: self._maskTaskStepChain(masked_result[req_name], req_info, "Task", mask_key) elif "StepChain" in req_info: self._maskTaskStepChain(masked_result[req_name], req_info,"Step", mask_key) return masked_result else: return result
def _mask_result(self, mask, result): if len(mask) == 1 and mask[0] == "DAS": mask = ReqMgrConfigDataCache.getConfig("DAS_RESULT_FILTER")["filter_list"] if len(mask) > 0: masked_result = {} for req_name, req_info in result.items(): masked_result.setdefault(req_name, {}) for mask_key in mask: masked_result[req_name].update({mask_key: req_info.get(mask_key, None)}) if "TaskChain" in req_info: self._maskTaskStepChain(masked_result[req_name], req_info, "Task", mask_key) elif "StepChain" in req_info: self._maskTaskStepChain(masked_result[req_name], req_info, "Step", mask_key) return masked_result else: return result
def create_updatable_splitting_format(split_info): """ _create_updatable_splitting_format_ Returns the workflow job splitting without parameters that cannot be updated in the POST call. """ splitInfo = [] splitSettings = ReqMgrConfigDataCache.getConfig("EDITABLE_SPLITTING_PARAM_CONFIG") for taskInfo in split_info: if taskInfo["taskType"] not in ["Cleanup", "LogCollect"]: splittingAlgo = taskInfo["splitAlgo"] submittedParams = taskInfo["splitParams"] splitParams = {} for param in submittedParams: validFlag, _ = _validate_split_param(splittingAlgo, param, splitSettings) if validFlag: splitParams[param] = taskInfo["splitParams"][param] taskInfo["splitParams"] = splitParams splitInfo.append(taskInfo) return splitInfo
def post(self, name): """ Parse job splitting parameters sent from the splitting parameter update page. Pull down the request and modify the new spec applying the updated splitting parameters. """ data = cherrypy.request.body.read() splittingInfo = json.loads(data) cherrypy.log("Updating job splitting for '%s' with these args: %s" % (name, splittingInfo)) helper = WMWorkloadHelper() try: helper.loadSpecFromCouch(self.reqdb_url, name) except Exception: raise cherrypy.HTTPError(404, "Cannot find workload for: %s" % name) splitSettings = ReqMgrConfigDataCache.getConfig("EDITABLE_SPLITTING_PARAM_CONFIG") for taskInfo in splittingInfo: splittingTask = taskInfo["taskName"] splittingAlgo = taskInfo["splitAlgo"] submittedParams = taskInfo["splitParams"] splitParams = {} for param in submittedParams: validFlag, castType = _validate_split_param(splittingAlgo, param, splitSettings) if validFlag: _assign_key_value(param, submittedParams[param], splitParams, castType) else: msg = "Parameter '%s' is not supported in the algorithm '%s'" % (param, splittingAlgo) raise cherrypy.HTTPError(400, msg) helper.setJobSplittingParameters(splittingTask, splittingAlgo, splitParams, updateOnly=True) # Now persist all these changes in the workload url = "%s/%s" % (self.reqdb_url, name) result = helper.saveCouchUrl(url) return result
def get(self, doc_name): """ """ config = ReqMgrConfigDataCache.getConfig(doc_name) return rows([config])