class ReqMgrTest(RESTBaseUnitTestWithDBBackend): """ Test WorkQueue Service client It will start WorkQueue RESTService Server DB sets from environment variable. Client DB sets from environment variable. This checks whether DS call makes without error and return the results. Not the correctness of functions. That will be tested in different module. """ def setFakeDN(self): # put into ReqMgr auxiliary database under "software" document scram/cmsms # which we'll need a little for request injection #Warning: this assumes the same structure in jenkins wmcore_root/test self.admin_header = getAuthHeader(self.test_authz_key.data, ADMIN_PERMISSION) self.create_header = getAuthHeader(self.test_authz_key.data, CREATE_PERMISSION) self.default_header = getAuthHeader(self.test_authz_key.data, DEFAULT_PERMISSION) self.assign_header = getAuthHeader(self.test_authz_key.data, ASSIGN_PERMISSION) self.default_status_header = getAuthHeader(self.test_authz_key.data, DEFAULT_STATUS_PERMISSION) def setUp(self): self.setConfig(config) self.setCouchDBs([(config.views.data.couch_reqmgr_db, "ReqMgr"), (config.views.data.couch_reqmgr_aux_db, None)]) self.setSchemaModules([]) RESTBaseUnitTestWithDBBackend.setUp(self) self.setFakeDN() requestPath = os.path.join(getWMBASE(), "test", "data", "ReqMgr", "requests", "DMWM") rerecoFile = open(os.path.join(requestPath, "ReReco.json"), 'r') rerecoArgs = JsonWrapper.load(rerecoFile) self.rerecoCreateArgs = rerecoArgs["createRequest"] self.rerecoAssignArgs = rerecoArgs["assignRequest"] cmsswDoc = {"_id": "software"} cmsswDoc[self.rerecoCreateArgs["ScramArch"]] = [] cmsswDoc[self.rerecoCreateArgs["ScramArch"]].append(self.rerecoCreateArgs["CMSSWVersion"]) insertDataToCouch(os.getenv("COUCHURL"), config.views.data.couch_reqmgr_aux_db, cmsswDoc) self.reqSvc = ReqMgr(self.jsonSender["host"]) self.reqSvc._noStale = True self.reqSvc['requests'].additionalHeaders = self.create_header def tearDown(self): RESTBaseUnitTestWithDBBackend.tearDown(self) def testRequestSimpleCycle(self): """ test request cycle with one request without composite get condition. post, get, put """ # test post method response = self.reqSvc.insertRequests(self.rerecoCreateArgs) self.assertEqual(len(response), 1) requestName = response[0]['request'] ## test get method # get by name response = self.reqSvc.getRequestByNames(requestName) self.assertEqual(response[requestName]['RequestPriority'], 10000) self.assertEqual(len(response), 1) # get by status response = self.reqSvc.getRequestByStatus('new') self.assertEqual(len(response), 1) print(response) self.reqSvc.updateRequestStatus(requestName, 'assignment-approved') response = self.reqSvc.getRequestByStatus('assignment-approved') self.assertEqual(len(response), 1) self.reqSvc.updateRequestProperty(requestName, {'RequestStatus': 'assigned', "AcquisitionEra": "TEST_ERA", "Team": "unittest", "SiteWhitelist": ["T1_US_CBS"], "SiteBlacklist": ["T1_US_FOX"]}) response = self.reqSvc.getRequestByStatus('assignment-approved') self.assertEqual(len(response), 0) response = self.reqSvc.getRequestByStatus('assigned') self.assertEqual(len(response), 1) self.assertEqual(response.values()[0]["SiteWhitelist"], ["T1_US_CBS"]) self.reqSvc.updateRequestStats(requestName, {'total_jobs': 100, 'input_lumis': 100, 'input_events': 100, 'input_num_files': 100})
class WorkQueueReqMgrInterface(object): """Helper class for ReqMgr interaction""" def __init__(self, **kwargs): if not kwargs.get('logger'): import logging kwargs['logger'] = logging self.logger = kwargs['logger'] # this will break all in one test self.reqMgr2 = ReqMgr(kwargs.get("reqmgr2_endpoint", None)) centralurl = kwargs.get("central_logdb_url", "") identifier = kwargs.get("log_reporter", "") # set the thread name before creat the log db. # only sets that when it is not set already myThread = threading.currentThread() if myThread.getName() == "MainThread": myThread.setName(self.__class__.__name__) self.logdb = LogDB(centralurl, identifier, logger=self.logger) self.previous_state = {} def __call__(self, queue): """Synchronize WorkQueue and RequestManager""" msg = '' try: # pull in new work work = self.queueNewRequests(queue) msg += "New Work: %d\n" % work except Exception: self.logger.exception("Error caught during RequestManager pull") try: # get additional open-running work extraWork = self.addNewElementsToOpenRequests(queue) msg += "Work added: %d\n" % extraWork except Exception: self.logger.exception("Error caught during RequestManager split") try: # report back to ReqMgr uptodate_elements = self.report(queue) msg += "Updated ReqMgr status for: %s\n" % ", ".join([x['RequestName'] for x in uptodate_elements]) except Exception: self.logger.exception("Error caught during RequestManager update") else: try: # Delete finished requests from WorkQueue self.deleteFinishedWork(queue, uptodate_elements) except Exception: self.logger.exception("Error caught during work deletion") queue.backend.recordTaskActivity('reqmgr_sync', msg) def queueNewRequests(self, queue): """Get requests from regMgr and queue to workqueue""" self.logger.info("Contacting Request manager for more work") work = 0 workLoads = [] if queue.params['DrainMode']: self.logger.info('Draining queue: Skip requesting work from ReqMgr') return 0 try: workLoads = self.getAvailableRequests() except Exception as ex: traceMsg = traceback.format_exc() msg = "Error contacting RequestManager: %s" % traceMsg self.logger.warning(msg) return 0 for team, reqName, workLoadUrl in workLoads: try: try: Lexicon.couchurl(workLoadUrl) except Exception as ex: # can throw many errors e.g. AttributeError, AssertionError etc. # check its not a local file if not os.path.exists(workLoadUrl): error = WorkQueueWMSpecError(None, "Workflow url validation error: %s" % str(ex)) raise error self.logger.info("Processing request %s at %s" % (reqName, workLoadUrl)) units = queue.queueWork(workLoadUrl, request=reqName, team=team) self.logdb.delete(reqName, "error", this_thread=True) except TERMINAL_EXCEPTIONS as ex: # fatal error - report back to ReqMgr self.logger.error('Permanent failure processing request "%s": %s' % (reqName, str(ex))) self.logger.info("Marking request %s as failed in ReqMgr" % reqName) self.reportRequestStatus(reqName, 'Failed', message=str(ex)) continue except (IOError, socket.error, CouchError, CouchConnectionError) as ex: # temporary problem - try again later msg = 'Error processing request "%s": will try again later.' % reqName msg += '\nError: "%s"' % str(ex) self.logger.info(msg) self.logdb.post(reqName, msg, 'error') continue except Exception as ex: # Log exception as it isnt a communication problem msg = 'Error processing request "%s": will try again later.' % reqName msg += '\nSee log for details.\nError: "%s"' % str(ex) self.logger.exception('Unknown error processing %s' % reqName) self.logdb.post(reqName, msg, 'error') continue try: self.reportRequestStatus(reqName, "acquired") except Exception as ex: self.logger.warning("Unable to update ReqMgr state: %s" % str(ex)) self.logger.warning('Will try again later') self.logger.info('%s units(s) queued for "%s"' % (units, reqName)) work += units self.logger.info("%s element(s) obtained from RequestManager" % work) return work def report(self, queue): """Report queue status to ReqMgr.""" new_state = {} uptodate_elements = [] now = time.time() elements = queue.statusInbox(dictKey="RequestName") if not elements: return new_state for ele in elements: ele = elements[ele][0] # 1 element tuple try: request = self.reqMgr2.getRequestByNames(ele['RequestName']) if not request: msg = 'Failed to get request "%s" from ReqMgr2. Will try again later.' % ele['RequestName'] self.logger.warning(msg) continue request = request[0][ele['RequestName']] if request['RequestStatus'] in ('failed', 'completed', 'announced', 'epic-FAILED', 'closed-out', 'rejected'): # requests can be done in reqmgr but running in workqueue # if request has been closed but agent cleanup actions # haven't been run (or agent has been retired) # Prune out obviously too old ones to avoid build up if queue.params.get('reqmgrCompleteGraceTime', -1) > 0: if (now - float(ele.updatetime)) > queue.params['reqmgrCompleteGraceTime']: # have to check all elements are at least running and are old enough request_elements = queue.statusInbox(WorkflowName=request['RequestName']) if not any( [x for x in request_elements if x['Status'] != 'Running' and not x.inEndState()]): last_update = max([float(x.updatetime) for x in request_elements]) if (now - last_update) > queue.params['reqmgrCompleteGraceTime']: self.logger.info( "Finishing request %s as it is done in reqmgr" % request['RequestName']) queue.doneWork(WorkflowName=request['RequestName']) continue else: pass # assume workqueue status will catch up later elif request['RequestStatus'] in ['aborted', 'force-complete']: queue.cancelWork(WorkflowName=request['RequestName']) # Check consistency of running-open/closed and the element closure status elif request['RequestStatus'] == 'running-open' and not ele.get('OpenForNewData', False): self.reportRequestStatus(ele['RequestName'], 'running-closed') elif request['RequestStatus'] == 'running-closed' and ele.get('OpenForNewData', False): queue.closeWork(ele['RequestName']) # we do not want to move the request to 'failed' status elif ele['Status'] == 'Failed': continue elif ele['Status'] not in self._reqMgrToWorkQueueStatus(request['RequestStatus']): self.reportElement(ele) uptodate_elements.append(ele) except Exception as ex: msg = 'Error talking to ReqMgr about request "%s": %s' % (ele['RequestName'], str(ex)) self.logger.exception(msg) return uptodate_elements def deleteFinishedWork(self, queue, elements): """Delete work from queue that is finished in ReqMgr""" finished = [] for element in elements: if element.inEndState() and self._workQueueToReqMgrStatus(element['Status']) in ('aborted', 'failed', 'completed', 'announced', 'epic-FAILED', 'closed-out', 'rejected'): finished.append(element['RequestName']) return queue.deleteWorkflows(*finished) def getAvailableRequests(self): """ Get available requests and sort by team and priority returns [(team, request_name, request_spec_url)] """ tempResults = self.reqMgr2.getRequestByStatus("assigned") filteredResults = [] for requests in tempResults: for request in requests.values(): filteredResults.append(request) filteredResults.sort(key=itemgetter('RequestPriority'), reverse=True) filteredResults.sort(key=lambda r: r["Teams"][0]) results = [(x["Teams"][0], x["RequestName"], x["RequestWorkflow"]) for x in filteredResults] return results def reportRequestStatus(self, request, status, message=None): """Change state in RequestManager Optionally, take a message to append to the request """ if message: self.logdb.post(request, str(message), 'info') reqmgrStatus = self._workQueueToReqMgrStatus(status) if reqmgrStatus: # only send known states try: self.reqMgr2.updateRequestStatus(request, reqmgrStatus) except Exception as ex: msg = "%s : fail to update status will try later: %s" % (request, str(ex)) msg += traceback.format_exc() self.logdb.post(request, msg, 'warning') raise ex return def _workQueueToReqMgrStatus(self, status): """Map WorkQueue Status to that reported to ReqMgr""" statusMapping = {'Acquired': 'acquired', 'Running': 'running-open', 'Failed': 'failed', 'Canceled': 'aborted', 'CancelRequested': 'aborted', 'Done': 'completed' } if status in statusMapping: # if wq status passed convert to reqmgr status return statusMapping[status] elif status in REQUEST_STATE_LIST: # if reqmgr status passed return reqmgr status return status else: # unknown status return None def _reqMgrToWorkQueueStatus(self, status): """Map ReqMgr status to that in a WorkQueue element, it is not a 1-1 relation""" statusMapping = {'acquired': ['Acquired'], 'running': ['Running'], 'running-open': ['Running'], 'running-closed': ['Running'], 'failed': ['Failed'], 'aborted': ['Canceled', 'CancelRequested'], 'force-complete': ['Canceled', 'CancelRequested'], 'completed': ['Done']} if status in statusMapping: return statusMapping[status] else: return [] def reportElement(self, element): """Report element to ReqMgr""" self.reportRequestStatus(element['RequestName'], element['Status']) def addNewElementsToOpenRequests(self, queue): """Add new elements to open requests which are in running-open state, only works adding new blocks from the input dataset""" self.logger.info("Checking Request Manager for open requests and closing old ones") # First close any open inbox element which hasn't found anything new in a while queue.closeWork() self.report(queue) work = 0 requests = [] # Drain mode, don't pull any work into open requests. They will be closed if the queue stays in drain long enough if queue.params['DrainMode']: self.logger.info('Draining queue: Skip requesting work from ReqMgr') return 0 try: requests = self.reqMgr2.getRequestByStatus("running-open", detail=False) except Exception as ex: traceMsg = traceback.format_exc() msg = "Error contacting RequestManager: %s" % traceMsg self.logger.warning(msg) return 0 for reqName in requests: try: self.logger.info("Processing request %s" % (reqName)) units = queue.addWork(requestName=reqName) self.logdb.delete(reqName, 'error', True) except (WorkQueueWMSpecError, WorkQueueNoWorkError) as ex: # fatal error - but at least it was split the first time. Log and skip. msg = 'Error adding further work to request "%s". Will try again later' % reqName msg += '\nError: "%s"' % str(ex) self.logger.info(msg) self.logdb.post(reqName, msg, 'error') continue except (IOError, socket.error, CouchError, CouchConnectionError) as ex: # temporary problem - try again later msg = 'Error processing request "%s": will try again later.' % reqName msg += '\nError: "%s"' % str(ex) self.logger.info(msg) self.logdb.post(reqName, msg, 'error') continue except Exception as ex: # Log exception as it isnt a communication problem msg = 'Error processing request "%s": will try again later.' % reqName msg += '\nSee log for details.\nError: "%s"' % str(ex) traceMsg = traceback.format_exc() msg = "%s\n%s" % (msg, traceMsg) self.logger.exception('Unknown error processing %s' % reqName) self.logdb.post(reqName, msg, 'error') continue self.logger.info('%s units(s) queued for "%s"' % (units, reqName)) work += units self.logger.info("%s element(s) added to open requests" % work) return work
class ReqMgrService(TemplatedPage): """ Request Manager web service class """ def __init__(self, app, config, mount): self.base = config.base self.rootdir = '/'.join(WMCore.__file__.split('/')[:-1]) if config and not isinstance(config, dict): web_config = config.dictionary_() if not config: web_config = {'base': self.base} TemplatedPage.__init__(self, web_config) imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd() + '/images') self.imgdir = web_config.get('imgdir', imgdir) cssdir = os.environ.get('RM_CSSPATH', os.getcwd() + '/css') self.cssdir = web_config.get('cssdir', cssdir) jsdir = os.environ.get('RM_JSPATH', os.getcwd() + '/js') self.jsdir = web_config.get('jsdir', jsdir) spdir = os.environ.get('RM_SPECPATH', os.getcwd() + '/specs') self.spdir = web_config.get('spdir', spdir) # read scripts area and initialize data-ops scripts self.sdir = os.environ.get('RM_SCRIPTS', os.getcwd() + '/scripts') self.sdir = web_config.get('sdir', self.sdir) self.sdict_thr = web_config.get('sdict_thr', 600) # put reasonable 10 min interval self.sdict = {'ts': time.time()} # placeholder for data-ops scripts self.update_scripts(force=True) # To be filled at run time self.cssmap = {} self.jsmap = {} self.imgmap = {} self.yuimap = {} std_specs_dir = os.path.join(self.rootdir, 'WMSpec/StdSpecs') self.std_specs = spec_list(std_specs_dir, 'WMSpec.StdSpecs') self.std_specs.sort() # Update CherryPy configuration mime_types = ['text/css'] mime_types += [ 'application/javascript', 'text/javascript', 'application/x-javascript', 'text/x-javascript' ] cherryconf.update({ 'tools.encode.on': True, 'tools.gzip.on': True, 'tools.gzip.mime_types': mime_types, }) self._cache = {} # initialize rest API statedir = '/tmp' app = RESTMain(config, statedir) # REST application mount = '/rest' # mount point for cherrypy service api = RestApiHub(app, config.reqmgr, mount) # initialize access to reqmgr2 APIs self.reqmgr = ReqMgr(config.reqmgr.reqmgr2_url) # only gets current view (This might cause to reponse time much longer, # If upto date view is not needed overwrite Fale) self.reqmgr._noStale = True # admin helpers self.admin_info = Info(app, api, config.reqmgr, mount=mount + '/info') self.admin_group = Group(app, api, config.reqmgr, mount=mount + '/group') self.admin_team = Team(app, api, config.reqmgr, mount=mount + '/team') # get fields which we'll use in templates cdict = config.reqmgr.dictionary_() self.couch_url = cdict.get('couch_host', '') self.couch_dbname = cdict.get('couch_reqmgr_db', '') self.couch_wdbname = cdict.get('couch_workload_summary_db', '') self.acdc_url = cdict.get('acdc_host', '') self.acdc_dbname = cdict.get('acdc_db', '') self.configcache_url = cdict.get('couch_config_cache_url', self.couch_url) self.dbs_url = cdict.get('dbs_url', '') self.dqm_url = cdict.get('dqm_url', '') self.sw_ver = cdict.get('default_sw_version', 'CMSSW_5_2_5') self.sw_arch = cdict.get('default_sw_scramarch', 'slc5_amd64_gcc434') def user(self): """ Return user name associated with this instance. """ try: return cherrypy.request.user['login'] except: return 'testuser' def user_dn(self): "Return user DN" try: return cherrypy.request.user['dn'] except: return '/CN/bla/foo' def update_scripts(self, force=False): "Update scripts dict" if force or abs(time.time() - self.sdict['ts']) > self.sdict_thr: for item in os.listdir(self.sdir): with open(os.path.join(self.sdir, item), 'r') as istream: self.sdict[item.split('.')[0]] = istream.read() self.sdict['ts'] = time.time() def abs_page(self, tmpl, content): """generate abstract page""" menu = self.templatepage('menu', menus=menus(), tmpl=tmpl) body = self.templatepage('generic', menu=menu, content=content) page = self.templatepage('main', content=body, user=self.user()) return page def page(self, content): """ Provide page wrapped with top/bottom templates. """ return self.templatepage('main', content=content) def error(self, content): "Generate common error page" content = self.templatepage('error', content=content) return self.abs_page('error', content) @expose def index(self, **kwds): """Main page""" content = self.templatepage('index', requests=ACTIVE_STATUS, rdict=REQUEST_STATE_TRANSITION) return self.abs_page('main', content) @expose def home(self, **kwds): """Main page""" return self.index(**kwds) ### Admin actions ### @expose def admin(self, **kwds): """admin page""" print "\n### ADMIN PAGE" rows = self.admin_info.get() print "rows", [r for r in rows] content = self.templatepage('admin') return self.abs_page('admin', content) @expose def add_user(self, **kwds): """add_user action""" rid = genid(kwds) status = "ok" # chagne to whatever it would be content = self.templatepage('confirm', ticket=rid, user=self.user(), status=status) return self.abs_page('admin', content) @expose def add_group(self, **kwds): """add_group action""" rows = self.admin_group.get() print "\n### GROUPS", [r for r in rows] rid = genid(kwds) status = "ok" # chagne to whatever it would be content = self.templatepage('confirm', ticket=rid, user=self.user(), status=status) return self.abs_page('admin', content) @expose def add_team(self, **kwds): """add_team action""" rows = self.admin_team.get() print "\n### TEAMS", kwds, [r for r in rows] print "request to add", kwds rid = genid(kwds) status = "ok" # chagne to whatever it would be content = self.templatepage('confirm', ticket=rid, user=self.user(), status=status) return self.abs_page('admin', content) ### Request actions ### @expose @checkargs(['status', 'sort']) def assign(self, **kwds): """assign page""" if not kwds: kwds = {} if 'status' not in kwds: kwds.update({'status': 'assignment-approved'}) docs = [] attrs = [ 'RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus' ] data = self.reqmgr.getRequestByStatus(statusList=[kwds['status']]) for key, val in data.items(): docs.append(request_attr(val, attrs)) sortby = kwds.get('sort', 'status') docs = [r for r in sort(docs, sortby)] misc_json = { 'CMSSW Releases': releases(), 'CMSSW architectures': architectures(), 'SubscriptionPriority': ['Low', 'Normal', 'High'], 'CustodialSubType': ['Move', 'Replica'], 'NonCustodialSubType': ['Move', 'Replica'], 'MinMergeSize': 2147483648, 'MaxMergeSize': 4294967296, 'MaxMergeEvents': 50000, 'MaxRSS': 20411724, 'MaxVSize': 20411724, 'SoftTimeout': 129600, 'GracePeriod': 300, 'BlockCloseMaxWaitTime': 66400, 'BlockCloseMaxFiles': 500, 'BlockCloseMaxEvents': 250000000, 'BlockCloseMaxSize': 5000000000000, 'AcquisitionEra': '', 'ProcessingVersion': 1, 'ProcessingString': '', 'MergedLFNBase': lfn_bases(), 'UnmergedLFNBase': lfn_unmerged_bases(), } filter_sort = self.templatepage('filter_sort') content = self.templatepage('assign', sort=sortby, filter_sort_table=filter_sort, sites=sites(), site_white_list=site_white_list(), site_black_list=site_black_list(), user=self.user(), user_dn=self.user_dn(), requests=docs, misc_table=json2table( misc_json, web_ui_names()), misc_json=json2form(misc_json, indent=2, keep_first_value=True)) return self.abs_page('assign', content) @expose @checkargs(['status', 'sort']) def approve(self, **kwds): """ Approve page: get list of request associated with user DN. Fetch their status list from ReqMgr and display if requests were seen by data-ops. """ if not kwds: kwds = {} if 'status' not in kwds: kwds.update({'status': 'new'}) kwds.update({'_nostale': True}) docs = [] attrs = [ 'RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus' ] data = self.reqmgr.getRequestByStatus(statusList=[kwds['status']]) for key, val in data.items(): docs.append(request_attr(val, attrs)) sortby = kwds.get('sort', 'status') docs = [r for r in sort(docs, sortby)] filter_sort = self.templatepage('filter_sort') content = self.templatepage('approve', requests=docs, date=tstamp(), sort=sortby, filter_sort_table=filter_sort) return self.abs_page('approve', content) @expose def create(self, **kwds): """create page""" # get list of standard specs from WMCore and new ones from local area #loc_specs_dir = os.path.join(self.spdir, 'Specs') # local specs #loc_specs = spec_list(loc_specs_dir, 'Specs') #all_specs = list(set(self.std_specs + loc_specs)) #all_specs.sort() all_specs = self.std_specs spec = kwds.get('form', '') if not spec: spec = self.std_specs[0] # make spec first in all_specs list if spec in all_specs: all_specs.remove(spec) all_specs = [spec] + all_specs jsondata = get_request_template_from_type(spec) # create templatized page out of provided forms self.update_scripts() content = self.templatepage( 'create', table=json2table(jsondata, web_ui_names()), jsondata=json2form(jsondata, indent=2, keep_first_value=True), name=spec, scripts=[s for s in self.sdict.keys() if s != 'ts'], specs=all_specs) return self.abs_page('create', content) def generate_objs(self, script, jsondict): """Generate objects from givem JSON template""" self.update_scripts() code = self.sdict.get(script, '') if code.find('def genobjs(jsondict)') == -1: return self.error( "Improper python snippet, your code should start with <b>def genobjs(jsondict)</b> function" ) exec(code) # code snippet must starts with genobjs function return [r for r in genobjs(jsondict)] @expose def fetch(self, rid, **kwds): "Fetch document for given id" rid = rid.replace('request-', '') doc = self.reqmgr.getRequestByNames(rid) transitions = [] if len(doc) == 1: try: doc = doc[rid] except: pass name = doc.get('RequestName', 'NA') title = 'Request %s' % name status = doc.get('RequestStatus', '') transitions = REQUEST_STATE_TRANSITION.get(status, []) if status in transitions: transitions.remove(status) content = self.templatepage('doc', title=title, status=status, name=name, table=json2table(doc, web_ui_names()), jsondata=json2form( doc, indent=2, keep_first_value=False), transitions=transitions) elif len(doc) > 1: jsondata = [pprint.pformat(d) for d in doc] content = self.templatepage('doc', title='Series of docs: %s' % rid, table="", jsondata=jsondata, transitions=transitions) else: doc = 'No request found for name=%s' % rid return self.abs_page('request', content) @expose def requests(self, **kwds): """Page showing requests""" if not kwds: kwds = {} if 'status' not in kwds: kwds.update({'status': 'acquired'}) results = self.reqmgr.getRequestByStatus(kwds['status']) docs = [] for key, doc in results.items(): docs.append(request_attr(doc)) sortby = kwds.get('sort', 'status') docs = [r for r in sort(docs, sortby)] filter_sort = self.templatepage('filter_sort') content = self.templatepage('requests', requests=docs, sort=sortby, status=kwds['status'], filter_sort_table=filter_sort) return self.abs_page('requests', content) @expose def request(self, **kwargs): "Get data example and expose it as json" dataset = kwargs.get('uinput', '') if not dataset: return {'error': 'no input dataset'} url = 'https://cmsweb.cern.ch/reqmgr/rest/outputdataset/%s' % dataset params = {} headers = {'Accept': 'application/json;text/json'} wdata = getdata(url, params) wdict = dict(date=time.ctime(), team='Team-A', status='Running', ID=genid(wdata)) winfo = self.templatepage('workflow', wdict=wdict, dataset=dataset, code=pprint.pformat(wdata)) content = self.templatepage('search', content=winfo) return self.abs_page('request', content) @expose def batch(self, **kwds): """batch page""" # TODO: we need a template for batch attributes # and read it from separate area, like DASMaps name = kwds.get('name', '') batch = {} if name: # batch = self.reqmgr.getBatchesByName(name) batch = { 'Name': 'Batch1', 'Description': 'Bla-bla', 'Creator': 'valya', 'Group': 'test', 'Workflows': ['workflow1', 'workflow2'], 'Attributes': { 'HeavyIon': ['true', 'false'] } } attributes = batch.get('Attributes', {}) workflows = batch.get('Workflows', []) description = batch.get('Description', '') creator = batch.get('Creator', self.user_dn()) content = self.templatepage('batch', name=name, attributes=json2table( attributes, web_ui_names()), workflows=workflows, creator=creator, description=description) return self.abs_page('batch', content) @expose def batches(self, **kwds): """Page showing batches""" if not kwds: kwds = {} if 'name' not in kwds: kwds.update({'name': ''}) sortby = kwds.get('sort', 'name') # results = self.reqmgr.getBatchesByName(kwds['name']) results = [ { 'Name': 'Batch1', 'Description': 'Bla-bla', 'Creator': 'valya', 'Group': 'test', 'Workflows': ['workflow1', 'workflow2'], 'Date': 'Fri Feb 13 10:36:41 EST 2015', 'Attributes': { 'HeavyIon': ['true', 'false'] } }, { 'Name': 'Batch2', 'Description': 'lksdjflksjdf', 'Creator': 'valya', 'Group': 'test', 'Workflows': ['workflow1', 'workflow2'], 'Date': 'Fri Feb 10 10:36:41 EST 2015', 'Attributes': { 'HeavyIon': ['true', 'false'] } }, ] docs = [r for r in sort(results, sortby)] filter_sort = self.templatepage('filter_sort') content = self.templatepage('batches', batches=docs, sort=sortby, filter_sort_table=filter_sort) return self.abs_page('batches', content) ### Aux methods ### @expose def put_request(self, *args, **kwds): "PUT request callback to reqmgr server, should be used in AJAX" reqname = kwds.get('RequestName', '') status = kwds.get('RequestStatus', '') if not reqname: msg = 'Unable to update request status, empty request name' raise cherrypy.HTTPError(406, msg) if not status: msg = 'Unable to update request status, empty status value' raise cherrypy.HTTPError(406, msg) return self.reqmgr.updateRequestStatus(reqname, status) @expose def images(self, *args, **kwargs): """ Serve static images. """ args = list(args) self.check_scripts(args, self.imgmap, self.imgdir) mime_types = [ '*/*', 'image/gif', 'image/png', 'image/jpg', 'image/jpeg' ] accepts = cherrypy.request.headers.elements('Accept') for accept in accepts: if accept.value in mime_types and len(args) == 1 \ and args[0] in self.imgmap: image = self.imgmap[args[0]] # use image extension to pass correct content type ctype = 'image/%s' % image.split('.')[-1] cherrypy.response.headers['Content-type'] = ctype return serve_file(image, content_type=ctype) def serve(self, kwds, imap, idir, datatype='', minimize=False): "Serve files for high level APIs (yui/css/js)" args = [] for key, val in kwds.items(): if key == 'f': # we only look-up files from given kwds dict if isinstance(val, list): args += val else: args.append(val) scripts = self.check_scripts(args, imap, idir) return self.serve_files(args, scripts, imap, datatype, minimize) @exposecss @tools.gzip() def css(self, **kwargs): """ Serve provided CSS files. They can be passed as f=file1.css&f=file2.css """ resource = kwargs.get('resource', 'css') if resource == 'css': return self.serve(kwargs, self.cssmap, self.cssdir, 'css', True) @exposejs @tools.gzip() def js(self, **kwargs): """ Serve provided JS scripts. They can be passed as f=file1.js&f=file2.js with optional resource parameter to speficy type of JS files, e.g. resource=yui. """ resource = kwargs.get('resource', 'js') if resource == 'js': return self.serve(kwargs, self.jsmap, self.jsdir) def serve_files(self, args, scripts, resource, datatype='', minimize=False): """ Return asked set of files for JS, YUI, CSS. """ idx = "-".join(scripts) if idx not in self._cache.keys(): data = '' if datatype == 'css': data = '@CHARSET "UTF-8";' for script in args: path = os.path.join(sys.path[0], resource[script]) path = os.path.normpath(path) ifile = open(path) data = "\n".join ([data, ifile.read().\ replace('@CHARSET "UTF-8";', '')]) ifile.close() if datatype == 'css': set_headers("text/css") if minimize: self._cache[idx] = minify(data) else: self._cache[idx] = data return self._cache[idx] def check_scripts(self, scripts, resource, path): """ Check a script is known to the resource map and that the script actually exists """ for script in scripts: if script not in resource.keys(): spath = os.path.normpath(os.path.join(path, script)) if os.path.isfile(spath): resource.update({script: spath}) return scripts
class WorkQueueReqMgrInterface(object): """Helper class for ReqMgr interaction""" def __init__(self, **kwargs): if not kwargs.get('logger'): import logging kwargs['logger'] = logging self.logger = kwargs['logger'] # this will break all in one test self.reqMgr2 = ReqMgr(kwargs.get("reqmgr2_endpoint", None)) centralurl = kwargs.get("central_logdb_url", "") identifier = kwargs.get("log_reporter", "") # set the thread name before creat the log db. # only sets that when it is not set already myThread = threading.currentThread() if myThread.getName() == "MainThread": myThread.setName(self.__class__.__name__) self.logdb = LogDB(centralurl, identifier, logger=self.logger) self.previous_state = {} def __call__(self, queue): """Synchronize WorkQueue and RequestManager""" msg = '' try: # pull in new work self.logger.info("queueing new work") work = self.queueNewRequests(queue) msg += "New Work: %d\n" % work except Exception as ex: errorMsg = "Error caught during RequestManager pull" self.logger.exception("%s: %s", errorMsg, str(ex)) try: # get additional open-running work self.logger.info("adding new element to open requests") extraWork = self.addNewElementsToOpenRequests(queue) msg += "Work added: %d\n" % extraWork except Exception as ex: errorMsg = "Error caught during RequestManager split" self.logger.exception("%s: %s", errorMsg, str(ex)) try: # report back to ReqMgr self.logger.info("cancel aborted requests") count = self.cancelWork(queue) self.logger.info("finised canceling requests") msg += "Work canceled: %s " % count except Exception as ex: errorMsg = "Error caught during canceling the request" self.logger.exception("%s: %s", errorMsg, str(ex)) queue.backend.recordTaskActivity('reqmgr_sync', msg) def queueNewRequests(self, queue): """Get requests from regMgr and queue to workqueue""" self.logger.info("Contacting Request manager for more work") work = 0 workLoads = [] try: workLoads = self.getAvailableRequests() except Exception as ex: traceMsg = traceback.format_exc() msg = "Error contacting RequestManager: %s" % traceMsg self.logger.warning(msg) return 0 for team, reqName, workLoadUrl in workLoads: try: try: Lexicon.couchurl(workLoadUrl) except Exception as ex: # can throw many errors e.g. AttributeError, AssertionError etc. # check its not a local file if not os.path.exists(workLoadUrl): error = WorkQueueWMSpecError( None, "Workflow url validation error: %s" % str(ex)) raise error self.logger.info("Processing request %s at %s" % (reqName, workLoadUrl)) units = queue.queueWork(workLoadUrl, request=reqName, team=team) self.logdb.delete(reqName, "error", this_thread=True, agent=False) except TERMINAL_EXCEPTIONS as ex: # fatal error - report back to ReqMgr self.logger.error( 'Permanent failure processing request "%s": %s' % (reqName, str(ex))) self.logger.info("Marking request %s as failed in ReqMgr" % reqName) self.reportRequestStatus(reqName, 'Failed', message=str(ex)) continue except (IOError, socket.error, CouchError, CouchConnectionError) as ex: # temporary problem - try again later msg = 'Error processing request "%s": will try again later.' % reqName msg += '\nError: "%s"' % str(ex) self.logger.info(msg) self.logdb.post(reqName, msg, 'error') continue except Exception as ex: # Log exception as it isnt a communication problem msg = 'Error processing request "%s": will try again later.' % reqName msg += '\nSee log for details.\nError: "%s"' % str(ex) self.logger.exception('Unknown error processing %s' % reqName) self.logdb.post(reqName, msg, 'error') continue self.logger.info('%s units(s) queued for "%s"' % (units, reqName)) work += units self.logger.info("%s element(s) obtained from RequestManager" % work) return work def cancelWork(self, queue): requests = self.reqMgr2.getRequestByStatus( ['aborted', 'force-complete'], detail=False) count = 0 for req in requests: try: queue.cancelWork(req) count += 1 except Exception as ex: msg = 'Error to cancel the request "%s": %s' % (req, str(ex)) self.logger.exception(msg) return count def report(self, queue): """Report queue status to ReqMgr.""" new_state = {} uptodate_elements = [] now = time.time() elements = queue.statusInbox(dictKey="RequestName") if not elements: return new_state for ele in elements: ele = elements[ele][0] # 1 element tuple try: request = self.reqMgr2.getRequestByNames(ele['RequestName']) if not request: msg = 'Failed to get request "%s" from ReqMgr2. Will try again later.' % ele[ 'RequestName'] self.logger.warning(msg) continue request = request[0][ele['RequestName']] if request['RequestStatus'] in ('failed', 'completed', 'announced', 'closed-out', 'rejected'): # requests can be done in reqmgr but running in workqueue # if request has been closed but agent cleanup actions # haven't been run (or agent has been retired) # Prune out obviously too old ones to avoid build up if queue.params.get('reqmgrCompleteGraceTime', -1) > 0: if (now - float(ele.updatetime) ) > queue.params['reqmgrCompleteGraceTime']: # have to check all elements are at least running and are old enough request_elements = queue.statusInbox( WorkflowName=request['RequestName']) if not any([ x for x in request_elements if x['Status'] != 'Running' and not x.inEndState() ]): last_update = max([ float(x.updatetime) for x in request_elements ]) if ( now - last_update ) > queue.params['reqmgrCompleteGraceTime']: self.logger.info( "Finishing request %s as it is done in reqmgr" % request['RequestName']) queue.doneWork( WorkflowName=request['RequestName']) continue else: pass # assume workqueue status will catch up later elif request['RequestStatus'] in ['aborted', 'force-complete']: queue.cancelWork(WorkflowName=request['RequestName']) # Check consistency of running-open/closed and the element closure status elif request['RequestStatus'] == 'running-open' and not ele.get( 'OpenForNewData', False): self.reportRequestStatus(ele['RequestName'], 'running-closed') elif request['RequestStatus'] == 'running-closed' and ele.get( 'OpenForNewData', False): queue.closeWork(ele['RequestName']) # we do not want to move the request to 'failed' status elif ele['Status'] == 'Failed': continue elif ele['Status'] not in self._reqMgrToWorkQueueStatus( request['RequestStatus']): self.reportElement(ele) uptodate_elements.append(ele) except Exception as ex: msg = 'Error talking to ReqMgr about request "%s": %s' % ( ele['RequestName'], str(ex)) self.logger.exception(msg) return uptodate_elements def deleteFinishedWork(self, queue, elements): """Delete work from queue that is finished in ReqMgr""" finished = [] for element in elements: if element.inEndState(): finished.append(element['RequestName']) return queue.deleteWorkflows(*finished) def getAvailableRequests(self): """ Get available requests and sort by team and priority returns [(team, request_name, request_spec_url)] """ tempResults = self.reqMgr2.getRequestByStatus("staged") filteredResults = [] for requests in tempResults: for request in requests.values(): filteredResults.append(request) filteredResults.sort(key=itemgetter('RequestPriority'), reverse=True) filteredResults.sort(key=lambda r: r["Team"]) results = [(x["Team"], x["RequestName"], x["RequestWorkflow"]) for x in filteredResults] return results def reportRequestStatus(self, request, status, message=None): """Change state in RequestManager Optionally, take a message to append to the request """ if message: logType = "error" if status == "Failed" else "info" self.logdb.post(request, str(message), logType) reqmgrStatus = self._workQueueToReqMgrStatus(status) if reqmgrStatus: # only send known states try: self.reqMgr2.updateRequestStatus(request, reqmgrStatus) except Exception as ex: msg = "%s : fail to update status will try later: %s" % ( request, str(ex)) msg += traceback.format_exc() self.logdb.post(request, msg, 'warning') raise ex return def _workQueueToReqMgrStatus(self, status): """Map WorkQueue Status to that reported to ReqMgr""" statusMapping = { 'Acquired': 'acquired', 'Running': 'running-open', 'Failed': 'failed', 'Canceled': 'aborted', 'CancelRequested': 'aborted', 'Done': 'completed' } if status in statusMapping: # if wq status passed convert to reqmgr status return statusMapping[status] elif status in REQUEST_STATE_LIST: # if reqmgr status passed return reqmgr status return status else: # unknown status return None def _reqMgrToWorkQueueStatus(self, status): """Map ReqMgr status to that in a WorkQueue element, it is not a 1-1 relation""" statusMapping = { 'acquired': ['Acquired'], 'running': ['Running'], 'running-open': ['Running'], 'running-closed': ['Running'], 'failed': ['Failed'], 'aborted': ['Canceled', 'CancelRequested'], 'force-complete': ['Canceled', 'CancelRequested'], 'completed': ['Done'] } if status in statusMapping: return statusMapping[status] else: return [] def reportElement(self, element): """Report element to ReqMgr""" self.reportRequestStatus(element['RequestName'], element['Status']) def addNewElementsToOpenRequests(self, queue): """Add new elements to open requests which are in running-open state, only works adding new blocks from the input dataset""" self.logger.info( "Checking Request Manager for open requests and closing old ones") work = 0 requests = [] try: requests = self.reqMgr2.getRequestByStatus("running-open", detail=False) except Exception as ex: traceMsg = traceback.format_exc() msg = "Error contacting RequestManager: %s" % traceMsg self.logger.warning(msg) return 0 for reqName in requests: try: self.logger.info("Processing request %s" % (reqName)) units = queue.addWork(requestName=reqName) self.logdb.delete(reqName, 'error', True, agent=False) except (WorkQueueWMSpecError, WorkQueueNoWorkError) as ex: # fatal error - but at least it was split the first time. Log and skip. msg = 'Error adding further work to request "%s". Will try again later' % reqName msg += '\nError: "%s"' % str(ex) self.logger.info(msg) self.logdb.post(reqName, msg, 'error') continue except (IOError, socket.error, CouchError, CouchConnectionError) as ex: # temporary problem - try again later msg = 'Error processing request "%s": will try again later.' % reqName msg += '\nError: "%s"' % str(ex) self.logger.info(msg) self.logdb.post(reqName, msg, 'error') continue except Exception as ex: # Log exception as it isnt a communication problem msg = 'Error processing request "%s": will try again later.' % reqName msg += '\nSee log for details.\nError: "%s"' % str(ex) traceMsg = traceback.format_exc() msg = "%s\n%s" % (msg, traceMsg) self.logger.exception('Unknown error processing %s' % reqName) self.logdb.post(reqName, msg, 'error') continue self.logger.info('%s units(s) queued for "%s"' % (units, reqName)) work += units self.logger.info("%s element(s) added to open requests" % work) return work
class ReqMgrService(TemplatedPage): """ Request Manager web service class """ def __init__(self, app, config, mount): self.base = config.base self.rootdir = '/'.join(WMCore.__file__.split('/')[:-1]) if config and not isinstance(config, dict): web_config = config.dictionary_() if not config: web_config = {'base': self.base} TemplatedPage.__init__(self, web_config) imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd()+'/images') self.imgdir = web_config.get('imgdir', imgdir) cssdir = os.environ.get('RM_CSSPATH', os.getcwd()+'/css') self.cssdir = web_config.get('cssdir', cssdir) jsdir = os.environ.get('RM_JSPATH', os.getcwd()+'/js') self.jsdir = web_config.get('jsdir', jsdir) spdir = os.environ.get('RM_SPECPATH', os.getcwd()+'/specs') self.spdir = web_config.get('spdir', spdir) # read scripts area and initialize data-ops scripts self.sdir = os.environ.get('RM_SCRIPTS', os.getcwd()+'/scripts') self.sdir = web_config.get('sdir', self.sdir) self.sdict_thr = web_config.get('sdict_thr', 600) # put reasonable 10 min interval self.sdict = {'ts':time.time()} # placeholder for data-ops scripts self.update_scripts(force=True) # To be filled at run time self.cssmap = {} self.jsmap = {} self.imgmap = {} self.yuimap = {} std_specs_dir = os.path.join(self.rootdir, 'WMSpec/StdSpecs') self.std_specs = spec_list(std_specs_dir, 'WMSpec.StdSpecs') self.std_specs.sort() # Update CherryPy configuration mime_types = ['text/css'] mime_types += ['application/javascript', 'text/javascript', 'application/x-javascript', 'text/x-javascript'] cherryconf.update({'tools.encode.on': True, 'tools.gzip.on': True, 'tools.gzip.mime_types': mime_types, }) self._cache = {} # initialize rest API statedir = '/tmp' app = RESTMain(config, statedir) # REST application mount = '/rest' # mount point for cherrypy service api = RestApiHub(app, config.reqmgr, mount) # initialize access to reqmgr2 APIs self.reqmgr = ReqMgr(config.reqmgr.reqmgr2_url) # only gets current view (This might cause to reponse time much longer, # If upto date view is not needed overwrite Fale) self.reqmgr._noStale = True # admin helpers self.admin_info = Info(app, api, config.reqmgr, mount=mount+'/info') self.admin_group = Group(app, api, config.reqmgr, mount=mount+'/group') self.admin_team = Team(app, api, config.reqmgr, mount=mount+'/team') # get fields which we'll use in templates cdict = config.reqmgr.dictionary_() self.couch_url = cdict.get('couch_host', '') self.couch_dbname = cdict.get('couch_reqmgr_db', '') self.couch_wdbname = cdict.get('couch_workload_summary_db', '') self.acdc_url = cdict.get('acdc_host', '') self.acdc_dbname = cdict.get('acdc_db', '') self.configcache_url = cdict.get('couch_config_cache_url', self.couch_url) self.dbs_url = cdict.get('dbs_url', '') self.dqm_url = cdict.get('dqm_url', '') self.sw_ver = cdict.get('default_sw_version', 'CMSSW_5_2_5') self.sw_arch = cdict.get('default_sw_scramarch', 'slc5_amd64_gcc434') def user(self): """ Return user name associated with this instance. """ try: return cherrypy.request.user['login'] except: return 'testuser' def user_dn(self): "Return user DN" try: return cherrypy.request.user['dn'] except: return '/CN/bla/foo' def update_scripts(self, force=False): "Update scripts dict" if force or abs(time.time()-self.sdict['ts']) > self.sdict_thr: for item in os.listdir(self.sdir): with open(os.path.join(self.sdir, item), 'r') as istream: self.sdict[item.split('.')[0]] = istream.read() self.sdict['ts'] = time.time() def abs_page(self, tmpl, content): """generate abstract page""" menu = self.templatepage('menu', menus=menus(), tmpl=tmpl) body = self.templatepage('generic', menu=menu, content=content) page = self.templatepage('main', content=body, user=self.user()) return page def page(self, content): """ Provide page wrapped with top/bottom templates. """ return self.templatepage('main', content=content) def error(self, content): "Generate common error page" content = self.templatepage('error', content=content) return self.abs_page('error', content) @expose def index(self, **kwds): """Main page""" content = self.templatepage('index', requests=ACTIVE_STATUS, rdict=REQUEST_STATE_TRANSITION) return self.abs_page('main', content) @expose def home(self, **kwds): """Main page""" return self.index(**kwds) ### Admin actions ### @expose def admin(self, **kwds): """admin page""" print "\n### ADMIN PAGE" rows = self.admin_info.get() print "rows", [r for r in rows] content = self.templatepage('admin') return self.abs_page('admin', content) @expose def add_user(self, **kwds): """add_user action""" rid = genid(kwds) status = "ok" # chagne to whatever it would be content = self.templatepage('confirm', ticket=rid, user=self.user(), status=status) return self.abs_page('admin', content) @expose def add_group(self, **kwds): """add_group action""" rows = self.admin_group.get() print "\n### GROUPS", [r for r in rows] rid = genid(kwds) status = "ok" # chagne to whatever it would be content = self.templatepage('confirm', ticket=rid, user=self.user(), status=status) return self.abs_page('admin', content) @expose def add_team(self, **kwds): """add_team action""" rows = self.admin_team.get() print "\n### TEAMS", kwds, [r for r in rows] print "request to add", kwds rid = genid(kwds) status = "ok" # chagne to whatever it would be content = self.templatepage('confirm', ticket=rid, user=self.user(), status=status) return self.abs_page('admin', content) ### Request actions ### @expose @checkargs(['status', 'sort']) def assign(self, **kwds): """assign page""" if not kwds: kwds = {} if 'status' not in kwds: kwds.update({'status': 'assignment-approved'}) docs = [] attrs = ['RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus'] data = self.reqmgr.getRequestByStatus(statusList=[kwds['status']]) for key, val in data.items(): docs.append(request_attr(val, attrs)) sortby = kwds.get('sort', 'status') docs = [r for r in sort(docs, sortby)] misc_json = {'CMSSW Releases':releases(), 'CMSSW architectures':architectures(), 'SubscriptionPriority':['Low', 'Normal', 'High'], 'CustodialSubType':['Move', 'Replica'], 'NonCustodialSubType':['Move', 'Replica'], 'MinMergeSize':2147483648, 'MaxMergeSize':4294967296, 'MaxMergeEvents':50000, 'MaxRSS':20411724, 'MaxVSize':20411724, 'SoftTimeout':129600, 'GracePeriod':300, 'BlockCloseMaxWaitTime':66400, 'BlockCloseMaxFiles':500, 'BlockCloseMaxEvents':250000000, 'BlockCloseMaxSize':5000000000000, 'AcquisitionEra':'', 'ProcessingVersion':1, 'ProcessingString':'', 'MergedLFNBase':lfn_bases(), 'UnmergedLFNBase':lfn_unmerged_bases(),} filter_sort = self.templatepage('filter_sort') content = self.templatepage('assign', sort=sortby, filter_sort_table=filter_sort, sites=sites(), site_white_list=site_white_list(), site_black_list=site_black_list(), user=self.user(), user_dn=self.user_dn(), requests=docs, misc_table=json2table(misc_json, web_ui_names()), misc_json=json2form(misc_json, indent=2, keep_first_value=True)) return self.abs_page('assign', content) @expose @checkargs(['status', 'sort']) def approve(self, **kwds): """ Approve page: get list of request associated with user DN. Fetch their status list from ReqMgr and display if requests were seen by data-ops. """ if not kwds: kwds = {} if 'status' not in kwds: kwds.update({'status': 'new'}) kwds.update({'_nostale':True}) docs = [] attrs = ['RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus'] data = self.reqmgr.getRequestByStatus(statusList=[kwds['status']]) for key, val in data.items(): docs.append(request_attr(val, attrs)) sortby = kwds.get('sort', 'status') docs = [r for r in sort(docs, sortby)] filter_sort = self.templatepage('filter_sort') content = self.templatepage('approve', requests=docs, date=tstamp(), sort=sortby, filter_sort_table=filter_sort) return self.abs_page('approve', content) @expose def create(self, **kwds): """create page""" # get list of standard specs from WMCore and new ones from local area #loc_specs_dir = os.path.join(self.spdir, 'Specs') # local specs #loc_specs = spec_list(loc_specs_dir, 'Specs') #all_specs = list(set(self.std_specs + loc_specs)) #all_specs.sort() all_specs = self.std_specs spec = kwds.get('form', '') if not spec: spec = self.std_specs[0] # make spec first in all_specs list if spec in all_specs: all_specs.remove(spec) all_specs = [spec] + all_specs jsondata = get_request_template_from_type(spec) # create templatized page out of provided forms self.update_scripts() content = self.templatepage('create', table=json2table(jsondata, web_ui_names()), jsondata=json2form(jsondata, indent=2, keep_first_value=True), name=spec, scripts=[s for s in self.sdict.keys() if s!='ts'], specs=all_specs) return self.abs_page('create', content) def generate_objs(self, script, jsondict): """Generate objects from givem JSON template""" self.update_scripts() code = self.sdict.get(script, '') if code.find('def genobjs(jsondict)') == -1: return self.error("Improper python snippet, your code should start with <b>def genobjs(jsondict)</b> function") exec(code) # code snippet must starts with genobjs function return [r for r in genobjs(jsondict)] @expose def fetch(self, rid, **kwds): "Fetch document for given id" rid = rid.replace('request-', '') doc = self.reqmgr.getRequestByNames(rid) transitions = [] if len(doc) == 1: try: doc = doc[rid] except: pass name = doc.get('RequestName', 'NA') title = 'Request %s' % name status = doc.get('RequestStatus', '') transitions = REQUEST_STATE_TRANSITION.get(status, []) if status in transitions: transitions.remove(status) content = self.templatepage('doc', title=title, status=status, name=name, table=json2table(doc, web_ui_names()), jsondata=json2form(doc, indent=2, keep_first_value=False), transitions=transitions) elif len(doc) > 1: jsondata = [pprint.pformat(d) for d in doc] content = self.templatepage('doc', title='Series of docs: %s' % rid, table="", jsondata=jsondata, transitions=transitions) else: doc = 'No request found for name=%s' % rid return self.abs_page('request', content) @expose def requests(self, **kwds): """Page showing requests""" if not kwds: kwds = {} if 'status' not in kwds: kwds.update({'status': 'acquired'}) results = self.reqmgr.getRequestByStatus(kwds['status']) docs = [] for key, doc in results.items(): docs.append(request_attr(doc)) sortby = kwds.get('sort', 'status') docs = [r for r in sort(docs, sortby)] filter_sort = self.templatepage('filter_sort') content = self.templatepage('requests', requests=docs, sort=sortby, status=kwds['status'], filter_sort_table=filter_sort) return self.abs_page('requests', content) @expose def request(self, **kwargs): "Get data example and expose it as json" dataset = kwargs.get('uinput', '') if not dataset: return {'error':'no input dataset'} url = 'https://cmsweb.cern.ch/reqmgr/rest/outputdataset/%s' % dataset params = {} headers = {'Accept': 'application/json;text/json'} wdata = getdata(url, params) wdict = dict(date=time.ctime(), team='Team-A', status='Running', ID=genid(wdata)) winfo = self.templatepage('workflow', wdict=wdict, dataset=dataset, code=pprint.pformat(wdata)) content = self.templatepage('search', content=winfo) return self.abs_page('request', content) @expose def batch(self, **kwds): """batch page""" # TODO: we need a template for batch attributes # and read it from separate area, like DASMaps name = kwds.get('name', '') batch = {} if name: # batch = self.reqmgr.getBatchesByName(name) batch = {'Name':'Batch1', 'Description': 'Bla-bla', 'Creator':'valya', 'Group':'test', 'Workflows':['workflow1', 'workflow2'], 'Attributes':{'HeavyIon':['true', 'false']}} attributes = batch.get('Attributes', {}) workflows = batch.get('Workflows', []) description = batch.get('Description', '') creator = batch.get('Creator', self.user_dn()) content = self.templatepage('batch', name=name, attributes=json2table(attributes, web_ui_names()), workflows=workflows, creator=creator, description=description) return self.abs_page('batch', content) @expose def batches(self, **kwds): """Page showing batches""" if not kwds: kwds = {} if 'name' not in kwds: kwds.update({'name': ''}) sortby = kwds.get('sort', 'name') # results = self.reqmgr.getBatchesByName(kwds['name']) results = [ {'Name':'Batch1', 'Description': 'Bla-bla', 'Creator':'valya', 'Group':'test', 'Workflows':['workflow1', 'workflow2'], 'Date': 'Fri Feb 13 10:36:41 EST 2015', 'Attributes':{'HeavyIon':['true', 'false']}}, {'Name':'Batch2', 'Description': 'lksdjflksjdf', 'Creator':'valya', 'Group':'test', 'Workflows':['workflow1', 'workflow2'], 'Date': 'Fri Feb 10 10:36:41 EST 2015', 'Attributes':{'HeavyIon':['true', 'false']}}, ] docs = [r for r in sort(results, sortby)] filter_sort = self.templatepage('filter_sort') content = self.templatepage('batches', batches=docs, sort=sortby, filter_sort_table=filter_sort) return self.abs_page('batches', content) ### Aux methods ### @expose def put_request(self, *args, **kwds): "PUT request callback to reqmgr server, should be used in AJAX" reqname = kwds.get('RequestName', '') status = kwds.get('RequestStatus', '') if not reqname: msg = 'Unable to update request status, empty request name' raise cherrypy.HTTPError(406, msg) if not status: msg = 'Unable to update request status, empty status value' raise cherrypy.HTTPError(406, msg) return self.reqmgr.updateRequestStatus(reqname, status) @expose def images(self, *args, **kwargs): """ Serve static images. """ args = list(args) self.check_scripts(args, self.imgmap, self.imgdir) mime_types = ['*/*', 'image/gif', 'image/png', 'image/jpg', 'image/jpeg'] accepts = cherrypy.request.headers.elements('Accept') for accept in accepts: if accept.value in mime_types and len(args) == 1 \ and args[0] in self.imgmap: image = self.imgmap[args[0]] # use image extension to pass correct content type ctype = 'image/%s' % image.split('.')[-1] cherrypy.response.headers['Content-type'] = ctype return serve_file(image, content_type=ctype) def serve(self, kwds, imap, idir, datatype='', minimize=False): "Serve files for high level APIs (yui/css/js)" args = [] for key, val in kwds.items(): if key == 'f': # we only look-up files from given kwds dict if isinstance(val, list): args += val else: args.append(val) scripts = self.check_scripts(args, imap, idir) return self.serve_files(args, scripts, imap, datatype, minimize) @exposecss @tools.gzip() def css(self, **kwargs): """ Serve provided CSS files. They can be passed as f=file1.css&f=file2.css """ resource = kwargs.get('resource', 'css') if resource == 'css': return self.serve(kwargs, self.cssmap, self.cssdir, 'css', True) @exposejs @tools.gzip() def js(self, **kwargs): """ Serve provided JS scripts. They can be passed as f=file1.js&f=file2.js with optional resource parameter to speficy type of JS files, e.g. resource=yui. """ resource = kwargs.get('resource', 'js') if resource == 'js': return self.serve(kwargs, self.jsmap, self.jsdir) def serve_files(self, args, scripts, resource, datatype='', minimize=False): """ Return asked set of files for JS, YUI, CSS. """ idx = "-".join(scripts) if idx not in self._cache.keys(): data = '' if datatype == 'css': data = '@CHARSET "UTF-8";' for script in args: path = os.path.join(sys.path[0], resource[script]) path = os.path.normpath(path) ifile = open(path) data = "\n".join ([data, ifile.read().\ replace('@CHARSET "UTF-8";', '')]) ifile.close() if datatype == 'css': set_headers("text/css") if minimize: self._cache[idx] = minify(data) else: self._cache[idx] = data return self._cache[idx] def check_scripts(self, scripts, resource, path): """ Check a script is known to the resource map and that the script actually exists """ for script in scripts: if script not in resource.keys(): spath = os.path.normpath(os.path.join(path, script)) if os.path.isfile(spath): resource.update({script: spath}) return scripts
class WorkQueueReqMgrInterface(): """Helper class for ReqMgr interaction""" def __init__(self, **kwargs): if not kwargs.get('logger'): import logging kwargs['logger'] = logging self.logger = kwargs['logger'] #TODO: (reqmgr2Only - remove this line when reqmgr is replaced) self.reqMgr = RequestManager(kwargs) #this will break all in one test self.reqMgr2 = ReqMgr(kwargs.get("reqmgr2_endpoint", None)) centralurl = kwargs.get("central_logdb_url", "") identifier = kwargs.get("log_reporter", "") # set the thread name before creat the log db. # only sets that when it is not set already myThread = threading.currentThread() if myThread.getName() == "MainThread": myThread.setName(self.__class__.__name__) self.logdb = LogDB(centralurl, identifier, logger=self.logger) self.previous_state = {} def __call__(self, queue): """Synchronize WorkQueue and RequestManager""" msg = '' try: # pull in new work work = self.queueNewRequests(queue) msg += "New Work: %d\n" % work except Exception: self.logger.exception("Error caught during RequestManager pull") try: # get additional open-running work extraWork = self.addNewElementsToOpenRequests(queue) msg += "Work added: %d\n" % extraWork except Exception: self.logger.exception("Error caught during RequestManager split") try: # report back to ReqMgr uptodate_elements = self.report(queue) msg += "Updated ReqMgr status for: %s\n" % ", ".join( [x['RequestName'] for x in uptodate_elements]) except Exception: self.logger.exception("Error caught during RequestManager update") else: try: # Delete finished requests from WorkQueue self.deleteFinishedWork(queue, uptodate_elements) except Exception: self.logger.exception("Error caught during work deletion") queue.backend.recordTaskActivity('reqmgr_sync', msg) def queueNewRequests(self, queue): """Get requests from regMgr and queue to workqueue""" self.logger.info("Contacting Request manager for more work") work = 0 workLoads = [] if queue.params['DrainMode']: self.logger.info( 'Draining queue: Skip requesting work from ReqMgr') return 0 try: workLoads = self.getAvailableRequests(queue.params['Teams']) except Exception as ex: traceMsg = traceback.format_exc() msg = "Error contacting RequestManager: %s" % traceMsg self.logger.warning(msg) return 0 for team, reqName, workLoadUrl in workLoads: # try: # self.reportRequestStatus(reqName, "negotiating") # except Exception, ex: # self.logger.error(""" # Unable to update ReqMgr state to negotiating: %s # Ignoring this request: %s""" % (str(ex), reqName)) # continue try: try: Lexicon.couchurl(workLoadUrl) except Exception as ex: # can throw many errors e.g. AttributeError, AssertionError etc. # check its not a local file if not os.path.exists(workLoadUrl): error = WorkQueueWMSpecError( None, "Workflow url validation error: %s" % str(ex)) raise error self.logger.info("Processing request %s at %s" % (reqName, workLoadUrl)) units = queue.queueWork(workLoadUrl, request=reqName, team=team) self.logdb.delete(reqName, "error", this_thread=True) except (WorkQueueWMSpecError, WorkQueueNoWorkError) as ex: # fatal error - report back to ReqMgr self.logger.info( 'Permanent failure processing request "%s": %s' % (reqName, str(ex))) self.logger.info("Marking request %s as failed in ReqMgr" % reqName) self.reportRequestStatus(reqName, 'Failed', message=str(ex)) continue except (IOError, socket.error, CouchError, CouchConnectionError) as ex: # temporary problem - try again later msg = 'Error processing request "%s": will try again later.' \ '\nError: "%s"' % (reqName, str(ex)) self.logger.info(msg) self.logdb.post(reqName, msg, 'error') continue except Exception as ex: # Log exception as it isnt a communication problem msg = 'Error processing request "%s": will try again later.' \ '\nSee log for details.\nError: "%s"' % (reqName, str(ex)) self.logger.exception('Unknown error processing %s' % reqName) self.logdb.post(reqName, msg, 'error') continue try: self.reportRequestStatus(reqName, "acquired") except Exception as ex: self.logger.warning("Unable to update ReqMgr state: %s" % str(ex)) self.logger.warning('Will try again later') self.logger.info('%s units(s) queued for "%s"' % (units, reqName)) work += units self.logger.info("%s element(s) obtained from RequestManager" % work) return work def report(self, queue): """Report queue status to ReqMgr.""" new_state = {} uptodate_elements = [] now = time.time() elements = queue.statusInbox(dictKey="RequestName") if not elements: return new_state for ele in elements: ele = elements[ele][0] # 1 element tuple try: request = self.reqMgr2.getRequestByNames(ele['RequestName']) if not request: msg = 'Failed to get request "%s" from ReqMgr2. Will try again later.' % ele[ 'RequestName'] self.logger.warning(msg) continue request = request[ele['RequestName']] if request['RequestStatus'] in ('failed', 'completed', 'announced', 'epic-FAILED', 'closed-out', 'rejected'): # requests can be done in reqmgr but running in workqueue # if request has been closed but agent cleanup actions # haven't been run (or agent has been retired) # Prune out obviously too old ones to avoid build up if queue.params.get('reqmgrCompleteGraceTime', -1) > 0: if (now - float(ele.updatetime) ) > queue.params['reqmgrCompleteGraceTime']: # have to check all elements are at least running and are old enough request_elements = queue.statusInbox( WorkflowName=request['RequestName']) if not any([ x for x in request_elements if x['Status'] != 'Running' and not x.inEndState() ]): last_update = max([ float(x.updatetime) for x in request_elements ]) if ( now - last_update ) > queue.params['reqmgrCompleteGraceTime']: self.logger.info( "Finishing request %s as it is done in reqmgr" % request['RequestName']) queue.doneWork( WorkflowName=request['RequestName']) continue else: pass # assume workqueue status will catch up later elif request['RequestStatus'] == 'aborted' or request[ 'RequestStatus'] == 'force-complete': queue.cancelWork(WorkflowName=request['RequestName']) # Check consistency of running-open/closed and the element closure status elif request['RequestStatus'] == 'running-open' and not ele.get( 'OpenForNewData', False): self.reportRequestStatus(ele['RequestName'], 'running-closed') elif request['RequestStatus'] == 'running-closed' and ele.get( 'OpenForNewData', False): queue.closeWork(ele['RequestName']) # update request status if necessary elif ele['Status'] not in self._reqMgrToWorkQueueStatus( request['RequestStatus']): self.reportElement(ele) uptodate_elements.append(ele) except Exception as ex: msg = 'Error talking to ReqMgr about request "%s": %s' traceMsg = traceback.format_exc() self.logger.error(msg % (ele['RequestName'], traceMsg)) return uptodate_elements def deleteFinishedWork(self, queue, elements): """Delete work from queue that is finished in ReqMgr""" finished = [] for element in elements: if self._workQueueToReqMgrStatus(element['Status']) in ('aborted', 'failed', 'completed', 'announced', 'epic-FAILED', 'closed-out', 'rejected') \ and element.inEndState(): finished.append(element['RequestName']) return queue.deleteWorkflows(*finished) def _getRequestsByTeamsAndStatus(self, status, teams=[]): """ TODO: now it assumes one team per requests - check whether this assumption is correct Check whether we actually use the team for this. Also switch to byteamandstatus couch call instead of """ requests = self.reqMgr2.getRequestByStatus(status) #Then sort by Team name then sort by Priority #https://docs.python.org/2/howto/sorting.html if teams and len(teams) > 0: results = {} for reqName, value in requests.items(): if value["Teams"][0] in teams: results[reqName] = value return results else: return requests def getAvailableRequests(self, teams): """ Get available requests for the given teams and sort by team and priority returns [(team, request_name, request_spec_url)] """ tempResults = self._getRequestsByTeamsAndStatus("assigned", teams).values() filteredResults = [] for request in tempResults: if "Teams" in request and len(request["Teams"]) == 1: filteredResults.append(request) self.logdb.delete(request["RequestName"], "error", this_thread=True) else: msg = "no team or more than one team (%s) are assigined: %s" % ( request.get("Teams", None), request["RequestName"]) self.logger.error(msg) self.logdb.post(request["RequestName"], msg, 'error') filteredResults.sort(key=itemgetter('RequestPriority'), reverse=True) filteredResults.sort(key=lambda r: r["Teams"][0]) results = [(x["Teams"][0], x["RequestName"], x["RequestWorkflow"]) for x in filteredResults] return results def reportRequestStatus(self, request, status, message=None): """Change state in RequestManager Optionally, take a message to append to the request """ if message: self.logdb.post(request, str(message), 'info') reqmgrStatus = self._workQueueToReqMgrStatus(status) if reqmgrStatus: # only send known states try: #TODO: try reqmgr1 call if it fails (reqmgr2Only - remove this line when reqmgr is replaced) self.reqMgr.reportRequestStatus(request, reqmgrStatus) # And replace with this (remove all Exceptins) #self.reqMgr2.updateRequestStatus(request, reqmgrStatus) except HTTPException as ex: # If we get an HTTPException of 404 means reqmgr2 request if ex.status == 404: # try reqmgr2 call msg = "%s : reqmgr2 request: %s" % (request, str(ex)) self.logdb.post(request, msg, 'info') self.reqMgr2.updateRequestStatus(request, reqmgrStatus) else: msg = "%s : fail to update status with HTTP error: %s" % ( request, str(ex)) self.logdb.post(request, msg, 'warning') raise ex except Exception as ex: msg = "%s : fail to update status will try later: %s" % ( request, str(ex)) self.logdb.post(request, msg, 'warning') raise ex def markAcquired(self, request, url=None): """Mark request acquired""" self.reqMgr.putWorkQueue(request, url) def _workQueueToReqMgrStatus(self, status): """Map WorkQueue Status to that reported to ReqMgr""" statusMapping = { 'Acquired': 'acquired', 'Running': 'running-open', 'Failed': 'failed', 'Canceled': 'aborted', 'CancelRequested': 'aborted', 'Done': 'completed' } if status in statusMapping: # if wq status passed convert to reqmgr status return statusMapping[status] elif status in REQUEST_STATE_LIST: # if reqmgr status passed return reqmgr status return status else: # unknown status return None def _reqMgrToWorkQueueStatus(self, status): """Map ReqMgr status to that in a WorkQueue element, it is not a 1-1 relation""" statusMapping = { 'acquired': ['Acquired'], 'running': ['Running'], 'running-open': ['Running'], 'running-closed': ['Running'], 'failed': ['Failed'], 'aborted': ['Canceled', 'CancelRequested'], 'force-complete': ['Canceled', 'CancelRequested'], 'completed': ['Done'] } if status in statusMapping: return statusMapping[status] else: return [] def reportElement(self, element): """Report element to ReqMgr""" self.reportRequestStatus(element['RequestName'], element['Status']) def addNewElementsToOpenRequests(self, queue): """Add new elements to open requests which are in running-open state, only works adding new blocks from the input dataset""" self.logger.info( "Checking Request Manager for open requests and closing old ones") # First close any open inbox element which hasn't found anything new in a while queue.closeWork() self.report(queue) work = 0 requests = [] # Drain mode, don't pull any work into open requests. They will be closed if the queue stays in drain long enough if queue.params['DrainMode']: self.logger.info( 'Draining queue: Skip requesting work from ReqMgr') return 0 try: requests = self._getRequestsByTeamsAndStatus( "running-open", queue.params['Teams']).keys() except Exception as ex: traceMsg = traceback.format_exc() msg = "Error contacting RequestManager: %s" % traceMsg self.logger.warning(msg) return 0 for reqName in requests: try: self.logger.info("Processing request %s" % (reqName)) units = queue.addWork(requestName=reqName) self.logdb.delete(request["RequestName"], 'error', True) except (WorkQueueWMSpecError, WorkQueueNoWorkError) as ex: # fatal error - but at least it was split the first time. Log and skip. msg = 'Error adding further work to request "%s". Will try again later' \ '\nError: "%s"' % (reqName, str(ex)) self.logger.info(msg) self.logdb.post(reqName, msg, 'error') continue except (IOError, socket.error, CouchError, CouchConnectionError) as ex: # temporary problem - try again later msg = 'Error processing request "%s": will try again later.' \ '\nError: "%s"' % (reqName, str(ex)) self.logger.info(msg) self.logdb.post(reqName, msg, 'error') continue except Exception as ex: # Log exception as it isnt a communication problem msg = 'Error processing request "%s": will try again later.' \ '\nSee log for details.\nError: "%s"' % (reqName, str(ex)) self.logger.exception('Unknown error processing %s' % reqName) self.logdb.post(reqName, msg, 'error') continue self.logger.info('%s units(s) queued for "%s"' % (units, reqName)) work += units self.logger.info("%s element(s) added to open requests" % work) return work
class ReqMgrService(TemplatedPage): """ Request Manager web service class """ def __init__(self, app, config, mount): self.base = config.base self.rootdir = '/'.join(WMCore.__file__.split('/')[:-1]) if config and not isinstance(config, dict): web_config = config.dictionary_() if not config: web_config = {'base': self.base} TemplatedPage.__init__(self, web_config) imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd() + '/images') self.imgdir = web_config.get('imgdir', imgdir) cssdir = os.environ.get('RM_CSSPATH', os.getcwd() + '/css') self.cssdir = web_config.get('cssdir', cssdir) jsdir = os.environ.get('RM_JSPATH', os.getcwd() + '/js') self.jsdir = web_config.get('jsdir', jsdir) spdir = os.environ.get('RM_SPECPATH', os.getcwd() + '/specs') self.spdir = web_config.get('spdir', spdir) # read scripts area and initialize data-ops scripts self.sdir = os.environ.get('RM_SCRIPTS', os.getcwd() + '/scripts') self.sdir = web_config.get('sdir', self.sdir) self.sdict_thr = web_config.get('sdict_thr', 600) # put reasonable 10 min interval self.sdict = {'ts': time.time()} # placeholder for data-ops scripts self.update_scripts(force=True) # To be filled at run time self.cssmap = {} self.jsmap = {} self.imgmap = {} self.yuimap = {} std_specs_dir = os.path.join(self.rootdir, 'WMSpec/StdSpecs') self.std_specs = spec_list(std_specs_dir) self.std_specs.sort() # Update CherryPy configuration mime_types = ['text/css'] mime_types += [ 'application/javascript', 'text/javascript', 'application/x-javascript', 'text/x-javascript' ] cherryconf.update({ 'tools.encode.on': True, 'tools.gzip.on': True, 'tools.gzip.mime_types': mime_types, }) self._cache = {} # initialize access to reqmgr2 APIs self.reqmgr_url = config.reqmgr.reqmgr2_url self.reqmgr = ReqMgr(self.reqmgr_url) # only gets current view (This might cause to reponse time much longer, # If upto date view is not needed overwrite Fale) self.reqmgr._noStale = True # get fields which we'll use in templates cdict = config.reqmgr.dictionary_() self.couch_url = cdict.get('couch_host', '') self.couch_dbname = cdict.get('couch_reqmgr_db', '') self.couch_wdbname = cdict.get('couch_workload_summary_db', '') self.acdc_url = cdict.get('acdc_host', '') self.acdc_dbname = cdict.get('acdc_db', '') self.configcache_url = cdict.get('couch_config_cache_url', self.couch_url) self.dbs_url = cdict.get('dbs_url', '') self.dqm_url = cdict.get('dqm_url', '') self.sw_ver = cdict.get('default_sw_version', 'CMSSW_7_6_1') self.sw_arch = cdict.get('default_sw_scramarch', 'slc6_amd64_gcc493') # LogDB holder centralurl = cdict.get("central_logdb_url", "") identifier = cdict.get("log_reporter", "reqmgr2") self.logdb = LogDB(centralurl, identifier) # local team cache which will request data from wmstats base, uri = self.reqmgr_url.split('://') base_url = '%s://%s' % (base, uri.split('/')[0]) self.wmstatsurl = cdict.get('wmstats_url', '%s/wmstatsserver' % base_url) if not self.wmstatsurl: raise Exception( 'ReqMgr2 configuration file does not provide wmstats url') self.team_cache = [] # fetch assignment arguments specification from StdBase self.assignArgs = StdBase().getWorkloadAssignArgs() self.assignArgs = { key: val['default'] for key, val in self.assignArgs.items() } def getTeams(self): "Helper function to get teams from wmstats or local cache" teams = self.team_cache url = '%s/data/teams' % self.wmstatsurl params = {} headers = {'Accept': 'application/json'} try: data = getdata(url, params, headers) if 'error' in data: print("WARNING: fail to get teams from %s" % url) print(data) teams = data.get('result', []) self.team_cache = teams except Exception as exp: print("WARNING: fail to get teams from %s" % url) print(str(exp)) return teams def update_scripts(self, force=False): "Update scripts dict" if force or abs(time.time() - self.sdict['ts']) > self.sdict_thr: for item in os.listdir(self.sdir): with open(os.path.join(self.sdir, item), 'r') as istream: self.sdict[item.split('.')[0]] = istream.read() self.sdict['ts'] = time.time() def abs_page(self, tmpl, content): """generate abstract page""" menu = self.templatepage('menu', menus=menus(), tmpl=tmpl) body = self.templatepage('generic', menu=menu, content=content) page = self.templatepage('main', content=body, user=user()) return page def page(self, content): """ Provide page wrapped with top/bottom templates. """ return self.templatepage('main', content=content) def error(self, content): "Generate common error page" content = self.templatepage('error', content=content) return self.abs_page('error', content) @expose def index(self): """Main page""" content = self.templatepage('index', requests=ACTIVE_STATUS, rdict=REQUEST_STATE_TRANSITION) return self.abs_page('main', content) @expose def home(self, **kwds): """Main page""" return self.index(**kwds) ### Request actions ### @expose @checkargs(['status', 'sort']) def assign(self, **kwds): """assign page""" if not kwds: kwds = {} if 'status' not in kwds: kwds.update({'status': 'assignment-approved'}) docs = [] attrs = [ 'RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus' ] dataResult = self.reqmgr.getRequestByStatus( statusList=[kwds['status']]) for data in dataResult: for val in data.values(): docs.append(request_attr(val, attrs)) sortby = kwds.get('sort', 'status') docs = [r for r in sort(docs, sortby)] assignDict = deepcopy(self.assignArgs) assignDict.update(getPropValueMap()) assignDict['Team'] = self.getTeams() filter_sort = self.templatepage('filter_sort') content = self.templatepage('assign', sort=sortby, filter_sort_table=filter_sort, sites=SITE_CACHE.getData(), site_white_list=site_white_list(), site_black_list=site_black_list(), user=user(), user_dn=user_dn(), requests=toString(docs), misc_table=json2table( assignDict, web_ui_names(), "all_attributes"), misc_json=json2form(assignDict, indent=2, keep_first_value=True)) return self.abs_page('assign', content) @expose @checkargs(['status', 'sort']) def approve(self, **kwds): """ Approve page: get list of request associated with user DN. Fetch their status list from ReqMgr and display if requests were seen by data-ops. """ if not kwds: kwds = {} if 'status' not in kwds: kwds.update({'status': 'new'}) kwds.update({'_nostale': True}) docs = [] attrs = [ 'RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus', 'Campaign' ] dataResult = self.reqmgr.getRequestByStatus( statusList=[kwds['status']]) for data in dataResult: for val in data.values(): docs.append(request_attr(val, attrs)) sortby = kwds.get('sort', 'status') docs = [r for r in sort(docs, sortby)] filter_sort = self.templatepage('filter_sort') content = self.templatepage('approve', requests=toString(docs), date=tstamp(), sort=sortby, filter_sort_table=filter_sort, gen_color=gen_color) return self.abs_page('approve', content) @expose def create(self, **kwds): """create page""" # get list of standard specs from WMCore and new ones from local area # loc_specs_dir = os.path.join(self.spdir, 'Specs') # local specs # loc_specs = spec_list(loc_specs_dir, 'Specs') # all_specs = list(set(self.std_specs + loc_specs)) # all_specs.sort() all_specs = list(self.std_specs) spec = kwds.get('form', '') if not spec: spec = self.std_specs[0] # make spec first in all_specs list if spec in all_specs: all_specs.remove(spec) all_specs = [spec] + all_specs jsondata = get_request_template_from_type(spec) # create templatized page out of provided forms self.update_scripts() content = self.templatepage( 'create', table=json2table(jsondata, web_ui_names(), jsondata), jsondata=json2form(jsondata, indent=2, keep_first_value=True), name=spec, scripts=[s for s in self.sdict.keys() if s != 'ts'], specs=all_specs) return self.abs_page('create', content) def generate_objs(self, script, jsondict): """Generate objects from givem JSON template""" self.update_scripts() code = self.sdict.get(script, '') if code.find('def genobjs(jsondict)') == -1: return self.error( "Improper python snippet, your code should start with <b>def genobjs(jsondict)</b> function" ) exec(code) # code snippet must starts with genobjs function return [r for r in genobjs(jsondict)] @expose def config(self, name): "Fetch config for given request name" result = self.reqmgr.getConfig(name) if len(result) == 1: result = result[0] else: result = 'Configuration not found for: %s' % name return result.replace('\n', '<br/>') @expose def fetch(self, rid): "Fetch document for given id" rid = rid.replace('request-', '') doc = self.reqmgr.getRequestByNames(rid) transitions = [] tst = time.time() # get request tasks tasks = self.reqmgr.getRequestTasks(rid) if len(doc) == 1: try: doc = doc[0][rid] except: pass name = doc.get('RequestName', 'NA') title = 'Request %s' % name status = doc.get('RequestStatus', '') transitions = REQUEST_STATE_TRANSITION.get(status, []) if status in transitions: transitions.remove(status) visible_attrs = get_modifiable_properties(status) filterout_attrs = get_protected_properties() # extend filterout list with "RequestStatus" since it is passed separately filterout_attrs.append("RequestStatus") for key, val in self.assignArgs.items(): if not doc.get(key): doc[key] = val if visible_attrs == "all_attributes": filteredDoc = doc for prop in filterout_attrs: if prop in filteredDoc: del filteredDoc[prop] else: filteredDoc = {} for prop in visible_attrs: filteredDoc[prop] = doc.get(prop, "") propValueMap = getPropValueMap() propValueMap['Team'] = self.getTeams() selected = {} for prop in propValueMap: if prop in filteredDoc: filteredDoc[prop], selected[prop] = reorder_list( propValueMap[prop], filteredDoc[prop]) content = self.templatepage( 'doc', title=title, status=status, name=name, rid=rid, tasks=json2form(tasks, indent=2, keep_first_value=False), table=json2table(filteredDoc, web_ui_names(), visible_attrs, selected), jsondata=json2form(doc, indent=2, keep_first_value=False), doc=json.dumps(doc), time=time, tasksConfigs=tasks_configs(doc, html=True), sTransition=state_transition(doc), pTransition=priority_transition(doc), transitions=transitions, ts=tst, user=user(), userdn=user_dn()) elif len(doc) > 1: jsondata = [pprint.pformat(d) for d in doc] content = self.templatepage('doc', title='Series of docs: %s' % rid, table="", jsondata=jsondata, time=time, tasksConfigs=tasks_configs(doc, html=True), sTransition=state_transition(doc), pTransition=priority_transition(doc), transitions=transitions, ts=tst, user=user(), userdn=user_dn()) else: doc = 'No request found for name=%s' % rid return self.abs_page('request', content) @expose def record2logdb(self, **kwds): """LogDB submission page""" print(kwds) request = kwds['request'] msg = kwds['message'] self.logdb.post(request, msg) msg = '<h6>Confirmation</h6>Your request has been entered to LogDB.' return self.abs_page('generic', msg) @expose def requests(self, **kwds): """Page showing requests""" if not kwds: kwds = {} if 'status' not in kwds: kwds.update({'status': 'acquired'}) dataResult = self.reqmgr.getRequestByStatus(kwds['status']) attrs = [ 'RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus', 'Campaign' ] docs = [] for data in dataResult: for doc in data.values(): docs.append(request_attr(doc, attrs)) sortby = kwds.get('sort', 'status') docs = [r for r in sort(docs, sortby)] filter_sort = self.templatepage('filter_sort') content = self.templatepage('requests', requests=toString(docs), sort=sortby, status=kwds['status'], filter_sort_table=filter_sort) return self.abs_page('requests', content) @expose def request(self, **kwargs): "Get data example and expose it as json" dataset = kwargs.get('uinput', '') if not dataset: return {'error': 'no input dataset'} url = 'https://cmsweb.cern.ch/reqmgr2/data/request?outputdataset=%s' % dataset params = {} headers = {'Accept': 'application/json'} wdata = getdata(url, params, headers) wdict = dict(date=time.ctime(), team='Team-A', status='Running', ID=genid(wdata)) winfo = self.templatepage('workflow', wdict=wdict, dataset=dataset, code=pprint.pformat(wdata)) content = self.templatepage('search', content=winfo) return self.abs_page('request', content) @expose def batch(self, **kwds): """batch page""" # TODO: we need a template for batch attributes # and read it from separate area, like DASMaps name = kwds.get('name', '') batch = {} if name: # batch = self.reqmgr.getBatchesByName(name) batch = { 'Name': 'Batch1', 'Description': 'Bla-bla', 'Creator': 'valya', 'Group': 'test', 'Workflows': ['workflow1', 'workflow2'], 'Attributes': { 'HeavyIon': ['true', 'false'] } } attributes = batch.get('Attributes', {}) workflows = batch.get('Workflows', []) description = batch.get('Description', '') creator = batch.get('Creator', user_dn()) content = self.templatepage('batch', name=name, attributes=json2table( attributes, web_ui_names()), workflows=workflows, creator=creator, description=description) return self.abs_page('batch', content) @expose def batches(self, **kwds): """Page showing batches""" if not kwds: kwds = {} if 'name' not in kwds: kwds.update({'name': ''}) sortby = kwds.get('sort', 'name') # results = self.reqmgr.getBatchesByName(kwds['name']) results = [ { 'Name': 'Batch1', 'Description': 'Bla-bla', 'Creator': 'valya', 'Group': 'test', 'Workflows': ['workflow1', 'workflow2'], 'Date': 'Fri Feb 13 10:36:41 EST 2015', 'Attributes': { 'HeavyIon': ['true', 'false'] } }, { 'Name': 'Batch2', 'Description': 'lksdjflksjdf', 'Creator': 'valya', 'Group': 'test', 'Workflows': ['workflow1', 'workflow2'], 'Date': 'Fri Feb 10 10:36:41 EST 2015', 'Attributes': { 'HeavyIon': ['true', 'false'] } }, ] docs = [r for r in sort(results, sortby)] filter_sort = self.templatepage('filter_sort') content = self.templatepage('batches', batches=docs, sort=sortby, filter_sort_table=filter_sort) return self.abs_page('batches', content) ### Aux methods ### @expose def put_request(self, **kwds): "PUT request callback to reqmgr server, should be used in AJAX" reqname = kwds.get('RequestName', '') status = kwds.get('RequestStatus', '') if not reqname: msg = 'Unable to update request status, empty request name' raise cherrypy.HTTPError(406, msg) if not status: msg = 'Unable to update request status, empty status value' raise cherrypy.HTTPError(406, msg) return self.reqmgr.updateRequestStatus(reqname, status) @expose def images(self, *args): """ Serve static images. """ args = list(args) check_scripts(args, self.imgmap, self.imgdir) mime_types = [ '*/*', 'image/gif', 'image/png', 'image/jpg', 'image/jpeg' ] accepts = cherrypy.request.headers.elements('Accept') for accept in accepts: if accept.value in mime_types and len(args) == 1 \ and args[0] in self.imgmap: image = self.imgmap[args[0]] # use image extension to pass correct content type ctype = 'image/%s' % image.split('.')[-1] cherrypy.response.headers['Content-type'] = ctype return serve_file(image, content_type=ctype) def serve(self, kwds, imap, idir, datatype='', minimize=False): "Serve files for high level APIs (yui/css/js)" args = [] for key, val in kwds.items(): if key == 'f': # we only look-up files from given kwds dict if isinstance(val, list): args += val else: args.append(val) scripts = check_scripts(args, imap, idir) return self.serve_files(args, scripts, imap, datatype, minimize) @exposecss @tools.gzip() def css(self, **kwargs): """ Serve provided CSS files. They can be passed as f=file1.css&f=file2.css """ resource = kwargs.get('resource', 'css') if resource == 'css': return self.serve(kwargs, self.cssmap, self.cssdir, 'css', True) @exposejs @tools.gzip() def js(self, **kwargs): """ Serve provided JS scripts. They can be passed as f=file1.js&f=file2.js with optional resource parameter to speficy type of JS files, e.g. resource=yui. """ resource = kwargs.get('resource', 'js') if resource == 'js': return self.serve(kwargs, self.jsmap, self.jsdir) def serve_files(self, args, scripts, resource, datatype='', minimize=False): """ Return asked set of files for JS, YUI, CSS. """ idx = "-".join(scripts) if idx not in self._cache.keys(): data = '' if datatype == 'css': data = '@CHARSET "UTF-8";' for script in args: path = os.path.join(sys.path[0], resource[script]) path = os.path.normpath(path) ifile = open(path) data = "\n".join([data, ifile.read(). \ replace('@CHARSET "UTF-8";', '')]) ifile.close() if datatype == 'css': set_headers("text/css") if minimize: self._cache[idx] = minify(data) else: self._cache[idx] = data return self._cache[idx]
class ReqMgrService(TemplatedPage): """ Request Manager web service class """ def __init__(self, app, config, mount): self.base = config.base self.rootdir = '/'.join(WMCore.__file__.split('/')[:-1]) if config and not isinstance(config, dict): web_config = config.dictionary_() if not config: web_config = {'base': self.base} TemplatedPage.__init__(self, web_config) imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd() + '/images') self.imgdir = web_config.get('imgdir', imgdir) cssdir = os.environ.get('RM_CSSPATH', os.getcwd() + '/css') self.cssdir = web_config.get('cssdir', cssdir) jsdir = os.environ.get('RM_JSPATH', os.getcwd() + '/js') self.jsdir = web_config.get('jsdir', jsdir) spdir = os.environ.get('RM_SPECPATH', os.getcwd() + '/specs') self.spdir = web_config.get('spdir', spdir) # read scripts area and initialize data-ops scripts self.sdir = os.environ.get('RM_SCRIPTS', os.getcwd() + '/scripts') self.sdir = web_config.get('sdir', self.sdir) self.sdict_thr = web_config.get('sdict_thr', 600) # put reasonable 10 min interval self.sdict = {'ts': time.time()} # placeholder for data-ops scripts self.update_scripts(force=True) # To be filled at run time self.cssmap = {} self.jsmap = {} self.imgmap = {} self.yuimap = {} std_specs_dir = os.path.join(self.rootdir, 'WMSpec/StdSpecs') self.std_specs = spec_list(std_specs_dir) self.std_specs.sort() # Update CherryPy configuration mime_types = ['text/css'] mime_types += ['application/javascript', 'text/javascript', 'application/x-javascript', 'text/x-javascript'] cherryconf.update({'tools.encode.on': True, 'tools.gzip.on': True, 'tools.gzip.mime_types': mime_types, }) self._cache = {} # initialize access to reqmgr2 APIs self.reqmgr_url = config.reqmgr.reqmgr2_url self.reqmgr = ReqMgr(self.reqmgr_url) # only gets current view (This might cause to reponse time much longer, # If upto date view is not needed overwrite Fale) self.reqmgr._noStale = True # get fields which we'll use in templates cdict = config.reqmgr.dictionary_() self.couch_url = cdict.get('couch_host', '') self.couch_dbname = cdict.get('couch_reqmgr_db', '') self.couch_wdbname = cdict.get('couch_workload_summary_db', '') self.acdc_url = cdict.get('acdc_host', '') self.acdc_dbname = cdict.get('acdc_db', '') self.configcache_url = cdict.get('couch_config_cache_url', self.couch_url) self.dbs_url = cdict.get('dbs_url', '') self.dqm_url = cdict.get('dqm_url', '') self.sw_ver = cdict.get('default_sw_version', 'CMSSW_7_6_1') self.sw_arch = cdict.get('default_sw_scramarch', 'slc6_amd64_gcc493') # LogDB holder centralurl = cdict.get("central_logdb_url", "") identifier = cdict.get("log_reporter", "reqmgr2") self.logdb = LogDB(centralurl, identifier) # local team cache which will request data from wmstats base, uri = self.reqmgr_url.split('://') base_url = '%s://%s' % (base, uri.split('/')[0]) self.wmstatsurl = cdict.get('wmstats_url', '%s/wmstatsserver' % base_url) if not self.wmstatsurl: raise Exception('ReqMgr2 configuration file does not provide wmstats url') self.team_cache = [] # fetch assignment arguments specification from StdBase self.assignArgs = StdBase().getWorkloadAssignArgs() self.assignArgs = {key: val['default'] for key, val in self.assignArgs.items()} def getTeams(self): "Helper function to get teams from wmstats or local cache" teams = self.team_cache url = '%s/data/teams' % self.wmstatsurl params = {} headers = {'Accept': 'application/json'} try: data = getdata(url, params, headers) if 'error' in data: print("WARNING: fail to get teams from %s" % url) print(data) teams = data.get('result', []) self.team_cache = teams except Exception as exp: print("WARNING: fail to get teams from %s" % url) print(str(exp)) return teams def update_scripts(self, force=False): "Update scripts dict" if force or abs(time.time() - self.sdict['ts']) > self.sdict_thr: for item in os.listdir(self.sdir): with open(os.path.join(self.sdir, item), 'r') as istream: self.sdict[item.split('.')[0]] = istream.read() self.sdict['ts'] = time.time() def abs_page(self, tmpl, content): """generate abstract page""" menu = self.templatepage('menu', menus=menus(), tmpl=tmpl) body = self.templatepage('generic', menu=menu, content=content) page = self.templatepage('main', content=body, user=user()) return page def page(self, content): """ Provide page wrapped with top/bottom templates. """ return self.templatepage('main', content=content) def error(self, content): "Generate common error page" content = self.templatepage('error', content=content) return self.abs_page('error', content) @expose def index(self): """Main page""" content = self.templatepage('index', requests=ACTIVE_STATUS, rdict=REQUEST_STATE_TRANSITION) return self.abs_page('main', content) @expose def home(self, **kwds): """Main page""" return self.index(**kwds) ### Request actions ### @expose @checkargs(['status', 'sort']) def assign(self, **kwds): """assign page""" if not kwds: kwds = {} if 'status' not in kwds: kwds.update({'status': 'assignment-approved'}) docs = [] attrs = ['RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus'] dataResult = self.reqmgr.getRequestByStatus(statusList=[kwds['status']]) for data in dataResult: for val in data.values(): docs.append(request_attr(val, attrs)) sortby = kwds.get('sort', 'status') docs = [r for r in sort(docs, sortby)] assignDict = deepcopy(self.assignArgs) assignDict.update(getPropValueMap()) assignDict['Team'] = self.getTeams() filter_sort = self.templatepage('filter_sort') content = self.templatepage('assign', sort=sortby, filter_sort_table=filter_sort, sites=SITE_CACHE.getData(), site_white_list=site_white_list(), site_black_list=site_black_list(), user=user(), user_dn=user_dn(), requests=toString(docs), misc_table=json2table(assignDict, web_ui_names(), "all_attributes"), misc_json=json2form(assignDict, indent=2, keep_first_value=True)) return self.abs_page('assign', content) @expose @checkargs(['status', 'sort']) def approve(self, **kwds): """ Approve page: get list of request associated with user DN. Fetch their status list from ReqMgr and display if requests were seen by data-ops. """ if not kwds: kwds = {} if 'status' not in kwds: kwds.update({'status': 'new'}) kwds.update({'_nostale': True}) docs = [] attrs = ['RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus', 'Campaign'] dataResult = self.reqmgr.getRequestByStatus(statusList=[kwds['status']]) for data in dataResult: for val in data.values(): docs.append(request_attr(val, attrs)) sortby = kwds.get('sort', 'status') docs = [r for r in sort(docs, sortby)] filter_sort = self.templatepage('filter_sort') content = self.templatepage('approve', requests=toString(docs), date=tstamp(), sort=sortby, filter_sort_table=filter_sort, gen_color=gen_color) return self.abs_page('approve', content) @expose def create(self, **kwds): """create page""" # get list of standard specs from WMCore and new ones from local area # loc_specs_dir = os.path.join(self.spdir, 'Specs') # local specs # loc_specs = spec_list(loc_specs_dir, 'Specs') # all_specs = list(set(self.std_specs + loc_specs)) # all_specs.sort() all_specs = list(self.std_specs) spec = kwds.get('form', '') if not spec: spec = self.std_specs[0] # make spec first in all_specs list if spec in all_specs: all_specs.remove(spec) all_specs = [spec] + all_specs jsondata = get_request_template_from_type(spec) # create templatized page out of provided forms self.update_scripts() content = self.templatepage('create', table=json2table(jsondata, web_ui_names(), jsondata), jsondata=json2form(jsondata, indent=2, keep_first_value=True), name=spec, scripts=[s for s in self.sdict.keys() if s != 'ts'], specs=all_specs) return self.abs_page('create', content) def generate_objs(self, script, jsondict): """Generate objects from givem JSON template""" self.update_scripts() code = self.sdict.get(script, '') if code.find('def genobjs(jsondict)') == -1: return self.error( "Improper python snippet, your code should start with <b>def genobjs(jsondict)</b> function") exec (code) # code snippet must starts with genobjs function return [r for r in genobjs(jsondict)] @expose def config(self, name): "Fetch config for given request name" result = self.reqmgr.getConfig(name) if len(result) == 1: result = result[0] else: result = 'Configuration not found for: %s' % name return result.replace('\n', '<br/>') @expose def fetch(self, rid): "Fetch document for given id" rid = rid.replace('request-', '') doc = self.reqmgr.getRequestByNames(rid) transitions = [] tst = time.time() # get request tasks tasks = self.reqmgr.getRequestTasks(rid) if len(doc) == 1: try: doc = doc[0][rid] except: pass name = doc.get('RequestName', 'NA') title = 'Request %s' % name status = doc.get('RequestStatus', '') transitions = REQUEST_STATE_TRANSITION.get(status, []) if status in transitions: transitions.remove(status) visible_attrs = get_modifiable_properties(status) filterout_attrs = get_protected_properties() # extend filterout list with "RequestStatus" since it is passed separately filterout_attrs.append("RequestStatus") for key, val in self.assignArgs.items(): if not doc.get(key): doc[key] = val if visible_attrs == "all_attributes": filteredDoc = doc for prop in filterout_attrs: if prop in filteredDoc: del filteredDoc[prop] else: filteredDoc = {} for prop in visible_attrs: filteredDoc[prop] = doc.get(prop, "") propValueMap = getPropValueMap() propValueMap['Team'] = self.getTeams() selected = {} for prop in propValueMap: if prop in filteredDoc: filteredDoc[prop], selected[prop] = reorder_list(propValueMap[prop], filteredDoc[prop]) content = self.templatepage('doc', title=title, status=status, name=name, rid=rid, tasks=json2form(tasks, indent=2, keep_first_value=False), table=json2table(filteredDoc, web_ui_names(), visible_attrs, selected), jsondata=json2form(doc, indent=2, keep_first_value=False), doc=json.dumps(doc), time=time, tasksConfigs=tasks_configs(doc, html=True), sTransition=state_transition(doc), pTransition=priority_transition(doc), transitions=transitions, humanStates=REQUEST_HUMAN_STATES, ts=tst, user=user(), userdn=user_dn()) elif len(doc) > 1: jsondata = [pprint.pformat(d) for d in doc] content = self.templatepage('doc', title='Series of docs: %s' % rid, table="", jsondata=jsondata, time=time, tasksConfigs=tasks_configs(doc, html=True), sTransition=state_transition(doc), pTransition=priority_transition(doc), transitions=transitions, humanStates=REQUEST_HUMAN_STATES, ts=tst, user=user(), userdn=user_dn()) else: doc = 'No request found for name=%s' % rid return self.abs_page('request', content) @expose def record2logdb(self, **kwds): """LogDB submission page""" print(kwds) request = kwds['request'] msg = kwds['message'] self.logdb.post(request, msg) msg = '<h6>Confirmation</h6>Your request has been entered to LogDB.' return self.abs_page('generic', msg) @expose def requests(self, **kwds): """Page showing requests""" if not kwds: kwds = {} if 'status' not in kwds: kwds.update({'status': 'acquired'}) dataResult = self.reqmgr.getRequestByStatus(kwds['status']) attrs = ['RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus', 'Campaign'] docs = [] for data in dataResult: for doc in data.values(): docs.append(request_attr(doc, attrs)) sortby = kwds.get('sort', 'status') docs = [r for r in sort(docs, sortby)] filter_sort = self.templatepage('filter_sort') content = self.templatepage('requests', requests=toString(docs), sort=sortby, status=kwds['status'], filter_sort_table=filter_sort) return self.abs_page('requests', content) @expose def request(self, **kwargs): "Get data example and expose it as json" dataset = kwargs.get('uinput', '') if not dataset: return {'error': 'no input dataset'} url = 'https://cmsweb.cern.ch/reqmgr2/data/request?outputdataset=%s' % dataset params = {} headers = {'Accept': 'application/json'} wdata = getdata(url, params, headers) wdict = dict(date=time.ctime(), team='Team-A', status='Running', ID=genid(wdata)) winfo = self.templatepage('workflow', wdict=wdict, dataset=dataset, code=pprint.pformat(wdata)) content = self.templatepage('search', content=winfo) return self.abs_page('request', content) @expose def batch(self, **kwds): """batch page""" # TODO: we need a template for batch attributes # and read it from separate area, like DASMaps name = kwds.get('name', '') batch = {} if name: # batch = self.reqmgr.getBatchesByName(name) batch = {'Name': 'Batch1', 'Description': 'Bla-bla', 'Creator': 'valya', 'Group': 'test', 'Workflows': ['workflow1', 'workflow2'], 'Attributes': {'HeavyIon': ['true', 'false']}} attributes = batch.get('Attributes', {}) workflows = batch.get('Workflows', []) description = batch.get('Description', '') creator = batch.get('Creator', user_dn()) content = self.templatepage('batch', name=name, attributes=json2table(attributes, web_ui_names()), workflows=workflows, creator=creator, description=description) return self.abs_page('batch', content) @expose def batches(self, **kwds): """Page showing batches""" if not kwds: kwds = {} if 'name' not in kwds: kwds.update({'name': ''}) sortby = kwds.get('sort', 'name') # results = self.reqmgr.getBatchesByName(kwds['name']) results = [ {'Name': 'Batch1', 'Description': 'Bla-bla', 'Creator': 'valya', 'Group': 'test', 'Workflows': ['workflow1', 'workflow2'], 'Date': 'Fri Feb 13 10:36:41 EST 2015', 'Attributes': {'HeavyIon': ['true', 'false']}}, {'Name': 'Batch2', 'Description': 'lksdjflksjdf', 'Creator': 'valya', 'Group': 'test', 'Workflows': ['workflow1', 'workflow2'], 'Date': 'Fri Feb 10 10:36:41 EST 2015', 'Attributes': {'HeavyIon': ['true', 'false']}}, ] docs = [r for r in sort(results, sortby)] filter_sort = self.templatepage('filter_sort') content = self.templatepage('batches', batches=docs, sort=sortby, filter_sort_table=filter_sort) return self.abs_page('batches', content) ### Aux methods ### @expose def put_request(self, **kwds): "PUT request callback to reqmgr server, should be used in AJAX" reqname = kwds.get('RequestName', '') status = kwds.get('RequestStatus', '') if not reqname: msg = 'Unable to update request status, empty request name' raise cherrypy.HTTPError(406, msg) if not status: msg = 'Unable to update request status, empty status value' raise cherrypy.HTTPError(406, msg) return self.reqmgr.updateRequestStatus(reqname, status) @expose def images(self, *args): """ Serve static images. """ args = list(args) check_scripts(args, self.imgmap, self.imgdir) mime_types = ['*/*', 'image/gif', 'image/png', 'image/jpg', 'image/jpeg'] accepts = cherrypy.request.headers.elements('Accept') for accept in accepts: if accept.value in mime_types and len(args) == 1 \ and args[0] in self.imgmap: image = self.imgmap[args[0]] # use image extension to pass correct content type ctype = 'image/%s' % image.split('.')[-1] cherrypy.response.headers['Content-type'] = ctype return serve_file(image, content_type=ctype) def serve(self, kwds, imap, idir, datatype='', minimize=False): "Serve files for high level APIs (yui/css/js)" args = [] for key, val in kwds.items(): if key == 'f': # we only look-up files from given kwds dict if isinstance(val, list): args += val else: args.append(val) scripts = check_scripts(args, imap, idir) return self.serve_files(args, scripts, imap, datatype, minimize) @exposecss @tools.gzip() def css(self, **kwargs): """ Serve provided CSS files. They can be passed as f=file1.css&f=file2.css """ resource = kwargs.get('resource', 'css') if resource == 'css': return self.serve(kwargs, self.cssmap, self.cssdir, 'css', True) @exposejs @tools.gzip() def js(self, **kwargs): """ Serve provided JS scripts. They can be passed as f=file1.js&f=file2.js with optional resource parameter to speficy type of JS files, e.g. resource=yui. """ resource = kwargs.get('resource', 'js') if resource == 'js': return self.serve(kwargs, self.jsmap, self.jsdir) def serve_files(self, args, scripts, resource, datatype='', minimize=False): """ Return asked set of files for JS, YUI, CSS. """ idx = "-".join(scripts) if idx not in self._cache.keys(): data = '' if datatype == 'css': data = '@CHARSET "UTF-8";' for script in args: path = os.path.join(sys.path[0], resource[script]) path = os.path.normpath(path) with open(path) as ifile: data = "\n".join([data, ifile.read().replace('@CHARSET "UTF-8";', '')]) if datatype == 'css': set_headers("text/css") if minimize: self._cache[idx] = minify(data) else: self._cache[idx] = data return self._cache[idx]
class ReqMgrTest(RESTBaseUnitTestWithDBBackend): """ Test WorkQueue Service client It will start WorkQueue RESTService Server DB sets from environment variable. Client DB sets from environment variable. This checks whether DS call makes without error and return the results. Not the correctness of functions. That will be tested in different module. """ def setFakeDN(self): # put into ReqMgr auxiliary database under "software" document scram/cmsms # which we'll need a little for request injection #Warning: this assumes the same structure in jenkins wmcore_root/test self.admin_header = getAuthHeader(self.test_authz_key.data, ADMIN_PERMISSION) self.create_header = getAuthHeader(self.test_authz_key.data, CREATE_PERMISSION) self.default_header = getAuthHeader(self.test_authz_key.data, DEFAULT_PERMISSION) self.assign_header = getAuthHeader(self.test_authz_key.data, ASSIGN_PERMISSION) self.default_status_header = getAuthHeader(self.test_authz_key.data, DEFAULT_STATUS_PERMISSION) def setUp(self): self.setConfig(config) self.setCouchDBs([(config.views.data.couch_reqmgr_db, "ReqMgr"), (config.views.data.couch_reqmgr_aux_db, None)]) self.setSchemaModules([]) RESTBaseUnitTestWithDBBackend.setUp(self) self.setFakeDN() requestPath = os.path.join(getWMBASE(), "test", "data", "ReqMgr", "requests", "DMWM") rerecoFile = open(os.path.join(requestPath, "ReReco.json"), 'r') rerecoArgs = json.load(rerecoFile) self.rerecoCreateArgs = rerecoArgs["createRequest"] self.rerecoAssignArgs = rerecoArgs["assignRequest"] cmsswDoc = {"_id": "software"} cmsswDoc[self.rerecoCreateArgs["ScramArch"]] = [] cmsswDoc[self.rerecoCreateArgs["ScramArch"]].append( self.rerecoCreateArgs["CMSSWVersion"]) insertDataToCouch(os.getenv("COUCHURL"), config.views.data.couch_reqmgr_aux_db, cmsswDoc) self.reqSvc = ReqMgr(self.jsonSender["host"]) self.reqSvc._noStale = True self.reqSvc['requests'].additionalHeaders = self.create_header def tearDown(self): RESTBaseUnitTestWithDBBackend.tearDown(self) def testRequestSimpleCycle(self): """ test request cycle with one request without composite get condition. post, get, put """ # test post method response = self.reqSvc.insertRequests(self.rerecoCreateArgs) self.assertEqual(len(response), 1) requestName = response[0]['request'] ## test get method # get by name response = self.reqSvc.getRequestByNames(requestName) self.assertEqual(response[requestName]['RequestPriority'], 10000) self.assertEqual(len(response), 1) # get by status response = self.reqSvc.getRequestByStatus('new') self.assertEqual(len(response), 1) print(response) self.reqSvc.updateRequestStatus(requestName, 'assignment-approved') response = self.reqSvc.getRequestByStatus('assignment-approved') self.assertEqual(len(response), 1) self.reqSvc.updateRequestProperty( requestName, { 'RequestStatus': 'assigned', "AcquisitionEra": "TEST_ERA", "Team": "unittest", "SiteWhitelist": ["T1_US_CBS"], "SiteBlacklist": ["T1_US_FOX"] }) response = self.reqSvc.getRequestByStatus('assignment-approved') self.assertEqual(len(response), 0) response = self.reqSvc.getRequestByStatus('assigned') self.assertEqual(len(response), 1) self.assertEqual(response.values()[0]["SiteWhitelist"], ["T1_US_CBS"]) self.reqSvc.updateRequestStats( requestName, { 'total_jobs': 100, 'input_lumis': 100, 'input_events': 100, 'input_num_files': 100 })
class MSTransferor(object): def __init__(self, microConfig, status, logger=None): """ Runs the basic setup and initialization for the MS Transferor module :param microConfig: microservice configuration """ self.msConfig = microConfig self.status = status self.uConfig = {} self.logger = getMSLogger(microConfig['verbose'], logger=logger) self.reqmgr2 = ReqMgr(microConfig['reqmgrUrl'], logger=self.logger) self.reqmgrAux = ReqMgrAux(microConfig['reqmgrUrl'], httpDict={'cacheduration': 60}, logger=self.logger) # eventually will change it to Rucio self.phedex = PhEDEx(httpDict={'cacheduration': 10 * 60}, dbsUrl=microConfig['dbsUrl'], logger=self.logger) def prep(self): """ Runs any preparation tasks before executing the actual algorithm. For now: * fetches the unified configuration :return: False if it fail to run this action, otherwise True """ self.uConfig = self.reqmgrAux.getUnifiedConfig(docName="config") return bool(self.uConfig) def execute(self): """ Executes the whole transferor logic :param reqStatus: request status to that matters for this module :return: """ if not self.prep(): self.logger.warning( "Failed to fetch the latest unified config. Skipping this cycle" ) return self.uConfig = self.uConfig[0] requestRecords = [] try: # get requests from ReqMgr2 data-service for given status requests = self.reqmgr2.getRequestByStatus([self.status], detail=True) if requests: requests = requests[0] self.logger.info("### transferor found %s requests in '%s' state", len(requests), self.status) if requests: for _, wfData in requests.iteritems(): requestRecords.append(self.requestRecord(wfData)) except Exception as err: self.logger.exception('### transferor error: %s', str(err)) if not requestRecords: return try: reqInfo = RequestInfo(self.msConfig, self.uConfig, self.logger) for reqSlice in grouper(requestRecords, 50): reqResults = reqInfo(reqSlice) self.logger.info("%d requests completely processed.", len(reqResults)) self.logger.info( "Working on the data subscription and status change...") # process all requests for req in reqResults: reqName = req['name'] # perform transfer tid = self.transferRequest(req) if tid: # Once all transfer requests were successfully made, update: assigned -> staging self.logger.debug( "### transfer request for %s successfull", reqName) self.change(req, 'staging', '### transferor') # if there is nothing to be transferred (no input at all), # then update the request status once again staging -> staged # self.change(req, 'staged', '### transferor') except Exception as err: # general error self.logger.exception('### transferor error: %s', str(err)) def post(self): """ Runs any post tasks before exiting the execution cycle :return: """ pass def requestRecord(self, wfData): """ Selects only important information for a request dictionary Returns: a dictionary """ datasets = [] if "TaskChain" in wfData or "StepChain" in wfData: innerDicts = [] for i in range( 1, wfData.get("TaskChain", wfData.get("StepChain")) + 1): innerDicts.append( wfData.get("Task%d" % i, wfData.get("Step%d" % i))) else: # ReReco and DQMHarvesting innerDicts = [wfData] for item in innerDicts: for key in ['InputDataset', 'MCPileup', 'DataPileup']: dataset = item.get(key) if dataset: datasets.append({'type': key, 'name': dataset}) return { 'name': wfData.get('RequestName'), 'reqStatus': wfData.get('RequestStatus'), 'SiteWhiteList': wfData.get('SiteWhitelist', []), 'SiteBlackList': wfData.get('SiteBlacklist', []), 'datasets': datasets, 'campaign': [] } def transferRequest(self, req): "Send request to Phedex and return status of request subscription" datasets = req.get('datasets', []) sites = req.get('sites', []) if datasets and sites: self.logger.debug("### creating subscription for: %s", pformat(req)) subscription = PhEDExSubscription(datasets, sites, self.msConfig['group']) # TODO: implement how to get transfer id tid = hashlib.md5(str(subscription)).hexdigest() # TODO: when ready enable submit subscription step # self.phedex.subscribe(subscription) return tid def change(self, req, reqStatus, prefix='###'): """ Change request status, internally it is done via PUT request to ReqMgr2: curl -X PUT -H "Content-Type: application/json" \ -d '{"RequestStatus":"staging", "RequestName":"bla-bla"}' \ https://xxx.yyy.zz/reqmgr2/data/request """ self.logger.debug('%s updating %s status to %s', prefix, req['name'], reqStatus) try: if not self.msConfig['readOnly']: self.reqmgr2.updateRequestStatus(req['name'], reqStatus) except Exception as err: self.logger.exception("Failed to change request status. Error: %s", str(err))
class MSManager(object): """ Entry point for the MicroServices. This class manages both transferor and monitor services/threads. """ def __init__(self, config=None, logger=None): """ Setup a bunch of things, like: * logger for this service * initialize all the necessary service helpers * fetch the unified configuration from central couch * update the unified configuration with some deployment and default settings * start both transfer and monitor threads :param config: reqmgr2ms service configuration :param logger: """ self.uConfig = {} self.config = config self.logger = getMSLogger(getattr(config, 'verbose', False), logger) self._parseConfig(config) self.logger.info("Configuration including default values:\n%s", self.msConfig) self.reqmgr2 = ReqMgr(self.msConfig['reqmgrUrl'], logger=self.logger) self.reqmgrAux = ReqMgrAux(self.msConfig['reqmgrUrl'], httpDict={'cacheduration': 60}, logger=self.logger) # transferor has to look at workflows in assigned status self.msTransferor = MSTransferor(self.msConfig, "assigned", logger=self.logger) ### Last but not least, get the threads started thname = 'MSTransferor' self.transfThread = start_new_thread( thname, daemon, (self.transferor, 'assigned', self.msConfig['interval'], self.logger)) self.logger.debug("### Running %s thread %s", thname, self.transfThread.running()) thname = 'MSTransferorMonit' self.monitThread = start_new_thread( thname, daemon, (self.monitor, 'staging', self.msConfig['interval'] * 2, self.logger)) self.logger.debug("+++ Running %s thread %s", thname, self.monitThread.running()) def _parseConfig(self, config): """ __parseConfig_ Parse the MicroService configuration and set any default values. :param config: config as defined in the deployment """ self.logger.info("Using the following config:\n%s", config) self.msConfig = {} self.msConfig['verbose'] = getattr(config, 'verbose', False) self.msConfig['group'] = getattr(config, 'group', 'DataOps') self.msConfig['interval'] = getattr(config, 'interval', 5 * 60) self.msConfig['readOnly'] = getattr(config, 'readOnly', True) self.msConfig['reqmgrUrl'] = getattr(config, 'reqmgr2Url', 'https://cmsweb.cern.ch/reqmgr2') self.msConfig['reqmgrCacheUrl'] = self.msConfig['reqmgrUrl'].replace( 'reqmgr2', 'couchdb/reqmgr_workload_cache') self.msConfig['phedexUrl'] = getattr( config, 'phedexUrl', 'https://cmsweb.cern.ch/phedex/datasvc/json/prod') self.msConfig['dbsUrl'] = getattr( config, 'dbsUrl', 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader') def transferor(self, reqStatus): """ MSManager transferor function. It performs Unified logic for data subscription and transfers requests from assigned to staging/staged state of ReqMgr2. For references see https://github.com/dmwm/WMCore/wiki/ReqMgr2-MicroService-Transferor """ startT = time.time() self.logger.info("Starting the transferor thread...") self.msTransferor.execute() self.logger.info("Total transferor execution time: %.2f secs", time.time() - startT) def monitor(self, reqStatus='staging'): """ MSManager monitoring function. It performs transfer requests from staging to staged state of ReqMgr2. For references see https://github.com/dmwm/WMCore/wiki/ReqMgr2-MicroService-Transferor """ startT = time.time() self.logger.info("Starting the monitor thread...") # First, fetch/update our unified configuration from reqmgr_aux db # Keep our own copy of the unified config to avoid race conditions self.uConfig = self.reqmgrAux.getUnifiedConfig(docName="config") if not self.uConfig: self.logger.warning( "Monitor failed to fetch the unified config. Skipping this cycle." ) return self.uConfig = self.uConfig[0] try: # get requests from ReqMgr2 data-service for given statue # here with detail=False we get back list of records requests = self.reqmgr2.getRequestByStatus([reqStatus], detail=False) self.logger.debug('+++ monit found %s requests in %s state', len(requests), reqStatus) requestStatus = {} # keep track of request statuses for reqName in requests: req = {'name': reqName, 'reqStatus': reqStatus} # get transfer IDs tids = self.getTransferIDs() # get transfer status transferStatuses = self.getTransferStatuses(tids) # get campaing and unified configuration campaign = self.requestCampaign(reqName) conf = self.requestConfiguration(reqName) self.logger.debug("+++ request %s campaing %s conf %s", req, campaign, conf) # if all transfers are completed, move the request status staging -> staged # completed = self.checkSubscription(request) completed = 100 # TMP if completed == 100: # all data are staged self.logger.debug( "+++ request %s all transfers are completed", req) self.change(req, 'staged', '+++ monit') # if pileup transfers are completed AND some input blocks are completed, move the request status staging -> staged elif self.pileupTransfersCompleted(tids): self.logger.debug( "+++ request %s pileup transfers are completed", req) self.change(req, 'staged', '+++ monit') # transfers not completed, just update the database with their completion else: self.logger.debug( "+++ request %s transfers are not completed", req) requestStatus[ req] = transferStatuses # TODO: implement update of transfer ids self.updateTransferIDs(requestStatus) except Exception as err: # general error self.logger.exception('+++ monit error: %s', str(err)) self.logger.info("Total monitor execution time: %.2f secs", time.time() - startT) def stop(self): "Stop MSManager" # stop MSTransferorMonit thread self.monitThread.stop() # stop MSTransferor thread self.transfThread.stop() # stop checkStatus thread status = self.transfThread.running() return status def getTransferIDsDoc(self): """ Get transfer ids document from backend. The document has the following form: { "wf_A": [record1, record2, ...], "wf_B": [....], } where each record has the following format: {"timestamp":000, "dataset":"/a/b/c", "type": "primary", "trainsferIDs": [1,2,3]} """ doc = {} return doc def updateTransferIDs(self, requestStatus): "Update transfer ids in backend" # TODO/Wait: https://github.com/dmwm/WMCore/issues/9198 # doc = self.getTransferIDsDoc() def getTransferIDs(self): "Get transfer ids from backend" # TODO/Wait: https://github.com/dmwm/WMCore/issues/9198 # meanwhile return transfer ids from internal store return [] def getTransferStatuses(self, tids): "get transfer statuses for given transfer IDs from backend" # transfer docs on backend has the following form # https://gist.github.com/amaltaro/72599f995b37a6e33566f3c749143154 statuses = {} for tid in tids: # TODO: I need to find request name from transfer ID # status = self.checkSubscription(request) status = 100 statuses[tid] = status return statuses def requestCampaign(self, req): "Return request campaign" return 'campaign_TODO' # TODO def requestConfiguration(self, req): "Return request configuration" return {} def pileupTransfersCompleted(self, tids): "Check if pileup transfers are completed" # TODO: add implementation return False def checkSubscription(self, req): "Send request to Phedex and return status of request subscription" sdict = {} for dataset in req.get('datasets', []): data = self.phedex.subscriptions(dataset=dataset, group=self.msConfig['group']) self.logger.debug("### dataset %s group %s", dataset, self.msConfig['group']) self.logger.debug("### subscription %s", data) for row in data['phedex']['dataset']: if row['name'] != dataset: continue nodes = [s['node'] for s in row['subscription']] rNodes = req.get('sites') self.logger.debug("### nodes %s %s", nodes, rNodes) subset = set(nodes) & set(rNodes) if subset == set(rNodes): sdict[dataset] = 1 else: pct = float(len(subset)) / float(len(set(rNodes))) sdict[dataset] = pct self.logger.debug("### sdict %s", sdict) tot = len(sdict.keys()) if not tot: return -1 # return percentage of completion return round(float(sum(sdict.values())) / float(tot), 2) * 100 def checkStatus(self, req): "Check status of request in local storage" self.logger.debug("### checkStatus of request: %s", req['name']) # check subscription status of the request # completed = self.checkSubscription(req) completed = 100 if completed == 100: # all data are staged self.logger.debug( "### request is completed, change its status and remove it from the store" ) self.change(req, 'staged', '### transferor') else: self.logger.debug("### request %s, completed %s", req, completed) def info(self, req): "Return info about given request" completed = self.checkSubscription(req) return {'request': req, 'status': completed} def delete(self, request): "Delete request in backend" pass def change(self, req, reqStatus, prefix='###'): """ Change request status, internally it is done via PUT request to ReqMgr2: curl -X PUT -H "Content-Type: application/json" \ -d '{"RequestStatus":"staging", "RequestName":"bla-bla"}' \ https://xxx.yyy.zz/reqmgr2/data/request """ self.logger.debug('%s updating %s status to %s', prefix, req['name'], reqStatus) try: if not self.msConfig['readOnly']: self.reqmgr2.updateRequestStatus(req['name'], reqStatus) except Exception as err: self.logger.exception("Failed to change request status. Error: %s", str(err))