def __getCache(self): Operations.__cacheLock.acquire() try: currentVersion = gConfigurationData.getVersion() if currentVersion != Operations.__cacheVersion: Operations.__cache = {} Operations.__cacheVersion = currentVersion cacheKey = (self.__vo, self.__setup) if cacheKey in Operations.__cache: return Operations.__cache[cacheKey] mergedCFG = CFG.CFG() for path in self.__getSearchPaths(): pathCFG = gConfigurationData.mergedCFG[path] if pathCFG: mergedCFG = mergedCFG.mergeWith(pathCFG) Operations.__cache[cacheKey] = mergedCFG return Operations.__cache[cacheKey] finally: try: Operations.__cacheLock.release() except thread.error: pass
def __loadDIRACCFG( self ): installPath = gConfig.getValue( '/LocalInstallation/TargetPath', gConfig.getValue( '/LocalInstallation/RootPath', '' ) ) if not installPath: installPath = rootPath cfgPath = os.path.join( installPath, 'etc', 'dirac.cfg' ) try: diracCFG = CFG.CFG().loadFromFile( cfgPath ) except Exception, excp: return S_ERROR( "Could not load dirac.cfg: %s" % str( excp ) )
def __getCache(self): Operations.__cacheLock.acquire() try: currentVersion = gConfigurationData.getVersion() if currentVersion != Operations.__cacheVersion: Operations.__cache = {} Operations.__cacheVersion = currentVersion cacheKey = (self.__threadData.vo, self.__threadData.setup) if cacheKey in Operations.__cache: return Operations.__cache[cacheKey] mergedCFG = CFG.CFG() for path in (self.__getDefaultPath(), self.__getSetupPath()): pathCFG = gConfigurationData.mergedCFG[path] if pathCFG: mergedCFG = mergedCFG.mergeWith(pathCFG) Operations.__cache[cacheKey] = mergedCFG return Operations.__cache[cacheKey] finally: Operations.__cacheLock.release()
def loadCFGFromRepository( self, svnPath ): remoteData = self.getSVNFileContents( svnPath ) return CFG.CFG().loadFromBuffer( remoteData )
def initDIRAC(rootPath, enableDebug=False): # CONFIGURATION OPTIONS HERE (note: all config options will override # any Pylons config options) configDict = {'webConfig': {}} configDict['webConfig']['dirac.webroot'] = rootPath diracRootPath = os.path.realpath(os.path.dirname( os.path.dirname(rootPath))) configDict['webConfig']['dirac.root'] = diracRootPath if diracRootPath not in sys.path: sys.path.append(diracRootPath) from DIRAC.FrameworkSystem.Client.Logger import gLogger gLogger.registerBackends(['stderr']) from DIRAC.Core.Base import Script Script.registerSwitch("r", "reload", "Reload for pylons") Script.localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") Script.localCfg.addDefaultEntry("LogColor", True) Script.initialize(script="Website", ignoreErrors=True, initializeMonitor=False) gLogger._systemName = "Framework" gLogger.initialize("Web", "/Website") gLogger.setLevel("VERBOSE") from DIRAC import gMonitor, gConfig, rootPath as droot from DIRAC.Core.Utilities import CFG from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions gMonitor.setComponentType(gMonitor.COMPONENT_WEB) gMonitor.initialize() gMonitor.registerActivity("pagesServed", "Pages served", "Framework", "pages", gMonitor.OP_SUM) gLogger.info("DIRAC Initialized") configDict['portalVersion'] = portalVersion(rootPath) gLogger.info("DIRAC portal version: %s" % configDict['portalVersion']) extModules = ['%sDIRAC' % module for module in getCSExtensions()] #Load web.cfg of modules cfgFilePaths = [os.path.join(droot, "etc", "web.cfg")] for extModule in extModules: gLogger.info("Adding web.cfg for %s extension" % extModule) extModulePath = os.path.join(diracRootPath, extModule) webCFGPath = os.path.join(extModulePath, "Web", "web.cfg") cfgFilePaths.append(webCFGPath) for systemDir in os.listdir(extModulePath): webCFGSystemPath = os.path.join(extModulePath, systemDir, "Web", "web.cfg") cfgFilePaths.append(webCFGSystemPath) webCFG = CFG.CFG() for webCFGPath in cfgFilePaths: if not os.path.isfile(webCFGPath): gLogger.warn("%s does not exist" % webCFGPath) else: gLogger.info("Loading %s" % webCFGPath) modCFG = CFG.CFG().loadFromFile(webCFGPath) if modCFG.getOption('Website/AbsoluteDefinition', False): gLogger.info("CFG %s is absolute" % webCFGPath) webCFG = modCFG else: webCFG = webCFG.mergeWith(modCFG) gConfig.loadCFG(webCFG) gLogger.showHeaders(True) gLogger._gLogger__initialized = False gLogger.initialize("Web", "/Website") #Define the controllers, templates and public directories for type in ('controllers', 'templates', 'public'): configDict[type] = [] for extModule in extModules: extModulePath = os.path.join(diracRootPath, extModule) typePath = os.path.join(extModulePath, "Web", type) if os.path.isdir(typePath): gLogger.info("Adding %s path for module %s" % (type, extModule)) configDict[type].append(typePath) for systemDir in os.listdir(extModulePath): systemTypePath = os.path.join(extModulePath, systemDir, "Web", type) if os.path.isdir(systemTypePath): gLogger.info("Adding %s path for system %s in module %s" % (type, systemDir, extModule)) configDict[type].append(systemTypePath) #End of extensions configDict[type].append(os.path.join(rootPath, type)) #Load debug.cfg? if enableDebug: debugCFGPath = os.path.join(rootPath, "debug.cfg") if os.path.isfile(debugCFGPath): gLogger.info("Loading debug cfg file at %s" % debugCFGPath) gConfig.loadFile(debugCFGPath) gLogger.info("Extension modules loaded") return configDict
def web_jobs(self): """ Retrieve a list of jobs matching the requirements, use: GET /jobs?<options> -- retrieve a list of jobs matching the requirements. * options: Any job attribute can also be defined as a restriction in a HTTP list form. For instance: Site=DIRAC.Site.com&Site=DIRAC.Site2.com&Status=Waiting * allOwners - show jobs from all owners instead of just the current user. By default is set to false. * maxJobs - maximum number of jobs to retrieve. By default is set to 100. * startJob - starting job for the query. By default is set to 0. GET /jobs/<jid> -- retrieve info about job with id=*jid* GET /jobs/<jid>/manifest -- retrieve the job manifest GET /jobs/<jid>/inputsandbox -- retrieve the job input sandbox GET /jobs/<jid>/outputsandbox -- retrieve the job output sandbox * jid - job identity number POST /jobs -- submit a job. The API expects a manifest to be sent as a JSON object. Files can also be sent as a multipart request. If files are sent, they will be added to the input sandbox and the manifest will be modified accordingly. An example of manifest can be: {Executable: "/bin/echo", Arguments: "Hello World", Sites: ["DIRAC.Site.com", "DIRAC.Site2.com"]} DELETE /jobs/<jid> -- kill a job. The user has to have privileges over a job. * jid - job identity number """ optns = self.overpath.strip('/').split('/') if len(optns) > 2: raise WErr(404, "Wrone way") __jid = re.match("([0-9]+)?", optns[0]).group() __obj = re.match("([a-z]+)?", optns[1]).group() if len(optns) > 1 else None self.loggin.info(__jid, '<<<') self.loggin.info(__obj, '<<<') # GET if self.request.method == 'GET': # manifest if __obj == "manifest": result = yield self.threadTask(self._getJobManifest, __jid) if not result.ok: self.log.error(result.msg) raise result self.finish(result.data) # outputsandbox, inputsandbox elif __obj in ("outputsandbox", "inputsandbox"): result = yield self.threadTask(self._getJobSB, __jid, __obj) if not result.ok: self.log.error(result.msg) raise result data = result.data self.clear() self.set_header("Content-Type", "application/x-tar") cacheTime = 86400 self.set_header("Expires", datetime.datetime.utcnow() + datetime.timedelta(seconds=cacheTime)) self.set_header("Cache-Control", "max-age=%d" % cacheTime) self.set_header("ETag", '"%s"' % hashlib.sha1(data).hexdigest) self.set_header("Content-Disposition", 'attachment; filename="%s-%s.tar.gz"' % (__jid, __obj)) self.finish(data) # summary elif __obj == 'summary': selDict = {} if 'allOwners' not in self.request.arguments: selDict[ 'Owner' ] = self.getUserName() rpc = RPCClient( "WorkloadManagement/JobMonitoring" ) if 'group' not in self.request.arguments: group = [ 'Status' ] else: group = self.request.arguments[ 'group' ] result = yield self.threadTask( rpc.getCounters, group, selDict ) if not result[ 'OK' ]: self.log.error( "Could not retrieve job counters", result[ 'Message' ] ) raise WErr( 500 ) data = {} for cDict, count in result[ 'Value' ]: cKey = "|".join( [ cDict[ k ] for k in group ] ) data[ cKey ] = count self.finish( data ) # history elif __obj == 'history': condDict = {} if 'allOwners' not in self.request.arguments: condDict['Owner'] = self.getUserName() timespan = 86400 if 'timeSpan' in self.request.arguments: try: timespan = int(self.request.arguments['timeSpan'][-1]) except ValueError: raise WErr(400, reason="timeSpan has to be an integer!") rpc = ReportsClient() end = datetime.datetime.utcnow() start = end - datetime.timedelta(seconds=timespan) result = yield self.threadTask(rpc.getReport, "WMSHistory", "NumberOfJobs", start, end, condDict, "Status") if not result['OK']: self.log.error(result['Message']) raise WErr(500) data = result['Value'] self.finish(data) # invalid elif __obj: raise WErr(404, "Invalid job object") # With/without job ID startJob = 0 maxJobs = 100 if __jid: selDict = {'JobID': int(__jid)} else: selDict = {} for convList in (self.ATTRIBUTES, self.FLAGS): for attrPair in convList: jAtt = attrPair[0] if jAtt in self.request.arguments: selDict[attrPair[1]] = self.request.arguments[jAtt] if 'allOwners' not in self.request.arguments: selDict['Owner'] = self.getUserName() if 'startJob' in self.request.arguments: try: startJob = max(startJob, int(self.request.arguments['startJob'][-1])) except ValueError: raise WErr(400, reason="startJob has to be an integer") if 'maxJobs' in self.request.arguments: try: maxJobs = max(maxJobs, int(self.request.arguments['maxJobs'][-1])) except ValueError: raise WErr(400, reason="maxJobs has to be an integer") result = yield self.threadTask(self._getJobs, selDict, startJob, maxJobs) if not result.ok: raise result data = result.data if not __jid: self.finish(data) return if data['entries'] == 0: raise WErr(404, "Unknown jid") self.finish(data['jobs'][0]) # POST elif self.request.method == 'POST': if __jid: self.send_error(404) return if 'manifest' not in self.request.arguments: raise WErr(400, "No manifest") manifests = [] for manifest in self.request.arguments['manifest']: try: manifest = json.loads(manifest) except ValueError: raise WErr(400, "Manifest is not JSON") if type(manifest) != types.DictType: raise WErr(400, "Manifest is not an associative array") manifests.append(manifest) # Upload sandbox files = self.request.files if files: result = yield self.threadTask(self.uploadSandbox, files) if not result.ok: self.log.error("Cannot upload sandbox: %s" % result.msg) raise result sb = result.data self.log.info("Uploaded to %s" % sb) for manifest in manifests: isb = manifest.get('InputSandbox', []) if type(isb) != types.ListType: isb = [isd] isb.append(sb) manifest['InputSandbox'] = isb # Send jobs jids = [] rpc = RPCClient('WorkloadManagement/JobManager') for manifest in manifests: jdl = dumpCFGAsJDL(CFG.CFG().loadFromDict(manifest)) result = yield self.threadTask(rpc.submitJob, str(jdl)) if not result['OK']: self.log.error("Could not submit job: %s" % result['Message']) raise WErr(500, result['Message']) data = result['Value'] if type(data) == types.ListType: jids.extend(data) else: jids.append(data) self.log.info("Got jids %s" % jids) self.finish({'jids': jids}) # DELETE elif self.request.method == 'DELETE': if not __jid: self.send_error(404) return try: __jid = int(__jid) except ValueError: raise WErr(400, "Invalid jid") rpc = RPCClient('WorkloadManagement/JobManager') if 'killonly' in self.request.arguments and self.request.arguments['killonly']: result = yield self.threadTask(rpc.killJob, [__jid]) else: result = yield self.threadTask(rpc.deleteJob, [__jid]) if not result['OK']: if 'NonauthorizedJobIDs' in result: # Not authorized raise WErr(401, "Not authorized") if 'InvalidJobIDs' in result: # Invalid jid raise WErr(400, "Invalid jid") if 'FailedJobIDs' in result: # "Could not delete JID" raise WErr(500, "Could not delete") self.finish({'jid': __jid})