def __getCache(self):
Operations.__cacheLock.acquire()
try:
currentVersion = gConfigurationData.getVersion()
if currentVersion != Operations.__cacheVersion:
Operations.__cache = {}
Operations.__cacheVersion = currentVersion
cacheKey = (self.__vo, self.__setup)
if cacheKey in Operations.__cache:
return Operations.__cache[cacheKey]
mergedCFG = CFG.CFG()
for path in self.__getSearchPaths():
pathCFG = gConfigurationData.mergedCFG[path]
if pathCFG:
mergedCFG = mergedCFG.mergeWith(pathCFG)
Operations.__cache[cacheKey] = mergedCFG
return Operations.__cache[cacheKey]
finally:
try:
Operations.__cacheLock.release()
except thread.error:
pass
def __loadDIRACCFG( self ):
installPath = gConfig.getValue( '/LocalInstallation/TargetPath',
gConfig.getValue( '/LocalInstallation/RootPath', '' ) )
if not installPath:
installPath = rootPath
cfgPath = os.path.join( installPath, 'etc', 'dirac.cfg' )
try:
diracCFG = CFG.CFG().loadFromFile( cfgPath )
except Exception, excp:
return S_ERROR( "Could not load dirac.cfg: %s" % str( excp ) )
def __getCache(self):
Operations.__cacheLock.acquire()
try:
currentVersion = gConfigurationData.getVersion()
if currentVersion != Operations.__cacheVersion:
Operations.__cache = {}
Operations.__cacheVersion = currentVersion
cacheKey = (self.__threadData.vo, self.__threadData.setup)
if cacheKey in Operations.__cache:
return Operations.__cache[cacheKey]
mergedCFG = CFG.CFG()
for path in (self.__getDefaultPath(), self.__getSetupPath()):
pathCFG = gConfigurationData.mergedCFG[path]
if pathCFG:
mergedCFG = mergedCFG.mergeWith(pathCFG)
Operations.__cache[cacheKey] = mergedCFG
return Operations.__cache[cacheKey]
finally:
Operations.__cacheLock.release()
def loadCFGFromRepository( self, svnPath ): remoteData = self.getSVNFileContents( svnPath ) return CFG.CFG().loadFromBuffer( remoteData )
def initDIRAC(rootPath, enableDebug=False):
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
configDict = {'webConfig': {}}
configDict['webConfig']['dirac.webroot'] = rootPath
diracRootPath = os.path.realpath(os.path.dirname(
os.path.dirname(rootPath)))
configDict['webConfig']['dirac.root'] = diracRootPath
if diracRootPath not in sys.path:
sys.path.append(diracRootPath)
from DIRAC.FrameworkSystem.Client.Logger import gLogger
gLogger.registerBackends(['stderr'])
from DIRAC.Core.Base import Script
Script.registerSwitch("r", "reload", "Reload for pylons")
Script.localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate",
"yes")
Script.localCfg.addDefaultEntry("LogColor", True)
Script.initialize(script="Website",
ignoreErrors=True,
initializeMonitor=False)
gLogger._systemName = "Framework"
gLogger.initialize("Web", "/Website")
gLogger.setLevel("VERBOSE")
from DIRAC import gMonitor, gConfig, rootPath as droot
from DIRAC.Core.Utilities import CFG
from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions
gMonitor.setComponentType(gMonitor.COMPONENT_WEB)
gMonitor.initialize()
gMonitor.registerActivity("pagesServed", "Pages served", "Framework",
"pages", gMonitor.OP_SUM)
gLogger.info("DIRAC Initialized")
configDict['portalVersion'] = portalVersion(rootPath)
gLogger.info("DIRAC portal version: %s" % configDict['portalVersion'])
extModules = ['%sDIRAC' % module for module in getCSExtensions()]
#Load web.cfg of modules
cfgFilePaths = [os.path.join(droot, "etc", "web.cfg")]
for extModule in extModules:
gLogger.info("Adding web.cfg for %s extension" % extModule)
extModulePath = os.path.join(diracRootPath, extModule)
webCFGPath = os.path.join(extModulePath, "Web", "web.cfg")
cfgFilePaths.append(webCFGPath)
for systemDir in os.listdir(extModulePath):
webCFGSystemPath = os.path.join(extModulePath, systemDir, "Web",
"web.cfg")
cfgFilePaths.append(webCFGSystemPath)
webCFG = CFG.CFG()
for webCFGPath in cfgFilePaths:
if not os.path.isfile(webCFGPath):
gLogger.warn("%s does not exist" % webCFGPath)
else:
gLogger.info("Loading %s" % webCFGPath)
modCFG = CFG.CFG().loadFromFile(webCFGPath)
if modCFG.getOption('Website/AbsoluteDefinition', False):
gLogger.info("CFG %s is absolute" % webCFGPath)
webCFG = modCFG
else:
webCFG = webCFG.mergeWith(modCFG)
gConfig.loadCFG(webCFG)
gLogger.showHeaders(True)
gLogger._gLogger__initialized = False
gLogger.initialize("Web", "/Website")
#Define the controllers, templates and public directories
for type in ('controllers', 'templates', 'public'):
configDict[type] = []
for extModule in extModules:
extModulePath = os.path.join(diracRootPath, extModule)
typePath = os.path.join(extModulePath, "Web", type)
if os.path.isdir(typePath):
gLogger.info("Adding %s path for module %s" %
(type, extModule))
configDict[type].append(typePath)
for systemDir in os.listdir(extModulePath):
systemTypePath = os.path.join(extModulePath, systemDir, "Web",
type)
if os.path.isdir(systemTypePath):
gLogger.info("Adding %s path for system %s in module %s" %
(type, systemDir, extModule))
configDict[type].append(systemTypePath)
#End of extensions
configDict[type].append(os.path.join(rootPath, type))
#Load debug.cfg?
if enableDebug:
debugCFGPath = os.path.join(rootPath, "debug.cfg")
if os.path.isfile(debugCFGPath):
gLogger.info("Loading debug cfg file at %s" % debugCFGPath)
gConfig.loadFile(debugCFGPath)
gLogger.info("Extension modules loaded")
return configDict
def web_jobs(self):
""" Retrieve a list of jobs matching the requirements, use:
GET /jobs?<options> -- retrieve a list of jobs matching the requirements.
* options:
Any job attribute can also be defined as a restriction in a HTTP list form. For instance:
Site=DIRAC.Site.com&Site=DIRAC.Site2.com&Status=Waiting
* allOwners - show jobs from all owners instead of just the current user. By default is set to false.
* maxJobs - maximum number of jobs to retrieve. By default is set to 100.
* startJob - starting job for the query. By default is set to 0.
GET /jobs/<jid> -- retrieve info about job with id=*jid*
GET /jobs/<jid>/manifest -- retrieve the job manifest
GET /jobs/<jid>/inputsandbox -- retrieve the job input sandbox
GET /jobs/<jid>/outputsandbox -- retrieve the job output sandbox
* jid - job identity number
POST /jobs -- submit a job. The API expects a manifest to be sent as a JSON object.
Files can also be sent as a multipart request. If files are sent,
they will be added to the input sandbox and the manifest will be modified accordingly.
An example of manifest can be:
{Executable: "/bin/echo",
Arguments: "Hello World",
Sites: ["DIRAC.Site.com", "DIRAC.Site2.com"]}
DELETE /jobs/<jid> -- kill a job. The user has to have privileges over a job.
* jid - job identity number
"""
optns = self.overpath.strip('/').split('/')
if len(optns) > 2:
raise WErr(404, "Wrone way")
__jid = re.match("([0-9]+)?", optns[0]).group()
__obj = re.match("([a-z]+)?", optns[1]).group() if len(optns) > 1 else None
self.loggin.info(__jid, '<<<')
self.loggin.info(__obj, '<<<')
# GET
if self.request.method == 'GET':
# manifest
if __obj == "manifest":
result = yield self.threadTask(self._getJobManifest, __jid)
if not result.ok:
self.log.error(result.msg)
raise result
self.finish(result.data)
# outputsandbox, inputsandbox
elif __obj in ("outputsandbox", "inputsandbox"):
result = yield self.threadTask(self._getJobSB, __jid, __obj)
if not result.ok:
self.log.error(result.msg)
raise result
data = result.data
self.clear()
self.set_header("Content-Type", "application/x-tar")
cacheTime = 86400
self.set_header("Expires", datetime.datetime.utcnow() + datetime.timedelta(seconds=cacheTime))
self.set_header("Cache-Control", "max-age=%d" % cacheTime)
self.set_header("ETag", '"%s"' % hashlib.sha1(data).hexdigest)
self.set_header("Content-Disposition", 'attachment; filename="%s-%s.tar.gz"' % (__jid, __obj))
self.finish(data)
# summary
elif __obj == 'summary':
selDict = {}
if 'allOwners' not in self.request.arguments:
selDict[ 'Owner' ] = self.getUserName()
rpc = RPCClient( "WorkloadManagement/JobMonitoring" )
if 'group' not in self.request.arguments:
group = [ 'Status' ]
else:
group = self.request.arguments[ 'group' ]
result = yield self.threadTask( rpc.getCounters, group, selDict )
if not result[ 'OK' ]:
self.log.error( "Could not retrieve job counters", result[ 'Message' ] )
raise WErr( 500 )
data = {}
for cDict, count in result[ 'Value' ]:
cKey = "|".join( [ cDict[ k ] for k in group ] )
data[ cKey ] = count
self.finish( data )
# history
elif __obj == 'history':
condDict = {}
if 'allOwners' not in self.request.arguments:
condDict['Owner'] = self.getUserName()
timespan = 86400
if 'timeSpan' in self.request.arguments:
try:
timespan = int(self.request.arguments['timeSpan'][-1])
except ValueError:
raise WErr(400, reason="timeSpan has to be an integer!")
rpc = ReportsClient()
end = datetime.datetime.utcnow()
start = end - datetime.timedelta(seconds=timespan)
result = yield self.threadTask(rpc.getReport, "WMSHistory", "NumberOfJobs", start, end, condDict, "Status")
if not result['OK']:
self.log.error(result['Message'])
raise WErr(500)
data = result['Value']
self.finish(data)
# invalid
elif __obj:
raise WErr(404, "Invalid job object")
# With/without job ID
startJob = 0
maxJobs = 100
if __jid:
selDict = {'JobID': int(__jid)}
else:
selDict = {}
for convList in (self.ATTRIBUTES, self.FLAGS):
for attrPair in convList:
jAtt = attrPair[0]
if jAtt in self.request.arguments:
selDict[attrPair[1]] = self.request.arguments[jAtt]
if 'allOwners' not in self.request.arguments:
selDict['Owner'] = self.getUserName()
if 'startJob' in self.request.arguments:
try:
startJob = max(startJob, int(self.request.arguments['startJob'][-1]))
except ValueError:
raise WErr(400, reason="startJob has to be an integer")
if 'maxJobs' in self.request.arguments:
try:
maxJobs = max(maxJobs, int(self.request.arguments['maxJobs'][-1]))
except ValueError:
raise WErr(400, reason="maxJobs has to be an integer")
result = yield self.threadTask(self._getJobs, selDict, startJob, maxJobs)
if not result.ok:
raise result
data = result.data
if not __jid:
self.finish(data)
return
if data['entries'] == 0:
raise WErr(404, "Unknown jid")
self.finish(data['jobs'][0])
# POST
elif self.request.method == 'POST':
if __jid:
self.send_error(404)
return
if 'manifest' not in self.request.arguments:
raise WErr(400, "No manifest")
manifests = []
for manifest in self.request.arguments['manifest']:
try:
manifest = json.loads(manifest)
except ValueError:
raise WErr(400, "Manifest is not JSON")
if type(manifest) != types.DictType:
raise WErr(400, "Manifest is not an associative array")
manifests.append(manifest)
# Upload sandbox
files = self.request.files
if files:
result = yield self.threadTask(self.uploadSandbox, files)
if not result.ok:
self.log.error("Cannot upload sandbox: %s" % result.msg)
raise result
sb = result.data
self.log.info("Uploaded to %s" % sb)
for manifest in manifests:
isb = manifest.get('InputSandbox', [])
if type(isb) != types.ListType:
isb = [isd]
isb.append(sb)
manifest['InputSandbox'] = isb
# Send jobs
jids = []
rpc = RPCClient('WorkloadManagement/JobManager')
for manifest in manifests:
jdl = dumpCFGAsJDL(CFG.CFG().loadFromDict(manifest))
result = yield self.threadTask(rpc.submitJob, str(jdl))
if not result['OK']:
self.log.error("Could not submit job: %s" % result['Message'])
raise WErr(500, result['Message'])
data = result['Value']
if type(data) == types.ListType:
jids.extend(data)
else:
jids.append(data)
self.log.info("Got jids %s" % jids)
self.finish({'jids': jids})
# DELETE
elif self.request.method == 'DELETE':
if not __jid:
self.send_error(404)
return
try:
__jid = int(__jid)
except ValueError:
raise WErr(400, "Invalid jid")
rpc = RPCClient('WorkloadManagement/JobManager')
if 'killonly' in self.request.arguments and self.request.arguments['killonly']:
result = yield self.threadTask(rpc.killJob, [__jid])
else:
result = yield self.threadTask(rpc.deleteJob, [__jid])
if not result['OK']:
if 'NonauthorizedJobIDs' in result:
# Not authorized
raise WErr(401, "Not authorized")
if 'InvalidJobIDs' in result:
# Invalid jid
raise WErr(400, "Invalid jid")
if 'FailedJobIDs' in result:
# "Could not delete JID"
raise WErr(500, "Could not delete")
self.finish({'jid': __jid})