def postJobs(): result = gOAManager.authorize() if not result[ 'OK' ]: bottle.abort( 401, result[ 'Message' ] ) request = bottle.request if len( request.files ): result = uploadSandbox( request.files ) if not result[ 'OK' ]: bottle.abort( 500, result[ 'Message' ] ) isb = result[ 'Value' ] else: isb = False jobs = [] wms = getWMSClient() for k in request.forms: origData = bottle.json_lds( request.forms[ k ] ) jobData = origData if isb: if 'InputSandbox' not in jobData: jobData[ 'InputSandbox' ] = [] jobData[ 'InputSandbox' ].append( isb ) cfg = CFG.CFG().loadFromDict( jobData ) jdl = dumpCFGAsJDL( cfg ) result = wms.submitJob( jdl ) if not result[ 'OK' ]: bottle.abort( 500, result[ 'Message' ] ) jobs.append( result[ 'Value' ] ) return { 'sandbox' : isb, 'jobs' : jobs }
def post( self, jid ): if jid: self.send_error( 404 ) return args = self.request.arguments if 'manifest' not in args: raise WErr( 400, "No manifest" ) manifests = [] for manifest in args[ 'manifest' ]: try: manifest = json.loads( manifest ) except ValueError: raise WErr( 400, "Manifest is not JSON" ) if type( manifest ) != types.DictType: raise WErr( 400, "Manifest is not an associative array" ) manifests.append( manifest ) #Upload sandbox files = self.request.files if files: result = yield self.threadTask( self.uploadSandbox, files ) if not result.ok: self.log.error( "Cannot upload sandbox: %s" % result.msg ) raise result sb = result.data self.log.info( "Uploaded to %s" % sb ) for manifest in manifests: isb = manifest.get( 'InputSandbox', [] ) if type( isb ) != types.ListType: isb = [ isd ] isb.append( sb ) manifest[ 'InputSandbox' ] = isb #Send jobs jids = [] rpc = RPCClient( 'WorkloadManagement/JobManager' ) for manifest in manifests: jdl = dumpCFGAsJDL( CFG.CFG().loadFromDict( manifest ) ) result = yield self.threadTask( rpc.submitJob, str( jdl ) ) if not result[ 'OK' ]: self.log.error( "Could not submit job: %s" % result[ 'Message' ] ) raise WErr( 500, result[ 'Message' ] ) data = result[ 'Value' ] if type( data ) == types.ListType: jids.extend( data ) else: jids.append( data ) self.log.info( "Got jids %s" % jids ) self.finish( { 'jids' : jids } )
def dumpAsJDL(self): return dumpCFGAsJDL(self.__manifest)
def JSON2JDL( jobData ): cfg = CFG.CFG().loadFromDicT( jobData ) jdl = dumpCFGAsJDL( cfg ) return jdl
def dumpDescriptionAsJDL( self ): return dumpCFGAsJDL( self.__description )
def dumpDescriptionAsJDL(self): return dumpCFGAsJDL(self.__description)
def web_jobs(self): """ Retrieve a list of jobs matching the requirements, use: GET /jobs?<options> -- retrieve a list of jobs matching the requirements. * options: Any job attribute can also be defined as a restriction in a HTTP list form. For instance: Site=DIRAC.Site.com&Site=DIRAC.Site2.com&Status=Waiting * allOwners - show jobs from all owners instead of just the current user. By default is set to false. * maxJobs - maximum number of jobs to retrieve. By default is set to 100. * startJob - starting job for the query. By default is set to 0. GET /jobs/<jid> -- retrieve info about job with id=*jid* GET /jobs/<jid>/manifest -- retrieve the job manifest GET /jobs/<jid>/inputsandbox -- retrieve the job input sandbox GET /jobs/<jid>/outputsandbox -- retrieve the job output sandbox * jid - job identity number POST /jobs -- submit a job. The API expects a manifest to be sent as a JSON object. Files can also be sent as a multipart request. If files are sent, they will be added to the input sandbox and the manifest will be modified accordingly. An example of manifest can be: {Executable: "/bin/echo", Arguments: "Hello World", Sites: ["DIRAC.Site.com", "DIRAC.Site2.com"]} DELETE /jobs/<jid> -- kill a job. The user has to have privileges over a job. * jid - job identity number """ optns = self.overpath.strip('/').split('/') if len(optns) > 2: raise WErr(404, "Wrone way") __jid = re.match("([0-9]+)?", optns[0]).group() __obj = re.match("([a-z]+)?", optns[1]).group() if len(optns) > 1 else None self.loggin.info(__jid, '<<<') self.loggin.info(__obj, '<<<') # GET if self.request.method == 'GET': # manifest if __obj == "manifest": result = yield self.threadTask(self._getJobManifest, __jid) if not result.ok: self.log.error(result.msg) raise result self.finish(result.data) # outputsandbox, inputsandbox elif __obj in ("outputsandbox", "inputsandbox"): result = yield self.threadTask(self._getJobSB, __jid, __obj) if not result.ok: self.log.error(result.msg) raise result data = result.data self.clear() self.set_header("Content-Type", "application/x-tar") cacheTime = 86400 self.set_header("Expires", datetime.datetime.utcnow() + datetime.timedelta(seconds=cacheTime)) self.set_header("Cache-Control", "max-age=%d" % cacheTime) self.set_header("ETag", '"%s"' % hashlib.sha1(data).hexdigest) self.set_header("Content-Disposition", 'attachment; filename="%s-%s.tar.gz"' % (__jid, __obj)) self.finish(data) # summary elif __obj == 'summary': selDict = {} if 'allOwners' not in self.request.arguments: selDict[ 'Owner' ] = self.getUserName() rpc = RPCClient( "WorkloadManagement/JobMonitoring" ) if 'group' not in self.request.arguments: group = [ 'Status' ] else: group = self.request.arguments[ 'group' ] result = yield self.threadTask( rpc.getCounters, group, selDict ) if not result[ 'OK' ]: self.log.error( "Could not retrieve job counters", result[ 'Message' ] ) raise WErr( 500 ) data = {} for cDict, count in result[ 'Value' ]: cKey = "|".join( [ cDict[ k ] for k in group ] ) data[ cKey ] = count self.finish( data ) # history elif __obj == 'history': condDict = {} if 'allOwners' not in self.request.arguments: condDict['Owner'] = self.getUserName() timespan = 86400 if 'timeSpan' in self.request.arguments: try: timespan = int(self.request.arguments['timeSpan'][-1]) except ValueError: raise WErr(400, reason="timeSpan has to be an integer!") rpc = ReportsClient() end = datetime.datetime.utcnow() start = end - datetime.timedelta(seconds=timespan) result = yield self.threadTask(rpc.getReport, "WMSHistory", "NumberOfJobs", start, end, condDict, "Status") if not result['OK']: self.log.error(result['Message']) raise WErr(500) data = result['Value'] self.finish(data) # invalid elif __obj: raise WErr(404, "Invalid job object") # With/without job ID startJob = 0 maxJobs = 100 if __jid: selDict = {'JobID': int(__jid)} else: selDict = {} for convList in (self.ATTRIBUTES, self.FLAGS): for attrPair in convList: jAtt = attrPair[0] if jAtt in self.request.arguments: selDict[attrPair[1]] = self.request.arguments[jAtt] if 'allOwners' not in self.request.arguments: selDict['Owner'] = self.getUserName() if 'startJob' in self.request.arguments: try: startJob = max(startJob, int(self.request.arguments['startJob'][-1])) except ValueError: raise WErr(400, reason="startJob has to be an integer") if 'maxJobs' in self.request.arguments: try: maxJobs = max(maxJobs, int(self.request.arguments['maxJobs'][-1])) except ValueError: raise WErr(400, reason="maxJobs has to be an integer") result = yield self.threadTask(self._getJobs, selDict, startJob, maxJobs) if not result.ok: raise result data = result.data if not __jid: self.finish(data) return if data['entries'] == 0: raise WErr(404, "Unknown jid") self.finish(data['jobs'][0]) # POST elif self.request.method == 'POST': if __jid: self.send_error(404) return if 'manifest' not in self.request.arguments: raise WErr(400, "No manifest") manifests = [] for manifest in self.request.arguments['manifest']: try: manifest = json.loads(manifest) except ValueError: raise WErr(400, "Manifest is not JSON") if type(manifest) != types.DictType: raise WErr(400, "Manifest is not an associative array") manifests.append(manifest) # Upload sandbox files = self.request.files if files: result = yield self.threadTask(self.uploadSandbox, files) if not result.ok: self.log.error("Cannot upload sandbox: %s" % result.msg) raise result sb = result.data self.log.info("Uploaded to %s" % sb) for manifest in manifests: isb = manifest.get('InputSandbox', []) if type(isb) != types.ListType: isb = [isd] isb.append(sb) manifest['InputSandbox'] = isb # Send jobs jids = [] rpc = RPCClient('WorkloadManagement/JobManager') for manifest in manifests: jdl = dumpCFGAsJDL(CFG.CFG().loadFromDict(manifest)) result = yield self.threadTask(rpc.submitJob, str(jdl)) if not result['OK']: self.log.error("Could not submit job: %s" % result['Message']) raise WErr(500, result['Message']) data = result['Value'] if type(data) == types.ListType: jids.extend(data) else: jids.append(data) self.log.info("Got jids %s" % jids) self.finish({'jids': jids}) # DELETE elif self.request.method == 'DELETE': if not __jid: self.send_error(404) return try: __jid = int(__jid) except ValueError: raise WErr(400, "Invalid jid") rpc = RPCClient('WorkloadManagement/JobManager') if 'killonly' in self.request.arguments and self.request.arguments['killonly']: result = yield self.threadTask(rpc.killJob, [__jid]) else: result = yield self.threadTask(rpc.deleteJob, [__jid]) if not result['OK']: if 'NonauthorizedJobIDs' in result: # Not authorized raise WErr(401, "Not authorized") if 'InvalidJobIDs' in result: # Invalid jid raise WErr(400, "Invalid jid") if 'FailedJobIDs' in result: # "Could not delete JID" raise WErr(500, "Could not delete") self.finish({'jid': __jid})