def __call__(self): statusDict = getMutedStatusInfo(self.logger) jobList = statusDict['jobList'] if self.splitting == 'Automatic' and statusDict['dbStatus'] == 'KILLED': msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Tasks using automatic splitting cannot be resubmitted after a kill." self.logger.info(msg) return None if not jobList: msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Status information is unavailable, will not proceed with the resubmission." msg += " Try again a few minutes later if the task has just been submitted." self.logger.info(msg) return None publicationEnabled = statusDict['publicationEnabled'] jobsPerStatus = statusDict['jobsPerStatus'] if self.options.publication: if not publicationEnabled: msg = "Publication was disabled for this task. Therefore, " msg += "there are no publications to resubmit." self.logger.info(msg) return None else: if "finished" not in jobsPerStatus: msg = "No files found to publish" self.logger.info(msg) return None self.jobids = self.processJobIds(jobList) configreq = self.getQueryParams() self.logger.info("Sending resubmit request to the server.") self.logger.debug("Submitting %s " % str(configreq)) configreq_encoded = self._encodeRequest(configreq) self.logger.debug("Encoded resubmit request: %s" % (configreq_encoded)) dictresult, _, _ = self.server.post(self.uri, data = configreq_encoded) self.logger.debug("Result: %s" % (dictresult)) self.logger.info("Resubmit request sent to the server.") if dictresult['result'][0]['result'] != 'ok': msg = "Server responded with: '%s'" % (dictresult['result'][0]['result']) self.logger.info(msg) returndict = {'status': 'FAILED'} else: if not self.options.wait: msg = "Please use 'crab status' to check how the resubmission process proceeds." msg += "\nNotice it may take a couple of minutes for the resubmission to get fully processed." self.logger.info(msg) else: targetTaskStatus = 'SUBMITTED' checkStatusLoop(self.logger, self.server, self.uri, self.cachedinfo['RequestName'], targetTaskStatus, self.name) returndict = {'status': 'SUCCESS'} return returndict
def __call__(self): serverFactory = CRABClient.Emulator.getEmulator('rest') server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version = __version__) if self.jobids: msg = "Requesting resubmission of jobs %s in task %s" % (self.jobids, self.cachedinfo['RequestName']) else: msg = "Requesting resubmission of failed jobs in task %s" % (self.cachedinfo['RequestName']) self.logger.debug(msg) configreq = {'workflow': self.cachedinfo['RequestName'], 'subresource': 'resubmit'} for attr_name in ['jobids', 'sitewhitelist', 'siteblacklist']: attr_value = getattr(self, attr_name) ## For 'jobids', 'sitewhitelist' and 'siteblacklist', attr_value is either a list of strings or None. if attr_value is not None: configreq[attr_name] = attr_value for attr_name in ['maxjobruntime', 'maxmemory', 'numcores', 'priority']: attr_value = getattr(self.options, attr_name) ## For 'maxjobruntime', 'maxmemory', 'numcores', and 'priority', attr_value is either an integer or None. if attr_value is not None: configreq[attr_name] = attr_value configreq['force'] = 1 if self.options.force else 0 configreq['publication'] = 1 if self.options.publication else 0 self.logger.info("Sending resubmit request to the server.") self.logger.debug("Submitting %s " % str(configreq)) configreq_encoded = self._encodeRequest(configreq) self.logger.debug("Encoded resubmit request: %s" % (configreq_encoded)) dictresult, status, reason = server.post(self.uri, data = configreq_encoded) self.logger.debug("Result: %s" % (dictresult)) if status != 200: msg = "Problem resubmitting the task to the server:\ninput:%s\noutput:%s\nreason:%s" \ % (str(configreq_encoded), str(dictresult), str(reason)) raise RESTCommunicationException(msg) self.logger.info("Resubmit request sent to the server.") if dictresult['result'][0]['result'] != 'ok': msg = "Server responded with: '%s'" % (dictresult['result'][0]['result']) self.logger.info(msg) returndict = {'status': 'FAILED'} else: if not self.options.wait: msg = "Please use 'crab status' to check how the resubmission process proceeds." msg += "\nNotice it may take a couple of minutes for the resubmission to get fully processed." self.logger.info(msg) else: targetTaskStatus = 'SUBMITTED' checkStatusLoop(self.logger, server, self.uri, self.cachedinfo['RequestName'], targetTaskStatus, self.name) returndict = {'status': 'SUCCESS'} return returndict
def __call__(self): self.logger.debug("Started submission") serverFactory = CRABClient.Emulator.getEmulator('rest') uniquerequestname = None self.logger.debug("Working on %s" % str(self.requestarea)) self.configreq = {'dryrun': 1 if self.options.dryrun else 0} for param in parametersMapping['on-server']: mustbetype = getattr(types, parametersMapping['on-server'][param]['type']) default = parametersMapping['on-server'][param]['default'] config_params = parametersMapping['on-server'][param]['config'] for config_param in config_params: attrs = config_param.split('.') temp = self.configuration for attr in attrs: temp = getattr(temp, attr, None) if temp is None: break if temp is not None: self.configreq[param] = temp break elif default is not None: self.configreq[param] = default temp = default else: ## Parameter not strictly required. pass ## Check that the requestname is of the right type. ## This is not checked in SubCommand.validateConfig(). if param == 'workflow': if isinstance(self.requestname, mustbetype): self.configreq['workflow'] = self.requestname ## Translate boolean flags into integers. elif param in ['savelogsflag', 'publication', 'publishgroupname', 'nonprodsw', 'useparent',\ 'ignorelocality', 'saveoutput', 'oneEventMode', 'nonvaliddata', 'ignoreglobalblacklist']: self.configreq[param] = 1 if temp else 0 ## Translate DBS URL aliases into DBS URLs. elif param in ['dbsurl', 'publishdbsurl']: if param == 'dbsurl': dbstype = 'reader' elif param == 'publishdbsurl': dbstype = 'writer' allowed_dbsurls = DBSURLS[dbstype].values() allowed_dbsurls_aliases = DBSURLS[dbstype].keys() if self.configreq[param] in allowed_dbsurls_aliases: self.configreq[param] = DBSURLS[dbstype][ self.configreq[param]] elif self.configreq[param].rstrip('/') in allowed_dbsurls: self.configreq[param] = self.configreq[param].rstrip('/') elif param == 'scriptexe' and 'scriptexe' in self.configreq: self.configreq[param] = os.path.basename(self.configreq[param]) jobconfig = {} #get the backend URLs from the server external configuration serverBackendURLs = server_info('backendurls', self.serverurl, self.proxyfilename, getUrl(self.instance, resource='info')) #if cacheSSL is specified in the server external configuration we will use it to upload the sandbox filecacheurl = serverBackendURLs[ 'cacheSSL'] if 'cacheSSL' in serverBackendURLs else None pluginParams = [ self.configuration, self.proxyfilename, self.logger, os.path.join(self.requestarea, 'inputs') ] crab_job_types = getJobTypes() if upper(self.configreq['jobtype']) in crab_job_types: plugjobtype = crab_job_types[upper( self.configreq['jobtype'])](*pluginParams) dummy_inputfiles, jobconfig = plugjobtype.run(filecacheurl) else: fullname = self.configreq['jobtype'] basename = os.path.basename(fullname).split('.')[0] plugin = addPlugin(fullname)[basename] pluginInst = plugin(*pluginParams) dummy_inputfiles, jobconfig = pluginInst.run() if self.configreq['publication']: non_edm_files = jobconfig['tfileoutfiles'] + jobconfig[ 'addoutputfiles'] if non_edm_files: msg = "%sWarning%s: The following output files will not be published, as they are not EDM files: %s" % ( colors.RED, colors.NORMAL, non_edm_files) self.logger.warning(msg) self.configreq.update(jobconfig) server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) self.logger.info("Sending the request to the server at %s" % self.serverurl) self.logger.debug("Submitting %s " % str(self.configreq)) ## TODO: this shouldn't be hard-coded. listParams = ['addoutputfiles', 'sitewhitelist', 'siteblacklist', 'blockwhitelist', 'blockblacklist', \ 'tfileoutfiles', 'edmoutfiles', 'runs', 'lumis', 'userfiles', 'scriptargs', 'extrajdl'] self.configreq_encoded = self._encodeRequest(self.configreq, listParams) self.logger.debug('Encoded submit request: %s' % (self.configreq_encoded)) dictresult, status, reason = server.put(self.uri, data=self.configreq_encoded) self.logger.debug("Result: %s" % dictresult) if status != 200: msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % ( str(self.configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) elif 'result' in dictresult: uniquerequestname = dictresult["result"][0]["RequestName"] else: msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \ % (str(self.configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) tmpsplit = self.serverurl.split(':') createCache(self.requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit) > 1 else '', uniquerequestname, voRole=self.voRole, voGroup=self.voGroup, instance=self.instance, originalConfig=self.configuration) self.logger.info( "%sSuccess%s: Your task has been delivered to the %s CRAB3 server." % (colors.GREEN, colors.NORMAL, self.instance)) if not (self.options.wait or self.options.dryrun): self.logger.info("Task name: %s" % uniquerequestname) projDir = os.path.join( getattr(self.configuration.General, 'workArea', '.'), self.requestname) self.logger.info("Project dir: %s" % projDir) self.logger.info( "Please use 'crab status -d %s' to check how the submission process proceeds.", projDir) else: targetTaskStatus = 'UPLOADED' if self.options.dryrun else 'SUBMITTED' checkStatusLoop(self.logger, server, self.uri, uniquerequestname, targetTaskStatus, self.name) if self.options.dryrun: self.printDryRunResults(*self.executeTestRun(filecacheurl)) self.logger.debug("About to return") return { 'requestname': self.requestname, 'uniquerequestname': uniquerequestname }
def __call__(self): self.logger.debug("Started submission") serverFactory = CRABClient.Emulator.getEmulator('rest') uniquerequestname = None self.logger.debug("Working on %s" % str(self.requestarea)) self.configreq = {'dryrun': 1 if self.options.dryrun else 0} for param in parametersMapping['on-server']: mustbetype = getattr(types, parametersMapping['on-server'][param]['type']) default = parametersMapping['on-server'][param]['default'] config_params = parametersMapping['on-server'][param]['config'] for config_param in config_params: attrs = config_param.split('.') temp = self.configuration for attr in attrs: temp = getattr(temp, attr, None) if temp is None: break if temp is not None: self.configreq[param] = temp break elif default is not None: self.configreq[param] = default temp = default else: ## Parameter not strictly required. pass ## Check that the requestname is of the right type. ## This is not checked in SubCommand.validateConfig(). if param == 'workflow': if isinstance(self.requestname, mustbetype): self.configreq['workflow'] = self.requestname ## Translate boolean flags into integers. elif param in ['savelogsflag', 'publication', 'publishgroupname', 'nonprodsw', 'useparent',\ 'ignorelocality', 'saveoutput', 'oneEventMode', 'nonvaliddata', 'ignoreglobalblacklist']: self.configreq[param] = 1 if temp else 0 ## Translate DBS URL aliases into DBS URLs. elif param in ['dbsurl', 'publishdbsurl']: if param == 'dbsurl': dbstype = 'reader' elif param == 'publishdbsurl': dbstype = 'writer' allowed_dbsurls = DBSURLS[dbstype].values() allowed_dbsurls_aliases = DBSURLS[dbstype].keys() if self.configreq[param] in allowed_dbsurls_aliases: self.configreq[param] = DBSURLS[dbstype][self.configreq[param]] elif self.configreq[param].rstrip('/') in allowed_dbsurls: self.configreq[param] = self.configreq[param].rstrip('/') elif param == 'scriptexe' and 'scriptexe' in self.configreq: self.configreq[param] = os.path.basename(self.configreq[param]) jobconfig = {} #get the backend URLs from the server external configuration serverBackendURLs = server_info('backendurls', self.serverurl, self.proxyfilename, getUrl(self.instance, resource='info')) #if cacheSSL is specified in the server external configuration we will use it to upload the sandbox filecacheurl = serverBackendURLs['cacheSSL'] if 'cacheSSL' in serverBackendURLs else None pluginParams = [self.configuration, self.logger, os.path.join(self.requestarea, 'inputs')] crab_job_types = getJobTypes() if upper(self.configreq['jobtype']) in crab_job_types: plugjobtype = crab_job_types[upper(self.configreq['jobtype'])](*pluginParams) dummy_inputfiles, jobconfig = plugjobtype.run(filecacheurl) else: fullname = self.configreq['jobtype'] basename = os.path.basename(fullname).split('.')[0] plugin = addPlugin(fullname)[basename] pluginInst = plugin(*pluginParams) dummy_inputfiles, jobconfig = pluginInst.run() if self.configreq['publication']: non_edm_files = jobconfig['tfileoutfiles'] + jobconfig['addoutputfiles'] if non_edm_files: msg = "%sWarning%s: The following output files will not be published, as they are not EDM files: %s" % (colors.RED, colors.NORMAL, non_edm_files) self.logger.warning(msg) self.configreq.update(jobconfig) server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) self.logger.info("Sending the request to the server") self.logger.debug("Submitting %s " % str(self.configreq)) ## TODO: this shouldn't be hard-coded. listParams = ['adduserfiles', 'addoutputfiles', 'sitewhitelist', 'siteblacklist', 'blockwhitelist', 'blockblacklist', \ 'tfileoutfiles', 'edmoutfiles', 'runs', 'lumis', 'userfiles', 'scriptargs', 'extrajdl'] self.configreq_encoded = self._encodeRequest(self.configreq, listParams) self.logger.debug('Encoded submit request: %s' % (self.configreq_encoded)) dictresult, status, reason = server.put(self.uri, data = self.configreq_encoded) self.logger.debug("Result: %s" % dictresult) if status != 200: msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (str(self.configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) elif 'result' in dictresult: uniquerequestname = dictresult["result"][0]["RequestName"] else: msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \ % (str(self.configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) tmpsplit = self.serverurl.split(':') createCache(self.requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit) > 1 else '', uniquerequestname, voRole=self.voRole, voGroup=self.voGroup, instance=self.instance, originalConfig=self.configuration) self.logger.info("%sSuccess%s: Your task has been delivered to the CRAB3 server." %(colors.GREEN, colors.NORMAL)) if not (self.options.wait or self.options.dryrun): self.logger.info("Task name: %s" % uniquerequestname) self.logger.info("Please use 'crab status' to check how the submission process proceeds.") else: targetTaskStatus = 'UPLOADED' if self.options.dryrun else 'SUBMITTED' checkStatusLoop(self.logger, server, self.uri, uniquerequestname, targetTaskStatus, self.name) if self.options.dryrun: self.printDryRunResults(*self.executeTestRun(filecacheurl)) self.logger.debug("About to return") return {'requestname': self.requestname , 'uniquerequestname': uniquerequestname}
def __call__(self): serverFactory = CRABClient.Emulator.getEmulator('rest') server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) if self.jobids: msg = "Requesting resubmission of jobs %s in task %s" % ( self.jobids, self.cachedinfo['RequestName']) else: msg = "Requesting resubmission of failed jobs in task %s" % ( self.cachedinfo['RequestName']) self.logger.debug(msg) configreq = { 'workflow': self.cachedinfo['RequestName'], 'subresource': 'resubmit' } for attr_name in ['jobids', 'sitewhitelist', 'siteblacklist']: attr_value = getattr(self, attr_name) ## For 'jobids', 'sitewhitelist' and 'siteblacklist', attr_value is either a list of strings or None. if attr_value is not None: configreq[attr_name] = attr_value for attr_name in [ 'maxjobruntime', 'maxmemory', 'numcores', 'priority' ]: attr_value = getattr(self.options, attr_name) ## For 'maxjobruntime', 'maxmemory', 'numcores', and 'priority', attr_value is either an integer or None. if attr_value is not None: configreq[attr_name] = attr_value configreq['force'] = 1 if self.options.force else 0 configreq['publication'] = 1 if self.options.publication else 0 self.logger.info("Sending resubmit request to the server.") self.logger.debug("Submitting %s " % str(configreq)) configreq_encoded = self._encodeRequest(configreq) self.logger.debug("Encoded resubmit request: %s" % (configreq_encoded)) dictresult, status, reason = server.post(self.uri, data=configreq_encoded) self.logger.debug("Result: %s" % (dictresult)) if status != 200: msg = "Problem resubmitting the task to the server:\ninput:%s\noutput:%s\nreason:%s" \ % (str(configreq_encoded), str(dictresult), str(reason)) raise RESTCommunicationException(msg) self.logger.info("Resubmit request sent to the server.") if dictresult['result'][0]['result'] != 'ok': msg = "Server responded with: '%s'" % ( dictresult['result'][0]['result']) self.logger.info(msg) returndict = {'status': 'FAILED'} else: if not self.options.wait: msg = "Please use 'crab status' to check how the resubmission process proceeds." msg += "\nNotice it may take a couple of minutes for the resubmission to get fully processed." self.logger.info(msg) else: targetTaskStatus = 'SUBMITTED' checkStatusLoop(self.logger, server, self.uri, self.cachedinfo['RequestName'], targetTaskStatus, self.name) returndict = {'status': 'SUCCESS'} return returndict