def __call__(self): valid = False configmsg = 'Default' self.logger.debug("Started submission") serverFactory = CRABClient.Emulator.getEmulator('rest') uniquerequestname = None self.logger.debug("Working on %s" % str(self.requestarea)) configreq = {'dryrun': 1 if self.options.dryrun else 0} for param in parametersMapping['on-server']: mustbetype = getattr(types, parametersMapping['on-server'][param]['type']) default = parametersMapping['on-server'][param]['default'] config_params = parametersMapping['on-server'][param]['config'] for config_param in config_params: attrs = config_param.split('.') temp = self.configuration for attr in attrs: temp = getattr(temp, attr, None) if temp is None: break if temp is not None: configreq[param] = temp break elif default is not None: configreq[param] = default temp = default else: ## Parameter not strictly required. pass ## Check that the requestname is of the right type. ## This is not checked in SubCommand.validateConfig(). if param == 'workflow': if mustbetype == type(self.requestname): configreq['workflow'] = self.requestname ## Translate boolean flags into integers. elif param in ['savelogsflag', 'publication', 'nonprodsw', 'useparent', 'ignorelocality', 'saveoutput', 'oneEventMode']: configreq[param] = 1 if temp else 0 ## Translate DBS URL aliases into DBS URLs. elif param in ['dbsurl', 'publishdbsurl']: if param == 'dbsurl': dbstype = 'reader' elif param == 'publishdbsurl': dbstype = 'writer' allowed_dbsurls = DBSURLS[dbstype].values() allowed_dbsurls_aliases = DBSURLS[dbstype].keys() if configreq[param] in allowed_dbsurls_aliases: configreq[param] = DBSURLS[dbstype][configreq[param]] elif configreq[param].rstrip('/') in allowed_dbsurls: configreq[param] = configreq[param].rstrip('/') elif param == 'scriptexe' and 'scriptexe' in configreq: configreq[param] = os.path.basename(configreq[param]) jobconfig = {} #get the backend URLs from the server external configuration serverBackendURLs = server_info('backendurls', self.serverurl, self.proxyfilename, getUrl(self.instance, resource='info')) #if cacheSSL is specified in the server external configuration we will use it to upload the sandbox (baseURL will be ignored) filecacheurl = serverBackendURLs['cacheSSL'] if 'cacheSSL' in serverBackendURLs else None pluginParams = [ self.configuration, self.logger, os.path.join(self.requestarea, 'inputs') ] crab_job_types = getJobTypes() if upper(configreq['jobtype']) in crab_job_types: plugjobtype = crab_job_types[upper(configreq['jobtype'])](*pluginParams) inputfiles, jobconfig, isbchecksum = plugjobtype.run(filecacheurl) else: fullname = configreq['jobtype'] basename = os.path.basename(fullname).split('.')[0] plugin = addPlugin(fullname)[basename] pluginInst = plugin(*pluginParams) inputfiles, jobconfig, isbchecksum = pluginInst.run() if configreq['publication']: non_edm_files = jobconfig['tfileoutfiles'] + jobconfig['addoutputfiles'] if non_edm_files: msg = "%sWarning%s: The following output files will not be published, as they are not EDM files: %s" % (colors.RED, colors.NORMAL, non_edm_files) self.logger.warning(msg) if not configreq['publishname']: configreq['publishname'] = isbchecksum else: configreq['publishname'] = "%s-%s" %(configreq['publishname'], isbchecksum) configreq.update(jobconfig) server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) self.logger.info("Sending the request to the server") self.logger.debug("Submitting %s " % str(configreq)) ## TODO: this shouldn't be hard-coded. listParams = ['adduserfiles', 'addoutputfiles', 'sitewhitelist', 'siteblacklist', 'blockwhitelist', 'blockblacklist', \ 'tfileoutfiles', 'edmoutfiles', 'runs', 'lumis', 'userfiles', 'scriptargs', 'extrajdl'] configreq_encoded = self._encodeRequest(configreq, listParams) self.logger.debug('Encoded submit request: %s' % (configreq_encoded)) dictresult, status, reason = server.put( self.uri, data = configreq_encoded) self.logger.debug("Result: %s" % dictresult) if status != 200: msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (str(configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) elif dictresult.has_key("result"): uniquerequestname = dictresult["result"][0]["RequestName"] else: msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \ % (str(configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) tmpsplit = self.serverurl.split(':') createCache(self.requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit)>1 else '', uniquerequestname, voRole=self.voRole, voGroup=self.voGroup, instance=self.instance, originalConfig = self.configuration) self.logger.info("%sSuccess%s: Your task has been delivered to the CRAB3 server." %(colors.GREEN, colors.NORMAL)) if not (self.options.wait or self.options.dryrun): self.logger.info("Task name: %s" % uniquerequestname) self.logger.info("Please use 'crab status' to check how the submission process proceeds.") else: targetTaskStatus = 'UPLOADED' if self.options.dryrun else 'SUBMITTED' self.checkStatusLoop(server, uniquerequestname, targetTaskStatus) if self.options.dryrun: self.printDryRunResults(*self.executeTestRun(filecacheurl)) self.logger.debug("About to return") return {'requestname' : self.requestname , 'uniquerequestname' : uniquerequestname }
def validateConfig(self): """ __validateConfig__ Checking if needed input parameters are there """ valid, msg = SubCommand.validateConfig(self) if not valid: return False, msg ## Check that Data.unitsPerjob is specified. if hasattr(self.configuration.Data, 'unitsPerJob'): try: float(self.configuration.Data.unitsPerJob) except ValueError: msg = "Invalid CRAB configuration: Parameter Data.unitsPerJob must be a valid number, not %s." % (self.configuration.Data.unitsPerJob) return False, msg ## Check that JobType.pluginName and JobType.externalPluginFile are not both specified. if hasattr(self.configuration.JobType, 'pluginName') and hasattr(self.configuration.JobType, 'externalPluginFile'): msg = "Invalid CRAB configuration: Only one of JobType.pluginName or JobType.externalPluginFile parameters can be specified." pluginName_default = getParamDefaultValue('JobType.pluginName') if pluginName_default: msg += "\nIf neither JobType.pluginName nor JobType.externalPluginFile would be specified," msg += " the default JobType.pluginName = '%s' would be used." % (pluginName_default) return False, msg ## Load the external plugin or check that the crab plugin is valid. external_plugin_name = getattr(self.configuration.JobType, 'externalPluginFile', None) crab_plugin_name = getattr(self.configuration.JobType, 'pluginName', None) crab_job_types = {'ANALYSIS': None, 'PRIVATEMC': None} #getJobTypes() if external_plugin_name: addPlugin(external_plugin_name) # Do we need to do this here? if crab_plugin_name: if upper(crab_plugin_name) not in crab_job_types: msg = "Invalid CRAB configuration: Parameter JobType.pluginName has an invalid value ('%s')." % (crab_plugin_name) msg += "\nAllowed values are: %s." % (", ".join(['%s' % job_type for job_type in crab_job_types.keys()])) return False, msg msg = "Will use CRAB %s plugin" % ("Analysis" if upper(crab_plugin_name) == 'ANALYSIS' else "PrivateMC") msg += " (i.e. will run %s job type)." % ("an analysis" if upper(crab_plugin_name) == 'ANALYSIS' else "a MC generation") self.logger.debug(msg) ## Check that the particular combination (Data.publication = True, General.transferOutputs = False) is not specified. if getattr(self.configuration.Data, 'publication', getParamDefaultValue('Data.publication')) and \ not getattr(self.configuration.General, 'transferOutputs', getParamDefaultValue('General.transferOutputs')): msg = "Invalid CRAB configuration: Data.publication is True, but General.transferOutputs is False." msg += "\nPublication can not be performed if the output files are not transferred to a permanent storage." return False, msg ## Check that a storage site is specified if General.transferOutputs = True or General.transferLogs = True. if not hasattr(self.configuration.Site, 'storageSite'): if getattr(self.configuration.General, 'transferLogs', getParamDefaultValue('General.transferLogs')) or \ getattr(self.configuration.General, 'transferOutputs', getParamDefaultValue('General.transferOutputs')): msg = "Invalid CRAB configuration: Parameter Site.storageSite is missing." return False, msg ## If an input dataset and a DBS URL are specified, check that the DBS URL is a good one. ## Also, if the DBS URL is 'phys0x', check that the input dataset tier is USER. if hasattr(self.configuration.Data, 'inputDBS'): if hasattr(self.configuration.Data, 'inputDataset'): msg = None dbs_urls_aliases = DBSURLS['reader'].keys() dbs_urls = DBSURLS['reader'].values() if (self.configuration.Data.inputDBS not in dbs_urls_aliases) and (self.configuration.Data.inputDBS.rstrip('/') not in dbs_urls): msg = "Invalid CRAB configuration: Parameter Data.inputDBS has an invalid value ('%s')." % (self.configuration.Data.inputDBS) msg += "\nAllowed values are: " msg += "\n ".join(["'%s' ('%s')" % (alias, url) for alias, url in DBSURLS['reader'].iteritems()]) local_dbs_urls_aliases = ['phys01', 'phys02', 'phys03'] local_dbs_urls = [DBSURLS['reader'][alias] for alias in local_dbs_urls_aliases if alias in DBSURLS['reader']] if self.configuration.Data.inputDBS in local_dbs_urls + local_dbs_urls_aliases: inputDataset_parts = self.configuration.Data.inputDataset.split('/') inputDataset_parts.pop(0) inputDataset_tier = inputDataset_parts[-1] if len(inputDataset_parts) == 3 else None user_data_tiers = ['USER'] if inputDataset_tier not in user_data_tiers: msg = "Invalid CRAB configuration: A local DBS instance '%s' was specified for reading an input dataset of tier %s." \ % (self.configuration.Data.inputDBS, inputDataset_tier) msg += "\nDatasets of tier different than %s must be read from the global DBS instance; this is, set Data.inputDBS = 'global'." \ % (", ".join(user_data_tiers[:-1]) + " or " + user_data_tiers[-1] if len(user_data_tiers) > 1 else user_data_tiers[0]) if msg: inputDBS_default = getParamDefaultValue('Data.inputDBS') if inputDBS_default: inputDBS_default, inputDBS_default_alias = self.getDBSURLAndAlias(inputDBS_default, 'reader') if inputDBS_default and inputDBS_default_alias: msg += "\nIf Data.inputDBS would not be specified, the default '%s' ('%s') would be used." % (inputDBS_default_alias, inputDBS_default) return False, msg ## If a publication DBS URL is specified and publication is ON, check that the DBS URL is a good one. if hasattr(self.configuration.Data, 'publishDBS'): if getattr(self.configuration.Data, 'publication', getParamDefaultValue('Data.publication')): dbs_urls = DBSURLS['writer'].values() dbs_urls_aliases = DBSURLS['writer'].keys() if (self.configuration.Data.publishDBS not in dbs_urls_aliases) and (self.configuration.Data.publishDBS.rstrip('/') not in dbs_urls): msg = "Invalid CRAB configuration: Parameter Data.publishDBS has an invalid value ('%s')." % (self.configuration.Data.publishDBS) msg += "\nAllowed values are: " msg += "\n ".join(["'%s' ('%s')" % (alias, url) for alias, url in DBSURLS['writer'].iteritems()]) publishDBS_default = getParamDefaultValue('Data.publishDBS') if publishDBS_default: publishDBS_default, publishDBS_default_alias = self.getDBSURLAndAlias(publishDBS_default, 'writer') if publishDBS_default and publishDBS_default_alias: msg += "\nIf Data.publishDBS would not be specified, the default '%s' ('%s') would be used." \ % (publishDBS_default_alias, publishDBS_default) return False, msg if hasattr(self.configuration.JobType, 'scriptExe'): if not os.path.isfile(self.configuration.JobType.scriptExe): msg = "Cannot find the file %s specified in the JobType.scriptExe configuration parameter." % (self.configuration.JobType.scriptExe) return False, msg return True, "Valid configuration"
def validateConfig(self): """ __validateConfig__ Checking if needed input parameters are there """ valid, msg = SubCommand.validateConfig(self) if not valid: return False, msg requestNameLenLimit = 100 if hasattr(self.configuration.General, 'requestName'): if len(self.configuration.General.requestName ) > requestNameLenLimit: msg = "Invalid CRAB configuration: Parameter General.requestName should not be longer than %d characters." % ( requestNameLenLimit) return False, msg splitting = getattr(self.configuration.Data, 'splitting', 'Automatic') autoSplitt = True if splitting == 'Automatic' else False autoSplittUnitsMin = 180 # 3 hours (defined also in TW config as 'minAutomaticRuntimeMins') autoSplittUnitsMax = 2700 # 45 hours ## Check that maxJobRuntimeMin is not used with Automatic splitting if autoSplitt and hasattr(self.configuration.JobType, 'maxJobRuntimeMin'): msg = "The 'maxJobRuntimeMin' parameter is not compatible with the 'Automatic' splitting mode (default)." return False, msg ## Check that --dryrun is not used with Automatic splitting if autoSplitt and self.options.dryrun: msg = "The 'dryrun' option is not compatible with the 'Automatic' splitting mode (default)." return False, msg ## Check that Data.unitsPerjob is specified. if hasattr(self.configuration.Data, 'unitsPerJob'): try: float(self.configuration.Data.unitsPerJob) except ValueError: msg = "Invalid CRAB configuration: Parameter Data.unitsPerJob must be a valid number, not %s." % ( self.configuration.Data.unitsPerJob) return False, msg if not int(self.configuration.Data.unitsPerJob) > 0: msg = "Invalid CRAB configuration: Parameter Data.unitsPerJob must be > 0, not %s." % ( self.configuration.Data.unitsPerJob) return False, msg if autoSplitt and ( self.configuration.Data.unitsPerJob > autoSplittUnitsMax or self.configuration.Data.unitsPerJob < autoSplittUnitsMin): msg = "Invalid CRAB configuration: In case of Automatic splitting, the Data.unitsPerJob parameter must be in the [%d, %d] minutes range. You asked for %d minutes." % ( autoSplittUnitsMin, autoSplittUnitsMax, self.configuration.Data.unitsPerJob) return False, msg elif not autoSplitt: # The default value is only valid for automatic splitting! msg = "Invalid CRAB configuration: Parameter Data.unitsPerJob is mandatory for '%s' splitting mode." % splitting return False, msg ## Check that JobType.pluginName and JobType.externalPluginFile are not both specified. if hasattr(self.configuration.JobType, 'pluginName') and hasattr( self.configuration.JobType, 'externalPluginFile'): msg = "Invalid CRAB configuration: Only one of JobType.pluginName or JobType.externalPluginFile parameters can be specified." pluginName_default = getParamDefaultValue('JobType.pluginName') if pluginName_default: msg += "\nIf neither JobType.pluginName nor JobType.externalPluginFile would be specified," msg += " the default JobType.pluginName = '%s' would be used." % ( pluginName_default) return False, msg ## Load the external plugin or check that the crab plugin is valid. external_plugin_name = getattr(self.configuration.JobType, 'externalPluginFile', None) crab_plugin_name = getattr(self.configuration.JobType, 'pluginName', None) crab_job_types = { 'ANALYSIS': None, 'PRIVATEMC': None, 'COPYCAT': None } #getJobTypes() if external_plugin_name: addPlugin(external_plugin_name) # Do we need to do this here? if crab_plugin_name: if upper(crab_plugin_name) not in crab_job_types: msg = "Invalid CRAB configuration: Parameter JobType.pluginName has an invalid value ('%s')." % ( crab_plugin_name) msg += "\nAllowed values are: %s." % (", ".join( ['%s' % job_type for job_type in crab_job_types.keys()])) return False, msg msg = "Will use CRAB %s plugin" % ("Analysis" if upper( crab_plugin_name) == 'ANALYSIS' else "PrivateMC") msg += " (i.e. will run %s job type)." % ("an analysis" if upper( crab_plugin_name) == 'ANALYSIS' else "a MC generation") self.logger.debug(msg) ## Check that the particular combination (Data.publication = True, General.transferOutputs = False) is not specified. if getattr(self.configuration.Data, 'publication', getParamDefaultValue('Data.publication')) and \ not getattr(self.configuration.General, 'transferOutputs', getParamDefaultValue('General.transferOutputs')): msg = "Invalid CRAB configuration: Data.publication is True, but General.transferOutputs is False." msg += "\nPublication can not be performed if the output files are not transferred to a permanent storage." return False, msg ## Check that a storage site is specified if General.transferOutputs = True or General.transferLogs = True. if not hasattr(self.configuration.Site, 'storageSite'): if getattr(self.configuration.General, 'transferLogs', getParamDefaultValue('General.transferLogs')) or \ getattr(self.configuration.General, 'transferOutputs', getParamDefaultValue('General.transferOutputs')): msg = "Invalid CRAB configuration: Parameter Site.storageSite is missing." return False, msg ## If an input dataset and a DBS URL are specified, check that the DBS URL is a good one. ## Also, if the DBS URL is 'phys0x', check that the input dataset tier is USER. if hasattr(self.configuration.Data, 'inputDBS'): if hasattr(self.configuration.Data, 'inputDataset'): msg = None dbs_urls_aliases = DBSURLS['reader'].keys() dbs_urls = DBSURLS['reader'].values() if (self.configuration.Data.inputDBS not in dbs_urls_aliases ) and (self.configuration.Data.inputDBS.rstrip('/') not in dbs_urls): msg = "Invalid CRAB configuration: Parameter Data.inputDBS has an invalid value ('%s')." % ( self.configuration.Data.inputDBS) msg += "\nAllowed values are: " msg += "\n ".join([ "'%s' ('%s')" % (alias, url) for alias, url in DBSURLS['reader'].iteritems() ]) local_dbs_urls_aliases = ['phys01', 'phys02', 'phys03'] local_dbs_urls = [ DBSURLS['reader'][alias] for alias in local_dbs_urls_aliases if alias in DBSURLS['reader'] ] if self.configuration.Data.inputDBS in local_dbs_urls + local_dbs_urls_aliases: inputDataset_parts = self.configuration.Data.inputDataset.split( '/') inputDataset_parts.pop(0) inputDataset_tier = inputDataset_parts[-1] if len( inputDataset_parts) == 3 else None user_data_tiers = ['USER'] if inputDataset_tier not in user_data_tiers: msg = "Invalid CRAB configuration: A local DBS instance '%s' was specified for reading an input dataset of tier %s." \ % (self.configuration.Data.inputDBS, inputDataset_tier) msg += "\nDatasets of tier different than %s must be read from the global DBS instance; this is, set Data.inputDBS = 'global'." \ % (", ".join(user_data_tiers[:-1]) + " or " + user_data_tiers[-1] if len(user_data_tiers) > 1 else user_data_tiers[0]) if msg: inputDBS_default = getParamDefaultValue('Data.inputDBS') if inputDBS_default: inputDBS_default, inputDBS_default_alias = self.getDBSURLAndAlias( inputDBS_default, 'reader') if inputDBS_default and inputDBS_default_alias: msg += "\nIf Data.inputDBS would not be specified, the default '%s' ('%s') would be used." % ( inputDBS_default_alias, inputDBS_default) return False, msg ## If a publication DBS URL is specified and publication is ON, check that the DBS URL is a good one. if hasattr(self.configuration.Data, 'publishDBS'): if getattr(self.configuration.Data, 'publication', getParamDefaultValue('Data.publication')): dbs_urls = DBSURLS['writer'].values() dbs_urls_aliases = DBSURLS['writer'].keys() if (self.configuration.Data.publishDBS not in dbs_urls_aliases ) and (self.configuration.Data.publishDBS.rstrip('/') not in dbs_urls): msg = "Invalid CRAB configuration: Parameter Data.publishDBS has an invalid value ('%s')." % ( self.configuration.Data.publishDBS) msg += "\nAllowed values are: " msg += "\n ".join([ "'%s' ('%s')" % (alias, url) for alias, url in DBSURLS['writer'].iteritems() ]) publishDBS_default = getParamDefaultValue( 'Data.publishDBS') if publishDBS_default: publishDBS_default, publishDBS_default_alias = self.getDBSURLAndAlias( publishDBS_default, 'writer') if publishDBS_default and publishDBS_default_alias: msg += "\nIf Data.publishDBS would not be specified, the default '%s' ('%s') would be used." \ % (publishDBS_default_alias, publishDBS_default) return False, msg if hasattr(self.configuration.JobType, 'scriptExe'): if not os.path.isfile(self.configuration.JobType.scriptExe): msg = "Cannot find the file %s specified in the JobType.scriptExe configuration parameter." % ( self.configuration.JobType.scriptExe) return False, msg ## If ignoreLocality is set, check that a sitewhilelist is present if getattr(self.configuration.Data, 'ignoreLocality', False): if not hasattr(self.configuration.Site, 'whitelist'): msg = "Invalid CRAB configuration:\n when ignoreLocality is set a valid site white list must be specified using the Site.whitelist parameter" return False, msg if hasattr(self.configuration.General, 'failureLimit'): msg = "You have specified deprecated parameter 'failureLimit' which will be removed in the near future." msg += "\nIf you really need it write a mail to hn-cms-computingTools explaining your use case." self.logger.warning("%sWARNING%s: %s" % (colors.RED, colors.NORMAL, msg)) return True, "Valid configuration"
def __call__(self): self.logger.debug("Started submission") serverFactory = CRABClient.Emulator.getEmulator('rest') uniquerequestname = None self.logger.debug("Working on %s" % str(self.requestarea)) self.configreq = {'dryrun': 1 if self.options.dryrun else 0} for param in parametersMapping['on-server']: mustbetype = getattr(types, parametersMapping['on-server'][param]['type']) default = parametersMapping['on-server'][param]['default'] config_params = parametersMapping['on-server'][param]['config'] for config_param in config_params: attrs = config_param.split('.') temp = self.configuration for attr in attrs: temp = getattr(temp, attr, None) if temp is None: break if temp is not None: self.configreq[param] = temp break elif default is not None: self.configreq[param] = default temp = default else: ## Parameter not strictly required. pass ## Check that the requestname is of the right type. ## This is not checked in SubCommand.validateConfig(). if param == 'workflow': if isinstance(self.requestname, mustbetype): self.configreq['workflow'] = self.requestname ## Translate boolean flags into integers. elif param in ['savelogsflag', 'publication', 'publishgroupname', 'nonprodsw', 'useparent',\ 'ignorelocality', 'saveoutput', 'oneEventMode', 'nonvaliddata', 'ignoreglobalblacklist']: self.configreq[param] = 1 if temp else 0 ## Translate DBS URL aliases into DBS URLs. elif param in ['dbsurl', 'publishdbsurl']: if param == 'dbsurl': dbstype = 'reader' elif param == 'publishdbsurl': dbstype = 'writer' allowed_dbsurls = DBSURLS[dbstype].values() allowed_dbsurls_aliases = DBSURLS[dbstype].keys() if self.configreq[param] in allowed_dbsurls_aliases: self.configreq[param] = DBSURLS[dbstype][ self.configreq[param]] elif self.configreq[param].rstrip('/') in allowed_dbsurls: self.configreq[param] = self.configreq[param].rstrip('/') elif param == 'scriptexe' and 'scriptexe' in self.configreq: self.configreq[param] = os.path.basename(self.configreq[param]) jobconfig = {} #get the backend URLs from the server external configuration serverBackendURLs = server_info('backendurls', self.serverurl, self.proxyfilename, getUrl(self.instance, resource='info')) #if cacheSSL is specified in the server external configuration we will use it to upload the sandbox filecacheurl = serverBackendURLs[ 'cacheSSL'] if 'cacheSSL' in serverBackendURLs else None pluginParams = [ self.configuration, self.proxyfilename, self.logger, os.path.join(self.requestarea, 'inputs') ] crab_job_types = getJobTypes() if upper(self.configreq['jobtype']) in crab_job_types: plugjobtype = crab_job_types[upper( self.configreq['jobtype'])](*pluginParams) dummy_inputfiles, jobconfig = plugjobtype.run(filecacheurl) else: fullname = self.configreq['jobtype'] basename = os.path.basename(fullname).split('.')[0] plugin = addPlugin(fullname)[basename] pluginInst = plugin(*pluginParams) dummy_inputfiles, jobconfig = pluginInst.run() if self.configreq['publication']: non_edm_files = jobconfig['tfileoutfiles'] + jobconfig[ 'addoutputfiles'] if non_edm_files: msg = "%sWarning%s: The following output files will not be published, as they are not EDM files: %s" % ( colors.RED, colors.NORMAL, non_edm_files) self.logger.warning(msg) self.configreq.update(jobconfig) server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) self.logger.info("Sending the request to the server at %s" % self.serverurl) self.logger.debug("Submitting %s " % str(self.configreq)) ## TODO: this shouldn't be hard-coded. listParams = ['addoutputfiles', 'sitewhitelist', 'siteblacklist', 'blockwhitelist', 'blockblacklist', \ 'tfileoutfiles', 'edmoutfiles', 'runs', 'lumis', 'userfiles', 'scriptargs', 'extrajdl'] self.configreq_encoded = self._encodeRequest(self.configreq, listParams) self.logger.debug('Encoded submit request: %s' % (self.configreq_encoded)) dictresult, status, reason = server.put(self.uri, data=self.configreq_encoded) self.logger.debug("Result: %s" % dictresult) if status != 200: msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % ( str(self.configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) elif 'result' in dictresult: uniquerequestname = dictresult["result"][0]["RequestName"] else: msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \ % (str(self.configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) tmpsplit = self.serverurl.split(':') createCache(self.requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit) > 1 else '', uniquerequestname, voRole=self.voRole, voGroup=self.voGroup, instance=self.instance, originalConfig=self.configuration) self.logger.info( "%sSuccess%s: Your task has been delivered to the %s CRAB3 server." % (colors.GREEN, colors.NORMAL, self.instance)) if not (self.options.wait or self.options.dryrun): self.logger.info("Task name: %s" % uniquerequestname) projDir = os.path.join( getattr(self.configuration.General, 'workArea', '.'), self.requestname) self.logger.info("Project dir: %s" % projDir) self.logger.info( "Please use 'crab status -d %s' to check how the submission process proceeds.", projDir) else: targetTaskStatus = 'UPLOADED' if self.options.dryrun else 'SUBMITTED' checkStatusLoop(self.logger, server, self.uri, uniquerequestname, targetTaskStatus, self.name) if self.options.dryrun: self.printDryRunResults(*self.executeTestRun(filecacheurl)) self.logger.debug("About to return") return { 'requestname': self.requestname, 'uniquerequestname': uniquerequestname }
def validateConfig(self): """ __validateConfig__ Checking if needed input parameters are there """ valid, msg = SubCommand.validateConfig(self) if not valid: return False, msg ## Check that Data.unitsPerjob is specified. if hasattr(self.configuration.Data, 'unitsPerJob'): try: float(self.configuration.Data.unitsPerJob) except ValueError: msg = "Invalid CRAB configuration: Parameter Data.unitsPerJob must be a valid number, not %s." % ( self.configuration.Data.unitsPerJob) return False, msg ## Check that JobType.pluginName and JobType.externalPluginFile are not both specified. if hasattr(self.configuration.JobType, 'pluginName') and hasattr( self.configuration.JobType, 'externalPluginFile'): msg = "Invalid CRAB configuration: Only one of JobType.pluginName or JobType.externalPluginFile parameters can be specified." pluginName_default = getParamDefaultValue('JobType.pluginName') if pluginName_default: msg += "\nIf neither JobType.pluginName nor JobType.externalPluginFile would be specified," msg += " the default JobType.pluginName = '%s' would be used." % ( pluginName_default) return False, msg ## Load the external plugin or check that the crab plugin is valid. external_plugin_name = getattr(self.configuration.JobType, 'externalPluginFile', None) crab_plugin_name = getattr(self.configuration.JobType, 'pluginName', None) crab_job_types = {'ANALYSIS': None, 'PRIVATEMC': None} #getJobTypes() if external_plugin_name: addPlugin(external_plugin_name) # Do we need to do this here? if crab_plugin_name: if upper(crab_plugin_name) not in crab_job_types: msg = "Invalid CRAB configuration: Parameter JobType.pluginName has an invalid value ('%s')." % ( crab_plugin_name) msg += "\nAllowed values are: %s." % (", ".join( ['%s' % job_type for job_type in crab_job_types.keys()])) return False, msg msg = "Will use CRAB %s plugin" % ("Analysis" if upper( crab_plugin_name) == 'ANALYSIS' else "PrivateMC") msg += " (i.e. will run %s job type)." % ("an analysis" if upper( crab_plugin_name) == 'ANALYSIS' else "a MC generation") self.logger.debug(msg) ## Check that the particular combination (Data.publication = True, General.transferOutputs = False) is not specified. if getattr(self.configuration.Data, 'publication', getParamDefaultValue('Data.publication')) and \ not getattr(self.configuration.General, 'transferOutputs', getParamDefaultValue('General.transferOutputs')): msg = "Invalid CRAB configuration: Data.publication is True, but General.transferOutputs is False." msg += "\nPublication can not be performed if the output files are not transferred to a permanent storage." return False, msg ## Check that a storage site is specified if General.transferOutputs = True or General.transferLogs = True. if not hasattr(self.configuration.Site, 'storageSite'): if getattr(self.configuration.General, 'transferLogs', getParamDefaultValue('General.transferLogs')) or \ getattr(self.configuration.General, 'transferOutputs', getParamDefaultValue('General.transferOutputs')): msg = "Invalid CRAB configuration: Parameter Site.storageSite is missing." return False, msg ## If an input dataset and a DBS URL are specified, check that the DBS URL is a good one. ## Also, if the DBS URL is 'phys0x', check that the input dataset tier is USER. if hasattr(self.configuration.Data, 'inputDBS'): if hasattr(self.configuration.Data, 'inputDataset'): msg = None dbs_urls_aliases = DBSURLS['reader'].keys() dbs_urls = DBSURLS['reader'].values() if (self.configuration.Data.inputDBS not in dbs_urls_aliases ) and (self.configuration.Data.inputDBS.rstrip('/') not in dbs_urls): msg = "Invalid CRAB configuration: Parameter Data.inputDBS has an invalid value ('%s')." % ( self.configuration.Data.inputDBS) msg += "\nAllowed values are: " msg += "\n ".join([ "'%s' ('%s')" % (alias, url) for alias, url in DBSURLS['reader'].iteritems() ]) local_dbs_urls_aliases = ['phys01', 'phys02', 'phys03'] local_dbs_urls = [ DBSURLS['reader'][alias] for alias in local_dbs_urls_aliases if alias in DBSURLS['reader'] ] if self.configuration.Data.inputDBS in local_dbs_urls + local_dbs_urls_aliases: inputDataset_parts = self.configuration.Data.inputDataset.split( '/') inputDataset_parts.pop(0) inputDataset_tier = inputDataset_parts[-1] if len( inputDataset_parts) == 3 else None user_data_tiers = ['USER'] if inputDataset_tier not in user_data_tiers: msg = "Invalid CRAB configuration: A local DBS instance '%s' was specified for reading an input dataset of tier %s." \ % (self.configuration.Data.inputDBS, inputDataset_tier) msg += "\nDatasets of tier different than %s must be read from the global DBS instance; this is, set Data.inputDBS = 'global'." \ % (", ".join(user_data_tiers[:-1]) + " or " + user_data_tiers[-1] if len(user_data_tiers) > 1 else user_data_tiers[0]) if msg: inputDBS_default = getParamDefaultValue('Data.inputDBS') if inputDBS_default: inputDBS_default, inputDBS_default_alias = self.getDBSURLAndAlias( inputDBS_default, 'reader') if inputDBS_default and inputDBS_default_alias: msg += "\nIf Data.inputDBS would not be specified, the default '%s' ('%s') would be used." % ( inputDBS_default_alias, inputDBS_default) return False, msg ## If a publication DBS URL is specified and publication is ON, check that the DBS URL is a good one. if hasattr(self.configuration.Data, 'publishDBS'): if getattr(self.configuration.Data, 'publication', getParamDefaultValue('Data.publication')): dbs_urls = DBSURLS['writer'].values() dbs_urls_aliases = DBSURLS['writer'].keys() if (self.configuration.Data.publishDBS not in dbs_urls_aliases ) and (self.configuration.Data.publishDBS.rstrip('/') not in dbs_urls): msg = "Invalid CRAB configuration: Parameter Data.publishDBS has an invalid value ('%s')." % ( self.configuration.Data.publishDBS) msg += "\nAllowed values are: " msg += "\n ".join([ "'%s' ('%s')" % (alias, url) for alias, url in DBSURLS['writer'].iteritems() ]) publishDBS_default = getParamDefaultValue( 'Data.publishDBS') if publishDBS_default: publishDBS_default, publishDBS_default_alias = self.getDBSURLAndAlias( publishDBS_default, 'writer') if publishDBS_default and publishDBS_default_alias: msg += "\nIf Data.publishDBS would not be specified, the default '%s' ('%s') would be used." \ % (publishDBS_default_alias, publishDBS_default) return False, msg if hasattr(self.configuration.JobType, 'scriptExe'): if not os.path.isfile(self.configuration.JobType.scriptExe): msg = "Cannot find the file %s specified in the JobType.scriptExe configuration parameter." % ( self.configuration.JobType.scriptExe) return False, msg return True, "Valid configuration"