def validateConfig(self): """ __validateConfig__ Checking if needed input parameters are there. Not all the commands require a configuration. """ ## Check that the configuration object has the sections we expect it to have. ## (WMCore already checks that attributes added to the configuration object are of type ConfigSection.) ## Even if not all configuration sections need to be there, we anyway request ## the user to add all the sections in the configuration file. if not hasattr(self.configuration, 'General'): msg = "Invalid CRAB configuration: Section 'General' is missing." return False, msg if not hasattr(self.configuration, 'JobType'): msg = "Invalid CRAB configuration: Section 'JobType' is missing." return False, msg if not hasattr(self.configuration, 'Data'): msg = "Invalid CRAB configuration: Section 'Data' is missing." return False, msg if not hasattr(self.configuration, 'Site'): msg = "Invalid CRAB configuration: Section 'Site' is missing." return False, msg ## Some parameters may have been renamed. Check here if the configuration file has an old ## parameter defined, and in that case tell the user what is the new parameter name. for old_param, new_param in renamedParams.iteritems(): if len(old_param.split('.')) != 2 or len(new_param['newParam'].split('.')) != 2: continue old_param_section, old_param_name = old_param.split('.') if hasattr(self.configuration, old_param_section) and hasattr(getattr(self.configuration, old_param_section), old_param_name): msg = "Invalid CRAB configuration: Parameter %s has been renamed to %s" % (old_param, new_param['newParam']) if new_param['version'] != None: msg += " starting from CRAB %s" % (new_param['version']) msg += "; please change your configuration file accordingly." return False, msg ## Check if there are unknown parameters (and try to suggest the correct parameter name). all_config_params = configParametersInfo.keys() SpellChecker.DICTIONARY = SpellChecker.train(all_config_params) for section in self.configuration.listSections_(): for attr in getattr(self.configuration, section).listSections_(): param = (section + '.' + attr) if not SpellChecker.is_correct(param): msg = "Invalid CRAB configuration: Parameter %s is not known." % (param) if SpellChecker.correct(param) != param: msg += " Maybe you mean %s?" % (SpellChecker.correct(param)) return False, msg ## Check that each parameter specified in the configuration file is of the ## type specified in the configuration map. ## Check that, if a parameter is a required one and it has no default value, ## then it must be specified in the configuration file. for paramName, paramInfo in configParametersInfo.iteritems(): requiredTypeName = paramInfo['type'] try: requiredType = getattr(types, requiredTypeName) except AttributeError: msg = "Invalid type %s specified in CRABClient configuration mapping for parameter %s." % (requiredTypeName, paramName) return False, msg attrs = paramName.split('.') obj = self.configuration while attrs and obj is not None: obj = getattr(obj, attrs.pop(0), None) if obj is not None: if not isinstance(obj, requiredType): msg = "Invalid CRAB configuration: Parameter %s requires a value of type %s (while a value of type %s was given)." \ % (paramName, str(requiredType), str(type(obj))) if paramName == "Data.userInputFiles": msg += "\nIn CRAB v3.3.14 the configuration parameter Data.userInputFiles has been modified to directly take a (python) list of primary input files." msg += " Previously it was taking the name of a local text file where the primary input files were listed." msg += " One can still use a text file and convert its content into a python list by doing Data.userInputFiles = list(open('my_list_of_files.txt'))" return False, msg elif requiredType == list: if not all(isinstance(arg, str) for arg in obj): msg = "Invalid CRAB configuration: Parameter %s has to be a list of strings." % (paramName) return False, msg elif getParamDefaultValue(paramName) is None and paramInfo['required']: msg = "Invalid CRAB configuration: Parameter %s is missing." % (paramName) return False, msg return True, "Valid configuration"
def __call__(self): valid = False configmsg = 'Default' if not os.path.isfile(self.options.config): raise MissingOptionException("Configuration file '%s' not found" % self.options.config) #store the configuration file in self.configuration self.loadConfig( self.options.config, self.args ) requestarea, requestname, self.logfile = createWorkArea( self.logger, getattr(self.configuration.General, 'workArea', None), getattr(self.configuration.General, 'requestName', None) ) self.logger.debug("Started submission") #determine the serverurl if self.options.server: self.serverurl = self.options.server elif getattr( self.configuration.General, 'serverUrl', None ) is not None: self.serverurl = self.configuration.General.serverUrl #TODO: For sure the server url should not be handled here. Find an intelligent way for this else: self.serverurl = 'http://cmsweb.cern.ch' if not hasattr( self.configuration.General, 'ufccacheUrl' ): self.configuration.General.ufccacheUrl = self.serverurl if not hasattr( self.configuration.General, 'configcacheUrl' ): #https is required because configcache does not use ServerInteractions self.configuration.General.configcacheUrl = 'https://' + self.serverurl + '/couchdb' if not hasattr( self.configuration.General, 'configcacheName' ): self.configuration.General.configcacheName = 'analysis_reqmgr_config_cache' self.createCache( self.serverurl ) ######### Check if the user provided unexpected parameters ######## #init the dictionary with all the known parameters SpellChecker.DICTIONARY = SpellChecker.train( [ val['config'] for _, val in self.requestmapper.iteritems() if val['config'] ] + \ [ x for x in self.otherConfigParams ] ) #iterate on the parameters provided by the user for section in self.configuration.listSections_(): for attr in getattr(self.configuration, section).listSections_(): par = (section + '.' + attr) #if the parameter is not know exit, but try to correct it before if not SpellChecker.is_correct( par ): msg = 'The parameter %s is not known.' % par msg += '' if SpellChecker.correct(par) == par else ' Did you mean %s?' % SpellChecker.correct(par) raise ConfigurationException(msg) #usertarball and cmsswconfig use this parameter and we should set it up in a correct way self.configuration.General.serverUrl = self.serverurl #delegating the proxy (creation done in SubCommand) self.voRole = getattr(self.configuration.User, "voRole", "") self.voGroup = getattr(self.configuration.User, "voGroup", "") self.handleProxy() uniquerequestname = None self.logger.debug("Working on %s" % str(requestarea)) configreq = {} for param in self.requestmapper: mustbetype = getattr(types, self.requestmapper[param]['type']) if self.requestmapper[param]['config']: attrs = self.requestmapper[param]['config'].split('.') temp = self.configuration for attr in attrs: temp = getattr(temp, attr, None) if temp is None: break if temp: if mustbetype == type(temp): configreq[param] = temp else: raise ConfigurationException(1, "Invalid type " + str(type(temp)) + " for parameter " + self.requestmapper[param]['config'] \ + ". It is needed a " + str(mustbetype) + ".") elif self.requestmapper[param]['default'] is not None: configreq[param] = self.requestmapper[param]['default'] elif self.requestmapper[param]['required']: raise ConfigurationException(1, "Missing parameter " + self.requestmapper[param]['config'] + " from the configuration.") else: ## parameter not strictly required pass if param == "workflow": if mustbetype == type(requestname): configreq["workflow"] = requestname elif param == "savelogsflag": configreq["savelogsflag"] = 1 if temp else 0 elif param == "publication": configreq["publication"] = 1 if temp else 0 elif param == "blacklistT1": blacklistT1 = self.voRole != 't1access' #if the user choose to remove the automatic T1 blacklisting and has not the t1acces role if getattr (self.configuration.Site, 'removeT1Blacklisting', False) and blacklistT1: self.logger.info("WARNING: You disabled the T1 automatic blacklisting without having the t1access role") blacklistT1 = False configreq["blacklistT1"] = 1 if blacklistT1 else 0 jobconfig = {} self.configuration.JobType.proxyfilename = self.proxyfilename self.configuration.JobType.capath = HTTPRequests.getCACertPath() pluginParams = [ self.configuration, self.logger, os.path.join(requestarea, 'inputs') ] if getattr(self.configuration.JobType, 'pluginName', None) is not None: jobtypes = getJobTypes() plugjobtype = jobtypes[upper(self.configuration.JobType.pluginName)](*pluginParams) inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq) else: fullname = self.configuration.JobType.externalPluginFile basename = os.path.basename(fullname).split('.')[0] plugin = addPlugin(fullname)[basename] pluginInst = plugin(*pluginParams) inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq) configreq['publishname'] = "%s-%s" %(configreq['publishname'], isbchecksum) configreq.update(jobconfig) server = HTTPRequests(self.serverurl, self.proxyfilename) self.logger.info("Sending the request to the server") self.logger.debug("Submitting %s " % str( configreq ) ) dictresult, status, reason = server.put( self.uri, data = self._encodeRequest(configreq) ) self.logger.debug("Result: %s" % dictresult) if status != 200: msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (str(configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) elif dictresult.has_key("result"): uniquerequestname = dictresult["result"][0]["RequestName"] else: msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \ % (str(configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) tmpsplit = self.serverurl.split(':') createCache( requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit)>1 else '', uniquerequestname, voRole = self.voRole, voGroup = self.voGroup ) self.logger.info("Submission completed") self.logger.debug("Request ID: %s " % uniquerequestname) self.logger.debug("Ended submission") return uniquerequestname
def __call__(self): valid = False configmsg = 'Default' self.logger.debug("Started submission") serverFactory = CRABClient.Emulator.getEmulator('rest') # Get some debug parameters ######### Check if the user provided unexpected parameters ######## #init the dictionary with all the known parameters all_config_params = [x for x in parameters_mapping['other-config-params']] for _, val in parameters_mapping['on-server'].iteritems(): if val['config']: all_config_params.extend(val['config']) SpellChecker.DICTIONARY = SpellChecker.train(all_config_params) #iterate on the parameters provided by the user for section in self.configuration.listSections_(): for attr in getattr(self.configuration, section).listSections_(): par = (section + '.' + attr) #if the parameter is not know exit, but try to correct it before if not SpellChecker.is_correct( par ): msg = 'The parameter %s is not known.\nPlease refer to <https://twiki.cern.ch/twiki/bin/view/CMSPublic/WorkBookCRAB3Tutorial#CRAB_configuration_parameters> for list of valid parameter.\nSee the ./crab.log file for more details"' % par msg += '' if SpellChecker.correct(par) == par else '\nOr maybe did you mean %s?' % SpellChecker.correct(par) raise ConfigurationException(msg) #usertarball and cmsswconfig use this parameter and we should set it up in a correct way self.configuration.General.serverUrl = self.serverurl uniquerequestname = None self.logger.debug("Working on %s" % str(self.requestarea)) configreq = {} for param in parameters_mapping['on-server']: mustbetype = getattr(types, parameters_mapping['on-server'][param]['type']) default = parameters_mapping['on-server'][param]['default'] config_params = parameters_mapping['on-server'][param]['config'] for config_param in config_params: attrs = config_param.split('.') temp = self.configuration for attr in attrs: temp = getattr(temp, attr, None) if temp is None: break if temp is not None: configreq[param] = temp break elif default is not None: configreq[param] = default temp = default else: ## Parameter not strictly required. pass ## Check that the requestname is of the right type. ## This is not checked in SubCommand.validateConfig(). if param == 'workflow': if mustbetype == type(self.requestname): configreq['workflow'] = self.requestname ## Translate boolean flags into integers. elif param in ['savelogsflag', 'publication', 'nonprodsw', 'useparent', 'ignorelocality', 'saveoutput', 'oneEventMode']: configreq[param] = 1 if temp else 0 ## Translate DBS URL aliases into DBS URLs. elif param in ['dbsurl', 'publishdbsurl']: if param == 'dbsurl': dbstype = 'reader' elif param == 'publishdbsurl': dbstype = 'writer' allowed_dbsurls = DBSURLS[dbstype].values() allowed_dbsurls_aliases = DBSURLS[dbstype].keys() if configreq[param] in allowed_dbsurls_aliases: configreq[param] = DBSURLS[dbstype][configreq[param]] elif configreq[param].rstrip('/') in allowed_dbsurls: configreq[param] = configreq[param].rstrip('/') elif param == 'scriptexe' and 'scriptexe' in configreq: configreq[param] = os.path.basename(configreq[param]) jobconfig = {} self.configuration.JobType.proxyfilename = self.proxyfilename self.configuration.JobType.capath = serverFactory.getCACertPath() #get the backend URLs from the server external configuration serverBackendURLs = server_info('backendurls', self.serverurl, self.proxyfilename, getUrl(self.instance, resource='info')) #if cacheSSL is specified in the server external configuration we will use it to upload the sandbox (baseURL will be ignored) self.configuration.JobType.filecacheurl = serverBackendURLs['cacheSSL'] if 'cacheSSL' in serverBackendURLs else None pluginParams = [ self.configuration, self.logger, os.path.join(self.requestarea, 'inputs') ] crab_job_types = getJobTypes() if upper(configreq['jobtype']) in crab_job_types: plugjobtype = crab_job_types[upper(configreq['jobtype'])](*pluginParams) inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq) else: fullname = configreq['jobtype'] basename = os.path.basename(fullname).split('.')[0] plugin = addPlugin(fullname)[basename] pluginInst = plugin(*pluginParams) inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq) if configreq['publication']: non_edm_files = jobconfig['tfileoutfiles'] + jobconfig['addoutputfiles'] if non_edm_files: msg = "%sWARNING%s: The following output files will not be published, as they are not EDM files: %s" % (colors.RED, colors.NORMAL, non_edm_files) self.logger.warning(msg) if not configreq['publishname']: configreq['publishname'] = isbchecksum else: configreq['publishname'] = "%s-%s" %(configreq['publishname'], isbchecksum) configreq.update(jobconfig) server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) self.logger.info("Sending the request to the server") self.logger.debug("Submitting %s " % str(configreq)) ## TODO: this shouldn't be hard-coded. listParams = ['adduserfiles', 'addoutputfiles', 'sitewhitelist', 'siteblacklist', 'blockwhitelist', 'blockblacklist', \ 'tfileoutfiles', 'edmoutfiles', 'runs', 'lumis', 'userfiles', 'scriptargs', 'extrajdl'] configreq_encoded = self._encodeRequest(configreq, listParams) self.logger.debug('Encoded submit request: %s' % (configreq_encoded)) dictresult, status, reason = server.put( self.uri, data = configreq_encoded) self.logger.debug("Result: %s" % dictresult) if status != 200: msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (str(configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) elif dictresult.has_key("result"): uniquerequestname = dictresult["result"][0]["RequestName"] else: msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \ % (str(configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) tmpsplit = self.serverurl.split(':') createCache(self.requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit)>1 else '', uniquerequestname, voRole=self.voRole, voGroup=self.voGroup, instance=self.instance, originalConfig = self.configuration) self.logger.info("%sSuccess%s: Your task has been delivered to the CRAB3 server." %(colors.GREEN, colors.NORMAL)) if not self.options.wait: self.logger.info("Task name: %s" % uniquerequestname) self.logger.info("Please use 'crab status' to check how the submission process proceed") if self.options.wait: self.checkStatusLoop(server,uniquerequestname) self.logger.debug("About to return") return {'requestname' : self.requestname , 'uniquerequestname' : uniquerequestname }
def validateConfig(self): """ __validateConfig__ Checking if needed input parameters are there. Not all the commands require a configuration. """ ## Check that the configuration object has the sections we expect it to have. ## (WMCore already checks that attributes added to the configuration object are of type ConfigSection.) ## Even if not all configuration sections need to be there, we anyway request ## the user to add all the sections in the configuration file. if not hasattr(self.configuration, 'General'): msg = "Invalid CRAB configuration: Section 'General' is missing." return False, msg if not hasattr(self.configuration, 'JobType'): msg = "Invalid CRAB configuration: Section 'JobType' is missing." return False, msg if not hasattr(self.configuration, 'Data'): msg = "Invalid CRAB configuration: Section 'Data' is missing." return False, msg if not hasattr(self.configuration, 'Site'): msg = "Invalid CRAB configuration: Section 'Site' is missing." return False, msg ## Some parameters may have been renamed. Check here if the configuration file has an old ## parameter defined, and in that case tell the user what is the new parameter name. for old_param, new_param in renamedParams.iteritems(): if len(old_param.split('.')) != 2 or len( new_param['newParam'].split('.')) != 2: continue old_param_section, old_param_name = old_param.split('.') if hasattr(self.configuration, old_param_section) and hasattr( getattr(self.configuration, old_param_section), old_param_name): msg = "Invalid CRAB configuration: Parameter %s has been renamed to %s" % ( old_param, new_param['newParam']) if new_param['version'] != None: msg += " starting from CRAB %s" % (new_param['version']) msg += "; please change your configuration file accordingly." return False, msg ## Check if there are unknown parameters (and try to suggest the correct parameter name). all_config_params = configParametersInfo.keys() SpellChecker.DICTIONARY = SpellChecker.train(all_config_params) for section in self.configuration.listSections_(): for attr in getattr(self.configuration, section).listSections_(): param = (section + '.' + attr) if not SpellChecker.is_correct(param): msg = "Invalid CRAB configuration: Parameter %s is not known." % ( param) if SpellChecker.correct(param) != param: msg += " Maybe you mean %s?" % ( SpellChecker.correct(param)) return False, msg ## Check that each parameter specified in the configuration file is of the ## type specified in the configuration map. ## Check that, if a parameter is a required one and it has no default value, ## then it must be specified in the configuration file. for paramName, paramInfo in configParametersInfo.iteritems(): requiredTypeName = paramInfo['type'] try: requiredType = getattr(types, requiredTypeName) except AttributeError: msg = "Invalid type %s specified in CRABClient configuration mapping for parameter %s." % ( requiredTypeName, paramName) return False, msg attrs = paramName.split('.') obj = self.configuration while attrs and obj is not None: obj = getattr(obj, attrs.pop(0), None) if obj is not None: if not isinstance(obj, requiredType): msg = "Invalid CRAB configuration: Parameter %s requires a value of type %s (while a value of type %s was given)." \ % (paramName, str(requiredType), str(type(obj))) if paramName == "Data.totalUnits" and isinstance( obj, float): continue if paramName == "Data.userInputFiles": msg += "\nIn CRAB v3.3.14 the configuration parameter Data.userInputFiles has been modified to directly take a (python) list of primary input files." msg += " Previously it was taking the name of a local text file where the primary input files were listed." msg += " One can still use a text file and convert its content into a python list by doing Data.userInputFiles = list(open('my_list_of_files.txt'))" return False, msg elif requiredType == list: if not all(isinstance(arg, str) for arg in obj): msg = "Invalid CRAB configuration: Parameter %s has to be a list of strings." % ( paramName) return False, msg elif getParamDefaultValue( paramName) is None and paramInfo['required']: msg = "Invalid CRAB configuration: Parameter %s is missing." % ( paramName) return False, msg return True, "Valid configuration"
def __call__(self): valid = False configmsg = 'Default' self.logger.debug("Started submission") # Get some debug parameters oneEventMode = hasattr(self.configuration, 'Debug') and \ getattr(self.configuration.Debug, 'oneEventMode') ######### Check if the user provided unexpected parameters ######## #init the dictionary with all the known parameters SpellChecker.DICTIONARY = SpellChecker.train( [ val['config'] for _, val in self.requestmapper.iteritems() if val['config'] ] + \ [ x for x in self.otherConfigParams ] ) #iterate on the parameters provided by the user for section in self.configuration.listSections_(): for attr in getattr(self.configuration, section).listSections_(): par = (section + '.' + attr) #if the parameter is not know exit, but try to correct it before if not SpellChecker.is_correct( par ): msg = 'The parameter %s is not known.' % par msg += '' if SpellChecker.correct(par) == par else ' Did you mean %s?' % SpellChecker.correct(par) raise ConfigurationException(msg) #usertarball and cmsswconfig use this parameter and we should set it up in a correct way self.configuration.General.serverUrl = self.serverurl uniquerequestname = None self.logger.debug("Working on %s" % str(self.requestarea)) configreq = {} for param in self.requestmapper: mustbetype = getattr(types, self.requestmapper[param]['type']) if self.requestmapper[param]['config']: attrs = self.requestmapper[param]['config'].split('.') temp = self.configuration for attr in attrs: temp = getattr(temp, attr, None) if temp is None: break if temp is not None: if mustbetype == type(temp): configreq[param] = temp else: raise ConfigurationException("Invalid type " + str(type(temp)) + " for parameter " + self.requestmapper[param]['config'] \ + ". It is needed a " + str(mustbetype) + ".") elif self.requestmapper[param]['default'] is not None: configreq[param] = self.requestmapper[param]['default'] temp = self.requestmapper[param]['default'] elif self.requestmapper[param]['required']: raise ConfigurationException("Missing parameter " + self.requestmapper[param]['config'] + " from the configuration.") else: ## parameter not strictly required pass if param == "workflow": if mustbetype == type(self.requestname): configreq["workflow"] = self.requestname elif param in ['savelogsflag','publication','nonprodsw','ignorelocality','saveoutput']:#TODO use clientmappig to do this configreq[param] = 1 if temp else 0 elif param in ['dbsurl','publishdbsurl']: if param == 'dbsurl': dbstype = 'reader' elif param == 'publishdbsurl': dbstype = 'writer' alloweddbsurls = DBSURLS[dbstype].values() alloweddbsurlsaliases = DBSURLS[dbstype].keys() if configreq[param] in alloweddbsurlsaliases: configreq[param] = DBSURLS[dbstype][configreq[param]] else: if configreq[param].rstrip('/') in alloweddbsurls: configreq[param] = configreq[param].rstrip('/') else: raise ConfigurationException("Invalid argument " + configreq[param] + " for parameter " + self.requestmapper[param]['config'] + " in the configuration.") if (configreq['saveoutput'] or configreq['savelogsflag']) and 'asyncdest' not in configreq: raise ConfigurationException("Missing parameter " + self.requestmapper['asyncdest']['config'] + " from the configuration.") # Add debug parameters to the configreq dict configreq['oneEventMode'] = int(oneEventMode) jobconfig = {} self.configuration.JobType.proxyfilename = self.proxyfilename self.configuration.JobType.capath = HTTPRequests.getCACertPath() #get the backend URLs from the server external configuration serverBackendURLs = server_info('backendurls', self.serverurl, self.proxyfilename, self.getUrl(self.instance, resource='info')) #if cacheSSL is specified in the server external configuration we will use it to upload the sandbox (baseURL will be ignored) self.configuration.JobType.filecacheurl = serverBackendURLs['cacheSSL'] if 'cacheSSL' in serverBackendURLs else None pluginParams = [ self.configuration, self.logger, os.path.join(self.requestarea, 'inputs') ] if getattr(self.configuration.JobType, 'pluginName', None) is not None: jobtypes = getJobTypes() plugjobtype = jobtypes[upper(self.configuration.JobType.pluginName)](*pluginParams) inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq) else: fullname = self.configuration.JobType.externalPluginFile basename = os.path.basename(fullname).split('.')[0] plugin = addPlugin(fullname)[basename] pluginInst = plugin(*pluginParams) inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq) if not configreq['publishname']: configreq['publishname'] = isbchecksum else: configreq['publishname'] = "%s-%s" %(configreq['publishname'], isbchecksum) configreq.update(jobconfig) server = HTTPRequests(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) self.logger.info("Sending the request to the server") self.logger.debug("Submitting %s " % str( configreq ) ) dictresult, status, reason = server.put( self.uri, data = self._encodeRequest(configreq) ) self.logger.debug("Result: %s" % dictresult) if status != 200: msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (str(configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) elif dictresult.has_key("result"): uniquerequestname = dictresult["result"][0]["RequestName"] else: msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \ % (str(configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) tmpsplit = self.serverurl.split(':') createCache(self.requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit)>1 else '', uniquerequestname, voRole=self.voRole, voGroup=self.voGroup, instance=self.instance, originalConfig = self.configuration) self.logger.info("%sSuccess%s: Your task has been delivered to the CRAB3 server." %(colors.GREEN, colors.NORMAL)) if not self.options.wait: self.logger.info("Please use 'crab status' to check how the submission process proceed") self.logger.debug("Request ID: %s " % uniquerequestname) if self.options.wait: self.checkStatusLoop(server,uniquerequestname) return uniquerequestname
def __call__(self): valid = False configmsg = 'Default' if not os.path.isfile(self.options.config): raise MissingOptionException("Configuration file '%s' not found" % self.options.config) #store the configuration file in self.configuration self.loadConfig(self.options.config, self.args) requestarea, requestname, self.logfile = createWorkArea( self.logger, getattr(self.configuration.General, 'workArea', None), getattr(self.configuration.General, 'requestName', None)) self.logger.debug("Started submission") #determine the serverurl if self.options.server: self.serverurl = self.options.server elif getattr(self.configuration.General, 'serverUrl', None) is not None: self.serverurl = self.configuration.General.serverUrl #TODO: For sure the server url should not be handled here. Find an intelligent way for this else: self.serverurl = 'http://cmsweb.cern.ch' if not hasattr(self.configuration.General, 'ufccacheUrl'): self.configuration.General.ufccacheUrl = self.serverurl if not hasattr(self.configuration.General, 'configcacheUrl'): #https is required because configcache does not use ServerInteractions self.configuration.General.configcacheUrl = 'https://' + self.serverurl + '/couchdb' if not hasattr(self.configuration.General, 'configcacheName'): self.configuration.General.configcacheName = 'analysis_reqmgr_config_cache' self.createCache(self.serverurl) ######### Check if the user provided unexpected parameters ######## #init the dictionary with all the known parameters SpellChecker.DICTIONARY = SpellChecker.train( [ val['config'] for _, val in self.requestmapper.iteritems() if val['config'] ] + \ [ x for x in self.otherConfigParams ] ) #iterate on the parameters provided by the user for section in self.configuration.listSections_(): for attr in getattr(self.configuration, section).listSections_(): par = (section + '.' + attr) #if the parameter is not know exit, but try to correct it before if not SpellChecker.is_correct(par): msg = 'The parameter %s is not known.' % par msg += '' if SpellChecker.correct( par ) == par else ' Did you mean %s?' % SpellChecker.correct( par) raise ConfigurationException(msg) #usertarball and cmsswconfig use this parameter and we should set it up in a correct way self.configuration.General.serverUrl = self.serverurl #delegating the proxy (creation done in SubCommand) self.voRole = getattr(self.configuration.User, "voRole", "") self.voGroup = getattr(self.configuration.User, "voGroup", "") self.handleProxy() uniquerequestname = None self.logger.debug("Working on %s" % str(requestarea)) configreq = {} for param in self.requestmapper: mustbetype = getattr(types, self.requestmapper[param]['type']) if self.requestmapper[param]['config']: attrs = self.requestmapper[param]['config'].split('.') temp = self.configuration for attr in attrs: temp = getattr(temp, attr, None) if temp is None: break if temp: if mustbetype == type(temp): configreq[param] = temp else: raise ConfigurationException(1, "Invalid type " + str(type(temp)) + " for parameter " + self.requestmapper[param]['config'] \ + ". It is needed a " + str(mustbetype) + ".") elif self.requestmapper[param]['default'] is not None: configreq[param] = self.requestmapper[param]['default'] elif self.requestmapper[param]['required']: raise ConfigurationException( 1, "Missing parameter " + self.requestmapper[param]['config'] + " from the configuration.") else: ## parameter not strictly required pass if param == "workflow": if mustbetype == type(requestname): configreq["workflow"] = requestname elif param == "savelogsflag": configreq["savelogsflag"] = 1 if temp else 0 elif param == "publication": configreq["publication"] = 1 if temp else 0 elif param == "blacklistT1": blacklistT1 = self.voRole != 't1access' #if the user choose to remove the automatic T1 blacklisting and has not the t1acces role if getattr(self.configuration.Site, 'removeT1Blacklisting', False) and blacklistT1: self.logger.info( "WARNING: You disabled the T1 automatic blacklisting without having the t1access role" ) blacklistT1 = False configreq["blacklistT1"] = 1 if blacklistT1 else 0 jobconfig = {} self.configuration.JobType.proxyfilename = self.proxyfilename self.configuration.JobType.capath = HTTPRequests.getCACertPath() pluginParams = [ self.configuration, self.logger, os.path.join(requestarea, 'inputs') ] if getattr(self.configuration.JobType, 'pluginName', None) is not None: jobtypes = getJobTypes() plugjobtype = jobtypes[upper( self.configuration.JobType.pluginName)](*pluginParams) inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq) else: fullname = self.configuration.JobType.externalPluginFile basename = os.path.basename(fullname).split('.')[0] plugin = addPlugin(fullname)[basename] pluginInst = plugin(*pluginParams) inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq) configreq['publishname'] = "%s-%s" % (configreq['publishname'], isbchecksum) configreq.update(jobconfig) server = HTTPRequests(self.serverurl, self.proxyfilename) self.logger.info("Sending the request to the server") self.logger.debug("Submitting %s " % str(configreq)) dictresult, status, reason = server.put( self.uri, data=self._encodeRequest(configreq)) self.logger.debug("Result: %s" % dictresult) if status != 200: msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % ( str(configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) elif dictresult.has_key("result"): uniquerequestname = dictresult["result"][0]["RequestName"] else: msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \ % (str(configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) tmpsplit = self.serverurl.split(':') createCache(requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit) > 1 else '', uniquerequestname, voRole=self.voRole, voGroup=self.voGroup) self.logger.info("Submission completed") self.logger.debug("Request ID: %s " % uniquerequestname) self.logger.debug("Ended submission") return uniquerequestname
def __call__(self): valid = False configmsg = 'Default' self.logger.debug("Started submission") serverFactory = CRABClient.Emulator.getEmulator('rest') # Get some debug parameters ######### Check if the user provided unexpected parameters ######## #init the dictionary with all the known parameters all_config_params = [ x for x in parameters_mapping['other-config-params'] ] for _, val in parameters_mapping['on-server'].iteritems(): if val['config']: all_config_params.extend(val['config']) SpellChecker.DICTIONARY = SpellChecker.train(all_config_params) #iterate on the parameters provided by the user for section in self.configuration.listSections_(): for attr in getattr(self.configuration, section).listSections_(): par = (section + '.' + attr) #if the parameter is not know exit, but try to correct it before if not SpellChecker.is_correct(par): msg = 'The parameter %s is not known.\nPlease refer to <https://twiki.cern.ch/twiki/bin/view/CMSPublic/WorkBookCRAB3Tutorial#CRAB_configuration_parameters> for list of valid parameter.\nSee the ./crab.log file for more details"' % par msg += '' if SpellChecker.correct( par ) == par else '\nOr maybe did you mean %s?' % SpellChecker.correct( par) raise ConfigurationException(msg) #usertarball and cmsswconfig use this parameter and we should set it up in a correct way self.configuration.General.serverUrl = self.serverurl uniquerequestname = None self.logger.debug("Working on %s" % str(self.requestarea)) configreq = {} for param in parameters_mapping['on-server']: mustbetype = getattr( types, parameters_mapping['on-server'][param]['type']) default = parameters_mapping['on-server'][param]['default'] config_params = parameters_mapping['on-server'][param]['config'] for config_param in config_params: attrs = config_param.split('.') temp = self.configuration for attr in attrs: temp = getattr(temp, attr, None) if temp is None: break if temp is not None: configreq[param] = temp break elif default is not None: configreq[param] = default temp = default else: ## Parameter not strictly required. pass ## Check that the requestname is of the right type. ## This is not checked in SubCommand.validateConfig(). if param == 'workflow': if mustbetype == type(self.requestname): configreq['workflow'] = self.requestname ## Translate boolean flags into integers. elif param in [ 'savelogsflag', 'publication', 'nonprodsw', 'useparent', 'ignorelocality', 'saveoutput', 'oneEventMode' ]: configreq[param] = 1 if temp else 0 ## Translate DBS URL aliases into DBS URLs. elif param in ['dbsurl', 'publishdbsurl']: if param == 'dbsurl': dbstype = 'reader' elif param == 'publishdbsurl': dbstype = 'writer' allowed_dbsurls = DBSURLS[dbstype].values() allowed_dbsurls_aliases = DBSURLS[dbstype].keys() if configreq[param] in allowed_dbsurls_aliases: configreq[param] = DBSURLS[dbstype][configreq[param]] elif configreq[param].rstrip('/') in allowed_dbsurls: configreq[param] = configreq[param].rstrip('/') elif param == 'scriptexe' and 'scriptexe' in configreq: configreq[param] = os.path.basename(configreq[param]) jobconfig = {} self.configuration.JobType.proxyfilename = self.proxyfilename self.configuration.JobType.capath = serverFactory.getCACertPath() #get the backend URLs from the server external configuration serverBackendURLs = server_info('backendurls', self.serverurl, self.proxyfilename, getUrl(self.instance, resource='info')) #if cacheSSL is specified in the server external configuration we will use it to upload the sandbox (baseURL will be ignored) self.configuration.JobType.filecacheurl = serverBackendURLs[ 'cacheSSL'] if 'cacheSSL' in serverBackendURLs else None pluginParams = [ self.configuration, self.logger, os.path.join(self.requestarea, 'inputs') ] crab_job_types = getJobTypes() if upper(configreq['jobtype']) in crab_job_types: plugjobtype = crab_job_types[upper( configreq['jobtype'])](*pluginParams) inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq) else: fullname = configreq['jobtype'] basename = os.path.basename(fullname).split('.')[0] plugin = addPlugin(fullname)[basename] pluginInst = plugin(*pluginParams) inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq) if configreq['publication']: non_edm_files = jobconfig['tfileoutfiles'] + jobconfig[ 'addoutputfiles'] if non_edm_files: msg = "%sWARNING%s: The following output files will not be published, as they are not EDM files: %s" % ( colors.RED, colors.NORMAL, non_edm_files) self.logger.warning(msg) if not configreq['publishname']: configreq['publishname'] = isbchecksum else: configreq['publishname'] = "%s-%s" % (configreq['publishname'], isbchecksum) configreq.update(jobconfig) server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) self.logger.info("Sending the request to the server") self.logger.debug("Submitting %s " % str(configreq)) ## TODO: this shouldn't be hard-coded. listParams = ['adduserfiles', 'addoutputfiles', 'sitewhitelist', 'siteblacklist', 'blockwhitelist', 'blockblacklist', \ 'tfileoutfiles', 'edmoutfiles', 'runs', 'lumis', 'userfiles', 'scriptargs', 'extrajdl'] configreq_encoded = self._encodeRequest(configreq, listParams) self.logger.debug('Encoded submit request: %s' % (configreq_encoded)) dictresult, status, reason = server.put(self.uri, data=configreq_encoded) self.logger.debug("Result: %s" % dictresult) if status != 200: msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % ( str(configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) elif dictresult.has_key("result"): uniquerequestname = dictresult["result"][0]["RequestName"] else: msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \ % (str(configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) tmpsplit = self.serverurl.split(':') createCache(self.requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit) > 1 else '', uniquerequestname, voRole=self.voRole, voGroup=self.voGroup, instance=self.instance, originalConfig=self.configuration) self.logger.info( "%sSuccess%s: Your task has been delivered to the CRAB3 server." % (colors.GREEN, colors.NORMAL)) if not self.options.wait: self.logger.info("Task name: %s" % uniquerequestname) self.logger.info( "Please use 'crab status' to check how the submission process proceed" ) if self.options.wait: self.checkStatusLoop(server, uniquerequestname) self.logger.debug("About to return") return { 'requestname': self.requestname, 'uniquerequestname': uniquerequestname }
def __call__(self): valid = False configmsg = 'Default' if not os.path.isfile(self.options.config): raise MissingOptionException("Configuration file '%s' not found" % self.options.config) self.logger.debug("Started submission") ######### Check if the user provided unexpected parameters ######## #init the dictionary with all the known parameters SpellChecker.DICTIONARY = SpellChecker.train( [ val['config'] for _, val in self.requestmapper.iteritems() if val['config'] ] + \ [ x for x in self.otherConfigParams ] ) #iterate on the parameters provided by the user for section in self.configuration.listSections_(): for attr in getattr(self.configuration, section).listSections_(): par = (section + '.' + attr) #if the parameter is not know exit, but try to correct it before if not SpellChecker.is_correct( par ): msg = 'The parameter %s is not known.' % par msg += '' if SpellChecker.correct(par) == par else ' Did you mean %s?' % SpellChecker.correct(par) raise ConfigurationException(msg) #usertarball and cmsswconfig use this parameter and we should set it up in a correct way self.configuration.General.serverUrl = self.serverurl uniquerequestname = None self.logger.debug("Working on %s" % str(self.requestarea)) configreq = {} for param in self.requestmapper: mustbetype = getattr(types, self.requestmapper[param]['type']) if self.requestmapper[param]['config']: attrs = self.requestmapper[param]['config'].split('.') temp = self.configuration for attr in attrs: temp = getattr(temp, attr, None) if temp is None: break if temp: if mustbetype == type(temp): configreq[param] = temp else: raise ConfigurationException("Invalid type " + str(type(temp)) + " for parameter " + self.requestmapper[param]['config'] \ + ". It is needed a " + str(mustbetype) + ".") elif self.requestmapper[param]['default'] is not None: configreq[param] = self.requestmapper[param]['default'] elif self.requestmapper[param]['required']: raise ConfigurationException("Missing parameter " + self.requestmapper[param]['config'] + " from the configuration.") else: ## parameter not strictly required pass if param == "workflow": if mustbetype == type(self.requestname): configreq["workflow"] = self.requestname elif param == "savelogsflag": configreq["savelogsflag"] = 1 if temp else 0 elif param == "publication": configreq["publication"] = 1 if temp else 0 elif param == "blacklistT1": blacklistT1 = self.voRole != 't1access' #if the user choose to remove the automatic T1 blacklisting and has not the t1acces role if getattr (self.configuration.Site, 'removeT1Blacklisting', False) and blacklistT1: self.logger.info("WARNING: You disabled the T1 automatic blacklisting without having the t1access role") blacklistT1 = False configreq["blacklistT1"] = 1 if blacklistT1 else 0 jobconfig = {} self.configuration.JobType.proxyfilename = self.proxyfilename self.configuration.JobType.capath = HTTPRequests.getCACertPath() #get the backend URLs from the server external configuration serverBackendURLs = server_info('backendurls', self.serverurl, self.proxyfilename, self.getUrl(self.instance, resource='info')) #if cacheSSL is specified in the server external configuration we will use it to upload the sandbox (baseURL will be ignored) self.configuration.JobType.filecacheurl = serverBackendURLs['cacheSSL'] if 'cacheSSL' in serverBackendURLs else None #otherwise we will contact the baseurl to get the cache hostname self.configuration.JobType.baseurl = serverBackendURLs['baseURL'] pluginParams = [ self.configuration, self.logger, os.path.join(self.requestarea, 'inputs') ] if getattr(self.configuration.JobType, 'pluginName', None) is not None: jobtypes = getJobTypes() plugjobtype = jobtypes[upper(self.configuration.JobType.pluginName)](*pluginParams) inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq) else: fullname = self.configuration.JobType.externalPluginFile basename = os.path.basename(fullname).split('.')[0] plugin = addPlugin(fullname)[basename] pluginInst = plugin(*pluginParams) inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq) if not configreq['publishname']: configreq['publishname'] = isbchecksum else: configreq['publishname'] = "%s-%s" %(configreq['publishname'], isbchecksum) configreq.update(jobconfig) server = HTTPRequests(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) self.logger.info("Sending the request to the server") self.logger.debug("Submitting %s " % str( configreq ) ) dictresult, status, reason = server.put( self.uri, data = self._encodeRequest(configreq) ) self.logger.debug("Result: %s" % dictresult) if status != 200: msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (str(configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) elif dictresult.has_key("result"): uniquerequestname = dictresult["result"][0]["RequestName"] else: msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \ % (str(configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) tmpsplit = self.serverurl.split(':') createCache(self.requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit)>1 else '', uniquerequestname, voRole=self.voRole, voGroup=self.voGroup, instance=self.instance) self.logger.info("Submission completed") self.logger.debug("Request ID: %s " % uniquerequestname) self.logger.debug("Ended submission") return uniquerequestname