def validateOptions(self): """ After doing the general options validation from the parent SubCommand class, do the validation of options that are specific to the submit command. """ ## First call validateOptions() from the SubCommand class. SubCommand.validateOptions(self) ## If no configuration file was passed as an option, try to extract it from the arguments. ## Assume that the arguments can only be: ## 1) the configuration file name, and ## 2) parameters to override in the configuration file. ## The last ones should all contain an '=' sign, so these are not candidates to be the ## configuration file argument. Also, the configuration file name should end with '.py'. ## If can not find a configuration file candidate, use the default 'crabConfig.py'. ## If find more than one candidate, raise ConfigurationException. if self.options.config is None: use_default = True if len(self.args): config_candidates = [(arg,i) for i,arg in enumerate(self.args) if '=' not in arg and arg[-3:] == '.py'] config_candidate_names = set([config_candidate_name for (config_candidate_name,_) in config_candidates]) if len(config_candidate_names) == 1: self.options.config = config_candidates[0][0] del self.args[config_candidates[0][1]] use_default = False elif len(config_candidate_names) > 1: self.logger.info('Unable to unambiguously extract the configuration file from the command-line arguments.') self.logger.info('Possible candidates are: %s' % list(config_candidate_names)) raise ConfigurationException('ERROR: Unable to extract configuration file from command-line arguments.') if use_default: self.options.config = 'crabConfig.py'
def __init__(self, logger, cmdargs=None): SubCommand.__init__(self, logger, cmdargs) self.phedex = PhEDEx( {"cert": self.proxyfilename, "key": self.proxyfilename, "logger": self.logger, "pycurl": True} ) self.lfnsaddprefix = None self.filename = None
def validateOptions(self): SubCommand.validateOptions(self) #check the format of jobids self.jobids = '' if getattr(self.options, 'jobids', None): self.jobids = validateJobids(self.options.jobids)
def validateOptions(self): SubCommand.validateOptions(self) if self.options.sitename is None: msg = "%sError%s: Please specify the site where to check the write permissions." % (colors.RED, colors.NORMAL) msg += " Use the --site option." ex = MissingOptionException(msg) ex.missingOption = "sitename" raise ex if hasattr(self.options, 'command') and self.options.command != None: AvailableCommands = ['LCG', 'GFAL'] self.command = self.options.command.upper() if self.command not in AvailableCommands: msg = "You specified to use %s command and it is not allowed. Available commands are: %s " % (self.command, str(AvailableCommands)) ex = ConfigurationException(msg) raise ex else: self.command = None if hasattr(self.options, 'checksum'): if re.match('^yes$|^no$', self.options.checksum): self.checksum = 'ADLER32' if self.options.checksum == 'yes' else None else: msg = "You specified to use %s checksum. Only lowercase yes/no is accepted to turn ADLER32 checksum" % self.options.checksum ex = ConfigurationException(msg) raise ex
def validateOptions(self): """ Check if the sitelist parameter is a comma separater list of cms sitenames, and put the strings to be passed to the server to self """ SubCommand.validateOptions(self) #Checking if the sites provided by the user are valid cmsnames. Doing this because with only the #server error handling we get: # Server answered with: Invalid input parameter # Reason is: Incorrect 'siteblacklist' parameter #which is not really user friendly. #Moreover, I prefer to be independent from Lexicon. I'll the regex here. sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression sn_rec = re.compile(sn_re) #sn_rec => SiteName_RegularExpressionCompiled self.sitewhitelist = '' self.siteblsklist = '' for siteList in ['sitewhitelist', 'siteblacklist']: result = '' paramVal = getattr(self.options, siteList, None) if paramVal: for site in paramVal.split(','): if not sn_rec.match(site): raise ConfigException("The sitename %s dows not look like a valid CMS name (not matching %s)" % (site, sn_re) ) result += "&%s=%s" % (siteList, site) setattr(self, siteList, result) #check the format of jobids self.jobids = '' if getattr(self.options, 'jobids', None): self.jobids = validateJobids(self.options.jobids)
def validateOptions(self): SubCommand.validateOptions(self) if self.options.jobid is not None: try: int(self.options.jobid) except ValueError: raise ClientException("The --jobid option has to be an integer")
def validateOptions(self): """ After doing the general options validation from the parent SubCommand class, do the validation of options that are specific to the submit command. """ ## First do the basic validation in the SubCommand. SubCommand.validateOptions(self) validateSubmitOptions(self.options, self.args)
def validateOptions(self): SubCommand.validateOptions(self) if self.options.jobid is not None: try: int(self.options.jobid) except ValueError: raise ClientException( "The --jobid option has to be an integer")
def validateOptions(self): """ Check if the sitelist parameter is a comma separater list of cms sitenames, and put the strings to be passed to the server to self """ SubCommand.validateOptions(self) #Checking if the sites provided by the user are valid cmsnames. Doing this because with only the #server error handling we get: # Server answered with: Invalid input parameter # Reason is: Incorrect 'siteblacklist' parameter #which is not really user friendly. #Moreover, I prefer to be independent from Lexicon. I'll the regex here. sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression sn_rec = re.compile(sn_re) #sn_rec => SiteName_RegularExpressionCompiled self.sitewhitelist = '' self.siteblsklist = '' for siteList in ['sitewhitelist', 'siteblacklist']: result = '' paramVal = getattr(self.options, siteList, None) if paramVal: for site in paramVal.split(','): if not sn_rec.match(site): raise ConfigException("The sitename %s dows not look like a valid CMS name (not matching %s)" % (site, sn_re) ) result += "&%s=%s" % (siteList, site) setattr(self, siteList, result) #check the format of jobids self.jobids = '' if getattr(self.options, 'jobids', None): self.jobids = validateJobids(self.options.jobids) # Sanity checks for task sizes. Limits are purposely fairly generous to provide some level of future-proofing. # The server may restrict further. self.numcores = None if self.options.numcores != None: if self.options.numcores < 1 or self.options.numcores > 128: raise ConfigException("The number of requested cores (%d) must be between 1 and 128." % (self.options.numcores)) self.numcores = str(self.options.numcores) self.maxjobruntime = None if self.options.maxjobruntime != None: if self.options.maxjobruntime < 1 or self.options.maxjobruntime > 336: raise ConfigException("The requested max job runtime (%d hours) must be between 1 and 336 hours." % self.options.maxjobruntime) self.maxjobruntime = str(self.options.maxjobruntime) self.maxmemory = None if self.options.maxmemory != None: if self.options.maxmemory < 30 or self.options.maxmemory > 1024*30: raise ConfigException("The requested per-job memory (%d MB) must be between 30 and 30720 MB." % self.options.maxmemory) self.maxmemory = str(self.options.maxmemory) self.priority = None if self.options.priority != None: self.priority = str(self.options.priority)
def validateOptions(self): SubCommand.validateOptions(self) if self.options.idle and (self.options.long or self.options.summary ): raise ConfigurationException("Idle option (-i) conflicts with -u, and -l") self.long = self.options.long self.json = self.options.json self.summary = self.options.summary self.idle = self.options.idle
def validateOptions(self): SubCommand.validateOptions(self) if self.options.sitename is None: msg = "%sError%s: Please specify the site where to check the write permissions." % (colors.RED, colors.NORMAL) msg += " Use the --site option." ex = MissingOptionException(msg) ex.missingOption = "sitename" raise ex
def validateOptions(self): """ Check if the output file is given and set as attribute """ SubCommand.validateOptions(self) if not re.match('^yes$|^no$', self.options.usedbs): raise ConfigurationException("--dbs option only accepts the yes and no values (--dbs=yes or --dbs=no)") self.usedbs = 1 if self.options.usedbs=='yes' else 0 setattr(self, 'outdir', getattr(self.options, 'outdir', None))
def __init__(self, logger, cmdargs=None): SubCommand.__init__(self, logger, cmdargs) self.phedex = PhEDEx({ "cert": self.proxyfilename, "key": self.proxyfilename, "logger": self.logger, "pycurl": True }) self.lfnsaddprefix = None self.filename = None
def __init__(self, logger, cmdargs=None): """ Class constructor. """ ## These parameters are defined in setOptions(). They correspond to what ## the user passed in the corresponding command line options. self.jobids = None self.sitewhitelist = None self.siteblacklist = None SubCommand.__init__(self, logger, cmdargs)
def __init__(self, logger, cmdargs = None): """ Class constructor. """ ## These parameters are defined in setOptions(). They correspond to what ## the user passed in the corresponding command line options. self.jobids = None self.sitewhitelist = None self.siteblacklist = None SubCommand.__init__(self, logger, cmdargs)
def validateOptions(self): SubCommand.validateOptions(self) if self.options.idle and (self.options.long or self.options.summary): raise ConfigurationException("Option --idle conflicts with --summary and --long") if self.options.sort is not None: sortOpts = ["state", "site", "runtime", "memory", "cpu", "retries", "waste", "exitcode"] if self.options.sort not in sortOpts: msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Only the following values are accepted for --sort option: %s" % (sortOpts) raise ConfigurationException(msg)
def validateOptions(self): """ Check if the output file is given and set as attribute """ SubCommand.validateOptions(self) if not re.match('^yes$|^no$', self.options.usedbs): raise ConfigurationException( "--dbs option only accepts the yes and no values (--dbs=yes or --dbs=no)" ) self.usedbs = 1 if self.options.usedbs == 'yes' else 0 setattr(self, 'outdir', getattr(self.options, 'outdir', None))
def validateOptions(self): SubCommand.validateOptions(self) if self.options.idle and (self.options.long or self.options.summary or self.options.publication): raise ConfigurationException("Idle option (-i) conflicts with -u, -l, and -p") if self.options.publication and (self.options.long or self.options.summary or self.options.idle): raise ConfigurationException("Publication option (-p) conflicts with -u, -l, and -i") self.long = self.options.long self.json = self.options.json self.summary = self.options.summary self.idle = self.options.idle self.publication = self.options.publication
def validateOptions(self): ## Do the options validation from SubCommand. try: SubCommand.validateOptions(self) except MissingOptionException as ex: if ex.missingOption == "task": msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Please provide a path to a CRAB project directory (use the -d/--dir option)" msg += " or to a log file (use the --logpath option)." ex = MissingOptionException(msg) ex.missingOption = "task" raise ex
def __init__(self, logger, cmdargs = None): """ Class constructor. """ self.jobids = None self.sitewhitelist = None self.siteblacklist = None self.maxjobruntime = None self.maxmemory = None self.numcores = None self.priority = None SubCommand.__init__(self, logger, cmdargs)
def validateOptions(self): """ Check if the sitelist parameter is a comma separater list of cms sitenames, and put the strings to be passed to the server to self """ SubCommand.validateOptions(self) ## Check the format of the jobids option. if getattr(self.options, 'jobids'): jobidstuple = validateJobids(self.options.jobids) self.jobids = [str(jobid) for (_, jobid) in jobidstuple] #Checking if the sites provided by the user are valid cmsnames. Doing this because with only the #server error handling we get: # Server answered with: Invalid input parameter # Reason is: Incorrect 'siteblacklist' parameter #which is not really user friendly. #Moreover, I prefer to be independent from Lexicon. I'll the regex here. sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression sn_rec = re.compile(sn_re) #sn_rec => SiteName_RegularExpressionCompiled for sitelist in ['sitewhitelist', 'siteblacklist']: if getattr(self.options, sitelist) is not None: for i, site_name in enumerate(getattr(self.options, sitelist).split(',')): if not sn_rec.match(site_name): msg = "The site name %s does not look like a valid CMS site name" % (site_name) msg += " (it is not matching the regular expression %s)." % (sn_re) raise ConfigurationException(msg) setattr(self, sitelist, getattr(self.options, sitelist).split(',')) ## Sanity checks for task sizes. Limits are purposely fairly generous to provide ## some level of future-proofing. The server may restrict further. if self.options.maxjobruntime is not None: if self.options.maxjobruntime < 60 or self.options.maxjobruntime > 336*60: msg = "The requested maximum job runtime (%d minutes) must be between 60 and 20160 minutes." % (self.options.maxjobruntime) raise ConfigurationException(msg) self.maxjobruntime = str(self.options.maxjobruntime) if self.options.maxmemory is not None: if self.options.maxmemory < 30 or self.options.maxmemory > 1024*30: msg = "The requested per-job memory (%d MB) must be between 30 and 30720 MB." % (self.options.maxmemory) raise ConfigurationException(msg) self.maxmemory = str(self.options.maxmemory) if self.options.numcores is not None: if self.options.numcores < 1 or self.options.numcores > 128: msg = "The requested number of cores (%d) must be between 1 and 128." % (self.options.numcores) raise ConfigurationException(msg) self.numcores = str(self.options.numcores) if self.options.priority is not None: self.priority = str(self.options.priority)
def validateOptions(self): """ Check if the output file is given and set as attribute """ SubCommand.validateOptions(self) if self.options.usedbs is not None: msg = "CRAB command option error: the option --dbs has been deprecated since CRAB v3.3.1603." raise UnknownOptionException(msg) recoveryMethods = ['notFinished', 'notPublished', 'failed'] if self.options.recovery not in recoveryMethods: msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " The --recovery option only accepts the following values: %s" % (recoveryMethods) raise ConfigurationException(msg)
def validateOptions(self): SubCommand.validateOptions(self) if self.options.idle and (self.options.long or self.options.summary): raise ConfigurationException( "Option --idle conflicts with --summary and --long") if self.options.sort is not None: sortOpts = [ "state", "site", "runtime", "memory", "cpu", "retries", "waste", "exitcode" ] if self.options.sort not in sortOpts: msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Only the following values are accepted for --sort option: %s" % ( sortOpts) raise ConfigurationException(msg)
def validateOptions(self): """ Check if the output file is given and set as attribute """ SubCommand.validateOptions(self) if self.options.usedbs is not None: msg = "CRAB command option error: the option --dbs has been deprecated since CRAB v3.3.1603." raise UnknownOptionException(msg) recoveryMethods = ['notFinished', 'notPublished', 'failed'] if self.options.recovery not in recoveryMethods: msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " The --recovery option only accepts the following values: %s" % ( recoveryMethods) raise ConfigurationException(msg)
def validateOptions(self): #Figuring out the destination directory SubCommand.validateOptions(self) self.dest = None if self.options.outputpath is not None: if not os.path.isabs( self.options.outputpath ): self.dest = os.path.abspath( self.options.outputpath ) else: self.dest = self.options.outputpath #convert all to -1 if getattr(self.options, 'quantity', None) == 'all': self.options.quantity = -1 #check the format of jobids if getattr(self.options, 'jobids', None): self.options.jobids = validateJobids(self.options.jobids)
def validateOptions(self): SubCommand.validateOptions(self) if self.options.idle and (self.options.long or self.options.summary ): raise ConfigurationException("Idle option (-i) conflicts with -u, and -l") self.json = self.options.json self.summary = self.options.summary self.idle = self.options.idle self.long = self.options.long acceptedsort = ["state", "site", "runtime", "memory", "cpu", "retries", "waste"] if hasattr(self.options , 'sort') and self.options.sort != None: if not self.long: raise ConfigurationException('%sError%s: Please use option --long together with --sort' % (colors.RED, colors.NORMAL)) elif not self.options.sort in acceptedsort: raise ConfigurationException('%sError%s: Only this value accepted for crab status sort: %s' % (colors.RED, colors.NORMAL, acceptedsort)) else: self.sort = self.options.sort
def validateOptions(self): """ After doing the general options validation from the parent SubCommand class, do the validation of options that are specific to the submit command. """ ## First call validateOptions() from the SubCommand class. SubCommand.validateOptions(self) ## If no configuration file was passed as an option, try to extract it from the arguments. ## Assume that the arguments can only be: ## 1) the configuration file name, and ## 2) parameters to override in the configuration file. ## The last ones should all contain an '=' sign, so these are not candidates to be the ## configuration file argument. Also, the configuration file name should end with '.py'. ## If can not find a configuration file candidate, use the default 'crabConfig.py'. ## If find more than one candidate, raise ConfigurationException. if self.options.config is None: use_default = True if len(self.args): config_candidates = [(arg, i) for i, arg in enumerate(self.args) if '=' not in arg and arg[-3:] == '.py'] config_candidate_names = set([ config_candidate_name for (config_candidate_name, _) in config_candidates ]) if len(config_candidate_names) == 1: self.options.config = config_candidates[0][0] del self.args[config_candidates[0][1]] use_default = False elif len(config_candidate_names) > 1: self.logger.info( 'Unable to unambiguously extract the configuration file from the command-line arguments.' ) self.logger.info('Possible candidates are: %s' % list(config_candidate_names)) raise ConfigurationException( 'ERROR: Unable to extract configuration file from command-line arguments.' ) if use_default: self.options.config = 'crabConfig.py'
def validateOptions(self): #Figuring out the destination directory SubCommand.validateOptions(self) self.dest = None if self.options.outputpath is not None: if not os.path.isabs( self.options.outputpath ): self.dest = os.path.abspath( self.options.outputpath ) else: self.dest = self.options.outputpath #convert all to -1 if getattr(self.options, 'quantity', None) == 'all': self.options.quantity = -1 #check the format of jobids if getattr(self.options, 'jobids', None): if re.compile('^\d+(,\d+)*$').match(self.options.jobids): self.options.jobids = [('jobids',jobid) for jobid in self.options.jobids.split(',')] else: raise ConfigurationException("The command line option jobids should be a comma separated list of integers")
def validateOptions(self): # Figuring out the destination directory SubCommand.validateOptions(self) self.dest = None if self.options.outputpath is not None: if re.match("^[a-z]+://", self.options.outputpath): self.dest = self.options.outputpath elif not os.path.isabs(self.options.outputpath): self.dest = os.path.abspath(self.options.outputpath) else: self.dest = self.options.outputpath # convert all to -1 if getattr(self.options, "quantity", None) == "all": self.options.quantity = -1 # check the format of jobids if getattr(self.options, "jobids", None): self.options.jobids = validateJobids(self.options.jobids) self.dump = self.options.dump
def validateOptions(self): #Figuring out the destination directory SubCommand.validateOptions(self) self.dest = None if self.options.outputpath is not None: if re.match("^[a-z]+://", self.options.outputpath): self.dest = self.options.outputpath elif not os.path.isabs(self.options.outputpath): self.dest = os.path.abspath(self.options.outputpath) else: self.dest = self.options.outputpath #convert all to -1 if getattr(self.options, 'quantity', None) == 'all': self.options.quantity = -1 #check the format of jobids if getattr(self.options, 'jobids', None): self.options.jobids = validateJobids(self.options.jobids) self.dump = self.options.dump
def validateOptions(self): ## Check if the --logpath option was used. If it was, don't require the task ## option. if self.options.logpath is not None: if not os.path.isfile(self.options.logpath): msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Could not find the log file %s." % (self.options.logpath) raise ConfigurationException(msg) self.cmdconf["requiresDirOption"] = False ## Do the options validation from SubCommand. try: SubCommand.validateOptions(self) except MissingOptionException as ex: if ex.missingOption == "task": msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Please provide a path to a CRAB project directory (use the -d/--dir option)" msg += " or to a log file (use the --logpath option)." ex = MissingOptionException(msg) ex.missingOption = "task" raise ex
def validateOptions(self): ## Check if the --logpath option was used. If it was, don't require the task ## option. if self.options.logpath is not None: if not os.path.isfile(self.options.logpath): msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Could not find the log file %s." % ( self.options.logpath) raise ConfigurationException(msg) self.cmdconf['requiresDirOption'] = False ## Do the options validation from SubCommand. try: SubCommand.validateOptions(self) except MissingOptionException as ex: if ex.missingOption == "task": msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Please provide a path to a CRAB project directory (use the -d/--dir option)" msg += " or to a log file (use the --logpath option)." ex = MissingOptionException(msg) ex.missingOption = "task" raise ex
def validateOptions(self): SubCommand.validateOptions(self) if hasattr(self.options, 'logpath') and self.options.logpath != None: if not path.isfile(self.options.logpath): msg = '%sError%s: Could not find the log file in the path: %s' % (colors.RED,colors.NORMAL,self.options.logpath) raise ConfigurationException(msg) elif hasattr(self.options, 'task'): self.cmdconf['requiresTaskOption'] = True self.cmdconf['useCache'] = True if self.options.task == None: if len(self.args) == 1 and self.args[0]: self.options.task = self.args[0] #for the case of 'crab uploadlog', user is using the .crab3 file if self.options.task == None and hasattr(self, 'crab3dic'): if self.crab3dic["taskname"] != None: self.options.task = self.crab3dic["taskname"] else: msg = '%sError%s: Please use the directory option -d or --logpath to specify which log to upload' % (colors.RED, colors.NORMAL) raise ConfigurationException(msg) else: msg = '%sError%s: Please use the directory option -d or --logpath to specify which log to upload' % (colors.RED, colors.NORMAL) raise ConfigurationException(msg)
def validateOptions(self): SubCommand.validateOptions(self) if self.options.sort is not None: sortOpts = [ "state", "site", "runtime", "memory", "cpu", "retries", "waste", "exitcode" ] if self.options.sort not in sortOpts: msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Only the following values are accepted for --sort option: %s" % ( sortOpts) raise ConfigurationException(msg) if self.options.jobids: jobidstuple = validateJobids(self.options.jobids) self.jobids = [str(jobid) for (_, jobid) in jobidstuple] if self.options.jobids and not (self.options.long or self.options.sort): raise ConfigurationException( "Parameter --jobids can only be used in combination " "with --long or --sort options.")
def validateOptions(self): #Figuring out the destination directory SubCommand.validateOptions(self) self.dest = None if self.options.outputpath is not None: if re.match("^[a-z]+://", self.options.outputpath): self.dest = self.options.outputpath elif not os.path.isabs(self.options.outputpath): self.dest = os.path.abspath(self.options.outputpath) else: self.dest = self.options.outputpath #convert all to -1 if getattr(self.options, 'quantity', None) == 'all': self.options.quantity = -1 #check the format of jobids if getattr(self.options, 'jobids', None): self.options.jobids = validateJobids(self.options.jobids) if hasattr(self.options, 'command') and self.options.command != None: AvailableCommands = ['LCG', 'GFAL'] self.command = self.options.command.upper() if self.command not in AvailableCommands: msg = "You specified to use %s command and it is not allowed. Available commands are: %s " % ( self.command, str(AvailableCommands)) ex = ConfigurationException(msg) raise ex else: self.command = None if hasattr(self.options, 'checksum'): if re.match('^yes$|^no$', self.options.checksum): self.checksum = 'ADLER32' if self.options.checksum == 'yes' else None else: msg = "You specified to use %s checksum. Only lowercase yes/no is accepted to turn ADLER32 checksum" % self.options.checksum ex = ConfigurationException(msg) raise ex
def validateOptions(self): #Figuring out the destination directory SubCommand.validateOptions(self) self.dest = None if self.options.outputpath is not None: if re.match("^[a-z]+://", self.options.outputpath): self.dest = self.options.outputpath elif not os.path.isabs( self.options.outputpath ): self.dest = os.path.abspath( self.options.outputpath ) else: self.dest = self.options.outputpath #convert all to -1 if getattr(self.options, 'quantity', None) == 'all': self.options.quantity = -1 #check the format of jobids if getattr(self.options, 'jobids', None): self.options.jobids = validateJobids(self.options.jobids) if hasattr(self.options, 'command') and self.options.command != None: AvailableCommands = ['LCG', 'GFAL'] self.command = self.options.command.upper() if self.command not in AvailableCommands: msg = "You specified to use %s command and it is not allowed. Available commands are: %s " % (self.command, str(AvailableCommands)) ex = ConfigurationException(msg) raise ex else: self.command = None if hasattr(self.options, 'checksum'): if re.match('^yes$|^no$', self.options.checksum): self.checksum = 'ADLER32' if self.options.checksum == 'yes' else None else: msg = "You specified to use %s checksum. Only lowercase yes/no is accepted to turn ADLER32 checksum" % self.options.checksum ex = ConfigurationException(msg) raise ex
def __init__(self, logger, cmdargs = None): SubCommand.__init__(self, logger, cmdargs, disable_interspersed_args = True)
def validateOptions(self): """ Check if the sitelist parameter is a comma separater list of cms sitenames, and put the strings to be passed to the server to self """ SubCommand.validateOptions(self) serverFactory = CRABClient.Emulator.getEmulator('rest') self.server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) uri = getUrl(self.instance, resource='task') crabDBInfo, _, _ = self.server.get(uri, data={ 'subresource': 'search', 'workflow': self.cachedinfo['RequestName'] }) self.splitting = getColumn(crabDBInfo, 'tm_split_algo') if self.options.publication: if self.options.sitewhitelist is not None or self.options.siteblacklist is not None or \ self.options.maxjobruntime is not None or self.options.maxmemory is not None or \ self.options.numcores is not None or self.options.priority is not None: msg = "The options --sitewhitelist, --siteblacklist," msg += " --maxjobruntime, --maxmemory, --numcores and --priority" msg += " can not be specified together with the option --publication." msg += " The last option is to only resubmit (failed) publications," msg += " in which case all of the first options make no sense." raise ConfigurationException(msg) if self.options.jobids: msg = "The option --jobids" msg += " can not be specified together with the option --publication." msg += " The last option is to only resubmit (failed) publications," msg += " which does not allow yet filtering on job ids (ALL failed publications will be resubmitted)." raise ConfigurationException(msg) if self.options.force: msg = "The option --force" msg += " can not be specified together with the option --publication." msg += " The last option is to only resubmit failed publications." msg += " Publications in a status other than 'failed' can not be resubmitted." raise ConfigurationException(msg) ## The --jobids option indicates which jobs have to be resubmitted. If it is not ## given, then all jobs in the task that are not running or successfully ## completed are resubmitted. If the user provides a list of job ids, then also ## successfully completed jobs can be resubmitted. ## Check the format of the jobids option. if self.options.jobids: jobidstuple = validateJobids(self.options.jobids, self.splitting != 'Automatic') self.jobids = [str(jobid) for (_, jobid) in jobidstuple] ## The --force option should not be accepted unless combined with a user-given ## list of job ids via --jobids. if self.options.force and not self.jobids: msg = "Option --force can only be used in combination with option --jobids." raise ConfigurationException(msg) ## Covention used for the job parameters that the user can set when doing job ## resubmission (i.e. siteblacklist, sitewhitelist, maxjobruntime, maxmemory, ## numcores and priority): ## - If the user doesn't set a parameter we don't pass it to the server and the ## the server copies the original value the parameter had at task submission. ## It copies it from the Task DB. Therefore we need to keep these parameters ## in separate columns of the Task DB containing their original values. ## - For the site black- and whitelists, if the user passes an empty string, ## e.g. --siteblacklist='', we pass to the server siteblacklist=empty and the ## server interprets this as and empty list ([]). If the user passes a given ## list of sites, this new list overwrittes the original one. ## - The values of the parameters are used only for the resubmitted jobs (for ## their first resubmission and all next automatic resubmissions). #Checking if the sites provided by the user are valid cmsnames. Doing this because with only the #server error handling we get: # Server answered with: Invalid input parameter # Reason is: Incorrect 'siteblacklist' parameter #which is not really user friendly. #Moreover, I prefer to be independent from Lexicon. I'll the regex here. sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression sn_rec = re.compile( sn_re) #sn_rec => SiteName_RegularExpressionCompiled for sitelist in ['sitewhitelist', 'siteblacklist']: if getattr(self.options, sitelist) is not None: if getattr(self.options, sitelist) != "": for site_name in getattr(self.options, sitelist).split(','): if '*' not in site_name and not sn_rec.match( site_name): msg = "The site name '%s' does not look like a valid CMS site name" % ( site_name) msg += " (it is not matching the regular expression '%s')." % ( sn_re) raise ConfigurationException(msg) setattr(self, sitelist, getattr(self.options, sitelist).split(',')) else: setattr(self, sitelist, []) ## Sanity checks for task sizes. Limits are purposely fairly generous to provide ## some level of future-proofing. The server may restrict further. if self.options.maxjobruntime is not None: if self.options.maxjobruntime < 60 or self.options.maxjobruntime > 336 * 60: msg = "The requested maximum job runtime (%d minutes) must be between 60 and 20160 minutes." % ( self.options.maxjobruntime) raise ConfigurationException(msg) if self.options.maxmemory is not None: if self.options.maxmemory < 30 or self.options.maxmemory > 1024 * 30: msg = "The requested per-job memory (%d MB) must be between 30 and 30720 MB." % ( self.options.maxmemory) raise ConfigurationException(msg) if self.options.numcores is not None: if self.options.numcores < 1 or self.options.numcores > 128: msg = "The requested number of cores (%d) must be between 1 and 128." % ( self.options.numcores) raise ConfigurationException(msg) if self.options.priority is not None: if self.options.priority < 1: msg = "The requested priority (%d) must be greater than 0." % ( self.options.priority) raise ConfigurationException(msg)
def validateOptions(self): SubCommand.validateOptions(self)
def validateOptions(self): """ Check if the output file is given and set as attribute """ SubCommand.validateOptions(self) setattr(self, 'outdir', getattr(self.options, 'outdir', None))
def __init__(self, logger, cmdargs=None): SubCommand.__init__(self, logger, cmdargs)
def validateConfig(self): """ __validateConfig__ Checking if needed input parameters are there """ valid, msg = SubCommand.validateConfig(self) if not valid: return False, msg ## Check that Data.unitsPerjob is specified. if hasattr(self.configuration.Data, 'unitsPerJob'): try: float(self.configuration.Data.unitsPerJob) except ValueError: msg = "Invalid CRAB configuration: Parameter Data.unitsPerJob must be a valid number, not %s." % (self.configuration.Data.unitsPerJob) return False, msg ## Check that JobType.pluginName and JobType.externalPluginFile are not both specified. if hasattr(self.configuration.JobType, 'pluginName') and hasattr(self.configuration.JobType, 'externalPluginFile'): msg = "Invalid CRAB configuration: Only one of JobType.pluginName or JobType.externalPluginFile parameters can be specified." pluginName_default = getParamDefaultValue('JobType.pluginName') if pluginName_default: msg += "\nIf neither JobType.pluginName nor JobType.externalPluginFile would be specified," msg += " the default JobType.pluginName = '%s' would be used." % (pluginName_default) return False, msg ## Load the external plugin or check that the crab plugin is valid. external_plugin_name = getattr(self.configuration.JobType, 'externalPluginFile', None) crab_plugin_name = getattr(self.configuration.JobType, 'pluginName', None) crab_job_types = {'ANALYSIS': None, 'PRIVATEMC': None} #getJobTypes() if external_plugin_name: addPlugin(external_plugin_name) # Do we need to do this here? if crab_plugin_name: if upper(crab_plugin_name) not in crab_job_types: msg = "Invalid CRAB configuration: Parameter JobType.pluginName has an invalid value ('%s')." % (crab_plugin_name) msg += "\nAllowed values are: %s." % (", ".join(['%s' % job_type for job_type in crab_job_types.keys()])) return False, msg msg = "Will use CRAB %s plugin" % ("Analysis" if upper(crab_plugin_name) == 'ANALYSIS' else "PrivateMC") msg += " (i.e. will run %s job type)." % ("an analysis" if upper(crab_plugin_name) == 'ANALYSIS' else "a MC generation") self.logger.debug(msg) ## Check that the particular combination (Data.publication = True, General.transferOutputs = False) is not specified. if getattr(self.configuration.Data, 'publication', getParamDefaultValue('Data.publication')) and \ not getattr(self.configuration.General, 'transferOutputs', getParamDefaultValue('General.transferOutputs')): msg = "Invalid CRAB configuration: Data.publication is True, but General.transferOutputs is False." msg += "\nPublication can not be performed if the output files are not transferred to a permanent storage." return False, msg ## Check that a storage site is specified if General.transferOutputs = True or General.transferLogs = True. if not hasattr(self.configuration.Site, 'storageSite'): if getattr(self.configuration.General, 'transferLogs', getParamDefaultValue('General.transferLogs')) or \ getattr(self.configuration.General, 'transferOutputs', getParamDefaultValue('General.transferOutputs')): msg = "Invalid CRAB configuration: Parameter Site.storageSite is missing." return False, msg ## If an input dataset and a DBS URL are specified, check that the DBS URL is a good one. ## Also, if the DBS URL is 'phys0x', check that the input dataset tier is USER. if hasattr(self.configuration.Data, 'inputDBS'): if hasattr(self.configuration.Data, 'inputDataset'): msg = None dbs_urls_aliases = DBSURLS['reader'].keys() dbs_urls = DBSURLS['reader'].values() if (self.configuration.Data.inputDBS not in dbs_urls_aliases) and (self.configuration.Data.inputDBS.rstrip('/') not in dbs_urls): msg = "Invalid CRAB configuration: Parameter Data.inputDBS has an invalid value ('%s')." % (self.configuration.Data.inputDBS) msg += "\nAllowed values are: " msg += "\n ".join(["'%s' ('%s')" % (alias, url) for alias, url in DBSURLS['reader'].iteritems()]) local_dbs_urls_aliases = ['phys01', 'phys02', 'phys03'] local_dbs_urls = [DBSURLS['reader'][alias] for alias in local_dbs_urls_aliases if alias in DBSURLS['reader']] if self.configuration.Data.inputDBS in local_dbs_urls + local_dbs_urls_aliases: inputDataset_parts = self.configuration.Data.inputDataset.split('/') inputDataset_parts.pop(0) inputDataset_tier = inputDataset_parts[-1] if len(inputDataset_parts) == 3 else None user_data_tiers = ['USER'] if inputDataset_tier not in user_data_tiers: msg = "Invalid CRAB configuration: A local DBS instance '%s' was specified for reading an input dataset of tier %s." \ % (self.configuration.Data.inputDBS, inputDataset_tier) msg += "\nDatasets of tier different than %s must be read from the global DBS instance; this is, set Data.inputDBS = 'global'." \ % (", ".join(user_data_tiers[:-1]) + " or " + user_data_tiers[-1] if len(user_data_tiers) > 1 else user_data_tiers[0]) if msg: inputDBS_default = getParamDefaultValue('Data.inputDBS') if inputDBS_default: inputDBS_default, inputDBS_default_alias = self.getDBSURLAndAlias(inputDBS_default, 'reader') if inputDBS_default and inputDBS_default_alias: msg += "\nIf Data.inputDBS would not be specified, the default '%s' ('%s') would be used." % (inputDBS_default_alias, inputDBS_default) return False, msg ## If a publication DBS URL is specified and publication is ON, check that the DBS URL is a good one. if hasattr(self.configuration.Data, 'publishDBS'): if getattr(self.configuration.Data, 'publication', getParamDefaultValue('Data.publication')): dbs_urls = DBSURLS['writer'].values() dbs_urls_aliases = DBSURLS['writer'].keys() if (self.configuration.Data.publishDBS not in dbs_urls_aliases) and (self.configuration.Data.publishDBS.rstrip('/') not in dbs_urls): msg = "Invalid CRAB configuration: Parameter Data.publishDBS has an invalid value ('%s')." % (self.configuration.Data.publishDBS) msg += "\nAllowed values are: " msg += "\n ".join(["'%s' ('%s')" % (alias, url) for alias, url in DBSURLS['writer'].iteritems()]) publishDBS_default = getParamDefaultValue('Data.publishDBS') if publishDBS_default: publishDBS_default, publishDBS_default_alias = self.getDBSURLAndAlias(publishDBS_default, 'writer') if publishDBS_default and publishDBS_default_alias: msg += "\nIf Data.publishDBS would not be specified, the default '%s' ('%s') would be used." \ % (publishDBS_default_alias, publishDBS_default) return False, msg if hasattr(self.configuration.JobType, 'scriptExe'): if not os.path.isfile(self.configuration.JobType.scriptExe): msg = "Cannot find the file %s specified in the JobType.scriptExe configuration parameter." % (self.configuration.JobType.scriptExe) return False, msg return True, "Valid configuration"
def validateOptions(self): SubCommand.validateOptions(self) if not hasattr(self.options, 'sitename') or self.options.sitename is None: self.logger.info("Missing site name, use '--site' option") raise MissingOptionException
def __init__(self, logger, cmdargs=None): SubCommand.__init__(self, logger, cmdargs, disable_interspersed_args=True) self.configreq = None self.configreq_encoded = None
def __init__(self, logger, cmdargs=None): self.jobids = None SubCommand.__init__(self, logger, cmdargs)
def validateOptions(self): """ Check if the sitelist parameter is a comma separater list of cms sitenames, and put the strings to be passed to the server to self """ SubCommand.validateOptions(self) if self.options.publication: if self.options.sitewhitelist is not None or self.options.siteblacklist is not None or \ self.options.maxjobruntime is not None or self.options.maxmemory is not None or \ self.options.numcores is not None or self.options.priority is not None: msg = "The options --sitewhitelist, --siteblacklist," msg += " --maxjobruntime, --maxmemory, --numcores and --priority" msg += " can not be specified together with the option --publication." msg += " The last option is to only resubmit (failed) publications," msg += " in which case all of the first options make no sense." raise ConfigurationException(msg) if self.options.jobids: msg = "The option --jobids" msg += " can not be specified together with the option --publication." msg += " The last option is to only resubmit (failed) publications," msg += " which does not allow yet filtering on job ids (ALL failed publications will be resubmitted)." raise ConfigurationException(msg) if self.options.force: msg = "The option --force" msg += " can not be specified together with the option --publication." msg += " The last option is to only resubmit failed publications." msg += " Publications in a status other than 'failed' can not be resubmitted." raise ConfigurationException(msg) ## The --jobids option indicates which jobs have to be resubmitted. If it is not ## given, then all jobs in the task that are not running or successfully ## completed are resubmitted. If the user provides a list of job ids, then also ## successfully completed jobs can be resubmitted. ## Check the format of the jobids option. if self.options.jobids: jobidstuple = validateJobids(self.options.jobids) self.jobids = [str(jobid) for (_, jobid) in jobidstuple] ## The --force option should not be accepted unless combined with a user-given ## list of job ids via --jobids. if self.options.force and not self.jobids: msg = "Option --force can only be used in combination with option --jobids." raise ConfigurationException(msg) ## Covention used for the job parameters that the user can set when doing job ## resubmission (i.e. siteblacklist, sitewhitelist, maxjobruntime, maxmemory, ## numcores and priority): ## - If the user doesn't set a parameter we don't pass it to the server and the ## the server copies the original value the parameter had at task submission. ## It copies it from the Task DB. Therefore we need to keep these parameters ## in separate columns of the Task DB containing their original values. ## - For the site black- and whitelists, if the user passes an empty string, ## e.g. --siteblacklist='', we pass to the server siteblacklist=empty and the ## server interprets this as and empty list ([]). If the user passes a given ## list of sites, this new list overwrittes the original one. ## - The values of the parameters are used only for the resubmitted jobs (for ## their first resubmission and all next automatic resubmissions). #Checking if the sites provided by the user are valid cmsnames. Doing this because with only the #server error handling we get: # Server answered with: Invalid input parameter # Reason is: Incorrect 'siteblacklist' parameter #which is not really user friendly. #Moreover, I prefer to be independent from Lexicon. I'll the regex here. sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression sn_rec = re.compile(sn_re) #sn_rec => SiteName_RegularExpressionCompiled for sitelist in ['sitewhitelist', 'siteblacklist']: if getattr(self.options, sitelist) is not None: if getattr(self.options, sitelist) != "": for site_name in getattr(self.options, sitelist).split(','): if '*' not in site_name and not sn_rec.match(site_name): msg = "The site name '%s' does not look like a valid CMS site name" % (site_name) msg += " (it is not matching the regular expression '%s')." % (sn_re) raise ConfigurationException(msg) setattr(self, sitelist, getattr(self.options, sitelist).split(',')) else: setattr(self, sitelist, []) ## Sanity checks for task sizes. Limits are purposely fairly generous to provide ## some level of future-proofing. The server may restrict further. if self.options.maxjobruntime is not None: if self.options.maxjobruntime < 60 or self.options.maxjobruntime > 336*60: msg = "The requested maximum job runtime (%d minutes) must be between 60 and 20160 minutes." % (self.options.maxjobruntime) raise ConfigurationException(msg) if self.options.maxmemory is not None: if self.options.maxmemory < 30 or self.options.maxmemory > 1024*30: msg = "The requested per-job memory (%d MB) must be between 30 and 30720 MB." % (self.options.maxmemory) raise ConfigurationException(msg) if self.options.numcores is not None: if self.options.numcores < 1 or self.options.numcores > 128: msg = "The requested number of cores (%d) must be between 1 and 128." % (self.options.numcores) raise ConfigurationException(msg) if self.options.priority is not None: if self.options.priority < 1: msg = "The requested priority (%d) must be greater than 0." % (self.options.priority) raise ConfigurationException(msg)
def validateConfig(self): """ __validateConfig__ Checking if needed input parameters are there """ valid, msg = SubCommand.validateConfig(self) if not valid: return False, msg ## Check that Data.unitsPerjob is specified. if hasattr(self.configuration.Data, 'unitsPerJob'): try: float(self.configuration.Data.unitsPerJob) except ValueError: msg = "Invalid CRAB configuration: Parameter Data.unitsPerJob must be a valid number, not %s." % ( self.configuration.Data.unitsPerJob) return False, msg ## Check that JobType.pluginName and JobType.externalPluginFile are not both specified. if hasattr(self.configuration.JobType, 'pluginName') and hasattr( self.configuration.JobType, 'externalPluginFile'): msg = "Invalid CRAB configuration: Only one of JobType.pluginName or JobType.externalPluginFile parameters can be specified." pluginName_default = getParamDefaultValue('JobType.pluginName') if pluginName_default: msg += "\nIf neither JobType.pluginName nor JobType.externalPluginFile would be specified," msg += " the default JobType.pluginName = '%s' would be used." % ( pluginName_default) return False, msg ## Load the external plugin or check that the crab plugin is valid. external_plugin_name = getattr(self.configuration.JobType, 'externalPluginFile', None) crab_plugin_name = getattr(self.configuration.JobType, 'pluginName', None) crab_job_types = {'ANALYSIS': None, 'PRIVATEMC': None} #getJobTypes() if external_plugin_name: addPlugin(external_plugin_name) # Do we need to do this here? if crab_plugin_name: if upper(crab_plugin_name) not in crab_job_types: msg = "Invalid CRAB configuration: Parameter JobType.pluginName has an invalid value ('%s')." % ( crab_plugin_name) msg += "\nAllowed values are: %s." % (", ".join( ['%s' % job_type for job_type in crab_job_types.keys()])) return False, msg msg = "Will use CRAB %s plugin" % ("Analysis" if upper( crab_plugin_name) == 'ANALYSIS' else "PrivateMC") msg += " (i.e. will run %s job type)." % ("an analysis" if upper( crab_plugin_name) == 'ANALYSIS' else "a MC generation") self.logger.debug(msg) ## Check that the particular combination (Data.publication = True, General.transferOutputs = False) is not specified. if getattr(self.configuration.Data, 'publication', getParamDefaultValue('Data.publication')) and \ not getattr(self.configuration.General, 'transferOutputs', getParamDefaultValue('General.transferOutputs')): msg = "Invalid CRAB configuration: Data.publication is True, but General.transferOutputs is False." msg += "\nPublication can not be performed if the output files are not transferred to a permanent storage." return False, msg ## Check that a storage site is specified if General.transferOutputs = True or General.transferLogs = True. if not hasattr(self.configuration.Site, 'storageSite'): if getattr(self.configuration.General, 'transferLogs', getParamDefaultValue('General.transferLogs')) or \ getattr(self.configuration.General, 'transferOutputs', getParamDefaultValue('General.transferOutputs')): msg = "Invalid CRAB configuration: Parameter Site.storageSite is missing." return False, msg ## If an input dataset and a DBS URL are specified, check that the DBS URL is a good one. ## Also, if the DBS URL is 'phys0x', check that the input dataset tier is USER. if hasattr(self.configuration.Data, 'inputDBS'): if hasattr(self.configuration.Data, 'inputDataset'): msg = None dbs_urls_aliases = DBSURLS['reader'].keys() dbs_urls = DBSURLS['reader'].values() if (self.configuration.Data.inputDBS not in dbs_urls_aliases ) and (self.configuration.Data.inputDBS.rstrip('/') not in dbs_urls): msg = "Invalid CRAB configuration: Parameter Data.inputDBS has an invalid value ('%s')." % ( self.configuration.Data.inputDBS) msg += "\nAllowed values are: " msg += "\n ".join([ "'%s' ('%s')" % (alias, url) for alias, url in DBSURLS['reader'].iteritems() ]) local_dbs_urls_aliases = ['phys01', 'phys02', 'phys03'] local_dbs_urls = [ DBSURLS['reader'][alias] for alias in local_dbs_urls_aliases if alias in DBSURLS['reader'] ] if self.configuration.Data.inputDBS in local_dbs_urls + local_dbs_urls_aliases: inputDataset_parts = self.configuration.Data.inputDataset.split( '/') inputDataset_parts.pop(0) inputDataset_tier = inputDataset_parts[-1] if len( inputDataset_parts) == 3 else None user_data_tiers = ['USER'] if inputDataset_tier not in user_data_tiers: msg = "Invalid CRAB configuration: A local DBS instance '%s' was specified for reading an input dataset of tier %s." \ % (self.configuration.Data.inputDBS, inputDataset_tier) msg += "\nDatasets of tier different than %s must be read from the global DBS instance; this is, set Data.inputDBS = 'global'." \ % (", ".join(user_data_tiers[:-1]) + " or " + user_data_tiers[-1] if len(user_data_tiers) > 1 else user_data_tiers[0]) if msg: inputDBS_default = getParamDefaultValue('Data.inputDBS') if inputDBS_default: inputDBS_default, inputDBS_default_alias = self.getDBSURLAndAlias( inputDBS_default, 'reader') if inputDBS_default and inputDBS_default_alias: msg += "\nIf Data.inputDBS would not be specified, the default '%s' ('%s') would be used." % ( inputDBS_default_alias, inputDBS_default) return False, msg ## If a publication DBS URL is specified and publication is ON, check that the DBS URL is a good one. if hasattr(self.configuration.Data, 'publishDBS'): if getattr(self.configuration.Data, 'publication', getParamDefaultValue('Data.publication')): dbs_urls = DBSURLS['writer'].values() dbs_urls_aliases = DBSURLS['writer'].keys() if (self.configuration.Data.publishDBS not in dbs_urls_aliases ) and (self.configuration.Data.publishDBS.rstrip('/') not in dbs_urls): msg = "Invalid CRAB configuration: Parameter Data.publishDBS has an invalid value ('%s')." % ( self.configuration.Data.publishDBS) msg += "\nAllowed values are: " msg += "\n ".join([ "'%s' ('%s')" % (alias, url) for alias, url in DBSURLS['writer'].iteritems() ]) publishDBS_default = getParamDefaultValue( 'Data.publishDBS') if publishDBS_default: publishDBS_default, publishDBS_default_alias = self.getDBSURLAndAlias( publishDBS_default, 'writer') if publishDBS_default and publishDBS_default_alias: msg += "\nIf Data.publishDBS would not be specified, the default '%s' ('%s') would be used." \ % (publishDBS_default_alias, publishDBS_default) return False, msg if hasattr(self.configuration.JobType, 'scriptExe'): if not os.path.isfile(self.configuration.JobType.scriptExe): msg = "Cannot find the file %s specified in the JobType.scriptExe configuration parameter." % ( self.configuration.JobType.scriptExe) return False, msg return True, "Valid configuration"
def __init__(self, logger, cmdargs=None): SubCommand.__init__(self, logger, cmdargs) self.filename = None self.subdir = None self.lfnPrefix = None
def __init__(self, logger, cmdargs=None): SubCommand.__init__(self, logger, cmdargs, disable_interspersed_args=True)
def __init__(self, logger, cmdargs=None): SubCommand.__init__(self, logger, cmdargs) self.destination = None #Save the ASO destintion from he DB when we download input files
def validateConfig(self): """ __validateConfig__ Checking if needed input parameters are there """ valid, msg = SubCommand.validateConfig(self) if not valid: return False, msg ## Check that the configuration object has the sections we expect it to have. ## (WMCore already checks that attributes added to the configuration object are of type ConfigSection.) ## Even if not all configuration sections need to be there, we anyway request ## the user to add all the sections in the configuration file. if not hasattr(self.configuration, 'General'): msg = "CRAB configuration problem: Section 'General' is missing" return False, msg if not hasattr(self.configuration, 'JobType'): msg = "CRAB configuration problem: Section 'JobType' is missing" return False, msg if not hasattr(self.configuration, 'Data'): msg = "CRAB configuration problem: Section 'Data' is missing" return False, msg if not hasattr(self.configuration, 'Site'): msg = "CRAB configuration problem: Section 'Site' is missing" return False, msg ## Some parameters may have been renamed. Check here if the configuration file has an old ## parameter defined, and in that case tell the user what is the new parameter name. for old_param, new_param in renamed_params.iteritems(): if len(old_param.split('.')) != 2 or len(new_param.split('.')) != 2: continue old_param_section, old_param_name = old_param.split('.') if hasattr(self.configuration, old_param_section) and hasattr(getattr(self.configuration, old_param_section), old_param_name): msg = "CRAB configuration problem: Parameter %s has been renamed to %s; please change your configuration file accordingly" % (old_param, new_param) return False, msg ## Check that Data.unitsPerjob is specified. if hasattr(self.configuration.Data, 'unitsPerJob'): try: float(self.configuration.Data.unitsPerJob) except ValueError: msg = "CRAB configuration problem: Parameter Data.unitsPerJob must be a valid number, not %s" % self.configuration.Data.unitsPerJob return False, msg ## Check that JobType.pluginName and JobType.externalPluginFile are not both specified. if hasattr(self.configuration.JobType, 'pluginName') and hasattr(self.configuration.JobType, 'externalPluginFile'): msg = "CRAB configuration problem: Only one of JobType.pluginName or JobType.externalPluginFile parameters can be specified" pluginName_default = getParamDefaultValue('JobType.pluginName') if pluginName_default: msg += "\nIf neither JobType.pluginName nor JobType.externalPluginFile would be specified, the default JobType.pluginName = '%s' would be used" \ % pluginName_default return False, msg ## Load the external plugin or check that the crab plugin is valid. external_plugin_name = getattr(self.configuration.JobType, 'externalPluginFile', None) crab_plugin_name = getattr(self.configuration.JobType, 'pluginName', None) crab_job_types = {'ANALYSIS': None, 'PRIVATEMC': None} #getJobTypes() if external_plugin_name: addPlugin(external_plugin_name) # Do we need to do this here? if crab_plugin_name and upper(crab_plugin_name) not in crab_job_types: msg = "CRAB configuration problem: Parameter JobType.pluginName has an invalid value '%s'" % crab_plugin_name msg += "\nAllowed values are: %s" % ", ".join(['%s' % job_type for job_type in crab_job_types.keys()]) return False, msg ## Check that the particular combination (Data.publication = True, General.transferOutputs = False) is not specified. if hasattr(self.configuration.Data, 'publication') and hasattr(self.configuration.General, 'transferOutputs'): if self.configuration.Data.publication and not self.configuration.General.transferOutputs: msg = "CRAB configuration problem: Data.publication is on, but General.transferOutputs is off" msg += "\nPublication can not be performed if the output files are not transferred to a permanent storage" return False, msg ## Check that a storage site is specified if General.transferOutputs = True or General.transferLogs = True. if not hasattr(self.configuration.Site, 'storageSite'): if (hasattr(self.configuration.General, 'transferOutputs') and self.configuration.General.transferOutputs) or \ (hasattr(self.configuration.General, 'transferLogs') and self.configuration.General.transferLogs): msg = "CRAB configuration problem: Parameter Site.storageSite is missing" return False, msg ## If an input dataset and a DBS URL are specified, check that the DBS URL is a good one. ## Also, if the DBS URL is 'phys0x', check that the input dataset tier is USER. if hasattr(self.configuration.Data, 'inputDBS'): if hasattr(self.configuration.Data, 'inputDataset'): msg = None dbs_urls_aliases = DBSURLS['reader'].keys() dbs_urls = DBSURLS['reader'].values() if (self.configuration.Data.inputDBS not in dbs_urls_aliases) and (self.configuration.Data.inputDBS.rstrip('/') not in dbs_urls): msg = "CRAB configuration problem: Parameter Data.inputDBS has an invalid value '%s'" % self.configuration.Data.inputDBS msg += "\nAllowed values are: " msg += "\n ".join(["'%s' ('%s')" % (alias, url) for alias, url in DBSURLS['reader'].iteritems()]) local_dbs_urls_aliases = ['phys01', 'phys02', 'phys03'] local_dbs_urls = [DBSURLS['reader'][alias] for alias in local_dbs_urls_aliases if alias in DBSURLS['reader']] if self.configuration.Data.inputDBS in local_dbs_urls + local_dbs_urls_aliases: inputDataset_parts = self.configuration.Data.inputDataset.split('/') inputDataset_parts.pop(0) inputDataset_tier = inputDataset_parts[-1] if len(inputDataset_parts) == 3 else None user_data_tiers = ['USER'] if inputDataset_tier not in user_data_tiers: msg = "CRAB configuration problem: A local DBS instance '%s' was specified for reading an input dataset of tier %s" \ % (self.configuration.Data.inputDBS, inputDataset_tier) msg += "\nDatasets of tier different than %s must be read from the global DBS instance; this is, set Data.inputDBS = 'global'" \ % (", ".join(user_data_tiers[:-1]) + " or " + user_data_tiers[-1] if len(user_data_tiers) > 1 else user_data_tiers[0]) if msg: inputDBS_default = getParamDefaultValue('Data.inputDBS') if inputDBS_default: inputDBS_default, inputDBS_default_alias = self.getDBSURLAndAlias(inputDBS_default, 'reader') if inputDBS_default and inputDBS_default_alias: msg += "\nIf Data.inputDBS would not be specified, the default '%s' ('%s') would be used" % (inputDBS_default_alias, inputDBS_default) return False, msg ## If a publication DBS URL is specified and publication is ON, check that the DBS URL is a good one. if hasattr(self.configuration.Data, 'publishDBS'): publication_default = getParamDefaultValue('Data.publication') if getattr(self.configuration.Data, 'publication', publication_default): dbs_urls = DBSURLS['writer'].values() dbs_urls_aliases = DBSURLS['writer'].keys() if (self.configuration.Data.publishDBS not in dbs_urls_aliases) and (self.configuration.Data.publishDBS.rstrip('/') not in dbs_urls): msg = "CRAB configuration problem: Parameter Data.publishDBS has an invalid value '%s'" % self.configuration.Data.publishDBS msg += "\nAllowed values are: " msg += "\n ".join(["'%s' ('%s')" % (alias, url) for alias, url in DBSURLS['writer'].iteritems()]) publishDBS_default = getParamDefaultValue('Data.publishDBS') if publishDBS_default: publishDBS_default, publishDBS_default_alias = self.getDBSURLAndAlias(publishDBS_default, 'writer') if publishDBS_default and publishDBS_default_alias: msg += "\nIf Data.publishDBS would not be specified, the default '%s' ('%s') would be used" \ % (publishDBS_default_alias, publishDBS_default) return False, msg if hasattr(self.configuration.JobType, 'scriptExe'): if not os.path.isfile(self.configuration.JobType.scriptExe): msg = "Cannot find the file %s specified in the scriptExe configuration parameter" % self.configuration.JobType.scriptExe return False, msg return True, "Valid configuration"
def __init__(self, logger, cmdargs=None): SubCommand.__init__(self, logger, cmdargs) self.remotecpLogile = None
def validateConfig(self): """ __validateConfig__ Checking if needed input parameters are there """ valid, msg = SubCommand.validateConfig(self) if not valid: return False, msg requestNameLenLimit = 100 if hasattr(self.configuration.General, 'requestName'): if len(self.configuration.General.requestName ) > requestNameLenLimit: msg = "Invalid CRAB configuration: Parameter General.requestName should not be longer than %d characters." % ( requestNameLenLimit) return False, msg splitting = getattr(self.configuration.Data, 'splitting', 'Automatic') autoSplitt = True if splitting == 'Automatic' else False autoSplittUnitsMin = 180 # 3 hours (defined also in TW config as 'minAutomaticRuntimeMins') autoSplittUnitsMax = 2700 # 45 hours ## Check that maxJobRuntimeMin is not used with Automatic splitting if autoSplitt and hasattr(self.configuration.JobType, 'maxJobRuntimeMin'): msg = "The 'maxJobRuntimeMin' parameter is not compatible with the 'Automatic' splitting mode (default)." return False, msg ## Check that --dryrun is not used with Automatic splitting if autoSplitt and self.options.dryrun: msg = "The 'dryrun' option is not compatible with the 'Automatic' splitting mode (default)." return False, msg ## Check that Data.unitsPerjob is specified. if hasattr(self.configuration.Data, 'unitsPerJob'): try: float(self.configuration.Data.unitsPerJob) except ValueError: msg = "Invalid CRAB configuration: Parameter Data.unitsPerJob must be a valid number, not %s." % ( self.configuration.Data.unitsPerJob) return False, msg if not int(self.configuration.Data.unitsPerJob) > 0: msg = "Invalid CRAB configuration: Parameter Data.unitsPerJob must be > 0, not %s." % ( self.configuration.Data.unitsPerJob) return False, msg if autoSplitt and ( self.configuration.Data.unitsPerJob > autoSplittUnitsMax or self.configuration.Data.unitsPerJob < autoSplittUnitsMin): msg = "Invalid CRAB configuration: In case of Automatic splitting, the Data.unitsPerJob parameter must be in the [%d, %d] minutes range. You asked for %d minutes." % ( autoSplittUnitsMin, autoSplittUnitsMax, self.configuration.Data.unitsPerJob) return False, msg elif not autoSplitt: # The default value is only valid for automatic splitting! msg = "Invalid CRAB configuration: Parameter Data.unitsPerJob is mandatory for '%s' splitting mode." % splitting return False, msg ## Check that JobType.pluginName and JobType.externalPluginFile are not both specified. if hasattr(self.configuration.JobType, 'pluginName') and hasattr( self.configuration.JobType, 'externalPluginFile'): msg = "Invalid CRAB configuration: Only one of JobType.pluginName or JobType.externalPluginFile parameters can be specified." pluginName_default = getParamDefaultValue('JobType.pluginName') if pluginName_default: msg += "\nIf neither JobType.pluginName nor JobType.externalPluginFile would be specified," msg += " the default JobType.pluginName = '%s' would be used." % ( pluginName_default) return False, msg ## Load the external plugin or check that the crab plugin is valid. external_plugin_name = getattr(self.configuration.JobType, 'externalPluginFile', None) crab_plugin_name = getattr(self.configuration.JobType, 'pluginName', None) crab_job_types = { 'ANALYSIS': None, 'PRIVATEMC': None, 'COPYCAT': None } #getJobTypes() if external_plugin_name: addPlugin(external_plugin_name) # Do we need to do this here? if crab_plugin_name: if upper(crab_plugin_name) not in crab_job_types: msg = "Invalid CRAB configuration: Parameter JobType.pluginName has an invalid value ('%s')." % ( crab_plugin_name) msg += "\nAllowed values are: %s." % (", ".join( ['%s' % job_type for job_type in crab_job_types.keys()])) return False, msg msg = "Will use CRAB %s plugin" % ("Analysis" if upper( crab_plugin_name) == 'ANALYSIS' else "PrivateMC") msg += " (i.e. will run %s job type)." % ("an analysis" if upper( crab_plugin_name) == 'ANALYSIS' else "a MC generation") self.logger.debug(msg) ## Check that the particular combination (Data.publication = True, General.transferOutputs = False) is not specified. if getattr(self.configuration.Data, 'publication', getParamDefaultValue('Data.publication')) and \ not getattr(self.configuration.General, 'transferOutputs', getParamDefaultValue('General.transferOutputs')): msg = "Invalid CRAB configuration: Data.publication is True, but General.transferOutputs is False." msg += "\nPublication can not be performed if the output files are not transferred to a permanent storage." return False, msg ## Check that a storage site is specified if General.transferOutputs = True or General.transferLogs = True. if not hasattr(self.configuration.Site, 'storageSite'): if getattr(self.configuration.General, 'transferLogs', getParamDefaultValue('General.transferLogs')) or \ getattr(self.configuration.General, 'transferOutputs', getParamDefaultValue('General.transferOutputs')): msg = "Invalid CRAB configuration: Parameter Site.storageSite is missing." return False, msg ## If an input dataset and a DBS URL are specified, check that the DBS URL is a good one. ## Also, if the DBS URL is 'phys0x', check that the input dataset tier is USER. if hasattr(self.configuration.Data, 'inputDBS'): if hasattr(self.configuration.Data, 'inputDataset'): msg = None dbs_urls_aliases = DBSURLS['reader'].keys() dbs_urls = DBSURLS['reader'].values() if (self.configuration.Data.inputDBS not in dbs_urls_aliases ) and (self.configuration.Data.inputDBS.rstrip('/') not in dbs_urls): msg = "Invalid CRAB configuration: Parameter Data.inputDBS has an invalid value ('%s')." % ( self.configuration.Data.inputDBS) msg += "\nAllowed values are: " msg += "\n ".join([ "'%s' ('%s')" % (alias, url) for alias, url in DBSURLS['reader'].iteritems() ]) local_dbs_urls_aliases = ['phys01', 'phys02', 'phys03'] local_dbs_urls = [ DBSURLS['reader'][alias] for alias in local_dbs_urls_aliases if alias in DBSURLS['reader'] ] if self.configuration.Data.inputDBS in local_dbs_urls + local_dbs_urls_aliases: inputDataset_parts = self.configuration.Data.inputDataset.split( '/') inputDataset_parts.pop(0) inputDataset_tier = inputDataset_parts[-1] if len( inputDataset_parts) == 3 else None user_data_tiers = ['USER'] if inputDataset_tier not in user_data_tiers: msg = "Invalid CRAB configuration: A local DBS instance '%s' was specified for reading an input dataset of tier %s." \ % (self.configuration.Data.inputDBS, inputDataset_tier) msg += "\nDatasets of tier different than %s must be read from the global DBS instance; this is, set Data.inputDBS = 'global'." \ % (", ".join(user_data_tiers[:-1]) + " or " + user_data_tiers[-1] if len(user_data_tiers) > 1 else user_data_tiers[0]) if msg: inputDBS_default = getParamDefaultValue('Data.inputDBS') if inputDBS_default: inputDBS_default, inputDBS_default_alias = self.getDBSURLAndAlias( inputDBS_default, 'reader') if inputDBS_default and inputDBS_default_alias: msg += "\nIf Data.inputDBS would not be specified, the default '%s' ('%s') would be used." % ( inputDBS_default_alias, inputDBS_default) return False, msg ## If a publication DBS URL is specified and publication is ON, check that the DBS URL is a good one. if hasattr(self.configuration.Data, 'publishDBS'): if getattr(self.configuration.Data, 'publication', getParamDefaultValue('Data.publication')): dbs_urls = DBSURLS['writer'].values() dbs_urls_aliases = DBSURLS['writer'].keys() if (self.configuration.Data.publishDBS not in dbs_urls_aliases ) and (self.configuration.Data.publishDBS.rstrip('/') not in dbs_urls): msg = "Invalid CRAB configuration: Parameter Data.publishDBS has an invalid value ('%s')." % ( self.configuration.Data.publishDBS) msg += "\nAllowed values are: " msg += "\n ".join([ "'%s' ('%s')" % (alias, url) for alias, url in DBSURLS['writer'].iteritems() ]) publishDBS_default = getParamDefaultValue( 'Data.publishDBS') if publishDBS_default: publishDBS_default, publishDBS_default_alias = self.getDBSURLAndAlias( publishDBS_default, 'writer') if publishDBS_default and publishDBS_default_alias: msg += "\nIf Data.publishDBS would not be specified, the default '%s' ('%s') would be used." \ % (publishDBS_default_alias, publishDBS_default) return False, msg if hasattr(self.configuration.JobType, 'scriptExe'): if not os.path.isfile(self.configuration.JobType.scriptExe): msg = "Cannot find the file %s specified in the JobType.scriptExe configuration parameter." % ( self.configuration.JobType.scriptExe) return False, msg ## If ignoreLocality is set, check that a sitewhilelist is present if getattr(self.configuration.Data, 'ignoreLocality', False): if not hasattr(self.configuration.Site, 'whitelist'): msg = "Invalid CRAB configuration:\n when ignoreLocality is set a valid site white list must be specified using the Site.whitelist parameter" return False, msg if hasattr(self.configuration.General, 'failureLimit'): msg = "You have specified deprecated parameter 'failureLimit' which will be removed in the near future." msg += "\nIf you really need it write a mail to hn-cms-computingTools explaining your use case." self.logger.warning("%sWARNING%s: %s" % (colors.RED, colors.NORMAL, msg)) return True, "Valid configuration"
def validateOptions(self): SubCommand.validateOptions(self) if self.options.scheddonly and self.options.cacheonly: self.logger.info('Options --schedd and --cache can not be specified simultaneously. No purging will be done.') raise ConfigException
def validateConfig(self): """ __validateConfig__ Checking if needed input parameters are there """ valid, msg = SubCommand.validateConfig(self) if not valid: return False, msg ## Check that the configuration object has the sections we expect it to have. ## (WMCore already checks that attributes added to the configuration object are of type ConfigSection.) ## Even if not all configuration sections need to be there, we anyway request ## the user to add all the sections in the configuration file. if not hasattr(self.configuration, 'General'): msg = "CRAB configuration problem: Section 'General' is missing" return False, msg if not hasattr(self.configuration, 'JobType'): msg = "CRAB configuration problem: Section 'JobType' is missing" return False, msg if not hasattr(self.configuration, 'Data'): msg = "CRAB configuration problem: Section 'Data' is missing" return False, msg if not hasattr(self.configuration, 'Site'): msg = "CRAB configuration problem: Section 'Site' is missing" return False, msg ## Some parameters may have been renamed. Check here if the configuration file has an old ## parameter defined, and in that case tell the user what is the new parameter name. for old_param, new_param in renamed_params.iteritems(): if len(old_param.split('.')) != 2 or len( new_param.split('.')) != 2: continue old_param_section, old_param_name = old_param.split('.') if hasattr(self.configuration, old_param_section) and hasattr( getattr(self.configuration, old_param_section), old_param_name): msg = "CRAB configuration problem: Parameter %s has been renamed to %s; please change your configuration file accordingly" % ( old_param, new_param) return False, msg ## Check that Data.unitsPerjob is specified. if hasattr(self.configuration.Data, 'unitsPerJob'): try: float(self.configuration.Data.unitsPerJob) except ValueError: msg = "CRAB configuration problem: Parameter Data.unitsPerJob must be a valid number, not %s" % self.configuration.Data.unitsPerJob return False, msg ## Check that JobType.pluginName and JobType.externalPluginFile are not both specified. if hasattr(self.configuration.JobType, 'pluginName') and hasattr( self.configuration.JobType, 'externalPluginFile'): msg = "CRAB configuration problem: Only one of JobType.pluginName or JobType.externalPluginFile parameters can be specified" pluginName_default = getParamDefaultValue('JobType.pluginName') if pluginName_default: msg += "\nIf neither JobType.pluginName nor JobType.externalPluginFile would be specified, the default JobType.pluginName = '%s' would be used" \ % pluginName_default return False, msg ## Load the external plugin or check that the crab plugin is valid. external_plugin_name = getattr(self.configuration.JobType, 'externalPluginFile', None) crab_plugin_name = getattr(self.configuration.JobType, 'pluginName', None) crab_job_types = {'ANALYSIS': None, 'PRIVATEMC': None} #getJobTypes() if external_plugin_name: addPlugin(external_plugin_name) # Do we need to do this here? if crab_plugin_name and upper(crab_plugin_name) not in crab_job_types: msg = "CRAB configuration problem: Parameter JobType.pluginName has an invalid value '%s'" % crab_plugin_name msg += "\nAllowed values are: %s" % ", ".join( ['%s' % job_type for job_type in crab_job_types.keys()]) return False, msg ## Check that the particular combination (Data.publication = True, General.transferOutputs = False) is not specified. if hasattr(self.configuration.Data, 'publication') and hasattr( self.configuration.General, 'transferOutputs'): if self.configuration.Data.publication and not self.configuration.General.transferOutputs: msg = "CRAB configuration problem: Data.publication is on, but General.transferOutputs is off" msg += "\nPublication can not be performed if the output files are not transferred to a permanent storage" return False, msg ## Check that a storage site is specified if General.transferOutputs = True or General.transferLogs = True. if not hasattr(self.configuration.Site, 'storageSite'): if (hasattr(self.configuration.General, 'transferOutputs') and self.configuration.General.transferOutputs) or \ (hasattr(self.configuration.General, 'transferLogs') and self.configuration.General.transferLogs): msg = "CRAB configuration problem: Parameter Site.storageSite is missing" return False, msg ## If an input dataset and a DBS URL are specified, check that the DBS URL is a good one. ## Also, if the DBS URL is 'phys0x', check that the input dataset tier is USER. if hasattr(self.configuration.Data, 'inputDBS'): if hasattr(self.configuration.Data, 'inputDataset'): msg = None dbs_urls_aliases = DBSURLS['reader'].keys() dbs_urls = DBSURLS['reader'].values() if (self.configuration.Data.inputDBS not in dbs_urls_aliases ) and (self.configuration.Data.inputDBS.rstrip('/') not in dbs_urls): msg = "CRAB configuration problem: Parameter Data.inputDBS has an invalid value '%s'" % self.configuration.Data.inputDBS msg += "\nAllowed values are: " msg += "\n ".join([ "'%s' ('%s')" % (alias, url) for alias, url in DBSURLS['reader'].iteritems() ]) local_dbs_urls_aliases = ['phys01', 'phys02', 'phys03'] local_dbs_urls = [ DBSURLS['reader'][alias] for alias in local_dbs_urls_aliases if alias in DBSURLS['reader'] ] if self.configuration.Data.inputDBS in local_dbs_urls + local_dbs_urls_aliases: inputDataset_parts = self.configuration.Data.inputDataset.split( '/') inputDataset_parts.pop(0) inputDataset_tier = inputDataset_parts[-1] if len( inputDataset_parts) == 3 else None user_data_tiers = ['USER'] if inputDataset_tier not in user_data_tiers: msg = "CRAB configuration problem: A local DBS instance '%s' was specified for reading an input dataset of tier %s" \ % (self.configuration.Data.inputDBS, inputDataset_tier) msg += "\nDatasets of tier different than %s must be read from the global DBS instance; this is, set Data.inputDBS = 'global'" \ % (", ".join(user_data_tiers[:-1]) + " or " + user_data_tiers[-1] if len(user_data_tiers) > 1 else user_data_tiers[0]) if msg: inputDBS_default = getParamDefaultValue('Data.inputDBS') if inputDBS_default: inputDBS_default, inputDBS_default_alias = self.getDBSURLAndAlias( inputDBS_default, 'reader') if inputDBS_default and inputDBS_default_alias: msg += "\nIf Data.inputDBS would not be specified, the default '%s' ('%s') would be used" % ( inputDBS_default_alias, inputDBS_default) return False, msg ## If a publication DBS URL is specified and publication is ON, check that the DBS URL is a good one. if hasattr(self.configuration.Data, 'publishDBS'): publication_default = getParamDefaultValue('Data.publication') if getattr(self.configuration.Data, 'publication', publication_default): dbs_urls = DBSURLS['writer'].values() dbs_urls_aliases = DBSURLS['writer'].keys() if (self.configuration.Data.publishDBS not in dbs_urls_aliases ) and (self.configuration.Data.publishDBS.rstrip('/') not in dbs_urls): msg = "CRAB configuration problem: Parameter Data.publishDBS has an invalid value '%s'" % self.configuration.Data.publishDBS msg += "\nAllowed values are: " msg += "\n ".join([ "'%s' ('%s')" % (alias, url) for alias, url in DBSURLS['writer'].iteritems() ]) publishDBS_default = getParamDefaultValue( 'Data.publishDBS') if publishDBS_default: publishDBS_default, publishDBS_default_alias = self.getDBSURLAndAlias( publishDBS_default, 'writer') if publishDBS_default and publishDBS_default_alias: msg += "\nIf Data.publishDBS would not be specified, the default '%s' ('%s') would be used" \ % (publishDBS_default_alias, publishDBS_default) return False, msg if hasattr(self.configuration.JobType, 'scriptExe'): if not os.path.isfile(self.configuration.JobType.scriptExe): msg = "Cannot find the file %s specified in the scriptExe configuration parameter" % self.configuration.JobType.scriptExe return False, msg return True, "Valid configuration"