def validateOptions(self): """ Check if the sitelist parameter is a comma separater list of cms sitenames, and put the strings to be passed to the server to self """ SubCommand.validateOptions(self) #Checking if the sites provided by the user are valid cmsnames. Doing this because with only the #server error handling we get: # Server answered with: Invalid input parameter # Reason is: Incorrect 'siteblacklist' parameter #which is not really user friendly. #Moreover, I prefer to be independent from Lexicon. I'll the regex here. sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression sn_rec = re.compile( sn_re) #sn_rec => SiteName_RegularExpressionCompiled self.sitewhitelist = '' self.siteblsklist = '' for siteList in ['sitewhitelist', 'siteblacklist']: result = '' paramVal = getattr(self.options, siteList, None) if paramVal: for site in paramVal.split(','): if not sn_rec.match(site): raise ConfigException( "The sitename %s dows not look like a valid CMS name (not matching %s)" % (site, sn_re)) result += "&%s=%s" % (siteList, site) setattr(self, siteList, result)
def validateOptions(self): SubCommand.validateOptions(self) if hasattr(self.options, 'logpath') and self.options.logpath != None: if not path.isfile(self.options.logpath): msg = '%sError%s: Could not find the log file in the path: %s' % ( colors.RED, colors.NORMAL, self.options.logpath) raise ConfigurationException(msg) elif hasattr(self.options, 'task'): self.cmdconf['requiresTaskOption'] = True self.cmdconf['useCache'] = True if self.options.task == None: if len(self.args) == 1 and self.args[0]: self.options.task = self.args[0] #for the case of 'crab uploadlog', user is using the .crab3 file if self.options.task == None and hasattr(self, 'crab3dic'): if self.crab3dic["taskname"] != None: self.options.task = self.crab3dic["taskname"] else: msg = '%sError%s: Please use the directory option -d or --logpath to specify which log to upload' % ( colors.RED, colors.NORMAL) raise ConfigurationException(msg) else: msg = '%sError%s: Please use the directory option -d or --logpath to specify which log to upload' % ( colors.RED, colors.NORMAL) raise ConfigurationException(msg)
def validateOptions(self): """ Check if the sitelist parameter is a comma separater list of cms sitenames, and put the strings to be passed to the server to self """ SubCommand.validateOptions(self) #Checking if the sites provided by the user are valid cmsnames. Doing this because with only the #server error handling we get: # Server answered with: Invalid input parameter # Reason is: Incorrect 'siteblacklist' parameter #which is not really user friendly. #Moreover, I prefer to be independent from Lexicon. I'll the regex here. sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression sn_rec = re.compile(sn_re) #sn_rec => SiteName_RegularExpressionCompiled self.sitewhitelist = '' self.siteblsklist = '' for siteList in ['sitewhitelist', 'siteblacklist']: result = '' paramVal = getattr(self.options, siteList, None) if paramVal: for site in paramVal.split(','): if not sn_rec.match(site): raise ConfigException("The sitename %s dows not look like a valid CMS name (not matching %s)" % (site, sn_re) ) result += "&%s=%s" % (siteList, site) setattr(self, siteList, result) #check the format of jobids self.jobids = '' if getattr(self.options, 'jobids', None): self.jobids = validateJobids(self.options.jobids)
def validateOptions(self): #Figuring out the destination directory SubCommand.validateOptions(self) self.dest = None if self.options.outputpath is not None: if not os.path.isabs(self.options.outputpath): self.dest = os.path.abspath(self.options.outputpath) else: self.dest = self.options.outputpath #convert all to -1 if getattr(self.options, 'quantity', None) == 'all': self.options.quantity = -1 #check the format of jobids if getattr(self.options, 'jobids', None): if re.compile('^\d+(,\d+)*$').match(self.options.jobids): self.options.jobids = [ ('jobids', jobid) for jobid in self.options.jobids.split(',') ] else: raise ConfigurationException( "The command line option jobids should be a comma separated list of integers" )
def validateOptions(self): SubCommand.validateOptions(self) if self.options.idle and (self.options.long or self.options.summary): raise ConfigurationException( "Idle option (-i) conflicts with -u, and -l") self.json = self.options.json self.summary = self.options.summary self.idle = self.options.idle self.long = self.options.long acceptedsort = [ "state", "site", "runtime", "memory", "cpu", "retries", "waste", "exitcode" ] if hasattr(self.options, 'sort') and self.options.sort != None: if not self.long: raise ConfigurationException( '%sError%s: Please use option --long together with --sort' % (colors.RED, colors.NORMAL)) elif not self.options.sort in acceptedsort: raise ConfigurationException( '%sError%s: Only these values are accepted for crab status --sort option: %s' % (colors.RED, colors.NORMAL, acceptedsort)) else: self.sort = self.options.sort
def validateOptions(self): SubCommand.validateOptions(self) #check the format of jobids self.jobids = '' if getattr(self.options, 'jobids', None): self.jobids = validateJobids(self.options.jobids)
def validateOptions(self): SubCommand.validateOptions(self) if self.options.sitename is None: msg = "%sError%s: Please specify the site where to check the write permissions." % (colors.RED, colors.NORMAL) msg += " Use the --site option." ex = MissingOptionException(msg) ex.missingOption = "sitename" raise ex if hasattr(self.options, 'command') and self.options.command != None: AvailableCommands = ['LCG', 'GFAL'] self.command = self.options.command.upper() if self.command not in AvailableCommands: msg = "You specified to use %s command and it is not allowed. Available commands are: %s " % (self.command, str(AvailableCommands)) ex = ConfigurationException(msg) raise ex else: self.command = None if hasattr(self.options, 'checksum'): if re.match('^yes$|^no$', self.options.checksum): self.checksum = 'ADLER32' if self.options.checksum == 'yes' else None else: msg = "You specified to use %s checksum. Only lowercase yes/no is accepted to turn ADLER32 checksum" % self.options.checksum ex = ConfigurationException(msg) raise ex
def validateOptions(self): SubCommand.validateOptions(self) if not hasattr(self.options, 'sitename') or self.options.sitename is None: self.logger.info("Missing site name, use '--site' option") raise MissingOptionException
def validateOptions(self): """ After doing the general options validation from the parent SubCommand class, do the validation of options that are specific to the submit command. """ ## First call validateOptions() from the SubCommand class. SubCommand.validateOptions(self) ## If no configuration file was passed as an option, try to extract it from the arguments. ## Assume that the arguments can only be: ## 1) the configuration file name, and ## 2) parameters to override in the configuration file. ## The last ones should all contain an '=' sign, so these are not candidates to be the ## configuration file argument. Also, the configuration file name should end with '.py'. ## If can not find a configuration file candidate, use the default 'crabConfig.py'. ## If find more than one candidate, raise ConfigurationException. if self.options.config is None: use_default = True if len(self.args): config_candidates = [(arg,i) for i,arg in enumerate(self.args) if '=' not in arg and arg[-3:] == '.py'] config_candidate_names = set([config_candidate_name for (config_candidate_name,_) in config_candidates]) if len(config_candidate_names) == 1: self.options.config = config_candidates[0][0] del self.args[config_candidates[0][1]] use_default = False elif len(config_candidate_names) > 1: self.logger.info('Unable to unambiguously extract the configuration file from the command-line arguments.') self.logger.info('Possible candidates are: %s' % list(config_candidate_names)) raise ConfigurationException('ERROR: Unable to extract configuration file from command-line arguments.') if use_default: self.options.config = 'crabConfig.py'
def validateOptions(self): SubCommand.validateOptions(self) #check the format of jobids self.jobids = '' if getattr(self.options, 'jobids', None): self.jobids = validateJobids(self.options.jobids)
def validateOptions(self): #Figuring out the destination directory SubCommand.validateOptions(self) if self.options.outputpath is not None: if re.match("^[a-z]+://", self.options.outputpath): self.dest = self.options.outputpath elif not os.path.isabs( self.options.outputpath ): self.dest = os.path.abspath( self.options.outputpath ) else: self.dest = self.options.outputpath #convert all to -1 if getattr(self.options, 'quantity', None) == 'all': self.options.quantity = -1 if hasattr(self.options, 'command') and self.options.command != None: AvailableCommands = ['LCG', 'GFAL'] self.command = self.options.command.upper() if self.command not in AvailableCommands: msg = "You specified to use %s command and it is not allowed. Available commands are: %s " % (self.command, str(AvailableCommands)) ex = ConfigurationException(msg) raise ex else: self.command = None if hasattr(self.options, 'checksum'): if re.match('^yes$|^no$', self.options.checksum): self.checksum = 'ADLER32' if self.options.checksum == 'yes' else None else: msg = "You specified to use %s checksum. Only lowercase yes/no is accepted to turn ADLER32 checksum" % self.options.checksum ex = ConfigurationException(msg) raise ex
def validateOptions(self): SubCommand.validateOptions(self) if self.options.scheddonly and self.options.cacheonly: self.logger.info( 'Options --schedd and --cache can not be specified simultaneously. No purging will be done.' ) raise ConfigException
def validateOptions(self): SubCommand.validateOptions(self) if self.options.jobid is not None: try: int(self.options.jobid) except ValueError: raise ClientException("The --jobid option has to be an integer")
def validateOptions(self): """ After doing the general options validation from the parent SubCommand class, do the validation of options that are specific to the submit command. """ ## First do the basic validation in the SubCommand. SubCommand.validateOptions(self) validateSubmitOptions(self.options, self.args)
def validateOptions(self): """ Check if the sitelist parameter is a comma separater list of cms sitenames, and put the strings to be passed to the server to self """ SubCommand.validateOptions(self) #Checking if the sites provided by the user are valid cmsnames. Doing this because with only the #server error handling we get: # Server answered with: Invalid input parameter # Reason is: Incorrect 'siteblacklist' parameter #which is not really user friendly. #Moreover, I prefer to be independent from Lexicon. I'll the regex here. sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression sn_rec = re.compile(sn_re) #sn_rec => SiteName_RegularExpressionCompiled self.sitewhitelist = '' self.siteblsklist = '' for siteList in ['sitewhitelist', 'siteblacklist']: result = '' paramVal = getattr(self.options, siteList, None) if paramVal: for site in paramVal.split(','): if not sn_rec.match(site): raise ConfigException("The sitename %s dows not look like a valid CMS name (not matching %s)" % (site, sn_re) ) result += "&%s=%s" % (siteList, site) setattr(self, siteList, result) #check the format of jobids self.jobids = '' if getattr(self.options, 'jobids', None): self.jobids = validateJobids(self.options.jobids) # Sanity checks for task sizes. Limits are purposely fairly generous to provide some level of future-proofing. # The server may restrict further. self.numcores = None if self.options.numcores != None: if self.options.numcores < 1 or self.options.numcores > 128: raise ConfigException("The number of requested cores (%d) must be between 1 and 128." % (self.options.numcores)) self.numcores = str(self.options.numcores) self.maxjobruntime = None if self.options.maxjobruntime != None: if self.options.maxjobruntime < 1 or self.options.maxjobruntime > 336: raise ConfigException("The requested max job runtime (%d hours) must be between 1 and 336 hours." % self.options.maxjobruntime) self.maxjobruntime = str(self.options.maxjobruntime) self.maxmemory = None if self.options.maxmemory != None: if self.options.maxmemory < 30 or self.options.maxmemory > 1024*30: raise ConfigException("The requested per-job memory (%d MB) must be between 30 and 30720 MB." % self.options.maxmemory) self.maxmemory = str(self.options.maxmemory) self.priority = None if self.options.priority != None: self.priority = str(self.options.priority)
def validateOptions(self): SubCommand.validateOptions(self) if self.options.sitename is None: msg = "%sError%s: Please specify the site where to check the write permissions." % (colors.RED, colors.NORMAL) msg += " Use the --site option." ex = MissingOptionException(msg) ex.missingOption = "sitename" raise ex
def validateOptions(self): SubCommand.validateOptions(self) if self.options.idle and (self.options.long or self.options.summary ): raise ConfigurationException("Idle option (-i) conflicts with -u, and -l") self.long = self.options.long self.json = self.options.json self.summary = self.options.summary self.idle = self.options.idle
def validateOptions(self): """ After doing the general options validation from the parent SubCommand class, do the validation of options that are specific to the submit command. """ ## First do the basic validation in the SubCommand. SubCommand.validateOptions(self) validateSubmitOptions(self.options, self.args)
def validateOptions(self): """ Check if the output file is given and set as attribute """ SubCommand.validateOptions(self) if not re.match('^yes$|^no$', self.options.usedbs): raise ConfigurationException("--dbs option only accepts the yes and no values (--dbs=yes or --dbs=no)") self.usedbs = 1 if self.options.usedbs=='yes' else 0 setattr(self, 'outdir', getattr(self.options, 'outdir', None))
def validateOptions(self): SubCommand.validateOptions(self) if self.options.idle and (self.options.long or self.options.summary): raise ConfigurationException("Option --idle conflicts with --summary and --long") if self.options.sort is not None: sortOpts = ["state", "site", "runtime", "memory", "cpu", "retries", "waste", "exitcode"] if self.options.sort not in sortOpts: msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Only the following values are accepted for --sort option: %s" % (sortOpts) raise ConfigurationException(msg)
def validateOptions(self): ## Do the options validation from SubCommand. try: SubCommand.validateOptions(self) except MissingOptionException as ex: if ex.missingOption == "task": msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Please provide a path to a CRAB project directory (use the -d/--dir option)" msg += " or to a log file (use the --logpath option)." ex = MissingOptionException(msg) ex.missingOption = "task" raise ex
def validateOptions(self): """ Check if the output file is given and set as attribute """ SubCommand.validateOptions(self) if not re.match('^yes$|^no$', self.options.usedbs): raise ConfigurationException( "--dbs option only accepts the yes and no values (--dbs=yes or --dbs=no)" ) self.usedbs = 1 if self.options.usedbs == 'yes' else 0 setattr(self, 'outdir', getattr(self.options, 'outdir', None))
def validateOptions(self): SubCommand.validateOptions(self) if self.options.idle and (self.options.long or self.options.summary or self.options.publication): raise ConfigurationException("Idle option (-i) conflicts with -u, -l, and -p") if self.options.publication and (self.options.long or self.options.summary or self.options.idle): raise ConfigurationException("Publication option (-p) conflicts with -u, -l, and -i") self.long = self.options.long self.json = self.options.json self.summary = self.options.summary self.idle = self.options.idle self.publication = self.options.publication
def validateOptions(self): """ Check if the sitelist parameter is a comma separater list of cms sitenames, and put the strings to be passed to the server to self """ SubCommand.validateOptions(self) ## Check the format of the jobids option. if getattr(self.options, 'jobids'): jobidstuple = validateJobids(self.options.jobids) self.jobids = [str(jobid) for (_, jobid) in jobidstuple] #Checking if the sites provided by the user are valid cmsnames. Doing this because with only the #server error handling we get: # Server answered with: Invalid input parameter # Reason is: Incorrect 'siteblacklist' parameter #which is not really user friendly. #Moreover, I prefer to be independent from Lexicon. I'll the regex here. sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression sn_rec = re.compile(sn_re) #sn_rec => SiteName_RegularExpressionCompiled for sitelist in ['sitewhitelist', 'siteblacklist']: if getattr(self.options, sitelist) is not None: for i, site_name in enumerate(getattr(self.options, sitelist).split(',')): if not sn_rec.match(site_name): msg = "The site name %s does not look like a valid CMS site name" % (site_name) msg += " (it is not matching the regular expression %s)." % (sn_re) raise ConfigurationException(msg) setattr(self, sitelist, getattr(self.options, sitelist).split(',')) ## Sanity checks for task sizes. Limits are purposely fairly generous to provide ## some level of future-proofing. The server may restrict further. if self.options.maxjobruntime is not None: if self.options.maxjobruntime < 60 or self.options.maxjobruntime > 336*60: msg = "The requested maximum job runtime (%d minutes) must be between 60 and 20160 minutes." % (self.options.maxjobruntime) raise ConfigurationException(msg) self.maxjobruntime = str(self.options.maxjobruntime) if self.options.maxmemory is not None: if self.options.maxmemory < 30 or self.options.maxmemory > 1024*30: msg = "The requested per-job memory (%d MB) must be between 30 and 30720 MB." % (self.options.maxmemory) raise ConfigurationException(msg) self.maxmemory = str(self.options.maxmemory) if self.options.numcores is not None: if self.options.numcores < 1 or self.options.numcores > 128: msg = "The requested number of cores (%d) must be between 1 and 128." % (self.options.numcores) raise ConfigurationException(msg) self.numcores = str(self.options.numcores) if self.options.priority is not None: self.priority = str(self.options.priority)
def validateOptions(self): """ Check if the output file is given and set as attribute """ SubCommand.validateOptions(self) if self.options.usedbs is not None: msg = "CRAB command option error: the option --dbs has been deprecated since CRAB v3.3.1603." raise UnknownOptionException(msg) recoveryMethods = ['notFinished', 'notPublished', 'failed'] if self.options.recovery not in recoveryMethods: msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " The --recovery option only accepts the following values: %s" % (recoveryMethods) raise ConfigurationException(msg)
def validateOptions(self): """ Check if the output file is given and set as attribute """ SubCommand.validateOptions(self) if self.options.usedbs is not None: msg = "CRAB command option error: the option --dbs has been deprecated since CRAB v3.3.1603." raise UnknownOptionException(msg) recoveryMethods = ['notFinished', 'notPublished', 'failed'] if self.options.recovery not in recoveryMethods: msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " The --recovery option only accepts the following values: %s" % ( recoveryMethods) raise ConfigurationException(msg)
def validateOptions(self): SubCommand.validateOptions(self) if self.options.idle and (self.options.long or self.options.summary): raise ConfigurationException( "Option --idle conflicts with --summary and --long") if self.options.sort is not None: sortOpts = [ "state", "site", "runtime", "memory", "cpu", "retries", "waste", "exitcode" ] if self.options.sort not in sortOpts: msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Only the following values are accepted for --sort option: %s" % ( sortOpts) raise ConfigurationException(msg)
def validateOptions(self): #Figuring out the destination directory SubCommand.validateOptions(self) self.dest = None if self.options.outputpath is not None: if not os.path.isabs( self.options.outputpath ): self.dest = os.path.abspath( self.options.outputpath ) else: self.dest = self.options.outputpath #convert all to -1 if getattr(self.options, 'quantity', None) == 'all': self.options.quantity = -1 #check the format of jobids if getattr(self.options, 'jobids', None): self.options.jobids = validateJobids(self.options.jobids)
def validateOptions(self): SubCommand.validateOptions(self) if self.options.idle and (self.options.long or self.options.summary ): raise ConfigurationException("Idle option (-i) conflicts with -u, and -l") self.json = self.options.json self.summary = self.options.summary self.idle = self.options.idle self.long = self.options.long acceptedsort = ["state", "site", "runtime", "memory", "cpu", "retries", "waste"] if hasattr(self.options , 'sort') and self.options.sort != None: if not self.long: raise ConfigurationException('%sError%s: Please use option --long together with --sort' % (colors.RED, colors.NORMAL)) elif not self.options.sort in acceptedsort: raise ConfigurationException('%sError%s: Only this value accepted for crab status sort: %s' % (colors.RED, colors.NORMAL, acceptedsort)) else: self.sort = self.options.sort
def validateOptions(self): """ After doing the general options validation from the parent SubCommand class, do the validation of options that are specific to the submit command. """ ## First call validateOptions() from the SubCommand class. SubCommand.validateOptions(self) ## If no configuration file was passed as an option, try to extract it from the arguments. ## Assume that the arguments can only be: ## 1) the configuration file name, and ## 2) parameters to override in the configuration file. ## The last ones should all contain an '=' sign, so these are not candidates to be the ## configuration file argument. Also, the configuration file name should end with '.py'. ## If can not find a configuration file candidate, use the default 'crabConfig.py'. ## If find more than one candidate, raise ConfigurationException. if self.options.config is None: use_default = True if len(self.args): config_candidates = [(arg, i) for i, arg in enumerate(self.args) if '=' not in arg and arg[-3:] == '.py'] config_candidate_names = set([ config_candidate_name for (config_candidate_name, _) in config_candidates ]) if len(config_candidate_names) == 1: self.options.config = config_candidates[0][0] del self.args[config_candidates[0][1]] use_default = False elif len(config_candidate_names) > 1: self.logger.info( 'Unable to unambiguously extract the configuration file from the command-line arguments.' ) self.logger.info('Possible candidates are: %s' % list(config_candidate_names)) raise ConfigurationException( 'ERROR: Unable to extract configuration file from command-line arguments.' ) if use_default: self.options.config = 'crabConfig.py'
def validateOptions(self): #Figuring out the destination directory SubCommand.validateOptions(self) self.dest = None if self.options.outputpath is not None: if not os.path.isabs( self.options.outputpath ): self.dest = os.path.abspath( self.options.outputpath ) else: self.dest = self.options.outputpath #convert all to -1 if getattr(self.options, 'quantity', None) == 'all': self.options.quantity = -1 #check the format of jobids if getattr(self.options, 'jobids', None): if re.compile('^\d+(,\d+)*$').match(self.options.jobids): self.options.jobids = [('jobids',jobid) for jobid in self.options.jobids.split(',')] else: raise ConfigurationException("The command line option jobids should be a comma separated list of integers")
def validateOptions(self): # Figuring out the destination directory SubCommand.validateOptions(self) self.dest = None if self.options.outputpath is not None: if re.match("^[a-z]+://", self.options.outputpath): self.dest = self.options.outputpath elif not os.path.isabs(self.options.outputpath): self.dest = os.path.abspath(self.options.outputpath) else: self.dest = self.options.outputpath # convert all to -1 if getattr(self.options, "quantity", None) == "all": self.options.quantity = -1 # check the format of jobids if getattr(self.options, "jobids", None): self.options.jobids = validateJobids(self.options.jobids) self.dump = self.options.dump
def validateOptions(self): #Figuring out the destination directory SubCommand.validateOptions(self) self.dest = None if self.options.outputpath is not None: if re.match("^[a-z]+://", self.options.outputpath): self.dest = self.options.outputpath elif not os.path.isabs(self.options.outputpath): self.dest = os.path.abspath(self.options.outputpath) else: self.dest = self.options.outputpath #convert all to -1 if getattr(self.options, 'quantity', None) == 'all': self.options.quantity = -1 #check the format of jobids if getattr(self.options, 'jobids', None): self.options.jobids = validateJobids(self.options.jobids) self.dump = self.options.dump
def validateOptions(self): ## Check if the --logpath option was used. If it was, don't require the task ## option. if self.options.logpath is not None: if not os.path.isfile(self.options.logpath): msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Could not find the log file %s." % (self.options.logpath) raise ConfigurationException(msg) self.cmdconf["requiresDirOption"] = False ## Do the options validation from SubCommand. try: SubCommand.validateOptions(self) except MissingOptionException as ex: if ex.missingOption == "task": msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Please provide a path to a CRAB project directory (use the -d/--dir option)" msg += " or to a log file (use the --logpath option)." ex = MissingOptionException(msg) ex.missingOption = "task" raise ex
def validateOptions(self): SubCommand.validateOptions(self) if self.options.sort is not None: sortOpts = [ "state", "site", "runtime", "memory", "cpu", "retries", "waste", "exitcode" ] if self.options.sort not in sortOpts: msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Only the following values are accepted for --sort option: %s" % ( sortOpts) raise ConfigurationException(msg) if self.options.jobids: jobidstuple = validateJobids(self.options.jobids) self.jobids = [str(jobid) for (_, jobid) in jobidstuple] if self.options.jobids and not (self.options.long or self.options.sort): raise ConfigurationException( "Parameter --jobids can only be used in combination " "with --long or --sort options.")
def validateOptions(self): ## Check if the --logpath option was used. If it was, don't require the task ## option. if self.options.logpath is not None: if not os.path.isfile(self.options.logpath): msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Could not find the log file %s." % ( self.options.logpath) raise ConfigurationException(msg) self.cmdconf['requiresDirOption'] = False ## Do the options validation from SubCommand. try: SubCommand.validateOptions(self) except MissingOptionException as ex: if ex.missingOption == "task": msg = "%sError%s:" % (colors.RED, colors.NORMAL) msg += " Please provide a path to a CRAB project directory (use the -d/--dir option)" msg += " or to a log file (use the --logpath option)." ex = MissingOptionException(msg) ex.missingOption = "task" raise ex
def validateOptions(self): SubCommand.validateOptions(self) if hasattr(self.options, 'logpath') and self.options.logpath != None: if not path.isfile(self.options.logpath): msg = '%sError%s: Could not find the log file in the path: %s' % (colors.RED,colors.NORMAL,self.options.logpath) raise ConfigurationException(msg) elif hasattr(self.options, 'task'): self.cmdconf['requiresTaskOption'] = True self.cmdconf['useCache'] = True if self.options.task == None: if len(self.args) == 1 and self.args[0]: self.options.task = self.args[0] #for the case of 'crab uploadlog', user is using the .crab3 file if self.options.task == None and hasattr(self, 'crab3dic'): if self.crab3dic["taskname"] != None: self.options.task = self.crab3dic["taskname"] else: msg = '%sError%s: Please use the directory option -d or --logpath to specify which log to upload' % (colors.RED, colors.NORMAL) raise ConfigurationException(msg) else: msg = '%sError%s: Please use the directory option -d or --logpath to specify which log to upload' % (colors.RED, colors.NORMAL) raise ConfigurationException(msg)
def validateOptions(self): #Figuring out the destination directory SubCommand.validateOptions(self) self.dest = None if self.options.outputpath is not None: if re.match("^[a-z]+://", self.options.outputpath): self.dest = self.options.outputpath elif not os.path.isabs( self.options.outputpath ): self.dest = os.path.abspath( self.options.outputpath ) else: self.dest = self.options.outputpath #convert all to -1 if getattr(self.options, 'quantity', None) == 'all': self.options.quantity = -1 #check the format of jobids if getattr(self.options, 'jobids', None): self.options.jobids = validateJobids(self.options.jobids) if hasattr(self.options, 'command') and self.options.command != None: AvailableCommands = ['LCG', 'GFAL'] self.command = self.options.command.upper() if self.command not in AvailableCommands: msg = "You specified to use %s command and it is not allowed. Available commands are: %s " % (self.command, str(AvailableCommands)) ex = ConfigurationException(msg) raise ex else: self.command = None if hasattr(self.options, 'checksum'): if re.match('^yes$|^no$', self.options.checksum): self.checksum = 'ADLER32' if self.options.checksum == 'yes' else None else: msg = "You specified to use %s checksum. Only lowercase yes/no is accepted to turn ADLER32 checksum" % self.options.checksum ex = ConfigurationException(msg) raise ex
def validateOptions(self): SubCommand.validateOptions(self)
def validateOptions(self): SubCommand.validateOptions(self) if not hasattr(self.options, 'sitename') or self.options.sitename is None: self.logger.info("Missing site name, use '--site' option") raise MissingOptionException
def validateOptions(self): SubCommand.validateOptions(self)
def validateOptions(self): """ Check if the sitelist parameter is a comma separater list of cms sitenames, and put the strings to be passed to the server to self """ SubCommand.validateOptions(self) if self.options.publication: if self.options.sitewhitelist is not None or self.options.siteblacklist is not None or \ self.options.maxjobruntime is not None or self.options.maxmemory is not None or \ self.options.numcores is not None or self.options.priority is not None: msg = "The options --sitewhitelist, --siteblacklist," msg += " --maxjobruntime, --maxmemory, --numcores and --priority" msg += " can not be specified together with the option --publication." msg += " The last option is to only resubmit (failed) publications," msg += " in which case all of the first options make no sense." raise ConfigurationException(msg) if self.options.jobids: msg = "The option --jobids" msg += " can not be specified together with the option --publication." msg += " The last option is to only resubmit (failed) publications," msg += " which does not allow yet filtering on job ids (ALL failed publications will be resubmitted)." raise ConfigurationException(msg) if self.options.force: msg = "The option --force" msg += " can not be specified together with the option --publication." msg += " The last option is to only resubmit failed publications." msg += " Publications in a status other than 'failed' can not be resubmitted." raise ConfigurationException(msg) ## The --jobids option indicates which jobs have to be resubmitted. If it is not ## given, then all jobs in the task that are not running or successfully ## completed are resubmitted. If the user provides a list of job ids, then also ## successfully completed jobs can be resubmitted. ## Check the format of the jobids option. if self.options.jobids: jobidstuple = validateJobids(self.options.jobids) self.jobids = [str(jobid) for (_, jobid) in jobidstuple] ## The --force option should not be accepted unless combined with a user-given ## list of job ids via --jobids. if self.options.force and not self.jobids: msg = "Option --force can only be used in combination with option --jobids." raise ConfigurationException(msg) ## Covention used for the job parameters that the user can set when doing job ## resubmission (i.e. siteblacklist, sitewhitelist, maxjobruntime, maxmemory, ## numcores and priority): ## - If the user doesn't set a parameter we don't pass it to the server and the ## the server copies the original value the parameter had at task submission. ## It copies it from the Task DB. Therefore we need to keep these parameters ## in separate columns of the Task DB containing their original values. ## - For the site black- and whitelists, if the user passes an empty string, ## e.g. --siteblacklist='', we pass to the server siteblacklist=empty and the ## server interprets this as and empty list ([]). If the user passes a given ## list of sites, this new list overwrittes the original one. ## - The values of the parameters are used only for the resubmitted jobs (for ## their first resubmission and all next automatic resubmissions). #Checking if the sites provided by the user are valid cmsnames. Doing this because with only the #server error handling we get: # Server answered with: Invalid input parameter # Reason is: Incorrect 'siteblacklist' parameter #which is not really user friendly. #Moreover, I prefer to be independent from Lexicon. I'll the regex here. sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression sn_rec = re.compile(sn_re) #sn_rec => SiteName_RegularExpressionCompiled for sitelist in ['sitewhitelist', 'siteblacklist']: if getattr(self.options, sitelist) is not None: if getattr(self.options, sitelist) != "": for site_name in getattr(self.options, sitelist).split(','): if '*' not in site_name and not sn_rec.match(site_name): msg = "The site name '%s' does not look like a valid CMS site name" % (site_name) msg += " (it is not matching the regular expression '%s')." % (sn_re) raise ConfigurationException(msg) setattr(self, sitelist, getattr(self.options, sitelist).split(',')) else: setattr(self, sitelist, []) ## Sanity checks for task sizes. Limits are purposely fairly generous to provide ## some level of future-proofing. The server may restrict further. if self.options.maxjobruntime is not None: if self.options.maxjobruntime < 60 or self.options.maxjobruntime > 336*60: msg = "The requested maximum job runtime (%d minutes) must be between 60 and 20160 minutes." % (self.options.maxjobruntime) raise ConfigurationException(msg) if self.options.maxmemory is not None: if self.options.maxmemory < 30 or self.options.maxmemory > 1024*30: msg = "The requested per-job memory (%d MB) must be between 30 and 30720 MB." % (self.options.maxmemory) raise ConfigurationException(msg) if self.options.numcores is not None: if self.options.numcores < 1 or self.options.numcores > 128: msg = "The requested number of cores (%d) must be between 1 and 128." % (self.options.numcores) raise ConfigurationException(msg) if self.options.priority is not None: if self.options.priority < 1: msg = "The requested priority (%d) must be greater than 0." % (self.options.priority) raise ConfigurationException(msg)
def validateOptions(self): """ Check if the output file is given and set as attribute """ SubCommand.validateOptions(self) setattr(self, 'outdir', getattr(self.options, 'outdir', None))
def validateOptions(self): SubCommand.validateOptions(self) if self.options.scheddonly and self.options.cacheonly: self.logger.info('Options --schedd and --cache can not be specified simultaneously. No purging will be done.') raise ConfigException
def validateOptions(self): """ Check if the sitelist parameter is a comma separater list of cms sitenames, and put the strings to be passed to the server to self """ SubCommand.validateOptions(self) serverFactory = CRABClient.Emulator.getEmulator('rest') self.server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) uri = getUrl(self.instance, resource='task') crabDBInfo, _, _ = self.server.get(uri, data={ 'subresource': 'search', 'workflow': self.cachedinfo['RequestName'] }) self.splitting = getColumn(crabDBInfo, 'tm_split_algo') if self.options.publication: if self.options.sitewhitelist is not None or self.options.siteblacklist is not None or \ self.options.maxjobruntime is not None or self.options.maxmemory is not None or \ self.options.numcores is not None or self.options.priority is not None: msg = "The options --sitewhitelist, --siteblacklist," msg += " --maxjobruntime, --maxmemory, --numcores and --priority" msg += " can not be specified together with the option --publication." msg += " The last option is to only resubmit (failed) publications," msg += " in which case all of the first options make no sense." raise ConfigurationException(msg) if self.options.jobids: msg = "The option --jobids" msg += " can not be specified together with the option --publication." msg += " The last option is to only resubmit (failed) publications," msg += " which does not allow yet filtering on job ids (ALL failed publications will be resubmitted)." raise ConfigurationException(msg) if self.options.force: msg = "The option --force" msg += " can not be specified together with the option --publication." msg += " The last option is to only resubmit failed publications." msg += " Publications in a status other than 'failed' can not be resubmitted." raise ConfigurationException(msg) ## The --jobids option indicates which jobs have to be resubmitted. If it is not ## given, then all jobs in the task that are not running or successfully ## completed are resubmitted. If the user provides a list of job ids, then also ## successfully completed jobs can be resubmitted. ## Check the format of the jobids option. if self.options.jobids: jobidstuple = validateJobids(self.options.jobids, self.splitting != 'Automatic') self.jobids = [str(jobid) for (_, jobid) in jobidstuple] ## The --force option should not be accepted unless combined with a user-given ## list of job ids via --jobids. if self.options.force and not self.jobids: msg = "Option --force can only be used in combination with option --jobids." raise ConfigurationException(msg) ## Covention used for the job parameters that the user can set when doing job ## resubmission (i.e. siteblacklist, sitewhitelist, maxjobruntime, maxmemory, ## numcores and priority): ## - If the user doesn't set a parameter we don't pass it to the server and the ## the server copies the original value the parameter had at task submission. ## It copies it from the Task DB. Therefore we need to keep these parameters ## in separate columns of the Task DB containing their original values. ## - For the site black- and whitelists, if the user passes an empty string, ## e.g. --siteblacklist='', we pass to the server siteblacklist=empty and the ## server interprets this as and empty list ([]). If the user passes a given ## list of sites, this new list overwrittes the original one. ## - The values of the parameters are used only for the resubmitted jobs (for ## their first resubmission and all next automatic resubmissions). #Checking if the sites provided by the user are valid cmsnames. Doing this because with only the #server error handling we get: # Server answered with: Invalid input parameter # Reason is: Incorrect 'siteblacklist' parameter #which is not really user friendly. #Moreover, I prefer to be independent from Lexicon. I'll the regex here. sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression sn_rec = re.compile( sn_re) #sn_rec => SiteName_RegularExpressionCompiled for sitelist in ['sitewhitelist', 'siteblacklist']: if getattr(self.options, sitelist) is not None: if getattr(self.options, sitelist) != "": for site_name in getattr(self.options, sitelist).split(','): if '*' not in site_name and not sn_rec.match( site_name): msg = "The site name '%s' does not look like a valid CMS site name" % ( site_name) msg += " (it is not matching the regular expression '%s')." % ( sn_re) raise ConfigurationException(msg) setattr(self, sitelist, getattr(self.options, sitelist).split(',')) else: setattr(self, sitelist, []) ## Sanity checks for task sizes. Limits are purposely fairly generous to provide ## some level of future-proofing. The server may restrict further. if self.options.maxjobruntime is not None: if self.options.maxjobruntime < 60 or self.options.maxjobruntime > 336 * 60: msg = "The requested maximum job runtime (%d minutes) must be between 60 and 20160 minutes." % ( self.options.maxjobruntime) raise ConfigurationException(msg) if self.options.maxmemory is not None: if self.options.maxmemory < 30 or self.options.maxmemory > 1024 * 30: msg = "The requested per-job memory (%d MB) must be between 30 and 30720 MB." % ( self.options.maxmemory) raise ConfigurationException(msg) if self.options.numcores is not None: if self.options.numcores < 1 or self.options.numcores > 128: msg = "The requested number of cores (%d) must be between 1 and 128." % ( self.options.numcores) raise ConfigurationException(msg) if self.options.priority is not None: if self.options.priority < 1: msg = "The requested priority (%d) must be greater than 0." % ( self.options.priority) raise ConfigurationException(msg)
def validateOptions(self): """ Check if the output file is given and set as attribute """ SubCommand.validateOptions(self) setattr(self, 'outdir', getattr(self.options, 'outdir', None))