def validateOptions(self):
"""
Check if the sitelist parameter is a comma separater list of cms sitenames,
and put the strings to be passed to the server to self
"""
SubCommand.validateOptions(self)
#Checking if the sites provided by the user are valid cmsnames. Doing this because with only the
#server error handling we get:
# Server answered with: Invalid input parameter
# Reason is: Incorrect 'siteblacklist' parameter
#which is not really user friendly.
#Moreover, I prefer to be independent from Lexicon. I'll the regex here.
sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression
sn_rec = re.compile(
sn_re) #sn_rec => SiteName_RegularExpressionCompiled
self.sitewhitelist = ''
self.siteblsklist = ''
for siteList in ['sitewhitelist', 'siteblacklist']:
result = ''
paramVal = getattr(self.options, siteList, None)
if paramVal:
for site in paramVal.split(','):
if not sn_rec.match(site):
raise ConfigException(
"The sitename %s dows not look like a valid CMS name (not matching %s)"
% (site, sn_re))
result += "&%s=%s" % (siteList, site)
setattr(self, siteList, result)
def validateOptions(self):
SubCommand.validateOptions(self)
if hasattr(self.options, 'logpath') and self.options.logpath != None:
if not path.isfile(self.options.logpath):
msg = '%sError%s: Could not find the log file in the path: %s' % (
colors.RED, colors.NORMAL, self.options.logpath)
raise ConfigurationException(msg)
elif hasattr(self.options, 'task'):
self.cmdconf['requiresTaskOption'] = True
self.cmdconf['useCache'] = True
if self.options.task == None:
if len(self.args) == 1 and self.args[0]:
self.options.task = self.args[0]
#for the case of 'crab uploadlog', user is using the .crab3 file
if self.options.task == None and hasattr(self, 'crab3dic'):
if self.crab3dic["taskname"] != None:
self.options.task = self.crab3dic["taskname"]
else:
msg = '%sError%s: Please use the directory option -d or --logpath to specify which log to upload' % (
colors.RED, colors.NORMAL)
raise ConfigurationException(msg)
else:
msg = '%sError%s: Please use the directory option -d or --logpath to specify which log to upload' % (
colors.RED, colors.NORMAL)
raise ConfigurationException(msg)
def validateOptions(self):
"""
Check if the sitelist parameter is a comma separater list of cms sitenames,
and put the strings to be passed to the server to self
"""
SubCommand.validateOptions(self)
#Checking if the sites provided by the user are valid cmsnames. Doing this because with only the
#server error handling we get:
# Server answered with: Invalid input parameter
# Reason is: Incorrect 'siteblacklist' parameter
#which is not really user friendly.
#Moreover, I prefer to be independent from Lexicon. I'll the regex here.
sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression
sn_rec = re.compile(sn_re) #sn_rec => SiteName_RegularExpressionCompiled
self.sitewhitelist = ''
self.siteblsklist = ''
for siteList in ['sitewhitelist', 'siteblacklist']:
result = ''
paramVal = getattr(self.options, siteList, None)
if paramVal:
for site in paramVal.split(','):
if not sn_rec.match(site):
raise ConfigException("The sitename %s dows not look like a valid CMS name (not matching %s)" % (site, sn_re) )
result += "&%s=%s" % (siteList, site)
setattr(self, siteList, result)
#check the format of jobids
self.jobids = ''
if getattr(self.options, 'jobids', None):
self.jobids = validateJobids(self.options.jobids)
def validateOptions(self):
#Figuring out the destination directory
SubCommand.validateOptions(self)
self.dest = None
if self.options.outputpath is not None:
if not os.path.isabs(self.options.outputpath):
self.dest = os.path.abspath(self.options.outputpath)
else:
self.dest = self.options.outputpath
#convert all to -1
if getattr(self.options, 'quantity', None) == 'all':
self.options.quantity = -1
#check the format of jobids
if getattr(self.options, 'jobids', None):
if re.compile('^\d+(,\d+)*$').match(self.options.jobids):
self.options.jobids = [
('jobids', jobid)
for jobid in self.options.jobids.split(',')
]
else:
raise ConfigurationException(
"The command line option jobids should be a comma separated list of integers"
)
def validateOptions(self):
SubCommand.validateOptions(self)
if self.options.idle and (self.options.long or self.options.summary):
raise ConfigurationException(
"Idle option (-i) conflicts with -u, and -l")
self.json = self.options.json
self.summary = self.options.summary
self.idle = self.options.idle
self.long = self.options.long
acceptedsort = [
"state", "site", "runtime", "memory", "cpu", "retries", "waste",
"exitcode"
]
if hasattr(self.options, 'sort') and self.options.sort != None:
if not self.long:
raise ConfigurationException(
'%sError%s: Please use option --long together with --sort'
% (colors.RED, colors.NORMAL))
elif not self.options.sort in acceptedsort:
raise ConfigurationException(
'%sError%s: Only these values are accepted for crab status --sort option: %s'
% (colors.RED, colors.NORMAL, acceptedsort))
else:
self.sort = self.options.sort
def validateOptions(self):
SubCommand.validateOptions(self)
#check the format of jobids
self.jobids = ''
if getattr(self.options, 'jobids', None):
self.jobids = validateJobids(self.options.jobids)
def validateOptions(self):
SubCommand.validateOptions(self)
if self.options.sitename is None:
msg = "%sError%s: Please specify the site where to check the write permissions." % (colors.RED, colors.NORMAL)
msg += " Use the --site option."
ex = MissingOptionException(msg)
ex.missingOption = "sitename"
raise ex
if hasattr(self.options, 'command') and self.options.command != None:
AvailableCommands = ['LCG', 'GFAL']
self.command = self.options.command.upper()
if self.command not in AvailableCommands:
msg = "You specified to use %s command and it is not allowed. Available commands are: %s " % (self.command, str(AvailableCommands))
ex = ConfigurationException(msg)
raise ex
else:
self.command = None
if hasattr(self.options, 'checksum'):
if re.match('^yes$|^no$', self.options.checksum):
self.checksum = 'ADLER32' if self.options.checksum == 'yes' else None
else:
msg = "You specified to use %s checksum. Only lowercase yes/no is accepted to turn ADLER32 checksum" % self.options.checksum
ex = ConfigurationException(msg)
raise ex
def validateOptions(self):
SubCommand.validateOptions(self)
if not hasattr(self.options,
'sitename') or self.options.sitename is None:
self.logger.info("Missing site name, use '--site' option")
raise MissingOptionException
def validateOptions(self):
"""
After doing the general options validation from the parent SubCommand class,
do the validation of options that are specific to the submit command.
"""
## First call validateOptions() from the SubCommand class.
SubCommand.validateOptions(self)
## If no configuration file was passed as an option, try to extract it from the arguments.
## Assume that the arguments can only be:
## 1) the configuration file name, and
## 2) parameters to override in the configuration file.
## The last ones should all contain an '=' sign, so these are not candidates to be the
## configuration file argument. Also, the configuration file name should end with '.py'.
## If can not find a configuration file candidate, use the default 'crabConfig.py'.
## If find more than one candidate, raise ConfigurationException.
if self.options.config is None:
use_default = True
if len(self.args):
config_candidates = [(arg,i) for i,arg in enumerate(self.args) if '=' not in arg and arg[-3:] == '.py']
config_candidate_names = set([config_candidate_name for (config_candidate_name,_) in config_candidates])
if len(config_candidate_names) == 1:
self.options.config = config_candidates[0][0]
del self.args[config_candidates[0][1]]
use_default = False
elif len(config_candidate_names) > 1:
self.logger.info('Unable to unambiguously extract the configuration file from the command-line arguments.')
self.logger.info('Possible candidates are: %s' % list(config_candidate_names))
raise ConfigurationException('ERROR: Unable to extract configuration file from command-line arguments.')
if use_default:
self.options.config = 'crabConfig.py'
def validateOptions(self):
SubCommand.validateOptions(self)
#check the format of jobids
self.jobids = ''
if getattr(self.options, 'jobids', None):
self.jobids = validateJobids(self.options.jobids)
def validateOptions(self):
#Figuring out the destination directory
SubCommand.validateOptions(self)
if self.options.outputpath is not None:
if re.match("^[a-z]+://", self.options.outputpath):
self.dest = self.options.outputpath
elif not os.path.isabs( self.options.outputpath ):
self.dest = os.path.abspath( self.options.outputpath )
else:
self.dest = self.options.outputpath
#convert all to -1
if getattr(self.options, 'quantity', None) == 'all':
self.options.quantity = -1
if hasattr(self.options, 'command') and self.options.command != None:
AvailableCommands = ['LCG', 'GFAL']
self.command = self.options.command.upper()
if self.command not in AvailableCommands:
msg = "You specified to use %s command and it is not allowed. Available commands are: %s " % (self.command, str(AvailableCommands))
ex = ConfigurationException(msg)
raise ex
else:
self.command = None
if hasattr(self.options, 'checksum'):
if re.match('^yes$|^no$', self.options.checksum):
self.checksum = 'ADLER32' if self.options.checksum == 'yes' else None
else:
msg = "You specified to use %s checksum. Only lowercase yes/no is accepted to turn ADLER32 checksum" % self.options.checksum
ex = ConfigurationException(msg)
raise ex
def validateOptions(self):
SubCommand.validateOptions(self)
if self.options.scheddonly and self.options.cacheonly:
self.logger.info(
'Options --schedd and --cache can not be specified simultaneously. No purging will be done.'
)
raise ConfigException
def validateOptions(self):
SubCommand.validateOptions(self)
if self.options.jobid is not None:
try:
int(self.options.jobid)
except ValueError:
raise ClientException("The --jobid option has to be an integer")
def validateOptions(self):
"""
After doing the general options validation from the parent SubCommand class,
do the validation of options that are specific to the submit command.
"""
## First do the basic validation in the SubCommand.
SubCommand.validateOptions(self)
validateSubmitOptions(self.options, self.args)
def validateOptions(self):
"""
Check if the sitelist parameter is a comma separater list of cms sitenames,
and put the strings to be passed to the server to self
"""
SubCommand.validateOptions(self)
#Checking if the sites provided by the user are valid cmsnames. Doing this because with only the
#server error handling we get:
# Server answered with: Invalid input parameter
# Reason is: Incorrect 'siteblacklist' parameter
#which is not really user friendly.
#Moreover, I prefer to be independent from Lexicon. I'll the regex here.
sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression
sn_rec = re.compile(sn_re) #sn_rec => SiteName_RegularExpressionCompiled
self.sitewhitelist = ''
self.siteblsklist = ''
for siteList in ['sitewhitelist', 'siteblacklist']:
result = ''
paramVal = getattr(self.options, siteList, None)
if paramVal:
for site in paramVal.split(','):
if not sn_rec.match(site):
raise ConfigException("The sitename %s dows not look like a valid CMS name (not matching %s)" % (site, sn_re) )
result += "&%s=%s" % (siteList, site)
setattr(self, siteList, result)
#check the format of jobids
self.jobids = ''
if getattr(self.options, 'jobids', None):
self.jobids = validateJobids(self.options.jobids)
# Sanity checks for task sizes. Limits are purposely fairly generous to provide some level of future-proofing.
# The server may restrict further.
self.numcores = None
if self.options.numcores != None:
if self.options.numcores < 1 or self.options.numcores > 128:
raise ConfigException("The number of requested cores (%d) must be between 1 and 128." % (self.options.numcores))
self.numcores = str(self.options.numcores)
self.maxjobruntime = None
if self.options.maxjobruntime != None:
if self.options.maxjobruntime < 1 or self.options.maxjobruntime > 336:
raise ConfigException("The requested max job runtime (%d hours) must be between 1 and 336 hours." % self.options.maxjobruntime)
self.maxjobruntime = str(self.options.maxjobruntime)
self.maxmemory = None
if self.options.maxmemory != None:
if self.options.maxmemory < 30 or self.options.maxmemory > 1024*30:
raise ConfigException("The requested per-job memory (%d MB) must be between 30 and 30720 MB." % self.options.maxmemory)
self.maxmemory = str(self.options.maxmemory)
self.priority = None
if self.options.priority != None:
self.priority = str(self.options.priority)
def validateOptions(self):
SubCommand.validateOptions(self)
if self.options.sitename is None:
msg = "%sError%s: Please specify the site where to check the write permissions." % (colors.RED, colors.NORMAL)
msg += " Use the --site option."
ex = MissingOptionException(msg)
ex.missingOption = "sitename"
raise ex
def validateOptions(self):
SubCommand.validateOptions(self)
if self.options.idle and (self.options.long or self.options.summary ):
raise ConfigurationException("Idle option (-i) conflicts with -u, and -l")
self.long = self.options.long
self.json = self.options.json
self.summary = self.options.summary
self.idle = self.options.idle
def validateOptions(self):
"""
After doing the general options validation from the parent SubCommand class,
do the validation of options that are specific to the submit command.
"""
## First do the basic validation in the SubCommand.
SubCommand.validateOptions(self)
validateSubmitOptions(self.options, self.args)
def validateOptions(self):
"""
Check if the output file is given and set as attribute
"""
SubCommand.validateOptions(self)
if not re.match('^yes$|^no$', self.options.usedbs):
raise ConfigurationException("--dbs option only accepts the yes and no values (--dbs=yes or --dbs=no)")
self.usedbs = 1 if self.options.usedbs=='yes' else 0
setattr(self, 'outdir', getattr(self.options, 'outdir', None))
def validateOptions(self):
SubCommand.validateOptions(self)
if self.options.idle and (self.options.long or self.options.summary):
raise ConfigurationException("Option --idle conflicts with --summary and --long")
if self.options.sort is not None:
sortOpts = ["state", "site", "runtime", "memory", "cpu", "retries", "waste", "exitcode"]
if self.options.sort not in sortOpts:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Only the following values are accepted for --sort option: %s" % (sortOpts)
raise ConfigurationException(msg)
def validateOptions(self):
## Do the options validation from SubCommand.
try:
SubCommand.validateOptions(self)
except MissingOptionException as ex:
if ex.missingOption == "task":
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Please provide a path to a CRAB project directory (use the -d/--dir option)"
msg += " or to a log file (use the --logpath option)."
ex = MissingOptionException(msg)
ex.missingOption = "task"
raise ex
def validateOptions(self):
"""
Check if the output file is given and set as attribute
"""
SubCommand.validateOptions(self)
if not re.match('^yes$|^no$', self.options.usedbs):
raise ConfigurationException(
"--dbs option only accepts the yes and no values (--dbs=yes or --dbs=no)"
)
self.usedbs = 1 if self.options.usedbs == 'yes' else 0
setattr(self, 'outdir', getattr(self.options, 'outdir', None))
def validateOptions(self):
SubCommand.validateOptions(self)
if self.options.idle and (self.options.long or self.options.summary or self.options.publication):
raise ConfigurationException("Idle option (-i) conflicts with -u, -l, and -p")
if self.options.publication and (self.options.long or self.options.summary or self.options.idle):
raise ConfigurationException("Publication option (-p) conflicts with -u, -l, and -i")
self.long = self.options.long
self.json = self.options.json
self.summary = self.options.summary
self.idle = self.options.idle
self.publication = self.options.publication
def validateOptions(self):
"""
Check if the sitelist parameter is a comma separater list of cms sitenames,
and put the strings to be passed to the server to self
"""
SubCommand.validateOptions(self)
## Check the format of the jobids option.
if getattr(self.options, 'jobids'):
jobidstuple = validateJobids(self.options.jobids)
self.jobids = [str(jobid) for (_, jobid) in jobidstuple]
#Checking if the sites provided by the user are valid cmsnames. Doing this because with only the
#server error handling we get:
# Server answered with: Invalid input parameter
# Reason is: Incorrect 'siteblacklist' parameter
#which is not really user friendly.
#Moreover, I prefer to be independent from Lexicon. I'll the regex here.
sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression
sn_rec = re.compile(sn_re) #sn_rec => SiteName_RegularExpressionCompiled
for sitelist in ['sitewhitelist', 'siteblacklist']:
if getattr(self.options, sitelist) is not None:
for i, site_name in enumerate(getattr(self.options, sitelist).split(',')):
if not sn_rec.match(site_name):
msg = "The site name %s does not look like a valid CMS site name" % (site_name)
msg += " (it is not matching the regular expression %s)." % (sn_re)
raise ConfigurationException(msg)
setattr(self, sitelist, getattr(self.options, sitelist).split(','))
## Sanity checks for task sizes. Limits are purposely fairly generous to provide
## some level of future-proofing. The server may restrict further.
if self.options.maxjobruntime is not None:
if self.options.maxjobruntime < 60 or self.options.maxjobruntime > 336*60:
msg = "The requested maximum job runtime (%d minutes) must be between 60 and 20160 minutes." % (self.options.maxjobruntime)
raise ConfigurationException(msg)
self.maxjobruntime = str(self.options.maxjobruntime)
if self.options.maxmemory is not None:
if self.options.maxmemory < 30 or self.options.maxmemory > 1024*30:
msg = "The requested per-job memory (%d MB) must be between 30 and 30720 MB." % (self.options.maxmemory)
raise ConfigurationException(msg)
self.maxmemory = str(self.options.maxmemory)
if self.options.numcores is not None:
if self.options.numcores < 1 or self.options.numcores > 128:
msg = "The requested number of cores (%d) must be between 1 and 128." % (self.options.numcores)
raise ConfigurationException(msg)
self.numcores = str(self.options.numcores)
if self.options.priority is not None:
self.priority = str(self.options.priority)
def validateOptions(self):
"""
Check if the output file is given and set as attribute
"""
SubCommand.validateOptions(self)
if self.options.usedbs is not None:
msg = "CRAB command option error: the option --dbs has been deprecated since CRAB v3.3.1603."
raise UnknownOptionException(msg)
recoveryMethods = ['notFinished', 'notPublished', 'failed']
if self.options.recovery not in recoveryMethods:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " The --recovery option only accepts the following values: %s" % (recoveryMethods)
raise ConfigurationException(msg)
def validateOptions(self):
"""
Check if the output file is given and set as attribute
"""
SubCommand.validateOptions(self)
if self.options.usedbs is not None:
msg = "CRAB command option error: the option --dbs has been deprecated since CRAB v3.3.1603."
raise UnknownOptionException(msg)
recoveryMethods = ['notFinished', 'notPublished', 'failed']
if self.options.recovery not in recoveryMethods:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " The --recovery option only accepts the following values: %s" % (
recoveryMethods)
raise ConfigurationException(msg)
def validateOptions(self):
SubCommand.validateOptions(self)
if self.options.idle and (self.options.long or self.options.summary):
raise ConfigurationException(
"Option --idle conflicts with --summary and --long")
if self.options.sort is not None:
sortOpts = [
"state", "site", "runtime", "memory", "cpu", "retries",
"waste", "exitcode"
]
if self.options.sort not in sortOpts:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Only the following values are accepted for --sort option: %s" % (
sortOpts)
raise ConfigurationException(msg)
def validateOptions(self):
#Figuring out the destination directory
SubCommand.validateOptions(self)
self.dest = None
if self.options.outputpath is not None:
if not os.path.isabs( self.options.outputpath ):
self.dest = os.path.abspath( self.options.outputpath )
else:
self.dest = self.options.outputpath
#convert all to -1
if getattr(self.options, 'quantity', None) == 'all':
self.options.quantity = -1
#check the format of jobids
if getattr(self.options, 'jobids', None):
self.options.jobids = validateJobids(self.options.jobids)
def validateOptions(self):
SubCommand.validateOptions(self)
if self.options.idle and (self.options.long or self.options.summary ):
raise ConfigurationException("Idle option (-i) conflicts with -u, and -l")
self.json = self.options.json
self.summary = self.options.summary
self.idle = self.options.idle
self.long = self.options.long
acceptedsort = ["state", "site", "runtime", "memory", "cpu", "retries", "waste"]
if hasattr(self.options , 'sort') and self.options.sort != None:
if not self.long:
raise ConfigurationException('%sError%s: Please use option --long together with --sort' % (colors.RED, colors.NORMAL))
elif not self.options.sort in acceptedsort:
raise ConfigurationException('%sError%s: Only this value accepted for crab status sort: %s' % (colors.RED, colors.NORMAL, acceptedsort))
else:
self.sort = self.options.sort
def validateOptions(self):
"""
After doing the general options validation from the parent SubCommand class,
do the validation of options that are specific to the submit command.
"""
## First call validateOptions() from the SubCommand class.
SubCommand.validateOptions(self)
## If no configuration file was passed as an option, try to extract it from the arguments.
## Assume that the arguments can only be:
## 1) the configuration file name, and
## 2) parameters to override in the configuration file.
## The last ones should all contain an '=' sign, so these are not candidates to be the
## configuration file argument. Also, the configuration file name should end with '.py'.
## If can not find a configuration file candidate, use the default 'crabConfig.py'.
## If find more than one candidate, raise ConfigurationException.
if self.options.config is None:
use_default = True
if len(self.args):
config_candidates = [(arg, i)
for i, arg in enumerate(self.args)
if '=' not in arg and arg[-3:] == '.py']
config_candidate_names = set([
config_candidate_name
for (config_candidate_name, _) in config_candidates
])
if len(config_candidate_names) == 1:
self.options.config = config_candidates[0][0]
del self.args[config_candidates[0][1]]
use_default = False
elif len(config_candidate_names) > 1:
self.logger.info(
'Unable to unambiguously extract the configuration file from the command-line arguments.'
)
self.logger.info('Possible candidates are: %s' %
list(config_candidate_names))
raise ConfigurationException(
'ERROR: Unable to extract configuration file from command-line arguments.'
)
if use_default:
self.options.config = 'crabConfig.py'
def validateOptions(self):
#Figuring out the destination directory
SubCommand.validateOptions(self)
self.dest = None
if self.options.outputpath is not None:
if not os.path.isabs( self.options.outputpath ):
self.dest = os.path.abspath( self.options.outputpath )
else:
self.dest = self.options.outputpath
#convert all to -1
if getattr(self.options, 'quantity', None) == 'all':
self.options.quantity = -1
#check the format of jobids
if getattr(self.options, 'jobids', None):
if re.compile('^\d+(,\d+)*$').match(self.options.jobids):
self.options.jobids = [('jobids',jobid) for jobid in self.options.jobids.split(',')]
else:
raise ConfigurationException("The command line option jobids should be a comma separated list of integers")
def validateOptions(self):
# Figuring out the destination directory
SubCommand.validateOptions(self)
self.dest = None
if self.options.outputpath is not None:
if re.match("^[a-z]+://", self.options.outputpath):
self.dest = self.options.outputpath
elif not os.path.isabs(self.options.outputpath):
self.dest = os.path.abspath(self.options.outputpath)
else:
self.dest = self.options.outputpath
# convert all to -1
if getattr(self.options, "quantity", None) == "all":
self.options.quantity = -1
# check the format of jobids
if getattr(self.options, "jobids", None):
self.options.jobids = validateJobids(self.options.jobids)
self.dump = self.options.dump
def validateOptions(self):
#Figuring out the destination directory
SubCommand.validateOptions(self)
self.dest = None
if self.options.outputpath is not None:
if re.match("^[a-z]+://", self.options.outputpath):
self.dest = self.options.outputpath
elif not os.path.isabs(self.options.outputpath):
self.dest = os.path.abspath(self.options.outputpath)
else:
self.dest = self.options.outputpath
#convert all to -1
if getattr(self.options, 'quantity', None) == 'all':
self.options.quantity = -1
#check the format of jobids
if getattr(self.options, 'jobids', None):
self.options.jobids = validateJobids(self.options.jobids)
self.dump = self.options.dump
def validateOptions(self):
## Check if the --logpath option was used. If it was, don't require the task
## option.
if self.options.logpath is not None:
if not os.path.isfile(self.options.logpath):
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Could not find the log file %s." % (self.options.logpath)
raise ConfigurationException(msg)
self.cmdconf["requiresDirOption"] = False
## Do the options validation from SubCommand.
try:
SubCommand.validateOptions(self)
except MissingOptionException as ex:
if ex.missingOption == "task":
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Please provide a path to a CRAB project directory (use the -d/--dir option)"
msg += " or to a log file (use the --logpath option)."
ex = MissingOptionException(msg)
ex.missingOption = "task"
raise ex
def validateOptions(self):
SubCommand.validateOptions(self)
if self.options.sort is not None:
sortOpts = [
"state", "site", "runtime", "memory", "cpu", "retries",
"waste", "exitcode"
]
if self.options.sort not in sortOpts:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Only the following values are accepted for --sort option: %s" % (
sortOpts)
raise ConfigurationException(msg)
if self.options.jobids:
jobidstuple = validateJobids(self.options.jobids)
self.jobids = [str(jobid) for (_, jobid) in jobidstuple]
if self.options.jobids and not (self.options.long
or self.options.sort):
raise ConfigurationException(
"Parameter --jobids can only be used in combination "
"with --long or --sort options.")
def validateOptions(self):
## Check if the --logpath option was used. If it was, don't require the task
## option.
if self.options.logpath is not None:
if not os.path.isfile(self.options.logpath):
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Could not find the log file %s." % (
self.options.logpath)
raise ConfigurationException(msg)
self.cmdconf['requiresDirOption'] = False
## Do the options validation from SubCommand.
try:
SubCommand.validateOptions(self)
except MissingOptionException as ex:
if ex.missingOption == "task":
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Please provide a path to a CRAB project directory (use the -d/--dir option)"
msg += " or to a log file (use the --logpath option)."
ex = MissingOptionException(msg)
ex.missingOption = "task"
raise ex
def validateOptions(self):
SubCommand.validateOptions(self)
if hasattr(self.options, 'logpath') and self.options.logpath != None:
if not path.isfile(self.options.logpath):
msg = '%sError%s: Could not find the log file in the path: %s' % (colors.RED,colors.NORMAL,self.options.logpath)
raise ConfigurationException(msg)
elif hasattr(self.options, 'task'):
self.cmdconf['requiresTaskOption'] = True
self.cmdconf['useCache'] = True
if self.options.task == None:
if len(self.args) == 1 and self.args[0]:
self.options.task = self.args[0]
#for the case of 'crab uploadlog', user is using the .crab3 file
if self.options.task == None and hasattr(self, 'crab3dic'):
if self.crab3dic["taskname"] != None:
self.options.task = self.crab3dic["taskname"]
else:
msg = '%sError%s: Please use the directory option -d or --logpath to specify which log to upload' % (colors.RED, colors.NORMAL)
raise ConfigurationException(msg)
else:
msg = '%sError%s: Please use the directory option -d or --logpath to specify which log to upload' % (colors.RED, colors.NORMAL)
raise ConfigurationException(msg)
def validateOptions(self):
#Figuring out the destination directory
SubCommand.validateOptions(self)
self.dest = None
if self.options.outputpath is not None:
if re.match("^[a-z]+://", self.options.outputpath):
self.dest = self.options.outputpath
elif not os.path.isabs( self.options.outputpath ):
self.dest = os.path.abspath( self.options.outputpath )
else:
self.dest = self.options.outputpath
#convert all to -1
if getattr(self.options, 'quantity', None) == 'all':
self.options.quantity = -1
#check the format of jobids
if getattr(self.options, 'jobids', None):
self.options.jobids = validateJobids(self.options.jobids)
if hasattr(self.options, 'command') and self.options.command != None:
AvailableCommands = ['LCG', 'GFAL']
self.command = self.options.command.upper()
if self.command not in AvailableCommands:
msg = "You specified to use %s command and it is not allowed. Available commands are: %s " % (self.command, str(AvailableCommands))
ex = ConfigurationException(msg)
raise ex
else:
self.command = None
if hasattr(self.options, 'checksum'):
if re.match('^yes$|^no$', self.options.checksum):
self.checksum = 'ADLER32' if self.options.checksum == 'yes' else None
else:
msg = "You specified to use %s checksum. Only lowercase yes/no is accepted to turn ADLER32 checksum" % self.options.checksum
ex = ConfigurationException(msg)
raise ex
def validateOptions(self):
SubCommand.validateOptions(self)
def validateOptions(self):
SubCommand.validateOptions(self)
if not hasattr(self.options, 'sitename') or self.options.sitename is None:
self.logger.info("Missing site name, use '--site' option")
raise MissingOptionException
def validateOptions(self):
SubCommand.validateOptions(self)
def validateOptions(self):
"""
Check if the sitelist parameter is a comma separater list of cms sitenames,
and put the strings to be passed to the server to self
"""
SubCommand.validateOptions(self)
if self.options.publication:
if self.options.sitewhitelist is not None or self.options.siteblacklist is not None or \
self.options.maxjobruntime is not None or self.options.maxmemory is not None or \
self.options.numcores is not None or self.options.priority is not None:
msg = "The options --sitewhitelist, --siteblacklist,"
msg += " --maxjobruntime, --maxmemory, --numcores and --priority"
msg += " can not be specified together with the option --publication."
msg += " The last option is to only resubmit (failed) publications,"
msg += " in which case all of the first options make no sense."
raise ConfigurationException(msg)
if self.options.jobids:
msg = "The option --jobids"
msg += " can not be specified together with the option --publication."
msg += " The last option is to only resubmit (failed) publications,"
msg += " which does not allow yet filtering on job ids (ALL failed publications will be resubmitted)."
raise ConfigurationException(msg)
if self.options.force:
msg = "The option --force"
msg += " can not be specified together with the option --publication."
msg += " The last option is to only resubmit failed publications."
msg += " Publications in a status other than 'failed' can not be resubmitted."
raise ConfigurationException(msg)
## The --jobids option indicates which jobs have to be resubmitted. If it is not
## given, then all jobs in the task that are not running or successfully
## completed are resubmitted. If the user provides a list of job ids, then also
## successfully completed jobs can be resubmitted.
## Check the format of the jobids option.
if self.options.jobids:
jobidstuple = validateJobids(self.options.jobids)
self.jobids = [str(jobid) for (_, jobid) in jobidstuple]
## The --force option should not be accepted unless combined with a user-given
## list of job ids via --jobids.
if self.options.force and not self.jobids:
msg = "Option --force can only be used in combination with option --jobids."
raise ConfigurationException(msg)
## Covention used for the job parameters that the user can set when doing job
## resubmission (i.e. siteblacklist, sitewhitelist, maxjobruntime, maxmemory,
## numcores and priority):
## - If the user doesn't set a parameter we don't pass it to the server and the
## the server copies the original value the parameter had at task submission.
## It copies it from the Task DB. Therefore we need to keep these parameters
## in separate columns of the Task DB containing their original values.
## - For the site black- and whitelists, if the user passes an empty string,
## e.g. --siteblacklist='', we pass to the server siteblacklist=empty and the
## server interprets this as and empty list ([]). If the user passes a given
## list of sites, this new list overwrittes the original one.
## - The values of the parameters are used only for the resubmitted jobs (for
## their first resubmission and all next automatic resubmissions).
#Checking if the sites provided by the user are valid cmsnames. Doing this because with only the
#server error handling we get:
# Server answered with: Invalid input parameter
# Reason is: Incorrect 'siteblacklist' parameter
#which is not really user friendly.
#Moreover, I prefer to be independent from Lexicon. I'll the regex here.
sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression
sn_rec = re.compile(sn_re) #sn_rec => SiteName_RegularExpressionCompiled
for sitelist in ['sitewhitelist', 'siteblacklist']:
if getattr(self.options, sitelist) is not None:
if getattr(self.options, sitelist) != "":
for site_name in getattr(self.options, sitelist).split(','):
if '*' not in site_name and not sn_rec.match(site_name):
msg = "The site name '%s' does not look like a valid CMS site name" % (site_name)
msg += " (it is not matching the regular expression '%s')." % (sn_re)
raise ConfigurationException(msg)
setattr(self, sitelist, getattr(self.options, sitelist).split(','))
else:
setattr(self, sitelist, [])
## Sanity checks for task sizes. Limits are purposely fairly generous to provide
## some level of future-proofing. The server may restrict further.
if self.options.maxjobruntime is not None:
if self.options.maxjobruntime < 60 or self.options.maxjobruntime > 336*60:
msg = "The requested maximum job runtime (%d minutes) must be between 60 and 20160 minutes." % (self.options.maxjobruntime)
raise ConfigurationException(msg)
if self.options.maxmemory is not None:
if self.options.maxmemory < 30 or self.options.maxmemory > 1024*30:
msg = "The requested per-job memory (%d MB) must be between 30 and 30720 MB." % (self.options.maxmemory)
raise ConfigurationException(msg)
if self.options.numcores is not None:
if self.options.numcores < 1 or self.options.numcores > 128:
msg = "The requested number of cores (%d) must be between 1 and 128." % (self.options.numcores)
raise ConfigurationException(msg)
if self.options.priority is not None:
if self.options.priority < 1:
msg = "The requested priority (%d) must be greater than 0." % (self.options.priority)
raise ConfigurationException(msg)
def validateOptions(self):
"""
Check if the output file is given and set as attribute
"""
SubCommand.validateOptions(self)
setattr(self, 'outdir', getattr(self.options, 'outdir', None))
def validateOptions(self):
SubCommand.validateOptions(self)
if self.options.scheddonly and self.options.cacheonly:
self.logger.info('Options --schedd and --cache can not be specified simultaneously. No purging will be done.')
raise ConfigException
def validateOptions(self):
"""
Check if the sitelist parameter is a comma separater list of cms sitenames,
and put the strings to be passed to the server to self
"""
SubCommand.validateOptions(self)
serverFactory = CRABClient.Emulator.getEmulator('rest')
self.server = serverFactory(self.serverurl,
self.proxyfilename,
self.proxyfilename,
version=__version__)
uri = getUrl(self.instance, resource='task')
crabDBInfo, _, _ = self.server.get(uri,
data={
'subresource':
'search',
'workflow':
self.cachedinfo['RequestName']
})
self.splitting = getColumn(crabDBInfo, 'tm_split_algo')
if self.options.publication:
if self.options.sitewhitelist is not None or self.options.siteblacklist is not None or \
self.options.maxjobruntime is not None or self.options.maxmemory is not None or \
self.options.numcores is not None or self.options.priority is not None:
msg = "The options --sitewhitelist, --siteblacklist,"
msg += " --maxjobruntime, --maxmemory, --numcores and --priority"
msg += " can not be specified together with the option --publication."
msg += " The last option is to only resubmit (failed) publications,"
msg += " in which case all of the first options make no sense."
raise ConfigurationException(msg)
if self.options.jobids:
msg = "The option --jobids"
msg += " can not be specified together with the option --publication."
msg += " The last option is to only resubmit (failed) publications,"
msg += " which does not allow yet filtering on job ids (ALL failed publications will be resubmitted)."
raise ConfigurationException(msg)
if self.options.force:
msg = "The option --force"
msg += " can not be specified together with the option --publication."
msg += " The last option is to only resubmit failed publications."
msg += " Publications in a status other than 'failed' can not be resubmitted."
raise ConfigurationException(msg)
## The --jobids option indicates which jobs have to be resubmitted. If it is not
## given, then all jobs in the task that are not running or successfully
## completed are resubmitted. If the user provides a list of job ids, then also
## successfully completed jobs can be resubmitted.
## Check the format of the jobids option.
if self.options.jobids:
jobidstuple = validateJobids(self.options.jobids,
self.splitting != 'Automatic')
self.jobids = [str(jobid) for (_, jobid) in jobidstuple]
## The --force option should not be accepted unless combined with a user-given
## list of job ids via --jobids.
if self.options.force and not self.jobids:
msg = "Option --force can only be used in combination with option --jobids."
raise ConfigurationException(msg)
## Covention used for the job parameters that the user can set when doing job
## resubmission (i.e. siteblacklist, sitewhitelist, maxjobruntime, maxmemory,
## numcores and priority):
## - If the user doesn't set a parameter we don't pass it to the server and the
## the server copies the original value the parameter had at task submission.
## It copies it from the Task DB. Therefore we need to keep these parameters
## in separate columns of the Task DB containing their original values.
## - For the site black- and whitelists, if the user passes an empty string,
## e.g. --siteblacklist='', we pass to the server siteblacklist=empty and the
## server interprets this as and empty list ([]). If the user passes a given
## list of sites, this new list overwrittes the original one.
## - The values of the parameters are used only for the resubmitted jobs (for
## their first resubmission and all next automatic resubmissions).
#Checking if the sites provided by the user are valid cmsnames. Doing this because with only the
#server error handling we get:
# Server answered with: Invalid input parameter
# Reason is: Incorrect 'siteblacklist' parameter
#which is not really user friendly.
#Moreover, I prefer to be independent from Lexicon. I'll the regex here.
sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression
sn_rec = re.compile(
sn_re) #sn_rec => SiteName_RegularExpressionCompiled
for sitelist in ['sitewhitelist', 'siteblacklist']:
if getattr(self.options, sitelist) is not None:
if getattr(self.options, sitelist) != "":
for site_name in getattr(self.options,
sitelist).split(','):
if '*' not in site_name and not sn_rec.match(
site_name):
msg = "The site name '%s' does not look like a valid CMS site name" % (
site_name)
msg += " (it is not matching the regular expression '%s')." % (
sn_re)
raise ConfigurationException(msg)
setattr(self, sitelist,
getattr(self.options, sitelist).split(','))
else:
setattr(self, sitelist, [])
## Sanity checks for task sizes. Limits are purposely fairly generous to provide
## some level of future-proofing. The server may restrict further.
if self.options.maxjobruntime is not None:
if self.options.maxjobruntime < 60 or self.options.maxjobruntime > 336 * 60:
msg = "The requested maximum job runtime (%d minutes) must be between 60 and 20160 minutes." % (
self.options.maxjobruntime)
raise ConfigurationException(msg)
if self.options.maxmemory is not None:
if self.options.maxmemory < 30 or self.options.maxmemory > 1024 * 30:
msg = "The requested per-job memory (%d MB) must be between 30 and 30720 MB." % (
self.options.maxmemory)
raise ConfigurationException(msg)
if self.options.numcores is not None:
if self.options.numcores < 1 or self.options.numcores > 128:
msg = "The requested number of cores (%d) must be between 1 and 128." % (
self.options.numcores)
raise ConfigurationException(msg)
if self.options.priority is not None:
if self.options.priority < 1:
msg = "The requested priority (%d) must be greater than 0." % (
self.options.priority)
raise ConfigurationException(msg)
def validateOptions(self):
"""
Check if the output file is given and set as attribute
"""
SubCommand.validateOptions(self)
setattr(self, 'outdir', getattr(self.options, 'outdir', None))