def validateOptions(self):
SubCommand.validateOptions(self)
if self.options.sitename is None:
msg = "%sError%s: Please specify the site where to check the write permissions." % (colors.RED, colors.NORMAL)
msg += " Use the --site option."
ex = MissingOptionException(msg)
ex.missingOption = "sitename"
raise ex
if hasattr(self.options, 'command') and self.options.command != None:
AvailableCommands = ['LCG', 'GFAL']
self.command = self.options.command.upper()
if self.command not in AvailableCommands:
msg = "You specified to use %s command and it is not allowed. Available commands are: %s " % (self.command, str(AvailableCommands))
ex = ConfigurationException(msg)
raise ex
else:
self.command = None
if hasattr(self.options, 'checksum'):
if re.match('^yes$|^no$', self.options.checksum):
self.checksum = 'ADLER32' if self.options.checksum == 'yes' else None
else:
msg = "You specified to use %s checksum. Only lowercase yes/no is accepted to turn ADLER32 checksum" % self.options.checksum
ex = ConfigurationException(msg)
raise ex
def validateOptions(self):
SubCommand.validateOptions(self)
if self.options.idle and (self.options.long or self.options.summary):
raise ConfigurationException(
"Idle option (-i) conflicts with -u, and -l")
self.json = self.options.json
self.summary = self.options.summary
self.idle = self.options.idle
self.long = self.options.long
acceptedsort = [
"state", "site", "runtime", "memory", "cpu", "retries", "waste",
"exitcode"
]
if hasattr(self.options, 'sort') and self.options.sort != None:
if not self.long:
raise ConfigurationException(
'%sError%s: Please use option --long together with --sort'
% (colors.RED, colors.NORMAL))
elif not self.options.sort in acceptedsort:
raise ConfigurationException(
'%sError%s: Only these values are accepted for crab status --sort option: %s'
% (colors.RED, colors.NORMAL, acceptedsort))
else:
self.sort = self.options.sort
def getLumiList(lumi_mask_name, logger=None):
"""
Takes a lumi-mask and returns a LumiList object.
lumi-mask: either an http address or a json file on disk.
"""
lumi_list = None
parts = urlparse(lumi_mask_name)
if parts[0] in ['http', 'https']:
if logger:
logger.debug('Downloading lumi-mask from %s' % lumi_mask_name)
try:
lumi_list = LumiList(url=lumi_mask_name)
except Exception as err:
raise ConfigurationException(
"CMSSW failed to get lumimask from URL. Please try to download the lumimask yourself and point to it in crabConfig;\n%s"
% str(err))
else:
if logger:
logger.debug('Reading lumi-mask from %s' % lumi_mask_name)
try:
lumi_list = LumiList(filename=lumi_mask_name)
except IOError as err:
raise ConfigurationException("Problem loading lumi-mask file; %s" %
str(err))
return lumi_list
def validateOptions(self):
SubCommand.validateOptions(self)
if hasattr(self.options, 'logpath') and self.options.logpath != None:
if not path.isfile(self.options.logpath):
msg = '%sError%s: Could not find the log file in the path: %s' % (
colors.RED, colors.NORMAL, self.options.logpath)
raise ConfigurationException(msg)
elif hasattr(self.options, 'task'):
self.cmdconf['requiresTaskOption'] = True
self.cmdconf['useCache'] = True
if self.options.task == None:
if len(self.args) == 1 and self.args[0]:
self.options.task = self.args[0]
#for the case of 'crab uploadlog', user is using the .crab3 file
if self.options.task == None and hasattr(self, 'crab3dic'):
if self.crab3dic["taskname"] != None:
self.options.task = self.crab3dic["taskname"]
else:
msg = '%sError%s: Please use the directory option -d or --logpath to specify which log to upload' % (
colors.RED, colors.NORMAL)
raise ConfigurationException(msg)
else:
msg = '%sError%s: Please use the directory option -d or --logpath to specify which log to upload' % (
colors.RED, colors.NORMAL)
raise ConfigurationException(msg)
def getLumiList(lumi_mask_name, logger=None):
"""
Takes a lumi-mask and returns a LumiList object.
lumi-mask: either an http address or a json file on disk.
"""
lumi_list = None
parts = urlparse.urlparse(lumi_mask_name)
if parts[0] in ['http', 'https']:
if logger:
logger.debug('Downloading lumi-mask from %s' % lumi_mask_name)
try:
lumi_list = LumiList(url=lumi_mask_name)
except urllib2.HTTPError as err:
raise ConfigurationException(
"Problem downloading lumi-mask file; %s %s" %
(err.code, err.msg))
else:
if logger:
logger.debug('Reading lumi-mask from %s' % lumi_mask_name)
try:
lumi_list = LumiList(filename=lumi_mask_name)
except IOError as err:
raise ConfigurationException("Problem loading lumi-mask file; %s" %
str(err))
return lumi_list
def processAndStoreJobIds(self):
"""
Call the status command to check that the jobids passed by the user are in a valid
state to retrieve files. Otherwise, if no jobids are passed by the user, populate the
list with all possible jobids.
Also store some information which is used later when deciding the correct pfn.
"""
statusDict = getMutedStatusInfo(self.logger)
jobList = statusDict['jobList']
if not jobList:
msg = "Cannot retrieve job list from the status command."
raise ClientException(msg)
transferringIds = [x[1] for x in jobList if x[0] in ['transferring', 'cooloff', 'held']]
finishedIds = [x[1] for x in jobList if x[0] in ['finished', 'failed', 'transferred']]
possibleJobIds = transferringIds + finishedIds
if self.options.jobids:
for jobid in self.options.jobids:
if not str(jobid[1]) in possibleJobIds:
raise ConfigurationException("The job with id %s is not in a valid state to retrieve output files" % jobid[1])
else:
## If the user does not give us jobids, set them to all possible ids.
self.options.jobids = []
for jobid in possibleJobIds:
self.options.jobids.append(('jobids', jobid))
if len(self.options.jobids) > 500:
msg = "You requested to process files for %d jobs." % len(self.options.jobids)
msg += "\nThe limit is 500. Please use the '--jobids'"
msg += "option to select up to 500 jobs."
raise ConfigurationException(msg)
self.transferringIds = transferringIds
def validateOptions(self):
if self.options.fromdate is not None and self.options.days is not None:
msg = "Options --fromdate and --days cannot be used together. Please specify only one of them."
raise ConfigurationException(msg)
if self.options.fromdate is None and self.options.days is None:
days = 30
self.date = date.today() - timedelta(days=days)
elif self.options.days is not None:
days = self.options.days
self.date = date.today() - timedelta(days=days)
elif self.options.fromdate is not None:
try:
datetime.strptime(self.options.fromdate, '%Y-%m-%d')
except ValueError:
msg = "Please enter date with format 'YYYY-MM-DD'. Example: crab tasks --fromdate=2014-01-01"
raise ConfigurationException(msg)
if len(self.options.fromdate) != 10:
msg = "Please enter date with format 'YYYY-MM-DD'. Example: crab tasks --fromdate=2014-01-01"
raise ConfigurationException(msg)
self.date = self.options.fromdate
if self.options.status is not None:
if self.options.status not in TASKDBSTATUSES:
msg = "Please enter a valid task status. Valid task statuses are: %s" % (
TASKDBSTATUSES)
raise ConfigurationException(msg)
def validateOptions(self):
#Figuring out the destination directory
SubCommand.validateOptions(self)
if self.options.outputpath is not None:
if re.match("^[a-z]+://", self.options.outputpath):
self.dest = self.options.outputpath
elif not os.path.isabs( self.options.outputpath ):
self.dest = os.path.abspath( self.options.outputpath )
else:
self.dest = self.options.outputpath
#convert all to -1
if getattr(self.options, 'quantity', None) == 'all':
self.options.quantity = -1
if hasattr(self.options, 'command') and self.options.command != None:
AvailableCommands = ['LCG', 'GFAL']
self.command = self.options.command.upper()
if self.command not in AvailableCommands:
msg = "You specified to use %s command and it is not allowed. Available commands are: %s " % (self.command, str(AvailableCommands))
ex = ConfigurationException(msg)
raise ex
else:
self.command = None
if hasattr(self.options, 'checksum'):
if re.match('^yes$|^no$', self.options.checksum):
self.checksum = 'ADLER32' if self.options.checksum == 'yes' else None
else:
msg = "You specified to use %s checksum. Only lowercase yes/no is accepted to turn ADLER32 checksum" % self.options.checksum
ex = ConfigurationException(msg)
raise ex
def hasLHESource(self):
"""
Returns a tuple containing a bool to indicate usage of an
LHESource and an integer for the number of input files.
"""
if bootstrapDone():
self.logger.debug("Getting lhe info from bootstrap cachefile.")
info = self.getCfgInfo()
return info['lheinfo']
isLHE, numFiles = False, 0
if getattr(self.fullConfig.process, 'source'):
source = self.fullConfig.process.source
try:
isLHE = str(source.type_()) == 'LHESource'
except AttributeError as ex:
msg = "\nInvalid CMSSW configuration: Failed to check if 'process.source' is of type 'LHESource': %s" % (
ex)
raise ConfigurationException(msg)
if isLHE:
if hasattr(source, 'fileNames'):
numFiles = len(source.fileNames)
else:
msg = "\nInvalid CMSSW configuration: Object 'process.source', of type 'LHESource', is missing attribute 'fileNames'."
raise ConfigurationException(msg)
return isLHE, numFiles
def validateOptions(self):
"""
Check if the sitelist parameter is a comma separater list of cms sitenames,
and put the strings to be passed to the server to self
"""
SubCommand.validateOptions(self)
## Check the format of the jobids option.
if getattr(self.options, 'jobids'):
jobidstuple = validateJobids(self.options.jobids)
self.jobids = [str(jobid) for (_, jobid) in jobidstuple]
#Checking if the sites provided by the user are valid cmsnames. Doing this because with only the
#server error handling we get:
# Server answered with: Invalid input parameter
# Reason is: Incorrect 'siteblacklist' parameter
#which is not really user friendly.
#Moreover, I prefer to be independent from Lexicon. I'll the regex here.
sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression
sn_rec = re.compile(sn_re) #sn_rec => SiteName_RegularExpressionCompiled
for sitelist in ['sitewhitelist', 'siteblacklist']:
if getattr(self.options, sitelist) is not None:
for i, site_name in enumerate(getattr(self.options, sitelist).split(',')):
if not sn_rec.match(site_name):
msg = "The site name %s does not look like a valid CMS site name" % (site_name)
msg += " (it is not matching the regular expression %s)." % (sn_re)
raise ConfigurationException(msg)
setattr(self, sitelist, getattr(self.options, sitelist).split(','))
## Sanity checks for task sizes. Limits are purposely fairly generous to provide
## some level of future-proofing. The server may restrict further.
if self.options.maxjobruntime is not None:
if self.options.maxjobruntime < 60 or self.options.maxjobruntime > 336*60:
msg = "The requested maximum job runtime (%d minutes) must be between 60 and 20160 minutes." % (self.options.maxjobruntime)
raise ConfigurationException(msg)
self.maxjobruntime = str(self.options.maxjobruntime)
if self.options.maxmemory is not None:
if self.options.maxmemory < 30 or self.options.maxmemory > 1024*30:
msg = "The requested per-job memory (%d MB) must be between 30 and 30720 MB." % (self.options.maxmemory)
raise ConfigurationException(msg)
self.maxmemory = str(self.options.maxmemory)
if self.options.numcores is not None:
if self.options.numcores < 1 or self.options.numcores > 128:
msg = "The requested number of cores (%d) must be between 1 and 128." % (self.options.numcores)
raise ConfigurationException(msg)
self.numcores = str(self.options.numcores)
if self.options.priority is not None:
self.priority = str(self.options.priority)
def run(self, *args, **kwargs):
"""
Override run() for JobType
"""
tarFilename, configArguments = super(PrivateMC,
self).run(*args, **kwargs)
configArguments['jobtype'] = 'PrivateMC'
lhe, nfiles = self.cmsswCfg.hasLHESource()
pool = self.cmsswCfg.hasPoolSource()
if lhe:
self.logger.debug("LHESource found in the CMSSW configuration.")
configArguments['generator'] = getattr(self.config.JobType,
'generator', 'lhe')
major, minor, _ = os.environ['CMSSW_VERSION'].split('_')[1:4]
warn = True
if int(major) >= 7:
if int(minor) >= 5:
warn = False
if nfiles > 1 and warn:
msg = "{0}Warning{1}: Using an LHESource with ".format(
colors.RED, colors.NORMAL)
msg += "more than one input file may not be supported by the CMSSW version used. "
msg += "Consider merging the LHE input files to guarantee complete processing."
self.logger.warning(msg)
if getattr(self.config.JobType, 'generator', '') == 'lhe' and not lhe:
msg = "Generator set to 'lhe' but "
if pool:
msg += "'PoolSource' instead of 'LHESource' present in parameter set. If you "
msg += "are processing files in EDMLHE format, please set 'JobType.pluginName' "
msg += "to 'Analysis'."
else:
msg += "no 'LHESource' found in parameter set. If you are processing a gridpack "
msg += "to produce EDMLHE files, please remove the parameter 'JobType.generator'."
raise ConfigurationException(msg)
elif pool:
msg = "Found a 'PoolSource' in the parameter set. But that's not compatible with PrivateMC. "
msg += "Please switch to either 'EmptySource' or 'LHESource' for event generation, "
msg += "or set 'JobType.pluginName' to 'Analysis'."
raise ConfigurationException(msg)
configArguments['primarydataset'] = getattr(self.config.Data,
'outputPrimaryDataset',
'CRAB_PrivateMC')
return tarFilename, configArguments
def processAndStoreJobIds(self):
"""
Call the status command to check that the jobids passed by the user are in a valid
state to retrieve files. Otherwise, if no jobids are passed by the user, populate the
list with all possible jobids.
Also store some information which is used later when deciding the correct pfn.
"""
mod = __import__('CRABClient.Commands.status2', fromlist='status2')
cmdobj = getattr(mod, 'status2')(self.logger)
_, jobList = cmdobj.__call__()
jobList = jobList['jobList']
transferringIds = [x[1] for x in jobList if x[0] in ['transferring', 'cooloff', 'held']]
finishedIds = [x[1] for x in jobList if x[0] in ['finished', 'failed', 'transferred']]
possibleJobIds = transferringIds + finishedIds
if self.options.jobids:
for jobid in self.options.jobids:
if not str(jobid[1]) in possibleJobIds:
raise ConfigurationException("The job with id %s is not in a valid state to retrieve output files" % jobid[1])
else:
## If the user does not give us jobids, set them to all possible ids.
self.options.jobids = []
for jobid in possibleJobIds:
self.options.jobids.append(('jobids', jobid))
self.transferringIds = transferringIds
def validateOptions(self):
SubCommand.validateOptions(self)
if self.options.idle and (self.options.long or self.options.summary):
raise ConfigurationException(
"Option --idle conflicts with --summary and --long")
if self.options.sort is not None:
sortOpts = [
"state", "site", "runtime", "memory", "cpu", "retries",
"waste", "exitcode"
]
if self.options.sort not in sortOpts:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Only the following values are accepted for --sort option: %s" % (
sortOpts)
raise ConfigurationException(msg)
def getConfiDict(self):
"""
Load the CRAB cache file (~/.crab3). If it doesn't exist, create one.
"""
crabCacheFileName = self.crabcachepath()
if not os.path.isfile(crabCacheFileName):
msg = "Could not find CRAB cache file %s; creating a new one." % (
crabCacheFileName)
self.logger.debug(msg)
configdict = {'crab_project_directory': ''}
crabCacheFileName_tmp = "%s.%s" % (crabCacheFileName, os.getpid())
with open(crabCacheFileName_tmp, 'w') as fd:
json.dump(configdict, fd)
os.rename(crabCacheFileName_tmp, crabCacheFileName)
return configdict
try:
msg = "Found CRAB cache file %s" % (crabCacheFileName)
self.logger.debug(msg)
with open(crabCacheFileName, 'r') as fd:
configdict = json.load(fd)
except ValueError:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Error loading CRAB cache file."
msg += " Try to do 'rm -rf %s' and run the crab command again." % (
crabCacheFileName)
raise ConfigurationException(msg)
if 'crab_project_directory' not in configdict:
configdict['crab_project_directory'] = configdict.get(
'taskname', '')
if 'taskname' in configdict:
del configdict['taskname']
return configdict
def validateOptions(self):
"""
__validateOptions__
Validate the command line options of the command.
Raise a ConfigurationException in case of error; don't do anything if ok.
"""
if self.cmdconf['requiresDirOption']:
if self.options.projdir is None:
if len(self.args) > 1:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " 'crab %s' command accepts at most 1 argument (a path to a CRAB project directory), %d given." % (
self.name, len(self.args))
raise ConfigurationException(msg)
elif len(self.args) == 1 and self.args[0]:
self.options.projdir = self.args.pop(0)
elif self.cmdconf['useCache'] and self.crab3dic.get(
'crab_project_directory'):
self.options.projdir = str(
self.crab3dic['crab_project_directory'])
if self.options.projdir is None:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Please indicate the CRAB project directory with --dir=<project-directory>."
ex = MissingOptionException(msg)
ex.missingOption = "task"
raise ex
if not os.path.isdir(self.options.projdir):
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " %s is not a valid CRAB project directory." % (
self.options.projdir)
raise ConfigurationException(msg)
## If an input project directory was given, load the request cache and take the
## server URL from it.
self.loadLocalCache()
## If the command does not take any arguments, but some arguments were passed,
## clear the arguments list and give a warning message saying that the given
## arguments will be ignored.
if not self.cmdconf['acceptsArguments'] and len(self.args):
msg = "%sWarning%s:" % (colors.RED, colors.NORMAL)
msg += " 'crab %s' command takes no arguments, %d given." % (
self.name, len(self.args))
msg += " Ignoring arguments %s." % (self.args)
self.logger.warning(msg)
self.args = []
def validateLogpathOption(self):
## Check if the --logpath option was used. If it was, don't require the task
## option.
if self.options.logpath is not None:
if not os.path.isfile(self.options.logpath):
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Could not find the log file %s." % (self.options.logpath)
raise ConfigurationException(msg)
self.cmdconf['requiresDirOption'] = False
def getUrl(self, instance='prod', resource='workflow'):
"""
Retrieve the url depending on the resource we are accessing and the instance.
"""
if instance in SERVICE_INSTANCES.keys():
return BASEURL + instance + '/' + resource
elif instance == 'private':
return BASEURL + 'dev' + '/' + resource
raise ConfigurationException('Error: only %s instances can be used.' % str(SERVICE_INSTANCES.keys()))
def getRunList(myrange):
"""
Take a string like '1,2,5-8' and return a list of integers [1,2,5,6,7,8].
"""
myrange = myrange.replace(' ', '')
if not myrange:
return []
try:
jobrange(myrange)
except AssertionError, ex:
raise ConfigurationException("Invalid runRange: %s" % myrange)
def validateOptions(self):
if self.options.cmptask is None:
raise MissingOptionException(
"%sError%s: Please use the --task option to specify the task name."
% (colors.RED, colors.NORMAL))
else:
regex = "^\d{6}_\d{6}_?([^\:]*)\:[a-zA-Z-]+_(crab_)?.+"
if not re.match(regex, self.options.cmptask):
msg = "%sError%s: Task name does not match the regular expression '%s'." % (
colors.RED, colors.NORMAL, regex)
raise ConfigurationException(msg)
def validateOptions(self):
if self.options.cmptask is None:
msg = "%sError%s: Please specify the task name for which to remake a CRAB project directory." % (colors.RED, colors.NORMAL)
msg += " Use the --task option."
ex = MissingOptionException(msg)
ex.missingOption = "cmptask"
raise ex
else:
regex = "^\d{6}_\d{6}_?([^\:]*)\:[a-zA-Z-]+_(crab_)?.+"
if not re.match(regex, self.options.cmptask):
msg = "%sError%s: Task name does not match the regular expression '%s'." % (colors.RED, colors.NORMAL, regex)
raise ConfigurationException(msg)
def checkUserJobids(self, statusCacheInfo, userJobids):
""" Checks that the job information taken from the status_cache file on the schedd
contains all of the jobids passed by the user.
"""
wrongJobIds = [
uJobid for uJobid in userJobids
if uJobid not in statusCacheInfo.keys()
]
if wrongJobIds:
raise ConfigurationException(
"The following jobids were not found in the task: %s" %
wrongJobIds)
def __init__(self, config, logger, workingdir):
self.logger = logger
## Before everything, check if the config is ok.
if config:
valid, msg = self.validateConfig(config)
if valid:
self.config = config
self.workdir = workingdir
else:
msg += "\nThe documentation about the CRAB configuration file can be found in"
msg += " https://twiki.cern.ch/twiki/bin/view/CMSPublic/CRAB3ConfigurationFile."
raise ConfigurationException(msg)
def validateOptions(self):
"""
Check if the output file is given and set as attribute
"""
SubCommand.validateOptions(self)
if not re.match('^yes$|^no$', self.options.usedbs):
raise ConfigurationException(
"--dbs option only accepts the yes and no values (--dbs=yes or --dbs=no)"
)
self.usedbs = 1 if self.options.usedbs == 'yes' else 0
setattr(self, 'outdir', getattr(self.options, 'outdir', None))
def loadConfig(self, configname, overrideargs = None):
"""
Load the configuration file
"""
## If the configuration is alredy an object it doesn't need to be loaded from the file.
if isinstance(configname, Configuration):
self.configuration = configname
valid, configmsg = self.validateConfig()
if not valid:
configmsg += "\nThe documentation about the CRAB configuration file can be found in"
configmsg += " https://twiki.cern.ch/twiki/bin/view/CMSPublic/CRAB3ConfigurationFile"
raise ConfigurationException(configmsg)
return
if not os.path.isfile(configname):
raise ConfigurationException("CRAB configuration file %s not found." % (configname))
self.logger.info("Will use CRAB configuration file %s" % (configname))
try:
self.logger.debug("Loading CRAB configuration file.")
self.configuration = loadConfigurationFile(os.path.abspath(configname))
## Overwrite configuration parameters passed as arguments in the command line.
if overrideargs:
for singlearg in overrideargs:
## The next line is needed, because we allow the config to be passed as argument
## instead via the --config option.
if singlearg == configname: continue
if len(singlearg.split('=',1)) == 1:
self.logger.info("Wrong format in command-line argument '%s'. Expected format is <section-name>.<parameter-name>=<parameter-value>." % (singlearg))
if len(singlearg) > 1 and singlearg[0] == '-':
self.logger.info("If the argument '%s' is an option to the %s command, try 'crab %s %s [value for %s option (if required)] [arguments]'." \
% (singlearg, self.__class__.__name__, self.__class__.__name__, singlearg, singlearg))
raise ConfigurationException("ERROR: Wrong command-line format.")
fullparname, parval = singlearg.split('=',1)
# now supporting just one sub params, eg: Data.inputFiles, User.email, ...
parnames = fullparname.split('.', 1)
if len(parnames) == 1:
self.logger.info("Wrong format in command-line argument '%s'. Expected format is <section-name>.<parameter-name>=<parameter-value>." % (singlearg))
raise ConfigurationException("ERROR: Wrong command-line format.")
self.configuration.section_(parnames[0])
type = configParametersInfo.get(fullparname, {}).get('type', 'undefined')
if type in ['undefined', 'StringType']:
setattr(getattr(self.configuration, parnames[0]), parnames[1], literal_eval("\'%s\'" % parval))
self.logger.debug("Overriden parameter %s with '%s'" % (fullparname, parval))
else:
setattr(getattr(self.configuration, parnames[0]), parnames[1], literal_eval("%s" % parval))
self.logger.debug("Overriden parameter %s with %s" % (fullparname, parval))
valid, configmsg = self.validateConfig() ## Subclasses of SubCommand overwrite this method if needed.
except RuntimeError as re:
configmsg = "Error while loading CRAB configuration:\n%s" % (self._extractReason(configname, re))
configmsg += "\nPlease refer to https://twiki.cern.ch/twiki/bin/view/CMSPublic/CRAB3CommonErrors#Syntax_error_in_CRAB_configurati"
configmsg += "\nSee the ./crab.log file for more details."
configmsg += "\nThe documentation about the CRAB configuration file can be found in"
configmsg += " https://twiki.cern.ch/twiki/bin/view/CMSPublic/CRAB3ConfigurationFile"
raise ConfigurationException(configmsg)
else:
if not valid:
configmsg += "\nThe documentation about the CRAB configuration file can be found in"
configmsg += " https://twiki.cern.ch/twiki/bin/view/CMSPublic/CRAB3ConfigurationFile"
raise ConfigurationException(configmsg)
def setDestination(self):
#Setting default destination if -o is not provided
if not self.dest:
self.dest = os.path.join(self.requestarea, 'results')
# Destination is a URL.
if re.match("^[a-z]+://", self.dest):
if not self.dest.endswith("/"):
self.dest += "/"
#Creating the destination directory if necessary
elif not os.path.exists(self.dest):
self.logger.debug("Creating directory %s " % self.dest)
os.makedirs(self.dest)
elif not os.path.isdir(self.dest):
raise ConfigurationException('Destination directory is a file')
def validateOptions(self):
SubCommand.validateOptions(self)
if self.options.sort is not None:
sortOpts = [
"state", "site", "runtime", "memory", "cpu", "retries",
"waste", "exitcode"
]
if self.options.sort not in sortOpts:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Only the following values are accepted for --sort option: %s" % (
sortOpts)
raise ConfigurationException(msg)
if self.options.jobids:
jobidstuple = validateJobids(self.options.jobids)
self.jobids = [str(jobid) for (_, jobid) in jobidstuple]
if self.options.jobids and not (self.options.long
or self.options.sort):
raise ConfigurationException(
"Parameter --jobids can only be used in combination "
"with --long or --sort options.")
def getUrl(instance='prod', resource='workflow'):
"""
Retrieve the url depending on the resource we are accessing and the instance.
As of August 2019 the Kubernetes server instance (k8s) points to preprod database instance
"""
if instance in SERVICE_INSTANCES.keys():
if instance == 'k8s':
return BASEURL + 'preprod' + '/' + resource
else:
return BASEURL + instance + '/' + resource
elif instance == 'private':
return BASEURL + 'dev' + '/' + resource
raise ConfigurationException(
'Error: only the following instances can be used: %s' %
str(SERVICE_INSTANCES.keys()))
def validateOptions(self):
"""
Check if the output file is given and set as attribute
"""
SubCommand.validateOptions(self)
if self.options.usedbs is not None:
msg = "CRAB command option error: the option --dbs has been deprecated since CRAB v3.3.1603."
raise UnknownOptionException(msg)
recoveryMethods = ['notFinished', 'notPublished', 'failed']
if self.options.recovery not in recoveryMethods:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " The --recovery option only accepts the following values: %s" % (
recoveryMethods)
raise ConfigurationException(msg)
def __init__(self, config, userConfig=None, logger=None):
global configurationCache
self.config = config
self.logger = logger if logger else logging
self.fullConfig = None
self.outputFile = None
if userConfig:
cfgBaseName = os.path.basename(userConfig).replace(".py", "")
cfgDirName = os.path.dirname(os.path.abspath(userConfig))
if not os.path.isfile(userConfig):
msg = "Cannot find CMSSW configuration file %s in %s" % (
userConfig, os.getcwd())
raise ConfigurationException(msg)
self.logger.info("Importing CMSSW configuration %s" % (userConfig))
pyCfgParams = getattr(self.config.JobType, 'pyCfgParams', [])
originalArgv = sys.argv
sys.argv = [userConfig]
if pyCfgParams:
sys.argv.extend(pyCfgParams)
msg = "Additional parameters for the CMSSW configuration are: %s" % (
pyCfgParams)
self.logger.debug(msg)
configFile, pathname, description = imp.find_module(
cfgBaseName, [cfgDirName])
cacheLine = (tuple(sys.path), tuple(pathname), tuple(sys.argv))
if cacheLine in configurationCache:
self.fullConfig = configurationCache[cacheLine]
configFile.close()
elif not bootstrapDone():
sys.path.append(os.getcwd())
try:
oldstdout = sys.stdout
sys.stdout = open(logger.logfile, 'a')
self.fullConfig = imp.load_module(cfgBaseName, configFile,
pathname, description)
finally:
sys.stdout.close()
sys.stdout = oldstdout
configFile.close()
configurationCache[cacheLine] = self.fullConfig
self.logger.info("Finished importing CMSSW configuration %s" %
(userConfig))
sys.argv = originalArgv
def __init__(self, config, proxyfilename, logger, workingdir, crabserver,
s3tester):
self.logger = logger
self.proxyfilename = proxyfilename
self.automaticAvail = False
self.crabserver = crabserver
self.s3tester = s3tester
## Before everything, check if the config is ok.
if config:
valid, msg = self.validateConfig(config)
if valid:
self.config = config
self.workdir = workingdir
else:
msg += "\nThe documentation about the CRAB configuration file can be found in"
msg += " https://twiki.cern.ch/twiki/bin/view/CMSPublic/CRAB3ConfigurationFile"
raise ConfigurationException(msg)