def getUsernameFromCRIC(proxyFileName=None):
"""
Retrieve username from CRIC by doing a query to
https://cms-cric.cern.ch/api/accounts/user/query/?json&preset=whoami
using the users proxy.
args:
proxyfile : string : the full patch to the file containing the user proxy
"""
## Path to certificates.
capath = os.environ[
'X509_CERT_DIR'] if 'X509_CERT_DIR' in os.environ else "/etc/grid-security/certificates"
# Path to user proxy
if not proxyFileName:
proxyFileName = getUserProxy()
if not proxyFileName:
msg = "Can't find user proxy file"
raise UsernameException(msg)
## Retrieve user info from CRIC. Note the curl must be executed in same env. (i.e. CMSSW) as crab
queryCmd = "curl -sS --capath %s --cert %s --key %s 'https://cms-cric.cern.ch/api/accounts/user/query/?json&preset=whoami'" %\
(capath, proxyFileName, proxyFileName)
stdout, stderr, rc = execute_command(queryCmd)
if rc or not stdout:
msg = "Error contacting CRIC."
msg += "\nDetails follow:"
msg += "\n Executed command: %s" % (queryCmd)
msg += "\n Stdout:\n %s" % (str(stdout).replace(
'\n', '\n '))
msg += "\n Stderr:\n %s" % (str(stderr).replace(
'\n', '\n '))
raise UsernameException(msg)
## Extract the username from the above command output.
parseCmd = "echo '%s' | tr ':,' '\n' | grep -A1 login | tail -1 | tr -d ' \n\"'" % (
str(stdout))
username, stderr, rc = execute_command(parseCmd)
if username == 'null' or not username:
msg = "Failed to retrieve username from CRIC."
msg += "\nDetails follow:"
msg += "\n Executed command: %s" % (queryCmd)
msg += "\n Stdout:\n %s" % (str(stdout).replace(
'\n', '\n '))
msg += "\n Parsed username: %s" % (username)
msg += "\n%sNote%s: Make sure you have the correct certificate mapped in your CERN account page" % (
colors.BOLD, colors.NORMAL)
msg += " (you can check what is the certificate you currently have mapped"
msg += " by looking at CERN Certificatiom Authority page."
msg += "\nFor instructions on how to map a certificate, see "
msg += "\n https://twiki.cern.ch/twiki/bin/view/CMSPublic/UsernameForCRAB#Adding_your_DN_to_your_profile"
raise UsernameException(msg)
return username
def getUserCertEndDate(self):
"""
Return the number of seconds until the expiration of the user cert
in .globus/usercert.pem or $X509_USER_CERT if set
"""
cmd = 'openssl x509 -noout -dates -in %s' % self.certLocation
stdout, stderr, rc = execute_command(cmd, logger=self.logger)
if rc != 0:
self.logger.error(stdout + '\n' + stderr)
msg = "\n".join(['Error executing %s:' % cmd, stdout, stderr])
raise ProxyCreationException(msg)
out = stdout.rstrip().split('notAfter=')[1]
possibleFormats = ['%b %d %H:%M:%S %Y %Z', '%b %d %H:%M:%S %Y %Z']
exptime = None
for frmt in possibleFormats:
try:
exptime = datetime.strptime(out, frmt)
except ValueError:
pass # try next format
if not exptime:
# If we cannot decode the output in any way print
# a message and fallback to voms-proxy-info command
self.logger.warning(
'Cannot decode "openssl x509 -noout -in %s -dates" date format.'
% self.certLocation)
timeleft = 0
else:
# if everything is fine then we are ready to return!!
timeleft = (exptime - datetime.utcnow()).total_seconds()
daystoexp = int(timeleft // (60. * 60 * 24))
return daystoexp
def initFromEnv(self):
""" Init the class taking the required information from the environment
"""
#self.command = 'scram' # SB I think this line is not needed
self["SCRAM_ARCH"] = None
if 'SCRAM_ARCH' in os.environ:
self["SCRAM_ARCH"] = os.environ["SCRAM_ARCH"]
else:
stdout, _, _ = execute_command(command='scram arch')
self["SCRAM_ARCH"] = stdout
try:
self["CMSSW_BASE"] = os.environ["CMSSW_BASE"]
self["CMSSW_VERSION"] = os.environ["CMSSW_VERSION"]
# Commenting these two out. I don't think they are really needed
# self.cmsswReleaseBase = os.environ["CMSSW_RELEASE_BASE"]
# self.localRT = os.environ["LOCALRT"]
except KeyError as ke:
self["CMSSW_BASE"] = None
self["CMSSW_VERSION"] = None
# self.cmsswReleaseBase = None
# self.localRT = None
msg = "Please make sure you have setup the CMS enviroment (cmsenv). Cannot find %s in your env" % str(
ke)
msg += "\nPlease refer to https://twiki.cern.ch/twiki/bin/view/CMSPublic/WorkBookCRAB3Tutorial#CMS_environment for how to setup the CMS enviroment."
raise EnvironmentException(msg)
def create(self, username=None, retrievers=None, validity=720):
"""
creates a new credential in myproxy.cern.ch
args: username: string: the username of the credential, usually the has of the user DN
args: retrievers: string: regexp indicating list of DN's authorized to retrieve this credential
args: validity: integer: how long this credential will be valid for in hours, default is 30 days
example of the command we want :
command : export GT_PROXY_MODE=rfc
myproxy-init -d -n -s myproxy.cern.ch
-x -R '/DC=ch/DC=cern/OU=computers/CN=vocms0105.cern.ch|/DC=ch/DC=cern/OU=computers/CN=crab-(preprod|prod|dev)-tw(02|01).cern.ch|/DC=ch/DC=cern/OU=computers/CN=(ddi|ddidk|mytw).cern.ch|/DC=ch/DC=cern/OU=computers/CN=stefanov(m|m2).cern.ch'
-x -Z '/DC=ch/DC=cern/OU=computers/CN=vocms0105.cern.ch|/DC=ch/DC=cern/OU=computers/CN=crab-(preprod|prod|dev)-tw(02|01).cern.ch|/DC=ch/DC=cern/OU=computers/CN=(ddi|ddidk|mytw).cern.ch|/DC=ch/DC=cern/OU=computers/CN=stefanov(m|m2).cern.ch'
-l 'be1f4dc5be8664cbd145bf008f5399adf42b086f'
-t 168:00 -c 3600:00
"""
cmd = 'export GT_PROXY_MODE=rfc ; myproxy-init -d -n -s myproxy.cern.ch'
cmd += ' -C %s' % self.certLocation
cmd += ' -y %s' % self.keyLocation
cmd += ' -x -R \'%s\'' % retrievers
cmd += ' -x -Z \'%s\'' % retrievers
cmd += ' -l %s' % username
cmd += ' -t 168 -c %s' % validity # validity of the retrieved proxy: 7 days = 168 hours
stdout, stderr, rc = execute_command(cmd, logger=self.logger)
if rc != 0:
self.logger.error(stdout + '\n' + stderr)
msg = "\n".join(['Error executing %s:' % cmd, stdout, stderr])
raise ProxyCreationException(msg)
def getSubject(self):
cmd = 'voms-proxy-info --identity --file %s' % self.proxyFile
stdout, stderr, rc = execute_command(cmd, logger=self.logger)
if rc != 0:
self.logger.error(stdout + '\n' + stderr)
msg = "\n".join(['Error executing %s:' % cmd, stdout, stderr])
raise ProxyCreationException(msg)
return stdout.rstrip()
def getTimeLeft(self):
cmd = 'voms-proxy-info --actimeleft --timeleft --file %s' % self.proxyFile
stdout, stderr, rc = execute_command(cmd, logger=self.logger)
if rc != 0:
self.logger.error(stdout + '\n' + stderr)
msg = "\n".join(['Error executing %s:' % cmd, stdout, stderr])
raise ProxyCreationException(msg)
# pick the shorter between actimeleft and timeleft
times = stdout.split('\n')
timeLeft = min(int(times[0]), int(times[1]))
return timeLeft
def delete(self, pfn, command):
undoScram = "which scram >/dev/null 2>&1 && eval `scram unsetenv -sh`"
rmcmd = undoScram + "; " + command + "'" + pfn + "'"
self.logger.info('Executing command: %s' % rmcmd)
self.logger.info('Please wait...')
delout, delerr, delexitcode = execute_command(rmcmd)
if delexitcode:
self.logger.info('Failed running delete command')
if delout:
self.logger.info(' Stdout:\n %s' %
str(delout).replace('\n', '\n '))
if delerr:
self.logger.info(' Stderr:\n %s' %
str(delerr).replace('\n', '\n '))
return delexitcode
def getGroupAndRole(self):
cmd = 'voms-proxy-info --fqan --file %s' % self.proxyFile
stdout, stderr, rc = execute_command(cmd, logger=self.logger)
if rc != 0:
self.logger.error(stdout + '\n' + stderr)
msg = "\n".join(['Error executing %s:' % cmd, stdout, stderr])
raise ProxyCreationException(msg)
fqans = str(stdout)
primaryFqan = fqans.split('\n')[0]
attributes = primaryFqan.split('/')
if len(attributes) > 4:
group = attributes[2]
role = attributes[3].split('=')[1]
else:
group = ''
role = attributes[2].split('=')[1]
return group, role
def getPFN(self, site='T2_CH_CERN', lfn='/store/user', username='******'):
# prepare a simply python script to resolve lfn2pfn via Rucio
template = """
from rucio.client import Client
client = Client()
rse = "{site}"
lfn = ["user.{username}:{lfn}"]
for operation in ['third_party_copy', 'write', 'read']:
try:
#print('Try Rucio lfn2pn with operation %s', operation)
out = client.lfns2pfns(rse, lfn, operation=operation)
break
except Exception as ex:
print("Failed to resolve LNF to PFN via Rucio. Error is:\\n %s" % str(ex))
if not out:
print("Failed to resolve LNF to PFN via Rucio.")
exit(1)
print(out[lfn[0]])
exit(0)
"""
rucioScript = template.format(site=site, username=username, lfn=lfn)
import tempfile
(_, scriptName) = tempfile.mkstemp(dir='/tmp',
prefix='lfn2pfn-',
suffix='.py')
with open(scriptName, 'w') as ofile:
ofile.write(rucioScript)
cmd = 'eval `scram unsetenv -sh`; '
cmd += 'source /cvmfs/cms.cern.ch/rucio/setup.sh 2>/dev/null; export RUCIO_ACCOUNT=%s; ' % username
cmd += 'python %s; ' % scriptName
rucioOut, rucioErr, exitcode = execute_command(cmd)
os.unlink(scriptName)
if exitcode:
self.logger.info('PFN lookup failed')
if rucioOut:
self.logger.info(' Stdout:\n %s' %
str(rucioOut).replace('\n', '\n '))
if rucioErr:
self.logger.info(' Stderr:\n %s' %
str(rucioErr).replace('\n', '\n '))
raise Exception
pfn = rucioOut.rstrip()
return pfn
def executeTestRun(self, inputArgs, jobnr):
""" Execute a test run calling CMSRunAnalysis.sh
"""
os.environ.update({
'CRAB3_RUNTIME_DEBUG': 'True',
'_CONDOR_JOB_AD': 'Job.submit'
})
optsList = [
os.path.join(os.getcwd(), 'TweakPSet.py'),
'-a %s' % inputArgs[jobnr - 1]['CRAB_Archive'],
'-o %s' % inputArgs[jobnr - 1]['CRAB_AdditionalOutputFiles'],
'--sourceURL=%s' % inputArgs[jobnr - 1]['CRAB_ISB'],
'--location=%s' % os.getcwd(),
'--inputFile=%s' % inputArgs[jobnr - 1]['inputFiles'],
'--runAndLumis=%s' % inputArgs[jobnr - 1]['runAndLumiMask'],
'--firstEvent=%s' % inputArgs[jobnr - 1]
['firstEvent'], #jobs goes from 1 to N, inputArgs from 0 to N-1
'--lastEvent=%s' % inputArgs[jobnr - 1]['lastEvent'],
'--firstLumi=%s' % inputArgs[jobnr - 1]['firstLumi'],
'--firstRun=%s' % inputArgs[jobnr - 1]['firstRun'],
'--seeding=%s' % inputArgs[jobnr - 1]['seeding'],
'--lheInputFiles=%s' % inputArgs[jobnr - 1]['lheInputFiles'],
'--oneEventMode=0',
'--eventsPerLumi=%s' % inputArgs[jobnr - 1]['eventsPerLumi'],
'--maxRuntime=-1',
'--jobNumber=%s' % (jobnr - 1),
'--cmsswVersion=%s' % inputArgs[jobnr - 1]['CRAB_JobSW'],
'--scramArch=%s' % inputArgs[jobnr - 1]['CRAB_JobArch'],
'--scriptExe=%s' % inputArgs[jobnr - 1]['scriptExe'],
'--scriptArgs=%s' % inputArgs[jobnr - 1]['scriptArgs'],
]
# from a python list to a string which can be used as shell command argument
opts = ''
for opt in optsList:
opts = opts + ' %s' % opt
command = 'sh CMSRunAnalysis.sh ' + opts
out, err, returncode = execute_command(command=command)
self.logger.debug(out)
self.logger.debug(err)
if returncode != 0:
raise ClientException(
'Failed to execute local test run:\n StdOut: %s\n StdErr: %s' %
(out, err))
def getInfo(self, username=None):
"""
returns information about a credential stored in myproxy.cern.ch
args: username: string: the username of the credential, usually the has of the user DN
"""
cmd = 'myproxy-info -s myproxy.cern.ch -l %s' % username
stdout, stderr, rc = execute_command(cmd, logger=self.logger)
if rc != 0:
self.logger.error(stdout + '\n' + stderr)
if rc > 0 or not stdout: # if there's no credential myproxy-info returns rc=1
return 0, ''
olines = stdout.rstrip().split('\n')
trustedRetrievalPolicy = olines[-2]
trustedRetrievers = trustedRetrievalPolicy.split(':')[1].strip()
times = olines[-1].split(':')
hours = int(times[1])
mins = int(times[2])
timeLeft = hours * 3600 + mins * 60 # let's ignore seconds
return timeLeft, trustedRetrievers
def cp(self, pfn, command):
abspath = os.path.abspath(self.filename)
if cmd_exist("gfal-copy") and self.command in [None, "GFAL"]:
abspath = "file://" + abspath
undoScram = "which scram >/dev/null 2>&1 && eval `scram unsetenv -sh`"
cpcmd = undoScram + "; " + command + abspath + " '" + pfn + "'"
self.logger.info('Executing command: %s' % cpcmd)
self.logger.info('Please wait...')
cpout, cperr, cpexitcode = execute_command(cpcmd)
if cpexitcode:
self.logger.info('Failed running copy command')
if cpout:
self.logger.info(' Stdout:\n %s' %
str(cpout).replace('\n', '\n '))
if cperr:
self.logger.info(' Stderr:\n %s' %
str(cperr).replace('\n', '\n '))
return cpout, cperr, cpexitcode
def curlGetFileFromURL(url, filename=None, proxyfilename=None, logger=None):
"""
Read the content of a URL into a file via curl
url: the link you would like to retrieve
filename: the local filename where the url is saved to. Defaults to the filename in the url
proxyfilename: the x509 proxy certificate to be used in case auth is required
returns: the exit code of the command if command failed, otherwise the HTTP code of the call
note that curl exits with status 0 if the HTTP calls fail,
"""
## Path to certificates.
capath = os.environ[
'X509_CERT_DIR'] if 'X509_CERT_DIR' in os.environ else "/etc/grid-security/certificates"
# send curl output to file and http_code to stdout
downloadCommand = 'curl -sS --capath %s --cert %s --key %s -o %s -w %%"{http_code}"' %\
(capath, proxyfilename, proxyfilename, filename)
downloadCommand += ' "%s"' % url
if logger:
logger.debug("Will execute:\n%s", downloadCommand)
stdout, stderr, rc = execute_command(downloadCommand, logger=logger)
errorDetails = ''
if rc != 0:
os.unlink(filename)
httpCode = 503
else:
httpCode = int(stdout)
if httpCode != 200:
with open(filename) as f:
errorDetails = f.read()
os.unlink(filename)
if logger:
logger.debug('exitcode: %s\nstdout: %s\nstderr: %s\nerror details: %s',
rc, stdout, stderr, errorDetails)
return httpCode
def create(self, timeLeftThreshold=720):
# is there a proxy already ?
# does it have correct group and role ?
# is it valid long enough ?
# all OK, do nothing
# need a new proxy
cmd = 'voms-proxy-init --rfc'
cmd += ' --cert %s' % self.certLocation
cmd += ' --key %s' % self.keyLocation
cmd += ' --out %s' % self.proxyFile
cmd += ' --valid %s' % self.desiredValidity
vomsString = 'cms:/cms'
if self.group:
vomsString += '/%s' % self.group
if self.role and self.role != 'NULL':
vomsString += '/Role=%s' % self.role
cmd += ' --voms %s' % vomsString
stdout, stderr, rc = execute_command(cmd,
logger=self.logger,
redirect=False)
if rc != 0:
self.logger.error(stdout + '\n' + stderr)
msg = "\n".join(['Error executing %s:' % cmd, stdout, stderr])
raise ProxyCreationException(msg)
def __init__(self, logger, cmdargs=None, disable_interspersed_args=False):
"""
Initialize common client parameters
"""
if not hasattr(self, 'name'):
self.name = self.__class__.__name__
ConfigCommand.__init__(self)
# The command logger.
self.logger = logger
self.logfile = self.logger.logfile
stdout, _, _ = execute_command(command='uname -a')
localSystem = stdout.strip()
try:
localOS, _, _ = execute_command('grep PRETTY_NAME /etc/os-release')
localOS = localOS.strip().split('=')[1].strip('"')
except Exception as ex: # pylint: disable=unused-variable
try:
localOS, _, _ = execute_command(command='lsb_release -d')
localOS = localOS.strip().split(':')[1].strip()
except Exception as ex: # pylint: disable=unused-variable
localOS = "Unknown Operating System"
self.logger.debug("CRAB Client version: %s", __version__)
self.logger.debug("Running on: " + localSystem + " - " + localOS)
self.logger.debug("Executing command: '%s'" % str(self.name))
# Get the command configuration.
self.cmdconf = commandsConfiguration.get(self.name)
if not self.cmdconf:
raise RuntimeError(
"Canot find command %s in commandsConfiguration inside ClientMapping. Are you a developer"
"trying to add a command without it's correspondant configuration?"
% self.name)
# Get the CRAB cache file.
self.cachedinfo = None
self.crab3dic = self.getConfiDict()
# The options parser.
self.parser = CRABCmdOptParser(self.name, self.__doc__,
disable_interspersed_args)
# Define the command options.
self.setSuperOptions()
# Parse the command options/arguments.
cmdargs = cmdargs or []
(self.options, self.args) = self.parser.parse_args(cmdargs)
self.transferringIds = None
self.dest = None
self.validateLogpathOption()
# Validate first the SubCommand options
SubCommand.validateOptions(self)
# then the config option for the submit command
self.validateConfigOption()
# Get the VO group/role from the command options (if the command requires these
# options).
proxyOptsSetPlace = {
'set_in': {
'group': "default",
'role': "default"
},
'for_set_use': ""
}
msgadd = []
self.voGroup, self.voRole = "", "NULL"
if self.cmdconf['requiresProxyVOOptions']:
proxyOptsSetPlace['for_set_use'] = "cmdopts"
if self.options.voGroup is not None:
self.voGroup = self.options.voGroup
proxyOptsSetPlace['set_in']['group'] = "cmdopts"
msgadd.append("VO group '%s'" % (self.voGroup))
if self.options.voRole is not None:
self.voRole = self.options.voRole if self.options.voRole != "" else "NULL"
proxyOptsSetPlace['set_in']['role'] = "cmdopts"
msgadd.append("VO role '%s'" % (self.voRole))
if msgadd:
msg = "Using %s as specified in the crab command options." % (
" and ".join(msgadd))
self.logger.debug(msg)
# Create the object that will do the proxy operations. We don't really care
# what VO role and group and server URL we pass to the constructor, because
# these are not used until we do the proxy delegation to the myproxy server.
# And this happens in handleProxy(), which is called after we load the
# configuration file and retrieve the final values for those parameters.
# handleProxy() takes care of passing those parameters to self.credentialHandler
self.credentialHandler = CredentialInteractions(self.logger)
# If the user didn't use the --proxy command line option, and if there isn't a
# valid proxy already, we will create a new one with the current VO role and group
# (as commented above, we don't really care what are the VO role and group so
# far).
self.proxyCreated = False
# If there is an input configuration file:
if hasattr(self.options, 'config') and self.options.config is not None:
proxyOptsSetPlace['for_set_use'] = "config"
# Load the configuration file and validate it.
self.loadConfig(self.options.config, self.args)
# Create the CRAB project directory.
self.requestarea, self.requestname, self.logfile = createWorkArea(
self.logger,
getattr(self.configuration.General, 'workArea', None),
getattr(self.configuration.General, 'requestName', None))
# Get the VO group/role from the configuration file.
msgadd = []
if hasattr(self.configuration, 'User') and hasattr(
self.configuration.User, 'voGroup'):
self.voGroup = self.configuration.User.voGroup
proxyOptsSetPlace['set_in']['group'] = "config"
msgadd.append("VO group '%s'" % (self.voGroup))
if hasattr(self.configuration, 'User') and hasattr(
self.configuration.User, 'voRole'):
self.voRole = self.configuration.User.voRole if self.configuration.User.voRole != "" else "NULL"
proxyOptsSetPlace['set_in']['role'] = "config"
msgadd.append("VO role '%s'" % (self.voRole))
if msgadd:
msg = "Using %s as specified in the CRAB configuration file." % (
" and ".join(msgadd))
self.logger.debug(msg)
# If the VO group/role was not given in the command options, take it from the request cache.
if self.cmdconf['requiresDirOption']:
self.setCachedProxy(proxyOptsSetPlace)
# If the server URL isn't already set, we check the args and then the config.
if not hasattr(self, 'serverurl') and self.cmdconf['requiresREST']:
self.instance, self.serverurl = self.serverInstance()
elif not self.cmdconf['requiresREST']:
self.instance, self.serverurl = None, None
# Update (or create) the CRAB cache file.
self.updateCRABCacheFile()
# At this point we check if there is a valid proxy, and
# eventually create a new one. If the proxy was not created by CRAB, we check that the
# VO role/group in the proxy are the same as specified by the user in the configuration
# file (or in the command line options). If it is not, we ask the user if he wants to
# overwrite the current proxy. If he doesn't want to overwrite it, we don't continue
# and ask him to provide the VO role/group as in the existing proxy.
# Finally, delegate the proxy to myproxy server.
self.handleVomsProxy(proxyOptsSetPlace)
# only if this command talks to the REST we create a CRABRest object to communicate with CRABServer
# and check/upate credentials on myproxy
# this is usually the first time that a call to the server is made, so where Emulator('rest') is initialized
# arguments to Emulator('rest') call must match those for HTTPRequest.__init__ in RESTInteractions.py
#server = CRABClient.Emulator.getEmulator('rest')(url=serverurl, localcert=proxyfilename, localkey=proxyfilename,
# version=__version__, retry=2, logger=logger)
if self.cmdconf['requiresREST']:
crabRest = CRABClient.Emulator.getEmulator('rest')
self.crabserver = crabRest(hostname=self.serverurl,
localcert=self.proxyfilename,
localkey=self.proxyfilename,
retry=2,
logger=self.logger,
verbose=False,
version=__version__,
userAgent='CRABClient')
self.crabserver.setDbInstance(self.instance)
# prepare also a test crabserver instance which will send tarballs to S3
self.s3tester = crabRest(hostname='cmsweb-testbed.cern.ch',
localcert=self.proxyfilename,
localkey=self.proxyfilename,
retry=0,
logger=self.logger,
verbose=False,
version=__version__,
userAgent='CRABClient')
self.s3tester.setDbInstance('preprod')
self.handleMyProxy()
# Validate the command options
self.validateOptions()
self.validateOptions()
# Log user command and options used for debuging purpose.
self.logger.debug('Command use: %s' % self.name)
self.logger.debug('Options use: %s' % cmdargs)
if self.cmdconf['requiresREST']:
self.checkversion()
self.defaultApi = 'workflow'
self.logger.debug("Instance is %s" % (self.instance))
self.logger.debug("Server base url is %s" % (self.serverurl))
if self.cmdconf['requiresREST']:
self.logger.debug("Command api %s" % (self.defaultApi))
def processWorker(self, input_, successfiles, failedfiles):
"""
_processWorker_
Runs a subprocessed command.
"""
logger = self.setSubprocessLog()
# Get this started
while True:
try:
myfile, work = input_.get()
except (EOFError, IOError):
crashMessage = "Hit EOF/IO in getting new work\n"
crashMessage += "Assuming this is a graceful break attempt."
print(crashMessage)
break
if work == 'STOP':
break
else:
fileid = myfile['pfn'].split('/')[-1]
dirpath = os.path.join(
self.options.destination,
myfile['suffix'] if 'suffix' in myfile else '')
url_input = bool(re.match("^[a-z]+://", dirpath))
if not os.path.isdir(dirpath) and not url_input:
os.makedirs(dirpath)
localFilename = os.path.join(dirpath, str(fileid))
command = work
logger.info("Retrieving %s " % fileid)
logger.debug("Executing %s" % command)
try:
stdout, stderr, returncode = execute_command(command=command)
except KeyboardInterrupt:
logger.info("Subprocess exit due to keyboard interrupt")
break
error = simpleOutputCheck(stderr)
logger.debug("Finish executing for file %s" % fileid)
if returncode != 0 or len(error) > 0:
logger.info("%sWarning%s: Failed retrieving %s" %
(colors.RED, colors.NORMAL, fileid))
#logger.debug(colors.RED +"Stderr: %s " %stderr+ colors.NORMAL)
for x in error:
logger.info(colors.RED + "\t %s" % x + colors.NORMAL)
failedfiles[fileid] = str(error)
logger.debug("Full stderr follows:\n%s" % stderr)
if "timed out" in stderr or "timed out" in stdout:
logger.info(
"%sWarning%s: Failed due to connection timeout" %
(colors.RED, colors.NORMAL))
logger.info(
"Please use the '--wait=<#seconds>' option to increase the connection timeout"
)
if "checksum" in stderr:
logger.info(
"%sWarning%s: as of 3.3.1510 CRAB3 is using an option to validate the checksum with lcg-cp/gfal-cp commands."
" You might get false positives since for some site this is not working."
" In that case please use the option --checksum=no" %
(colors.RED, colors.NORMAL))
if os.path.isfile(localFilename) and os.path.getsize(
localFilename) != myfile['size']:
logger.debug("File %s has the wrong size, deleting it" %
fileid)
try:
os.remove(localFilename)
except OSError as ex:
logger.debug(
"%sWarning%s: Cannot remove the file because of: %s"
% (colors.RED, colors.NORMAL, ex))
try:
time.sleep(60)
except KeyboardInterrupt:
logger.info("Subprocess exit due to keyboard interrupt")
break
else:
logger.info("%sSuccess%s: Success in retrieving %s " %
(colors.GREEN, colors.NORMAL, fileid))
successfiles[fileid] = 'Successfully retrieved'
return
def executeTestRun(self, filecacheurl, uniquerequestname):
"""
Downloads the dry run tarball from the User File Cache and unpacks it in a temporary directory.
Runs a trial to obtain the performance report. Repeats trial with successively larger input events
until a job length of maxSeconds is reached (this improves accuracy for fast-running CMSSW parameter sets.)
"""
cwd = os.getcwd()
try:
tmpDir = tempfile.mkdtemp()
self.logger.info(
'Created temporary directory for dry run sandbox in %s' %
tmpDir)
os.chdir(tmpDir)
if 'S3' in filecacheurl.upper():
downloadFromS3(crabserver=self.crabserver,
filepath=os.path.join(tmpDir,
'dry-run-sandbox.tar.gz'),
objecttype='runtimefiles',
taskname=uniquerequestname,
logger=self.logger)
else:
ufc = CRABClient.Emulator.getEmulator('ufc')({
'endpoint': filecacheurl,
"pycurl": True
})
ufc.downloadLog('dry-run-sandbox.tar.gz',
output=os.path.join(tmpDir,
'dry-run-sandbox.tar.gz'))
for name in [
'dry-run-sandbox.tar.gz', 'InputFiles.tar.gz',
'CMSRunAnalysis.tar.gz', 'sandbox.tar.gz'
]:
tf = tarfile.open(os.path.join(tmpDir, name))
tf.extractall(tmpDir)
tf.close()
os.environ.update({
'CRAB3_RUNTIME_DEBUG': 'True',
'_CONDOR_JOB_AD': 'Job.submit'
})
with open('splitting-summary.json') as f:
splitting = json.load(f)
if self.options.skipEstimates:
return splitting, None
self.logger.info('Executing test, please wait...')
events = 10
totalJobSeconds = 0
maxSeconds = 25
while totalJobSeconds < maxSeconds:
optsList = getCMSRunAnalysisOpts('Job.submit',
'RunJobs.dag',
job=1,
events=events)
# from a python list to a string which can be used as shell command argument
opts = ''
for opt in optsList:
opts = opts + ' %s' % opt
command = 'sh CMSRunAnalysis.sh ' + opts
out, err, returncode = execute_command(command=command)
self.logger.debug(out)
if returncode != 0:
raise ClientException(
'Dry run failed to execute local test run:\n StdOut: %s\n StdErr: %s'
% (out, err))
#Once this https://github.com/dmwm/CRABServer/pull/4938 will get merged the job will be executed inside the CMSSW dir
#Therefore the 'jobReport.json' will not be in the cwd. We will delete these three lines of code in the future
jobReport = 'jobReport.json'
if not os.path.isfile(jobReport):
jobReport = os.path.join(self.configreq["jobsw"],
jobReport)
with open(jobReport) as f:
report = json.load(f)['steps']['cmsRun']['performance']
events += (maxSeconds / float(report['cpu']['AvgEventTime']))
totalJobSeconds = float(report['cpu']['TotalJobTime'])
finally:
os.chdir(cwd)
shutil.rmtree(tmpDir)
return splitting, report
def getDBSPublicationInfo_viaDasGoclient(self, outputDatasets):
"""
reimplemenation of getDBSPublicationInfo with dasgoclient. Remove dependencey from DBS
Get the lumis and number of events in the published output datasets.
"""
res = {}
res['outputDatasets'] = {}
for outputDataset in outputDatasets:
res['outputDatasets'][outputDataset] = {
'lumis': {},
'numEvents': 0
}
dbsInstance = "instance=prod/phys03"
query = "'run,lumi dataset=%s %s'" % (outputDataset, dbsInstance)
dasgo = "dasgoclient --query " + query + " --json"
runlumilist = {}
stdout, stderr, returncode = execute_command(command=dasgo,
logger=self.logger)
if returncode or not stdout:
self.logger.error('Failed running command %s. Exitcode is %s' %
(dasgo, returncode))
if stdout:
self.logger.error(' Stdout:\n %s' %
str(stdout).replace('\n', '\n '))
if stderr:
self.logger.error(' Stderr:\n %s' %
str(stderr).replace('\n', '\n '))
else:
result = json.loads(stdout)
for record in result:
run = record['run'][0]['run_number']
lumis = record['lumi'][0]['number']
runlumilist.setdefault(str(run), []).extend(lumis)
# convert to a LumiList object
outputDatasetLumis = LumiList(
runsAndLumis=runlumilist).getCompactList()
res['outputDatasets'][outputDataset]['lumis'] = outputDatasetLumis
# get total events in dataset
query = "'summary dataset=%s %s'" % (outputDataset, dbsInstance)
dasgo = "dasgoclient --query " + query + " --json"
stdout, stderr, returncode = execute_command(command=dasgo,
logger=self.logger)
if returncode or not stdout:
self.logger.error('Failed running command %s. Exitcode is %s' %
(dasgo, returncode))
if stdout:
self.logger.error(' Stdout:\n %s' %
str(stdout).replace('\n', '\n '))
if stderr:
self.logger.error(' Stderr:\n %s' %
str(stderr).replace('\n', '\n '))
total_events = 0
else:
result = json.loads(stdout)
if result:
total_events = result[0]['summary'][0]['nevents']
else:
total_events = 0
res['outputDatasets'][outputDataset]['numEvents'] = total_events
return res
def __call__(self):
username = getUsername(self.proxyfilename, logger=self.logger)
if hasattr(self.options, 'userlfn') and self.options.userlfn != None:
self.lfnPrefix = self.options.userlfn
else:
## If the user didn't provide an LFN path where to check the write permission,
## assume he/she wants to check in /store/user/<username>
self.logger.info(
'Will check write permission in the default location /store/user/<username>'
)
self.lfnPrefix = '/store/user/' + username
## Check that the location where we want to check write permission
## is one where the user will be allowed to stageout.
self.logger.info("Validating LFN %s...", self.lfnPrefix)
# if an error message is needed later, prepare it now to keep code below tidy
errMsg = "Refusing to check write permission in %s, because this is not an allowed LFN for stageout." % (
self.lfnPrefix)
errMsg += "\nThe LFN must start with either"
errMsg += " '/store/user/<username>/' or '/store/group/<groupname>/'"
errMsg += " (or '/store/local/<something>/' if publication is off),"
errMsg += " where username is your username as registered in CMS"
errMsg += " (i.e. the username of your CERN primary account)."
errMsg += "\nLFN %s is not valid." % (self.lfnPrefix)
if not checkOutLFN(self.lfnPrefix, username):
self.logger.info(errMsg)
return {'status': 'FAILED'}
else:
self.logger.info("LFN %s is valid.", self.lfnPrefix)
cp_cmd = ""
if cmd_exist("gfal-copy") and cmd_exist(
"gfal-rm") and self.command in [None, "GFAL"]:
self.logger.info(
"Will use `gfal-copy`, `gfal-rm` commands for checking write permissions"
)
cp_cmd = "gfal-copy -p -v -t 180 "
delfile_cmd = "gfal-rm -v -t 180 "
deldir_cmd = "gfal-rm -r -v -t 180 "
if self.checksum:
cp_cmd += "-K %s " % self.checksum
elif cmd_exist("lcg-cp") and cmd_exist("lcg-del"):
self.logger.info(
"Will use `lcg-cp`, `lcg-del` commands for checking write permissions"
)
cp_cmd = "lcg-cp -v -b -D srmv2 --connect-timeout 180 "
delfile_cmd = "lcg-del --connect-timeout 180 -b -l -D srmv2 "
deldir_cmd = "lcg-del -d --connect-timeout 180 -b -l -D srmv2 "
if self.checksum:
cp_cmd += "--checksum-type %s " % self.checksum
else:
self.logger.info("Neither gfal nor lcg command was found")
return {'status': 'FAILED'}
self.logger.info('Will check write permission in %s on site %s',
self.lfnPrefix, self.options.sitename)
timestamp = str(time.strftime("%Y%m%d_%H%M%S"))
self.filename = 'crab3checkwrite_' + timestamp + '.tmp'
self.subdir = 'crab3checkwrite_' + timestamp
lfn = self.lfnPrefix + '/' + self.subdir + '/' + self.filename
site = self.options.sitename
try:
pfn = self.getPFN(site=site, lfn=lfn, username=username)
except Exception:
return {'status': 'FAILED'}
self.createFile()
self.logger.info("Will use PFN: %s", pfn)
dirpfn = pfn[:len(pfn) - len(self.filename)]
self.logger.info(
'\nAttempting to create (dummy) directory %s and copy (dummy) file %s to %s\n'
% (self.subdir, self.filename, self.lfnPrefix))
cpout, cperr, cpexitcode = self.cp(pfn, cp_cmd)
if cpexitcode == 0:
self.logger.info(
'\nSuccessfully created directory %s and copied file %s to %s'
% (self.subdir, self.filename, self.lfnPrefix))
self.logger.info('\nAttempting to delete file %s\n' % (pfn))
delexitcode = self.delete(pfn, delfile_cmd)
if delexitcode:
self.logger.info('\nFailed to delete file %s' % (pfn))
finalmsg = '%sError%s: CRAB3 is able to copy but unable to delete file in %s on site %s. Asynchronous Stage Out with CRAB3 will fail.' % (
colors.RED, colors.NORMAL, self.lfnPrefix,
self.options.sitename)
finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
else:
self.logger.info('\nSuccessfully deleted file %s' % (pfn))
self.logger.info('\nAttempting to delete directory %s\n' %
(dirpfn))
delexitcode = self.delete(dirpfn, deldir_cmd)
if delexitcode:
self.logger.info('\nFailed to delete directory %s' %
(dirpfn))
finalmsg = '%sError%s: CRAB3 is able to copy but unable to delete directory in %s on site %s. Asynchronous Stage Out with CRAB3 will fail.' % (
colors.RED, colors.NORMAL, self.lfnPrefix,
self.options.sitename)
finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
else:
self.logger.info('\nSuccessfully deleted directory %s' %
(dirpfn))
finalmsg = '%sSuccess%s: Able to write in %s on site %s' % (
colors.GREEN, colors.NORMAL, self.lfnPrefix,
self.options.sitename)
returndict = {'status': 'SUCCESS'}
else:
if 'Permission denied' in cperr or 'mkdir: cannot create directory' in cperr:
finalmsg = '%sError%s: Unable to write in %s on site %s' % (
colors.RED, colors.NORMAL, self.lfnPrefix,
self.options.sitename)
finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
elif 'timeout' in cpout or 'timeout' in cperr:
self.logger.info('Connection time out.')
finalmsg = '\nUnable to check write permission in %s on site %s' % (
self.lfnPrefix, self.options.sitename)
finalmsg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
else:
finalmsg = 'Unable to check write permission in %s on site %s' % (
self.lfnPrefix, self.options.sitename)
finalmsg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
self.removeFile()
self.logger.info('\nCheckwrite Result:')
self.logger.info(finalmsg)
if returndict['status'] == 'FAILED':
self.logger.info(
'%sNote%s: You cannot write to a site if you did not ask permission.'
% (colors.BOLD, colors.NORMAL))
if 'CH_CERN' in self.options.sitename:
dbgmsg = '%sAdditional diagnostic info for CERN EOS%s\n' % (
colors.RED, colors.NORMAL)
dbgcmd = "echo '== id ==>:';id"
dbgcmd += ";echo '== voms-proxy-info -all ==>:';voms-proxy-info -all"
dbgcmd += ";which uberftp > /dev/null 2>&1 && echo '== uberftp eoscmsftp.cern.ch pwd ==>:'"
dbgcmd += ";which uberftp > /dev/null 2>&1 && uberftp eoscmsftp.cern.ch pwd"
dbgcmd += ";which uberftp > /dev/null 2>&1 || echo 'WARNING uberftp command not found. To get additional diagnostic info'"
dbgcmd += ";which uberftp > /dev/null 2>&1 || echo ' log on lxplus, get a proxy and execute:'"
dbgcmd += ";which uberftp > /dev/null 2>&1 || echo ' uberftp eoscmsftp.cern.ch pwd'"
#self.logger.info('Executing command: %s' % cmd)
#self.logger.info('Please wait...')
output, _, _ = execute_command(command=dbgcmd)
dbgmsg += output
self.logger.info(dbgmsg)
return returndict
def makeRequest(self, uri=None, data=None, verb='GET'):
"""
Make a request to the remote database for a given URI. The type of
request will determine the action taken by the server (be careful with
DELETE!).
Returns a tuple of the data from the server, decoded using the
appropriate method, the response status and the response reason, to be
used in error handling.
You can override the method to encode/decode your data by passing in an
encoding/decoding function to this method. Your encoded data must end up
as a string.
"""
data = data or {}
# Quoting the uri since it can contain the request name, and therefore spaces (see #2557)
uri = urllibQuote(uri)
caCertPath = self.getCACertPath()
url = 'https://' + self['host'] + uri
# if it is a dictionary, we need to encode it to string
if isinstance(data, dict):
data = encodeRequest(data)
self.logger.debug("Encoded data for curl request: %s" % data)
fh, path = tempfile.mkstemp(dir='/tmp', prefix='curlData')
with open(path, 'w') as f:
f.write(data)
if verb in ['GET', 'HEAD']:
url = url + '?' + data
command = ''
command += 'curl -v -X {0}'.format(verb)
command += ' -H "User-Agent: %s/%s"' % (self['userAgent'],
self['version'])
command += ' -H "Accept: */*"'
command += ' --data @%s' % path
command += ' --cert "%s"' % self['cert']
command += ' --key "%s"' % self['key']
command += ' --capath "%s"' % caCertPath
command += ' "%s" | tee /dev/stderr ' % url
# retries this up at least 3 times, or up to self['retry'] times for range of exit codes
# retries are counted AFTER 1st try, so call is made up to nRetries+1 times !
nRetries = max(2, self['retry'])
for i in range(nRetries + 1):
stdout, stderr, curlExitCode = execute_command(command=command,
logger=self.logger)
http_code, http_reason = parseResponseHeader(stderr)
if curlExitCode != 0 or http_code != 200:
if (i < 2) or (retriableError(http_code, curlExitCode) and
(i < self['retry'])):
sleeptime = 20 * (i + 1) + random.randint(-10, 10)
msg = "Sleeping %s seconds after HTTP error.\nError:\n:%s" % (
sleeptime, stderr)
self.logger.debug(msg)
else:
# this was the last retry
msg = "Fatal error trying to connect to %s using %s." % (
url, data)
self.logger.info(msg)
raise RESTInterfaceException(stderr)
else:
try:
curlResult = json.loads(stdout)
except Exception as ex:
msg = "Fatal error reading data from %s using %s: \n%s" % (
url, data, ex)
raise Exception(msg)
else:
break
return curlResult, http_code, http_reason
