def getUsernameFromCRIC(proxyFileName=None): """ Retrieve username from CRIC by doing a query to https://cms-cric.cern.ch/api/accounts/user/query/?json&preset=whoami using the users proxy. args: proxyfile : string : the full patch to the file containing the user proxy """ ## Path to certificates. capath = os.environ[ 'X509_CERT_DIR'] if 'X509_CERT_DIR' in os.environ else "/etc/grid-security/certificates" # Path to user proxy if not proxyFileName: proxyFileName = getUserProxy() if not proxyFileName: msg = "Can't find user proxy file" raise UsernameException(msg) ## Retrieve user info from CRIC. Note the curl must be executed in same env. (i.e. CMSSW) as crab queryCmd = "curl -sS --capath %s --cert %s --key %s 'https://cms-cric.cern.ch/api/accounts/user/query/?json&preset=whoami'" %\ (capath, proxyFileName, proxyFileName) stdout, stderr, rc = execute_command(queryCmd) if rc or not stdout: msg = "Error contacting CRIC." msg += "\nDetails follow:" msg += "\n Executed command: %s" % (queryCmd) msg += "\n Stdout:\n %s" % (str(stdout).replace( '\n', '\n ')) msg += "\n Stderr:\n %s" % (str(stderr).replace( '\n', '\n ')) raise UsernameException(msg) ## Extract the username from the above command output. parseCmd = "echo '%s' | tr ':,' '\n' | grep -A1 login | tail -1 | tr -d ' \n\"'" % ( str(stdout)) username, stderr, rc = execute_command(parseCmd) if username == 'null' or not username: msg = "Failed to retrieve username from CRIC." msg += "\nDetails follow:" msg += "\n Executed command: %s" % (queryCmd) msg += "\n Stdout:\n %s" % (str(stdout).replace( '\n', '\n ')) msg += "\n Parsed username: %s" % (username) msg += "\n%sNote%s: Make sure you have the correct certificate mapped in your CERN account page" % ( colors.BOLD, colors.NORMAL) msg += " (you can check what is the certificate you currently have mapped" msg += " by looking at CERN Certificatiom Authority page." msg += "\nFor instructions on how to map a certificate, see " msg += "\n https://twiki.cern.ch/twiki/bin/view/CMSPublic/UsernameForCRAB#Adding_your_DN_to_your_profile" raise UsernameException(msg) return username
def getUserCertEndDate(self): """ Return the number of seconds until the expiration of the user cert in .globus/usercert.pem or $X509_USER_CERT if set """ cmd = 'openssl x509 -noout -dates -in %s' % self.certLocation stdout, stderr, rc = execute_command(cmd, logger=self.logger) if rc != 0: self.logger.error(stdout + '\n' + stderr) msg = "\n".join(['Error executing %s:' % cmd, stdout, stderr]) raise ProxyCreationException(msg) out = stdout.rstrip().split('notAfter=')[1] possibleFormats = ['%b %d %H:%M:%S %Y %Z', '%b %d %H:%M:%S %Y %Z'] exptime = None for frmt in possibleFormats: try: exptime = datetime.strptime(out, frmt) except ValueError: pass # try next format if not exptime: # If we cannot decode the output in any way print # a message and fallback to voms-proxy-info command self.logger.warning( 'Cannot decode "openssl x509 -noout -in %s -dates" date format.' % self.certLocation) timeleft = 0 else: # if everything is fine then we are ready to return!! timeleft = (exptime - datetime.utcnow()).total_seconds() daystoexp = int(timeleft // (60. * 60 * 24)) return daystoexp
def initFromEnv(self): """ Init the class taking the required information from the environment """ #self.command = 'scram' # SB I think this line is not needed self["SCRAM_ARCH"] = None if 'SCRAM_ARCH' in os.environ: self["SCRAM_ARCH"] = os.environ["SCRAM_ARCH"] else: stdout, _, _ = execute_command(command='scram arch') self["SCRAM_ARCH"] = stdout try: self["CMSSW_BASE"] = os.environ["CMSSW_BASE"] self["CMSSW_VERSION"] = os.environ["CMSSW_VERSION"] # Commenting these two out. I don't think they are really needed # self.cmsswReleaseBase = os.environ["CMSSW_RELEASE_BASE"] # self.localRT = os.environ["LOCALRT"] except KeyError as ke: self["CMSSW_BASE"] = None self["CMSSW_VERSION"] = None # self.cmsswReleaseBase = None # self.localRT = None msg = "Please make sure you have setup the CMS enviroment (cmsenv). Cannot find %s in your env" % str( ke) msg += "\nPlease refer to https://twiki.cern.ch/twiki/bin/view/CMSPublic/WorkBookCRAB3Tutorial#CMS_environment for how to setup the CMS enviroment." raise EnvironmentException(msg)
def create(self, username=None, retrievers=None, validity=720): """ creates a new credential in myproxy.cern.ch args: username: string: the username of the credential, usually the has of the user DN args: retrievers: string: regexp indicating list of DN's authorized to retrieve this credential args: validity: integer: how long this credential will be valid for in hours, default is 30 days example of the command we want : command : export GT_PROXY_MODE=rfc myproxy-init -d -n -s myproxy.cern.ch -x -R '/DC=ch/DC=cern/OU=computers/CN=vocms0105.cern.ch|/DC=ch/DC=cern/OU=computers/CN=crab-(preprod|prod|dev)-tw(02|01).cern.ch|/DC=ch/DC=cern/OU=computers/CN=(ddi|ddidk|mytw).cern.ch|/DC=ch/DC=cern/OU=computers/CN=stefanov(m|m2).cern.ch' -x -Z '/DC=ch/DC=cern/OU=computers/CN=vocms0105.cern.ch|/DC=ch/DC=cern/OU=computers/CN=crab-(preprod|prod|dev)-tw(02|01).cern.ch|/DC=ch/DC=cern/OU=computers/CN=(ddi|ddidk|mytw).cern.ch|/DC=ch/DC=cern/OU=computers/CN=stefanov(m|m2).cern.ch' -l 'be1f4dc5be8664cbd145bf008f5399adf42b086f' -t 168:00 -c 3600:00 """ cmd = 'export GT_PROXY_MODE=rfc ; myproxy-init -d -n -s myproxy.cern.ch' cmd += ' -C %s' % self.certLocation cmd += ' -y %s' % self.keyLocation cmd += ' -x -R \'%s\'' % retrievers cmd += ' -x -Z \'%s\'' % retrievers cmd += ' -l %s' % username cmd += ' -t 168 -c %s' % validity # validity of the retrieved proxy: 7 days = 168 hours stdout, stderr, rc = execute_command(cmd, logger=self.logger) if rc != 0: self.logger.error(stdout + '\n' + stderr) msg = "\n".join(['Error executing %s:' % cmd, stdout, stderr]) raise ProxyCreationException(msg)
def getSubject(self): cmd = 'voms-proxy-info --identity --file %s' % self.proxyFile stdout, stderr, rc = execute_command(cmd, logger=self.logger) if rc != 0: self.logger.error(stdout + '\n' + stderr) msg = "\n".join(['Error executing %s:' % cmd, stdout, stderr]) raise ProxyCreationException(msg) return stdout.rstrip()
def getTimeLeft(self): cmd = 'voms-proxy-info --actimeleft --timeleft --file %s' % self.proxyFile stdout, stderr, rc = execute_command(cmd, logger=self.logger) if rc != 0: self.logger.error(stdout + '\n' + stderr) msg = "\n".join(['Error executing %s:' % cmd, stdout, stderr]) raise ProxyCreationException(msg) # pick the shorter between actimeleft and timeleft times = stdout.split('\n') timeLeft = min(int(times[0]), int(times[1])) return timeLeft
def delete(self, pfn, command): undoScram = "which scram >/dev/null 2>&1 && eval `scram unsetenv -sh`" rmcmd = undoScram + "; " + command + "'" + pfn + "'" self.logger.info('Executing command: %s' % rmcmd) self.logger.info('Please wait...') delout, delerr, delexitcode = execute_command(rmcmd) if delexitcode: self.logger.info('Failed running delete command') if delout: self.logger.info(' Stdout:\n %s' % str(delout).replace('\n', '\n ')) if delerr: self.logger.info(' Stderr:\n %s' % str(delerr).replace('\n', '\n ')) return delexitcode
def getGroupAndRole(self): cmd = 'voms-proxy-info --fqan --file %s' % self.proxyFile stdout, stderr, rc = execute_command(cmd, logger=self.logger) if rc != 0: self.logger.error(stdout + '\n' + stderr) msg = "\n".join(['Error executing %s:' % cmd, stdout, stderr]) raise ProxyCreationException(msg) fqans = str(stdout) primaryFqan = fqans.split('\n')[0] attributes = primaryFqan.split('/') if len(attributes) > 4: group = attributes[2] role = attributes[3].split('=')[1] else: group = '' role = attributes[2].split('=')[1] return group, role
def getPFN(self, site='T2_CH_CERN', lfn='/store/user', username='******'): # prepare a simply python script to resolve lfn2pfn via Rucio template = """ from rucio.client import Client client = Client() rse = "{site}" lfn = ["user.{username}:{lfn}"] for operation in ['third_party_copy', 'write', 'read']: try: #print('Try Rucio lfn2pn with operation %s', operation) out = client.lfns2pfns(rse, lfn, operation=operation) break except Exception as ex: print("Failed to resolve LNF to PFN via Rucio. Error is:\\n %s" % str(ex)) if not out: print("Failed to resolve LNF to PFN via Rucio.") exit(1) print(out[lfn[0]]) exit(0) """ rucioScript = template.format(site=site, username=username, lfn=lfn) import tempfile (_, scriptName) = tempfile.mkstemp(dir='/tmp', prefix='lfn2pfn-', suffix='.py') with open(scriptName, 'w') as ofile: ofile.write(rucioScript) cmd = 'eval `scram unsetenv -sh`; ' cmd += 'source /cvmfs/cms.cern.ch/rucio/setup.sh 2>/dev/null; export RUCIO_ACCOUNT=%s; ' % username cmd += 'python %s; ' % scriptName rucioOut, rucioErr, exitcode = execute_command(cmd) os.unlink(scriptName) if exitcode: self.logger.info('PFN lookup failed') if rucioOut: self.logger.info(' Stdout:\n %s' % str(rucioOut).replace('\n', '\n ')) if rucioErr: self.logger.info(' Stderr:\n %s' % str(rucioErr).replace('\n', '\n ')) raise Exception pfn = rucioOut.rstrip() return pfn
def executeTestRun(self, inputArgs, jobnr): """ Execute a test run calling CMSRunAnalysis.sh """ os.environ.update({ 'CRAB3_RUNTIME_DEBUG': 'True', '_CONDOR_JOB_AD': 'Job.submit' }) optsList = [ os.path.join(os.getcwd(), 'TweakPSet.py'), '-a %s' % inputArgs[jobnr - 1]['CRAB_Archive'], '-o %s' % inputArgs[jobnr - 1]['CRAB_AdditionalOutputFiles'], '--sourceURL=%s' % inputArgs[jobnr - 1]['CRAB_ISB'], '--location=%s' % os.getcwd(), '--inputFile=%s' % inputArgs[jobnr - 1]['inputFiles'], '--runAndLumis=%s' % inputArgs[jobnr - 1]['runAndLumiMask'], '--firstEvent=%s' % inputArgs[jobnr - 1] ['firstEvent'], #jobs goes from 1 to N, inputArgs from 0 to N-1 '--lastEvent=%s' % inputArgs[jobnr - 1]['lastEvent'], '--firstLumi=%s' % inputArgs[jobnr - 1]['firstLumi'], '--firstRun=%s' % inputArgs[jobnr - 1]['firstRun'], '--seeding=%s' % inputArgs[jobnr - 1]['seeding'], '--lheInputFiles=%s' % inputArgs[jobnr - 1]['lheInputFiles'], '--oneEventMode=0', '--eventsPerLumi=%s' % inputArgs[jobnr - 1]['eventsPerLumi'], '--maxRuntime=-1', '--jobNumber=%s' % (jobnr - 1), '--cmsswVersion=%s' % inputArgs[jobnr - 1]['CRAB_JobSW'], '--scramArch=%s' % inputArgs[jobnr - 1]['CRAB_JobArch'], '--scriptExe=%s' % inputArgs[jobnr - 1]['scriptExe'], '--scriptArgs=%s' % inputArgs[jobnr - 1]['scriptArgs'], ] # from a python list to a string which can be used as shell command argument opts = '' for opt in optsList: opts = opts + ' %s' % opt command = 'sh CMSRunAnalysis.sh ' + opts out, err, returncode = execute_command(command=command) self.logger.debug(out) self.logger.debug(err) if returncode != 0: raise ClientException( 'Failed to execute local test run:\n StdOut: %s\n StdErr: %s' % (out, err))
def getInfo(self, username=None): """ returns information about a credential stored in myproxy.cern.ch args: username: string: the username of the credential, usually the has of the user DN """ cmd = 'myproxy-info -s myproxy.cern.ch -l %s' % username stdout, stderr, rc = execute_command(cmd, logger=self.logger) if rc != 0: self.logger.error(stdout + '\n' + stderr) if rc > 0 or not stdout: # if there's no credential myproxy-info returns rc=1 return 0, '' olines = stdout.rstrip().split('\n') trustedRetrievalPolicy = olines[-2] trustedRetrievers = trustedRetrievalPolicy.split(':')[1].strip() times = olines[-1].split(':') hours = int(times[1]) mins = int(times[2]) timeLeft = hours * 3600 + mins * 60 # let's ignore seconds return timeLeft, trustedRetrievers
def cp(self, pfn, command): abspath = os.path.abspath(self.filename) if cmd_exist("gfal-copy") and self.command in [None, "GFAL"]: abspath = "file://" + abspath undoScram = "which scram >/dev/null 2>&1 && eval `scram unsetenv -sh`" cpcmd = undoScram + "; " + command + abspath + " '" + pfn + "'" self.logger.info('Executing command: %s' % cpcmd) self.logger.info('Please wait...') cpout, cperr, cpexitcode = execute_command(cpcmd) if cpexitcode: self.logger.info('Failed running copy command') if cpout: self.logger.info(' Stdout:\n %s' % str(cpout).replace('\n', '\n ')) if cperr: self.logger.info(' Stderr:\n %s' % str(cperr).replace('\n', '\n ')) return cpout, cperr, cpexitcode
def curlGetFileFromURL(url, filename=None, proxyfilename=None, logger=None): """ Read the content of a URL into a file via curl url: the link you would like to retrieve filename: the local filename where the url is saved to. Defaults to the filename in the url proxyfilename: the x509 proxy certificate to be used in case auth is required returns: the exit code of the command if command failed, otherwise the HTTP code of the call note that curl exits with status 0 if the HTTP calls fail, """ ## Path to certificates. capath = os.environ[ 'X509_CERT_DIR'] if 'X509_CERT_DIR' in os.environ else "/etc/grid-security/certificates" # send curl output to file and http_code to stdout downloadCommand = 'curl -sS --capath %s --cert %s --key %s -o %s -w %%"{http_code}"' %\ (capath, proxyfilename, proxyfilename, filename) downloadCommand += ' "%s"' % url if logger: logger.debug("Will execute:\n%s", downloadCommand) stdout, stderr, rc = execute_command(downloadCommand, logger=logger) errorDetails = '' if rc != 0: os.unlink(filename) httpCode = 503 else: httpCode = int(stdout) if httpCode != 200: with open(filename) as f: errorDetails = f.read() os.unlink(filename) if logger: logger.debug('exitcode: %s\nstdout: %s\nstderr: %s\nerror details: %s', rc, stdout, stderr, errorDetails) return httpCode
def create(self, timeLeftThreshold=720): # is there a proxy already ? # does it have correct group and role ? # is it valid long enough ? # all OK, do nothing # need a new proxy cmd = 'voms-proxy-init --rfc' cmd += ' --cert %s' % self.certLocation cmd += ' --key %s' % self.keyLocation cmd += ' --out %s' % self.proxyFile cmd += ' --valid %s' % self.desiredValidity vomsString = 'cms:/cms' if self.group: vomsString += '/%s' % self.group if self.role and self.role != 'NULL': vomsString += '/Role=%s' % self.role cmd += ' --voms %s' % vomsString stdout, stderr, rc = execute_command(cmd, logger=self.logger, redirect=False) if rc != 0: self.logger.error(stdout + '\n' + stderr) msg = "\n".join(['Error executing %s:' % cmd, stdout, stderr]) raise ProxyCreationException(msg)
def __init__(self, logger, cmdargs=None, disable_interspersed_args=False): """ Initialize common client parameters """ if not hasattr(self, 'name'): self.name = self.__class__.__name__ ConfigCommand.__init__(self) # The command logger. self.logger = logger self.logfile = self.logger.logfile stdout, _, _ = execute_command(command='uname -a') localSystem = stdout.strip() try: localOS, _, _ = execute_command('grep PRETTY_NAME /etc/os-release') localOS = localOS.strip().split('=')[1].strip('"') except Exception as ex: # pylint: disable=unused-variable try: localOS, _, _ = execute_command(command='lsb_release -d') localOS = localOS.strip().split(':')[1].strip() except Exception as ex: # pylint: disable=unused-variable localOS = "Unknown Operating System" self.logger.debug("CRAB Client version: %s", __version__) self.logger.debug("Running on: " + localSystem + " - " + localOS) self.logger.debug("Executing command: '%s'" % str(self.name)) # Get the command configuration. self.cmdconf = commandsConfiguration.get(self.name) if not self.cmdconf: raise RuntimeError( "Canot find command %s in commandsConfiguration inside ClientMapping. Are you a developer" "trying to add a command without it's correspondant configuration?" % self.name) # Get the CRAB cache file. self.cachedinfo = None self.crab3dic = self.getConfiDict() # The options parser. self.parser = CRABCmdOptParser(self.name, self.__doc__, disable_interspersed_args) # Define the command options. self.setSuperOptions() # Parse the command options/arguments. cmdargs = cmdargs or [] (self.options, self.args) = self.parser.parse_args(cmdargs) self.transferringIds = None self.dest = None self.validateLogpathOption() # Validate first the SubCommand options SubCommand.validateOptions(self) # then the config option for the submit command self.validateConfigOption() # Get the VO group/role from the command options (if the command requires these # options). proxyOptsSetPlace = { 'set_in': { 'group': "default", 'role': "default" }, 'for_set_use': "" } msgadd = [] self.voGroup, self.voRole = "", "NULL" if self.cmdconf['requiresProxyVOOptions']: proxyOptsSetPlace['for_set_use'] = "cmdopts" if self.options.voGroup is not None: self.voGroup = self.options.voGroup proxyOptsSetPlace['set_in']['group'] = "cmdopts" msgadd.append("VO group '%s'" % (self.voGroup)) if self.options.voRole is not None: self.voRole = self.options.voRole if self.options.voRole != "" else "NULL" proxyOptsSetPlace['set_in']['role'] = "cmdopts" msgadd.append("VO role '%s'" % (self.voRole)) if msgadd: msg = "Using %s as specified in the crab command options." % ( " and ".join(msgadd)) self.logger.debug(msg) # Create the object that will do the proxy operations. We don't really care # what VO role and group and server URL we pass to the constructor, because # these are not used until we do the proxy delegation to the myproxy server. # And this happens in handleProxy(), which is called after we load the # configuration file and retrieve the final values for those parameters. # handleProxy() takes care of passing those parameters to self.credentialHandler self.credentialHandler = CredentialInteractions(self.logger) # If the user didn't use the --proxy command line option, and if there isn't a # valid proxy already, we will create a new one with the current VO role and group # (as commented above, we don't really care what are the VO role and group so # far). self.proxyCreated = False # If there is an input configuration file: if hasattr(self.options, 'config') and self.options.config is not None: proxyOptsSetPlace['for_set_use'] = "config" # Load the configuration file and validate it. self.loadConfig(self.options.config, self.args) # Create the CRAB project directory. self.requestarea, self.requestname, self.logfile = createWorkArea( self.logger, getattr(self.configuration.General, 'workArea', None), getattr(self.configuration.General, 'requestName', None)) # Get the VO group/role from the configuration file. msgadd = [] if hasattr(self.configuration, 'User') and hasattr( self.configuration.User, 'voGroup'): self.voGroup = self.configuration.User.voGroup proxyOptsSetPlace['set_in']['group'] = "config" msgadd.append("VO group '%s'" % (self.voGroup)) if hasattr(self.configuration, 'User') and hasattr( self.configuration.User, 'voRole'): self.voRole = self.configuration.User.voRole if self.configuration.User.voRole != "" else "NULL" proxyOptsSetPlace['set_in']['role'] = "config" msgadd.append("VO role '%s'" % (self.voRole)) if msgadd: msg = "Using %s as specified in the CRAB configuration file." % ( " and ".join(msgadd)) self.logger.debug(msg) # If the VO group/role was not given in the command options, take it from the request cache. if self.cmdconf['requiresDirOption']: self.setCachedProxy(proxyOptsSetPlace) # If the server URL isn't already set, we check the args and then the config. if not hasattr(self, 'serverurl') and self.cmdconf['requiresREST']: self.instance, self.serverurl = self.serverInstance() elif not self.cmdconf['requiresREST']: self.instance, self.serverurl = None, None # Update (or create) the CRAB cache file. self.updateCRABCacheFile() # At this point we check if there is a valid proxy, and # eventually create a new one. If the proxy was not created by CRAB, we check that the # VO role/group in the proxy are the same as specified by the user in the configuration # file (or in the command line options). If it is not, we ask the user if he wants to # overwrite the current proxy. If he doesn't want to overwrite it, we don't continue # and ask him to provide the VO role/group as in the existing proxy. # Finally, delegate the proxy to myproxy server. self.handleVomsProxy(proxyOptsSetPlace) # only if this command talks to the REST we create a CRABRest object to communicate with CRABServer # and check/upate credentials on myproxy # this is usually the first time that a call to the server is made, so where Emulator('rest') is initialized # arguments to Emulator('rest') call must match those for HTTPRequest.__init__ in RESTInteractions.py #server = CRABClient.Emulator.getEmulator('rest')(url=serverurl, localcert=proxyfilename, localkey=proxyfilename, # version=__version__, retry=2, logger=logger) if self.cmdconf['requiresREST']: crabRest = CRABClient.Emulator.getEmulator('rest') self.crabserver = crabRest(hostname=self.serverurl, localcert=self.proxyfilename, localkey=self.proxyfilename, retry=2, logger=self.logger, verbose=False, version=__version__, userAgent='CRABClient') self.crabserver.setDbInstance(self.instance) # prepare also a test crabserver instance which will send tarballs to S3 self.s3tester = crabRest(hostname='cmsweb-testbed.cern.ch', localcert=self.proxyfilename, localkey=self.proxyfilename, retry=0, logger=self.logger, verbose=False, version=__version__, userAgent='CRABClient') self.s3tester.setDbInstance('preprod') self.handleMyProxy() # Validate the command options self.validateOptions() self.validateOptions() # Log user command and options used for debuging purpose. self.logger.debug('Command use: %s' % self.name) self.logger.debug('Options use: %s' % cmdargs) if self.cmdconf['requiresREST']: self.checkversion() self.defaultApi = 'workflow' self.logger.debug("Instance is %s" % (self.instance)) self.logger.debug("Server base url is %s" % (self.serverurl)) if self.cmdconf['requiresREST']: self.logger.debug("Command api %s" % (self.defaultApi))
def processWorker(self, input_, successfiles, failedfiles): """ _processWorker_ Runs a subprocessed command. """ logger = self.setSubprocessLog() # Get this started while True: try: myfile, work = input_.get() except (EOFError, IOError): crashMessage = "Hit EOF/IO in getting new work\n" crashMessage += "Assuming this is a graceful break attempt." print(crashMessage) break if work == 'STOP': break else: fileid = myfile['pfn'].split('/')[-1] dirpath = os.path.join( self.options.destination, myfile['suffix'] if 'suffix' in myfile else '') url_input = bool(re.match("^[a-z]+://", dirpath)) if not os.path.isdir(dirpath) and not url_input: os.makedirs(dirpath) localFilename = os.path.join(dirpath, str(fileid)) command = work logger.info("Retrieving %s " % fileid) logger.debug("Executing %s" % command) try: stdout, stderr, returncode = execute_command(command=command) except KeyboardInterrupt: logger.info("Subprocess exit due to keyboard interrupt") break error = simpleOutputCheck(stderr) logger.debug("Finish executing for file %s" % fileid) if returncode != 0 or len(error) > 0: logger.info("%sWarning%s: Failed retrieving %s" % (colors.RED, colors.NORMAL, fileid)) #logger.debug(colors.RED +"Stderr: %s " %stderr+ colors.NORMAL) for x in error: logger.info(colors.RED + "\t %s" % x + colors.NORMAL) failedfiles[fileid] = str(error) logger.debug("Full stderr follows:\n%s" % stderr) if "timed out" in stderr or "timed out" in stdout: logger.info( "%sWarning%s: Failed due to connection timeout" % (colors.RED, colors.NORMAL)) logger.info( "Please use the '--wait=<#seconds>' option to increase the connection timeout" ) if "checksum" in stderr: logger.info( "%sWarning%s: as of 3.3.1510 CRAB3 is using an option to validate the checksum with lcg-cp/gfal-cp commands." " You might get false positives since for some site this is not working." " In that case please use the option --checksum=no" % (colors.RED, colors.NORMAL)) if os.path.isfile(localFilename) and os.path.getsize( localFilename) != myfile['size']: logger.debug("File %s has the wrong size, deleting it" % fileid) try: os.remove(localFilename) except OSError as ex: logger.debug( "%sWarning%s: Cannot remove the file because of: %s" % (colors.RED, colors.NORMAL, ex)) try: time.sleep(60) except KeyboardInterrupt: logger.info("Subprocess exit due to keyboard interrupt") break else: logger.info("%sSuccess%s: Success in retrieving %s " % (colors.GREEN, colors.NORMAL, fileid)) successfiles[fileid] = 'Successfully retrieved' return
def executeTestRun(self, filecacheurl, uniquerequestname): """ Downloads the dry run tarball from the User File Cache and unpacks it in a temporary directory. Runs a trial to obtain the performance report. Repeats trial with successively larger input events until a job length of maxSeconds is reached (this improves accuracy for fast-running CMSSW parameter sets.) """ cwd = os.getcwd() try: tmpDir = tempfile.mkdtemp() self.logger.info( 'Created temporary directory for dry run sandbox in %s' % tmpDir) os.chdir(tmpDir) if 'S3' in filecacheurl.upper(): downloadFromS3(crabserver=self.crabserver, filepath=os.path.join(tmpDir, 'dry-run-sandbox.tar.gz'), objecttype='runtimefiles', taskname=uniquerequestname, logger=self.logger) else: ufc = CRABClient.Emulator.getEmulator('ufc')({ 'endpoint': filecacheurl, "pycurl": True }) ufc.downloadLog('dry-run-sandbox.tar.gz', output=os.path.join(tmpDir, 'dry-run-sandbox.tar.gz')) for name in [ 'dry-run-sandbox.tar.gz', 'InputFiles.tar.gz', 'CMSRunAnalysis.tar.gz', 'sandbox.tar.gz' ]: tf = tarfile.open(os.path.join(tmpDir, name)) tf.extractall(tmpDir) tf.close() os.environ.update({ 'CRAB3_RUNTIME_DEBUG': 'True', '_CONDOR_JOB_AD': 'Job.submit' }) with open('splitting-summary.json') as f: splitting = json.load(f) if self.options.skipEstimates: return splitting, None self.logger.info('Executing test, please wait...') events = 10 totalJobSeconds = 0 maxSeconds = 25 while totalJobSeconds < maxSeconds: optsList = getCMSRunAnalysisOpts('Job.submit', 'RunJobs.dag', job=1, events=events) # from a python list to a string which can be used as shell command argument opts = '' for opt in optsList: opts = opts + ' %s' % opt command = 'sh CMSRunAnalysis.sh ' + opts out, err, returncode = execute_command(command=command) self.logger.debug(out) if returncode != 0: raise ClientException( 'Dry run failed to execute local test run:\n StdOut: %s\n StdErr: %s' % (out, err)) #Once this https://github.com/dmwm/CRABServer/pull/4938 will get merged the job will be executed inside the CMSSW dir #Therefore the 'jobReport.json' will not be in the cwd. We will delete these three lines of code in the future jobReport = 'jobReport.json' if not os.path.isfile(jobReport): jobReport = os.path.join(self.configreq["jobsw"], jobReport) with open(jobReport) as f: report = json.load(f)['steps']['cmsRun']['performance'] events += (maxSeconds / float(report['cpu']['AvgEventTime'])) totalJobSeconds = float(report['cpu']['TotalJobTime']) finally: os.chdir(cwd) shutil.rmtree(tmpDir) return splitting, report
def getDBSPublicationInfo_viaDasGoclient(self, outputDatasets): """ reimplemenation of getDBSPublicationInfo with dasgoclient. Remove dependencey from DBS Get the lumis and number of events in the published output datasets. """ res = {} res['outputDatasets'] = {} for outputDataset in outputDatasets: res['outputDatasets'][outputDataset] = { 'lumis': {}, 'numEvents': 0 } dbsInstance = "instance=prod/phys03" query = "'run,lumi dataset=%s %s'" % (outputDataset, dbsInstance) dasgo = "dasgoclient --query " + query + " --json" runlumilist = {} stdout, stderr, returncode = execute_command(command=dasgo, logger=self.logger) if returncode or not stdout: self.logger.error('Failed running command %s. Exitcode is %s' % (dasgo, returncode)) if stdout: self.logger.error(' Stdout:\n %s' % str(stdout).replace('\n', '\n ')) if stderr: self.logger.error(' Stderr:\n %s' % str(stderr).replace('\n', '\n ')) else: result = json.loads(stdout) for record in result: run = record['run'][0]['run_number'] lumis = record['lumi'][0]['number'] runlumilist.setdefault(str(run), []).extend(lumis) # convert to a LumiList object outputDatasetLumis = LumiList( runsAndLumis=runlumilist).getCompactList() res['outputDatasets'][outputDataset]['lumis'] = outputDatasetLumis # get total events in dataset query = "'summary dataset=%s %s'" % (outputDataset, dbsInstance) dasgo = "dasgoclient --query " + query + " --json" stdout, stderr, returncode = execute_command(command=dasgo, logger=self.logger) if returncode or not stdout: self.logger.error('Failed running command %s. Exitcode is %s' % (dasgo, returncode)) if stdout: self.logger.error(' Stdout:\n %s' % str(stdout).replace('\n', '\n ')) if stderr: self.logger.error(' Stderr:\n %s' % str(stderr).replace('\n', '\n ')) total_events = 0 else: result = json.loads(stdout) if result: total_events = result[0]['summary'][0]['nevents'] else: total_events = 0 res['outputDatasets'][outputDataset]['numEvents'] = total_events return res
def __call__(self): username = getUsername(self.proxyfilename, logger=self.logger) if hasattr(self.options, 'userlfn') and self.options.userlfn != None: self.lfnPrefix = self.options.userlfn else: ## If the user didn't provide an LFN path where to check the write permission, ## assume he/she wants to check in /store/user/<username> self.logger.info( 'Will check write permission in the default location /store/user/<username>' ) self.lfnPrefix = '/store/user/' + username ## Check that the location where we want to check write permission ## is one where the user will be allowed to stageout. self.logger.info("Validating LFN %s...", self.lfnPrefix) # if an error message is needed later, prepare it now to keep code below tidy errMsg = "Refusing to check write permission in %s, because this is not an allowed LFN for stageout." % ( self.lfnPrefix) errMsg += "\nThe LFN must start with either" errMsg += " '/store/user/<username>/' or '/store/group/<groupname>/'" errMsg += " (or '/store/local/<something>/' if publication is off)," errMsg += " where username is your username as registered in CMS" errMsg += " (i.e. the username of your CERN primary account)." errMsg += "\nLFN %s is not valid." % (self.lfnPrefix) if not checkOutLFN(self.lfnPrefix, username): self.logger.info(errMsg) return {'status': 'FAILED'} else: self.logger.info("LFN %s is valid.", self.lfnPrefix) cp_cmd = "" if cmd_exist("gfal-copy") and cmd_exist( "gfal-rm") and self.command in [None, "GFAL"]: self.logger.info( "Will use `gfal-copy`, `gfal-rm` commands for checking write permissions" ) cp_cmd = "gfal-copy -p -v -t 180 " delfile_cmd = "gfal-rm -v -t 180 " deldir_cmd = "gfal-rm -r -v -t 180 " if self.checksum: cp_cmd += "-K %s " % self.checksum elif cmd_exist("lcg-cp") and cmd_exist("lcg-del"): self.logger.info( "Will use `lcg-cp`, `lcg-del` commands for checking write permissions" ) cp_cmd = "lcg-cp -v -b -D srmv2 --connect-timeout 180 " delfile_cmd = "lcg-del --connect-timeout 180 -b -l -D srmv2 " deldir_cmd = "lcg-del -d --connect-timeout 180 -b -l -D srmv2 " if self.checksum: cp_cmd += "--checksum-type %s " % self.checksum else: self.logger.info("Neither gfal nor lcg command was found") return {'status': 'FAILED'} self.logger.info('Will check write permission in %s on site %s', self.lfnPrefix, self.options.sitename) timestamp = str(time.strftime("%Y%m%d_%H%M%S")) self.filename = 'crab3checkwrite_' + timestamp + '.tmp' self.subdir = 'crab3checkwrite_' + timestamp lfn = self.lfnPrefix + '/' + self.subdir + '/' + self.filename site = self.options.sitename try: pfn = self.getPFN(site=site, lfn=lfn, username=username) except Exception: return {'status': 'FAILED'} self.createFile() self.logger.info("Will use PFN: %s", pfn) dirpfn = pfn[:len(pfn) - len(self.filename)] self.logger.info( '\nAttempting to create (dummy) directory %s and copy (dummy) file %s to %s\n' % (self.subdir, self.filename, self.lfnPrefix)) cpout, cperr, cpexitcode = self.cp(pfn, cp_cmd) if cpexitcode == 0: self.logger.info( '\nSuccessfully created directory %s and copied file %s to %s' % (self.subdir, self.filename, self.lfnPrefix)) self.logger.info('\nAttempting to delete file %s\n' % (pfn)) delexitcode = self.delete(pfn, delfile_cmd) if delexitcode: self.logger.info('\nFailed to delete file %s' % (pfn)) finalmsg = '%sError%s: CRAB3 is able to copy but unable to delete file in %s on site %s. Asynchronous Stage Out with CRAB3 will fail.' % ( colors.RED, colors.NORMAL, self.lfnPrefix, self.options.sitename) finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.' returndict = {'status': 'FAILED'} else: self.logger.info('\nSuccessfully deleted file %s' % (pfn)) self.logger.info('\nAttempting to delete directory %s\n' % (dirpfn)) delexitcode = self.delete(dirpfn, deldir_cmd) if delexitcode: self.logger.info('\nFailed to delete directory %s' % (dirpfn)) finalmsg = '%sError%s: CRAB3 is able to copy but unable to delete directory in %s on site %s. Asynchronous Stage Out with CRAB3 will fail.' % ( colors.RED, colors.NORMAL, self.lfnPrefix, self.options.sitename) finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.' returndict = {'status': 'FAILED'} else: self.logger.info('\nSuccessfully deleted directory %s' % (dirpfn)) finalmsg = '%sSuccess%s: Able to write in %s on site %s' % ( colors.GREEN, colors.NORMAL, self.lfnPrefix, self.options.sitename) returndict = {'status': 'SUCCESS'} else: if 'Permission denied' in cperr or 'mkdir: cannot create directory' in cperr: finalmsg = '%sError%s: Unable to write in %s on site %s' % ( colors.RED, colors.NORMAL, self.lfnPrefix, self.options.sitename) finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.' returndict = {'status': 'FAILED'} elif 'timeout' in cpout or 'timeout' in cperr: self.logger.info('Connection time out.') finalmsg = '\nUnable to check write permission in %s on site %s' % ( self.lfnPrefix, self.options.sitename) finalmsg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.' returndict = {'status': 'FAILED'} else: finalmsg = 'Unable to check write permission in %s on site %s' % ( self.lfnPrefix, self.options.sitename) finalmsg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.' returndict = {'status': 'FAILED'} self.removeFile() self.logger.info('\nCheckwrite Result:') self.logger.info(finalmsg) if returndict['status'] == 'FAILED': self.logger.info( '%sNote%s: You cannot write to a site if you did not ask permission.' % (colors.BOLD, colors.NORMAL)) if 'CH_CERN' in self.options.sitename: dbgmsg = '%sAdditional diagnostic info for CERN EOS%s\n' % ( colors.RED, colors.NORMAL) dbgcmd = "echo '== id ==>:';id" dbgcmd += ";echo '== voms-proxy-info -all ==>:';voms-proxy-info -all" dbgcmd += ";which uberftp > /dev/null 2>&1 && echo '== uberftp eoscmsftp.cern.ch pwd ==>:'" dbgcmd += ";which uberftp > /dev/null 2>&1 && uberftp eoscmsftp.cern.ch pwd" dbgcmd += ";which uberftp > /dev/null 2>&1 || echo 'WARNING uberftp command not found. To get additional diagnostic info'" dbgcmd += ";which uberftp > /dev/null 2>&1 || echo ' log on lxplus, get a proxy and execute:'" dbgcmd += ";which uberftp > /dev/null 2>&1 || echo ' uberftp eoscmsftp.cern.ch pwd'" #self.logger.info('Executing command: %s' % cmd) #self.logger.info('Please wait...') output, _, _ = execute_command(command=dbgcmd) dbgmsg += output self.logger.info(dbgmsg) return returndict
def makeRequest(self, uri=None, data=None, verb='GET'): """ Make a request to the remote database for a given URI. The type of request will determine the action taken by the server (be careful with DELETE!). Returns a tuple of the data from the server, decoded using the appropriate method, the response status and the response reason, to be used in error handling. You can override the method to encode/decode your data by passing in an encoding/decoding function to this method. Your encoded data must end up as a string. """ data = data or {} # Quoting the uri since it can contain the request name, and therefore spaces (see #2557) uri = urllibQuote(uri) caCertPath = self.getCACertPath() url = 'https://' + self['host'] + uri # if it is a dictionary, we need to encode it to string if isinstance(data, dict): data = encodeRequest(data) self.logger.debug("Encoded data for curl request: %s" % data) fh, path = tempfile.mkstemp(dir='/tmp', prefix='curlData') with open(path, 'w') as f: f.write(data) if verb in ['GET', 'HEAD']: url = url + '?' + data command = '' command += 'curl -v -X {0}'.format(verb) command += ' -H "User-Agent: %s/%s"' % (self['userAgent'], self['version']) command += ' -H "Accept: */*"' command += ' --data @%s' % path command += ' --cert "%s"' % self['cert'] command += ' --key "%s"' % self['key'] command += ' --capath "%s"' % caCertPath command += ' "%s" | tee /dev/stderr ' % url # retries this up at least 3 times, or up to self['retry'] times for range of exit codes # retries are counted AFTER 1st try, so call is made up to nRetries+1 times ! nRetries = max(2, self['retry']) for i in range(nRetries + 1): stdout, stderr, curlExitCode = execute_command(command=command, logger=self.logger) http_code, http_reason = parseResponseHeader(stderr) if curlExitCode != 0 or http_code != 200: if (i < 2) or (retriableError(http_code, curlExitCode) and (i < self['retry'])): sleeptime = 20 * (i + 1) + random.randint(-10, 10) msg = "Sleeping %s seconds after HTTP error.\nError:\n:%s" % ( sleeptime, stderr) self.logger.debug(msg) else: # this was the last retry msg = "Fatal error trying to connect to %s using %s." % ( url, data) self.logger.info(msg) raise RESTInterfaceException(stderr) else: try: curlResult = json.loads(stdout) except Exception as ex: msg = "Fatal error reading data from %s using %s: \n%s" % ( url, data, ex) raise Exception(msg) else: break return curlResult, http_code, http_reason