def run(self, requestConfig): """ Override run() for JobType """ configArguments = {'addoutputfiles' : [], 'adduserfiles' : [], 'tfileoutfiles' : [], 'edmoutfiles' : [], } # Get SCRAM environment scram = ScramEnvironment(logger=self.logger) configArguments.update({'jobarch' : scram.scramArch, 'jobsw' : scram.cmsswVersion, }) # Build tarball if self.workdir: tarUUID = PandaInterface.wrappedUuidGen() self.logger.debug('UNIQUE NAME: tarUUID %s ' % tarUUID) if len(tarUUID): tarFilename = os.path.join(self.workdir, tarUUID +'default.tgz') cfgOutputName = os.path.join(self.workdir, 'CMSSW_cfg.py') else: raise EnvironmentException('Problem with uuidgen while preparing for Sandbox upload.') else: _dummy, tarFilename = tempfile.mkstemp(suffix='.tgz') _dummy, cfgOutputName = tempfile.mkstemp(suffix='_cfg.py') #configArguments['userisburl'] = 'https://'+ self.config.General.ufccacheUrl + '/crabcache/file?hashkey=' + uploadResults['hashkey']#XXX hardcoded #configArguments['userisburl'] = 'INSERTuserisburl'#XXX hardcoded if getattr(self.config.Data, 'inputDataset', None): configArguments['inputdata'] = self.config.Data.inputDataset # configArguments['ProcessingVersion'] = getattr(self.config.Data, 'processingVersion', None) # Create CMSSW config self.logger.debug("self.config: %s" % self.config) self.logger.debug("self.config.JobType.psetName: %s" % self.config.JobType.psetName) cmsswCfg = CMSSWConfig(config=self.config, logger=self.logger, userConfig=self.config.JobType.psetName) # Interogate CMSSW config and user config for output file names, for now no use for edmFiles or TFiles here. analysisFiles, edmFiles = cmsswCfg.outputFiles() self.logger.debug("TFiles %s and EDM Files %s will be collected" % (analysisFiles, edmFiles)) configArguments['tfileoutfiles'] = analysisFiles configArguments['edmoutfiles'] = edmFiles outputFiles = getattr(self.config.JobType, 'outputFiles', []) self.logger.debug("User files %s will be collected" % outputFiles) configArguments['addoutputfiles'].extend(outputFiles) # Write out CMSSW config cmsswCfg.writeFile(cfgOutputName) with UserTarball(name=tarFilename, logger=self.logger, config=self.config) as tb: inputFiles = getattr(self.config.JobType, 'inputFiles', []) tb.addFiles(userFiles=inputFiles, cfgOutputName=cfgOutputName) configArguments['adduserfiles'] = [os.path.basename(f) for f in inputFiles] uploadResults = tb.upload() self.logger.debug("Result uploading input files: %s " % str(uploadResults)) configArguments['cachefilename'] = uploadResults[1] configArguments['cacheurl'] = uploadResults[0] isbchecksum = uploadResults[2] # Upload lumi mask if it exists lumiMaskName = getattr(self.config.Data, 'lumiMask', None) if lumiMaskName: self.logger.debug("Attaching lumi mask %s to the request" % lumiMaskName) lumiDict = getLumiMask(config=self.config, logger=self.logger) configArguments['runs'] = lumiDict.keys() #for each run we'll encode the lumis as a string representing a list of integers #[[1,2],[5,5]] ==> '1,2,5,5' configArguments['lumis'] = [ str(reduce(lambda x,y: x+y, \ lumiDict[run]))[1:-1].replace(' ','') \ for run in configArguments['runs'] ] configArguments['jobtype'] = 'Analysis' return tarFilename, configArguments, isbchecksum
def run(self, requestConfig): """ Override run() for JobType """ configArguments = { 'addoutputfiles': [], 'adduserfiles': [], 'tfileoutfiles': [], 'edmoutfiles': [], } # Get SCRAM environment scram = ScramEnvironment(logger=self.logger) configArguments.update({ 'jobarch': scram.scramArch, 'jobsw': scram.cmsswVersion, }) # Build tarball if self.workdir: tarFilename = os.path.join( self.workdir, PandaInterface.wrappedUuidGen() + 'default.tgz') cfgOutputName = os.path.join(self.workdir, 'CMSSW_cfg.py') else: _dummy, tarFilename = tempfile.mkstemp(suffix='.tgz') _dummy, cfgOutputName = tempfile.mkstemp(suffix='_cfg.py') #configArguments['userisburl'] = 'https://'+ self.config.General.ufccacheUrl + '/crabcache/file?hashkey=' + uploadResults['hashkey']#XXX hardcoded #configArguments['userisburl'] = 'INSERTuserisburl'#XXX hardcoded if getattr(self.config.Data, 'inputDataset', None): configArguments['inputdata'] = self.config.Data.inputDataset # configArguments['ProcessingVersion'] = getattr(self.config.Data, 'processingVersion', None) # Create CMSSW config self.logger.debug("self.config: %s" % self.config) self.logger.debug("self.config.JobType.psetName: %s" % self.config.JobType.psetName) cmsswCfg = CMSSWConfig(config=self.config, logger=self.logger, userConfig=self.config.JobType.psetName) # Interogate CMSSW config and user config for output file names, for now no use for edmFiles or TFiles here. analysisFiles, edmFiles = cmsswCfg.outputFiles() self.logger.debug("TFiles %s and EDM Files %s will be collected" % (analysisFiles, edmFiles)) configArguments['tfileoutfiles'] = analysisFiles configArguments['edmoutfiles'] = edmFiles outputFiles = getattr(self.config.JobType, 'outputFiles', []) self.logger.debug("User files %s will be collected" % outputFiles) configArguments['addoutputfiles'].extend(outputFiles) # Write out CMSSW config cmsswCfg.writeFile(cfgOutputName) with UserTarball(name=tarFilename, logger=self.logger, config=self.config) as tb: inputFiles = getattr(self.config.JobType, 'inputFiles', []) tb.addFiles(userFiles=inputFiles, cfgOutputName=cfgOutputName) configArguments['adduserfiles'] = [ os.path.basename(f) for f in inputFiles ] uploadResults = tb.upload() self.logger.debug("Result uploading input files: %s " % str(uploadResults)) configArguments['cachefilename'] = uploadResults[1] configArguments['cacheurl'] = uploadResults[0] isbchecksum = uploadResults[2] # Upload lumi mask if it exists lumiMaskName = getattr(self.config.Data, 'lumiMask', None) if lumiMaskName: self.logger.debug("Attaching lumi mask %s to the request" % lumiMaskName) lumiDict = getLumiMask(config=self.config, logger=self.logger) configArguments['runs'] = lumiDict.keys() #for each run we'll encode the lumis as a string representing a list of integers #[[1,2],[5,5]] ==> '1,2,5,5' configArguments['lumis'] = [ str(reduce(lambda x,y: x+y, \ lumiDict[run]))[1:-1].replace(' ','') \ for run in configArguments['runs'] ] configArguments['jobtype'] = 'Analysis' return tarFilename, configArguments, isbchecksum