def createInitialReport(job, logLocation): """ _createInitialReport_ Create an initial job report with the base information in it. """ try: siteCfg = loadSiteLocalConfig() except SiteConfigError: # For now, assume that we did this on purpose msg = "Couldn't find SiteConfig" logging.error(msg) # TODO: Make less goatballs for testing purposes return report = Report.Report() report.data.WMAgentJobID = job.get('id', None) report.data.WMAgentJobName = job.get('name', None) report.data.pnn = siteCfg.localStageOut.get('phedex-node', 'Unknown') report.data.siteName = getattr(siteCfg, 'siteName', 'Unknown') report.data.hostName = socket.gethostname() report.data.ceName = getSyncCE() report.data.completed = False report.setTaskName(taskName=job.get('task', 'TaskNotFound')) # Not so fond of this, but we have to put the master # report way up at the top so it's returned if the # job fails early reportPath = os.path.join(os.getcwd(), '../', logLocation) report.save(reportPath) return
def testSlcPhedexNodesEqualPhedexApiNodes(self): """ For each site, verify that the stageout node specified in site-local-config.xml is the same as the one returned by the PhEDEx api. """ os.environ["CMS_PATH"] = "/cvmfs/cms.cern.ch" phedex = PhEDEx() nodes = phedex.getNodeMap()["phedex"]["node"] # Make a dict for translating the se names into regular site names. node_map = {} for node in nodes: node_map[str(node[u"se"])] = str(node[str(u"name")]) for d in os.listdir("/cvmfs/cms.cern.ch/SITECONF/"): # Only T0_, T1_... folders are needed if d[0] == "T": os.environ[ 'WMAGENT_SITE_CONFIG_OVERRIDE'] = '/cvmfs/cms.cern.ch/SITECONF/%s/JobConfig/site-local-config.xml' % ( d) try: slc = loadSiteLocalConfig() except SiteConfigError as e: print e.args[0] phedexNode = slc.localStageOut.get("phedex-node") # If slc is correct, perform check if "se-name" in slc.localStageOut and slc.localStageOut[ "se-name"] in node_map and phedexNode != None: self.assertEqual(phedexNode, node_map[slc.localStageOut["se-name"]], \ "Error: Node specified in SLC (%s) doesn't match node returned by PhEDEx api (%s)." \ % (phedexNode, node_map[slc.localStageOut["se-name"]])) return
def createInitialReport(job, task, logLocation): """ _createInitialReport_ Create an initial job report with the base information in it. """ try: siteCfg = loadSiteLocalConfig() except SiteConfigError: # For now, assume that we did this on purpose msg = "Couldn't find SiteConfig" logging.error(msg) #TODO: Make less goatballs for testing purposes return report = Report.Report() report.data.WMAgentJobID = job.get('id', None) report.data.WMAgentJobName = job.get('name', None) report.data.pnn = siteCfg.localStageOut.get('phedex-node', 'Unknown') report.data.siteName = getattr(siteCfg, 'siteName', 'Unknown') report.data.hostName = socket.gethostname() report.data.ceName = getSyncCE() report.data.completed = False report.setTaskName(taskName=job.get('task', 'TaskNotFound')) # Not so fond of this, but we have to put the master # report way up at the top so it's returned if the # job fails early reportPath = os.path.join(os.getcwd(), '../', logLocation) report.save(reportPath) return
def send_initial_dashboard_update(data, config): # Dashboard does not like Unicode, just ASCII encoding syncid = str(config['monitoring']['syncid']) try: if os.environ.get("PARROT_ENABLED", "FALSE") == "TRUE": raise ValueError() sync_ce = loadSiteLocalConfig().siteName except Exception: for envvar in ["GLIDEIN_Gatekeeper", "OSG_HOSTNAME", "CONDORCE_COLLECTOR_HOST"]: if envvar in os.environ: sync_ce = os.environ[envvar] break else: host = socket.getfqdn() sync_ce = config['default host'] if host.rsplit('.')[-2:] == sync_ce.rsplit('.')[-2:]: sync_ce = config['default ce'] else: sync_ce = 'Unknown' logger.info("using sync CE {}".format(sync_ce)) parameters = { 'ExeStart': str(config['executable']), 'NCores': config.get('cores', 1), 'SyncCE': sync_ce, 'SyncGridJobId': syncid, 'WNHostName': socket.getfqdn() } monitor(parameters)
def _handlePileup(self): """ Handle pileup settings. """ # find out local site SE name siteConfig = loadSiteLocalConfig() seLocalName = siteConfig.localStageOut["se-name"] print "Running on site '%s', local SE name: '%s'" % ( siteConfig.siteName, seLocalName) pileupDict = self._getPileupConfigFromJson() # 2011-02-03 according to the most recent version of instructions, we do # want to differentiate between "MixingModule" and "DataMixingModule" mixModules, dataMixModules = self._getPileupMixingModules() # 2011-02-03 # on the contrary to the initial instructions (wave), there are # going to be only two types of pileup input datasets: "data" or "mc" # unlike all previous places where pileupType handled in a flexible # way as specified in the configuration passed by the user, here are # the two pileupTypes hardcoded: and we are going to add the "mc" # datasets to "MixingModule"s and only add the "data" datasets to the # "DataMixingModule"s # if the user in the configuration specifies different pileup types # than "data" or "mc", the following call will not modify anything self._processPileupMixingModules(pileupDict, seLocalName, dataMixModules, "data") self._processPileupMixingModules(pileupDict, seLocalName, mixModules, "mc")
def testSlcPhedexNodesEqualPhedexApiNodes(self): """ For each site, verify that the stageout node specified in site-local-config.xml is the same as the one returned by the PhEDEx api. """ os.environ["CMS_PATH"] = "/cvmfs/cms.cern.ch" phedex = PhEDEx() nodes = phedex.getNodeMap()["phedex"]["node"] # Make a dict for translating the se names into regular site names. node_map = {} for node in nodes: node_map[str(node[u"se"])] = str(node[str(u"name")]) for d in os.listdir("/cvmfs/cms.cern.ch/SITECONF/"): # Only T0_, T1_... folders are needed if d[0] == "T": os.environ['WMAGENT_SITE_CONFIG_OVERRIDE'] ='/cvmfs/cms.cern.ch/SITECONF/%s/JobConfig/site-local-config.xml' % (d) try: slc = loadSiteLocalConfig() except SiteConfigError as e: print e.args[0] phedexNode = slc.localStageOut.get("phedex-node") # If slc is correct, perform check if "se-name" in slc.localStageOut and slc.localStageOut["se-name"] in node_map and phedexNode != None: self.assertEqual(phedexNode, node_map[slc.localStageOut["se-name"]], \ "Error: Node specified in SLC (%s) doesn't match node returned by PhEDEx api (%s)." \ % (phedexNode, node_map[slc.localStageOut["se-name"]])) return
def createErrorReport(exitCode, errorType, errorDetails=None, logLocation="Report.0.pkl"): """ _createErrorReport_ Create a report if something fails inside the Bootstrap This creates a dummy step called 'CRITICAL' and sticks the error in there. """ try: siteCfg = loadSiteLocalConfig() except SiteConfigError: # For now, assume that we did this on purpose msg = "Couldn't find SiteConfig" logging.error(msg) # TODO: Make this not suck goatballs when you are just running tests return report = Report.Report() report.data.pnn = siteCfg.localStageOut.get('phedex-node', 'Unknown') report.data.siteName = getattr(siteCfg, 'siteName', 'Unknown') report.data.hostName = socket.gethostname() report.data.ceName = getSyncCE() report.data.completed = False report.addError(stepName='CRITICAL', exitCode=exitCode, errorType=errorType, errorDetails=errorDetails) reportPath = os.path.join(os.getcwd(), '../', logLocation) report.save(reportPath) return
def __init__(self, **overrideParams): self.override = False self.overrideConf = overrideParams if overrideParams != {}: self.override = True self.fallbacks = [] # // # // Try an get the TFC for the site #// self.tfc = None self.numberOfRetries = 3 self.retryPauseTime = 600 # // # // If override isnt None, we dont need SiteCfg, if it is #// then we need siteCfg otherwise we are dead. if self.override == False: self.siteCfg = loadSiteLocalConfig() if self.override: self.initialiseOverride() else: self.initialiseSiteConf()
def __init__(self, **overrideParams): self.override = False self.logger = overrideParams.pop("logger", logging.getLogger()) self.overrideConf = overrideParams if overrideParams != {}: self.override = True # // # // Try an get the TFC for the site # // self.tfc = None from WMCore.Storage.SiteLocalConfig import loadSiteLocalConfig self.numberOfRetries = 3 self.retryPauseTime = 600 self.pnn = None self.fallbacks = [] # // # // If override isnt None, we dont need SiteCfg, if it is # // then we need siteCfg otherwise we are dead. if self.override == False: self.siteCfg = loadSiteLocalConfig() if self.override: self.initialiseOverride() else: self.initialiseSiteConf()
def testSlcPhedexNodesEqualPhedexApiNodes(self): """ For each site, verify that the stageout node specified in site-local-config.xml is the same as the one returned by the PhEDEx api. """ os.environ["CMS_PATH"] = "/cvmfs/cms.cern.ch" nodes = ['FIXME'] for d in os.listdir("/cvmfs/cms.cern.ch/SITECONF/"): # Only T0_, T1_... folders are needed if d[0] == "T": os.environ[ 'WMAGENT_SITE_CONFIG_OVERRIDE'] = '/cvmfs/cms.cern.ch/SITECONF/%s/JobConfig/site-local-config.xml' % ( d) try: slc = loadSiteLocalConfig() except SiteConfigError as e: print(e.args[0]) phedexNode = slc.localStageOut.get("phedex-node") self.assertTrue( phedexNode in nodes, "Error: Node specified in SLC (%s) not in list returned by PhEDEx api" % phedexNode) return
def _handlePileup(self): """ Handle pileup settings. """ # find out local site SE name siteConfig = loadSiteLocalConfig() seLocalName = siteConfig.localStageOut["se-name"] print "Running on site '%s', local SE name: '%s'" % (siteConfig.siteName, seLocalName) pileupDict = self._getPileupConfigFromJson() # 2011-02-03 according to the most recent version of instructions, we do # want to differentiate between "MixingModule" and "DataMixingModule" mixModules, dataMixModules = self._getPileupMixingModules() # 2011-02-03 # on the contrary to the initial instructions (wave), there are # going to be only two types of pileup input datasets: "data" or "mc" # unlike all previous places where pileupType handled in a flexible # way as specified in the configuration passed by the user, here are # the two pileupTypes hardcoded: and we are going to add the "mc" # datasets to "MixingModule"s and only add the "data" datasets to the # "DataMixingModule"s # if the user in the configuration specifies different pileup types # than "data" or "mc", the following call will not modify anything self._processPileupMixingModules(pileupDict, seLocalName, dataMixModules, "data") self._processPileupMixingModules(pileupDict, seLocalName, mixModules, "mc")
def createInitialReport(job, reportName): """ _createInitialReport_ Create an initial job report with the base information in it. """ try: siteCfg = loadSiteLocalConfig() except SiteConfigError: # For now, assume that we did this on purpose msg = "Couldn't find SiteConfig" logging.error(msg) # TODO: Make less goatballs for testing purposes return report = Report.Report() report.data.WMAgentJobID = job.get('id', None) report.data.WMAgentJobName = job.get('name', None) report.data.pnn = siteCfg.localStageOut.get('phedex-node', 'Unknown') report.data.siteName = getattr(siteCfg, 'siteName', 'Unknown') report.data.hostName = socket.gethostname() report.data.ceName = getSyncCE() # TODO: need to check what format it returns and what features need to extract. # currently # $MACHINEFEATURES/hs06: HS06 score of the host # $MACHINEFEATURES/total_cpu: number of configured job slots # $JOBFEATURES/hs06_job: HS06 score available to your job # $JOBFEATURES/allocated_cpu: number of allocated slots (=8 in case of a multicore job machineFeaturesFile = os.environ.get('MACHINEFEATURES') report.data.machineFeatures = {} if machineFeaturesFile: report.data.machineFeatures['hs06'] = readFloatFromFile( "%s/hs06" % machineFeaturesFile) report.data.machineFeatures['total_cpu'] = readFloatFromFile( "%s/total_cpu" % machineFeaturesFile) jobFeaturesFile = os.environ.get('JOBFEATURES') report.data.jobFeatures = {} if jobFeaturesFile: report.data.jobFeatures['hs06_job'] = readFloatFromFile( "%s/hs06_job" % jobFeaturesFile) report.data.jobFeatures['allocated_cpu'] = readFloatFromFile( "%s/allocated_cpu" % jobFeaturesFile) report.data.completed = False report.setTaskName(taskName=job.get('task', 'TaskNotFound')) # Not so fond of this, but we have to put the master # report way up at the top so it's returned if the # job fails early reportPath = os.path.join(os.getcwd(), '../', reportName) report.save(reportPath) return
def getSEName(self): """ Using SiteLocalConfig, get the SEName """ from WMCore.Storage.SiteLocalConfig import loadSiteLocalConfig self.siteCfg = loadSiteLocalConfig()
def createInitialReport(job, reportName): """ _createInitialReport_ Create an initial job report with the base information in it. """ try: siteCfg = loadSiteLocalConfig() except SiteConfigError: # For now, assume that we did this on purpose msg = "Couldn't find SiteConfig" logging.error(msg) # TODO: Make less goatballs for testing purposes return report = Report.Report() report.data.WMAgentJobID = job.get('id', None) report.data.WMAgentJobName = job.get('name', None) report.data.pnn = siteCfg.localStageOut.get('phedex-node', 'Unknown') report.data.siteName = getattr(siteCfg, 'siteName', 'Unknown') report.data.hostName = socket.gethostname() report.data.ceName = getSyncCE() # TODO: need to check what format it returns and what features need to extract. # currently # $MACHINEFEATURES/hs06: HS06 score of the host # $MACHINEFEATURES/total_cpu: number of configured job slots # $JOBFEATURES/hs06_job: HS06 score available to your job # $JOBFEATURES/allocated_cpu: number of allocated slots (=8 in case of a multicore job machineFeaturesFile = os.environ.get('MACHINEFEATURES') report.data.machineFeatures = {} if machineFeaturesFile: report.data.machineFeatures['hs06'] = readFloatFromFile("%s/hs06" % machineFeaturesFile) report.data.machineFeatures['total_cpu'] = readFloatFromFile("%s/total_cpu" % machineFeaturesFile) jobFeaturesFile = os.environ.get('JOBFEATURES') report.data.jobFeatures = {} if jobFeaturesFile: report.data.jobFeatures['hs06_job'] = readFloatFromFile("%s/hs06_job" % jobFeaturesFile) report.data.jobFeatures['allocated_cpu'] = readFloatFromFile("%s/allocated_cpu" % jobFeaturesFile) report.data.completed = False report.setTaskName(taskName=job.get('task', 'TaskNotFound')) # Not so fond of this, but we have to put the master # report way up at the top so it's returned if the # job fails early reportPath = os.path.join(os.getcwd(), '../', reportName) report.save(reportPath) return
def testLoadingConfigFromOverridenEnvVarriable(self): """ test SiteLocalConfig module method loadSiteLocalConfig when loading site config from location defined by WMAGENT_SITE_CONFIG_OVERRIDE env. variable """ vandyConfigFileName = os.path.join( getTestBase(), "WMCore_t/Storage_t", "T3_US_Vanderbilt_SiteLocalConfig.xml") os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = vandyConfigFileName mySiteConfig = loadSiteLocalConfig() self.assertEqual(mySiteConfig.siteName, "T3_US_Vanderbilt", "Error: Wrong site name.")
def testLoadingConfigFromOverridenEnvVarriable(self): """ test SiteLocalConfig module method loadSiteLocalConfig when loading site config from location defined by WMAGENT_SITE_CONFIG_OVERRIDE env. variable """ vandyConfigFileName = os.path.join(getTestBase(), "WMCore_t/Storage_t", "T3_US_Vanderbilt_SiteLocalConfig.xml") os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = vandyConfigFileName mySiteConfig = loadSiteLocalConfig() self.assertEqual(mySiteConfig.siteName, "T3_US_Vanderbilt", "Error: Wrong site name.")
def __init__(self, interval=300, username=None, commonname=None): self.interval = interval self.__previous = 0 self.__states = {} self.username = username if username else self.__get_user() self.commonname = commonname if commonname else self.__get_distinguished_name().rsplit('/CN=', 1)[1] self.__cmssw_version = 'Unknown' self.__executable = 'Unknown' self.__dash = None try: self._ce = loadSiteLocalConfig().siteName except SiteConfigError: logger.error("can't load siteconfig, defaulting to hostname") self._ce = socket.getfqdn()
def __init__(self, interval=300, username=None, commonname=None): self.interval = interval self.__previous = 0 self.__states = {} self.username = username if username else self.__get_user() self.commonname = commonname if commonname else self.__get_distinguished_name( ).rsplit('/CN=', 1)[1] self.__cmssw_version = 'Unknown' self.__executable = 'Unknown' try: self._ce = loadSiteLocalConfig().siteName except SiteConfigError: logger.error("can't load siteconfig, defaulting to hostname") self._ce = socket.getfqdn()
def __init__(self, **overrideParams): logging.info("StageOutMgr::__init__()") self.overrideConf = overrideParams # Figure out if any of the override parameters apply to stage-out self.override = False if overrideParams != {}: logging.info("StageOutMgr::__init__(): Override: %s", overrideParams) checkParams = ["command", "option", "phedex-node", "lfn-prefix"] for param in checkParams: if param in self.overrideConf.keys(): self.override = True if not self.override: logging.info( "=======StageOut Override: These are not the parameters you are looking for" ) self.substituteGUID = True self.fallbacks = [] # // # // Try an get the TFC for the site # // self.tfc = None self.numberOfRetries = 3 self.retryPauseTime = 600 from WMCore.Storage.SiteLocalConfig import loadSiteLocalConfig # // # // If override isnt None, we dont need SiteCfg, if it is # // then we need siteCfg otherwise we are dead. if self.override == False: self.siteCfg = loadSiteLocalConfig() if self.override: self.initialiseOverride() else: self.initialiseSiteConf() self.failed = {} self.completedFiles = {} return
def __init__(self, **overrideParams): print "StageOutMgr::__init__()" self.overrideConf = overrideParams # Figure out if any of the override parameters apply to stage-out self.override = False if overrideParams != {}: print "StageOutMgr::__init__(): Override: %s" % overrideParams checkParams = ["command", "option", "se-name", "lfn-prefix"] for param in checkParams: if param in self.overrideConf.keys(): self.override = True if not self.override: print "=======StageOut Override: These are not the parameters you are looking for" self.substituteGUID = True self.fallbacks = [] # // # // Try an get the TFC for the site #// self.tfc = None self.numberOfRetries = 3 self.retryPauseTime = 600 from WMCore.Storage.SiteLocalConfig import loadSiteLocalConfig # // # // If override isnt None, we dont need SiteCfg, if it is #// then we need siteCfg otherwise we are dead. if self.override == False: self.siteCfg = loadSiteLocalConfig() if self.override: self.initialiseOverride() else: self.initialiseSiteConf() self.failed = {} self.completedFiles = {} return
def loadSiteConf(self): """ _loadSiteConf_ Read the site conf file """ if not os.environ.has_key("CMS_PATH"): msg = "CMS_PATH Not Set: Cannot find SiteConf" self.summary['SiteConf'] = "Failed: CMS_PATH not set" raise RuntimeError, msg try: self.siteConf = loadSiteLocalConfig() except Exception, ex: msg = "Error loading Site Conf File: %s" % str(ex) self.summary['SiteConf'] = "Failed: Cannot load SiteConf" raise RuntimeError, msg
def getSyncCE(default=socket.gethostname()): """ _getSyncCE_ Tries to get the site name from the localSite config, if it doesn't find it or it finds an empty string then we check the environment variables. Worst case scenario we give the Worker node. """ try: siteConfig = loadSiteLocalConfig() result = siteConfig.siteName if result: return result except SiteConfigError: logging.error( "Couldn't find the site config, looking for the CE elsewhere") result = socket.gethostname() if 'GLOBUS_GRAM_JOB_CONTACT' in os.environ: # // # // OSG, Sync CE from Globus ID #// val = os.environ['GLOBUS_GRAM_JOB_CONTACT'] try: host = val.split("https://", 1)[1] host = host.split(":")[0] result = host except: pass return result if 'NORDUGRID_CE' in os.environ: # // # // ARC, Sync CE from env. var. submitted with the job by JobSubmitter #// return os.environ['NORDUGRID_CE'] return result
def __init__(self, numberOfRetries = 3, retryPauseTime=15, **overrideParams): # set defaults self.failed = {} self.completedFiles = {} self.override = False self.overrideConf = overrideParams self.substituteGUID = True self.defaultMethod = {} self.fallbacks = [] self.tfc = None self.numberOfRetries = numberOfRetries self.retryPauseTime = retryPauseTime if overrideParams != {}: log.critical("Override: %s" % overrideParams) self.override = True self.initialiseOverride() else: self.siteCfg = loadSiteLocalConfig() self.initialiseSiteConf()
def __init__(self, numberOfRetries=3, retryPauseTime=15, **overrideParams): # set defaults self.failed = {} self.completedFiles = {} self.override = False self.overrideConf = overrideParams self.substituteGUID = True self.defaultMethod = {} self.fallbacks = [] self.tfc = None self.numberOfRetries = numberOfRetries self.retryPauseTime = retryPauseTime if overrideParams != {}: log.critical("Override: %s" % overrideParams) self.override = True self.initialiseOverride() else: self.siteCfg = loadSiteLocalConfig() self.initialiseSiteConf()
def getSyncCE(default = socket.gethostname()): """ _getSyncCE_ Tries to get the site name from the localSite config, if it doesn't find it or it finds an empty string then we check the environment variables. Worst case scenario we give the Worker node. """ try: siteConfig = loadSiteLocalConfig() result = siteConfig.siteName if result: return result except SiteConfigError: logging.error("Couldn't find the site config, looking for the CE elsewhere") result = socket.gethostname() if 'GLOBUS_GRAM_JOB_CONTACT' in os.environ: # // # // OSG, Sync CE from Globus ID #// val = os.environ['GLOBUS_GRAM_JOB_CONTACT'] try: host = val.split("https://", 1)[1] host = host.split(":")[0] result = host except: pass return result if 'NORDUGRID_CE' in os.environ: # // # // ARC, Sync CE from env. var. submitted with the job by JobSubmitter #// return os.environ['NORDUGRID_CE'] return result
def testSlcPhedexNodesEqualPhedexApiNodes(self): """ For each site, verify that the stageout node specified in site-local-config.xml is the same as the one returned by the PhEDEx api. """ os.environ["CMS_PATH"] = "/cvmfs/cms.cern.ch" phedex = PhEDEx() nodes = [node[u'name'] for node in phedex.getNodeMap()["phedex"]["node"]] for d in os.listdir("/cvmfs/cms.cern.ch/SITECONF/"): # Only T0_, T1_... folders are needed if d[0] == "T": os.environ['WMAGENT_SITE_CONFIG_OVERRIDE'] ='/cvmfs/cms.cern.ch/SITECONF/%s/JobConfig/site-local-config.xml' % (d) try: slc = loadSiteLocalConfig() except SiteConfigError as e: print(e.args[0]) phedexNode = slc.localStageOut.get("phedex-node") self.assertTrue(phedexNode in nodes, "Error: Node specified in SLC (%s) not in list returned by PhEDEx api" % phedexNode) return
def createErrorReport(exitCode, errorType, errorDetails=None, logLocation="Report.0.pkl"): """ _createErrorReport_ Create a report if something fails inside the Bootstrap This creates a dummy step called 'CRITICAL' and sticks the error in there. """ try: siteCfg = loadSiteLocalConfig() except SiteConfigError: # For now, assume that we did this on purpose msg = "Couldn't find SiteConfig" logging.error(msg) #TODO: Make this not suck goatballs when you are just running tests return report = Report.Report() report.data.seName = siteCfg.localStageOut.get('se-name', socket.gethostname()) report.data.pnn = siteCfg.localStageOut.get('phedex-node', 'Unknown') report.data.siteName = getattr(siteCfg, 'siteName', 'Unknown') report.data.hostName = socket.gethostname() report.data.ceName = getSyncCE() report.data.completed = False report.addError(stepName='CRITICAL', exitCode=exitCode, errorType=errorType, errorDetails=errorDetails) reportPath = os.path.join(os.getcwd(), '../', logLocation) report.save(reportPath) return
def __init__(self, config): util.Timing.__init__(self, 'dash', 'handler', 'updates', 'elk', 'transfers', 'cleanup', 'propagate', 'sqlite') self.config = config self.basedirs = [config.base_directory, config.startup_directory] self.workdir = config.workdir self._storage = config.storage self.statusfile = os.path.join(self.workdir, 'status.json') self.siteconf = os.path.join(self.workdir, 'siteconf') self.parrot_path = os.path.dirname(util.which('parrot_run')) self.parrot_bin = os.path.join(self.workdir, 'bin') self.parrot_lib = os.path.join(self.workdir, 'lib') self.__algo = Algo(config) self.__host = socket.getfqdn() try: siteconf = loadSiteLocalConfig() self.__ce = siteconf.siteName self.__se = siteconf.localStageOutPNN() self.__frontier_proxy = siteconf.frontierProxies[0] except (SiteConfigError, IndexError): logger.error("can't load siteconfig, defaulting to hostname") self.__ce = socket.getfqdn() self.__se = socket.getfqdn() try: self.__frontier_proxy = os.environ['HTTP_PROXY'] except KeyError: logger.error("can't determine proxy for Frontier via $HTTP_PROXY") sys.exit(1) try: with open('/etc/cvmfs/default.local') as f: lines = f.readlines() except IOError: lines = [] for l in lines: m = re.match('\s*CVMFS_HTTP_PROXY\s*=\s*[\'"]?(.*)[\'"]?', l) if m: self.__cvmfs_proxy = m.group(1).strip("\"'") break else: try: self.__cvmfs_proxy = os.environ['HTTP_PROXY'] except KeyError: logger.error("can't determine proxy for CVMFS via $HTTP_PROXY") sys.exit(1) logger.debug("using {} as proxy for CVMFS".format(self.__cvmfs_proxy)) logger.debug("using {} as proxy for Frontier".format(self.__frontier_proxy)) logger.debug("using {} as osg_version".format(self.config.advanced.osg_version)) util.sendemail("Your Lobster project has started!", self.config) self.__taskhandlers = {} self.__store = unit.UnitStore(self.config) self.__setup_inputs() self.copy_siteconf() create = not util.checkpoint(self.workdir, 'id') if create: self.taskid = 'lobster_{0}_{1}'.format( self.config.label, sha1(str(datetime.datetime.utcnow())).hexdigest()[-16:]) util.register_checkpoint(self.workdir, 'id', self.taskid) shutil.copy(self.config.base_configuration, os.path.join(self.workdir, 'config.py')) else: self.taskid = util.checkpoint(self.workdir, 'id') util.register_checkpoint(self.workdir, 'RESTARTED', str(datetime.datetime.utcnow())) if not util.checkpoint(self.workdir, 'executable'): # We can actually have more than one exe name (one per task label) # Set 'cmsRun' if any of the tasks are of that type, # or use cmd command if all tasks execute the same cmd, # or use 'noncmsRun' if task cmds are different # Using this for dashboard exe name reporting cmsconfigs = [wflow.pset for wflow in self.config.workflows] cmds = [wflow.command for wflow in self.config.workflows] if any(cmsconfigs): exename = 'cmsRun' elif all(x == cmds[0] and x is not None for x in cmds): exename = cmds[0] else: exename = 'noncmsRun' util.register_checkpoint(self.workdir, 'executable', exename) for wflow in self.config.workflows: if create and not util.checkpoint(self.workdir, wflow.label): wflow.setup(self.workdir, self.basedirs) logger.info("querying backend for {0}".format(wflow.label)) with fs.alternative(): dataset_info = wflow.dataset.get_info() logger.info("registering {0} in database".format(wflow.label)) self.__store.register_dataset(wflow, dataset_info, wflow.category.runtime) util.register_checkpoint(self.workdir, wflow.label, 'REGISTERED') elif os.path.exists(os.path.join(wflow.workdir, 'running')): for id in self.get_taskids(wflow.label): util.move(wflow.workdir, id, 'failed') for wflow in self.config.workflows: if wflow.parent: getattr(self.config.workflows, wflow.parent.label).register(wflow) if create: total_units = wflow.dataset.total_units * len(wflow.unique_arguments) self.__store.register_dependency(wflow.label, wflow.parent.label, total_units) if not util.checkpoint(self.workdir, 'sandbox cmssw version'): util.register_checkpoint(self.workdir, 'sandbox', 'CREATED') versions = set([w.version for w in self.config.workflows]) if len(versions) == 1: util.register_checkpoint(self.workdir, 'sandbox cmssw version', list(versions)[0]) if self.config.elk: if create: categories = {wflow.category.name: [] for wflow in self.config.workflows} for category in categories: for workflow in self.config.workflows: if workflow.category.name == category: categories[category].append(workflow.label) self.config.elk.create(categories) else: self.config.elk.resume() self.config.advanced.dashboard.setup(self.config) if create: self.config.save() self.config.advanced.dashboard.register_run() else: self.config.advanced.dashboard.update_task_status( (id_, dash.ABORTED) for id_ in self.__store.reset_units() ) for p in (self.parrot_bin, self.parrot_lib): if not os.path.exists(p): os.makedirs(p) for exe in ('parrot_run', 'chirp', 'chirp_put', 'chirp_get'): shutil.copy(util.which(exe), self.parrot_bin) subprocess.check_call(["strip", os.path.join(self.parrot_bin, exe)]) p_helper = os.path.join(os.path.dirname(self.parrot_path), 'lib', 'lib64', 'libparrot_helper.so') shutil.copy(p_helper, self.parrot_lib)
def handlePileup(self): """ _handlePileup_ Handle pileup settings. There has been stored pileup configuration stored in a JSON file as a result of DBS querrying when running PileupFetcher, this method loads this configuration from sandbox and returns it as dictionary. The PileupFetcher was called by WorkQueue which creates job's sandbox and sandbox gets migrated to the worker node. External script iterates over all modules and over all pileup configuration types. The only considered types are "data" and "mc" (input to this method). If other pileup types are specified by the user, the method doesn't modify anything. The method considers only files which are present on this local PNN. The job will use only those, unless it was told to trust the PU site location (trustPUSitelists=True), in this case ALL the blocks/files will be added to the PSet and files will be read via AAA. Dataset, divided into blocks, may not have all blocks present on a particular PNN. However, all files belonging into a block will be present when reported by DBS. The structure of the pileupDict: PileupFetcher._queryDbsAndGetPileupConfig """ # find out local site SE name siteConfig = loadSiteLocalConfig() PhEDExNodeName = siteConfig.localStageOut["phedex-node"] self.logger.info("Running on site '%s', local PNN: '%s'", siteConfig.siteName, PhEDExNodeName) jsonPileupConfig = os.path.join(self.stepSpace.location, "pileupconf.json") # Load pileup json try: with open(jsonPileupConfig) as jdata: pileupDict = json.load(jdata) except IOError: m = "Could not read pileup JSON configuration file: '%s'" % jsonPileupConfig raise RuntimeError(m) # Create a json with a list of files and events available # after dealing with PhEDEx/AAA logic newPileupDict = {} fileList = [] eventsAvailable = 0 for pileupType in self.step.data.pileup.listSections_(): useAAA = True if getattr(self.jobBag, 'trustPUSitelists', False) else False self.logger.info("Pileup set to read data remotely: %s", useAAA) for blockName in sorted(pileupDict[pileupType].keys()): blockDict = pileupDict[pileupType][blockName] if PhEDExNodeName in blockDict["PhEDExNodeNames"] or useAAA: eventsAvailable += int(blockDict.get('NumberOfEvents', 0)) for fileLFN in blockDict["FileList"]: fileList.append(str(fileLFN)) newPileupDict[pileupType] = { "eventsAvailable": eventsAvailable, "FileList": fileList } newJsonPileupConfig = os.path.join(self.stepSpace.location, "CMSSWPileupConfig.json") self.logger.info("Generating json for CMSSW pileup script") try: with open(newJsonPileupConfig, 'wb') as f: json.dump(newPileupDict, f) except Exception as ex: self.logger.exception("Error writing out process filelist json:") raise ex procScript = "cmssw_handle_pileup.py" cmd = "%s --input_pkl %s --output_pkl %s --pileup_dict %s" % ( procScript, os.path.join(self.stepSpace.location, self.configPickle), os.path.join(self.stepSpace.location, self.configPickle), newJsonPileupConfig) if getattr(self.jobBag, "skipPileupEvents", None): randomSeed = self.job['task'] skipPileupEvents = self.jobBag.skipPileupEvents cmd += " --skip_pileup_events %s --random_seed %s" % ( skipPileupEvents, randomSeed) self.scramRun(cmd) return
# modifications of job input files pileupConfig = {"data": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"], "mc": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"]} dbsUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" setupScript.step.setupPileup(pileupConfig, dbsUrl) # SetupCMSSWPset pileup handling will be consulting SiteLocalConfig # to determine StorageElement (SE) name the job is running on # SiteLocalConfig loads the site-local-config.xml file from env. # variable defined location ; if the variable is not defined already, set it # obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above # thick with SE name is not effective if not os.getenv("WMAGENT_SITE_CONFIG_OVERRIDE", None): os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = siteLocalConfig # find out local site name from the testing local site config, # will be needed later siteConfig = loadSiteLocalConfig() seLocalName = siteConfig.localStageOut["se-name"] print "Running on site '%s', local SE name: '%s'" % (siteConfig.siteName, seLocalName) # before calling the script, SetupCMSSWPset will try to load JSON # pileup configuration file, need to create it in self.testDir fetcher = PileupFetcher() fetcher.setWorkingDirectory(self.testDir) fetcher._createPileupConfigFile(setupScript.step) setupScript() # now test all modifications carried out in SetupCMSSWPset.__call__ # which will also test that CMSSWStepHelper.setupPileup run correctly mixModules, dataMixModules = setupScript._getPileupMixingModules()
def __init__(self, config): util.Timing.__init__(self, 'dash', 'handler', 'updates', 'elk', 'transfers', 'cleanup', 'propagate', 'sqlite') self.config = config self.basedirs = [config.base_directory, config.startup_directory] self.workdir = config.workdir self._storage = config.storage self.statusfile = os.path.join(self.workdir, 'status.json') self.siteconf = os.path.join(self.workdir, 'siteconf') self.parrot_path = os.path.dirname(util.which('parrot_run')) self.parrot_bin = os.path.join(self.workdir, 'bin') self.parrot_lib = os.path.join(self.workdir, 'lib') self.__algo = Algo(config) self.__host = socket.getfqdn() try: siteconf = loadSiteLocalConfig() self.__ce = siteconf.siteName self.__se = siteconf.localStageOutPNN() self.__frontier_proxy = siteconf.frontierProxies[0] except (SiteConfigError, IndexError): logger.error("can't load siteconfig, defaulting to hostname") self.__ce = socket.getfqdn() self.__se = socket.getfqdn() try: self.__frontier_proxy = os.environ['HTTP_PROXY'] except KeyError: logger.error( "can't determine proxy for Frontier via $HTTP_PROXY") sys.exit(1) try: with open('/etc/cvmfs/default.local') as f: lines = f.readlines() except IOError: lines = [] for l in lines: m = re.match('\s*CVMFS_HTTP_PROXY\s*=\s*[\'"]?(.*)[\'"]?', l) if m: self.__cvmfs_proxy = m.group(1).strip("\"'") break else: try: self.__cvmfs_proxy = os.environ['HTTP_PROXY'] except KeyError: logger.error("can't determine proxy for CVMFS via $HTTP_PROXY") sys.exit(1) logger.debug("using {} as proxy for CVMFS".format(self.__cvmfs_proxy)) logger.debug("using {} as proxy for Frontier".format( self.__frontier_proxy)) logger.debug("using {} as osg_version".format( self.config.advanced.osg_version)) util.sendemail("Your Lobster project has started!", self.config) self.__taskhandlers = {} self.__store = unit.UnitStore(self.config) self.__setup_inputs() self.copy_siteconf() create = not util.checkpoint(self.workdir, 'id') if create: self.taskid = 'lobster_{0}_{1}'.format( self.config.label, sha1(str(datetime.datetime.utcnow())).hexdigest()[-16:]) util.register_checkpoint(self.workdir, 'id', self.taskid) shutil.copy(self.config.base_configuration, os.path.join(self.workdir, 'config.py')) else: self.taskid = util.checkpoint(self.workdir, 'id') util.register_checkpoint(self.workdir, 'RESTARTED', str(datetime.datetime.utcnow())) if not util.checkpoint(self.workdir, 'executable'): # We can actually have more than one exe name (one per task label) # Set 'cmsRun' if any of the tasks are of that type, # or use cmd command if all tasks execute the same cmd, # or use 'noncmsRun' if task cmds are different # Using this for dashboard exe name reporting cmsconfigs = [wflow.pset for wflow in self.config.workflows] cmds = [wflow.command for wflow in self.config.workflows] if any(cmsconfigs): exename = 'cmsRun' elif all(x == cmds[0] and x is not None for x in cmds): exename = cmds[0] else: exename = 'noncmsRun' util.register_checkpoint(self.workdir, 'executable', exename) for wflow in self.config.workflows: if create and not util.checkpoint(self.workdir, wflow.label): wflow.setup(self.workdir, self.basedirs) logger.info("querying backend for {0}".format(wflow.label)) with fs.alternative(): dataset_info = wflow.dataset.get_info() logger.info("registering {0} in database".format(wflow.label)) self.__store.register_dataset(wflow, dataset_info, wflow.category.runtime) util.register_checkpoint(self.workdir, wflow.label, 'REGISTERED') elif os.path.exists(os.path.join(wflow.workdir, 'running')): for id in self.get_taskids(wflow.label): util.move(wflow.workdir, id, 'failed') for wflow in self.config.workflows: if wflow.parent: getattr(self.config.workflows, wflow.parent.label).register(wflow) if create: total_units = wflow.dataset.total_units * len( wflow.unique_arguments) self.__store.register_dependency(wflow.label, wflow.parent.label, total_units) if not util.checkpoint(self.workdir, 'sandbox cmssw version'): util.register_checkpoint(self.workdir, 'sandbox', 'CREATED') versions = set([w.version for w in self.config.workflows]) if len(versions) == 1: util.register_checkpoint(self.workdir, 'sandbox cmssw version', list(versions)[0]) if self.config.elk: if create: categories = { wflow.category.name: [] for wflow in self.config.workflows } for category in categories: for workflow in self.config.workflows: if workflow.category.name == category: categories[category].append(workflow.label) self.config.elk.create(categories) else: self.config.elk.resume() self.config.advanced.dashboard.setup(self.config) if create: self.config.save() self.config.advanced.dashboard.register_run() else: self.config.advanced.dashboard.update_task_status( (id_, dash.ABORTED) for id_ in self.__store.reset_units()) for p in (self.parrot_bin, self.parrot_lib): if not os.path.exists(p): os.makedirs(p) for exe in ('parrot_run', 'chirp', 'chirp_put', 'chirp_get'): shutil.copy(util.which(exe), self.parrot_bin) subprocess.check_call( ["strip", os.path.join(self.parrot_bin, exe)]) p_helper = os.path.join(os.path.dirname(self.parrot_path), 'lib', 'lib64', 'libparrot_helper.so') shutil.copy(p_helper, self.parrot_lib)
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO" ] } dbsUrl = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet" setupScript.step.setupPileup(pileupConfig, dbsUrl) # SetupCMSSWPset pileup handling will be consulting SiteLocalConfig # to determine StorageElement (SE) name the job is running on # SiteLocalConfig loads the site-local-config.xml file from env. # variable defined location ; if the variable is not defined already, set it # obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above # thick with SE name is not effective if not os.getenv("WMAGENT_SITE_CONFIG_OVERRIDE", None): os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = siteLocalConfig # find out local site name from the testing local site config, # will be needed later siteConfig = loadSiteLocalConfig() seLocalName = siteConfig.localStageOut["se-name"] print "Running on site '%s', local SE name: '%s'" % ( siteConfig.siteName, seLocalName) # before calling the script, SetupCMSSWPset will try to load JSON # pileup configuration file, need to create it in self.testDir fetcher = PileupFetcher() fetcher.setWorkingDirectory(self.testDir) fetcher._createPileupConfigFile(setupScript.step) setupScript() # now test all modifications carried out in SetupCMSSWPset.__call__ # which will also test that CMSSWStepHelper.setupPileup run correctly mixModules, dataMixModules = setupScript._getPileupMixingModules()
def testPileupSetup(self): """ Test the pileup setting. reference (setupScript.process instance): in test/python/WMCore_t/WMRuntime_t/Scripts_t/WMTaskSpace/cmsRun1/PSet.py """ try: from dbs.apis.dbsClient import DbsApi except ImportError as ex: raise unittest.SkipTest # this is modified and shortened version of # WMCore/test/python/WMCore_t/Misc_t/site-local-config.xml # since the dataset name in question (below) is only present at # storm-fe-cms.cr.cnaf.infn.it, need to make the test think it's its local SE siteLocalConfigContent = \ """ <site-local-config> <site name="-SOME-SITE-NAME-"> <event-data> <catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/> </event-data> <local-stage-out> <!-- original cmssrm.fnal.gov --> <phedex-node value="T2_CH_CERN"/> <command value="test-copy"/> <catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/> </local-stage-out> <calib-data> <frontier-connect> <load balance="proxies"/> <proxy url="http://cmsfrontier1.fnal.gov:3128"/> <proxy url="http://cmsfrontier2.fnal.gov:3128"/> </frontier-connect> </calib-data> </site> </site-local-config> """ siteLocalConfig = os.path.join(self.testDir, "test-site-local-config.xml") f = open(siteLocalConfig, 'w') f.write(siteLocalConfigContent) f.close() from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset setupScript = SetupCMSSWPset() setupScript.step = self.createTestStep() setupScript.stepSpace = ConfigSection(name="stepSpace") setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1") setupScript.job = self.createTestJob() # define pileup configuration # despite of the implementation considering whichever type of pileup, # only "data" and "mc" types are eventually considered and lead to any # modifications of job input files pileupConfig = { "data": [ "/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO" ], "mc": [ "/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO" ] } dbsUrl = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader" setupScript.step.setupPileup(pileupConfig, dbsUrl) # SetupCMSSWPset pileup handling will be consulting SiteLocalConfig # to determine StorageElement (SE) name the job is running on # SiteLocalConfig loads the site-local-config.xml file from env. # variable defined location ; if the variable is not defined already, set it # obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above # thick with SE name is not effective if not os.getenv("WMAGENT_SITE_CONFIG_OVERRIDE", None): os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = siteLocalConfig # find out local site name from the testing local site config, # will be needed later siteConfig = loadSiteLocalConfig() seLocalName = siteConfig.localStageOut["phedex-node"] print("Running on site '%s', local SE name: '%s'" % (siteConfig.siteName, seLocalName)) # before calling the script, SetupCMSSWPset will try to load JSON # pileup configuration file, need to create it in self.testDir fetcher = PileupFetcher() fetcher.setWorkingDirectory(self.testDir) fetcher.createPileupConfigFile(setupScript.step) setupScript() # now test all modifications carried out in SetupCMSSWPset.__call__ # which will also test that CMSSWStepHelper.setupPileup run correctly mixModules, dataMixModules = setupScript._getPileupMixingModules() # load in the pileup configuration in the form of dict which # PileupFetcher previously saved in a JSON file pileupDict = setupScript._getPileupConfigFromJson() # get the sub dict for particular pileup type # for pileupDict structure description - see PileupFetcher._queryDbsAndGetPileupConfig for pileupType, modules in zip(("data", "mc"), (dataMixModules, mixModules)): # getting KeyError here - above pileupConfig is not correct - need # to have these two types of pile type d = pileupDict[pileupType] self._mixingModulesInputFilesTest(modules, d, seLocalName)
def testPileupSetup(self): """ Test the pileup setting. reference (setupScript.process instance): in test/python/WMCore_t/WMRuntime_t/Scripts_t/WMTaskSpace/cmsRun1/PSet.py """ try: from dbs.apis.dbsClient import DbsApi except ImportError as ex: raise nose.SkipTest # this is modified and shortened version of # WMCore/test/python/WMCore_t/Misc_t/site-local-config.xml # since the dataset name in question (below) is only present at # storm-fe-cms.cr.cnaf.infn.it, need to make the test think it's its local SE siteLocalConfigContent = \ """ <site-local-config> <site name="-SOME-SITE-NAME-"> <event-data> <catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/> </event-data> <local-stage-out> <!-- original cmssrm.fnal.gov --> <phedex-node value="T2_CH_CERN"/> <command value="test-copy"/> <catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/> </local-stage-out> <calib-data> <frontier-connect> <load balance="proxies"/> <proxy url="http://cmsfrontier1.fnal.gov:3128"/> <proxy url="http://cmsfrontier2.fnal.gov:3128"/> </frontier-connect> </calib-data> </site> </site-local-config> """ siteLocalConfig = os.path.join(self.testDir, "test-site-local-config.xml") f = open(siteLocalConfig, 'w') f.write(siteLocalConfigContent) f.close() from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset setupScript = SetupCMSSWPset() setupScript.step = self.createTestStep() setupScript.stepSpace = ConfigSection(name = "stepSpace") setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1") setupScript.job = self.createTestJob() # define pileup configuration # despite of the implementation considering whichever type of pileup, # only "data" and "mc" types are eventually considered and lead to any # modifications of job input files pileupConfig = {"data": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"], "mc": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"]} dbsUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" setupScript.step.setupPileup(pileupConfig, dbsUrl) # SetupCMSSWPset pileup handling will be consulting SiteLocalConfig # to determine StorageElement (SE) name the job is running on # SiteLocalConfig loads the site-local-config.xml file from env. # variable defined location ; if the variable is not defined already, set it # obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above # thick with SE name is not effective if not os.getenv("WMAGENT_SITE_CONFIG_OVERRIDE", None): os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = siteLocalConfig # find out local site name from the testing local site config, # will be needed later siteConfig = loadSiteLocalConfig() seLocalName = siteConfig.localStageOut["phedex-node"] print("Running on site '%s', local SE name: '%s'" % (siteConfig.siteName, seLocalName)) # before calling the script, SetupCMSSWPset will try to load JSON # pileup configuration file, need to create it in self.testDir fetcher = PileupFetcher() fetcher.setWorkingDirectory(self.testDir) fetcher._createPileupConfigFile(setupScript.step, fakeSites=['T1_US_FNAL']) setupScript() # now test all modifications carried out in SetupCMSSWPset.__call__ # which will also test that CMSSWStepHelper.setupPileup run correctly mixModules, dataMixModules = setupScript._getPileupMixingModules() # load in the pileup configuration in the form of dict which # PileupFetcher previously saved in a JSON file pileupDict = setupScript._getPileupConfigFromJson() # get the sub dict for particular pileup type # for pileupDict structure description - see PileupFetcher._queryDbsAndGetPileupConfig for pileupType, modules in zip(("data", "mc"), (dataMixModules, mixModules)): # getting KeyError here - above pileupConfig is not correct - need # to have these two types of pile type d = pileupDict[pileupType] self._mixingModulesInputFilesTest(modules, d, seLocalName)