def createInitialReport(job, logLocation):
"""
_createInitialReport_
Create an initial job report with the base
information in it.
"""
try:
siteCfg = loadSiteLocalConfig()
except SiteConfigError:
# For now, assume that we did this on purpose
msg = "Couldn't find SiteConfig"
logging.error(msg)
# TODO: Make less goatballs for testing purposes
return
report = Report.Report()
report.data.WMAgentJobID = job.get('id', None)
report.data.WMAgentJobName = job.get('name', None)
report.data.pnn = siteCfg.localStageOut.get('phedex-node', 'Unknown')
report.data.siteName = getattr(siteCfg, 'siteName', 'Unknown')
report.data.hostName = socket.gethostname()
report.data.ceName = getSyncCE()
report.data.completed = False
report.setTaskName(taskName=job.get('task', 'TaskNotFound'))
# Not so fond of this, but we have to put the master
# report way up at the top so it's returned if the
# job fails early
reportPath = os.path.join(os.getcwd(), '../', logLocation)
report.save(reportPath)
return
def testSlcPhedexNodesEqualPhedexApiNodes(self):
"""
For each site, verify that the stageout node specified in
site-local-config.xml is the same as the one returned by the PhEDEx api.
"""
os.environ["CMS_PATH"] = "/cvmfs/cms.cern.ch"
phedex = PhEDEx()
nodes = phedex.getNodeMap()["phedex"]["node"]
# Make a dict for translating the se names into regular site names.
node_map = {}
for node in nodes:
node_map[str(node[u"se"])] = str(node[str(u"name")])
for d in os.listdir("/cvmfs/cms.cern.ch/SITECONF/"):
# Only T0_, T1_... folders are needed
if d[0] == "T":
os.environ[
'WMAGENT_SITE_CONFIG_OVERRIDE'] = '/cvmfs/cms.cern.ch/SITECONF/%s/JobConfig/site-local-config.xml' % (
d)
try:
slc = loadSiteLocalConfig()
except SiteConfigError as e:
print e.args[0]
phedexNode = slc.localStageOut.get("phedex-node")
# If slc is correct, perform check
if "se-name" in slc.localStageOut and slc.localStageOut[
"se-name"] in node_map and phedexNode != None:
self.assertEqual(phedexNode, node_map[slc.localStageOut["se-name"]], \
"Error: Node specified in SLC (%s) doesn't match node returned by PhEDEx api (%s)." \
% (phedexNode, node_map[slc.localStageOut["se-name"]]))
return
def createInitialReport(job, task, logLocation):
"""
_createInitialReport_
Create an initial job report with the base
information in it.
"""
try:
siteCfg = loadSiteLocalConfig()
except SiteConfigError:
# For now, assume that we did this on purpose
msg = "Couldn't find SiteConfig"
logging.error(msg)
#TODO: Make less goatballs for testing purposes
return
report = Report.Report()
report.data.WMAgentJobID = job.get('id', None)
report.data.WMAgentJobName = job.get('name', None)
report.data.pnn = siteCfg.localStageOut.get('phedex-node', 'Unknown')
report.data.siteName = getattr(siteCfg, 'siteName', 'Unknown')
report.data.hostName = socket.gethostname()
report.data.ceName = getSyncCE()
report.data.completed = False
report.setTaskName(taskName=job.get('task', 'TaskNotFound'))
# Not so fond of this, but we have to put the master
# report way up at the top so it's returned if the
# job fails early
reportPath = os.path.join(os.getcwd(), '../', logLocation)
report.save(reportPath)
return
def send_initial_dashboard_update(data, config):
# Dashboard does not like Unicode, just ASCII encoding
syncid = str(config['monitoring']['syncid'])
try:
if os.environ.get("PARROT_ENABLED", "FALSE") == "TRUE":
raise ValueError()
sync_ce = loadSiteLocalConfig().siteName
except Exception:
for envvar in ["GLIDEIN_Gatekeeper", "OSG_HOSTNAME", "CONDORCE_COLLECTOR_HOST"]:
if envvar in os.environ:
sync_ce = os.environ[envvar]
break
else:
host = socket.getfqdn()
sync_ce = config['default host']
if host.rsplit('.')[-2:] == sync_ce.rsplit('.')[-2:]:
sync_ce = config['default ce']
else:
sync_ce = 'Unknown'
logger.info("using sync CE {}".format(sync_ce))
parameters = {
'ExeStart': str(config['executable']),
'NCores': config.get('cores', 1),
'SyncCE': sync_ce,
'SyncGridJobId': syncid,
'WNHostName': socket.getfqdn()
}
monitor(parameters)
def _handlePileup(self):
"""
Handle pileup settings.
"""
# find out local site SE name
siteConfig = loadSiteLocalConfig()
seLocalName = siteConfig.localStageOut["se-name"]
print "Running on site '%s', local SE name: '%s'" % (
siteConfig.siteName, seLocalName)
pileupDict = self._getPileupConfigFromJson()
# 2011-02-03 according to the most recent version of instructions, we do
# want to differentiate between "MixingModule" and "DataMixingModule"
mixModules, dataMixModules = self._getPileupMixingModules()
# 2011-02-03
# on the contrary to the initial instructions (wave), there are
# going to be only two types of pileup input datasets: "data" or "mc"
# unlike all previous places where pileupType handled in a flexible
# way as specified in the configuration passed by the user, here are
# the two pileupTypes hardcoded: and we are going to add the "mc"
# datasets to "MixingModule"s and only add the "data" datasets to the
# "DataMixingModule"s
# if the user in the configuration specifies different pileup types
# than "data" or "mc", the following call will not modify anything
self._processPileupMixingModules(pileupDict, seLocalName,
dataMixModules, "data")
self._processPileupMixingModules(pileupDict, seLocalName, mixModules,
"mc")
def testSlcPhedexNodesEqualPhedexApiNodes(self):
"""
For each site, verify that the stageout node specified in
site-local-config.xml is the same as the one returned by the PhEDEx api.
"""
os.environ["CMS_PATH"] = "/cvmfs/cms.cern.ch"
phedex = PhEDEx()
nodes = phedex.getNodeMap()["phedex"]["node"]
# Make a dict for translating the se names into regular site names.
node_map = {}
for node in nodes:
node_map[str(node[u"se"])] = str(node[str(u"name")])
for d in os.listdir("/cvmfs/cms.cern.ch/SITECONF/"):
# Only T0_, T1_... folders are needed
if d[0] == "T":
os.environ['WMAGENT_SITE_CONFIG_OVERRIDE'] ='/cvmfs/cms.cern.ch/SITECONF/%s/JobConfig/site-local-config.xml' % (d)
try:
slc = loadSiteLocalConfig()
except SiteConfigError as e:
print e.args[0]
phedexNode = slc.localStageOut.get("phedex-node")
# If slc is correct, perform check
if "se-name" in slc.localStageOut and slc.localStageOut["se-name"] in node_map and phedexNode != None:
self.assertEqual(phedexNode, node_map[slc.localStageOut["se-name"]], \
"Error: Node specified in SLC (%s) doesn't match node returned by PhEDEx api (%s)." \
% (phedexNode, node_map[slc.localStageOut["se-name"]]))
return
def createErrorReport(exitCode, errorType, errorDetails=None,
logLocation="Report.0.pkl"):
"""
_createErrorReport_
Create a report if something fails inside the Bootstrap
This creates a dummy step called 'CRITICAL' and
sticks the error in there.
"""
try:
siteCfg = loadSiteLocalConfig()
except SiteConfigError:
# For now, assume that we did this on purpose
msg = "Couldn't find SiteConfig"
logging.error(msg)
# TODO: Make this not suck goatballs when you are just running tests
return
report = Report.Report()
report.data.pnn = siteCfg.localStageOut.get('phedex-node', 'Unknown')
report.data.siteName = getattr(siteCfg, 'siteName', 'Unknown')
report.data.hostName = socket.gethostname()
report.data.ceName = getSyncCE()
report.data.completed = False
report.addError(stepName='CRITICAL', exitCode=exitCode,
errorType=errorType, errorDetails=errorDetails)
reportPath = os.path.join(os.getcwd(), '../', logLocation)
report.save(reportPath)
return
def __init__(self, **overrideParams):
self.override = False
self.overrideConf = overrideParams
if overrideParams != {}:
self.override = True
self.fallbacks = []
# //
# // Try an get the TFC for the site
#//
self.tfc = None
self.numberOfRetries = 3
self.retryPauseTime = 600
# //
# // If override isnt None, we dont need SiteCfg, if it is
#// then we need siteCfg otherwise we are dead.
if self.override == False:
self.siteCfg = loadSiteLocalConfig()
if self.override:
self.initialiseOverride()
else:
self.initialiseSiteConf()
def __init__(self, **overrideParams):
self.override = False
self.logger = overrideParams.pop("logger", logging.getLogger())
self.overrideConf = overrideParams
if overrideParams != {}:
self.override = True
# //
# // Try an get the TFC for the site
# //
self.tfc = None
from WMCore.Storage.SiteLocalConfig import loadSiteLocalConfig
self.numberOfRetries = 3
self.retryPauseTime = 600
self.pnn = None
self.fallbacks = []
# //
# // If override isnt None, we dont need SiteCfg, if it is
# // then we need siteCfg otherwise we are dead.
if self.override == False:
self.siteCfg = loadSiteLocalConfig()
if self.override:
self.initialiseOverride()
else:
self.initialiseSiteConf()
def testSlcPhedexNodesEqualPhedexApiNodes(self):
"""
For each site, verify that the stageout node specified in
site-local-config.xml is the same as the one returned by the PhEDEx api.
"""
os.environ["CMS_PATH"] = "/cvmfs/cms.cern.ch"
nodes = ['FIXME']
for d in os.listdir("/cvmfs/cms.cern.ch/SITECONF/"):
# Only T0_, T1_... folders are needed
if d[0] == "T":
os.environ[
'WMAGENT_SITE_CONFIG_OVERRIDE'] = '/cvmfs/cms.cern.ch/SITECONF/%s/JobConfig/site-local-config.xml' % (
d)
try:
slc = loadSiteLocalConfig()
except SiteConfigError as e:
print(e.args[0])
phedexNode = slc.localStageOut.get("phedex-node")
self.assertTrue(
phedexNode in nodes,
"Error: Node specified in SLC (%s) not in list returned by PhEDEx api"
% phedexNode)
return
def _handlePileup(self):
"""
Handle pileup settings.
"""
# find out local site SE name
siteConfig = loadSiteLocalConfig()
seLocalName = siteConfig.localStageOut["se-name"]
print "Running on site '%s', local SE name: '%s'" % (siteConfig.siteName, seLocalName)
pileupDict = self._getPileupConfigFromJson()
# 2011-02-03 according to the most recent version of instructions, we do
# want to differentiate between "MixingModule" and "DataMixingModule"
mixModules, dataMixModules = self._getPileupMixingModules()
# 2011-02-03
# on the contrary to the initial instructions (wave), there are
# going to be only two types of pileup input datasets: "data" or "mc"
# unlike all previous places where pileupType handled in a flexible
# way as specified in the configuration passed by the user, here are
# the two pileupTypes hardcoded: and we are going to add the "mc"
# datasets to "MixingModule"s and only add the "data" datasets to the
# "DataMixingModule"s
# if the user in the configuration specifies different pileup types
# than "data" or "mc", the following call will not modify anything
self._processPileupMixingModules(pileupDict, seLocalName, dataMixModules, "data")
self._processPileupMixingModules(pileupDict, seLocalName, mixModules, "mc")
def __init__(self, **overrideParams):
self.override = False
self.overrideConf = overrideParams
if overrideParams != {}:
self.override = True
self.fallbacks = []
# //
# // Try an get the TFC for the site
#//
self.tfc = None
self.numberOfRetries = 3
self.retryPauseTime = 600
# //
# // If override isnt None, we dont need SiteCfg, if it is
#// then we need siteCfg otherwise we are dead.
if self.override == False:
self.siteCfg = loadSiteLocalConfig()
if self.override:
self.initialiseOverride()
else:
self.initialiseSiteConf()
def createInitialReport(job, reportName):
"""
_createInitialReport_
Create an initial job report with the base
information in it.
"""
try:
siteCfg = loadSiteLocalConfig()
except SiteConfigError:
# For now, assume that we did this on purpose
msg = "Couldn't find SiteConfig"
logging.error(msg)
# TODO: Make less goatballs for testing purposes
return
report = Report.Report()
report.data.WMAgentJobID = job.get('id', None)
report.data.WMAgentJobName = job.get('name', None)
report.data.pnn = siteCfg.localStageOut.get('phedex-node', 'Unknown')
report.data.siteName = getattr(siteCfg, 'siteName', 'Unknown')
report.data.hostName = socket.gethostname()
report.data.ceName = getSyncCE()
# TODO: need to check what format it returns and what features need to extract.
# currently
# $MACHINEFEATURES/hs06: HS06 score of the host
# $MACHINEFEATURES/total_cpu: number of configured job slots
# $JOBFEATURES/hs06_job: HS06 score available to your job
# $JOBFEATURES/allocated_cpu: number of allocated slots (=8 in case of a multicore job
machineFeaturesFile = os.environ.get('MACHINEFEATURES')
report.data.machineFeatures = {}
if machineFeaturesFile:
report.data.machineFeatures['hs06'] = readFloatFromFile(
"%s/hs06" % machineFeaturesFile)
report.data.machineFeatures['total_cpu'] = readFloatFromFile(
"%s/total_cpu" % machineFeaturesFile)
jobFeaturesFile = os.environ.get('JOBFEATURES')
report.data.jobFeatures = {}
if jobFeaturesFile:
report.data.jobFeatures['hs06_job'] = readFloatFromFile(
"%s/hs06_job" % jobFeaturesFile)
report.data.jobFeatures['allocated_cpu'] = readFloatFromFile(
"%s/allocated_cpu" % jobFeaturesFile)
report.data.completed = False
report.setTaskName(taskName=job.get('task', 'TaskNotFound'))
# Not so fond of this, but we have to put the master
# report way up at the top so it's returned if the
# job fails early
reportPath = os.path.join(os.getcwd(), '../', reportName)
report.save(reportPath)
return
def getSEName(self):
"""
Using SiteLocalConfig, get the SEName
"""
from WMCore.Storage.SiteLocalConfig import loadSiteLocalConfig
self.siteCfg = loadSiteLocalConfig()
def getSEName(self):
"""
Using SiteLocalConfig, get the SEName
"""
from WMCore.Storage.SiteLocalConfig import loadSiteLocalConfig
self.siteCfg = loadSiteLocalConfig()
def createInitialReport(job, reportName):
"""
_createInitialReport_
Create an initial job report with the base
information in it.
"""
try:
siteCfg = loadSiteLocalConfig()
except SiteConfigError:
# For now, assume that we did this on purpose
msg = "Couldn't find SiteConfig"
logging.error(msg)
# TODO: Make less goatballs for testing purposes
return
report = Report.Report()
report.data.WMAgentJobID = job.get('id', None)
report.data.WMAgentJobName = job.get('name', None)
report.data.pnn = siteCfg.localStageOut.get('phedex-node', 'Unknown')
report.data.siteName = getattr(siteCfg, 'siteName', 'Unknown')
report.data.hostName = socket.gethostname()
report.data.ceName = getSyncCE()
# TODO: need to check what format it returns and what features need to extract.
# currently
# $MACHINEFEATURES/hs06: HS06 score of the host
# $MACHINEFEATURES/total_cpu: number of configured job slots
# $JOBFEATURES/hs06_job: HS06 score available to your job
# $JOBFEATURES/allocated_cpu: number of allocated slots (=8 in case of a multicore job
machineFeaturesFile = os.environ.get('MACHINEFEATURES')
report.data.machineFeatures = {}
if machineFeaturesFile:
report.data.machineFeatures['hs06'] = readFloatFromFile("%s/hs06" % machineFeaturesFile)
report.data.machineFeatures['total_cpu'] = readFloatFromFile("%s/total_cpu" % machineFeaturesFile)
jobFeaturesFile = os.environ.get('JOBFEATURES')
report.data.jobFeatures = {}
if jobFeaturesFile:
report.data.jobFeatures['hs06_job'] = readFloatFromFile("%s/hs06_job" % jobFeaturesFile)
report.data.jobFeatures['allocated_cpu'] = readFloatFromFile("%s/allocated_cpu" % jobFeaturesFile)
report.data.completed = False
report.setTaskName(taskName=job.get('task', 'TaskNotFound'))
# Not so fond of this, but we have to put the master
# report way up at the top so it's returned if the
# job fails early
reportPath = os.path.join(os.getcwd(), '../', reportName)
report.save(reportPath)
return
def testLoadingConfigFromOverridenEnvVarriable(self):
"""
test SiteLocalConfig module method loadSiteLocalConfig when loading
site config from location defined by WMAGENT_SITE_CONFIG_OVERRIDE
env. variable
"""
vandyConfigFileName = os.path.join(
getTestBase(), "WMCore_t/Storage_t",
"T3_US_Vanderbilt_SiteLocalConfig.xml")
os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = vandyConfigFileName
mySiteConfig = loadSiteLocalConfig()
self.assertEqual(mySiteConfig.siteName, "T3_US_Vanderbilt",
"Error: Wrong site name.")
def testLoadingConfigFromOverridenEnvVarriable(self):
"""
test SiteLocalConfig module method loadSiteLocalConfig when loading
site config from location defined by WMAGENT_SITE_CONFIG_OVERRIDE
env. variable
"""
vandyConfigFileName = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T3_US_Vanderbilt_SiteLocalConfig.xml")
os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = vandyConfigFileName
mySiteConfig = loadSiteLocalConfig()
self.assertEqual(mySiteConfig.siteName, "T3_US_Vanderbilt",
"Error: Wrong site name.")
def __init__(self, interval=300, username=None, commonname=None):
self.interval = interval
self.__previous = 0
self.__states = {}
self.username = username if username else self.__get_user()
self.commonname = commonname if commonname else self.__get_distinguished_name().rsplit('/CN=', 1)[1]
self.__cmssw_version = 'Unknown'
self.__executable = 'Unknown'
self.__dash = None
try:
self._ce = loadSiteLocalConfig().siteName
except SiteConfigError:
logger.error("can't load siteconfig, defaulting to hostname")
self._ce = socket.getfqdn()
def __init__(self, interval=300, username=None, commonname=None):
self.interval = interval
self.__previous = 0
self.__states = {}
self.username = username if username else self.__get_user()
self.commonname = commonname if commonname else self.__get_distinguished_name(
).rsplit('/CN=', 1)[1]
self.__cmssw_version = 'Unknown'
self.__executable = 'Unknown'
try:
self._ce = loadSiteLocalConfig().siteName
except SiteConfigError:
logger.error("can't load siteconfig, defaulting to hostname")
self._ce = socket.getfqdn()
def __init__(self, **overrideParams):
logging.info("StageOutMgr::__init__()")
self.overrideConf = overrideParams
# Figure out if any of the override parameters apply to stage-out
self.override = False
if overrideParams != {}:
logging.info("StageOutMgr::__init__(): Override: %s",
overrideParams)
checkParams = ["command", "option", "phedex-node", "lfn-prefix"]
for param in checkParams:
if param in self.overrideConf.keys():
self.override = True
if not self.override:
logging.info(
"=======StageOut Override: These are not the parameters you are looking for"
)
self.substituteGUID = True
self.fallbacks = []
# //
# // Try an get the TFC for the site
# //
self.tfc = None
self.numberOfRetries = 3
self.retryPauseTime = 600
from WMCore.Storage.SiteLocalConfig import loadSiteLocalConfig
# //
# // If override isnt None, we dont need SiteCfg, if it is
# // then we need siteCfg otherwise we are dead.
if self.override == False:
self.siteCfg = loadSiteLocalConfig()
if self.override:
self.initialiseOverride()
else:
self.initialiseSiteConf()
self.failed = {}
self.completedFiles = {}
return
def __init__(self, **overrideParams):
print "StageOutMgr::__init__()"
self.overrideConf = overrideParams
# Figure out if any of the override parameters apply to stage-out
self.override = False
if overrideParams != {}:
print "StageOutMgr::__init__(): Override: %s" % overrideParams
checkParams = ["command", "option", "se-name", "lfn-prefix"]
for param in checkParams:
if param in self.overrideConf.keys():
self.override = True
if not self.override:
print "=======StageOut Override: These are not the parameters you are looking for"
self.substituteGUID = True
self.fallbacks = []
# //
# // Try an get the TFC for the site
#//
self.tfc = None
self.numberOfRetries = 3
self.retryPauseTime = 600
from WMCore.Storage.SiteLocalConfig import loadSiteLocalConfig
# //
# // If override isnt None, we dont need SiteCfg, if it is
#// then we need siteCfg otherwise we are dead.
if self.override == False:
self.siteCfg = loadSiteLocalConfig()
if self.override:
self.initialiseOverride()
else:
self.initialiseSiteConf()
self.failed = {}
self.completedFiles = {}
return
def loadSiteConf(self):
"""
_loadSiteConf_
Read the site conf file
"""
if not os.environ.has_key("CMS_PATH"):
msg = "CMS_PATH Not Set: Cannot find SiteConf"
self.summary['SiteConf'] = "Failed: CMS_PATH not set"
raise RuntimeError, msg
try:
self.siteConf = loadSiteLocalConfig()
except Exception, ex:
msg = "Error loading Site Conf File: %s" % str(ex)
self.summary['SiteConf'] = "Failed: Cannot load SiteConf"
raise RuntimeError, msg
def getSyncCE(default=socket.gethostname()):
"""
_getSyncCE_
Tries to get the site name from the localSite config, if it doesn't find it
or it finds an empty string then we check the environment
variables. Worst case scenario we give the Worker node.
"""
try:
siteConfig = loadSiteLocalConfig()
result = siteConfig.siteName
if result:
return result
except SiteConfigError:
logging.error(
"Couldn't find the site config, looking for the CE elsewhere")
result = socket.gethostname()
if 'GLOBUS_GRAM_JOB_CONTACT' in os.environ:
# //
# // OSG, Sync CE from Globus ID
#//
val = os.environ['GLOBUS_GRAM_JOB_CONTACT']
try:
host = val.split("https://", 1)[1]
host = host.split(":")[0]
result = host
except:
pass
return result
if 'NORDUGRID_CE' in os.environ:
# //
# // ARC, Sync CE from env. var. submitted with the job by JobSubmitter
#//
return os.environ['NORDUGRID_CE']
return result
def __init__(self, numberOfRetries = 3, retryPauseTime=15, **overrideParams):
# set defaults
self.failed = {}
self.completedFiles = {}
self.override = False
self.overrideConf = overrideParams
self.substituteGUID = True
self.defaultMethod = {}
self.fallbacks = []
self.tfc = None
self.numberOfRetries = numberOfRetries
self.retryPauseTime = retryPauseTime
if overrideParams != {}:
log.critical("Override: %s" % overrideParams)
self.override = True
self.initialiseOverride()
else:
self.siteCfg = loadSiteLocalConfig()
self.initialiseSiteConf()
def __init__(self, numberOfRetries=3, retryPauseTime=15, **overrideParams):
# set defaults
self.failed = {}
self.completedFiles = {}
self.override = False
self.overrideConf = overrideParams
self.substituteGUID = True
self.defaultMethod = {}
self.fallbacks = []
self.tfc = None
self.numberOfRetries = numberOfRetries
self.retryPauseTime = retryPauseTime
if overrideParams != {}:
log.critical("Override: %s" % overrideParams)
self.override = True
self.initialiseOverride()
else:
self.siteCfg = loadSiteLocalConfig()
self.initialiseSiteConf()
def getSyncCE(default = socket.gethostname()):
"""
_getSyncCE_
Tries to get the site name from the localSite config, if it doesn't find it
or it finds an empty string then we check the environment
variables. Worst case scenario we give the Worker node.
"""
try:
siteConfig = loadSiteLocalConfig()
result = siteConfig.siteName
if result:
return result
except SiteConfigError:
logging.error("Couldn't find the site config, looking for the CE elsewhere")
result = socket.gethostname()
if 'GLOBUS_GRAM_JOB_CONTACT' in os.environ:
# //
# // OSG, Sync CE from Globus ID
#//
val = os.environ['GLOBUS_GRAM_JOB_CONTACT']
try:
host = val.split("https://", 1)[1]
host = host.split(":")[0]
result = host
except:
pass
return result
if 'NORDUGRID_CE' in os.environ:
# //
# // ARC, Sync CE from env. var. submitted with the job by JobSubmitter
#//
return os.environ['NORDUGRID_CE']
return result
def testSlcPhedexNodesEqualPhedexApiNodes(self):
"""
For each site, verify that the stageout node specified in
site-local-config.xml is the same as the one returned by the PhEDEx api.
"""
os.environ["CMS_PATH"] = "/cvmfs/cms.cern.ch"
phedex = PhEDEx()
nodes = [node[u'name'] for node in phedex.getNodeMap()["phedex"]["node"]]
for d in os.listdir("/cvmfs/cms.cern.ch/SITECONF/"):
# Only T0_, T1_... folders are needed
if d[0] == "T":
os.environ['WMAGENT_SITE_CONFIG_OVERRIDE'] ='/cvmfs/cms.cern.ch/SITECONF/%s/JobConfig/site-local-config.xml' % (d)
try:
slc = loadSiteLocalConfig()
except SiteConfigError as e:
print(e.args[0])
phedexNode = slc.localStageOut.get("phedex-node")
self.assertTrue(phedexNode in nodes,
"Error: Node specified in SLC (%s) not in list returned by PhEDEx api" % phedexNode)
return
def createErrorReport(exitCode,
errorType,
errorDetails=None,
logLocation="Report.0.pkl"):
"""
_createErrorReport_
Create a report if something fails inside the Bootstrap
This creates a dummy step called 'CRITICAL' and
sticks the error in there.
"""
try:
siteCfg = loadSiteLocalConfig()
except SiteConfigError:
# For now, assume that we did this on purpose
msg = "Couldn't find SiteConfig"
logging.error(msg)
#TODO: Make this not suck goatballs when you are just running tests
return
report = Report.Report()
report.data.seName = siteCfg.localStageOut.get('se-name',
socket.gethostname())
report.data.pnn = siteCfg.localStageOut.get('phedex-node', 'Unknown')
report.data.siteName = getattr(siteCfg, 'siteName', 'Unknown')
report.data.hostName = socket.gethostname()
report.data.ceName = getSyncCE()
report.data.completed = False
report.addError(stepName='CRITICAL',
exitCode=exitCode,
errorType=errorType,
errorDetails=errorDetails)
reportPath = os.path.join(os.getcwd(), '../', logLocation)
report.save(reportPath)
return
def __init__(self, config):
util.Timing.__init__(self, 'dash', 'handler', 'updates', 'elk', 'transfers', 'cleanup', 'propagate', 'sqlite')
self.config = config
self.basedirs = [config.base_directory, config.startup_directory]
self.workdir = config.workdir
self._storage = config.storage
self.statusfile = os.path.join(self.workdir, 'status.json')
self.siteconf = os.path.join(self.workdir, 'siteconf')
self.parrot_path = os.path.dirname(util.which('parrot_run'))
self.parrot_bin = os.path.join(self.workdir, 'bin')
self.parrot_lib = os.path.join(self.workdir, 'lib')
self.__algo = Algo(config)
self.__host = socket.getfqdn()
try:
siteconf = loadSiteLocalConfig()
self.__ce = siteconf.siteName
self.__se = siteconf.localStageOutPNN()
self.__frontier_proxy = siteconf.frontierProxies[0]
except (SiteConfigError, IndexError):
logger.error("can't load siteconfig, defaulting to hostname")
self.__ce = socket.getfqdn()
self.__se = socket.getfqdn()
try:
self.__frontier_proxy = os.environ['HTTP_PROXY']
except KeyError:
logger.error("can't determine proxy for Frontier via $HTTP_PROXY")
sys.exit(1)
try:
with open('/etc/cvmfs/default.local') as f:
lines = f.readlines()
except IOError:
lines = []
for l in lines:
m = re.match('\s*CVMFS_HTTP_PROXY\s*=\s*[\'"]?(.*)[\'"]?', l)
if m:
self.__cvmfs_proxy = m.group(1).strip("\"'")
break
else:
try:
self.__cvmfs_proxy = os.environ['HTTP_PROXY']
except KeyError:
logger.error("can't determine proxy for CVMFS via $HTTP_PROXY")
sys.exit(1)
logger.debug("using {} as proxy for CVMFS".format(self.__cvmfs_proxy))
logger.debug("using {} as proxy for Frontier".format(self.__frontier_proxy))
logger.debug("using {} as osg_version".format(self.config.advanced.osg_version))
util.sendemail("Your Lobster project has started!", self.config)
self.__taskhandlers = {}
self.__store = unit.UnitStore(self.config)
self.__setup_inputs()
self.copy_siteconf()
create = not util.checkpoint(self.workdir, 'id')
if create:
self.taskid = 'lobster_{0}_{1}'.format(
self.config.label,
sha1(str(datetime.datetime.utcnow())).hexdigest()[-16:])
util.register_checkpoint(self.workdir, 'id', self.taskid)
shutil.copy(self.config.base_configuration, os.path.join(self.workdir, 'config.py'))
else:
self.taskid = util.checkpoint(self.workdir, 'id')
util.register_checkpoint(self.workdir, 'RESTARTED', str(datetime.datetime.utcnow()))
if not util.checkpoint(self.workdir, 'executable'):
# We can actually have more than one exe name (one per task label)
# Set 'cmsRun' if any of the tasks are of that type,
# or use cmd command if all tasks execute the same cmd,
# or use 'noncmsRun' if task cmds are different
# Using this for dashboard exe name reporting
cmsconfigs = [wflow.pset for wflow in self.config.workflows]
cmds = [wflow.command for wflow in self.config.workflows]
if any(cmsconfigs):
exename = 'cmsRun'
elif all(x == cmds[0] and x is not None for x in cmds):
exename = cmds[0]
else:
exename = 'noncmsRun'
util.register_checkpoint(self.workdir, 'executable', exename)
for wflow in self.config.workflows:
if create and not util.checkpoint(self.workdir, wflow.label):
wflow.setup(self.workdir, self.basedirs)
logger.info("querying backend for {0}".format(wflow.label))
with fs.alternative():
dataset_info = wflow.dataset.get_info()
logger.info("registering {0} in database".format(wflow.label))
self.__store.register_dataset(wflow, dataset_info, wflow.category.runtime)
util.register_checkpoint(self.workdir, wflow.label, 'REGISTERED')
elif os.path.exists(os.path.join(wflow.workdir, 'running')):
for id in self.get_taskids(wflow.label):
util.move(wflow.workdir, id, 'failed')
for wflow in self.config.workflows:
if wflow.parent:
getattr(self.config.workflows, wflow.parent.label).register(wflow)
if create:
total_units = wflow.dataset.total_units * len(wflow.unique_arguments)
self.__store.register_dependency(wflow.label, wflow.parent.label, total_units)
if not util.checkpoint(self.workdir, 'sandbox cmssw version'):
util.register_checkpoint(self.workdir, 'sandbox', 'CREATED')
versions = set([w.version for w in self.config.workflows])
if len(versions) == 1:
util.register_checkpoint(self.workdir, 'sandbox cmssw version', list(versions)[0])
if self.config.elk:
if create:
categories = {wflow.category.name: [] for wflow in self.config.workflows}
for category in categories:
for workflow in self.config.workflows:
if workflow.category.name == category:
categories[category].append(workflow.label)
self.config.elk.create(categories)
else:
self.config.elk.resume()
self.config.advanced.dashboard.setup(self.config)
if create:
self.config.save()
self.config.advanced.dashboard.register_run()
else:
self.config.advanced.dashboard.update_task_status(
(id_, dash.ABORTED) for id_ in self.__store.reset_units()
)
for p in (self.parrot_bin, self.parrot_lib):
if not os.path.exists(p):
os.makedirs(p)
for exe in ('parrot_run', 'chirp', 'chirp_put', 'chirp_get'):
shutil.copy(util.which(exe), self.parrot_bin)
subprocess.check_call(["strip", os.path.join(self.parrot_bin, exe)])
p_helper = os.path.join(os.path.dirname(self.parrot_path), 'lib', 'lib64', 'libparrot_helper.so')
shutil.copy(p_helper, self.parrot_lib)
def handlePileup(self):
"""
_handlePileup_
Handle pileup settings.
There has been stored pileup configuration stored in a JSON file
as a result of DBS querrying when running PileupFetcher,
this method loads this configuration from sandbox and returns it
as dictionary.
The PileupFetcher was called by WorkQueue which creates job's sandbox
and sandbox gets migrated to the worker node.
External script iterates over all modules and over all pileup configuration types.
The only considered types are "data" and "mc" (input to this method).
If other pileup types are specified by the user, the method doesn't
modify anything.
The method considers only files which are present on this local PNN.
The job will use only those, unless it was told to trust the PU site
location (trustPUSitelists=True), in this case ALL the blocks/files
will be added to the PSet and files will be read via AAA.
Dataset, divided into blocks, may not have all blocks present on a
particular PNN. However, all files belonging into a block will be
present when reported by DBS.
The structure of the pileupDict: PileupFetcher._queryDbsAndGetPileupConfig
"""
# find out local site SE name
siteConfig = loadSiteLocalConfig()
PhEDExNodeName = siteConfig.localStageOut["phedex-node"]
self.logger.info("Running on site '%s', local PNN: '%s'",
siteConfig.siteName, PhEDExNodeName)
jsonPileupConfig = os.path.join(self.stepSpace.location,
"pileupconf.json")
# Load pileup json
try:
with open(jsonPileupConfig) as jdata:
pileupDict = json.load(jdata)
except IOError:
m = "Could not read pileup JSON configuration file: '%s'" % jsonPileupConfig
raise RuntimeError(m)
# Create a json with a list of files and events available
# after dealing with PhEDEx/AAA logic
newPileupDict = {}
fileList = []
eventsAvailable = 0
for pileupType in self.step.data.pileup.listSections_():
useAAA = True if getattr(self.jobBag, 'trustPUSitelists',
False) else False
self.logger.info("Pileup set to read data remotely: %s", useAAA)
for blockName in sorted(pileupDict[pileupType].keys()):
blockDict = pileupDict[pileupType][blockName]
if PhEDExNodeName in blockDict["PhEDExNodeNames"] or useAAA:
eventsAvailable += int(blockDict.get('NumberOfEvents', 0))
for fileLFN in blockDict["FileList"]:
fileList.append(str(fileLFN))
newPileupDict[pileupType] = {
"eventsAvailable": eventsAvailable,
"FileList": fileList
}
newJsonPileupConfig = os.path.join(self.stepSpace.location,
"CMSSWPileupConfig.json")
self.logger.info("Generating json for CMSSW pileup script")
try:
with open(newJsonPileupConfig, 'wb') as f:
json.dump(newPileupDict, f)
except Exception as ex:
self.logger.exception("Error writing out process filelist json:")
raise ex
procScript = "cmssw_handle_pileup.py"
cmd = "%s --input_pkl %s --output_pkl %s --pileup_dict %s" % (
procScript, os.path.join(self.stepSpace.location,
self.configPickle),
os.path.join(self.stepSpace.location,
self.configPickle), newJsonPileupConfig)
if getattr(self.jobBag, "skipPileupEvents", None):
randomSeed = self.job['task']
skipPileupEvents = self.jobBag.skipPileupEvents
cmd += " --skip_pileup_events %s --random_seed %s" % (
skipPileupEvents, randomSeed)
self.scramRun(cmd)
return
# modifications of job input files
pileupConfig = {"data": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"],
"mc": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"]}
dbsUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
setupScript.step.setupPileup(pileupConfig, dbsUrl)
# SetupCMSSWPset pileup handling will be consulting SiteLocalConfig
# to determine StorageElement (SE) name the job is running on
# SiteLocalConfig loads the site-local-config.xml file from env.
# variable defined location ; if the variable is not defined already, set it
# obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above
# thick with SE name is not effective
if not os.getenv("WMAGENT_SITE_CONFIG_OVERRIDE", None):
os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = siteLocalConfig
# find out local site name from the testing local site config,
# will be needed later
siteConfig = loadSiteLocalConfig()
seLocalName = siteConfig.localStageOut["se-name"]
print "Running on site '%s', local SE name: '%s'" % (siteConfig.siteName, seLocalName)
# before calling the script, SetupCMSSWPset will try to load JSON
# pileup configuration file, need to create it in self.testDir
fetcher = PileupFetcher()
fetcher.setWorkingDirectory(self.testDir)
fetcher._createPileupConfigFile(setupScript.step)
setupScript()
# now test all modifications carried out in SetupCMSSWPset.__call__
# which will also test that CMSSWStepHelper.setupPileup run correctly
mixModules, dataMixModules = setupScript._getPileupMixingModules()
def __init__(self, config):
util.Timing.__init__(self, 'dash', 'handler', 'updates', 'elk',
'transfers', 'cleanup', 'propagate', 'sqlite')
self.config = config
self.basedirs = [config.base_directory, config.startup_directory]
self.workdir = config.workdir
self._storage = config.storage
self.statusfile = os.path.join(self.workdir, 'status.json')
self.siteconf = os.path.join(self.workdir, 'siteconf')
self.parrot_path = os.path.dirname(util.which('parrot_run'))
self.parrot_bin = os.path.join(self.workdir, 'bin')
self.parrot_lib = os.path.join(self.workdir, 'lib')
self.__algo = Algo(config)
self.__host = socket.getfqdn()
try:
siteconf = loadSiteLocalConfig()
self.__ce = siteconf.siteName
self.__se = siteconf.localStageOutPNN()
self.__frontier_proxy = siteconf.frontierProxies[0]
except (SiteConfigError, IndexError):
logger.error("can't load siteconfig, defaulting to hostname")
self.__ce = socket.getfqdn()
self.__se = socket.getfqdn()
try:
self.__frontier_proxy = os.environ['HTTP_PROXY']
except KeyError:
logger.error(
"can't determine proxy for Frontier via $HTTP_PROXY")
sys.exit(1)
try:
with open('/etc/cvmfs/default.local') as f:
lines = f.readlines()
except IOError:
lines = []
for l in lines:
m = re.match('\s*CVMFS_HTTP_PROXY\s*=\s*[\'"]?(.*)[\'"]?', l)
if m:
self.__cvmfs_proxy = m.group(1).strip("\"'")
break
else:
try:
self.__cvmfs_proxy = os.environ['HTTP_PROXY']
except KeyError:
logger.error("can't determine proxy for CVMFS via $HTTP_PROXY")
sys.exit(1)
logger.debug("using {} as proxy for CVMFS".format(self.__cvmfs_proxy))
logger.debug("using {} as proxy for Frontier".format(
self.__frontier_proxy))
logger.debug("using {} as osg_version".format(
self.config.advanced.osg_version))
util.sendemail("Your Lobster project has started!", self.config)
self.__taskhandlers = {}
self.__store = unit.UnitStore(self.config)
self.__setup_inputs()
self.copy_siteconf()
create = not util.checkpoint(self.workdir, 'id')
if create:
self.taskid = 'lobster_{0}_{1}'.format(
self.config.label,
sha1(str(datetime.datetime.utcnow())).hexdigest()[-16:])
util.register_checkpoint(self.workdir, 'id', self.taskid)
shutil.copy(self.config.base_configuration,
os.path.join(self.workdir, 'config.py'))
else:
self.taskid = util.checkpoint(self.workdir, 'id')
util.register_checkpoint(self.workdir, 'RESTARTED',
str(datetime.datetime.utcnow()))
if not util.checkpoint(self.workdir, 'executable'):
# We can actually have more than one exe name (one per task label)
# Set 'cmsRun' if any of the tasks are of that type,
# or use cmd command if all tasks execute the same cmd,
# or use 'noncmsRun' if task cmds are different
# Using this for dashboard exe name reporting
cmsconfigs = [wflow.pset for wflow in self.config.workflows]
cmds = [wflow.command for wflow in self.config.workflows]
if any(cmsconfigs):
exename = 'cmsRun'
elif all(x == cmds[0] and x is not None for x in cmds):
exename = cmds[0]
else:
exename = 'noncmsRun'
util.register_checkpoint(self.workdir, 'executable', exename)
for wflow in self.config.workflows:
if create and not util.checkpoint(self.workdir, wflow.label):
wflow.setup(self.workdir, self.basedirs)
logger.info("querying backend for {0}".format(wflow.label))
with fs.alternative():
dataset_info = wflow.dataset.get_info()
logger.info("registering {0} in database".format(wflow.label))
self.__store.register_dataset(wflow, dataset_info,
wflow.category.runtime)
util.register_checkpoint(self.workdir, wflow.label,
'REGISTERED')
elif os.path.exists(os.path.join(wflow.workdir, 'running')):
for id in self.get_taskids(wflow.label):
util.move(wflow.workdir, id, 'failed')
for wflow in self.config.workflows:
if wflow.parent:
getattr(self.config.workflows,
wflow.parent.label).register(wflow)
if create:
total_units = wflow.dataset.total_units * len(
wflow.unique_arguments)
self.__store.register_dependency(wflow.label,
wflow.parent.label,
total_units)
if not util.checkpoint(self.workdir, 'sandbox cmssw version'):
util.register_checkpoint(self.workdir, 'sandbox', 'CREATED')
versions = set([w.version for w in self.config.workflows])
if len(versions) == 1:
util.register_checkpoint(self.workdir, 'sandbox cmssw version',
list(versions)[0])
if self.config.elk:
if create:
categories = {
wflow.category.name: []
for wflow in self.config.workflows
}
for category in categories:
for workflow in self.config.workflows:
if workflow.category.name == category:
categories[category].append(workflow.label)
self.config.elk.create(categories)
else:
self.config.elk.resume()
self.config.advanced.dashboard.setup(self.config)
if create:
self.config.save()
self.config.advanced.dashboard.register_run()
else:
self.config.advanced.dashboard.update_task_status(
(id_, dash.ABORTED) for id_ in self.__store.reset_units())
for p in (self.parrot_bin, self.parrot_lib):
if not os.path.exists(p):
os.makedirs(p)
for exe in ('parrot_run', 'chirp', 'chirp_put', 'chirp_get'):
shutil.copy(util.which(exe), self.parrot_bin)
subprocess.check_call(
["strip", os.path.join(self.parrot_bin, exe)])
p_helper = os.path.join(os.path.dirname(self.parrot_path), 'lib',
'lib64', 'libparrot_helper.so')
shutil.copy(p_helper, self.parrot_lib)
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
]
}
dbsUrl = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
setupScript.step.setupPileup(pileupConfig, dbsUrl)
# SetupCMSSWPset pileup handling will be consulting SiteLocalConfig
# to determine StorageElement (SE) name the job is running on
# SiteLocalConfig loads the site-local-config.xml file from env.
# variable defined location ; if the variable is not defined already, set it
# obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above
# thick with SE name is not effective
if not os.getenv("WMAGENT_SITE_CONFIG_OVERRIDE", None):
os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = siteLocalConfig
# find out local site name from the testing local site config,
# will be needed later
siteConfig = loadSiteLocalConfig()
seLocalName = siteConfig.localStageOut["se-name"]
print "Running on site '%s', local SE name: '%s'" % (
siteConfig.siteName, seLocalName)
# before calling the script, SetupCMSSWPset will try to load JSON
# pileup configuration file, need to create it in self.testDir
fetcher = PileupFetcher()
fetcher.setWorkingDirectory(self.testDir)
fetcher._createPileupConfigFile(setupScript.step)
setupScript()
# now test all modifications carried out in SetupCMSSWPset.__call__
# which will also test that CMSSWStepHelper.setupPileup run correctly
mixModules, dataMixModules = setupScript._getPileupMixingModules()
def testPileupSetup(self):
"""
Test the pileup setting.
reference (setupScript.process instance):
in test/python/WMCore_t/WMRuntime_t/Scripts_t/WMTaskSpace/cmsRun1/PSet.py
"""
try:
from dbs.apis.dbsClient import DbsApi
except ImportError as ex:
raise unittest.SkipTest
# this is modified and shortened version of
# WMCore/test/python/WMCore_t/Misc_t/site-local-config.xml
# since the dataset name in question (below) is only present at
# storm-fe-cms.cr.cnaf.infn.it, need to make the test think it's its local SE
siteLocalConfigContent = \
"""
<site-local-config>
<site name="-SOME-SITE-NAME-">
<event-data>
<catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/>
</event-data>
<local-stage-out>
<!-- original cmssrm.fnal.gov -->
<phedex-node value="T2_CH_CERN"/>
<command value="test-copy"/>
<catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/>
</local-stage-out>
<calib-data>
<frontier-connect>
<load balance="proxies"/>
<proxy url="http://cmsfrontier1.fnal.gov:3128"/>
<proxy url="http://cmsfrontier2.fnal.gov:3128"/>
</frontier-connect>
</calib-data>
</site>
</site-local-config>
"""
siteLocalConfig = os.path.join(self.testDir,
"test-site-local-config.xml")
f = open(siteLocalConfig, 'w')
f.write(siteLocalConfigContent)
f.close()
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name="stepSpace")
setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1")
setupScript.job = self.createTestJob()
# define pileup configuration
# despite of the implementation considering whichever type of pileup,
# only "data" and "mc" types are eventually considered and lead to any
# modifications of job input files
pileupConfig = {
"data": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
],
"mc": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
]
}
dbsUrl = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader"
setupScript.step.setupPileup(pileupConfig, dbsUrl)
# SetupCMSSWPset pileup handling will be consulting SiteLocalConfig
# to determine StorageElement (SE) name the job is running on
# SiteLocalConfig loads the site-local-config.xml file from env.
# variable defined location ; if the variable is not defined already, set it
# obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above
# thick with SE name is not effective
if not os.getenv("WMAGENT_SITE_CONFIG_OVERRIDE", None):
os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = siteLocalConfig
# find out local site name from the testing local site config,
# will be needed later
siteConfig = loadSiteLocalConfig()
seLocalName = siteConfig.localStageOut["phedex-node"]
print("Running on site '%s', local SE name: '%s'" %
(siteConfig.siteName, seLocalName))
# before calling the script, SetupCMSSWPset will try to load JSON
# pileup configuration file, need to create it in self.testDir
fetcher = PileupFetcher()
fetcher.setWorkingDirectory(self.testDir)
fetcher.createPileupConfigFile(setupScript.step)
setupScript()
# now test all modifications carried out in SetupCMSSWPset.__call__
# which will also test that CMSSWStepHelper.setupPileup run correctly
mixModules, dataMixModules = setupScript._getPileupMixingModules()
# load in the pileup configuration in the form of dict which
# PileupFetcher previously saved in a JSON file
pileupDict = setupScript._getPileupConfigFromJson()
# get the sub dict for particular pileup type
# for pileupDict structure description - see PileupFetcher._queryDbsAndGetPileupConfig
for pileupType, modules in zip(("data", "mc"),
(dataMixModules, mixModules)):
# getting KeyError here - above pileupConfig is not correct - need
# to have these two types of pile type
d = pileupDict[pileupType]
self._mixingModulesInputFilesTest(modules, d, seLocalName)
def testPileupSetup(self):
"""
Test the pileup setting.
reference (setupScript.process instance):
in test/python/WMCore_t/WMRuntime_t/Scripts_t/WMTaskSpace/cmsRun1/PSet.py
"""
try:
from dbs.apis.dbsClient import DbsApi
except ImportError as ex:
raise nose.SkipTest
# this is modified and shortened version of
# WMCore/test/python/WMCore_t/Misc_t/site-local-config.xml
# since the dataset name in question (below) is only present at
# storm-fe-cms.cr.cnaf.infn.it, need to make the test think it's its local SE
siteLocalConfigContent = \
"""
<site-local-config>
<site name="-SOME-SITE-NAME-">
<event-data>
<catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/>
</event-data>
<local-stage-out>
<!-- original cmssrm.fnal.gov -->
<phedex-node value="T2_CH_CERN"/>
<command value="test-copy"/>
<catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/>
</local-stage-out>
<calib-data>
<frontier-connect>
<load balance="proxies"/>
<proxy url="http://cmsfrontier1.fnal.gov:3128"/>
<proxy url="http://cmsfrontier2.fnal.gov:3128"/>
</frontier-connect>
</calib-data>
</site>
</site-local-config>
"""
siteLocalConfig = os.path.join(self.testDir, "test-site-local-config.xml")
f = open(siteLocalConfig, 'w')
f.write(siteLocalConfigContent)
f.close()
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name = "stepSpace")
setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1")
setupScript.job = self.createTestJob()
# define pileup configuration
# despite of the implementation considering whichever type of pileup,
# only "data" and "mc" types are eventually considered and lead to any
# modifications of job input files
pileupConfig = {"data": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"],
"mc": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"]}
dbsUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
setupScript.step.setupPileup(pileupConfig, dbsUrl)
# SetupCMSSWPset pileup handling will be consulting SiteLocalConfig
# to determine StorageElement (SE) name the job is running on
# SiteLocalConfig loads the site-local-config.xml file from env.
# variable defined location ; if the variable is not defined already, set it
# obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above
# thick with SE name is not effective
if not os.getenv("WMAGENT_SITE_CONFIG_OVERRIDE", None):
os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = siteLocalConfig
# find out local site name from the testing local site config,
# will be needed later
siteConfig = loadSiteLocalConfig()
seLocalName = siteConfig.localStageOut["phedex-node"]
print("Running on site '%s', local SE name: '%s'" % (siteConfig.siteName, seLocalName))
# before calling the script, SetupCMSSWPset will try to load JSON
# pileup configuration file, need to create it in self.testDir
fetcher = PileupFetcher()
fetcher.setWorkingDirectory(self.testDir)
fetcher._createPileupConfigFile(setupScript.step, fakeSites=['T1_US_FNAL'])
setupScript()
# now test all modifications carried out in SetupCMSSWPset.__call__
# which will also test that CMSSWStepHelper.setupPileup run correctly
mixModules, dataMixModules = setupScript._getPileupMixingModules()
# load in the pileup configuration in the form of dict which
# PileupFetcher previously saved in a JSON file
pileupDict = setupScript._getPileupConfigFromJson()
# get the sub dict for particular pileup type
# for pileupDict structure description - see PileupFetcher._queryDbsAndGetPileupConfig
for pileupType, modules in zip(("data", "mc"), (dataMixModules, mixModules)):
# getting KeyError here - above pileupConfig is not correct - need
# to have these two types of pile type
d = pileupDict[pileupType]
self._mixingModulesInputFilesTest(modules, d, seLocalName)
