def main():
problemElems = getProblematicRequests()
print("Found %d bad elements that needs fixup" % len(problemElems))
if not problemElems:
print("Nothing to fix, contact a developer if the problem persists...")
return 0
cric = CRIC()
dbsUrl = "https://cmsweb.cern.ch/dbs/prod/phys03/DBSReader"
dbs = DBSReader(dbsUrl)
for elem in problemElems:
print("Handling id: %s, with inputs: %s" % (elem.id, elem['Inputs']))
for dataItem in elem['Inputs']:
if isDataset(dataItem):
pnns = dbs.listDatasetLocation(dataItem, dbsOnly=True)
else:
pnns = dbs.listFileBlockLocation(dataItem, dbsOnly=True)
psns = cric.PNNstoPSNs(pnns)
print(" PNNs: %s map to PSNs: %s" % (pnns, psns))
elem['Inputs'][dataItem] = psns
backend.saveElements(*problemElems)
print("Done")
return 0
def __init__(self, **args):
StartPolicyInterface.__init__(self, **args)
self.args.setdefault('SliceType', 'NumberOfFiles')
self.args.setdefault('SliceSize', 1)
self.args.setdefault('SplittingAlgo', 'LumiBased')
self.lumiType = "NumberOfLumis"
# Define how to handle the different splitting algorithms
self.algoMapping = {
'Harvest': self.singleChunk,
'ParentlessMergeBySize': self.singleChunk,
'MinFileBased': self.singleChunk,
'LumiBased': self.singleChunk,
'EventAwareLumiBased': self.singleChunk,
'EventBased': self.singleChunk
}
self.unsupportedAlgos = ['WMBSMergeBySize', 'SiblingProcessingBased']
self.defaultAlgo = self.fixedSizeChunk
self.sites = []
if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv(
"WMCORE_USE_CRIC", False):
self.cric = CRIC()
else:
self.cric = None
self.siteDB = SiteDB()
def setUp(self):
"""
_setUp_
Setup couchdb and the test environment
"""
super(ResubmitBlockTest, self).setUp()
self.group = 'unknown'
self.user = '******'
# Set external test helpers
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")
# Define test environment
self.couchUrl = os.environ["COUCHURL"]
self.acdcDBName = 'resubmitblock_t'
self.validLocations = [
'T2_US_Nebraska', 'T1_US_FNAL_Disk', 'T1_UK_RAL_Disk'
]
self.siteWhitelist = ['T2_XX_SiteA']
cric = CRIC()
# Convert phedex node name to a valid processing site name
self.PSNs = cric.PNNstoPSNs(self.validLocations)
self.workflowName = 'dballest_ReReco_workflow'
couchServer = CouchServer(dburl=self.couchUrl)
self.acdcDB = couchServer.connectDatabase(self.acdcDBName,
create=False)
user = makeUser(self.group, '*****@*****.**', self.couchUrl,
self.acdcDBName)
user.create()
return
def wrapped_func(*args, **kwargs):
if 'cric' in services and (
not args[0].allCMSNames.sites or
(args[0].allCMSNames.cachetime + 1800 < mktime(gmtime()))):
args[0].allCMSNames = CMSSitesCache(sites=CRIC().getAllPSNs(),
cachetime=mktime(gmtime()))
args[0].allPNNNames = CMSSitesCache(
sites=CRIC().getAllPhEDExNodeNames(),
cachetime=mktime(gmtime()))
if 'phedex' in services and not args[0].phedex:
phdict = args[0].phedexargs
phdict.update({'cert': serverCert, 'key': serverKey})
args[0].phedex = PhEDEx(responseType='xml', dict=phdict)
if 'centralconfig' in services and (
not args[0].centralcfg.centralconfig or
(args[0].centralcfg.cachetime + 1800 < mktime(gmtime()))):
args[0].centralcfg = ConfigCache(
centralconfig=getCentralConfig(
extconfigurl=args[0].config.extconfigurl,
mode=args[0].config.mode),
cachetime=mktime(gmtime()))
if 'servercert' in services:
args[0].serverCert = serverCert
args[0].serverKey = serverKey
return func(*args, **kwargs)
def __init__(self, **args):
StartPolicyInterface.__init__(self, **args)
self.args.setdefault('SliceType', 'NumberOfRuns')
self.args.setdefault('SliceSize', 1)
self.lumiType = "NumberOfLumis"
self.sites = []
self.cric = CRIC()
def __init__(self, msConfig, mode, logger=None):
"""
Runs the basic setup and initialization for the MSOutput module
:microConfig: microservice configuration
:mode: MSOutput Run mode:
- MSOutputConsumer:
Reads The workflow and transfer subscriptions from MongoDB and
makes transfer subscriptions.
- MSOutputProducer:
Fetches Workflows in a given status from Reqmgr2 then creates
and uploads the documents to MongoDB.
"""
super(MSOutput, self).__init__(msConfig, logger)
self.mode = mode
self.msConfig.setdefault("limitRequestsPerCycle", 500)
self.msConfig.setdefault("verbose", True)
self.msConfig.setdefault("interval", 600)
self.msConfig.setdefault("services", ['output'])
self.msConfig.setdefault("defaultDataManSys", "DDM")
self.msConfig.setdefault("defaultGroup", "DataOps")
self.msConfig.setdefault("enableAggSubscr", True)
self.msConfig.setdefault("enableDataPlacement", False)
self.msConfig.setdefault("excludeDataTier", ['NANOAOD', 'NANOAODSIM'])
self.msConfig.setdefault("rucioAccount", 'wma_test')
self.msConfig.setdefault("mongoDBUrl", 'mongodb://localhost')
self.msConfig.setdefault("mongoDBPort", 8230)
self.msConfig.setdefault("streamerBufferFile", None)
self.uConfig = {}
self.emailAlert = EmailAlert(self.msConfig)
self.cric = CRIC(logger=self.logger)
self.uConfig = {}
self.campaigns = {}
self.psn2pnnMap = {}
msOutIndex = IndexModel('RequestName', unique=True)
msOutDBConfig = {
'database':
'msOutDB',
'server':
self.msConfig['mongoDBUrl'],
'port':
self.msConfig['mongoDBPort'],
'logger':
self.logger,
'create':
True,
'collections': [('msOutRelValColl', msOutIndex),
('msOutNonRelValColl', msOutIndex)]
}
self.msOutDB = MongoDB(**msOutDBConfig).msOutDB
self.msOutRelValColl = self.msOutDB['msOutRelValColl']
self.msOutNonRelValColl = self.msOutDB['msOutNonRelValColl']
self.ddm = DDM(
url=self.msConfig['ddmUrl'],
logger=self.logger,
enableDataPlacement=self.msConfig['enableDataPlacement'])
def setUp(self):
"""
Setup for unit tests
"""
super(CRICTest, self).setUp()
self.myCRIC = CRIC()
if PY3:
self.assertItemsEqual = self.assertCountEqual
def __init__(self, *args, **kwargs):
TaskAction.__init__(self, *args, **kwargs)
with self.config.TaskWorker.envForCMSWEB:
configDict = {
"cacheduration": 1,
"pycurl": True
} # cache duration is in hours
self.resourceCatalog = CRIC(logger=self.logger,
configDict=configDict)
def sites():
"Return known CMS site list from CRIC"
try:
# Download a list of all the sites from CRIC
cric = CRIC()
site_list = sorted(cric.getAllPSNs())
except Exception as exc:
msg = "ERROR: Could not retrieve sites from CRIC, reason: %s" % str(exc)
raise Exception(msg)
return site_list
def sites():
"Return known CMS site list from CRIC"
try:
# Download a list of all the sites from CRIC
cric = CRIC()
site_list = sorted(cric.getAllPSNs())
except Exception as exc:
msg = "ERROR: Could not retrieve sites from CRIC, reason: %s" % str(
exc)
raise Exception(msg)
return site_list
def pnns():
"""
Returns all PhEDEx node names, excluding Buffer endpoints
"""
cric = CRIC()
try:
pnn_list = sorted(cric.getAllPhEDExNodeNames(excludeBuffer=True))
except Exception as exc:
msg = "ERROR: Could not retrieve PNNs from CRIC, reason: %s" % str(exc)
raise Exception(msg)
return pnn_list
def __init__(self, **args):
StartPolicyInterface.__init__(self, **args)
self.args.setdefault('SliceType', 'NumberOfRuns')
self.args.setdefault('SliceSize', 1)
self.lumiType = "NumberOfLumis"
self.sites = []
if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv(
"WMCORE_USE_CRIC", False):
self.cric = CRIC()
else:
self.cric = None
self.siteDB = SiteDB()
def __init__(self, **args):
StartPolicyInterface.__init__(self, **args)
self.args.setdefault('SliceType', 'NumberOfFiles')
self.args.setdefault('SliceSize', 1)
self.lumiType = "NumberOfLumis"
# Initialize a list of sites where the data is
self.sites = []
# Initialize modifiers of the policy
self.blockBlackListModifier = []
self.cric = CRIC()
def pnns():
"""
Returns all PhEDEx node names, excluding Buffer endpoints
"""
cric = CRIC()
try:
pnn_list = sorted(cric.getAllPhEDExNodeNames(excludeBuffer=True))
except Exception as exc:
msg = "ERROR: Could not retrieve PNNs from CRIC, reason: %s" % str(exc)
raise Exception(msg)
return pnn_list
def sites():
"Return known CMS site list from SiteDB"
try:
# Download a list of all the sites from SiteDB, uses v2 API.
if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv("WMCORE_USE_CRIC", False):
cric = CRIC()
site_list = sorted(cric.getAllPSNs())
else:
sitedb = SiteDBJSON()
site_list = sorted(sitedb.getAllCMSNames())
except Exception as exc:
msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % str(exc)
raise Exception(msg)
return site_list
def getDNFromUserName(username, log, ckey=None, cert=None):
"""
Parse site string to know the fts server to use
"""
dn = ''
with newX509env(X509_USER_CERT=cert, X509_USER_KEY=ckey):
resourceCatalog = CRIC(logger=log)
try:
dn = resourceCatalog.userNameDn(username)
except:
log.error("CRIC URL cannot be accessed")
if not dn:
log.error("user does not exist")
return dn
def cmsSiteNames():
"""Get all cms sites"""
global __cmsSiteNames
if __cmsSiteNames:
return __cmsSiteNames
global __cric
if not __cric:
__cric = CRIC()
try:
__cmsSiteNames = __cric.getAllPSNs()
except Exception:
pass
return __cmsSiteNames
def cmsSiteNames():
"""Get all cms sites"""
global __cmsSiteNames
if __cmsSiteNames:
return __cmsSiteNames
global __cric
if not __cric:
__cric = CRIC()
try:
__cmsSiteNames = __cric.getAllPSNs()
except Exception:
pass
return __cmsSiteNames
def pnns():
"""
Returns all PhEDEx node names, excluding Buffer endpoints
"""
if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv("WMCORE_USE_CRIC", False):
sitedb = CRIC() # FIXME: rename it to cric
else:
sitedb = SiteDBJSON()
try:
pnn_list = sorted(sitedb.getAllPhEDExNodeNames(excludeBuffer=True))
except Exception as exc:
msg = "ERROR: Could not retrieve PNNs from SiteDB, reason: %s" % str(exc)
raise Exception(msg)
return pnn_list
def getDNFromUserName(username, log, ckey = None, cert = None):
"""
Parse site string to know the fts server to use
"""
dn = ''
with newX509env(X509_USER_CERT=cert,X509_USER_KEY=ckey):
configDict = {"cacheduration": 1, "pycurl": True} # cache duration is in hours
resourceCatalog = CRIC(logger=self.logger, configDict=configDict)
try:
dn = resourceCatalog.userNameDn(username)
except :
log.error("CRIC URL cannot be accessed")
if not dn:
log.error("user does not exist")
return dn
def __init__(self, msConfig, logger=None):
"""
Runs the basic setup and initialization for the MS Transferor module
:param microConfig: microservice configuration
"""
super(MSTransferor, self).__init__(msConfig, logger=logger)
# minimum percentage completion for dataset/blocks subscribed
self.msConfig.setdefault("minPercentCompletion", 99)
# minimum available storage to consider a resource good for receiving data
self.msConfig.setdefault("minimumThreshold", 1 * (1000**4)) # 1TB
# limit MSTransferor to this amount of requests per cycle
self.msConfig.setdefault("limitRequestsPerCycle", 500)
# Send warning messages for any data transfer above this threshold.
# Set to negative to ignore.
self.msConfig.setdefault("warningTransferThreshold",
100. * (1000**4)) # 100TB
# weight expression for the input replication rules
self.msConfig.setdefault("rucioRuleWeight", 'ddm_quota')
quotaAccount = self.msConfig["rucioAccount"]
self.rseQuotas = RSEQuotas(
quotaAccount,
self.msConfig["quotaUsage"],
minimumThreshold=self.msConfig["minimumThreshold"],
verbose=self.msConfig['verbose'],
logger=logger)
self.reqInfo = RequestInfo(self.msConfig, self.rucio, self.logger)
self.cric = CRIC(logger=self.logger)
self.inputMap = {
"InputDataset": "primary",
"MCPileup": "secondary",
"DataPileup": "secondary"
}
self.uConfig = {}
self.campaigns = {}
self.psn2pnnMap = {}
self.pnn2psnMap = {}
self.dsetCounter = 0
self.blockCounter = 0
# service name used to route alerts via AlertManager
self.alertServiceName = "ms-transferor"
self.alertManagerUrl = self.msConfig.get("alertManagerUrl", None)
self.alertManagerApi = AlertManagerAPI(self.alertManagerUrl,
logger=logger)
def execute(self, *args, **kwargs):
self.logger.info(
"Data discovery and splitting for %s using user-provided files" %
kwargs['task']['tm_taskname'])
userfiles = kwargs['task']['tm_user_files']
splitting = kwargs['task']['tm_split_algo']
total_units = kwargs['task']['tm_totalunits']
if not userfiles or splitting != 'FileBased':
if not userfiles:
msg = "No files specified to process for task %s." % kwargs[
'task']['tm_taskname']
if splitting != 'FileBased':
msg = "Data.splitting must be set to 'FileBased' when using a custom set of files."
raise TaskWorkerException(msg)
if hasattr(self.config.Sites, 'available'):
locations = self.config.Sites.available
else:
with self.config.TaskWorker.envForCMSWEB:
configDict = {
"cacheduration": 1,
"pycurl": True
} # cache duration is in hours
resourceCatalog = CRIC(logger=self.logger,
configDict=configDict)
locations = resourceCatalog.getAllPSNs()
userFileset = Fileset(name=kwargs['task']['tm_taskname'])
self.logger.info("There are %d files specified by the user." %
len(userfiles))
if total_units > 0:
self.logger.info("Will run over the first %d files." % total_units)
file_counter = 0
for userfile, idx in zip(userfiles, range(len(userfiles))):
newFile = File(userfile, size=1000, events=1)
newFile.setLocation(locations)
newFile.addRun(Run(1, idx))
newFile["block"] = 'UserFilesFakeBlock'
newFile["first_event"] = 1
newFile["last_event"] = 2
userFileset.addFile(newFile)
file_counter += 1
if total_units > 0 and file_counter >= total_units:
break
return Result(task=kwargs['task'], result=userFileset)
def __init__(self, **args):
StartPolicyInterface.__init__(self, **args)
self.args.setdefault('SliceType', 'NumberOfFiles')
self.args.setdefault('SliceSize', 1)
self.lumiType = "NumberOfLumis"
# Initialize a list of sites where the data is
self.sites = []
# Initialize modifiers of the policy
self.blockBlackListModifier = []
if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv(
"WMCORE_USE_CRIC", False):
self.cric = CRIC()
else:
self.cric = None
self.siteDB = SiteDB()
def __init__(self, msConfig, logger=None):
"""
Runs the basic setup and initialization for the MS Transferor module
:param microConfig: microservice configuration
"""
super(MSTransferor, self).__init__(msConfig, logger=logger)
# minimum percentage completion for dataset/blocks subscribed
self.msConfig.setdefault("minPercentCompletion", 99)
# minimum available storage to consider a resource good for receiving data
self.msConfig.setdefault("minimumThreshold", 1 * (1000**4)) # 1TB
# limit MSTransferor to this amount of requests per cycle
self.msConfig.setdefault("limitRequestsPerCycle", 500)
# Send warning messages for any data transfer above this threshold.
# Set to negative to ignore.
self.msConfig.setdefault("warningTransferThreshold",
100. * (1000**4)) # 100TB
# Set default email settings
self.msConfig.setdefault("toAddr",
"*****@*****.**")
self.msConfig.setdefault("fromAddr", "*****@*****.**")
self.msConfig.setdefault("smtpServer", "localhost")
quotaAccount = self.msConfig["rucioAccount"]
self.rseQuotas = RSEQuotas(
quotaAccount,
self.msConfig["quotaUsage"],
minimumThreshold=self.msConfig["minimumThreshold"],
verbose=self.msConfig['verbose'],
logger=logger)
self.reqInfo = RequestInfo(self.msConfig, self.rucio, self.logger)
self.cric = CRIC(logger=self.logger)
self.inputMap = {
"InputDataset": "primary",
"MCPileup": "secondary",
"DataPileup": "secondary"
}
self.uConfig = {}
self.campaigns = {}
self.psn2pnnMap = {}
self.pnn2psnMap = {}
self.dsetCounter = 0
self.blockCounter = 0
self.emailAlert = EmailAlert(self.msConfig)
def __init__(self, msConfig, logger=None):
"""
Runs the basic setup and initialization for the MS Transferor module
:param microConfig: microservice configuration
"""
super(MSTransferor, self).__init__(msConfig, logger)
# url for fetching the storage quota
self.msConfig.setdefault("detoxUrl",
"http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/SitesInfo.txt")
# minimum percentage completion for dataset/blocks subscribed
self.msConfig.setdefault("minPercentCompletion", 99)
# minimum available storage to consider a resource good for receiving data
self.msConfig.setdefault("minimumThreshold", 1 * (1000 ** 4)) # 1TB
# limit MSTransferor to this amount of requests per cycle
self.msConfig.setdefault("limitRequestsPerCycle", 500)
# Send warning messages for any data transfer above this threshold.
# Set to negative to ignore.
self.msConfig.setdefault("warningTransferThreshold", 100. * (1000 ** 4)) # 100TB
# Set default email settings
self.msConfig.setdefault("toAddr", "*****@*****.**")
self.msConfig.setdefault("fromAddr", "*****@*****.**")
self.msConfig.setdefault("smtpServer", "localhost")
# enable or not the PhEDEx requests auto-approval (!request_only)
if self.msConfig.setdefault("phedexRequestOnly", True):
self.msConfig["phedexRequestOnly"] = "y"
else:
self.msConfig["phedexRequestOnly"] = "n"
self.rseQuotas = RSEQuotas(self.msConfig['detoxUrl'], self.msConfig["quotaAccount"],
self.msConfig["quotaUsage"], useRucio=self.msConfig["useRucio"],
minimumThreshold=self.msConfig["minimumThreshold"],
verbose=self.msConfig['verbose'], logger=logger)
self.reqInfo = RequestInfo(msConfig, logger)
self.cric = CRIC(logger=self.logger)
self.inputMap = {"InputDataset": "primary",
"MCPileup": "secondary",
"DataPileup": "secondary"}
self.uConfig = {}
self.campaigns = {}
self.psn2pnnMap = {}
self.dsetCounter = 0
self.blockCounter = 0
self.emailAlert = EmailAlert(self.msConfig)
class MakeFakeFileSet(TaskAction):
"""This is needed to make WMCore.JobSplitting lib working...
do not like very much. Given that all is fake here I am
quite sure we only need total number of events said that I
set all the other parmas to dummy values. We may want to set
them in the future"""
def __init__(self, *args, **kwargs):
TaskAction.__init__(self, *args, **kwargs)
with self.config.TaskWorker.envForCMSWEB:
configDict = {
"cacheduration": 1,
"pycurl": True
} # cache duration is in hours
self.resourceCatalog = CRIC(logger=self.logger,
configDict=configDict)
def getListOfSites(self):
""" Get the list of sites to use for PrivateMC workflows.
For the moment we are filtering out T1_ since they are precious resources
and don't want to overtake production (WMAgent) jobs there. In the
future we would like to take this list from the SSB.
"""
with self.config.TaskWorker.envForCMSWEB:
sites = self.resourceCatalog.getAllPSNs()
filteredSites = [site for site in sites if not site.startswith("T1_")]
return filteredSites
#even though args is not used we call "handler.actionWork(args, kwargs)" in Handler
def execute(self, *args, **kwargs): #pylint: disable=unused-argument
# since https://github.com/dmwm/CRABServer/issues/5633 totalunits can be a float
# but that would confuse WMCore, therefore cast to int
totalevents = int(kwargs['task']['tm_totalunits'])
firstEvent = 1
lastEvent = totalevents
firstLumi = 1
lastLumi = 10
# Set a default of 100 events per lumi. This is set as a task
# property, as the splitting considers it independently of the file
# information provided by the fake dataset.
if not kwargs['task']['tm_events_per_lumi']:
kwargs['task']['tm_events_per_lumi'] = 100
#MC comes with only one MCFakeFile
singleMCFileset = Fileset(name="MCFakeFileSet")
newFile = File("MCFakeFile", size=1000, events=totalevents)
newFile.setLocation(self.getListOfSites())
newFile.addRun(Run(1, *range(firstLumi, lastLumi + 1)))
newFile["block"] = 'MCFakeBlock'
newFile["first_event"] = firstEvent
newFile["last_event"] = lastEvent
singleMCFileset.addFile(newFile)
return Result(task=kwargs['task'], result=singleMCFileset)
def getDatasetLocationsByAccount(self, dataset, account):
"""
Returns the dataset locations for the given account in terms of computing element (not RSE name).
This function assumes that the returned RSE expression includes only one RSE
"""
try:
rules = self.list_did_rules(self.scope, dataset)
RSEs = []
for rule in rules:
if rule['account'] == account:
RSEs.append(rule['rse_expression'])
#RSE to CE conversion
cric = CRIC()
CEs = cric.PNNstoPSNs(RSEs)
except Exception as e:
print "Exception while getting the dataset location"
print(str(e))
return []
return CEs
class MakeFakeFileSet(TaskAction):
"""This is needed to make WMCore.JobSplitting lib working...
do not like very much. Given that all is fake here I am
quite sure we only need total number of events said that I
set all the other parmas to dummy values. We may want to set
them in the future"""
def __init__(self, *args, **kwargs):
TaskAction.__init__(self, *args, **kwargs)
with self.config.TaskWorker.envForCMSWEB:
configDict = {"cacheduration": 1, "pycurl": True} # cache duration is in hours
self.resourceCatalog = CRIC(logger=self.logger, configDict=configDict)
def getListOfSites(self):
""" Get the list of sites to use for PrivateMC workflows.
For the moment we are filtering out T1_ since they are precious resources
and don't want to overtake production (WMAgent) jobs there. In the
future we would like to take this list from the SSB.
"""
with self.config.TaskWorker.envForCMSWEB:
sites = self.resourceCatalog.getAllPSNs()
filteredSites = [site for site in sites if not site.startswith("T1_")]
return filteredSites
#even though args is not used we call "handler.actionWork(args, kwargs)" in Handler
def execute(self, *args, **kwargs): #pylint: disable=unused-argument
# since https://github.com/dmwm/CRABServer/issues/5633 totalunits can be a float
# but that would confuse WMCore, therefore cast to int
totalevents = int(kwargs['task']['tm_totalunits'])
firstEvent = 1
lastEvent = totalevents
firstLumi = 1
lastLumi = 10
# Set a default of 100 events per lumi. This is set as a task
# property, as the splitting considers it independently of the file
# information provided by the fake dataset.
if not kwargs['task']['tm_events_per_lumi']:
kwargs['task']['tm_events_per_lumi'] = 100
#MC comes with only one MCFakeFile
singleMCFileset = Fileset(name = "MCFakeFileSet")
newFile = File("MCFakeFile", size = 1000, events = totalevents)
newFile.setLocation(self.getListOfSites())
newFile.addRun(Run(1, *range(firstLumi, lastLumi + 1)))
newFile["block"] = 'MCFakeBlock'
newFile["first_event"] = firstEvent
newFile["last_event"] = lastEvent
singleMCFileset.addFile(newFile)
return Result(task=kwargs['task'], result=singleMCFileset)
def __init__(self, **args):
PolicyInterface.__init__(self, **args)
self.workQueueElements = []
self.wmspec = None
self.team = None
self.initialTask = None
self.splitParams = None
self.dbs_pool = {}
self.data = {}
self.lumi = None
self.couchdb = None
self.rejectedWork = [] # List of inputs that were rejected
self.badWork = [
] # list of bad work unit (e.g. without any valid files)
self.pileupData = {}
self.cric = CRIC()
if usingRucio():
self.rucio = Rucio(self.args['rucioAcct'],
configDict={'logger': self.logger})
else:
self.phedex = PhEDEx() # this will go away eventually
def wrapped_func(*args, **kwargs):
if 'cric' in services and (
not args[0].allCMSNames.sites or
(args[0].allCMSNames.cachetime + 1800 < mktime(gmtime()))):
args[0].allCMSNames = CMSSitesCache(sites=CRIC().getAllPSNs(),
cachetime=mktime(gmtime()))
args[0].allPNNNames = CMSSitesCache(
sites=CRIC().getAllPhEDExNodeNames(),
cachetime=mktime(gmtime()))
if 'centralconfig' in services and (
not args[0].centralcfg.centralconfig or
(args[0].centralcfg.cachetime + 1800 < mktime(gmtime()))):
args[0].centralcfg = ConfigCache(
centralconfig=getCentralConfig(
extconfigurl=args[0].config.extconfigurl,
mode=args[0].config.mode),
cachetime=mktime(gmtime()))
if 'servercert' in services:
args[0].serverCert = serverCert
args[0].serverKey = serverKey
return func(*args, **kwargs)
def cmsSiteNames():
"""Get all cms sites"""
global __cmsSiteNames
if __cmsSiteNames:
return __cmsSiteNames
logging.info("cmsSiteNames Using CRIC Service: %s", __USE_CRIC)
global __sitedb
if not __sitedb:
if __USE_CRIC:
from WMCore.Services.CRIC.CRIC import CRIC
__sitedb = CRIC()
else:
from WMCore.Services.SiteDB.SiteDB import SiteDBJSON as SiteDB
__sitedb = SiteDB()
try:
if __USE_CRIC:
__cmsSiteNames = __sitedb.getAllPSNs()
else:
__cmsSiteNames = __sitedb.getAllCMSNames()
except Exception:
pass
return __cmsSiteNames
def __init__(self, msConfig, logger=None):
"""
Runs the basic setup and initialization for the MS Transferor module
:param microConfig: microservice configuration
"""
super(MSTransferor, self).__init__(msConfig, logger)
# url for fetching the storage quota
self.msConfig.setdefault(
"detoxUrl",
"http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/SitesInfo.txt")
# minimum percentage completion for dataset/blocks subscribed
self.msConfig.setdefault("minPercentCompletion", 99)
# minimum available storage to consider a resource good for receiving data
self.msConfig.setdefault("minimumThreshold", 1 * (1000**4)) # 1TB
# limit MSTransferor to this amount of requests per cycle
self.msConfig.setdefault("limitRequestsPerCycle", 500)
self.rseQuotas = RSEQuotas(
self.msConfig['detoxUrl'],
self.msConfig["quotaAccount"],
self.msConfig["quotaUsage"],
useRucio=self.msConfig["useRucio"],
minimumThreshold=self.msConfig["minimumThreshold"],
verbose=self.msConfig['verbose'],
logger=logger)
self.reqInfo = RequestInfo(msConfig, logger)
self.cric = CRIC(logger=self.logger)
self.inputMap = {
"InputDataset": "primary",
"MCPileup": "secondary",
"DataPileup": "secondary"
}
self.uConfig = {}
self.campaigns = {}
self.psn2pnnMap = {}
self.dsetCounter = 0
self.blockCounter = 0
def execute(self, *args, **kwargs):
self.logger.info("Data discovery and splitting for %s using user-provided files" % kwargs['task']['tm_taskname'])
userfiles = kwargs['task']['tm_user_files']
splitting = kwargs['task']['tm_split_algo']
total_units = kwargs['task']['tm_totalunits']
if not userfiles or splitting != 'FileBased':
if not userfiles:
msg = "No files specified to process for task %s." % kwargs['task']['tm_taskname']
if splitting != 'FileBased':
msg = "Data.splitting must be set to 'FileBased' when using a custom set of files."
raise TaskWorkerException(msg)
if hasattr(self.config.Sites, 'available'):
locations = self.config.Sites.available
else:
with self.config.TaskWorker.envForCMSWEB :
configDict = {"cacheduration": 1, "pycurl": True} # cache duration is in hours
resourceCatalog = CRIC(logger=self.logger, configDict=configDict)
locations = resourceCatalog.getAllPSNs()
userFileset = Fileset(name = kwargs['task']['tm_taskname'])
self.logger.info("There are %d files specified by the user." % len(userfiles))
if total_units > 0:
self.logger.info("Will run over the first %d files." % total_units)
file_counter = 0
for userfile, idx in zip(userfiles, range(len(userfiles))):
newFile = File(userfile, size = 1000, events = 1)
newFile.setLocation(locations)
newFile.addRun(Run(1, idx))
newFile["block"] = 'UserFilesFakeBlock'
newFile["first_event"] = 1
newFile["last_event"] = 2
userFileset.addFile(newFile)
file_counter += 1
if total_units > 0 and file_counter >= total_units:
break
return Result(task = kwargs['task'], result = userFileset)
def __init__(self, **args):
# We need to pop this object instance from args because otherwise
# the super class blows up when doing a deepcopy(args)
self.rucio = args.pop("rucioObject", None)
PolicyInterface.__init__(self, **args)
self.workQueueElements = []
self.wmspec = None
self.team = None
self.initialTask = None
self.splitParams = None
self.dbs_pool = {}
self.data = {}
self.lumi = None
self.couchdb = None
self.rejectedWork = [] # List of inputs that were rejected
self.badWork = [
] # list of bad work unit (e.g. without any valid files)
self.pileupData = {}
self.cric = CRIC()
# FIXME: for the moment, it will always use the default value
self.rucioAcct = self.args.get("rucioAcct", "wmcore_transferor")
if not self.rucio:
self.rucio = Rucio(self.rucioAcct,
configDict={'logger': self.logger})
def testConfig(self):
"""
Test service attributes and the override mechanism
"""
self.assertEqual(self.myCRIC['endpoint'], 'https://cms-cric.cern.ch/')
self.assertEqual(self.myCRIC['cacheduration'], 1)
self.assertEqual(self.myCRIC['accept_type'], 'application/json')
self.assertEqual(self.myCRIC['content_type'], 'application/json')
self.assertEqual(self.myCRIC['requests'].pycurl, None)
newParams = {"cacheduration": 100, "pycurl": True}
cric = CRIC(url='https://BLAH.cern.ch/', configDict=newParams)
self.assertEqual(cric['endpoint'], 'https://BLAH.cern.ch/')
self.assertEqual(cric['cacheduration'], newParams['cacheduration'])
self.assertTrue(cric['requests'].pycurl)
{"site_name": "T2_XX_SiteC", "type": "phedex", "alias": "T2_XX_SiteC"},
],
'data-processing': [
{"phedex_name": "T2_XX_SiteA", "psn_name": "T2_XX_SiteA"},
{"phedex_name": "T2_XX_SiteB", "psn_name": "T2_XX_SiteB"},
{"phedex_name": "T2_XX_SiteC", "psn_name": "T2_XX_SiteC"},
]
}
lookup = {}
outFile = 'CRICMockData.json'
outFilename = os.path.join(getTestBase(), '..', 'data', 'Mock', outFile)
print("Creating the file %s with mocked CRIC data" % outFilename)
cric = CRIC()
for callname in calls:
print("Querying %s ..." % callname)
try:
if callname == 'people':
result = cric._CRICUserQuery(callname)
else:
result = cric._CRICSiteQuery(callname)
except HTTPError as exc:
result = 'Raises HTTPError'
if callname in ["site-names", "data-processing"]:
result.extend(additionals[callname])
else:
result = result
lookup.update({callname:result})
class Dataset(StartPolicyInterface):
"""Split elements into datasets"""
def __init__(self, **args):
StartPolicyInterface.__init__(self, **args)
self.args.setdefault('SliceType', 'NumberOfRuns')
self.args.setdefault('SliceSize', 1)
self.lumiType = "NumberOfLumis"
self.sites = []
self.cric = CRIC()
def split(self):
"""Apply policy to spec"""
work = set() if self.args['SliceType'] == 'NumberOfRuns' else 0
numFiles = 0
numEvents = 0
numLumis = 0
datasetPath = self.initialTask.getInputDatasetPath()
# dataset splitting can't have its data selection overridden
if self.data and self.data.keys() != [datasetPath]:
raise RuntimeError("Can't provide different data to split with")
blocks = self.validBlocks(self.initialTask, self.dbs())
if not blocks:
return
for block in blocks:
if self.args['SliceType'] == 'NumberOfRuns':
work = work.union(block[self.args['SliceType']])
else:
work += float(block[self.args['SliceType']])
numLumis += int(block[self.lumiType])
numFiles += int(block['NumberOfFiles'])
numEvents += int(block['NumberOfEvents'])
if self.args['SliceType'] == 'NumberOfRuns':
numJobs = ceil(len(work) / float(self.args['SliceSize']))
else:
numJobs = ceil(float(work) / float(self.args['SliceSize']))
# parentage
parentFlag = True if self.initialTask.parentProcessingFlag() else False
self.newQueueElement(Inputs={datasetPath: self.data.get(datasetPath, [])},
ParentFlag=parentFlag,
NumberOfLumis=numLumis,
NumberOfFiles=numFiles,
NumberOfEvents=numEvents,
Jobs=numJobs,
NoInputUpdate=self.initialTask.getTrustSitelists().get('trustlists'),
NoPileupUpdate=self.initialTask.getTrustSitelists().get('trustPUlists')
)
def validate(self):
"""Check args and spec work with block splitting"""
StartPolicyInterface.validateCommon(self)
if not self.initialTask.inputDataset():
raise WorkQueueWMSpecError(self.wmspec, 'No input dataset')
def validBlocks(self, task, dbs):
"""Return blocks that pass the input data restriction"""
datasetPath = task.getInputDatasetPath()
Lexicon.dataset(datasetPath) # check dataset name
validBlocks = []
locations = None
blockWhiteList = task.inputBlockWhitelist()
blockBlackList = task.inputBlockBlacklist()
runWhiteList = task.inputRunWhitelist()
runBlackList = task.inputRunBlacklist()
lumiMask = task.getLumiMask()
if lumiMask:
maskedBlocks = self.getMaskedBlocks(task, dbs, datasetPath)
for blockName in dbs.listFileBlocks(datasetPath):
# check block restrictions
if blockWhiteList and blockName not in blockWhiteList:
continue
if blockName in blockBlackList:
continue
blockSummary = dbs.getDBSSummaryInfo(block=blockName)
if int(blockSummary.get('NumberOfFiles', 0)) == 0:
logging.warning("Block %s being rejected for lack of valid files to process", blockName)
self.badWork.append(blockName)
continue
if self.args['SliceType'] == 'NumberOfRuns':
blockSummary['NumberOfRuns'] = dbs.listRuns(block=blockName)
# check lumi restrictions
if lumiMask:
if blockName not in maskedBlocks:
logging.warning("Block %s doesn't pass the lumi mask constraints", blockName)
self.rejectedWork.append(blockName)
continue
acceptedLumiCount = sum([len(maskedBlocks[blockName][lfn].getLumis()) for lfn in maskedBlocks[blockName]])
ratioAccepted = 1. * acceptedLumiCount / float(blockSummary['NumberOfLumis'])
maskedRuns = [maskedBlocks[blockName][lfn].getRuns() for lfn in maskedBlocks[blockName]]
acceptedRuns = set(lumiMask.getRuns()).intersection(set().union(*maskedRuns))
blockSummary['NumberOfFiles'] = len(maskedBlocks[blockName])
blockSummary['NumberOfEvents'] = float(blockSummary['NumberOfEvents']) * ratioAccepted
blockSummary[self.lumiType] = acceptedLumiCount
blockSummary['NumberOfRuns'] = acceptedRuns
# check run restrictions
elif runWhiteList or runBlackList:
runs = set(dbs.listRuns(block=blockName))
# multi run blocks need special account, requires more DBS calls
recalculateLumiCounts = True if len(runs) > 1 else False
# apply blacklist and whitelist
runs = runs.difference(runBlackList)
if runWhiteList:
runs = runs.intersection(runWhiteList)
# any runs left are ones we will run on, if none ignore block
if not runs:
logging.warning("Block %s doesn't pass the runs constraints", blockName)
self.rejectedWork.append(blockName)
continue
if recalculateLumiCounts:
# Recalculate the number of files, lumis and ~events accepted
acceptedLumiCount = 0
acceptedEventCount = 0
acceptedFileCount = 0
fileInfo = dbs.listFilesInBlock(fileBlockName=blockName)
for fileEntry in fileInfo:
acceptedFile = False
for lumiInfo in fileEntry['LumiList']:
if lumiInfo['RunNumber'] in runs:
acceptedFile = True
acceptedLumiCount += len(lumiInfo['LumiSectionNumber'])
if acceptedFile:
acceptedFileCount += 1
acceptedEventCount += fileEntry['NumberOfEvents']
else:
acceptedLumiCount = blockSummary["NumberOfLumis"]
acceptedFileCount = blockSummary['NumberOfFiles']
acceptedEventCount = blockSummary['NumberOfEvents']
blockSummary[self.lumiType] = acceptedLumiCount
blockSummary['NumberOfFiles'] = acceptedFileCount
blockSummary['NumberOfEvents'] = acceptedEventCount
blockSummary['NumberOfRuns'] = runs
validBlocks.append(blockSummary)
if locations is None:
locations = set(dbs.listFileBlockLocation(blockName))
else:
locations = locations.intersection(dbs.listFileBlockLocation(blockName))
# all needed blocks present at these sites
if task.getTrustSitelists().get('trustlists'):
siteWhitelist = task.siteWhitelist()
siteBlacklist = task.siteBlacklist()
self.sites = makeLocationsList(siteWhitelist, siteBlacklist)
self.data[datasetPath] = self.sites
elif locations:
self.data[datasetPath] = list(set(self.cric.PNNstoPSNs(locations)))
return validBlocks
def setUp(self):
"""
Setup for unit tests
"""
super(CRICTest, self).setUp()
self.myCRIC = CRIC()
def __init__(self, *args, **kwargs):
TaskAction.__init__(self, *args, **kwargs)
with self.config.TaskWorker.envForCMSWEB:
configDict = {"cacheduration": 1, "pycurl": True} # cache duration is in hours
self.resourceCatalog = CRIC(logger=self.logger, configDict=configDict)
class CRICTest(EmulatedUnitTestCase):
"""
Unit tests for SiteScreening module
"""
def __init__(self, methodName='runTest'):
super(CRICTest, self).__init__(methodName=methodName)
def setUp(self):
"""
Setup for unit tests
"""
super(CRICTest, self).setUp()
self.myCRIC = CRIC()
def testConfig(self):
"""
Test service attributes and the override mechanism
"""
self.assertEqual(self.myCRIC['endpoint'], 'https://cms-cric.cern.ch/')
self.assertEqual(self.myCRIC['cacheduration'], 1)
self.assertEqual(self.myCRIC['accept_type'], 'application/json')
self.assertEqual(self.myCRIC['content_type'], 'application/json')
self.assertEqual(self.myCRIC['requests'].pycurl, None)
newParams = {"cacheduration": 100, "pycurl": True}
cric = CRIC(url='https://BLAH.cern.ch/', configDict=newParams)
self.assertEqual(cric['endpoint'], 'https://BLAH.cern.ch/')
self.assertEqual(cric['cacheduration'], newParams['cacheduration'])
self.assertTrue(cric['requests'].pycurl)
@attr('integration')
def testWhoAmI(self):
"""
Test user mapping information from the request headers
"""
print("This test only works with service certificates, not user proxies")
self.assertTrue(self.myCRIC.whoAmI()) # empty list if nothing
def testUserNameDN(self):
"""
Tests user name to DN lookup
"""
expectedDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=amaltaro/CN=718748/CN=Alan Malta Rodrigues"
username = "******"
dn = self.myCRIC.userNameDn(username)
self.assertEqual(dn, expectedDN)
def testUserNameDNWithApostrophe(self):
"""
Test DN lookup with an apostrophe
"""
expectedDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=gdimperi/CN=728001/CN=Giulia D'Imperio"
username = "******"
dn = self.myCRIC.userNameDn(username)
self.assertEqual(dn, expectedDN)
def testGetAllPSNs(self):
"""
Test API which fetches all PSNs
"""
result = self.myCRIC.getAllPSNs()
self.assertNotIn('T1_US_FNAL_Disk', result)
self.assertNotIn('T1_US_FNAL_MSS', result)
self.assertIn('T1_US_FNAL', result)
self.assertIn('T2_CH_CERN', result)
self.assertIn('T2_CH_CERN_HLT', result)
t1s = [psn for psn in result if psn.startswith('T1_')]
self.assertTrue(len(t1s) < 10)
t2s = [psn for psn in result if psn.startswith('T2_')]
self.assertTrue(len(t2s) > 30)
t3s = [psn for psn in result if psn.startswith('T3_')]
self.assertTrue(len(t3s) > 10)
self.assertTrue(len(result) > 70)
return
def testGetAllPhEDExNodeNames(self):
"""
Test API to get all PhEDEx Node Names
"""
result = self.myCRIC.getAllPhEDExNodeNames(excludeBuffer=True)
self.assertFalse([pnn for pnn in result if pnn.endswith('_Buffer')])
result = self.myCRIC.getAllPhEDExNodeNames(excludeBuffer=False)
self.assertTrue(len([pnn for pnn in result if pnn.endswith('_Buffer')]) > 5)
result = self.myCRIC.getAllPhEDExNodeNames(pattern='T1.*', excludeBuffer=True)
self.assertFalse([pnn for pnn in result if not pnn.startswith('T1_')])
self.assertTrue(len(result) > 10)
result = self.myCRIC.getAllPhEDExNodeNames(excludeBuffer=True)
self.assertTrue([pnn for pnn in result if pnn.startswith('T1_')])
self.assertTrue([pnn for pnn in result if pnn.startswith('T2_')])
self.assertTrue([pnn for pnn in result if pnn.startswith('T3_')])
self.assertTrue(len(result) > 60)
result1 = self.myCRIC.getAllPhEDExNodeNames(pattern='.*', excludeBuffer=True)
self.assertTrue([pnn for pnn in result1 if pnn.startswith('T1_')])
self.assertTrue([pnn for pnn in result1 if pnn.startswith('T2_')])
self.assertTrue([pnn for pnn in result1 if pnn.startswith('T3_')])
self.assertTrue(len(result) > 60)
self.assertItemsEqual(result, result1)
# test a few PSNs
self.assertNotIn('T1_US_FNAL', result)
self.assertNotIn('T2_CH_CERN_HLT', result)
# test a few PNNs
self.assertIn('T1_US_FNAL_Disk', result)
self.assertIn('T1_US_FNAL_MSS', result)
self.assertIn('T2_CH_CERN', result)
return
def testPNNstoPSNs(self):
"""
Test mapping PhEDEx Node Names to Processing Site Names
"""
self.assertEqual(self.myCRIC.PNNstoPSNs([]), [])
self.assertItemsEqual(self.myCRIC.PNNstoPSNs(['T1_US_FNAL_MSS']), [])
self.assertItemsEqual(self.myCRIC.PNNstoPSNs(['T1_US_FNAL_Disk']), ['T1_US_FNAL'])
pnns = ['T1_US_FNAL_Disk', 'T1_US_FNAL_Buffer', 'T1_US_FNAL_MSS']
self.assertItemsEqual(self.myCRIC.PNNstoPSNs(pnns), ['T1_US_FNAL'])
pnns = ['T2_CH_CERN', 'T2_DE_DESY']
psns = ['T2_CH_CERN_HLT', 'T2_DE_DESY', 'T2_CH_CERN']
self.assertItemsEqual(self.myCRIC.PNNstoPSNs(pnns), psns)
psns = ['T2_UK_London_IC', 'T3_UK_London_QMUL', 'T3_UK_SGrid_Oxford', 'T3_UK_London_RHUL', 'T3_UK_ScotGrid_GLA']
self.assertItemsEqual(self.myCRIC.PNNstoPSNs('T2_UK_London_IC'), psns)
pnns = ['T2_UK_London_IC', 'T2_US_Purdue']
self.assertItemsEqual(self.myCRIC.PNNstoPSNs(pnns), psns + ['T2_US_Purdue'])
return
def testPSNstoPNNs(self):
"""
Test mapping Processing Site Names to PhEDEx Node Names
"""
self.assertEqual(self.myCRIC.PSNstoPNNs([]), [])
# test a few PNNs
self.assertItemsEqual(self.myCRIC.PSNstoPNNs('T1_US_FNAL_Disk'), [])
self.assertItemsEqual(self.myCRIC.PSNstoPNNs(['T1_DE_KIT_MSS']), [])
# test a few PSNs
self.assertItemsEqual(self.myCRIC.PSNstoPNNs(['T1_US_FNAL']), ['T1_US_FNAL_Disk', 'T3_US_FNALLPC'])
self.assertItemsEqual(self.myCRIC.PSNstoPNNs(['T2_CH_CERN_HLT']), ['T2_CH_CERN'])
pnns = ['T2_CH_CERN', 'T2_CH_CERNBOX']
self.assertItemsEqual(self.myCRIC.PSNstoPNNs(['T2_CH_CERN']), pnns)
psns = ['T2_CH_CERN', 'T2_CH_CERN_HLT']
self.assertItemsEqual(self.myCRIC.PSNstoPNNs(psns), pnns)
self.assertItemsEqual(self.myCRIC.PSNstoPNNs(['T2_UK_London_IC']), ['T2_UK_London_IC'])
pnns = ['T2_UK_London_IC', 'T2_UK_London_Brunel', 'T2_UK_SGrid_RALPP']
self.assertItemsEqual(self.myCRIC.PSNstoPNNs('T3_UK_London_QMUL'), pnns)
self.assertItemsEqual(self.myCRIC.PSNstoPNNs('T3_UK_ScotGrid_GLA'), pnns)
pnns = ['T2_UK_London_IC', 'T2_UK_SGrid_RALPP']
self.assertItemsEqual(self.myCRIC.PSNstoPNNs('T3_UK_SGrid_Oxford'), pnns)
self.assertItemsEqual(self.myCRIC.PSNstoPNNs('T3_UK_London_RHUL'), pnns)
self.assertItemsEqual(self.myCRIC.PSNstoPNNs(['T2_US_Purdue']), ['T2_US_Purdue'])
return
def testPSNtoPNNMap(self):
"""
Test API to get a map of PSNs to PNNs
"""
with self.assertRaises(TypeError):
self.myCRIC.PSNtoPNNMap(1)
with self.assertRaises(TypeError):
self.myCRIC.PSNtoPNNMap({'config': 'blah'})
with self.assertRaises(TypeError):
self.myCRIC.PSNtoPNNMap(['blah'])
result = self.myCRIC.PSNtoPNNMap()
self.assertTrue([psn for psn in result.keys() if psn.startswith('T1_')])
self.assertTrue([psn for psn in result.keys() if psn.startswith('T2_')])
self.assertTrue([psn for psn in result.keys() if psn.startswith('T3_')])
self.assertTrue(len(result) > 50)
result = self.myCRIC.PSNtoPNNMap(psnPattern='T1.*')
self.assertFalse([psn for psn in result.keys() if not psn.startswith('T1_')])
self.assertTrue(len(result) < 10)
result = self.myCRIC.PSNtoPNNMap(psnPattern='T2.*')
self.assertFalse([psn for psn in result.keys() if not psn.startswith('T2_')])
self.assertTrue(len(result) > 10)
result = self.myCRIC.PSNtoPNNMap(psnPattern='T3.*')
self.assertFalse([psn for psn in result.keys() if not psn.startswith('T3_')])
self.assertTrue(len(result) > 10)
result = self.myCRIC.PSNtoPNNMap('T2_CH_CERN$')
self.assertItemsEqual(result.keys(), ['T2_CH_CERN'])
self.assertItemsEqual(result['T2_CH_CERN'], ['T2_CH_CERNBOX', 'T2_CH_CERN'])
# test a exact site name, which is treated as a regex and yields a confusing result!!!
result = self.myCRIC.PSNtoPNNMap('T2_CH_CERN')
self.assertItemsEqual(result.keys(), ['T2_CH_CERN', 'T2_CH_CERN_HLT'])
self.assertItemsEqual(result['T2_CH_CERN'], ['T2_CH_CERNBOX', 'T2_CH_CERN'])
self.assertItemsEqual(result['T2_CH_CERN_HLT'], ['T2_CH_CERN'])
result = self.myCRIC.PSNtoPNNMap('T2_CH_CERN_HLT')
self.assertItemsEqual(result.keys(), ['T2_CH_CERN_HLT'])
self.assertItemsEqual(result['T2_CH_CERN_HLT'], ['T2_CH_CERN'])
result = self.myCRIC.PSNtoPNNMap('T1_US_FNAL')
self.assertItemsEqual(result.keys(), ['T1_US_FNAL'])
self.assertItemsEqual(result['T1_US_FNAL'], ['T1_US_FNAL_Disk', 'T3_US_FNALLPC'])
# test a PNN as input, expecting nothing mapped to it
self.assertItemsEqual(self.myCRIC.PSNtoPNNMap('T1_US_FNAL_Disk'), {})
return
