def testIsTapeRSE(self):
"""
Test the `isTapeRSE` utilitarian function
"""
self.assertTrue(Rucio.isTapeRSE("T1_US_FNAL_Tape"))
self.assertFalse(Rucio.isTapeRSE("T1_US_FNAL_Disk"))
self.assertFalse(Rucio.isTapeRSE("T1_US_FNAL_Disk_Test"))
self.assertFalse(Rucio.isTapeRSE("T1_US_FNAL_Tape_Test"))
self.assertFalse(Rucio.isTapeRSE(""))
def testDropTapeRSEs(self):
"""
Test the `dropTapeRSEs` utilitarian function
"""
tapeOnly = ["T1_US_FNAL_Tape", "T1_ES_PIC_Tape"]
diskOnly = ["T1_US_FNAL_Disk", "T1_US_FNAL_Disk_Test", "T2_CH_CERN"]
mixed = ["T1_US_FNAL_Tape", "T1_US_FNAL_Disk", "T1_US_FNAL_Disk_Test", "T1_ES_PIC_Tape"]
self.assertItemsEqual(Rucio.dropTapeRSEs(tapeOnly), [])
self.assertItemsEqual(Rucio.dropTapeRSEs(diskOnly), diskOnly)
self.assertItemsEqual(Rucio.dropTapeRSEs(mixed), ["T1_US_FNAL_Disk", "T1_US_FNAL_Disk_Test"])
def testMetaDataValidation(self):
"""
Test the `validateMetaData` validation function
"""
for thisProj in Rucio.RUCIO_VALID_PROJECT:
response = Rucio.validateMetaData("any_DID_name", dict(project=thisProj), self.myRucio.logger)
self.assertTrue(response)
# test with no "project" meta data at all
response = Rucio.validateMetaData("any_DID_name", dict(), self.myRucio.logger)
self.assertTrue(response)
# now an invalid "project" meta data
response = Rucio.validateMetaData("any_DID_name", dict(project="mistake"), self.myRucio.logger)
self.assertFalse(response)
def testConfig(self):
"""
Test service attributes and the override mechanism
"""
for key in self.defaultArgs:
self.assertEqual(getattr(self.myRucio.cli, key),
self.defaultArgs[key])
self.assertTrue(
getattr(self.myRucio.cli,
"user_agent").startswith("wmcore-client/"))
self.assertTrue(
getattr(self.client, "user_agent").startswith("rucio-clients/"))
newParams = {
"host": 'http://cms-rucio-int.cern.ch',
"auth_host": 'https://cms-rucio-auth-int.cern.ch',
"auth_type": "x509",
"account": self.acct,
"ca_cert": False,
"timeout": 5
}
newKeys = list(newParams)
rucio = Rucio.Rucio(newParams['account'],
hostUrl=newParams['host'],
authUrl=newParams['auth_host'],
configDict=newParams)
for key in newKeys:
self.assertEqual(getattr(rucio.cli, key), newParams[key])
def setUp(self):
"init test class"
self.maxDiff = None
self.msConfig = {
"verbose": True,
"interval": 1 * 60,
"services": ['ruleCleaner'],
"rucioWmaAcct": 'wma_test',
"rucioAccount": 'wma_test',
'reqmgr2Url': 'https://cmsweb-testbed.cern.ch/reqmgr2',
'msOutputUrl': 'https://cmsweb-testbed.cern.ch/ms-output',
'reqmgrCacheUrl':
'https://cmsweb-testbed.cern.ch/couchdb/reqmgr_workload_cache',
'phedexUrl':
'https://cmsweb-testbed.cern.ch/phedex/datasvc/json/prod',
'dbsUrl':
'https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader',
'rucioUrl': 'http://cmsrucio-int.cern.ch',
'rucioAuthUrl': 'https://cmsrucio-auth-int.cern.ch'
}
self.creds = {
"client_cert": os.getenv("X509_USER_CERT", "Unknown"),
"client_key": os.getenv("X509_USER_KEY", "Unknown")
}
self.rucioConfigDict = {
"rucio_host": self.msConfig['rucioUrl'],
"auth_host": self.msConfig['rucioAuthUrl'],
"auth_type": "x509",
"account": self.msConfig['rucioAccount'],
"ca_cert": False,
"timeout": 30,
"request_retries": 3,
"creds": self.creds
}
self.reqStatus = ['announced', 'aborted-completed', 'rejected']
self.msRuleCleaner = MSRuleCleaner(self.msConfig)
self.msRuleCleaner.resetCounters()
self.msRuleCleaner.rucio = Rucio.Rucio(
self.msConfig['rucioAccount'],
hostUrl=self.rucioConfigDict['rucio_host'],
authUrl=self.rucioConfigDict['auth_host'],
configDict=self.rucioConfigDict)
self.taskChainFile = getTestFile(
'data/ReqMgr/requests/Static/TaskChainRequestDump.json')
self.stepChainFile = getTestFile(
'data/ReqMgr/requests/Static/StepChainRequestDump.json')
self.reqRecordsFile = getTestFile(
'data/ReqMgr/requests/Static/BatchRequestsDump.json')
with open(self.reqRecordsFile) as fd:
self.reqRecords = json.load(fd)
with open(self.taskChainFile) as fd:
self.taskChainReq = json.load(fd)
with open(self.stepChainFile) as fd:
self.stepChainReq = json.load(fd)
super(MSRuleCleanerTest, self).setUp()
def createProductionRucio(self, account=None):
"""Create an object pointing to the Rucio Production instance"""
# NOTE: do your best to avoid testing against production!!!
newParams = {"auth_type": "x509", "ca_cert": False, "timeout": 50}
prodRucio = Rucio.Rucio(account,
hostUrl='http://cms-rucio.cern.ch',
authUrl='https://cms-rucio-auth.cern.ch',
configDict=newParams)
return prodRucio
def setUp(self):
""" init test class """
self.maxDiff = None
self.msConfig = {
'couch_host': 'https://cmsweb-testbed.cern.ch/couchdb',
'couch_wmstats_db': 'wmstats',
'enableRealMode': False,
'interval': 28800,
'limitRSEsPerInstance': 200,
'limitTiersPerInstance': ['T1', 'T2', 'T3'],
'limitFilesPerRSE': -1,
'manager': 'WMCore.MicroService.MSManager.MSManager',
'object': 'WMCore.MicroService.Service.RestApiHub.RestApiHub',
'reqmgr2Url': 'https://cmsweb-testbed.cern.ch/reqmgr2',
'rseExpr': 'cms_type=real&rse_type=DISK',
'rucioAccount': 'wmcore_transferor',
'rucioUrl': 'http://cms-rucio-int.cern.ch',
'rucioAuthUrl': 'https://cms-rucio-auth-int.cern.ch',
'rucioConMon': 'https://cmsweb-testbed.cern.ch/rucioconmon',
'services': ['unmerged'],
'verbose': True,
'wmstatsUrl': 'https://cmsweb-testbed.cern.ch/wmstatsserver',
'dirFilterIncl': [],
'dirFilterExcl': [],
'emulateGfal2': True
}
self.creds = {
"client_cert": os.getenv("X509_USER_CERT", "Unknown"),
"client_key": os.getenv("X509_USER_KEY", "Unknown")
}
self.rucioConfigDict = {
"rucio_host": self.msConfig['rucioUrl'],
"auth_host": self.msConfig['rucioAuthUrl'],
"auth_type": "x509",
"account": self.msConfig['rucioAccount'],
"ca_cert": False,
"timeout": 30,
"request_retries": 3,
"creds": self.creds
}
self.msUnmerged = MSUnmerged(self.msConfig)
self.msUnmerged.rucioConMon = RucioConMonEmul()
self.msUnmerged.wmstatsSvc = WMStatsServerEmul()
self.msUnmerged.resetCounters()
self.msUnmerged.rucio = Rucio.Rucio(
self.msConfig['rucioAccount'],
hostUrl=self.rucioConfigDict['rucio_host'],
authUrl=self.rucioConfigDict['auth_host'],
configDict=self.rucioConfigDict)
super(MSUnmergedTest, self).setUp()
if PY3:
self.assertItemsEqual = self.assertCountEqual
def setUp(self):
"""
Setup for unit tests
"""
super(RucioTest, self).setUp()
self.myRucio = Rucio.Rucio(self.acct,
hostUrl=self.defaultArgs['host'],
authUrl=self.defaultArgs['auth_host'],
configDict=self.defaultArgs)
self.client = testClient(rucio_host=self.defaultArgs['host'],
auth_host=self.defaultArgs['auth_host'],
account=self.acct,
ca_cert=self.defaultArgs['ca_cert'],
auth_type=self.defaultArgs['auth_type'],
creds=self.defaultArgs['creds'],
timeout=self.defaultArgs['timeout'])
def testProdGetPFN(self):
"""
Test `getPFN` method using the production server, hence set
as an integration test not to be executed by jenkins
"""
newParams = {"auth_type": "x509", "ca_cert": False, "timeout": 50}
prodRucio = Rucio.Rucio("wmcore_transferor",
hostUrl='http://cms-rucio.cern.ch',
authUrl='https://cms-rucio-auth.cern.ch',
configDict=newParams)
# simplest call (from Alan)
cernTestDefaultPrefix = "gsiftp://eoscmsftp.cern.ch:2811/eos/cms"
testLfn = "/store"
resp = prodRucio.getPFN(site="T2_CH_CERN", lfns=testLfn)
self.assertEqual(resp[testLfn], cernTestDefaultPrefix + testLfn)
# see comment in testGetPFN() function above about testing with lfns=""
resp = prodRucio.getPFN(site="T2_CH_CERN", lfns="")
self.assertItemsEqual(resp, {u'': u'gsiftp://eoscmsftp.cern.ch:2811/'})
# possible additional tests (from Stefano)
lfn1 = '/store/afile'
lfn2 = '/store/mc/afile'
resp = prodRucio.getPFN(site="T2_CH_CERN", lfns=lfn1)
self.assertEqual(resp[lfn1], cernTestDefaultPrefix + lfn1)
# test with a list of LFN's
resp = prodRucio.getPFN(site="T2_CH_CERN", lfns=[lfn1, lfn2])
self.assertEqual(len(resp), 2)
self.assertEqual(resp[lfn1], cernTestDefaultPrefix + lfn1)
self.assertEqual(resp[lfn2], cernTestDefaultPrefix + lfn2)
# test different protocols
resp = prodRucio.getPFN(site="T2_US_Nebraska",
lfns=lfn1,
protocol='gsiftp')
self.assertEqual(
resp[lfn1],
"gsiftp://red-gridftp.unl.edu:2811/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms"
+ lfn1)
resp = prodRucio.getPFN(site="T2_US_Nebraska",
lfns=lfn1,
protocol='davs')
self.assertEqual(resp[lfn1], "davs://xrootd-local.unl.edu:1094" + lfn1)
def testGetReplicaInfoForBlocksRucio(self):
"""
Test `getReplicaInfoForBlocks` method, however not using
the output compatibility with PhEDEx
"""
theseArgs = self.defaultArgs.copy()
theseArgs['phedexCompatible'] = False
myRucio = Rucio.Rucio(self.acct,
hostUrl=theseArgs['host'],
authUrl=theseArgs['auth_host'],
configDict=theseArgs)
res = myRucio.getReplicaInfoForBlocks(dataset=DSET)
self.assertTrue(isinstance(res, list))
self.assertTrue(
len(res) >= 1) # at this very moment, there are 11 replicas
blocks = [item['name'] for item in res]
self.assertTrue(BLOCK in blocks)
for item in res:
self.assertTrue(len(item['replica']) > 0)
def testGetPileupLockedAndAvailable(self):
"""
Test `getPileupLockedAndAvailable` method
"""
# as much as I dislike it, we need to use the production instance...
newParams = {
"host": 'http://cms-rucio.cern.ch',
"auth_host": 'https://cms-rucio-auth.cern.ch',
"auth_type": "x509",
"account": "wmcore_transferor",
"ca_cert": False,
"timeout": 5
}
prodRucio = Rucio.Rucio(newParams['account'],
hostUrl=newParams['host'],
authUrl=newParams['auth_host'],
configDict=newParams)
resp = prodRucio.getPileupLockedAndAvailable(PUDSET, "transfer_ops")
# this dataset contains 10 blocks
self.assertEqual(len(resp), 10)
self.assertTrue(PUBLOCK in resp)
# with more than 10 block replicas in the grid
for block, rses in viewitems(resp):
self.assertTrue(len(rses) > 5)
def testListDataRules2(self):
"""
Test `listDataRules` method with data from production
"""
newParams = {
"host": 'http://cms-rucio.cern.ch',
"auth_host": 'https://cms-rucio-auth.cern.ch',
"auth_type": "x509",
"account": "wmcore_transferor",
"ca_cert": False,
"timeout": 50
}
prodRucio = Rucio.Rucio(newParams['account'],
hostUrl=newParams['host'],
authUrl=newParams['auth_host'],
configDict=newParams)
resp = prodRucio.listDataRules(name=DSET2, account="transfer_ops")
self.assertEqual(len(resp), 5)
accts = set([rule['account'] for rule in resp])
self.assertItemsEqual(accts, ['transfer_ops'])
resp = prodRucio.listDataRules(name=DSET2, account="wmcore_transferor")
self.assertItemsEqual(resp, [])
def testGetDataLockedAndAvailable(self):
"""
Test `getDataLockedAndAvailable` method
"""
# as much as I dislike it, we need to use the production instance...
newParams = {
"host": 'http://cms-rucio.cern.ch',
"auth_host": 'https://cms-rucio-auth.cern.ch',
"auth_type": "x509",
"account": "wmcore_transferor",
"ca_cert": False,
"timeout": 50
}
prodRucio = Rucio.Rucio(newParams['account'],
hostUrl=newParams['host'],
authUrl=newParams['auth_host'],
configDict=newParams)
# This is a very heavy check, with ~25k blocks
PUDSET = "/Neutrino_E-10_gun/RunIISpring15PrePremix-PUMoriond17_80X_mcRun2_asymptotic_2016_TrancheIV_v2-v2/GEN-SIM-DIGI-RAW"
resp = prodRucio.getDataLockedAndAvailable(name=PUDSET)
self.assertItemsEqual(resp, ['T1_US_FNAL_Disk'])
# matches rules for this container and transfer_ops account, returns an union of the blocks RSEs
resp = prodRucio.getDataLockedAndAvailable(name=DSET2,
account="transfer_ops")
self.assertItemsEqual(resp, [
'T2_CH_CSCS', 'T1_FR_CCIN2P3_Disk', 'T1_IT_CNAF_Disk',
'T2_US_Purdue', 'T1_RU_JINR_Disk', 'T2_FI_HIP', 'T2_UK_London_IC',
'T2_FR_GRIF_LLR', 'T2_US_Nebraska', 'T2_IT_Bari', 'T2_FR_IPHC',
'T2_DE_DESY', 'T2_BE_IIHE', 'T1_DE_KIT_Disk', 'T2_BR_SPRACE',
'T1_US_FNAL_Disk', 'T1_UK_RAL_Disk', 'T2_US_Vanderbilt'
])
# matches the same rules as above (all multi RSEs), but returns an intersection of the RSEs
resp = prodRucio.getDataLockedAndAvailable(name=DSET2,
account="transfer_ops",
grouping="ALL")
# FIXME: there is a serious block distribution problem with Rucio, there should be 6 disk RSE locations..
self.assertItemsEqual(resp, ['T1_US_FNAL_Disk'])
# there are no rules for the blocks, but 6 copies for the container level
resp = prodRucio.getDataLockedAndAvailable(name=BLOCK2,
account="transfer_ops",
grouping="ALL")
self.assertItemsEqual(resp, [
'T1_IT_CNAF_Disk', 'T1_RU_JINR_Disk', 'T1_UK_RAL_Disk',
'T1_US_FNAL_Disk', 'T2_IT_Bari', 'T2_US_Vanderbilt'
])
# return the 6 copies for the container level, plus the tape one(s)
resp = prodRucio.getDataLockedAndAvailable(name=BLOCK2,
account="transfer_ops",
grouping="ALL",
returnTape=True)
self.assertItemsEqual(resp, [
'T1_IT_CNAF_Disk', 'T1_RU_JINR_Disk', 'T1_UK_RAL_Disk',
'T1_US_FNAL_Disk', 'T2_IT_Bari', 'T2_US_Vanderbilt',
'T1_IT_CNAF_Tape'
])
# there are no rules for the blocks, but 6 copies for the container level
resp = prodRucio.getDataLockedAndAvailable(name=BLOCK2,
account="transfer_ops",
grouping="DATASET")
self.assertItemsEqual(resp, [])