def _getDatasetLocation(self, dset, blockDict):
"""
Given a dataset name, query PhEDEx or Rucio and resolve the block location
:param dset: string with the dataset name
:param blockDict: dictionary with DBS summary info
:return: update blockDict in place
"""
if usingRucio():
blockReplicas = self.rucio.getPileupLockedAndAvailable(
dset, account=self.rucioAcct)
for blockName, blockLocation in blockReplicas.viewitems():
try:
blockDict[blockName]['PhEDExNodeNames'] = list(
blockLocation)
except KeyError:
logging.warning(
"Block '%s' present in Rucio but not in DBS",
blockName)
else:
blockReplicasInfo = self.phedex.getReplicaPhEDExNodesForBlocks(
dataset=dset, complete='y')
for block in blockReplicasInfo:
try:
blockDict[block]['PhEDExNodeNames'] = list(
blockReplicasInfo[block])
except KeyError:
logging.warning(
"Block '%s' does not have any complete PhEDEx replica",
block)
def __init__(self, msConfig, logger=None):
"""
Provides setup for MSTransferor and MSMonitor classes
:param config: MS service configuration
:param logger: logger object (optional)
"""
self.logger = getMSLogger(getattr(msConfig, 'verbose', False), logger)
self.msConfig = msConfig
self.logger.info("Configuration including default values:\n%s",
self.msConfig)
self.reqmgr2 = ReqMgr(self.msConfig['reqmgr2Url'], logger=self.logger)
self.reqmgrAux = ReqMgrAux(self.msConfig['reqmgr2Url'],
httpDict={'cacheduration': 1.0},
logger=self.logger)
# hard code it to production DBS otherwise PhEDEx subscribe API fails to match TMDB data
dbsUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
if usingRucio():
# FIXME: we cannot use Rucio in write mode yet
# self.rucio = Rucio(self.msConfig['rucioAccount'], configDict={"logger": self.logger})
self.phedex = PhEDEx(httpDict={'cacheduration': 0.5},
dbsUrl=dbsUrl,
logger=self.logger)
else:
self.phedex = PhEDEx(httpDict={'cacheduration': 0.5},
dbsUrl=dbsUrl,
logger=self.logger)
def __init__(self):
"""
Prepare module setup
"""
super(PileupFetcher, self).__init__()
if usingRucio():
# FIXME: find a way to pass the Rucio account name to this fetcher module
self.rucioAcct = "wmcore_transferor"
self.rucio = Rucio(self.rucioAcct)
else:
self.phedex = PhEDEx() # this will go away eventually
def __init__(self):
"""
Prepare module setup
"""
super(PileupFetcher, self).__init__()
if usingRucio():
# Too much work to pass the rucio account name all the way to here
# just use the production rucio account for resolving pileup location
self.rucio = Rucio("wma_prod",
configDict={'phedexCompatible': False})
else:
self.phedex = PhEDEx() # this will go away eventually
def __init__(self, **args):
PolicyInterface.__init__(self, **args)
self.workQueueElements = []
self.wmspec = None
self.team = None
self.initialTask = None
self.splitParams = None
self.dbs_pool = {}
self.data = {}
self.lumi = None
self.couchdb = None
self.rejectedWork = [] # List of inputs that were rejected
self.badWork = [
] # list of bad work unit (e.g. without any valid files)
self.pileupData = {}
self.cric = CRIC()
if usingRucio():
self.rucio = Rucio(self.args['rucioAcct'],
configDict={'logger': self.logger})
else:
self.phedex = PhEDEx() # this will go away eventually
def testUsingRucio(self):
"""
Test the usingRucio function.
"""
self.assertFalse(usingRucio())
os.environ['WMAGENT_USE_RUCIO'] = 'FALSE'
self.assertFalse(usingRucio())
os.environ['WMAGENT_USE_RUCIO'] = 'TRUE'
self.assertTrue(usingRucio())
os.environ.pop('WMAGENT_USE_RUCIO')
self.assertFalse(usingRucio())
os.environ['WMCORE_USE_RUCIO'] = 'TRUE'
self.assertTrue(usingRucio())
os.environ.pop('WMCORE_USE_RUCIO')
self.assertFalse(usingRucio())