def __init__(self, msConfig, **kwargs):
"""
Provides setup for MSTransferor and MSMonitor classes
:param config: MS service configuration
:param kwargs: can be used to skip the initialization of specific services, such as:
logger: logger object
skipReqMgr: boolean to skip ReqMgr initialization
skipReqMgrAux: boolean to skip ReqMgrAux initialization
skipRucio: boolean to skip Rucio initialization
"""
self.logger = getMSLogger(getattr(msConfig, 'verbose', False), kwargs.get("logger"))
self.msConfig = msConfig
self.logger.info("Configuration including default values:\n%s", self.msConfig)
if not kwargs.get("skipReqMgr", False):
self.reqmgr2 = ReqMgr(self.msConfig['reqmgr2Url'], logger=self.logger)
if not kwargs.get("skipReqMgrAux", False):
self.reqmgrAux = ReqMgrAux(self.msConfig['reqmgr2Url'],
httpDict={'cacheduration': 1.0}, logger=self.logger)
self.phedex = None
self.rucio = None
if not kwargs.get("skipRucio", False):
self.rucio = Rucio(acct=self.msConfig['rucioAccount'],
hostUrl=self.msConfig['rucioUrl'],
authUrl=self.msConfig['rucioAuthUrl'],
configDict={"logger": self.logger, "user_agent": "wmcore-microservices"})
def testConfig(self):
"""
Test service attributes and the override mechanism
"""
for key in self.defaultArgs:
self.assertEqual(getattr(self.myRucio.cli, key),
self.defaultArgs[key])
self.assertTrue(
getattr(self.myRucio.cli,
"user_agent").startswith("wmcore-client/"))
self.assertTrue(
getattr(self.client, "user_agent").startswith("rucio-clients/"))
newParams = {
"host": 'http://cms-rucio-int.cern.ch',
"auth_host": 'https://cms-rucio-auth-int.cern.ch',
"auth_type": "x509",
"account": self.acct,
"ca_cert": False,
"timeout": 5,
"phedexCompatible": False
}
newKeys = newParams.keys()
newKeys.remove("phedexCompatible")
rucio = Rucio(newParams['account'],
hostUrl=newParams['host'],
authUrl=newParams['auth_host'],
configDict=newParams)
self.assertEqual(getattr(rucio, "phedexCompat"), False)
for key in newKeys:
self.assertEqual(getattr(rucio.cli, key), newParams[key])
def __init__(self):
"""
Prepare module setup
"""
super(PileupFetcher, self).__init__()
# FIXME: find a way to pass the Rucio account name to this fetcher module
self.rucioAcct = "wmcore_transferor"
self.rucio = Rucio(self.rucioAcct)
def __init__(self, msConfig, logger=None):
super(MSMonitor, self).__init__(msConfig, logger=logger)
# update interval is used to check records in CouchDB and update them
# after this interval, default 6h
self.updateInterval = self.msConfig.get('updateInterval', 6 * 60 * 60)
self.rucio = Rucio(acct=self.msConfig['rucioAccount'],
hostUrl=self.msConfig['rucioUrl'],
authUrl=self.msConfig['rucioAuthUrl'],
configDict={"logger": self.logger, "user_agent": "WMCore-MSMonitor"})
def __init__(self, config):
"""
___init___
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.enabled = config.RucioInjector.enabled
# dataset rule creation has a larger polling cycle
self.pollRules = config.RucioInjector.pollIntervalRules
self.lastRulesExecTime = 0
self.createBlockRules = config.RucioInjector.createBlockRules
self.containerDiskRuleParams = config.RucioInjector.containerDiskRuleParams
self.containerDiskRuleRSEExpr = config.RucioInjector.containerDiskRuleRSEExpr
self.skipRulesForTiers = config.RucioInjector.skipRulesForTiers
self.listTiersToInject = config.RucioInjector.listTiersToInject
if config.RucioInjector.metaDIDProject not in RUCIO_VALID_PROJECT:
msg = "Component configured with an invalid 'project' DID: %s"
raise RucioInjectorException(msg % config.RucioInjector.metaDIDProject)
self.metaDIDProject = dict(project=config.RucioInjector.metaDIDProject)
# setup cache for container and blocks (containers can be much longer, make 6 days now)
self.containersCache = MemoryCache(config.RucioInjector.cacheExpiration * 3, set())
self.blocksCache = MemoryCache(config.RucioInjector.cacheExpiration, set())
self.scope = getattr(config.RucioInjector, "scope", "cms")
self.rucioAcct = config.RucioInjector.rucioAccount
self.rucio = Rucio(acct=self.rucioAcct,
hostUrl=config.RucioInjector.rucioUrl,
authUrl=config.RucioInjector.rucioAuthUrl,
configDict={'logger': self.logger})
# metadata dictionary information to be added to block/container rules
# cannot be a python dictionary, but a JSON string instead
self.metaData = json.dumps(dict(agentHost=config.Agent.hostName,
userAgent=config.Agent.agentName))
self.testRSEs = config.RucioInjector.RSEPostfix
self.filesToRecover = []
# output data placement has a different behaviour between T0 and Production agents
if hasattr(config, "Tier0Feeder"):
logging.info("RucioInjector running on a T0 WMAgent")
self.isT0agent = True
else:
self.isT0agent = False
if not self.listTiersToInject:
logging.info("Component configured to inject all the data tiers")
else:
logging.info("Component configured to only inject data for data tiers: %s",
self.listTiersToInject)
logging.info("Component configured to skip container rule creation for data tiers: %s",
self.skipRulesForTiers)
logging.info("Component configured to create block rules: %s", self.createBlockRules)
def __init__(self):
"""
Prepare module setup
"""
super(PileupFetcher, self).__init__()
if usingRucio():
# FIXME: find a way to pass the Rucio account name to this fetcher module
self.rucioAcct = "wmcore_transferor"
self.rucio = Rucio(self.rucioAcct)
else:
self.phedex = PhEDEx() # this will go away eventually
def __init__(self):
"""
Prepare module setup
"""
super(PileupFetcher, self).__init__()
if usingRucio():
# Too much work to pass the rucio account name all the way to here
# just use the production rucio account for resolving pileup location
self.rucio = Rucio("wma_prod",
configDict={'phedexCompatible': False})
else:
self.phedex = PhEDEx() # this will go away eventually
def getFromRucio(dataset, logger):
"""
Using the WMCore Rucio object and fetch all the blocks and files
for a given container.
Returns a dictionary key'ed by the block name, value is the amount of files.
"""
rucio = Rucio(acct=RUCIO_ACCT,
hostUrl=RUCIO_HOST,
authUrl=RUCIO_AUTH,
configDict={'logger': logger})
result = dict()
for block in rucio.getBlocksInContainer(dataset):
data = rucio.getDID(block)
result.setdefault(block, data['length'])
return result
def __init__(self, config):
"""
___init___
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.enabled = config.RucioInjector.enabled
# dataset rule creation has a larger polling cycle
self.pollRules = config.RucioInjector.pollIntervalRules
self.lastRulesExecTime = 0
self.createBlockRules = config.RucioInjector.createBlockRules
self.skipRulesForTiers = config.RucioInjector.skipRulesForTiers
self.listTiersToInject = config.RucioInjector.listTiersToInject
# setup cache for container and blocks (containers can be much longer, make 6 days now)
self.containersCache = MemoryCache(
config.RucioInjector.cacheExpiration * 3, set())
self.blocksCache = MemoryCache(config.RucioInjector.cacheExpiration,
set())
self.scope = getattr(config.RucioInjector, "scope", "cms")
self.rucioAcct = config.RucioInjector.rucioAccount
self.rucio = Rucio(acct=self.rucioAcct,
hostUrl=config.RucioInjector.rucioUrl,
authUrl=config.RucioInjector.rucioAuthUrl,
configDict={'logger': self.logger})
# metadata dictionary information to be added to block/container rules
# cannot be a python dictionary, but a JSON string instead
self.metaData = json.dumps(
dict(agentHost=config.Agent.hostName,
userAgent=config.Agent.agentName))
self.testRSEs = config.RucioInjector.RSEPostfix
self.filesToRecover = []
logging.info(
"Component configured to only inject data for data tiers: %s",
self.listTiersToInject)
logging.info(
"Component configured to skip container rule creation for data tiers: %s",
self.skipRulesForTiers)
logging.info("Component configured to create block rules: %s",
self.createBlockRules)
def _getDatasetLocation(self, dset, blockDict):
"""
Given a dataset name, query PhEDEx or Rucio and resolve the block location
:param dset: string with the dataset name
:param blockDict: dictionary with DBS summary info
:return: update blockDict in place
"""
# initialize Rucio here to avoid this authentication on T0-WMAgent
self.rucio = Rucio(self.rucioAcct)
blockReplicas = self.rucio.getPileupLockedAndAvailable(
dset, account=self.rucioAcct)
for blockName, blockLocation in viewitems(blockReplicas):
try:
blockDict[blockName]['PhEDExNodeNames'] = list(blockLocation)
except KeyError:
logging.warning("Block '%s' present in Rucio but not in DBS",
blockName)
def setUp(self):
"""
Setup for unit tests
"""
super(RucioTest, self).setUp()
self.myRucio = Rucio(self.acct,
hostUrl=self.defaultArgs['host'],
authUrl=self.defaultArgs['auth_host'],
configDict=self.defaultArgs)
self.client = testClient(rucio_host=self.defaultArgs['host'],
auth_host=self.defaultArgs['auth_host'],
account=self.acct,
ca_cert=self.defaultArgs['ca_cert'],
auth_type=self.defaultArgs['auth_type'],
creds=self.defaultArgs['creds'],
timeout=self.defaultArgs['timeout'])
def testGetReplicaInfoForBlocksRucio(self):
"""
Test `getReplicaInfoForBlocks` method, however not using
the output compatibility with PhEDEx
"""
theseArgs = self.defaultArgs.copy()
theseArgs['phedexCompatible'] = False
myRucio = Rucio(self.acct,
hostUrl=theseArgs['host'],
authUrl=theseArgs['auth_host'],
configDict=theseArgs)
res = myRucio.getReplicaInfoForBlocks(dataset=DSET)
self.assertTrue(isinstance(res, list))
self.assertTrue(len(res) >= 1) # at this very moment, there are 11 replicas
blocks = [item['name'] for item in res]
self.assertTrue(BLOCK in blocks)
for item in res:
self.assertTrue(len(item['replica']) > 0)
def __init__(self, msConfig, **kwargs):
"""
Provides setup for MSTransferor and MSMonitor classes
:param config: MS service configuration
:param kwargs: can be used to skip the initialization of specific services, such as:
logger: logger object
skipReqMgr: boolean to skip ReqMgr initialization
skipReqMgrAux: boolean to skip ReqMgrAux initialization
skipRucio: boolean to skip Rucio initialization
skipPhEDEx: boolean to skip PhEDEx initialization
"""
self.logger = getMSLogger(getattr(msConfig, 'verbose', False),
kwargs.get("logger"))
self.msConfig = msConfig
self.logger.info("Configuration including default values:\n%s",
self.msConfig)
if not kwargs.get("skipReqMgr", False):
self.reqmgr2 = ReqMgr(self.msConfig['reqmgr2Url'],
logger=self.logger)
if not kwargs.get("skipReqMgrAux", False):
self.reqmgrAux = ReqMgrAux(self.msConfig['reqmgr2Url'],
httpDict={'cacheduration': 1.0},
logger=self.logger)
self.phedex = None
self.rucio = None
if self.msConfig.get('useRucio',
False) and not kwargs.get("skipRucio", False):
self.rucio = Rucio(acct=self.msConfig['rucioAccount'],
hostUrl=self.msConfig['rucioUrl'],
authUrl=self.msConfig['rucioAuthUrl'],
configDict={
"logger": self.logger,
"user_agent": "wmcore-microservices"
})
elif not kwargs.get("skipPhEDEx", False):
# hard code it to production DBS otherwise PhEDEx subscribe API fails to match TMDB data
dbsUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
self.phedex = PhEDEx(httpDict={'cacheduration': 0.5},
dbsUrl=dbsUrl,
logger=self.logger)
def setUp(self):
"""
_setUp_
"""
super(WMBSHelperTest, self).setUp()
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setupCouch("wmbshelper_t/jobs", "JobDump")
self.testInit.setupCouch("wmbshelper_t/fwjrs", "FWJRDump")
self.testInit.setupCouch("config_test", "GroupUser", "ConfigCache")
os.environ["COUCHDB"] = "wmbshelper_t"
self.testInit.setSchema(customModules=[
"WMCore.WMBS", "WMComponent.DBS3Buffer", "WMCore.BossAir",
"WMCore.ResourceControl"
],
useDefault=False)
self.workDir = self.testInit.generateWorkDir()
self.wmspec = self.createWMSpec()
self.topLevelTask = getFirstTask(self.wmspec)
self.inputDataset = self.topLevelTask.inputDataset()
self.dataset = self.topLevelTask.getInputDatasetPath()
self.dbs = DBSReader(self.inputDataset.dbsurl)
self.rucioAcct = "wmcore_transferor"
self.rucio = Rucio(self.rucioAcct)
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=threading.currentThread().logger,
dbinterface=threading.currentThread().dbi)
self.configFile = EmulatorSetup.setupWMAgentConfig()
self.config = loadConfigurationFile(self.configFile)
self.config.component_("JobSubmitter")
self.config.JobSubmitter.submitDir = self.workDir
self.config.JobSubmitter.submitScript = os.path.join(
getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh')
return
def __init__(self, **kwargs):
if not kwargs.get('logger'):
import logging
kwargs['logger'] = logging
self.logger = kwargs['logger']
self.rucio = Rucio(kwargs.get("rucioAccount", "wmcore_transferor"),
configDict=dict(logger=self.logger))
# this will break all in one test
self.reqMgr2 = ReqMgr(kwargs.get("reqmgr2_endpoint", None))
centralurl = kwargs.get("central_logdb_url", "")
identifier = kwargs.get("log_reporter", "")
# set the thread name before creat the log db.
# only sets that when it is not set already
myThread = threading.currentThread()
if myThread.getName() == "MainThread":
myThread.setName(self.__class__.__name__)
self.logdb = LogDB(centralurl, identifier, logger=self.logger)
def __init__(self, **args):
PolicyInterface.__init__(self, **args)
self.workQueueElements = []
self.wmspec = None
self.team = None
self.initialTask = None
self.splitParams = None
self.dbs_pool = {}
self.data = {}
self.lumi = None
self.couchdb = None
self.rejectedWork = [] # List of inputs that were rejected
self.badWork = [
] # list of bad work unit (e.g. without any valid files)
self.pileupData = {}
self.cric = CRIC()
if usingRucio():
self.rucio = Rucio(self.args['rucioAcct'],
configDict={'logger': self.logger})
else:
self.phedex = PhEDEx() # this will go away eventually
def __init__(self, **args):
# We need to pop this object instance from args because otherwise
# the super class blows up when doing a deepcopy(args)
self.rucio = args.pop("rucioObject", None)
PolicyInterface.__init__(self, **args)
self.workQueueElements = []
self.wmspec = None
self.team = None
self.initialTask = None
self.splitParams = None
self.dbs_pool = {}
self.data = {}
self.lumi = None
self.couchdb = None
self.rejectedWork = [] # List of inputs that were rejected
self.badWork = [
] # list of bad work unit (e.g. without any valid files)
self.pileupData = {}
self.cric = CRIC()
# FIXME: for the moment, it will always use the default value
self.rucioAcct = self.args.get("rucioAcct", "wmcore_transferor")
if not self.rucio:
self.rucio = Rucio(self.rucioAcct,
configDict={'logger': self.logger})
def _queryAndCompareWithDBS(self, pileupDict, pileupConfig, dbsUrl):
"""
pileupDict is a Python dictionary containing particular pileup
configuration information. Query DBS on given dataset contained
now in both input pileupConfig as well as in the pileupDict
and compare values.
"""
self.assertItemsEqual(list(pileupDict), list(pileupConfig))
reader = DBS3Reader(dbsUrl)
rucioObj = Rucio(self.rucioAcct)
# now query DBS and compare the blocks and files from DBS
# against those returned by the PileupFetcher
for pileupType, datasets in viewitems(pileupConfig):
# this is from the pileup configuration produced by PileupFetcher
blockDict = pileupDict[pileupType]
for dataset in datasets:
dbsBlocks = reader.listFileBlocks(dataset=dataset)
rucioBlocksLocation = rucioObj.getPileupLockedAndAvailable(dataset,
account=self.rucioAcct)
# first, validate the number of blocks and their names
self.assertItemsEqual(list(blockDict), dbsBlocks)
self.assertItemsEqual(list(blockDict), list(rucioBlocksLocation))
# now validate the block location between Rucio and PileupFetcher
for block, blockLocation in viewitems(blockDict):
self.assertItemsEqual(blockLocation['PhEDExNodeNames'], rucioBlocksLocation[block])
# finally, validate the files
fileList = []
# now get list of files in the block
dbsFiles = reader.listFilesInBlock(block)
for dbsFile in dbsFiles:
fileList.append(dbsFile["LogicalFileName"])
self.assertItemsEqual(blockDict[block]["FileList"], fileList)
def executeInternal(self, *args, **kwargs):
self.logger.info(
"Data discovery with DBS") ## to be changed into debug
dbsurl = self.config.Services.DBSUrl
if kwargs['task']['tm_dbs_url']:
dbsurl = kwargs['task']['tm_dbs_url']
self.dbs = DBSReader(dbsurl)
self.dbsInstance = self.dbs.dbs.serverinfo()["dbs_instance"]
isUserDataset = self.dbsInstance.split('/')[1] != 'global'
# where to look locations in pre-Rucio world
PhEDExOrDBS = 'PhEDEx' if not isUserDataset else 'DBS origin site'
taskName = kwargs['task']['tm_taskname']
userProxy = kwargs['task']['user_proxy']
self.logger.debug("Data discovery through %s for %s", self.dbs,
taskName)
inputDataset = kwargs['task']['tm_input_dataset']
secondaryDataset = kwargs['task'].get('tm_secondary_input_dataset',
None)
self.checkDatasetStatus(inputDataset, kwargs)
if secondaryDataset:
self.checkDatasetStatus(secondaryDataset, kwargs)
try:
# Get the list of blocks for the locations.
# The WMCore DBS3 implementation makes one call to DBS for each block
# when using locations=True so we are using locations=False and looking up location later
blocks = [
x['Name'] for x in self.dbs.getFileBlocksInfo(inputDataset,
locations=False)
]
if secondaryDataset:
secondaryBlocks = [
x['Name']
for x in self.dbs.getFileBlocksInfo(secondaryDataset,
locations=False)
]
except DBSReaderError as dbsexc:
# dataset not found in DBS is a known use case
if str(dbsexc).find('No matching data'):
raise TaskWorkerException(
"CRAB could not find dataset %s in this DBS instance: %s" %
inputDataset, dbsurl)
raise
## Create a map for block's locations: for each block get the list of locations.
## Note: listFileBlockLocation() gets first the locations from PhEDEx, and if no
## locations are found it gets the original locations from DBS. So it should
## never be the case at this point that some blocks have no locations.
## locationsMap is a dictionary, key=blockName, value=list of PhedexNodes, example:
## {'/JetHT/Run2016B-PromptReco-v2/AOD#b10179dc-3723-11e6-9aa5-001e67abf228': [u'T1_IT_CNAF_Buffer', u'T2_US_Wisconsin', u'T1_IT_CNAF_MSS', u'T2_BE_UCL'],
## '/JetHT/Run2016B-PromptReco-v2/AOD#89b03ca6-1dc9-11e6-b567-001e67ac06a0': [u'T1_IT_CNAF_Buffer', u'T2_US_Wisconsin', u'T1_IT_CNAF_MSS', u'T2_BE_UCL']}
# For now apply Rucio data location only to NANOAOD*
# in time useRucioForLocations may become a more rich expression
isNano = blocks[0].split("#")[0].split("/")[-1] in [
"NANOAOD", "NANOAODSIM"
]
if isNano:
self.logger.info(
"NANOAOD* datset. Will use Rucio for data location")
useRucioForLocations = isNano
locationsFoundWithRucio = False
if not useRucioForLocations:
self.logger.info("Will not use Rucio for this dataset")
# if locations should be in Rucio, try it first and fall back to old ways if Rucio calls fail
# of if they return no locations (possible Rucio teething pain). If Rucio returns a list, trust it.
if useRucioForLocations:
locationsMap = {}
scope = "cms"
# If the dataset is a USER one, use the Rucio user scope to find it
# TODO: we need a way to enable users to indicate others user scopes as source
if isUserDataset:
scope = "user.%s" % kwargs['task']['tm_username']
rucio_config_dict = {
"phedexCompatible": True,
"auth_type": "x509",
"ca_cert": self.config.Services.Rucio_caPath,
"logger": self.logger,
"creds": {
"client_cert": self.config.TaskWorker.cmscert,
"client_key": self.config.TaskWorker.cmskey
}
}
try:
self.logger.info("Initializing Rucio client")
# WMCore is awfully verbose
with tempSetLogLevel(logger=self.logger, level=logging.ERROR):
rucioClient = Rucio(
self.config.Services.Rucio_account,
hostUrl=self.config.Services.Rucio_host,
authUrl=self.config.Services.Rucio_authUrl,
configDict=rucio_config_dict)
rucioClient.whoAmI()
self.logger.info(
"Looking up data location with Rucio in %s scope.", scope)
with tempSetLogLevel(logger=self.logger, level=logging.ERROR):
locations = rucioClient.getReplicaInfoForBlocks(
scope=scope, block=list(blocks))
except Exception as exc:
msg = "Rucio lookup failed with\n%s" % str(exc)
# TODO when removing fall-back to PhEDEx, this should be a fatal error
# raise TaskWorkerException(msg)
self.logger.warn(msg)
locations = None
# TODO when removing fall-back to PhEDEx, above code will raise if it fails, therefore
# the following "if" must be removed and the code shifted left
if locations:
located_blocks = locations['phedex']['block']
for element in located_blocks:
if element[
'replica']: # only fill map for blocks which have at least one location
locationsMap.update({
element['name']:
[x['node'] for x in element['replica']]
})
if locationsMap:
locationsFoundWithRucio = True
else:
msg = "No locations found with Rucio for this dataset"
# since NANO* are not in PhEDEx, this should be a fatal error
if isNano:
raise TaskWorkerException(msg)
else:
# note it down and try with PhEDEx
self.logger.warn(msg)
if not locationsFoundWithRucio: # fall back to pre-Rucio methods
try:
self.logger.info("Looking up data locations using %s",
PhEDExOrDBS)
locationsMap = self.dbs.listFileBlockLocation(
list(blocks), dbsOnly=isUserDataset)
except Exception as ex:
raise TaskWorkerException(
"The CRAB3 server backend could not get the location of the files from dbs nor phedex nor rucio.\n"+\
"This is could be a temporary phedex/rucio/dbs glitch, please try to submit a new task (resubmit will not work)"+\
" and contact the experts if the error persists.\nError reason: %s" % str(ex)
)
# only fill map for blocks which have at least one location
locationsMap = {
key: value
for key, value in locationsMap.iteritems() if value
}
if secondaryDataset:
secondaryLocationsMap = {}
# see https://github.com/dmwm/CRABServer/issues/6075#issuecomment-641569446
self.logger.info(
"Trying data location of secondary blocks with Rucio")
try:
locations = rucioClient.getReplicaInfoForBlocks(
scope=scope, block=list(secondaryBlocks))
except Exception as exc:
locations = None
secondaryLocationsMap = {}
self.logger.warn("Rucio lookup failed with. %s", exc)
if locations:
located_blocks = locations['phedex']['block']
for element in located_blocks:
if element[
'replica']: # only fill map for blocks which have at least one location
secondaryLocationsMap.update({
element['name']:
[x['node'] for x in element['replica']]
})
if not secondaryLocationsMap:
msg = "No locations found with Rucio for secondaryDataset."
# TODO when removing fall-back to PhEDEx, this should be a fatal error
# raise TaskWorkerException(msg)
self.logger.warn(msg)
self.logger.info(
"Trying data location of secondary blocks with PhEDEx")
try:
secondaryLocationsMap = self.dbs.listFileBlockLocation(
list(secondaryBlocks), dbsOnly=isUserDataset)
except Exception as ex:
raise TaskWorkerException(
"The CRAB3 server backend could not get the location of the secondary dataset files from dbs or phedex or rucio.\n" + \
"This is could be a temporary phedex/rucio/dbs glitch, please try to submit a new task (resubmit will not work)" + \
" and contact the experts if the error persists.\nError reason: %s" % str(ex)
)
# only fill map for blocks which have at least one location
secondaryLocationsMap = {
key: value
for key, value in secondaryLocationsMap.iteritems()
if value
}
# From now on code is not dependent from having used Rucio or PhEDEx
blocksWithLocation = locationsMap.keys()
if secondaryDataset:
secondaryBlocksWithLocation = secondaryLocationsMap.keys()
self.keepOnlyDisks(locationsMap)
if not locationsMap:
msg = "Task could not be submitted because there is no DISK replica for dataset %s" % inputDataset
if self.tapeLocations:
msg += "\nN.B.: the input dataset is stored at %s, but those are TAPE locations." % ', '.join(
sorted(self.tapeLocations))
# submit request to DDM
ddmRequest = None
ddmServer = self.config.TaskWorker.DDMServer
try:
ddmRequest = blocksRequest(blocksWithLocation,
ddmServer,
self.config.TaskWorker.cmscert,
self.config.TaskWorker.cmskey,
verbose=False)
except HTTPException as hte:
self.logger.exception(hte)
msg += "\nThe automatic stage-out failed, please try again later. If the error persists contact the experts and provide this error message:"
msg += "\nHTTP Error while contacting the DDM server %s:\n%s" % (
ddmServer, str(hte))
msg += "\nHTTP Headers are: %s" % hte.headers
msg += "\nYou might want to contact your physics group if you need a disk replica."
raise TaskWorkerException(msg, retry=True)
self.logger.info("Contacted %s using %s and %s, got:\n%s",
self.config.TaskWorker.DDMServer,
self.config.TaskWorker.cmscert,
self.config.TaskWorker.cmskey, ddmRequest)
# The query above returns a JSON with a format {"result": "OK", "message": "Copy requested", "data": [{"request_id": 18, "site": <site>, "item": [<list of blocks>], "group": "AnalysisOps", "n": 1, "status": "new", "first_request": "2018-02-26 23:57:37", "last_request": "2018-02-26 23:57:37", "request_count": 1}]}
if ddmRequest["result"] == "OK":
# set status to TAPERECALL
tapeRecallStatus = 'TAPERECALL'
ddmReqId = ddmRequest["data"][0]["request_id"]
configreq = {
'workflow': taskName,
'taskstatus': tapeRecallStatus,
'ddmreqid': ddmReqId,
'subresource': 'addddmreqid',
}
try:
tapeRecallStatusSet = self.server.post(
self.restURInoAPI + '/task',
data=urllib.urlencode(configreq))
except HTTPException as hte:
self.logger.exception(hte)
msg = "HTTP Error while contacting the REST Interface %s:\n%s" % (
self.config.TaskWorker.restHost, str(hte))
msg += "\nSetting %s status and DDM request ID (%d) failed for task %s" % (
tapeRecallStatus, ddmReqId, taskName)
msg += "\nHTTP Headers are: %s" % hte.headers
raise TaskWorkerException(msg, retry=True)
msg += "\nA disk replica has been requested on %s to CMS DDM (request ID: %d)" % (
ddmRequest["data"][0]["first_request"], ddmReqId)
if tapeRecallStatusSet[2] == "OK":
self.logger.info("Status for task %s set to '%s'",
taskName, tapeRecallStatus)
msg += "\nThis task will be automatically submitted as soon as the stage-out is completed."
self.uploadWarning(msg, userProxy, taskName)
raise TapeDatasetException(msg)
else:
msg += ", please try again in two days."
else:
msg += "\nThe disk replica request failed with this error:\n %s" % ddmRequest[
"message"]
msg += "\nPlease, check DAS (https://cmsweb.cern.ch/das) and make sure the dataset is accessible on DISK."
raise TaskWorkerException(msg)
# will not need lumi info if user has asked for split by file with no run/lumi mask
splitAlgo = kwargs['task']['tm_split_algo']
lumiMask = kwargs['task']['tm_split_args']['lumis']
runRange = kwargs['task']['tm_split_args']['runs']
needLumiInfo = splitAlgo != 'FileBased' or lumiMask != [] or runRange != []
# secondary dataset access relies on run/lumi info
if secondaryDataset:
needLumiInfo = True
if needLumiInfo:
self.checkBlocksSize(
blocksWithLocation
) # Interested only in blocks with locations, 'blocks' may contain invalid ones and trigger an Exception
if secondaryDataset:
self.checkBlocksSize(secondaryBlocksWithLocation)
try:
filedetails = self.dbs.listDatasetFileDetails(
inputDataset,
getParents=True,
getLumis=needLumiInfo,
validFileOnly=0)
if secondaryDataset:
moredetails = self.dbs.listDatasetFileDetails(
secondaryDataset,
getParents=False,
getLumis=needLumiInfo,
validFileOnly=0)
for secfilename, secinfos in moredetails.items():
secinfos['lumiobj'] = LumiList(
runsAndLumis=secinfos['Lumis'])
self.logger.info(
"Beginning to match files from secondary dataset")
for dummyFilename, infos in filedetails.items():
infos['Parents'] = []
lumis = LumiList(runsAndLumis=infos['Lumis'])
for secfilename, secinfos in moredetails.items():
if lumis & secinfos['lumiobj']:
infos['Parents'].append(secfilename)
self.logger.info("Done matching files from secondary dataset")
kwargs['task']['tm_use_parent'] = 1
except Exception as ex: #TODO should we catch HttpException instead?
self.logger.exception(ex)
raise TaskWorkerException("The CRAB3 server backend could not contact DBS to get the files details (Lumis, events, etc).\n"+\
"This is could be a temporary DBS glitch. Please try to submit a new task (resubmit will not work)"+\
" and contact the experts if the error persists.\nError reason: %s" % str(ex))
#TODO addo the nodes phedex so the user can check themselves
if not filedetails:
raise TaskWorkerException(("Cannot find any file inside the dataset. Please, check your dataset in DAS, %s.\n" +\
"Aborting submission. Resubmitting your task will not help.") %\
("https://cmsweb.cern.ch/das/request?instance=%s&input=dataset=%s") %\
(self.dbsInstance, inputDataset))
## Format the output creating the data structures required by WMCore. Filters out invalid files,
## files whose block has no location, and figures out the PSN
result = self.formatOutput(task=kwargs['task'],
requestname=taskName,
datasetfiles=filedetails,
locations=locationsMap,
tempDir=kwargs['tempDir'])
if not result.result:
raise TaskWorkerException((
"Cannot find any valid file inside the dataset. Please, check your dataset in DAS, %s.\n"
+ "Aborting submission. Resubmitting your task will not help."
) % (
"https://cmsweb.cern.ch/das/request?instance=%s&input=dataset=%s"
) % (self.dbsInstance, inputDataset))
self.logger.debug("Got %s files", len(result.result.getFiles()))
return result
def loggerSetup(logLevel=logging.INFO):
"""
Return a logger which writes everything to stdout.
"""
logger = logging.getLogger(__name__)
outHandler = logging.StreamHandler(sys.stdout)
outHandler.setFormatter(logging.Formatter("%(asctime)s:%(levelname)s:%(module)s: %(message)s"))
outHandler.setLevel(logLevel)
logger.addHandler(outHandler)
logger.setLevel(logLevel)
return logger
if __name__ == '__main__':
args = parseArgs()
logger = loggerSetup()
rucio = Rucio(acct=RUCIO_ACCT, hostUrl=RUCIO_URL, authUrl=RUCIO_AUTH_URL,
configDict={"logger": logger, "user_agent": "amaltaro/makeRucioRules"})
rule = {'copies': 1,
'activity': 'Production Input',
'lifetime': None,
'account': RUCIO_ACCT,
'grouping': "ALL",
'comment': 'WMCore MSTransferor input data placement'}
logger.info("\nCreating rule for DID: %s, with RSE: %s and other attrs: %s",
args.container, args.rse, rule)
resp = rucio.createReplicationRule(args.container, args.rse, **rule)
logger.info("Response: %s", resp)