def wrapped_func(*args, **kwargs):
if 'cric' in services and (
not args[0].allCMSNames.sites or
(args[0].allCMSNames.cachetime + 1800 < mktime(gmtime()))):
args[0].allCMSNames = CMSSitesCache(sites=CRIC().getAllPSNs(),
cachetime=mktime(gmtime()))
args[0].allPNNNames = CMSSitesCache(
sites=CRIC().getAllPhEDExNodeNames(),
cachetime=mktime(gmtime()))
if 'phedex' in services and not args[0].phedex:
phdict = args[0].phedexargs
phdict.update({'cert': serverCert, 'key': serverKey})
args[0].phedex = PhEDEx(responseType='xml', dict=phdict)
if 'centralconfig' in services and (
not args[0].centralcfg.centralconfig or
(args[0].centralcfg.cachetime + 1800 < mktime(gmtime()))):
args[0].centralcfg = ConfigCache(
centralconfig=getCentralConfig(
extconfigurl=args[0].config.extconfigurl,
mode=args[0].config.mode),
cachetime=mktime(gmtime()))
if 'servercert' in services:
args[0].serverCert = serverCert
args[0].serverKey = serverKey
return func(*args, **kwargs)
def __init__(self, **args):
StartPolicyInterface.__init__(self, **args)
self.args.setdefault('SliceType', 'NumberOfRuns')
self.args.setdefault('SliceSize', 1)
self.lumiType = "NumberOfLumis"
self.sites = []
self.cric = CRIC()
def __init__(self, **args):
StartPolicyInterface.__init__(self, **args)
self.args.setdefault('SliceType', 'NumberOfFiles')
self.args.setdefault('SliceSize', 1)
self.args.setdefault('SplittingAlgo', 'LumiBased')
self.lumiType = "NumberOfLumis"
# Define how to handle the different splitting algorithms
self.algoMapping = {
'Harvest': self.singleChunk,
'ParentlessMergeBySize': self.singleChunk,
'MinFileBased': self.singleChunk,
'LumiBased': self.singleChunk,
'EventAwareLumiBased': self.singleChunk,
'EventBased': self.singleChunk
}
self.unsupportedAlgos = ['WMBSMergeBySize', 'SiblingProcessingBased']
self.defaultAlgo = self.fixedSizeChunk
self.sites = []
if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv(
"WMCORE_USE_CRIC", False):
self.cric = CRIC()
else:
self.cric = None
self.siteDB = SiteDB()
def setUp(self):
"""
_setUp_
Setup couchdb and the test environment
"""
super(ResubmitBlockTest, self).setUp()
self.group = 'unknown'
self.user = '******'
# Set external test helpers
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")
# Define test environment
self.couchUrl = os.environ["COUCHURL"]
self.acdcDBName = 'resubmitblock_t'
self.validLocations = [
'T2_US_Nebraska', 'T1_US_FNAL_Disk', 'T1_UK_RAL_Disk'
]
self.siteWhitelist = ['T2_XX_SiteA']
cric = CRIC()
# Convert phedex node name to a valid processing site name
self.PSNs = cric.PNNstoPSNs(self.validLocations)
self.workflowName = 'dballest_ReReco_workflow'
couchServer = CouchServer(dburl=self.couchUrl)
self.acdcDB = couchServer.connectDatabase(self.acdcDBName,
create=False)
user = makeUser(self.group, '*****@*****.**', self.couchUrl,
self.acdcDBName)
user.create()
return
def main():
problemElems = getProblematicRequests()
print("Found %d bad elements that needs fixup" % len(problemElems))
if not problemElems:
print("Nothing to fix, contact a developer if the problem persists...")
return 0
cric = CRIC()
dbsUrl = "https://cmsweb.cern.ch/dbs/prod/phys03/DBSReader"
dbs = DBSReader(dbsUrl)
for elem in problemElems:
print("Handling id: %s, with inputs: %s" % (elem.id, elem['Inputs']))
for dataItem in elem['Inputs']:
if isDataset(dataItem):
pnns = dbs.listDatasetLocation(dataItem, dbsOnly=True)
else:
pnns = dbs.listFileBlockLocation(dataItem, dbsOnly=True)
psns = cric.PNNstoPSNs(pnns)
print(" PNNs: %s map to PSNs: %s" % (pnns, psns))
elem['Inputs'][dataItem] = psns
backend.saveElements(*problemElems)
print("Done")
return 0
def __init__(self, msConfig, mode, logger=None):
"""
Runs the basic setup and initialization for the MSOutput module
:microConfig: microservice configuration
:mode: MSOutput Run mode:
- MSOutputConsumer:
Reads The workflow and transfer subscriptions from MongoDB and
makes transfer subscriptions.
- MSOutputProducer:
Fetches Workflows in a given status from Reqmgr2 then creates
and uploads the documents to MongoDB.
"""
super(MSOutput, self).__init__(msConfig, logger)
self.mode = mode
self.msConfig.setdefault("limitRequestsPerCycle", 500)
self.msConfig.setdefault("verbose", True)
self.msConfig.setdefault("interval", 600)
self.msConfig.setdefault("services", ['output'])
self.msConfig.setdefault("defaultDataManSys", "DDM")
self.msConfig.setdefault("defaultGroup", "DataOps")
self.msConfig.setdefault("enableAggSubscr", True)
self.msConfig.setdefault("enableDataPlacement", False)
self.msConfig.setdefault("excludeDataTier", ['NANOAOD', 'NANOAODSIM'])
self.msConfig.setdefault("rucioAccount", 'wma_test')
self.msConfig.setdefault("mongoDBUrl", 'mongodb://localhost')
self.msConfig.setdefault("mongoDBPort", 8230)
self.msConfig.setdefault("streamerBufferFile", None)
self.uConfig = {}
self.emailAlert = EmailAlert(self.msConfig)
self.cric = CRIC(logger=self.logger)
self.uConfig = {}
self.campaigns = {}
self.psn2pnnMap = {}
msOutIndex = IndexModel('RequestName', unique=True)
msOutDBConfig = {
'database':
'msOutDB',
'server':
self.msConfig['mongoDBUrl'],
'port':
self.msConfig['mongoDBPort'],
'logger':
self.logger,
'create':
True,
'collections': [('msOutRelValColl', msOutIndex),
('msOutNonRelValColl', msOutIndex)]
}
self.msOutDB = MongoDB(**msOutDBConfig).msOutDB
self.msOutRelValColl = self.msOutDB['msOutRelValColl']
self.msOutNonRelValColl = self.msOutDB['msOutNonRelValColl']
self.ddm = DDM(
url=self.msConfig['ddmUrl'],
logger=self.logger,
enableDataPlacement=self.msConfig['enableDataPlacement'])
def setUp(self):
"""
Setup for unit tests
"""
super(CRICTest, self).setUp()
self.myCRIC = CRIC()
if PY3:
self.assertItemsEqual = self.assertCountEqual
def __init__(self, *args, **kwargs):
TaskAction.__init__(self, *args, **kwargs)
with self.config.TaskWorker.envForCMSWEB:
configDict = {
"cacheduration": 1,
"pycurl": True
} # cache duration is in hours
self.resourceCatalog = CRIC(logger=self.logger,
configDict=configDict)
def sites():
"Return known CMS site list from CRIC"
try:
# Download a list of all the sites from CRIC
cric = CRIC()
site_list = sorted(cric.getAllPSNs())
except Exception as exc:
msg = "ERROR: Could not retrieve sites from CRIC, reason: %s" % str(
exc)
raise Exception(msg)
return site_list
def __init__(self, **args):
StartPolicyInterface.__init__(self, **args)
self.args.setdefault('SliceType', 'NumberOfRuns')
self.args.setdefault('SliceSize', 1)
self.lumiType = "NumberOfLumis"
self.sites = []
if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv(
"WMCORE_USE_CRIC", False):
self.cric = CRIC()
else:
self.cric = None
self.siteDB = SiteDB()
def wrapped_func(*args, **kwargs):
if 'cric' in services and (
not args[0].allCMSNames.sites or
(args[0].allCMSNames.cachetime + 1800 < mktime(gmtime()))):
args[0].allCMSNames = CMSSitesCache(sites=CRIC().getAllPSNs(),
cachetime=mktime(gmtime()))
args[0].allPNNNames = CMSSitesCache(
sites=CRIC().getAllPhEDExNodeNames(),
cachetime=mktime(gmtime()))
if 'centralconfig' in services and (
not args[0].centralcfg.centralconfig or
(args[0].centralcfg.cachetime + 1800 < mktime(gmtime()))):
args[0].centralcfg = ConfigCache(
centralconfig=getCentralConfig(
extconfigurl=args[0].config.extconfigurl,
mode=args[0].config.mode),
cachetime=mktime(gmtime()))
if 'servercert' in services:
args[0].serverCert = serverCert
args[0].serverKey = serverKey
return func(*args, **kwargs)
def pnns():
"""
Returns all PhEDEx node names, excluding Buffer endpoints
"""
cric = CRIC()
try:
pnn_list = sorted(cric.getAllPhEDExNodeNames(excludeBuffer=True))
except Exception as exc:
msg = "ERROR: Could not retrieve PNNs from CRIC, reason: %s" % str(exc)
raise Exception(msg)
return pnn_list
def __init__(self, **args):
StartPolicyInterface.__init__(self, **args)
self.args.setdefault('SliceType', 'NumberOfFiles')
self.args.setdefault('SliceSize', 1)
self.lumiType = "NumberOfLumis"
# Initialize a list of sites where the data is
self.sites = []
# Initialize modifiers of the policy
self.blockBlackListModifier = []
self.cric = CRIC()
def sites():
"Return known CMS site list from SiteDB"
try:
# Download a list of all the sites from SiteDB, uses v2 API.
if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv("WMCORE_USE_CRIC", False):
cric = CRIC()
site_list = sorted(cric.getAllPSNs())
else:
sitedb = SiteDBJSON()
site_list = sorted(sitedb.getAllCMSNames())
except Exception as exc:
msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % str(exc)
raise Exception(msg)
return site_list
def cmsSiteNames():
"""Get all cms sites"""
global __cmsSiteNames
if __cmsSiteNames:
return __cmsSiteNames
global __cric
if not __cric:
__cric = CRIC()
try:
__cmsSiteNames = __cric.getAllPSNs()
except Exception:
pass
return __cmsSiteNames
def getDNFromUserName(username, log, ckey=None, cert=None):
"""
Parse site string to know the fts server to use
"""
dn = ''
with newX509env(X509_USER_CERT=cert, X509_USER_KEY=ckey):
resourceCatalog = CRIC(logger=log)
try:
dn = resourceCatalog.userNameDn(username)
except:
log.error("CRIC URL cannot be accessed")
if not dn:
log.error("user does not exist")
return dn
def pnns():
"""
Returns all PhEDEx node names, excluding Buffer endpoints
"""
if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv("WMCORE_USE_CRIC", False):
sitedb = CRIC() # FIXME: rename it to cric
else:
sitedb = SiteDBJSON()
try:
pnn_list = sorted(sitedb.getAllPhEDExNodeNames(excludeBuffer=True))
except Exception as exc:
msg = "ERROR: Could not retrieve PNNs from SiteDB, reason: %s" % str(exc)
raise Exception(msg)
return pnn_list
def testConfig(self):
"""
Test service attributes and the override mechanism
"""
self.assertEqual(self.myCRIC['endpoint'], 'https://cms-cric.cern.ch/')
self.assertEqual(self.myCRIC['cacheduration'], 1)
self.assertEqual(self.myCRIC['accept_type'], 'application/json')
self.assertEqual(self.myCRIC['content_type'], 'application/json')
self.assertEqual(self.myCRIC['requests'].pycurl, None)
newParams = {"cacheduration": 100, "pycurl": True}
cric = CRIC(url='https://BLAH.cern.ch/', configDict=newParams)
self.assertEqual(cric['endpoint'], 'https://BLAH.cern.ch/')
self.assertEqual(cric['cacheduration'], newParams['cacheduration'])
self.assertTrue(cric['requests'].pycurl)
def __init__(self, msConfig, logger=None):
"""
Runs the basic setup and initialization for the MS Transferor module
:param microConfig: microservice configuration
"""
super(MSTransferor, self).__init__(msConfig, logger=logger)
# minimum percentage completion for dataset/blocks subscribed
self.msConfig.setdefault("minPercentCompletion", 99)
# minimum available storage to consider a resource good for receiving data
self.msConfig.setdefault("minimumThreshold", 1 * (1000**4)) # 1TB
# limit MSTransferor to this amount of requests per cycle
self.msConfig.setdefault("limitRequestsPerCycle", 500)
# Send warning messages for any data transfer above this threshold.
# Set to negative to ignore.
self.msConfig.setdefault("warningTransferThreshold",
100. * (1000**4)) # 100TB
# weight expression for the input replication rules
self.msConfig.setdefault("rucioRuleWeight", 'ddm_quota')
quotaAccount = self.msConfig["rucioAccount"]
self.rseQuotas = RSEQuotas(
quotaAccount,
self.msConfig["quotaUsage"],
minimumThreshold=self.msConfig["minimumThreshold"],
verbose=self.msConfig['verbose'],
logger=logger)
self.reqInfo = RequestInfo(self.msConfig, self.rucio, self.logger)
self.cric = CRIC(logger=self.logger)
self.inputMap = {
"InputDataset": "primary",
"MCPileup": "secondary",
"DataPileup": "secondary"
}
self.uConfig = {}
self.campaigns = {}
self.psn2pnnMap = {}
self.pnn2psnMap = {}
self.dsetCounter = 0
self.blockCounter = 0
# service name used to route alerts via AlertManager
self.alertServiceName = "ms-transferor"
self.alertManagerUrl = self.msConfig.get("alertManagerUrl", None)
self.alertManagerApi = AlertManagerAPI(self.alertManagerUrl,
logger=logger)
def execute(self, *args, **kwargs):
self.logger.info(
"Data discovery and splitting for %s using user-provided files" %
kwargs['task']['tm_taskname'])
userfiles = kwargs['task']['tm_user_files']
splitting = kwargs['task']['tm_split_algo']
total_units = kwargs['task']['tm_totalunits']
if not userfiles or splitting != 'FileBased':
if not userfiles:
msg = "No files specified to process for task %s." % kwargs[
'task']['tm_taskname']
if splitting != 'FileBased':
msg = "Data.splitting must be set to 'FileBased' when using a custom set of files."
raise TaskWorkerException(msg)
if hasattr(self.config.Sites, 'available'):
locations = self.config.Sites.available
else:
with self.config.TaskWorker.envForCMSWEB:
configDict = {
"cacheduration": 1,
"pycurl": True
} # cache duration is in hours
resourceCatalog = CRIC(logger=self.logger,
configDict=configDict)
locations = resourceCatalog.getAllPSNs()
userFileset = Fileset(name=kwargs['task']['tm_taskname'])
self.logger.info("There are %d files specified by the user." %
len(userfiles))
if total_units > 0:
self.logger.info("Will run over the first %d files." % total_units)
file_counter = 0
for userfile, idx in zip(userfiles, range(len(userfiles))):
newFile = File(userfile, size=1000, events=1)
newFile.setLocation(locations)
newFile.addRun(Run(1, idx))
newFile["block"] = 'UserFilesFakeBlock'
newFile["first_event"] = 1
newFile["last_event"] = 2
userFileset.addFile(newFile)
file_counter += 1
if total_units > 0 and file_counter >= total_units:
break
return Result(task=kwargs['task'], result=userFileset)
def __init__(self, msConfig, logger=None):
"""
Runs the basic setup and initialization for the MS Transferor module
:param microConfig: microservice configuration
"""
super(MSTransferor, self).__init__(msConfig, logger=logger)
# minimum percentage completion for dataset/blocks subscribed
self.msConfig.setdefault("minPercentCompletion", 99)
# minimum available storage to consider a resource good for receiving data
self.msConfig.setdefault("minimumThreshold", 1 * (1000**4)) # 1TB
# limit MSTransferor to this amount of requests per cycle
self.msConfig.setdefault("limitRequestsPerCycle", 500)
# Send warning messages for any data transfer above this threshold.
# Set to negative to ignore.
self.msConfig.setdefault("warningTransferThreshold",
100. * (1000**4)) # 100TB
# Set default email settings
self.msConfig.setdefault("toAddr",
"*****@*****.**")
self.msConfig.setdefault("fromAddr", "*****@*****.**")
self.msConfig.setdefault("smtpServer", "localhost")
quotaAccount = self.msConfig["rucioAccount"]
self.rseQuotas = RSEQuotas(
quotaAccount,
self.msConfig["quotaUsage"],
minimumThreshold=self.msConfig["minimumThreshold"],
verbose=self.msConfig['verbose'],
logger=logger)
self.reqInfo = RequestInfo(self.msConfig, self.rucio, self.logger)
self.cric = CRIC(logger=self.logger)
self.inputMap = {
"InputDataset": "primary",
"MCPileup": "secondary",
"DataPileup": "secondary"
}
self.uConfig = {}
self.campaigns = {}
self.psn2pnnMap = {}
self.pnn2psnMap = {}
self.dsetCounter = 0
self.blockCounter = 0
self.emailAlert = EmailAlert(self.msConfig)
def __init__(self, **args):
StartPolicyInterface.__init__(self, **args)
self.args.setdefault('SliceType', 'NumberOfFiles')
self.args.setdefault('SliceSize', 1)
self.lumiType = "NumberOfLumis"
# Initialize a list of sites where the data is
self.sites = []
# Initialize modifiers of the policy
self.blockBlackListModifier = []
if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv(
"WMCORE_USE_CRIC", False):
self.cric = CRIC()
else:
self.cric = None
self.siteDB = SiteDB()
def __init__(self, msConfig, logger=None):
"""
Runs the basic setup and initialization for the MS Transferor module
:param microConfig: microservice configuration
"""
super(MSTransferor, self).__init__(msConfig, logger)
# url for fetching the storage quota
self.msConfig.setdefault("detoxUrl",
"http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/SitesInfo.txt")
# minimum percentage completion for dataset/blocks subscribed
self.msConfig.setdefault("minPercentCompletion", 99)
# minimum available storage to consider a resource good for receiving data
self.msConfig.setdefault("minimumThreshold", 1 * (1000 ** 4)) # 1TB
# limit MSTransferor to this amount of requests per cycle
self.msConfig.setdefault("limitRequestsPerCycle", 500)
# Send warning messages for any data transfer above this threshold.
# Set to negative to ignore.
self.msConfig.setdefault("warningTransferThreshold", 100. * (1000 ** 4)) # 100TB
# Set default email settings
self.msConfig.setdefault("toAddr", "*****@*****.**")
self.msConfig.setdefault("fromAddr", "*****@*****.**")
self.msConfig.setdefault("smtpServer", "localhost")
# enable or not the PhEDEx requests auto-approval (!request_only)
if self.msConfig.setdefault("phedexRequestOnly", True):
self.msConfig["phedexRequestOnly"] = "y"
else:
self.msConfig["phedexRequestOnly"] = "n"
self.rseQuotas = RSEQuotas(self.msConfig['detoxUrl'], self.msConfig["quotaAccount"],
self.msConfig["quotaUsage"], useRucio=self.msConfig["useRucio"],
minimumThreshold=self.msConfig["minimumThreshold"],
verbose=self.msConfig['verbose'], logger=logger)
self.reqInfo = RequestInfo(msConfig, logger)
self.cric = CRIC(logger=self.logger)
self.inputMap = {"InputDataset": "primary",
"MCPileup": "secondary",
"DataPileup": "secondary"}
self.uConfig = {}
self.campaigns = {}
self.psn2pnnMap = {}
self.dsetCounter = 0
self.blockCounter = 0
self.emailAlert = EmailAlert(self.msConfig)
def getDatasetLocationsByAccount(self, dataset, account):
"""
Returns the dataset locations for the given account in terms of computing element (not RSE name).
This function assumes that the returned RSE expression includes only one RSE
"""
try:
rules = self.list_did_rules(self.scope, dataset)
RSEs = []
for rule in rules:
if rule['account'] == account:
RSEs.append(rule['rse_expression'])
#RSE to CE conversion
cric = CRIC()
CEs = cric.PNNstoPSNs(RSEs)
except Exception as e:
print "Exception while getting the dataset location"
print(str(e))
return []
return CEs
def __init__(self, **args):
PolicyInterface.__init__(self, **args)
self.workQueueElements = []
self.wmspec = None
self.team = None
self.initialTask = None
self.splitParams = None
self.dbs_pool = {}
self.data = {}
self.lumi = None
self.couchdb = None
self.rejectedWork = [] # List of inputs that were rejected
self.badWork = [
] # list of bad work unit (e.g. without any valid files)
self.pileupData = {}
self.cric = CRIC()
if usingRucio():
self.rucio = Rucio(self.args['rucioAcct'],
configDict={'logger': self.logger})
else:
self.phedex = PhEDEx() # this will go away eventually
def __init__(self, msConfig, logger=None):
"""
Runs the basic setup and initialization for the MS Transferor module
:param microConfig: microservice configuration
"""
super(MSTransferor, self).__init__(msConfig, logger)
# url for fetching the storage quota
self.msConfig.setdefault(
"detoxUrl",
"http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/SitesInfo.txt")
# minimum percentage completion for dataset/blocks subscribed
self.msConfig.setdefault("minPercentCompletion", 99)
# minimum available storage to consider a resource good for receiving data
self.msConfig.setdefault("minimumThreshold", 1 * (1000**4)) # 1TB
# limit MSTransferor to this amount of requests per cycle
self.msConfig.setdefault("limitRequestsPerCycle", 500)
self.rseQuotas = RSEQuotas(
self.msConfig['detoxUrl'],
self.msConfig["quotaAccount"],
self.msConfig["quotaUsage"],
useRucio=self.msConfig["useRucio"],
minimumThreshold=self.msConfig["minimumThreshold"],
verbose=self.msConfig['verbose'],
logger=logger)
self.reqInfo = RequestInfo(msConfig, logger)
self.cric = CRIC(logger=self.logger)
self.inputMap = {
"InputDataset": "primary",
"MCPileup": "secondary",
"DataPileup": "secondary"
}
self.uConfig = {}
self.campaigns = {}
self.psn2pnnMap = {}
self.dsetCounter = 0
self.blockCounter = 0
def cmsSiteNames():
"""Get all cms sites"""
global __cmsSiteNames
if __cmsSiteNames:
return __cmsSiteNames
logging.info("cmsSiteNames Using CRIC Service: %s", __USE_CRIC)
global __sitedb
if not __sitedb:
if __USE_CRIC:
from WMCore.Services.CRIC.CRIC import CRIC
__sitedb = CRIC()
else:
from WMCore.Services.SiteDB.SiteDB import SiteDBJSON as SiteDB
__sitedb = SiteDB()
try:
if __USE_CRIC:
__cmsSiteNames = __sitedb.getAllPSNs()
else:
__cmsSiteNames = __sitedb.getAllCMSNames()
except Exception:
pass
return __cmsSiteNames
def __init__(self, **args):
# We need to pop this object instance from args because otherwise
# the super class blows up when doing a deepcopy(args)
self.rucio = args.pop("rucioObject", None)
PolicyInterface.__init__(self, **args)
self.workQueueElements = []
self.wmspec = None
self.team = None
self.initialTask = None
self.splitParams = None
self.dbs_pool = {}
self.data = {}
self.lumi = None
self.couchdb = None
self.rejectedWork = [] # List of inputs that were rejected
self.badWork = [
] # list of bad work unit (e.g. without any valid files)
self.pileupData = {}
self.cric = CRIC()
# FIXME: for the moment, it will always use the default value
self.rucioAcct = self.args.get("rucioAcct", "wmcore_transferor")
if not self.rucio:
self.rucio = Rucio(self.rucioAcct,
configDict={'logger': self.logger})
def setUp(self):
"""
Setup for unit tests
"""
super(CRICTest, self).setUp()
self.myCRIC = CRIC()
def formatOutput(self, task, requestname, datasetfiles, locations,
tempDir):
"""
Receives as input the result of the data location
discovery operations and fill up the WMCore objects.
"""
self.logger.debug(" Formatting data discovery output ")
wmfiles = []
event_counter = 0
lumi_counter = 0
uniquelumis = set()
datasetLumis = {}
blocksWithNoLocations = set()
## Loop over the sorted list of files.
configDict = {
"cacheduration": 1,
"pycurl": True
} # cache duration is in hours
with tempSetLogLevel(logger=self.logger, level=logging.ERROR):
resourceCatalog = CRIC(logger=self.logger, configDict=configDict)
# can't affort one message from CRIC per file, unless critical !
with tempSetLogLevel(logger=self.logger, level=logging.ERROR):
for lfn, infos in datasetfiles.iteritems():
## Skip the file if it is not in VALID state.
if not infos.get('ValidFile', True):
self.logger.warning("Skipping invalid file %s", lfn)
continue
## Skip the file if the block has not been found or has no locations.
if not infos['BlockName'] in locations or not locations[
infos['BlockName']]:
self.logger.warning(
"Skipping %s because its block (%s) has no locations",
lfn, infos['BlockName'])
blocksWithNoLocations.add(infos['BlockName'])
continue
if task['tm_use_parent'] == 1 and len(infos['Parents']) == 0:
self.logger.warning(
"Skipping %s because it has no parents")
continue
## Create a WMCore File object.
size = infos['FileSize']
checksums = {
'Checksum': infos['Checksum'],
'Adler32': infos['Adler32'],
'Md5': infos['Md5']
}
wmfile = File(lfn=lfn,
events=infos['NumberOfEvents'],
size=size,
checksums=checksums,
parents=infos['Parents'])
wmfile['block'] = infos['BlockName']
try:
wmfile['locations'] = resourceCatalog.PNNstoPSNs(
locations[wmfile['block']])
except Exception as ex:
self.logger.error(
"Impossible translating %s to a CMS name through CMS Resource Catalog",
locations[wmfile['block']])
self.logger.error("got this exception:\n %s", ex)
raise
wmfile['workflow'] = requestname
event_counter += infos['NumberOfEvents']
for run, lumis in infos['Lumis'].iteritems():
datasetLumis.setdefault(run, []).extend(lumis)
wmfile.addRun(Run(run, *lumis))
for lumi in lumis:
uniquelumis.add((run, lumi))
lumi_counter += len(lumis)
wmfiles.append(wmfile)
if blocksWithNoLocations:
msg = "%d blocks will be skipped because are not completely replicated on DISK: %s" % (
len(blocksWithNoLocations), list(blocksWithNoLocations))
self.logger.warning(msg)
self.uploadWarning(msg, task['user_proxy'], task['tm_taskname'])
uniquelumis = len(uniquelumis)
self.logger.debug('Tot events found: %d', event_counter)
self.logger.debug('Tot lumis found: %d', uniquelumis)
self.logger.debug('Duplicate lumis found: %d',
(lumi_counter - uniquelumis))
self.logger.debug('Tot files found: %d', len(wmfiles))
self.logger.debug(
"Starting to create compact lumilists for input dataset")
datasetLumiList = LumiList(runsAndLumis=datasetLumis)
datasetLumis = datasetLumiList.getCompactList()
datasetDuplicateLumis = datasetLumiList.getDuplicates().getCompactList(
)
self.logger.debug(
"Finished to create compact lumilists for input dataset")
with open(os.path.join(tempDir, "input_dataset_lumis.json"),
"w") as fd:
json.dump(datasetLumis, fd)
with open(os.path.join(tempDir, "input_dataset_duplicate_lumis.json"),
"w") as fd:
json.dump(datasetDuplicateLumis, fd)
return Result(task=task,
result=Fileset(name='FilesToSplit', files=set(wmfiles)))