def wrapped_func(*args, **kwargs): if 'cric' in services and ( not args[0].allCMSNames.sites or (args[0].allCMSNames.cachetime + 1800 < mktime(gmtime()))): args[0].allCMSNames = CMSSitesCache(sites=CRIC().getAllPSNs(), cachetime=mktime(gmtime())) args[0].allPNNNames = CMSSitesCache( sites=CRIC().getAllPhEDExNodeNames(), cachetime=mktime(gmtime())) if 'phedex' in services and not args[0].phedex: phdict = args[0].phedexargs phdict.update({'cert': serverCert, 'key': serverKey}) args[0].phedex = PhEDEx(responseType='xml', dict=phdict) if 'centralconfig' in services and ( not args[0].centralcfg.centralconfig or (args[0].centralcfg.cachetime + 1800 < mktime(gmtime()))): args[0].centralcfg = ConfigCache( centralconfig=getCentralConfig( extconfigurl=args[0].config.extconfigurl, mode=args[0].config.mode), cachetime=mktime(gmtime())) if 'servercert' in services: args[0].serverCert = serverCert args[0].serverKey = serverKey return func(*args, **kwargs)
def __init__(self, **args): StartPolicyInterface.__init__(self, **args) self.args.setdefault('SliceType', 'NumberOfRuns') self.args.setdefault('SliceSize', 1) self.lumiType = "NumberOfLumis" self.sites = [] self.cric = CRIC()
def __init__(self, **args): StartPolicyInterface.__init__(self, **args) self.args.setdefault('SliceType', 'NumberOfFiles') self.args.setdefault('SliceSize', 1) self.args.setdefault('SplittingAlgo', 'LumiBased') self.lumiType = "NumberOfLumis" # Define how to handle the different splitting algorithms self.algoMapping = { 'Harvest': self.singleChunk, 'ParentlessMergeBySize': self.singleChunk, 'MinFileBased': self.singleChunk, 'LumiBased': self.singleChunk, 'EventAwareLumiBased': self.singleChunk, 'EventBased': self.singleChunk } self.unsupportedAlgos = ['WMBSMergeBySize', 'SiblingProcessingBased'] self.defaultAlgo = self.fixedSizeChunk self.sites = [] if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv( "WMCORE_USE_CRIC", False): self.cric = CRIC() else: self.cric = None self.siteDB = SiteDB()
def setUp(self): """ _setUp_ Setup couchdb and the test environment """ super(ResubmitBlockTest, self).setUp() self.group = 'unknown' self.user = '******' # Set external test helpers self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser") # Define test environment self.couchUrl = os.environ["COUCHURL"] self.acdcDBName = 'resubmitblock_t' self.validLocations = [ 'T2_US_Nebraska', 'T1_US_FNAL_Disk', 'T1_UK_RAL_Disk' ] self.siteWhitelist = ['T2_XX_SiteA'] cric = CRIC() # Convert phedex node name to a valid processing site name self.PSNs = cric.PNNstoPSNs(self.validLocations) self.workflowName = 'dballest_ReReco_workflow' couchServer = CouchServer(dburl=self.couchUrl) self.acdcDB = couchServer.connectDatabase(self.acdcDBName, create=False) user = makeUser(self.group, '*****@*****.**', self.couchUrl, self.acdcDBName) user.create() return
def main(): problemElems = getProblematicRequests() print("Found %d bad elements that needs fixup" % len(problemElems)) if not problemElems: print("Nothing to fix, contact a developer if the problem persists...") return 0 cric = CRIC() dbsUrl = "https://cmsweb.cern.ch/dbs/prod/phys03/DBSReader" dbs = DBSReader(dbsUrl) for elem in problemElems: print("Handling id: %s, with inputs: %s" % (elem.id, elem['Inputs'])) for dataItem in elem['Inputs']: if isDataset(dataItem): pnns = dbs.listDatasetLocation(dataItem, dbsOnly=True) else: pnns = dbs.listFileBlockLocation(dataItem, dbsOnly=True) psns = cric.PNNstoPSNs(pnns) print(" PNNs: %s map to PSNs: %s" % (pnns, psns)) elem['Inputs'][dataItem] = psns backend.saveElements(*problemElems) print("Done") return 0
def __init__(self, msConfig, mode, logger=None): """ Runs the basic setup and initialization for the MSOutput module :microConfig: microservice configuration :mode: MSOutput Run mode: - MSOutputConsumer: Reads The workflow and transfer subscriptions from MongoDB and makes transfer subscriptions. - MSOutputProducer: Fetches Workflows in a given status from Reqmgr2 then creates and uploads the documents to MongoDB. """ super(MSOutput, self).__init__(msConfig, logger) self.mode = mode self.msConfig.setdefault("limitRequestsPerCycle", 500) self.msConfig.setdefault("verbose", True) self.msConfig.setdefault("interval", 600) self.msConfig.setdefault("services", ['output']) self.msConfig.setdefault("defaultDataManSys", "DDM") self.msConfig.setdefault("defaultGroup", "DataOps") self.msConfig.setdefault("enableAggSubscr", True) self.msConfig.setdefault("enableDataPlacement", False) self.msConfig.setdefault("excludeDataTier", ['NANOAOD', 'NANOAODSIM']) self.msConfig.setdefault("rucioAccount", 'wma_test') self.msConfig.setdefault("mongoDBUrl", 'mongodb://localhost') self.msConfig.setdefault("mongoDBPort", 8230) self.msConfig.setdefault("streamerBufferFile", None) self.uConfig = {} self.emailAlert = EmailAlert(self.msConfig) self.cric = CRIC(logger=self.logger) self.uConfig = {} self.campaigns = {} self.psn2pnnMap = {} msOutIndex = IndexModel('RequestName', unique=True) msOutDBConfig = { 'database': 'msOutDB', 'server': self.msConfig['mongoDBUrl'], 'port': self.msConfig['mongoDBPort'], 'logger': self.logger, 'create': True, 'collections': [('msOutRelValColl', msOutIndex), ('msOutNonRelValColl', msOutIndex)] } self.msOutDB = MongoDB(**msOutDBConfig).msOutDB self.msOutRelValColl = self.msOutDB['msOutRelValColl'] self.msOutNonRelValColl = self.msOutDB['msOutNonRelValColl'] self.ddm = DDM( url=self.msConfig['ddmUrl'], logger=self.logger, enableDataPlacement=self.msConfig['enableDataPlacement'])
def setUp(self): """ Setup for unit tests """ super(CRICTest, self).setUp() self.myCRIC = CRIC() if PY3: self.assertItemsEqual = self.assertCountEqual
def __init__(self, *args, **kwargs): TaskAction.__init__(self, *args, **kwargs) with self.config.TaskWorker.envForCMSWEB: configDict = { "cacheduration": 1, "pycurl": True } # cache duration is in hours self.resourceCatalog = CRIC(logger=self.logger, configDict=configDict)
def sites(): "Return known CMS site list from CRIC" try: # Download a list of all the sites from CRIC cric = CRIC() site_list = sorted(cric.getAllPSNs()) except Exception as exc: msg = "ERROR: Could not retrieve sites from CRIC, reason: %s" % str( exc) raise Exception(msg) return site_list
def __init__(self, **args): StartPolicyInterface.__init__(self, **args) self.args.setdefault('SliceType', 'NumberOfRuns') self.args.setdefault('SliceSize', 1) self.lumiType = "NumberOfLumis" self.sites = [] if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv( "WMCORE_USE_CRIC", False): self.cric = CRIC() else: self.cric = None self.siteDB = SiteDB()
def wrapped_func(*args, **kwargs): if 'cric' in services and ( not args[0].allCMSNames.sites or (args[0].allCMSNames.cachetime + 1800 < mktime(gmtime()))): args[0].allCMSNames = CMSSitesCache(sites=CRIC().getAllPSNs(), cachetime=mktime(gmtime())) args[0].allPNNNames = CMSSitesCache( sites=CRIC().getAllPhEDExNodeNames(), cachetime=mktime(gmtime())) if 'centralconfig' in services and ( not args[0].centralcfg.centralconfig or (args[0].centralcfg.cachetime + 1800 < mktime(gmtime()))): args[0].centralcfg = ConfigCache( centralconfig=getCentralConfig( extconfigurl=args[0].config.extconfigurl, mode=args[0].config.mode), cachetime=mktime(gmtime())) if 'servercert' in services: args[0].serverCert = serverCert args[0].serverKey = serverKey return func(*args, **kwargs)
def pnns(): """ Returns all PhEDEx node names, excluding Buffer endpoints """ cric = CRIC() try: pnn_list = sorted(cric.getAllPhEDExNodeNames(excludeBuffer=True)) except Exception as exc: msg = "ERROR: Could not retrieve PNNs from CRIC, reason: %s" % str(exc) raise Exception(msg) return pnn_list
def __init__(self, **args): StartPolicyInterface.__init__(self, **args) self.args.setdefault('SliceType', 'NumberOfFiles') self.args.setdefault('SliceSize', 1) self.lumiType = "NumberOfLumis" # Initialize a list of sites where the data is self.sites = [] # Initialize modifiers of the policy self.blockBlackListModifier = [] self.cric = CRIC()
def sites(): "Return known CMS site list from SiteDB" try: # Download a list of all the sites from SiteDB, uses v2 API. if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv("WMCORE_USE_CRIC", False): cric = CRIC() site_list = sorted(cric.getAllPSNs()) else: sitedb = SiteDBJSON() site_list = sorted(sitedb.getAllCMSNames()) except Exception as exc: msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % str(exc) raise Exception(msg) return site_list
def cmsSiteNames(): """Get all cms sites""" global __cmsSiteNames if __cmsSiteNames: return __cmsSiteNames global __cric if not __cric: __cric = CRIC() try: __cmsSiteNames = __cric.getAllPSNs() except Exception: pass return __cmsSiteNames
def getDNFromUserName(username, log, ckey=None, cert=None): """ Parse site string to know the fts server to use """ dn = '' with newX509env(X509_USER_CERT=cert, X509_USER_KEY=ckey): resourceCatalog = CRIC(logger=log) try: dn = resourceCatalog.userNameDn(username) except: log.error("CRIC URL cannot be accessed") if not dn: log.error("user does not exist") return dn
def pnns(): """ Returns all PhEDEx node names, excluding Buffer endpoints """ if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv("WMCORE_USE_CRIC", False): sitedb = CRIC() # FIXME: rename it to cric else: sitedb = SiteDBJSON() try: pnn_list = sorted(sitedb.getAllPhEDExNodeNames(excludeBuffer=True)) except Exception as exc: msg = "ERROR: Could not retrieve PNNs from SiteDB, reason: %s" % str(exc) raise Exception(msg) return pnn_list
def testConfig(self): """ Test service attributes and the override mechanism """ self.assertEqual(self.myCRIC['endpoint'], 'https://cms-cric.cern.ch/') self.assertEqual(self.myCRIC['cacheduration'], 1) self.assertEqual(self.myCRIC['accept_type'], 'application/json') self.assertEqual(self.myCRIC['content_type'], 'application/json') self.assertEqual(self.myCRIC['requests'].pycurl, None) newParams = {"cacheduration": 100, "pycurl": True} cric = CRIC(url='https://BLAH.cern.ch/', configDict=newParams) self.assertEqual(cric['endpoint'], 'https://BLAH.cern.ch/') self.assertEqual(cric['cacheduration'], newParams['cacheduration']) self.assertTrue(cric['requests'].pycurl)
def __init__(self, msConfig, logger=None): """ Runs the basic setup and initialization for the MS Transferor module :param microConfig: microservice configuration """ super(MSTransferor, self).__init__(msConfig, logger=logger) # minimum percentage completion for dataset/blocks subscribed self.msConfig.setdefault("minPercentCompletion", 99) # minimum available storage to consider a resource good for receiving data self.msConfig.setdefault("minimumThreshold", 1 * (1000**4)) # 1TB # limit MSTransferor to this amount of requests per cycle self.msConfig.setdefault("limitRequestsPerCycle", 500) # Send warning messages for any data transfer above this threshold. # Set to negative to ignore. self.msConfig.setdefault("warningTransferThreshold", 100. * (1000**4)) # 100TB # weight expression for the input replication rules self.msConfig.setdefault("rucioRuleWeight", 'ddm_quota') quotaAccount = self.msConfig["rucioAccount"] self.rseQuotas = RSEQuotas( quotaAccount, self.msConfig["quotaUsage"], minimumThreshold=self.msConfig["minimumThreshold"], verbose=self.msConfig['verbose'], logger=logger) self.reqInfo = RequestInfo(self.msConfig, self.rucio, self.logger) self.cric = CRIC(logger=self.logger) self.inputMap = { "InputDataset": "primary", "MCPileup": "secondary", "DataPileup": "secondary" } self.uConfig = {} self.campaigns = {} self.psn2pnnMap = {} self.pnn2psnMap = {} self.dsetCounter = 0 self.blockCounter = 0 # service name used to route alerts via AlertManager self.alertServiceName = "ms-transferor" self.alertManagerUrl = self.msConfig.get("alertManagerUrl", None) self.alertManagerApi = AlertManagerAPI(self.alertManagerUrl, logger=logger)
def execute(self, *args, **kwargs): self.logger.info( "Data discovery and splitting for %s using user-provided files" % kwargs['task']['tm_taskname']) userfiles = kwargs['task']['tm_user_files'] splitting = kwargs['task']['tm_split_algo'] total_units = kwargs['task']['tm_totalunits'] if not userfiles or splitting != 'FileBased': if not userfiles: msg = "No files specified to process for task %s." % kwargs[ 'task']['tm_taskname'] if splitting != 'FileBased': msg = "Data.splitting must be set to 'FileBased' when using a custom set of files." raise TaskWorkerException(msg) if hasattr(self.config.Sites, 'available'): locations = self.config.Sites.available else: with self.config.TaskWorker.envForCMSWEB: configDict = { "cacheduration": 1, "pycurl": True } # cache duration is in hours resourceCatalog = CRIC(logger=self.logger, configDict=configDict) locations = resourceCatalog.getAllPSNs() userFileset = Fileset(name=kwargs['task']['tm_taskname']) self.logger.info("There are %d files specified by the user." % len(userfiles)) if total_units > 0: self.logger.info("Will run over the first %d files." % total_units) file_counter = 0 for userfile, idx in zip(userfiles, range(len(userfiles))): newFile = File(userfile, size=1000, events=1) newFile.setLocation(locations) newFile.addRun(Run(1, idx)) newFile["block"] = 'UserFilesFakeBlock' newFile["first_event"] = 1 newFile["last_event"] = 2 userFileset.addFile(newFile) file_counter += 1 if total_units > 0 and file_counter >= total_units: break return Result(task=kwargs['task'], result=userFileset)
def __init__(self, msConfig, logger=None): """ Runs the basic setup and initialization for the MS Transferor module :param microConfig: microservice configuration """ super(MSTransferor, self).__init__(msConfig, logger=logger) # minimum percentage completion for dataset/blocks subscribed self.msConfig.setdefault("minPercentCompletion", 99) # minimum available storage to consider a resource good for receiving data self.msConfig.setdefault("minimumThreshold", 1 * (1000**4)) # 1TB # limit MSTransferor to this amount of requests per cycle self.msConfig.setdefault("limitRequestsPerCycle", 500) # Send warning messages for any data transfer above this threshold. # Set to negative to ignore. self.msConfig.setdefault("warningTransferThreshold", 100. * (1000**4)) # 100TB # Set default email settings self.msConfig.setdefault("toAddr", "*****@*****.**") self.msConfig.setdefault("fromAddr", "*****@*****.**") self.msConfig.setdefault("smtpServer", "localhost") quotaAccount = self.msConfig["rucioAccount"] self.rseQuotas = RSEQuotas( quotaAccount, self.msConfig["quotaUsage"], minimumThreshold=self.msConfig["minimumThreshold"], verbose=self.msConfig['verbose'], logger=logger) self.reqInfo = RequestInfo(self.msConfig, self.rucio, self.logger) self.cric = CRIC(logger=self.logger) self.inputMap = { "InputDataset": "primary", "MCPileup": "secondary", "DataPileup": "secondary" } self.uConfig = {} self.campaigns = {} self.psn2pnnMap = {} self.pnn2psnMap = {} self.dsetCounter = 0 self.blockCounter = 0 self.emailAlert = EmailAlert(self.msConfig)
def __init__(self, **args): StartPolicyInterface.__init__(self, **args) self.args.setdefault('SliceType', 'NumberOfFiles') self.args.setdefault('SliceSize', 1) self.lumiType = "NumberOfLumis" # Initialize a list of sites where the data is self.sites = [] # Initialize modifiers of the policy self.blockBlackListModifier = [] if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv( "WMCORE_USE_CRIC", False): self.cric = CRIC() else: self.cric = None self.siteDB = SiteDB()
def __init__(self, msConfig, logger=None): """ Runs the basic setup and initialization for the MS Transferor module :param microConfig: microservice configuration """ super(MSTransferor, self).__init__(msConfig, logger) # url for fetching the storage quota self.msConfig.setdefault("detoxUrl", "http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/SitesInfo.txt") # minimum percentage completion for dataset/blocks subscribed self.msConfig.setdefault("minPercentCompletion", 99) # minimum available storage to consider a resource good for receiving data self.msConfig.setdefault("minimumThreshold", 1 * (1000 ** 4)) # 1TB # limit MSTransferor to this amount of requests per cycle self.msConfig.setdefault("limitRequestsPerCycle", 500) # Send warning messages for any data transfer above this threshold. # Set to negative to ignore. self.msConfig.setdefault("warningTransferThreshold", 100. * (1000 ** 4)) # 100TB # Set default email settings self.msConfig.setdefault("toAddr", "*****@*****.**") self.msConfig.setdefault("fromAddr", "*****@*****.**") self.msConfig.setdefault("smtpServer", "localhost") # enable or not the PhEDEx requests auto-approval (!request_only) if self.msConfig.setdefault("phedexRequestOnly", True): self.msConfig["phedexRequestOnly"] = "y" else: self.msConfig["phedexRequestOnly"] = "n" self.rseQuotas = RSEQuotas(self.msConfig['detoxUrl'], self.msConfig["quotaAccount"], self.msConfig["quotaUsage"], useRucio=self.msConfig["useRucio"], minimumThreshold=self.msConfig["minimumThreshold"], verbose=self.msConfig['verbose'], logger=logger) self.reqInfo = RequestInfo(msConfig, logger) self.cric = CRIC(logger=self.logger) self.inputMap = {"InputDataset": "primary", "MCPileup": "secondary", "DataPileup": "secondary"} self.uConfig = {} self.campaigns = {} self.psn2pnnMap = {} self.dsetCounter = 0 self.blockCounter = 0 self.emailAlert = EmailAlert(self.msConfig)
def getDatasetLocationsByAccount(self, dataset, account): """ Returns the dataset locations for the given account in terms of computing element (not RSE name). This function assumes that the returned RSE expression includes only one RSE """ try: rules = self.list_did_rules(self.scope, dataset) RSEs = [] for rule in rules: if rule['account'] == account: RSEs.append(rule['rse_expression']) #RSE to CE conversion cric = CRIC() CEs = cric.PNNstoPSNs(RSEs) except Exception as e: print "Exception while getting the dataset location" print(str(e)) return [] return CEs
def __init__(self, **args): PolicyInterface.__init__(self, **args) self.workQueueElements = [] self.wmspec = None self.team = None self.initialTask = None self.splitParams = None self.dbs_pool = {} self.data = {} self.lumi = None self.couchdb = None self.rejectedWork = [] # List of inputs that were rejected self.badWork = [ ] # list of bad work unit (e.g. without any valid files) self.pileupData = {} self.cric = CRIC() if usingRucio(): self.rucio = Rucio(self.args['rucioAcct'], configDict={'logger': self.logger}) else: self.phedex = PhEDEx() # this will go away eventually
def __init__(self, msConfig, logger=None): """ Runs the basic setup and initialization for the MS Transferor module :param microConfig: microservice configuration """ super(MSTransferor, self).__init__(msConfig, logger) # url for fetching the storage quota self.msConfig.setdefault( "detoxUrl", "http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/SitesInfo.txt") # minimum percentage completion for dataset/blocks subscribed self.msConfig.setdefault("minPercentCompletion", 99) # minimum available storage to consider a resource good for receiving data self.msConfig.setdefault("minimumThreshold", 1 * (1000**4)) # 1TB # limit MSTransferor to this amount of requests per cycle self.msConfig.setdefault("limitRequestsPerCycle", 500) self.rseQuotas = RSEQuotas( self.msConfig['detoxUrl'], self.msConfig["quotaAccount"], self.msConfig["quotaUsage"], useRucio=self.msConfig["useRucio"], minimumThreshold=self.msConfig["minimumThreshold"], verbose=self.msConfig['verbose'], logger=logger) self.reqInfo = RequestInfo(msConfig, logger) self.cric = CRIC(logger=self.logger) self.inputMap = { "InputDataset": "primary", "MCPileup": "secondary", "DataPileup": "secondary" } self.uConfig = {} self.campaigns = {} self.psn2pnnMap = {} self.dsetCounter = 0 self.blockCounter = 0
def cmsSiteNames(): """Get all cms sites""" global __cmsSiteNames if __cmsSiteNames: return __cmsSiteNames logging.info("cmsSiteNames Using CRIC Service: %s", __USE_CRIC) global __sitedb if not __sitedb: if __USE_CRIC: from WMCore.Services.CRIC.CRIC import CRIC __sitedb = CRIC() else: from WMCore.Services.SiteDB.SiteDB import SiteDBJSON as SiteDB __sitedb = SiteDB() try: if __USE_CRIC: __cmsSiteNames = __sitedb.getAllPSNs() else: __cmsSiteNames = __sitedb.getAllCMSNames() except Exception: pass return __cmsSiteNames
def __init__(self, **args): # We need to pop this object instance from args because otherwise # the super class blows up when doing a deepcopy(args) self.rucio = args.pop("rucioObject", None) PolicyInterface.__init__(self, **args) self.workQueueElements = [] self.wmspec = None self.team = None self.initialTask = None self.splitParams = None self.dbs_pool = {} self.data = {} self.lumi = None self.couchdb = None self.rejectedWork = [] # List of inputs that were rejected self.badWork = [ ] # list of bad work unit (e.g. without any valid files) self.pileupData = {} self.cric = CRIC() # FIXME: for the moment, it will always use the default value self.rucioAcct = self.args.get("rucioAcct", "wmcore_transferor") if not self.rucio: self.rucio = Rucio(self.rucioAcct, configDict={'logger': self.logger})
def setUp(self): """ Setup for unit tests """ super(CRICTest, self).setUp() self.myCRIC = CRIC()
def formatOutput(self, task, requestname, datasetfiles, locations, tempDir): """ Receives as input the result of the data location discovery operations and fill up the WMCore objects. """ self.logger.debug(" Formatting data discovery output ") wmfiles = [] event_counter = 0 lumi_counter = 0 uniquelumis = set() datasetLumis = {} blocksWithNoLocations = set() ## Loop over the sorted list of files. configDict = { "cacheduration": 1, "pycurl": True } # cache duration is in hours with tempSetLogLevel(logger=self.logger, level=logging.ERROR): resourceCatalog = CRIC(logger=self.logger, configDict=configDict) # can't affort one message from CRIC per file, unless critical ! with tempSetLogLevel(logger=self.logger, level=logging.ERROR): for lfn, infos in datasetfiles.iteritems(): ## Skip the file if it is not in VALID state. if not infos.get('ValidFile', True): self.logger.warning("Skipping invalid file %s", lfn) continue ## Skip the file if the block has not been found or has no locations. if not infos['BlockName'] in locations or not locations[ infos['BlockName']]: self.logger.warning( "Skipping %s because its block (%s) has no locations", lfn, infos['BlockName']) blocksWithNoLocations.add(infos['BlockName']) continue if task['tm_use_parent'] == 1 and len(infos['Parents']) == 0: self.logger.warning( "Skipping %s because it has no parents") continue ## Create a WMCore File object. size = infos['FileSize'] checksums = { 'Checksum': infos['Checksum'], 'Adler32': infos['Adler32'], 'Md5': infos['Md5'] } wmfile = File(lfn=lfn, events=infos['NumberOfEvents'], size=size, checksums=checksums, parents=infos['Parents']) wmfile['block'] = infos['BlockName'] try: wmfile['locations'] = resourceCatalog.PNNstoPSNs( locations[wmfile['block']]) except Exception as ex: self.logger.error( "Impossible translating %s to a CMS name through CMS Resource Catalog", locations[wmfile['block']]) self.logger.error("got this exception:\n %s", ex) raise wmfile['workflow'] = requestname event_counter += infos['NumberOfEvents'] for run, lumis in infos['Lumis'].iteritems(): datasetLumis.setdefault(run, []).extend(lumis) wmfile.addRun(Run(run, *lumis)) for lumi in lumis: uniquelumis.add((run, lumi)) lumi_counter += len(lumis) wmfiles.append(wmfile) if blocksWithNoLocations: msg = "%d blocks will be skipped because are not completely replicated on DISK: %s" % ( len(blocksWithNoLocations), list(blocksWithNoLocations)) self.logger.warning(msg) self.uploadWarning(msg, task['user_proxy'], task['tm_taskname']) uniquelumis = len(uniquelumis) self.logger.debug('Tot events found: %d', event_counter) self.logger.debug('Tot lumis found: %d', uniquelumis) self.logger.debug('Duplicate lumis found: %d', (lumi_counter - uniquelumis)) self.logger.debug('Tot files found: %d', len(wmfiles)) self.logger.debug( "Starting to create compact lumilists for input dataset") datasetLumiList = LumiList(runsAndLumis=datasetLumis) datasetLumis = datasetLumiList.getCompactList() datasetDuplicateLumis = datasetLumiList.getDuplicates().getCompactList( ) self.logger.debug( "Finished to create compact lumilists for input dataset") with open(os.path.join(tempDir, "input_dataset_lumis.json"), "w") as fd: json.dump(datasetLumis, fd) with open(os.path.join(tempDir, "input_dataset_duplicate_lumis.json"), "w") as fd: json.dump(datasetDuplicateLumis, fd) return Result(task=task, result=Fileset(name='FilesToSplit', files=set(wmfiles)))