def main(): problemElems = getProblematicRequests() print("Found %d bad elements that needs fixup" % len(problemElems)) if not problemElems: print("Nothing to fix, contact a developer if the problem persists...") return 0 cric = CRIC() dbsUrl = "https://cmsweb.cern.ch/dbs/prod/phys03/DBSReader" dbs = DBSReader(dbsUrl) for elem in problemElems: print("Handling id: %s, with inputs: %s" % (elem.id, elem['Inputs'])) for dataItem in elem['Inputs']: if isDataset(dataItem): pnns = dbs.listDatasetLocation(dataItem, dbsOnly=True) else: pnns = dbs.listFileBlockLocation(dataItem, dbsOnly=True) psns = cric.PNNstoPSNs(pnns) print(" PNNs: %s map to PSNs: %s" % (pnns, psns)) elem['Inputs'][dataItem] = psns backend.saveElements(*problemElems) print("Done") return 0
def __init__(self, **args): StartPolicyInterface.__init__(self, **args) self.args.setdefault('SliceType', 'NumberOfFiles') self.args.setdefault('SliceSize', 1) self.args.setdefault('SplittingAlgo', 'LumiBased') self.lumiType = "NumberOfLumis" # Define how to handle the different splitting algorithms self.algoMapping = { 'Harvest': self.singleChunk, 'ParentlessMergeBySize': self.singleChunk, 'MinFileBased': self.singleChunk, 'LumiBased': self.singleChunk, 'EventAwareLumiBased': self.singleChunk, 'EventBased': self.singleChunk } self.unsupportedAlgos = ['WMBSMergeBySize', 'SiblingProcessingBased'] self.defaultAlgo = self.fixedSizeChunk self.sites = [] if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv( "WMCORE_USE_CRIC", False): self.cric = CRIC() else: self.cric = None self.siteDB = SiteDB()
def setUp(self): """ _setUp_ Setup couchdb and the test environment """ super(ResubmitBlockTest, self).setUp() self.group = 'unknown' self.user = '******' # Set external test helpers self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser") # Define test environment self.couchUrl = os.environ["COUCHURL"] self.acdcDBName = 'resubmitblock_t' self.validLocations = [ 'T2_US_Nebraska', 'T1_US_FNAL_Disk', 'T1_UK_RAL_Disk' ] self.siteWhitelist = ['T2_XX_SiteA'] cric = CRIC() # Convert phedex node name to a valid processing site name self.PSNs = cric.PNNstoPSNs(self.validLocations) self.workflowName = 'dballest_ReReco_workflow' couchServer = CouchServer(dburl=self.couchUrl) self.acdcDB = couchServer.connectDatabase(self.acdcDBName, create=False) user = makeUser(self.group, '*****@*****.**', self.couchUrl, self.acdcDBName) user.create() return
def wrapped_func(*args, **kwargs): if 'cric' in services and ( not args[0].allCMSNames.sites or (args[0].allCMSNames.cachetime + 1800 < mktime(gmtime()))): args[0].allCMSNames = CMSSitesCache(sites=CRIC().getAllPSNs(), cachetime=mktime(gmtime())) args[0].allPNNNames = CMSSitesCache( sites=CRIC().getAllPhEDExNodeNames(), cachetime=mktime(gmtime())) if 'phedex' in services and not args[0].phedex: phdict = args[0].phedexargs phdict.update({'cert': serverCert, 'key': serverKey}) args[0].phedex = PhEDEx(responseType='xml', dict=phdict) if 'centralconfig' in services and ( not args[0].centralcfg.centralconfig or (args[0].centralcfg.cachetime + 1800 < mktime(gmtime()))): args[0].centralcfg = ConfigCache( centralconfig=getCentralConfig( extconfigurl=args[0].config.extconfigurl, mode=args[0].config.mode), cachetime=mktime(gmtime())) if 'servercert' in services: args[0].serverCert = serverCert args[0].serverKey = serverKey return func(*args, **kwargs)
def __init__(self, **args): StartPolicyInterface.__init__(self, **args) self.args.setdefault('SliceType', 'NumberOfRuns') self.args.setdefault('SliceSize', 1) self.lumiType = "NumberOfLumis" self.sites = [] self.cric = CRIC()
def __init__(self, msConfig, mode, logger=None): """ Runs the basic setup and initialization for the MSOutput module :microConfig: microservice configuration :mode: MSOutput Run mode: - MSOutputConsumer: Reads The workflow and transfer subscriptions from MongoDB and makes transfer subscriptions. - MSOutputProducer: Fetches Workflows in a given status from Reqmgr2 then creates and uploads the documents to MongoDB. """ super(MSOutput, self).__init__(msConfig, logger) self.mode = mode self.msConfig.setdefault("limitRequestsPerCycle", 500) self.msConfig.setdefault("verbose", True) self.msConfig.setdefault("interval", 600) self.msConfig.setdefault("services", ['output']) self.msConfig.setdefault("defaultDataManSys", "DDM") self.msConfig.setdefault("defaultGroup", "DataOps") self.msConfig.setdefault("enableAggSubscr", True) self.msConfig.setdefault("enableDataPlacement", False) self.msConfig.setdefault("excludeDataTier", ['NANOAOD', 'NANOAODSIM']) self.msConfig.setdefault("rucioAccount", 'wma_test') self.msConfig.setdefault("mongoDBUrl", 'mongodb://localhost') self.msConfig.setdefault("mongoDBPort", 8230) self.msConfig.setdefault("streamerBufferFile", None) self.uConfig = {} self.emailAlert = EmailAlert(self.msConfig) self.cric = CRIC(logger=self.logger) self.uConfig = {} self.campaigns = {} self.psn2pnnMap = {} msOutIndex = IndexModel('RequestName', unique=True) msOutDBConfig = { 'database': 'msOutDB', 'server': self.msConfig['mongoDBUrl'], 'port': self.msConfig['mongoDBPort'], 'logger': self.logger, 'create': True, 'collections': [('msOutRelValColl', msOutIndex), ('msOutNonRelValColl', msOutIndex)] } self.msOutDB = MongoDB(**msOutDBConfig).msOutDB self.msOutRelValColl = self.msOutDB['msOutRelValColl'] self.msOutNonRelValColl = self.msOutDB['msOutNonRelValColl'] self.ddm = DDM( url=self.msConfig['ddmUrl'], logger=self.logger, enableDataPlacement=self.msConfig['enableDataPlacement'])
def setUp(self): """ Setup for unit tests """ super(CRICTest, self).setUp() self.myCRIC = CRIC() if PY3: self.assertItemsEqual = self.assertCountEqual
def __init__(self, *args, **kwargs): TaskAction.__init__(self, *args, **kwargs) with self.config.TaskWorker.envForCMSWEB: configDict = { "cacheduration": 1, "pycurl": True } # cache duration is in hours self.resourceCatalog = CRIC(logger=self.logger, configDict=configDict)
def sites(): "Return known CMS site list from CRIC" try: # Download a list of all the sites from CRIC cric = CRIC() site_list = sorted(cric.getAllPSNs()) except Exception as exc: msg = "ERROR: Could not retrieve sites from CRIC, reason: %s" % str(exc) raise Exception(msg) return site_list
def sites(): "Return known CMS site list from CRIC" try: # Download a list of all the sites from CRIC cric = CRIC() site_list = sorted(cric.getAllPSNs()) except Exception as exc: msg = "ERROR: Could not retrieve sites from CRIC, reason: %s" % str( exc) raise Exception(msg) return site_list
def pnns(): """ Returns all PhEDEx node names, excluding Buffer endpoints """ cric = CRIC() try: pnn_list = sorted(cric.getAllPhEDExNodeNames(excludeBuffer=True)) except Exception as exc: msg = "ERROR: Could not retrieve PNNs from CRIC, reason: %s" % str(exc) raise Exception(msg) return pnn_list
def __init__(self, **args): StartPolicyInterface.__init__(self, **args) self.args.setdefault('SliceType', 'NumberOfRuns') self.args.setdefault('SliceSize', 1) self.lumiType = "NumberOfLumis" self.sites = [] if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv( "WMCORE_USE_CRIC", False): self.cric = CRIC() else: self.cric = None self.siteDB = SiteDB()
def __init__(self, **args): StartPolicyInterface.__init__(self, **args) self.args.setdefault('SliceType', 'NumberOfFiles') self.args.setdefault('SliceSize', 1) self.lumiType = "NumberOfLumis" # Initialize a list of sites where the data is self.sites = [] # Initialize modifiers of the policy self.blockBlackListModifier = [] self.cric = CRIC()
def pnns(): """ Returns all PhEDEx node names, excluding Buffer endpoints """ cric = CRIC() try: pnn_list = sorted(cric.getAllPhEDExNodeNames(excludeBuffer=True)) except Exception as exc: msg = "ERROR: Could not retrieve PNNs from CRIC, reason: %s" % str(exc) raise Exception(msg) return pnn_list
def sites(): "Return known CMS site list from SiteDB" try: # Download a list of all the sites from SiteDB, uses v2 API. if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv("WMCORE_USE_CRIC", False): cric = CRIC() site_list = sorted(cric.getAllPSNs()) else: sitedb = SiteDBJSON() site_list = sorted(sitedb.getAllCMSNames()) except Exception as exc: msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % str(exc) raise Exception(msg) return site_list
def getDNFromUserName(username, log, ckey=None, cert=None): """ Parse site string to know the fts server to use """ dn = '' with newX509env(X509_USER_CERT=cert, X509_USER_KEY=ckey): resourceCatalog = CRIC(logger=log) try: dn = resourceCatalog.userNameDn(username) except: log.error("CRIC URL cannot be accessed") if not dn: log.error("user does not exist") return dn
def cmsSiteNames(): """Get all cms sites""" global __cmsSiteNames if __cmsSiteNames: return __cmsSiteNames global __cric if not __cric: __cric = CRIC() try: __cmsSiteNames = __cric.getAllPSNs() except Exception: pass return __cmsSiteNames
def cmsSiteNames(): """Get all cms sites""" global __cmsSiteNames if __cmsSiteNames: return __cmsSiteNames global __cric if not __cric: __cric = CRIC() try: __cmsSiteNames = __cric.getAllPSNs() except Exception: pass return __cmsSiteNames
def pnns(): """ Returns all PhEDEx node names, excluding Buffer endpoints """ if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv("WMCORE_USE_CRIC", False): sitedb = CRIC() # FIXME: rename it to cric else: sitedb = SiteDBJSON() try: pnn_list = sorted(sitedb.getAllPhEDExNodeNames(excludeBuffer=True)) except Exception as exc: msg = "ERROR: Could not retrieve PNNs from SiteDB, reason: %s" % str(exc) raise Exception(msg) return pnn_list
def getDNFromUserName(username, log, ckey = None, cert = None): """ Parse site string to know the fts server to use """ dn = '' with newX509env(X509_USER_CERT=cert,X509_USER_KEY=ckey): configDict = {"cacheduration": 1, "pycurl": True} # cache duration is in hours resourceCatalog = CRIC(logger=self.logger, configDict=configDict) try: dn = resourceCatalog.userNameDn(username) except : log.error("CRIC URL cannot be accessed") if not dn: log.error("user does not exist") return dn
def __init__(self, msConfig, logger=None): """ Runs the basic setup and initialization for the MS Transferor module :param microConfig: microservice configuration """ super(MSTransferor, self).__init__(msConfig, logger=logger) # minimum percentage completion for dataset/blocks subscribed self.msConfig.setdefault("minPercentCompletion", 99) # minimum available storage to consider a resource good for receiving data self.msConfig.setdefault("minimumThreshold", 1 * (1000**4)) # 1TB # limit MSTransferor to this amount of requests per cycle self.msConfig.setdefault("limitRequestsPerCycle", 500) # Send warning messages for any data transfer above this threshold. # Set to negative to ignore. self.msConfig.setdefault("warningTransferThreshold", 100. * (1000**4)) # 100TB # weight expression for the input replication rules self.msConfig.setdefault("rucioRuleWeight", 'ddm_quota') quotaAccount = self.msConfig["rucioAccount"] self.rseQuotas = RSEQuotas( quotaAccount, self.msConfig["quotaUsage"], minimumThreshold=self.msConfig["minimumThreshold"], verbose=self.msConfig['verbose'], logger=logger) self.reqInfo = RequestInfo(self.msConfig, self.rucio, self.logger) self.cric = CRIC(logger=self.logger) self.inputMap = { "InputDataset": "primary", "MCPileup": "secondary", "DataPileup": "secondary" } self.uConfig = {} self.campaigns = {} self.psn2pnnMap = {} self.pnn2psnMap = {} self.dsetCounter = 0 self.blockCounter = 0 # service name used to route alerts via AlertManager self.alertServiceName = "ms-transferor" self.alertManagerUrl = self.msConfig.get("alertManagerUrl", None) self.alertManagerApi = AlertManagerAPI(self.alertManagerUrl, logger=logger)
def execute(self, *args, **kwargs): self.logger.info( "Data discovery and splitting for %s using user-provided files" % kwargs['task']['tm_taskname']) userfiles = kwargs['task']['tm_user_files'] splitting = kwargs['task']['tm_split_algo'] total_units = kwargs['task']['tm_totalunits'] if not userfiles or splitting != 'FileBased': if not userfiles: msg = "No files specified to process for task %s." % kwargs[ 'task']['tm_taskname'] if splitting != 'FileBased': msg = "Data.splitting must be set to 'FileBased' when using a custom set of files." raise TaskWorkerException(msg) if hasattr(self.config.Sites, 'available'): locations = self.config.Sites.available else: with self.config.TaskWorker.envForCMSWEB: configDict = { "cacheduration": 1, "pycurl": True } # cache duration is in hours resourceCatalog = CRIC(logger=self.logger, configDict=configDict) locations = resourceCatalog.getAllPSNs() userFileset = Fileset(name=kwargs['task']['tm_taskname']) self.logger.info("There are %d files specified by the user." % len(userfiles)) if total_units > 0: self.logger.info("Will run over the first %d files." % total_units) file_counter = 0 for userfile, idx in zip(userfiles, range(len(userfiles))): newFile = File(userfile, size=1000, events=1) newFile.setLocation(locations) newFile.addRun(Run(1, idx)) newFile["block"] = 'UserFilesFakeBlock' newFile["first_event"] = 1 newFile["last_event"] = 2 userFileset.addFile(newFile) file_counter += 1 if total_units > 0 and file_counter >= total_units: break return Result(task=kwargs['task'], result=userFileset)
def __init__(self, **args): StartPolicyInterface.__init__(self, **args) self.args.setdefault('SliceType', 'NumberOfFiles') self.args.setdefault('SliceSize', 1) self.lumiType = "NumberOfLumis" # Initialize a list of sites where the data is self.sites = [] # Initialize modifiers of the policy self.blockBlackListModifier = [] if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv( "WMCORE_USE_CRIC", False): self.cric = CRIC() else: self.cric = None self.siteDB = SiteDB()
def __init__(self, msConfig, logger=None): """ Runs the basic setup and initialization for the MS Transferor module :param microConfig: microservice configuration """ super(MSTransferor, self).__init__(msConfig, logger=logger) # minimum percentage completion for dataset/blocks subscribed self.msConfig.setdefault("minPercentCompletion", 99) # minimum available storage to consider a resource good for receiving data self.msConfig.setdefault("minimumThreshold", 1 * (1000**4)) # 1TB # limit MSTransferor to this amount of requests per cycle self.msConfig.setdefault("limitRequestsPerCycle", 500) # Send warning messages for any data transfer above this threshold. # Set to negative to ignore. self.msConfig.setdefault("warningTransferThreshold", 100. * (1000**4)) # 100TB # Set default email settings self.msConfig.setdefault("toAddr", "*****@*****.**") self.msConfig.setdefault("fromAddr", "*****@*****.**") self.msConfig.setdefault("smtpServer", "localhost") quotaAccount = self.msConfig["rucioAccount"] self.rseQuotas = RSEQuotas( quotaAccount, self.msConfig["quotaUsage"], minimumThreshold=self.msConfig["minimumThreshold"], verbose=self.msConfig['verbose'], logger=logger) self.reqInfo = RequestInfo(self.msConfig, self.rucio, self.logger) self.cric = CRIC(logger=self.logger) self.inputMap = { "InputDataset": "primary", "MCPileup": "secondary", "DataPileup": "secondary" } self.uConfig = {} self.campaigns = {} self.psn2pnnMap = {} self.pnn2psnMap = {} self.dsetCounter = 0 self.blockCounter = 0 self.emailAlert = EmailAlert(self.msConfig)
def __init__(self, msConfig, logger=None): """ Runs the basic setup and initialization for the MS Transferor module :param microConfig: microservice configuration """ super(MSTransferor, self).__init__(msConfig, logger) # url for fetching the storage quota self.msConfig.setdefault("detoxUrl", "http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/SitesInfo.txt") # minimum percentage completion for dataset/blocks subscribed self.msConfig.setdefault("minPercentCompletion", 99) # minimum available storage to consider a resource good for receiving data self.msConfig.setdefault("minimumThreshold", 1 * (1000 ** 4)) # 1TB # limit MSTransferor to this amount of requests per cycle self.msConfig.setdefault("limitRequestsPerCycle", 500) # Send warning messages for any data transfer above this threshold. # Set to negative to ignore. self.msConfig.setdefault("warningTransferThreshold", 100. * (1000 ** 4)) # 100TB # Set default email settings self.msConfig.setdefault("toAddr", "*****@*****.**") self.msConfig.setdefault("fromAddr", "*****@*****.**") self.msConfig.setdefault("smtpServer", "localhost") # enable or not the PhEDEx requests auto-approval (!request_only) if self.msConfig.setdefault("phedexRequestOnly", True): self.msConfig["phedexRequestOnly"] = "y" else: self.msConfig["phedexRequestOnly"] = "n" self.rseQuotas = RSEQuotas(self.msConfig['detoxUrl'], self.msConfig["quotaAccount"], self.msConfig["quotaUsage"], useRucio=self.msConfig["useRucio"], minimumThreshold=self.msConfig["minimumThreshold"], verbose=self.msConfig['verbose'], logger=logger) self.reqInfo = RequestInfo(msConfig, logger) self.cric = CRIC(logger=self.logger) self.inputMap = {"InputDataset": "primary", "MCPileup": "secondary", "DataPileup": "secondary"} self.uConfig = {} self.campaigns = {} self.psn2pnnMap = {} self.dsetCounter = 0 self.blockCounter = 0 self.emailAlert = EmailAlert(self.msConfig)
class MakeFakeFileSet(TaskAction): """This is needed to make WMCore.JobSplitting lib working... do not like very much. Given that all is fake here I am quite sure we only need total number of events said that I set all the other parmas to dummy values. We may want to set them in the future""" def __init__(self, *args, **kwargs): TaskAction.__init__(self, *args, **kwargs) with self.config.TaskWorker.envForCMSWEB: configDict = { "cacheduration": 1, "pycurl": True } # cache duration is in hours self.resourceCatalog = CRIC(logger=self.logger, configDict=configDict) def getListOfSites(self): """ Get the list of sites to use for PrivateMC workflows. For the moment we are filtering out T1_ since they are precious resources and don't want to overtake production (WMAgent) jobs there. In the future we would like to take this list from the SSB. """ with self.config.TaskWorker.envForCMSWEB: sites = self.resourceCatalog.getAllPSNs() filteredSites = [site for site in sites if not site.startswith("T1_")] return filteredSites #even though args is not used we call "handler.actionWork(args, kwargs)" in Handler def execute(self, *args, **kwargs): #pylint: disable=unused-argument # since https://github.com/dmwm/CRABServer/issues/5633 totalunits can be a float # but that would confuse WMCore, therefore cast to int totalevents = int(kwargs['task']['tm_totalunits']) firstEvent = 1 lastEvent = totalevents firstLumi = 1 lastLumi = 10 # Set a default of 100 events per lumi. This is set as a task # property, as the splitting considers it independently of the file # information provided by the fake dataset. if not kwargs['task']['tm_events_per_lumi']: kwargs['task']['tm_events_per_lumi'] = 100 #MC comes with only one MCFakeFile singleMCFileset = Fileset(name="MCFakeFileSet") newFile = File("MCFakeFile", size=1000, events=totalevents) newFile.setLocation(self.getListOfSites()) newFile.addRun(Run(1, *range(firstLumi, lastLumi + 1))) newFile["block"] = 'MCFakeBlock' newFile["first_event"] = firstEvent newFile["last_event"] = lastEvent singleMCFileset.addFile(newFile) return Result(task=kwargs['task'], result=singleMCFileset)
def getDatasetLocationsByAccount(self, dataset, account): """ Returns the dataset locations for the given account in terms of computing element (not RSE name). This function assumes that the returned RSE expression includes only one RSE """ try: rules = self.list_did_rules(self.scope, dataset) RSEs = [] for rule in rules: if rule['account'] == account: RSEs.append(rule['rse_expression']) #RSE to CE conversion cric = CRIC() CEs = cric.PNNstoPSNs(RSEs) except Exception as e: print "Exception while getting the dataset location" print(str(e)) return [] return CEs
class MakeFakeFileSet(TaskAction): """This is needed to make WMCore.JobSplitting lib working... do not like very much. Given that all is fake here I am quite sure we only need total number of events said that I set all the other parmas to dummy values. We may want to set them in the future""" def __init__(self, *args, **kwargs): TaskAction.__init__(self, *args, **kwargs) with self.config.TaskWorker.envForCMSWEB: configDict = {"cacheduration": 1, "pycurl": True} # cache duration is in hours self.resourceCatalog = CRIC(logger=self.logger, configDict=configDict) def getListOfSites(self): """ Get the list of sites to use for PrivateMC workflows. For the moment we are filtering out T1_ since they are precious resources and don't want to overtake production (WMAgent) jobs there. In the future we would like to take this list from the SSB. """ with self.config.TaskWorker.envForCMSWEB: sites = self.resourceCatalog.getAllPSNs() filteredSites = [site for site in sites if not site.startswith("T1_")] return filteredSites #even though args is not used we call "handler.actionWork(args, kwargs)" in Handler def execute(self, *args, **kwargs): #pylint: disable=unused-argument # since https://github.com/dmwm/CRABServer/issues/5633 totalunits can be a float # but that would confuse WMCore, therefore cast to int totalevents = int(kwargs['task']['tm_totalunits']) firstEvent = 1 lastEvent = totalevents firstLumi = 1 lastLumi = 10 # Set a default of 100 events per lumi. This is set as a task # property, as the splitting considers it independently of the file # information provided by the fake dataset. if not kwargs['task']['tm_events_per_lumi']: kwargs['task']['tm_events_per_lumi'] = 100 #MC comes with only one MCFakeFile singleMCFileset = Fileset(name = "MCFakeFileSet") newFile = File("MCFakeFile", size = 1000, events = totalevents) newFile.setLocation(self.getListOfSites()) newFile.addRun(Run(1, *range(firstLumi, lastLumi + 1))) newFile["block"] = 'MCFakeBlock' newFile["first_event"] = firstEvent newFile["last_event"] = lastEvent singleMCFileset.addFile(newFile) return Result(task=kwargs['task'], result=singleMCFileset)
def __init__(self, **args): PolicyInterface.__init__(self, **args) self.workQueueElements = [] self.wmspec = None self.team = None self.initialTask = None self.splitParams = None self.dbs_pool = {} self.data = {} self.lumi = None self.couchdb = None self.rejectedWork = [] # List of inputs that were rejected self.badWork = [ ] # list of bad work unit (e.g. without any valid files) self.pileupData = {} self.cric = CRIC() if usingRucio(): self.rucio = Rucio(self.args['rucioAcct'], configDict={'logger': self.logger}) else: self.phedex = PhEDEx() # this will go away eventually
def wrapped_func(*args, **kwargs): if 'cric' in services and ( not args[0].allCMSNames.sites or (args[0].allCMSNames.cachetime + 1800 < mktime(gmtime()))): args[0].allCMSNames = CMSSitesCache(sites=CRIC().getAllPSNs(), cachetime=mktime(gmtime())) args[0].allPNNNames = CMSSitesCache( sites=CRIC().getAllPhEDExNodeNames(), cachetime=mktime(gmtime())) if 'centralconfig' in services and ( not args[0].centralcfg.centralconfig or (args[0].centralcfg.cachetime + 1800 < mktime(gmtime()))): args[0].centralcfg = ConfigCache( centralconfig=getCentralConfig( extconfigurl=args[0].config.extconfigurl, mode=args[0].config.mode), cachetime=mktime(gmtime())) if 'servercert' in services: args[0].serverCert = serverCert args[0].serverKey = serverKey return func(*args, **kwargs)
def cmsSiteNames(): """Get all cms sites""" global __cmsSiteNames if __cmsSiteNames: return __cmsSiteNames logging.info("cmsSiteNames Using CRIC Service: %s", __USE_CRIC) global __sitedb if not __sitedb: if __USE_CRIC: from WMCore.Services.CRIC.CRIC import CRIC __sitedb = CRIC() else: from WMCore.Services.SiteDB.SiteDB import SiteDBJSON as SiteDB __sitedb = SiteDB() try: if __USE_CRIC: __cmsSiteNames = __sitedb.getAllPSNs() else: __cmsSiteNames = __sitedb.getAllCMSNames() except Exception: pass return __cmsSiteNames
def __init__(self, msConfig, logger=None): """ Runs the basic setup and initialization for the MS Transferor module :param microConfig: microservice configuration """ super(MSTransferor, self).__init__(msConfig, logger) # url for fetching the storage quota self.msConfig.setdefault( "detoxUrl", "http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/SitesInfo.txt") # minimum percentage completion for dataset/blocks subscribed self.msConfig.setdefault("minPercentCompletion", 99) # minimum available storage to consider a resource good for receiving data self.msConfig.setdefault("minimumThreshold", 1 * (1000**4)) # 1TB # limit MSTransferor to this amount of requests per cycle self.msConfig.setdefault("limitRequestsPerCycle", 500) self.rseQuotas = RSEQuotas( self.msConfig['detoxUrl'], self.msConfig["quotaAccount"], self.msConfig["quotaUsage"], useRucio=self.msConfig["useRucio"], minimumThreshold=self.msConfig["minimumThreshold"], verbose=self.msConfig['verbose'], logger=logger) self.reqInfo = RequestInfo(msConfig, logger) self.cric = CRIC(logger=self.logger) self.inputMap = { "InputDataset": "primary", "MCPileup": "secondary", "DataPileup": "secondary" } self.uConfig = {} self.campaigns = {} self.psn2pnnMap = {} self.dsetCounter = 0 self.blockCounter = 0
def execute(self, *args, **kwargs): self.logger.info("Data discovery and splitting for %s using user-provided files" % kwargs['task']['tm_taskname']) userfiles = kwargs['task']['tm_user_files'] splitting = kwargs['task']['tm_split_algo'] total_units = kwargs['task']['tm_totalunits'] if not userfiles or splitting != 'FileBased': if not userfiles: msg = "No files specified to process for task %s." % kwargs['task']['tm_taskname'] if splitting != 'FileBased': msg = "Data.splitting must be set to 'FileBased' when using a custom set of files." raise TaskWorkerException(msg) if hasattr(self.config.Sites, 'available'): locations = self.config.Sites.available else: with self.config.TaskWorker.envForCMSWEB : configDict = {"cacheduration": 1, "pycurl": True} # cache duration is in hours resourceCatalog = CRIC(logger=self.logger, configDict=configDict) locations = resourceCatalog.getAllPSNs() userFileset = Fileset(name = kwargs['task']['tm_taskname']) self.logger.info("There are %d files specified by the user." % len(userfiles)) if total_units > 0: self.logger.info("Will run over the first %d files." % total_units) file_counter = 0 for userfile, idx in zip(userfiles, range(len(userfiles))): newFile = File(userfile, size = 1000, events = 1) newFile.setLocation(locations) newFile.addRun(Run(1, idx)) newFile["block"] = 'UserFilesFakeBlock' newFile["first_event"] = 1 newFile["last_event"] = 2 userFileset.addFile(newFile) file_counter += 1 if total_units > 0 and file_counter >= total_units: break return Result(task = kwargs['task'], result = userFileset)
def __init__(self, **args): # We need to pop this object instance from args because otherwise # the super class blows up when doing a deepcopy(args) self.rucio = args.pop("rucioObject", None) PolicyInterface.__init__(self, **args) self.workQueueElements = [] self.wmspec = None self.team = None self.initialTask = None self.splitParams = None self.dbs_pool = {} self.data = {} self.lumi = None self.couchdb = None self.rejectedWork = [] # List of inputs that were rejected self.badWork = [ ] # list of bad work unit (e.g. without any valid files) self.pileupData = {} self.cric = CRIC() # FIXME: for the moment, it will always use the default value self.rucioAcct = self.args.get("rucioAcct", "wmcore_transferor") if not self.rucio: self.rucio = Rucio(self.rucioAcct, configDict={'logger': self.logger})
def testConfig(self): """ Test service attributes and the override mechanism """ self.assertEqual(self.myCRIC['endpoint'], 'https://cms-cric.cern.ch/') self.assertEqual(self.myCRIC['cacheduration'], 1) self.assertEqual(self.myCRIC['accept_type'], 'application/json') self.assertEqual(self.myCRIC['content_type'], 'application/json') self.assertEqual(self.myCRIC['requests'].pycurl, None) newParams = {"cacheduration": 100, "pycurl": True} cric = CRIC(url='https://BLAH.cern.ch/', configDict=newParams) self.assertEqual(cric['endpoint'], 'https://BLAH.cern.ch/') self.assertEqual(cric['cacheduration'], newParams['cacheduration']) self.assertTrue(cric['requests'].pycurl)
{"site_name": "T2_XX_SiteC", "type": "phedex", "alias": "T2_XX_SiteC"}, ], 'data-processing': [ {"phedex_name": "T2_XX_SiteA", "psn_name": "T2_XX_SiteA"}, {"phedex_name": "T2_XX_SiteB", "psn_name": "T2_XX_SiteB"}, {"phedex_name": "T2_XX_SiteC", "psn_name": "T2_XX_SiteC"}, ] } lookup = {} outFile = 'CRICMockData.json' outFilename = os.path.join(getTestBase(), '..', 'data', 'Mock', outFile) print("Creating the file %s with mocked CRIC data" % outFilename) cric = CRIC() for callname in calls: print("Querying %s ..." % callname) try: if callname == 'people': result = cric._CRICUserQuery(callname) else: result = cric._CRICSiteQuery(callname) except HTTPError as exc: result = 'Raises HTTPError' if callname in ["site-names", "data-processing"]: result.extend(additionals[callname]) else: result = result lookup.update({callname:result})
class Dataset(StartPolicyInterface): """Split elements into datasets""" def __init__(self, **args): StartPolicyInterface.__init__(self, **args) self.args.setdefault('SliceType', 'NumberOfRuns') self.args.setdefault('SliceSize', 1) self.lumiType = "NumberOfLumis" self.sites = [] self.cric = CRIC() def split(self): """Apply policy to spec""" work = set() if self.args['SliceType'] == 'NumberOfRuns' else 0 numFiles = 0 numEvents = 0 numLumis = 0 datasetPath = self.initialTask.getInputDatasetPath() # dataset splitting can't have its data selection overridden if self.data and self.data.keys() != [datasetPath]: raise RuntimeError("Can't provide different data to split with") blocks = self.validBlocks(self.initialTask, self.dbs()) if not blocks: return for block in blocks: if self.args['SliceType'] == 'NumberOfRuns': work = work.union(block[self.args['SliceType']]) else: work += float(block[self.args['SliceType']]) numLumis += int(block[self.lumiType]) numFiles += int(block['NumberOfFiles']) numEvents += int(block['NumberOfEvents']) if self.args['SliceType'] == 'NumberOfRuns': numJobs = ceil(len(work) / float(self.args['SliceSize'])) else: numJobs = ceil(float(work) / float(self.args['SliceSize'])) # parentage parentFlag = True if self.initialTask.parentProcessingFlag() else False self.newQueueElement(Inputs={datasetPath: self.data.get(datasetPath, [])}, ParentFlag=parentFlag, NumberOfLumis=numLumis, NumberOfFiles=numFiles, NumberOfEvents=numEvents, Jobs=numJobs, NoInputUpdate=self.initialTask.getTrustSitelists().get('trustlists'), NoPileupUpdate=self.initialTask.getTrustSitelists().get('trustPUlists') ) def validate(self): """Check args and spec work with block splitting""" StartPolicyInterface.validateCommon(self) if not self.initialTask.inputDataset(): raise WorkQueueWMSpecError(self.wmspec, 'No input dataset') def validBlocks(self, task, dbs): """Return blocks that pass the input data restriction""" datasetPath = task.getInputDatasetPath() Lexicon.dataset(datasetPath) # check dataset name validBlocks = [] locations = None blockWhiteList = task.inputBlockWhitelist() blockBlackList = task.inputBlockBlacklist() runWhiteList = task.inputRunWhitelist() runBlackList = task.inputRunBlacklist() lumiMask = task.getLumiMask() if lumiMask: maskedBlocks = self.getMaskedBlocks(task, dbs, datasetPath) for blockName in dbs.listFileBlocks(datasetPath): # check block restrictions if blockWhiteList and blockName not in blockWhiteList: continue if blockName in blockBlackList: continue blockSummary = dbs.getDBSSummaryInfo(block=blockName) if int(blockSummary.get('NumberOfFiles', 0)) == 0: logging.warning("Block %s being rejected for lack of valid files to process", blockName) self.badWork.append(blockName) continue if self.args['SliceType'] == 'NumberOfRuns': blockSummary['NumberOfRuns'] = dbs.listRuns(block=blockName) # check lumi restrictions if lumiMask: if blockName not in maskedBlocks: logging.warning("Block %s doesn't pass the lumi mask constraints", blockName) self.rejectedWork.append(blockName) continue acceptedLumiCount = sum([len(maskedBlocks[blockName][lfn].getLumis()) for lfn in maskedBlocks[blockName]]) ratioAccepted = 1. * acceptedLumiCount / float(blockSummary['NumberOfLumis']) maskedRuns = [maskedBlocks[blockName][lfn].getRuns() for lfn in maskedBlocks[blockName]] acceptedRuns = set(lumiMask.getRuns()).intersection(set().union(*maskedRuns)) blockSummary['NumberOfFiles'] = len(maskedBlocks[blockName]) blockSummary['NumberOfEvents'] = float(blockSummary['NumberOfEvents']) * ratioAccepted blockSummary[self.lumiType] = acceptedLumiCount blockSummary['NumberOfRuns'] = acceptedRuns # check run restrictions elif runWhiteList or runBlackList: runs = set(dbs.listRuns(block=blockName)) # multi run blocks need special account, requires more DBS calls recalculateLumiCounts = True if len(runs) > 1 else False # apply blacklist and whitelist runs = runs.difference(runBlackList) if runWhiteList: runs = runs.intersection(runWhiteList) # any runs left are ones we will run on, if none ignore block if not runs: logging.warning("Block %s doesn't pass the runs constraints", blockName) self.rejectedWork.append(blockName) continue if recalculateLumiCounts: # Recalculate the number of files, lumis and ~events accepted acceptedLumiCount = 0 acceptedEventCount = 0 acceptedFileCount = 0 fileInfo = dbs.listFilesInBlock(fileBlockName=blockName) for fileEntry in fileInfo: acceptedFile = False for lumiInfo in fileEntry['LumiList']: if lumiInfo['RunNumber'] in runs: acceptedFile = True acceptedLumiCount += len(lumiInfo['LumiSectionNumber']) if acceptedFile: acceptedFileCount += 1 acceptedEventCount += fileEntry['NumberOfEvents'] else: acceptedLumiCount = blockSummary["NumberOfLumis"] acceptedFileCount = blockSummary['NumberOfFiles'] acceptedEventCount = blockSummary['NumberOfEvents'] blockSummary[self.lumiType] = acceptedLumiCount blockSummary['NumberOfFiles'] = acceptedFileCount blockSummary['NumberOfEvents'] = acceptedEventCount blockSummary['NumberOfRuns'] = runs validBlocks.append(blockSummary) if locations is None: locations = set(dbs.listFileBlockLocation(blockName)) else: locations = locations.intersection(dbs.listFileBlockLocation(blockName)) # all needed blocks present at these sites if task.getTrustSitelists().get('trustlists'): siteWhitelist = task.siteWhitelist() siteBlacklist = task.siteBlacklist() self.sites = makeLocationsList(siteWhitelist, siteBlacklist) self.data[datasetPath] = self.sites elif locations: self.data[datasetPath] = list(set(self.cric.PNNstoPSNs(locations))) return validBlocks
def setUp(self): """ Setup for unit tests """ super(CRICTest, self).setUp() self.myCRIC = CRIC()
def __init__(self, *args, **kwargs): TaskAction.__init__(self, *args, **kwargs) with self.config.TaskWorker.envForCMSWEB: configDict = {"cacheduration": 1, "pycurl": True} # cache duration is in hours self.resourceCatalog = CRIC(logger=self.logger, configDict=configDict)
class CRICTest(EmulatedUnitTestCase): """ Unit tests for SiteScreening module """ def __init__(self, methodName='runTest'): super(CRICTest, self).__init__(methodName=methodName) def setUp(self): """ Setup for unit tests """ super(CRICTest, self).setUp() self.myCRIC = CRIC() def testConfig(self): """ Test service attributes and the override mechanism """ self.assertEqual(self.myCRIC['endpoint'], 'https://cms-cric.cern.ch/') self.assertEqual(self.myCRIC['cacheduration'], 1) self.assertEqual(self.myCRIC['accept_type'], 'application/json') self.assertEqual(self.myCRIC['content_type'], 'application/json') self.assertEqual(self.myCRIC['requests'].pycurl, None) newParams = {"cacheduration": 100, "pycurl": True} cric = CRIC(url='https://BLAH.cern.ch/', configDict=newParams) self.assertEqual(cric['endpoint'], 'https://BLAH.cern.ch/') self.assertEqual(cric['cacheduration'], newParams['cacheduration']) self.assertTrue(cric['requests'].pycurl) @attr('integration') def testWhoAmI(self): """ Test user mapping information from the request headers """ print("This test only works with service certificates, not user proxies") self.assertTrue(self.myCRIC.whoAmI()) # empty list if nothing def testUserNameDN(self): """ Tests user name to DN lookup """ expectedDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=amaltaro/CN=718748/CN=Alan Malta Rodrigues" username = "******" dn = self.myCRIC.userNameDn(username) self.assertEqual(dn, expectedDN) def testUserNameDNWithApostrophe(self): """ Test DN lookup with an apostrophe """ expectedDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=gdimperi/CN=728001/CN=Giulia D'Imperio" username = "******" dn = self.myCRIC.userNameDn(username) self.assertEqual(dn, expectedDN) def testGetAllPSNs(self): """ Test API which fetches all PSNs """ result = self.myCRIC.getAllPSNs() self.assertNotIn('T1_US_FNAL_Disk', result) self.assertNotIn('T1_US_FNAL_MSS', result) self.assertIn('T1_US_FNAL', result) self.assertIn('T2_CH_CERN', result) self.assertIn('T2_CH_CERN_HLT', result) t1s = [psn for psn in result if psn.startswith('T1_')] self.assertTrue(len(t1s) < 10) t2s = [psn for psn in result if psn.startswith('T2_')] self.assertTrue(len(t2s) > 30) t3s = [psn for psn in result if psn.startswith('T3_')] self.assertTrue(len(t3s) > 10) self.assertTrue(len(result) > 70) return def testGetAllPhEDExNodeNames(self): """ Test API to get all PhEDEx Node Names """ result = self.myCRIC.getAllPhEDExNodeNames(excludeBuffer=True) self.assertFalse([pnn for pnn in result if pnn.endswith('_Buffer')]) result = self.myCRIC.getAllPhEDExNodeNames(excludeBuffer=False) self.assertTrue(len([pnn for pnn in result if pnn.endswith('_Buffer')]) > 5) result = self.myCRIC.getAllPhEDExNodeNames(pattern='T1.*', excludeBuffer=True) self.assertFalse([pnn for pnn in result if not pnn.startswith('T1_')]) self.assertTrue(len(result) > 10) result = self.myCRIC.getAllPhEDExNodeNames(excludeBuffer=True) self.assertTrue([pnn for pnn in result if pnn.startswith('T1_')]) self.assertTrue([pnn for pnn in result if pnn.startswith('T2_')]) self.assertTrue([pnn for pnn in result if pnn.startswith('T3_')]) self.assertTrue(len(result) > 60) result1 = self.myCRIC.getAllPhEDExNodeNames(pattern='.*', excludeBuffer=True) self.assertTrue([pnn for pnn in result1 if pnn.startswith('T1_')]) self.assertTrue([pnn for pnn in result1 if pnn.startswith('T2_')]) self.assertTrue([pnn for pnn in result1 if pnn.startswith('T3_')]) self.assertTrue(len(result) > 60) self.assertItemsEqual(result, result1) # test a few PSNs self.assertNotIn('T1_US_FNAL', result) self.assertNotIn('T2_CH_CERN_HLT', result) # test a few PNNs self.assertIn('T1_US_FNAL_Disk', result) self.assertIn('T1_US_FNAL_MSS', result) self.assertIn('T2_CH_CERN', result) return def testPNNstoPSNs(self): """ Test mapping PhEDEx Node Names to Processing Site Names """ self.assertEqual(self.myCRIC.PNNstoPSNs([]), []) self.assertItemsEqual(self.myCRIC.PNNstoPSNs(['T1_US_FNAL_MSS']), []) self.assertItemsEqual(self.myCRIC.PNNstoPSNs(['T1_US_FNAL_Disk']), ['T1_US_FNAL']) pnns = ['T1_US_FNAL_Disk', 'T1_US_FNAL_Buffer', 'T1_US_FNAL_MSS'] self.assertItemsEqual(self.myCRIC.PNNstoPSNs(pnns), ['T1_US_FNAL']) pnns = ['T2_CH_CERN', 'T2_DE_DESY'] psns = ['T2_CH_CERN_HLT', 'T2_DE_DESY', 'T2_CH_CERN'] self.assertItemsEqual(self.myCRIC.PNNstoPSNs(pnns), psns) psns = ['T2_UK_London_IC', 'T3_UK_London_QMUL', 'T3_UK_SGrid_Oxford', 'T3_UK_London_RHUL', 'T3_UK_ScotGrid_GLA'] self.assertItemsEqual(self.myCRIC.PNNstoPSNs('T2_UK_London_IC'), psns) pnns = ['T2_UK_London_IC', 'T2_US_Purdue'] self.assertItemsEqual(self.myCRIC.PNNstoPSNs(pnns), psns + ['T2_US_Purdue']) return def testPSNstoPNNs(self): """ Test mapping Processing Site Names to PhEDEx Node Names """ self.assertEqual(self.myCRIC.PSNstoPNNs([]), []) # test a few PNNs self.assertItemsEqual(self.myCRIC.PSNstoPNNs('T1_US_FNAL_Disk'), []) self.assertItemsEqual(self.myCRIC.PSNstoPNNs(['T1_DE_KIT_MSS']), []) # test a few PSNs self.assertItemsEqual(self.myCRIC.PSNstoPNNs(['T1_US_FNAL']), ['T1_US_FNAL_Disk', 'T3_US_FNALLPC']) self.assertItemsEqual(self.myCRIC.PSNstoPNNs(['T2_CH_CERN_HLT']), ['T2_CH_CERN']) pnns = ['T2_CH_CERN', 'T2_CH_CERNBOX'] self.assertItemsEqual(self.myCRIC.PSNstoPNNs(['T2_CH_CERN']), pnns) psns = ['T2_CH_CERN', 'T2_CH_CERN_HLT'] self.assertItemsEqual(self.myCRIC.PSNstoPNNs(psns), pnns) self.assertItemsEqual(self.myCRIC.PSNstoPNNs(['T2_UK_London_IC']), ['T2_UK_London_IC']) pnns = ['T2_UK_London_IC', 'T2_UK_London_Brunel', 'T2_UK_SGrid_RALPP'] self.assertItemsEqual(self.myCRIC.PSNstoPNNs('T3_UK_London_QMUL'), pnns) self.assertItemsEqual(self.myCRIC.PSNstoPNNs('T3_UK_ScotGrid_GLA'), pnns) pnns = ['T2_UK_London_IC', 'T2_UK_SGrid_RALPP'] self.assertItemsEqual(self.myCRIC.PSNstoPNNs('T3_UK_SGrid_Oxford'), pnns) self.assertItemsEqual(self.myCRIC.PSNstoPNNs('T3_UK_London_RHUL'), pnns) self.assertItemsEqual(self.myCRIC.PSNstoPNNs(['T2_US_Purdue']), ['T2_US_Purdue']) return def testPSNtoPNNMap(self): """ Test API to get a map of PSNs to PNNs """ with self.assertRaises(TypeError): self.myCRIC.PSNtoPNNMap(1) with self.assertRaises(TypeError): self.myCRIC.PSNtoPNNMap({'config': 'blah'}) with self.assertRaises(TypeError): self.myCRIC.PSNtoPNNMap(['blah']) result = self.myCRIC.PSNtoPNNMap() self.assertTrue([psn for psn in result.keys() if psn.startswith('T1_')]) self.assertTrue([psn for psn in result.keys() if psn.startswith('T2_')]) self.assertTrue([psn for psn in result.keys() if psn.startswith('T3_')]) self.assertTrue(len(result) > 50) result = self.myCRIC.PSNtoPNNMap(psnPattern='T1.*') self.assertFalse([psn for psn in result.keys() if not psn.startswith('T1_')]) self.assertTrue(len(result) < 10) result = self.myCRIC.PSNtoPNNMap(psnPattern='T2.*') self.assertFalse([psn for psn in result.keys() if not psn.startswith('T2_')]) self.assertTrue(len(result) > 10) result = self.myCRIC.PSNtoPNNMap(psnPattern='T3.*') self.assertFalse([psn for psn in result.keys() if not psn.startswith('T3_')]) self.assertTrue(len(result) > 10) result = self.myCRIC.PSNtoPNNMap('T2_CH_CERN$') self.assertItemsEqual(result.keys(), ['T2_CH_CERN']) self.assertItemsEqual(result['T2_CH_CERN'], ['T2_CH_CERNBOX', 'T2_CH_CERN']) # test a exact site name, which is treated as a regex and yields a confusing result!!! result = self.myCRIC.PSNtoPNNMap('T2_CH_CERN') self.assertItemsEqual(result.keys(), ['T2_CH_CERN', 'T2_CH_CERN_HLT']) self.assertItemsEqual(result['T2_CH_CERN'], ['T2_CH_CERNBOX', 'T2_CH_CERN']) self.assertItemsEqual(result['T2_CH_CERN_HLT'], ['T2_CH_CERN']) result = self.myCRIC.PSNtoPNNMap('T2_CH_CERN_HLT') self.assertItemsEqual(result.keys(), ['T2_CH_CERN_HLT']) self.assertItemsEqual(result['T2_CH_CERN_HLT'], ['T2_CH_CERN']) result = self.myCRIC.PSNtoPNNMap('T1_US_FNAL') self.assertItemsEqual(result.keys(), ['T1_US_FNAL']) self.assertItemsEqual(result['T1_US_FNAL'], ['T1_US_FNAL_Disk', 'T3_US_FNALLPC']) # test a PNN as input, expecting nothing mapped to it self.assertItemsEqual(self.myCRIC.PSNtoPNNMap('T1_US_FNAL_Disk'), {}) return