Ejemplo n.º 1
0
    def testEmulator(self):

        EmulatorHelper.setEmulators(True, True, True, True)
        self.assertEqual(PhEDEx().wrapped.__module__,
                         'WMQuality.Emulators.PhEDExClient.PhEDEx')
        self.assertEqual(DBSReader(self.globalDBS).wrapped.__module__,
                         'WMQuality.Emulators.DBSClient.DBSReader')
        self.assertEqual(SiteDBJSON().wrapped.__module__,
                         'WMQuality.Emulators.SiteDBClient.SiteDB')
        self.assertEqual(RequestManager().wrapped.__module__,
                         'WMQuality.Emulators.RequestManagerClient.RequestManager')

        self.assertEqual(PhEDEx().__class__.__name__, 'PhEDEx')
        self.assertEqual(DBSReader(self.globalDBS).__class__.__name__, 'DBSReader')
        self.assertEqual(SiteDBJSON().__class__.__name__, 'SiteDBJSON')
        self.assertEqual(RequestManager().__class__.__name__, 'RequestManager')

        EmulatorHelper.resetEmulators()
        self.assertEqual(PhEDEx().wrapped.__module__,
                         'WMCore.Services.PhEDEx.PhEDEx')
        self.assertEqual(DBSReader(self.globalDBS).wrapped.__module__,
                         'WMCore.Services.DBS.DBS2Reader')
        self.assertEqual(SiteDBJSON().wrapped.__module__,
                         'WMCore.Services.SiteDB.SiteDB')
        self.assertEqual(RequestManager().wrapped.__module__,
                         'WMCore.Services.RequestManager.RequestManager')

        self.assertEqual(PhEDEx().__class__.__name__, 'PhEDEx')
        self.assertEqual(DBSReader(self.globalDBS).__class__.__name__, 'DBS2Reader')
        self.assertEqual(SiteDBJSON().__class__.__name__, 'SiteDBJSON')
        self.assertEqual(RequestManager().__class__.__name__, 'RequestManager')
Ejemplo n.º 2
0
    def testXMLJSON(self):
        """
        Test XML and JSON in the same scope
        """
        site = 'T1_US_FNAL_Buffer'
        httpDict = {
            'endpoint': "https://cmsweb.cern.ch/phedex/datasvc/json/test"
        }
        phedexJSON = PhEDEx(responseType='json', httpDict=httpDict)
        httpDict = {
            'endpoint': "https://cmsweb.cern.ch/phedex/datasvc/xml/test"
        }
        phedexXML = PhEDEx(responseType='xml', httpDict=httpDict)

        phedexXML.getNodeTFC(site)
        tfc_file = phedexXML.cacheFileName('tfc', inputdata={'node': site})
        tfc_map = {}
        tfc_map[site] = readTFC(tfc_file)
        pfn = tfc_map[site].matchLFN('srmv2',
                                     '/store/user/jblow/dir/test.root')

        self.assertTrue(
            pfn ==
            'srm://cmssrm.fnal.gov:8443/srm/managerv2?SFN=/11/store/user/jblow/dir/test.root'
        )

        self.assertTrue(
            phedexJSON.getNodeSE('T1_US_FNAL_Buffer') == 'cmssrm.fnal.gov')
Ejemplo n.º 3
0
    def __init__(self, msConfig, logger=None):
        """
        Provides setup for MSTransferor and MSMonitor classes

        :param config: MS service configuration
        :param logger: logger object (optional)
        """
        self.logger = getMSLogger(getattr(msConfig, 'verbose', False), logger)
        self.msConfig = msConfig
        self.logger.info("Configuration including default values:\n%s",
                         self.msConfig)

        self.reqmgr2 = ReqMgr(self.msConfig['reqmgr2Url'], logger=self.logger)
        self.reqmgrAux = ReqMgrAux(self.msConfig['reqmgr2Url'],
                                   httpDict={'cacheduration': 1.0},
                                   logger=self.logger)

        # hard code it to production DBS otherwise PhEDEx subscribe API fails to match TMDB data
        dbsUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
        if usingRucio():
            # FIXME: we cannot use Rucio in write mode yet
            # self.rucio = Rucio(self.msConfig['rucioAccount'], configDict={"logger": self.logger})
            self.phedex = PhEDEx(httpDict={'cacheduration': 0.5},
                                 dbsUrl=dbsUrl,
                                 logger=self.logger)
        else:
            self.phedex = PhEDEx(httpDict={'cacheduration': 0.5},
                                 dbsUrl=dbsUrl,
                                 logger=self.logger)
Ejemplo n.º 4
0
 def testGetEmulators(self):
     from WMCore.Services.PhEDEx.PhEDEx import PhEDEx
     phedexJSON = PhEDEx(responseType='json')
     self.assertTrue(hasattr(phedexJSON, '_testNonExistentInEmulator'))
     EmulatorHelper.setEmulators(phedex=True,
                                 dbs=True,
                                 siteDB=True,
                                 requestMgr=False)
     phedexJSON2 = PhEDEx(responseType='json')
     self.assertFalse(hasattr(phedexJSON2, '_testNonExistentInEmulator'))
     EmulatorHelper.resetEmulators()
     phedexJSON2 = PhEDEx(responseType='json')
     self.assertTrue(hasattr(phedexJSON2, '_testNonExistentInEmulator'))
Ejemplo n.º 5
0
    def __init__(self, config):
        """
        ___init___

        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.config = config
        self.phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl}, "json")
        self.dbsUrl = config.DBSInterface.globalDBSUrl
        self.group = getattr(config.PhEDExInjector, "group", "DataOps")

        # This will be used to map SE names which are stored in the DBSBuffer to
        # PhEDEx node names.  The first key will be the "kind" which consists
        # of one of the following: MSS, Disk, Buffer.  The next key will be the
        # SE name.
        self.seMap = {}
        self.nodeNames = []

        self.diskSites = getattr(config.PhEDExInjector, "diskSites", ["storm-fe-cms.cr.cnaf.infn.it",
                                                                      "srm-cms-disk.gridpp.rl.ac.uk"])

        # initialize the alert framework (if available - config.Alert present)
        #    self.sendAlert will be then be available
        self.initAlerts(compName = "PhEDExInjector")

        self.filesToRecover = None
Ejemplo n.º 6
0
    def __init__(self, config):
        BaseWorkerThread.__init__(self)
        self.config = config.FilesCleaner
        self.logger.debug('Configuration loaded')

        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging
            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)
        self.logger.debug('Configuration loaded')
        config_server = CouchServer(dburl=self.config.config_couch_instance)
        self.config_db = config_server.connectDatabase(
            self.config.config_database)
        self.logger.debug('Connected to files DB')
        self.phedex = PhEDEx(responseType='xml')
        self.log_dir = '%s/logs/%s/%s/%s' % ( self.config.componentDir, \
 str(datetime.datetime.now().month), str(datetime.datetime.now().year), "Ops")
        try:
            os.makedirs(self.log_dir)
        except OSError, e:
            if e.errno == errno.EEXIST:
                pass
            else:
                self.logger.error('Unknown error in mkdir' % e.errno)
                raise
Ejemplo n.º 7
0
    def testSlcPhedexNodesEqualPhedexApiNodes(self):
        """
        For each site, verify that the stageout node specified in
        site-local-config.xml is the same as the one returned by the PhEDEx api.
        """
        os.environ["CMS_PATH"] = "/cvmfs/cms.cern.ch"

        phedex = PhEDEx()
        nodes = phedex.getNodeMap()["phedex"]["node"]

        # Make a dict for translating the se names into regular site names.
        node_map = {}
        for node in nodes:
            node_map[str(node[u"se"])] = str(node[str(u"name")])

        for d in os.listdir("/cvmfs/cms.cern.ch/SITECONF/"):
            # Only T0_, T1_... folders are needed
            if d[0] == "T":
                os.environ[
                    'WMAGENT_SITE_CONFIG_OVERRIDE'] = '/cvmfs/cms.cern.ch/SITECONF/%s/JobConfig/site-local-config.xml' % (
                        d)
                try:
                    slc = loadSiteLocalConfig()
                except SiteConfigError as e:
                    print e.args[0]
                phedexNode = slc.localStageOut.get("phedex-node")
                # If slc is correct, perform check
                if "se-name" in slc.localStageOut and slc.localStageOut[
                        "se-name"] in node_map and phedexNode != None:
                    self.assertEqual(phedexNode, node_map[slc.localStageOut["se-name"]], \
                            "Error: Node specified in SLC (%s) doesn't match node returned by PhEDEx api (%s)." \
                            % (phedexNode, node_map[slc.localStageOut["se-name"]]))

        return
Ejemplo n.º 8
0
    def __init__(self, config):
        """
        Initialise class members
        """
        #Need a better way to test this without turning off this next line
        BaseWorkerThread.__init__(self)
        #logging.basicConfig(format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',datefmt = '%m-%d %H:%M')
        #self.logger = logging.getLogger()
        # self.logger is set up by the BaseWorkerThread, we just set it's level

        self.config = config.AsyncTransfer
        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging
            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)

        self.logger.debug('Configuration loaded')
        server = CouchServer(dburl=self.config.couch_instance,
                             ckey=self.config.opsProxy,
                             cert=self.config.opsProxy)
        self.db = server.connectDatabase(self.config.files_database)
        config_server = CouchServer(dburl=self.config.config_couch_instance)
        self.config_db = config_server.connectDatabase(
            self.config.config_database)
        self.logger.debug('Connected to CouchDB')
        self.pool = Pool(processes=self.config.pool_size)
        try:
            self.phedex = PhEDEx(responseType='xml')
        except Exception, e:
            self.logger.exception('PhEDEx exception: %s' % e)
Ejemplo n.º 9
0
 def keepOnlyDisks(self, locationsMap):
     phedex = PhEDEx()  # TODO use certs from the config!
     # get all the PNNs that are of kind 'Disk'
     try:
         diskLocations = set([
             pnn['name'] for pnn in phedex.getNodeMap()['phedex']['node']
             if pnn['kind'] == 'Disk'
         ])
     except HTTPException as ex:
         self.logger.error(ex.headers)
         raise TaskWorkerException("The CRAB3 server backend could not contact phedex to get the list of site storages.\n"+\
                             "This is could be a temporary phedex glitch, please try to submit a new task (resubmit will not work)"+\
                             " and contact the experts if the error persists.\nError reason: %s" % str(ex)) # TODO addo the nodes phedex so the user can check themselves
     diskLocationsMap = {}
     for block, locations in locationsMap.iteritems():
         locations[:] = [
             x for x in locations if x != 'T3_CH_CERN_OpenData'
         ]  # ignore OpenData until it is accessible by CRAB
         if set(locations) & diskLocations:
             # at least some locations are disk
             diskLocationsMap[block] = locationsMap[block]
         else:
             # no locations are in the disk list, assume that they are tape
             self.tapeLocations = self.tapeLocations.union(
                 set(locations) - diskLocations)
     locationsMap.clear()  # remove all blocks
     locationsMap.update(
         diskLocationsMap)  # add only blocks with disk locations
Ejemplo n.º 10
0
    def testSlcPhedexNodesEqualPhedexApiNodes(self):
        """
        For each site, verify that the stageout node specified in
        site-local-config.xml is the same as the one returned by the PhEDEx api.
        """
        os.environ["CMS_PATH"] = "/cvmfs/cms.cern.ch"

        phedex = PhEDEx()
        nodes = [
            node[u'name'] for node in phedex.getNodeMap()["phedex"]["node"]
        ]

        for d in os.listdir("/cvmfs/cms.cern.ch/SITECONF/"):
            # Only T0_, T1_... folders are needed
            if d[0] == "T":
                os.environ[
                    'WMAGENT_SITE_CONFIG_OVERRIDE'] = '/cvmfs/cms.cern.ch/SITECONF/%s/JobConfig/site-local-config.xml' % (
                        d)
                try:
                    slc = loadSiteLocalConfig()
                except SiteConfigError as e:
                    print(e.args[0])
                phedexNode = slc.localStageOut.get("phedex-node")
                self.assertTrue(
                    phedexNode in nodes,
                    "Error: Node specified in SLC (%s) not in list returned by PhEDEx api"
                    % phedexNode)
        return
Ejemplo n.º 11
0
    def __init__(self, config):
        """
        ___init___

        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.dbsUrl = config.DBSInterface.globalDBSUrl
        self.phedexGroup = config.PhEDExInjector.phedexGroup

        self.pollCounter = 0
        self.subFrequency = None
        if getattr(config.PhEDExInjector, "subscribeDatasets", False):
            pollInterval = config.PhEDExInjector.pollInterval
            subInterval = config.PhEDExInjector.subscribeInterval
            self.subFrequency = max(1, int(round(subInterval / pollInterval)))
            logging.info("SubscribeDataset and deleteBlocks will run every %d polling cycles", self.subFrequency)
            # subscribe on first cycle
            self.pollCounter = self.subFrequency - 1

        # retrieving the node mappings is fickle and can fail quite often
        self.phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl},
                             "json", dbsUrl=self.dbsUrl)
        try:
            nodeMappings = self.phedex.getNodeMap()
        except:
            time.sleep(2)
            try:
                nodeMappings = self.phedex.getNodeMap()
            except:
                time.sleep(4)
                nodeMappings = self.phedex.getNodeMap()

        # This will be used to map SE names which are stored in the DBSBuffer to
        # PhEDEx node names.  The first key will be the "kind" which consists
        # of one of the following: MSS, Disk, Buffer.  The next key will be the
        # SE name.
        self.seMap = {}
        self.nodeNames = []
        for node in nodeMappings["phedex"]["node"]:
            if node["kind"] not in self.seMap:
                self.seMap[node["kind"]] = {}
            logging.info("Adding mapping %s -> %s", node["se"], node["name"])
            self.seMap[node["kind"]][node["se"]] = node["name"]
            self.nodeNames.append(node["name"])

        self.phedexNodes = {'MSS': [], 'Disk': []}
        for node in nodeMappings["phedex"]["node"]:
            if node["kind"] in ["MSS", "Disk"]:
                self.phedexNodes[node["kind"]].append(node["name"])

        self.blocksToRecover = []

        # X-component configuration is BAD! But it will only be here during the
        # Rucio commissioning within WM
        self.listTiersToSkip = config.RucioInjector.listTiersToInject
        logging.info("Component configured to skip data injection for data tiers: %s",
                     self.listTiersToSkip)

        return
Ejemplo n.º 12
0
    def __init__(self, config):
        """
        ___init___

        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl},
                             "json")
        self.siteDB = SiteDBJSON()
        self.dbsUrl = config.DBSInterface.globalDBSUrl
        self.group = getattr(config.PhEDExInjector, "group", "DataOps")
        self.safeMode = getattr(config.PhEDExInjector, "safeOperationMode",
                                False)
        self.replicaOnly = getattr(config.PhEDExInjector, "replicaOnly", False)

        # Subscribed state in the DBSBuffer table for datasets
        self.terminalSubscriptionState = 1
        if self.safeMode:
            self.terminalSubscriptionState = 2

        # We will map node names to CMS names, that what the spec will have.
        # If a CMS name is associated to many PhEDEx node then choose the MSS option
        self.cmsToPhedexMap = {}

        # initialize the alert framework (if available - config.Alert present)
        #    self.sendAlert will be then be available

        self.initAlerts(compName="PhEDExInjector")
Ejemplo n.º 13
0
    def setUp(self):
        "init test class"
        self.group = 'DataOps'
        self.interval = 2
        self.rmgr = RequestManager(group=self.group,
                                   interval=self.interval,
                                   verbose=True)
        self.phedex = PhEDEx()

        # get some subscriptions from PhEDEx to play with
        data = self.phedex.subscriptions(group=self.group)
        for datasetInfo in data['phedex']['dataset']:
            dataset = datasetInfo.get('name')
            print("### dataset info from phedex, #files %s" %
                  datasetInfo.get('files', 0))
            # now use the same logic in as in Transferor, i.e. look-up dataset/group subscription
            data = self.phedex.subscriptions(dataset=dataset, group=self.group)
            if not data['phedex']['dataset']:
                print(
                    "### skip this dataset since no subscription data is available"
                )
                continue
            nodes = [
                i['node'] for r in data['phedex']['dataset']
                for i in r['subscription']
            ]
            print("### nodes", nodes)
            # create fake requests with dataset/nodes info
            rdict1 = dict(datasets=[dataset], sites=nodes, name='req1')
            rdict2 = dict(datasets=[dataset], sites=nodes, name='req2')
            self.requests = {'req1': rdict1, 'req2': rdict2}
            break
Ejemplo n.º 14
0
    def setUp(self):
        """
        _setUp_

        Install the DBSBuffer schema into the database and connect to PhEDEx.
        """
        self.phedexURL = "https://cmsweb.cern.ch/phedex/datasvc/json/test"
        self.dbsURL = "http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet"

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection(destroyAllDatabase=True)

        self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer"],
                                useDefault=False)

        myThread = threading.currentThread()
        daofactory = DAOFactory(package="WMComponent.DBSBuffer.Database",
                                logger=myThread.logger,
                                dbinterface=myThread.dbi)

        locationAction = daofactory(classname="DBSBufferFiles.AddLocation")
        locationAction.execute(siteName="srm-cms.cern.ch")
        locationAction.execute(siteName="se.fnal.gov")

        self.testFilesA = []
        self.testFilesB = []
        self.testDatasetA = "/%s/PromptReco-v1/RECO" % makeUUID()
        self.testDatasetB = "/%s/CRUZET11-v1/RAW" % makeUUID()
        self.phedex = PhEDEx({"endpoint": self.phedexURL}, "json")

        return
Ejemplo n.º 15
0
 def wrapped_func(*args, **kwargs):
     if 'cric' in services and (
             not args[0].allCMSNames.sites or
         (args[0].allCMSNames.cachetime + 1800 < mktime(gmtime()))):
         args[0].allCMSNames = CMSSitesCache(sites=CRIC().getAllPSNs(),
                                             cachetime=mktime(gmtime()))
         args[0].allPNNNames = CMSSitesCache(
             sites=CRIC().getAllPhEDExNodeNames(),
             cachetime=mktime(gmtime()))
     if 'phedex' in services and not args[0].phedex:
         phdict = args[0].phedexargs
         phdict.update({'cert': serverCert, 'key': serverKey})
         args[0].phedex = PhEDEx(responseType='xml', dict=phdict)
     if 'centralconfig' in services and (
             not args[0].centralcfg.centralconfig or
         (args[0].centralcfg.cachetime + 1800 < mktime(gmtime()))):
         args[0].centralcfg = ConfigCache(
             centralconfig=getCentralConfig(
                 extconfigurl=args[0].config.extconfigurl,
                 mode=args[0].config.mode),
             cachetime=mktime(gmtime()))
     if 'servercert' in services:
         args[0].serverCert = serverCert
         args[0].serverKey = serverKey
     return func(*args, **kwargs)
Ejemplo n.º 16
0
    def preInitialization(self):
        pollInterval = self.config.PhEDExInjector.pollInterval
        subInterval = self.config.PhEDExInjector.subscribeInterval
        logging.info("Setting poll interval to %s seconds for inject",
                     pollInterval)

        # retrieving the node mappings is fickle and can fail quite often
        # hence only do it once (with retries) and pass it to the workers
        phedex = PhEDEx({"endpoint": self.config.PhEDExInjector.phedexurl},
                        "json")
        try:
            nodeMappings = phedex.getNodeMap()
        except Exception:
            time.sleep(2)
            try:
                nodeMappings = phedex.getNodeMap()
            except Exception:
                time.sleep(4)
                nodeMappings = phedex.getNodeMap()

        myThread = threading.currentThread()
        myThread.workerThreadManager.addWorker(
            PhEDExInjectorPoller(self.config, phedex, nodeMappings),
            pollInterval)

        if getattr(self.config.PhEDExInjector, "subscribeDatasets", False):
            # wait a bit for first poll cycle of PhEDExInjectorPoller to complete
            # hopefully avoids intermingled logs (which can be confusing)
            time.sleep(2)
            logging.info("Setting poll interval to %s seconds for subscribe",
                         subInterval)
            myThread.workerThreadManager.addWorker(
                PhEDExInjectorSubscriber(self.config, phedex, nodeMappings),
                subInterval)
Ejemplo n.º 17
0
    def _queryDbsAndGetPileupConfig(self, stepHelper, dbsReader):
        """
        Method iterates over components of the pileup configuration input
        and queries DBS. Then iterates over results from DBS.

        There needs to be a list of files and their locations for each
        dataset name.
        Use dbsReader
        the result data structure is a Python dict following dictionary:
            FileList is a list of LFNs

        {"pileupTypeA": {"BlockA": {"FileList": [], "PhEDExNodeNames": []},
                         "BlockB": {"FileList": [], "PhEDExNodeName": []}, ....}

        this structure preserves knowledge of where particular files of dataset
        are physically (list of PNNs) located. DBS only lists sites which
        have all files belonging to blocks but e.g. BlockA of dataset DS1 may
        be located at site1 and BlockB only at site2 - it's possible that only
        a subset of the blocks in a dataset will be at a site.

        """
        # only production PhEDEx is connected (This can be moved to init method
        phedex = PhEDEx()
        node_filter = set(['UNKNOWN', None])

        resultDict = {}
        # iterate over input pileup types (e.g. "cosmics", "minbias")
        for pileupType in stepHelper.data.pileup.listSections_():
            # the format here is: step.data.pileup.cosmics.dataset = [/some/data/set]
            datasets = getattr(getattr(stepHelper.data.pileup, pileupType),
                               "dataset")
            # each dataset input can generally be a list, iterate over dataset names
            blockDict = {}
            for dataset in datasets:

                blockFileInfo = dbsReader.getFileListByDataset(dataset=dataset,
                                                               detail=True)

                for fileInfo in blockFileInfo:
                    blockDict.setdefault(fileInfo['block_name'], {
                        'FileList': [],
                        'NumberOfEvents': 0,
                        'PhEDExNodeNames': []
                    })
                    blockDict[fileInfo['block_name']]['FileList'].append(
                        {'logical_file_name': fileInfo['logical_file_name']})
                    blockDict[fileInfo['block_name']][
                        'NumberOfEvents'] += fileInfo['event_count']

                blockReplicasInfo = phedex.getReplicaPhEDExNodesForBlocks(
                    dataset=dataset, complete='y')
                for block in blockReplicasInfo:
                    nodes = set(blockReplicasInfo[block]) - node_filter
                    blockDict[block]['PhEDExNodeNames'] = list(nodes)
                    blockDict[block]['FileList'] = sorted(
                        blockDict[block]['FileList'])

            resultDict[pileupType] = blockDict
        return resultDict
Ejemplo n.º 18
0
    def _queryAndCompareWithDBS(self, pileupDict, defaultArguments, dbsUrl):
        """
        pileupDict is a Python dictionary containing particular pileup
        configuration information. Query DBS on given dataset contained
        now in both input defaultArguments as well as in the pileupDict
        and compare values.

        """
        reader = DBS3Reader(dbsUrl)
        phedex = PhEDEx()

        inputArgs = defaultArguments["PileupConfig"]

        self.assertEqual(len(inputArgs), len(pileupDict),
                         "Number of pileup types different.")
        for pileupType in inputArgs:
            m = ("pileup type '%s' not in PileupFetcher-produced pileup "
                 "configuration: '%s'" % (pileupType, pileupDict))
            self.assertTrue(pileupType in pileupDict, m)

        # now query DBS for compare actual results on files lists for each
        # pileup type and dataset and location (storage element names)
        # pileupDict is saved in the file and now comparing items of this
        # configuration with actual DBS results, the structure of pileupDict:
        #    {"pileupTypeA": {"BlockA": {"FileList": [], "PhEDExNodeNames": []},
        #                     "BlockB": {"FileList": [], "PhEDExNodeNames": []}, ....}
        for pileupType, datasets in inputArgs.items():
            # this is from the pileup configuration produced by PileupFetcher
            blockDict = pileupDict[pileupType]

            for dataset in datasets:
                dbsFileBlocks = reader.listFileBlocks(dataset=dataset)
                blocksLocation = phedex.getReplicaPhEDExNodesForBlocks(
                    dataset=dataset, complete='y')
                for dbsFileBlockName in dbsFileBlocks:
                    fileList = []
                    pnns = set()
                    for pnn in blocksLocation[dbsFileBlockName]:
                        pnns.add(pnn)
                    # now get list of files in the block
                    dbsFiles = reader.listFilesInBlock(dbsFileBlockName)
                    for dbsFile in dbsFiles:
                        fileList.append(dbsFile["LogicalFileName"])
                    # now compare the sets:
                    m = ("PNNs don't agree for pileup type '%s', "
                         "dataset '%s' in configuration: '%s'" %
                         (pileupType, dataset, pileupDict))
                    self.assertEqual(
                        set(blockDict[dbsFileBlockName]["PhEDExNodeNames"]),
                        pnns, m)
                    m = (
                        "FileList don't agree for pileup type '%s', dataset '%s' "
                        " in configuration: '%s'" %
                        (pileupType, dataset, pileupDict))
                    storedFileList = [
                        item['logical_file_name']
                        for item in blockDict[dbsFileBlockName]["FileList"]
                    ]
                    self.assertItemsEqual(storedFileList, fileList, m)
Ejemplo n.º 19
0
    def __init__(self, config):
        """
        ___init___

        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.dbsUrl = config.DBSInterface.globalDBSUrl

        self.pollCounter = 0
        self.subFrequency = None
        if getattr(config.PhEDExInjector, "subscribeDatasets", False):
            pollInterval = config.PhEDExInjector.pollInterval
            subInterval = config.PhEDExInjector.subscribeInterval
            self.subFrequency = max(1, int(round(subInterval / pollInterval)))
            logging.info(
                "SubscribeDataset and deleteBlocks will run every %d polling cycles",
                self.subFrequency)
            # subscribe on first cycle
            self.pollCounter = self.subFrequency - 1

        # retrieving the node mappings is fickle and can fail quite often
        self.phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl},
                             "json")
        try:
            nodeMappings = self.phedex.getNodeMap()
        except:
            time.sleep(2)
            try:
                nodeMappings = self.phedex.getNodeMap()
            except:
                time.sleep(4)
                nodeMappings = self.phedex.getNodeMap()

        # This will be used to map SE names which are stored in the DBSBuffer to
        # PhEDEx node names.  The first key will be the "kind" which consists
        # of one of the following: MSS, Disk, Buffer.  The next key will be the
        # SE name.
        self.seMap = {}
        self.nodeNames = []
        for node in nodeMappings["phedex"]["node"]:
            if node["kind"] not in self.seMap:
                self.seMap[node["kind"]] = {}
            logging.info("Adding mapping %s -> %s", node["se"], node["name"])
            self.seMap[node["kind"]][node["se"]] = node["name"]
            self.nodeNames.append(node["name"])

        self.phedexNodes = {'MSS': [], 'Disk': []}
        for node in nodeMappings["phedex"]["node"]:
            if node["kind"] in ["MSS", "Disk"]:
                self.phedexNodes[node["kind"]].append(node["name"])

        # initialize the alert framework (if available - config.Alert present)
        #    self.sendAlert will be then be available
        self.initAlerts(compName="PhEDExInjector")

        self.blocksToRecover = []

        return
Ejemplo n.º 20
0
def get_tfc_rules(site):
    """
    Get the TFC regexp for a given site.
    """
    phedex = PhEDEx(responseType='xml')
    phedex.getNodeTFC(site)
    tfc_file = phedex.cacheFileName('tfc', inputdata={'node': site})

    return readTFC(tfc_file)
Ejemplo n.º 21
0
 def __init__(self, logger, cmdargs=None):
     SubCommand.__init__(self, logger, cmdargs)
     self.phedex = PhEDEx({
         "cert": self.proxyfilename,
         "key": self.proxyfilename,
         "logger": self.logger
     })
     self.lfnsaddprefix = None
     self.filename = None
Ejemplo n.º 22
0
    def __init__(self,config):
        self.br=Browser()

        self.config = config
        
        # Initialise connections
        self.phedex = PhEDEx({"endpoint":"https://cmsweb.cern.ch/phedex/datasvc/json/prod/"}, "json")
        self.dbsPhys01 = DbsApi(url = dbs_base_url+"phys01/DBSReader/")
        self.dbsPhys02 = DbsApi(url = dbs_base_url+"phys02/DBSReader/")
        self.dbsPhys03 = DbsApi(url = dbs_base_url+"phys03/DBSReader/")
Ejemplo n.º 23
0
 def keepOnlyDisks(self, locationsMap):
     phedex = PhEDEx() #TODO use certs from the config!
     #get all the PNN that are of kind disk
     try:
         diskLocations = set([pnn['name'] for pnn in phedex.getNodeMap()['phedex']['node'] if pnn['kind']=='Disk'])
     except Exception, ex: #TODO should we catch HttpException instead?
         self.logger.exception(ex)
         raise TaskWorkerException("The CRAB3 server backend could not contact phedex to get the list of site storages.\n"+\
                             "This is could be a temporary phedex glitch, please try to submit a new task (resubmit will not work)"+\
                             " and contact the experts if the error persists.\nError reason: %s" % str(ex)) #TODO addo the nodes phedex so the user can check themselves
Ejemplo n.º 24
0
    def setUp(self):
        """
        _setUp_

        Initialize the PhEDEx API to point at the test server.
        """
        self.dbsTestUrl = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader"
        self.phedexApi = PhEDEx()

        return
Ejemplo n.º 25
0
    def setUp(self):
        """
        _setUp_

        Initialize the PhEDEx API to point at the test server.
        """
        phedexTestDS = "https://cmsweb.cern.ch/phedex/datasvc/json/test"
        self.dbsTestUrl = "http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet"
        self.phedexApi = PhEDEx({"endpoint": phedexTestDS, "method": "POST"})
        return
Ejemplo n.º 26
0
 def __init__(self):
     """
     Prepare module setup
     """
     super(PileupFetcher, self).__init__()
     if usingRucio():
         # FIXME: find a way to pass the Rucio account name to this fetcher module
         self.rucioAcct = "wmcore_transferor"
         self.rucio = Rucio(self.rucioAcct)
     else:
         self.phedex = PhEDEx()  # this will go away eventually
Ejemplo n.º 27
0
def remoteLFNPrefix(site, lfn=''):
    """
    Convert a site name to the relevant remote LFN prefix
    """
    from WMCore.Services.PhEDEx.PhEDEx import PhEDEx
    phedexJSON = PhEDEx(responseType='json')

    seName = phedexJSON.getNodeSE(site)
    uri = phedexJSON.getPFN(nodes=[site], lfns=[lfn])[(site, lfn)]

    return uri.replace(lfn,
                       ''), seName  # Don't want the actual LFN, just prefix
Ejemplo n.º 28
0
    def __init__(self, url, **contact):

        # instantiate dbs api object
        try:
            self.dbs = DbsApi(url, **contact)
        except dbsClientException as ex:
            msg = "Error in DBSReader with DbsApi\n"
            msg += "%s\n" % formatEx3(ex)
            raise DBSReaderError(msg)

        # connection to PhEDEx (Use default endpoint url)
        self.phedex = PhEDEx(responseType="json")
Ejemplo n.º 29
0
 def __init__(self):
     """
     Prepare module setup
     """
     super(PileupFetcher, self).__init__()
     if usingRucio():
         # Too much work to pass the rucio account name all the way to here
         # just use the production rucio account for resolving pileup location
         self.rucio = Rucio("wma_prod",
                            configDict={'phedexCompatible': False})
     else:
         self.phedex = PhEDEx()  # this will go away eventually
Ejemplo n.º 30
0
def main():
    """
    _main_
    """
    # Start services
    if 'WMAGENT_CONFIG' not in os.environ:
        os.environ[
            'WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
    connectToDB()
    myPhEDEx = PhEDEx()
    myThread = threading.currentThread()
    print "Please remember to shutdown the PhEDExInjector first, you have 10 seconds before the script starts."
    time.sleep(10)

    # Get the files that the PhEDExInjector would look for
    formatter = DBFormatter(logging, myThread.dbi)
    formatter.sql = query
    results = formatter.execute()
    sortedBlocks = defaultdict(set)
    for lfn, block in results:
        sortedBlocks[block].add(lfn)

    # Check with block-level calls
    foundFiles = set()
    for block in sortedBlocks:
        result = myPhEDEx._getResult('data', args={'block': block}, verb='GET')
        for dbs in result['phedex']['dbs']:
            for dataset in dbs['dataset']:
                blockChunk = dataset['block']
                for blockInfo in blockChunk:
                    for fileInfo in blockInfo['file']:
                        if fileInfo['lfn'] in sortedBlocks[block]:
                            foundFiles.add(fileInfo['lfn'])
    if not foundFiles:
        print "I didn't find an abnormal file, feel free to panic!. Please contact a developer."
        return 0
    print "Found %d files that are already registered in PhEDEx but the buffer doesn't know" % len(
        foundFiles)
    print "Fixing them now..."
    # Fix it!
    binds = []
    for lfn in foundFiles:
        binds.append({'lfn': lfn})
    formatter.dbi.processData(modification,
                              binds,
                              conn=None,
                              transaction=False,
                              returnCursor=False)
    print "Fixed them! :)"
    print "You can restart the PhEDExInjector now, have a nice day!"
    return 0