Ejemplo n.º 1
0
    def __init__(self, config):
        """
        ___init___

        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.config = config
        self.phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl}, "json")
        self.dbsUrl = config.DBSInterface.globalDBSUrl
        self.group = getattr(config.PhEDExInjector, "group", "DataOps")

        # This will be used to map SE names which are stored in the DBSBuffer to
        # PhEDEx node names.  The first key will be the "kind" which consists
        # of one of the following: MSS, Disk, Buffer.  The next key will be the
        # SE name.
        self.seMap = {}
        self.nodeNames = []

        self.diskSites = getattr(config.PhEDExInjector, "diskSites", ["storm-fe-cms.cr.cnaf.infn.it",
                                                                      "srm-cms-disk.gridpp.rl.ac.uk"])

        # initialize the alert framework (if available - config.Alert present)
        #    self.sendAlert will be then be available
        self.initAlerts(compName = "PhEDExInjector")

        self.filesToRecover = None
Ejemplo n.º 2
0
    def __init__(self, config):
        BaseWorkerThread.__init__(self)
        myThread = threading.currentThread()

        #DAO factory for WMBS objects
        self.daoFactory = DAOFactory(package = "WMCore.WMBS", \
                                     logger = logging,
                                     dbinterface = myThread.dbi)

        self.config = config

        #Libraries
        self.resourceControl = ResourceControl()

        self.changeState = ChangeState(self.config)
        self.repollCount = getattr(self.config.JobSubmitter, 'repollCount',
                                   10000)

        # BossAir
        self.bossAir = BossAirAPI(config=self.config)

        # Additions for caching-based JobSubmitter
        self.workflowTimestamps = {}
        self.cachedJobIDs = set()
        self.cachedJobs = {}
        self.jobDataCache = {}
        self.jobsToPackage = {}
        self.sandboxPackage = {}
        self.siteKeys = {}
        self.locationDict = {}
        self.cmsNames = {}
        self.drainSites = []
        self.sortedSites = []
        self.packageSize = getattr(self.config.JobSubmitter, 'packageSize',
                                   500)
        self.collSize = getattr(self.config.JobSubmitter, 'collectionSize',
                                self.packageSize * 1000)

        # initialize the alert framework (if available)
        self.initAlerts(compName="JobSubmitter")

        try:
            if not getattr(self.config.JobSubmitter, 'submitDir', None):
                self.config.JobSubmitter.submitDir = self.config.JobSubmitter.componentDir
            self.packageDir = os.path.join(self.config.JobSubmitter.submitDir,
                                           'packages')

            if not os.path.exists(self.packageDir):
                os.makedirs(self.packageDir)
        except Exception, ex:
            msg = "Error while trying to create packageDir %s\n!"
            msg += str(ex)
            logging.error(msg)
            self.sendAlert(6, msg=msg)
            try:
                logging.debug("PackageDir: %s" % self.packageDir)
                logging.debug("Config: %s" % config)
            except:
                pass
            raise JobSubmitterPollerException(msg)
Ejemplo n.º 3
0
 def __init__(self, config):
     """
     initialize properties specified from config
     """
     BaseWorkerThread.__init__(self)
     self.config = config
     self.drainAPI = DrainStatusAPI()
Ejemplo n.º 4
0
    def __init__(self, config):
        """
        Initialise class members
        """
        # Need a better way to test this without turning off this next line
        BaseWorkerThread.__init__(self)
        # logging.basicConfig(format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',datefmt = '%m-%d %H:%M')
        # self.logger = logging.getLogger()
        # self.logger is set up by the BaseWorkerThread, we just set it's level

        self.config = config.AsyncTransfer
        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging

            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)

        self.logger.debug("Configuration loaded")
        server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
        self.db = server.connectDatabase(self.config.files_database)
        config_server = CouchServer(dburl=self.config.config_couch_instance)
        self.config_db = config_server.connectDatabase(self.config.config_database)
        self.logger.debug("Connected to CouchDB")
        self.pool = Pool(processes=self.config.pool_size)
        try:
            self.phedex = PhEDEx(responseType="xml")
        except Exception, e:
            self.logger.exception("PhEDEx exception: %s" % e)
Ejemplo n.º 5
0
    def __init__(self, config):
        """
        Initialise class members
        """

        # Used for creating connections/transactions
        myThread = threading.currentThread()
        self.threadID = myThread.ident
        self.dbi = myThread.dbi

        self.logger = myThread.logger
        
        BaseWorkerThread.__init__(self)
        # get the db owner
        self.config  = config
        dbconfig = config.section_("CoreDatabase")
        self.dbowner=dbconfig.dbowner
       
        self.newBlock = False
        self.datasetCache = {}
        self.datasetCache['primDs']={}
        self.datasetCache['dataset']={}
        self.datasetCache['acquisitionEra']={}
        self.datasetCache['processingVersion']={}
        self.datasetCache['phyGrp']={}
        self.datasetCache['dataTier']={}
        self.datasetCache['primDsTp']={}
        self.datasetCache['datasetAccTp']={}
        self.datasetCache['processedDs']={}
        self.datasetCache['relVer']={}
        self.datasetCache['pHash']={}
        self.datasetCache['appExe']={}
        self.datasetCache['conf']={} #key=app:rel:phash
Ejemplo n.º 6
0
    def __init__(self, config, phedex, nodeMappings):
        """
        ___init___

        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.config = config
        self.phedex = phedex
        self.dbsUrl = config.DBSInterface.globalDBSUrl
        self.group = getattr(config.PhEDExInjector, "group", "DataOps")

        # This will be used to map SE names which are stored in the DBSBuffer to
        # PhEDEx node names.  The first key will be the "kind" which consists
        # of one of the following: MSS, Disk, Buffer.  The next key will be the
        # SE name.
        self.seMap = {}
        self.nodeNames = []
        for node in nodeMappings["phedex"]["node"]:
            if node["kind"] not in self.seMap:
                self.seMap[node["kind"]] = {}
            logging.info("Adding mapping %s -> %s", node["se"], node["name"])
            self.seMap[node["kind"]][node["se"]] = node["name"]
            self.nodeNames.append(node["name"])

        self.diskSites = getattr(config.PhEDExInjector, "diskSites", ["storm-fe-cms.cr.cnaf.infn.it",
                                                                      "srm-cms-disk.gridpp.rl.ac.uk"])

        # initialize the alert framework (if available - config.Alert present)
        #    self.sendAlert will be then be available
        self.initAlerts(compName = "PhEDExInjector")

        self.blocksToRecover = []
Ejemplo n.º 7
0
 def __init__(self, queue, config):
     """
     Initialise class members
     """
     BaseWorkerThread.__init__(self)
     self.queue = queue
     self.config = config
Ejemplo n.º 8
0
    def __init__(self, config):
        """
        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.config = config

        myThread = threading.currentThread()

        self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi)
        self.changeState = ChangeState(self.config)

        self.maxRetries = self.config.ErrorHandler.maxRetries
        if not isinstance(self.maxRetries, dict):
            self.maxRetries = {"default": self.maxRetries}
        if "default" not in self.maxRetries:
            raise ErrorHandlerException("Max retries for the default job type must be specified")

        self.maxProcessSize = getattr(self.config.ErrorHandler, "maxProcessSize", 250)
        self.exitCodes = getattr(self.config.ErrorHandler, "failureExitCodes", [])
        self.maxFailTime = getattr(self.config.ErrorHandler, "maxFailTime", 32 * 3600)
        self.readFWJR = getattr(self.config.ErrorHandler, "readFWJR", False)
        self.passCodes = getattr(self.config.ErrorHandler, "passExitCodes", [])

        self.getJobs = self.daoFactory(classname="Jobs.GetAllJobs")
        self.idLoad = self.daoFactory(classname="Jobs.LoadFromIDWithType")
        self.loadAction = self.daoFactory(classname="Jobs.LoadForErrorHandler")

        self.dataCollection = DataCollectionService(url=config.ACDC.couchurl, database=config.ACDC.database)

        return
Ejemplo n.º 9
0
    def __init__(self, config):
        """
        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.config = config

        myThread = threading.currentThread()

        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        self.changeState = ChangeState(self.config)

        self.maxRetries     = self.config.ErrorHandler.maxRetries
        if not isinstance(self.maxRetries, dict):
            self.maxRetries = {'default' : self.maxRetries}
        if 'default' not in self.maxRetries:
            raise ErrorHandlerException('Max retries for the default job type must be specified')

        self.maxProcessSize = getattr(self.config.ErrorHandler, 'maxProcessSize', 250)
        self.exitCodes      = getattr(self.config.ErrorHandler, 'failureExitCodes', [])
        self.maxFailTime    = getattr(self.config.ErrorHandler, 'maxFailTime', 32 * 3600)
        self.readFWJR       = getattr(self.config.ErrorHandler, 'readFWJR', False)
        self.passCodes      = getattr(self.config.ErrorHandler, 'passExitCodes', [])

        self.getJobs    = self.daoFactory(classname = "Jobs.GetAllJobs")
        self.idLoad     = self.daoFactory(classname = "Jobs.LoadFromIDWithType")
        self.loadAction = self.daoFactory(classname = "Jobs.LoadForErrorHandler")

        self.dataCollection = DataCollectionService(url = config.ACDC.couchurl,
                                                    database = config.ACDC.database)

        return
Ejemplo n.º 10
0
    def __init__(self, config):
        """
        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.config = config

        myThread = threading.currentThread()

        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)

        pluginPath = getattr(self.config.RetryManager, "pluginPath",
                             "WMComponent.RetryManager.PlugIns")
        self.pluginFactory = WMFactory("plugins", pluginPath)
        
        self.changeState = ChangeState(self.config)
        self.getJobs     = self.daoFactory(classname = "Jobs.GetAllJobs")
        
        # initialize the alert framework (if available) (self.sendAlert())
        self.initAlerts(compName = "RetryManager")        

        try:
            pluginName  = getattr(self.config.RetryManager, 'pluginName', 'DefaultRetryAlgo')
            self.plugin = self.pluginFactory.loadObject(classname = pluginName,
                                                        args = config)
        except Exception, ex:
            msg =  "Error loading plugin %s on path %s\n" % (pluginName, pluginPath)
            msg += str(ex)
            logging.error(msg)
            self.sendAlert(6, msg = msg)
            raise RetryManagerException(msg)
Ejemplo n.º 11
0
 def __init__(self, config):
     """
     Initialize config
     """
     BaseWorkerThread.__init__(self)
     # set the workqueue service for REST call
     self.config = config
Ejemplo n.º 12
0
    def __init__(self, config):
        """
        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.config = config

        myThread = threading.currentThread()

        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        self.changeState = ChangeState(self.config)

        self.maxRetries     = self.config.ErrorHandler.maxRetries
        self.maxProcessSize = getattr(self.config.ErrorHandler, 'maxProcessSize', 250)

        self.getJobs = self.daoFactory(classname = "Jobs.GetAllJobs")

        self.dataCollection = DataCollectionService(url = config.ACDC.couchurl,
                                                    database = config.ACDC.database)

        # initialize the alert framework (if available - config.Alert present)
        #    self.sendAlert will be then be available    
        self.initAlerts(compName = "ErrorHandler")        
        
        return
Ejemplo n.º 13
0
 def __init__(self, resources):
     """
     Initialise class members
     """
     BaseWorkerThread.__init__(self)
     self.resources = resources
     self.wq = localQueue()
Ejemplo n.º 14
0
    def __init__(self, config, phedex, nodeMappings):
        """
        ___init___

        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.config = config
        self.phedex = phedex
        self.dbsUrl = config.DBSInterface.globalDBSUrl
        self.group = getattr(config.PhEDExInjector, "group", "DataOps")

        # This will be used to map SE names which are stored in the DBSBuffer to
        # PhEDEx node names.  The first key will be the "kind" which consists
        # of one of the following: MSS, Disk, Buffer.  The next key will be the
        # SE name.
        self.seMap = {}
        self.nodeNames = []
        for node in nodeMappings["phedex"]["node"]:
            if node["kind"] not in self.seMap:
                self.seMap[node["kind"]] = {}
            logging.info("Adding mapping %s -> %s", node["se"], node["name"])
            self.seMap[node["kind"]][node["se"]] = node["name"]
            self.nodeNames.append(node["name"])

        self.diskSites = getattr(config.PhEDExInjector, "diskSites", ["storm-fe-cms.cr.cnaf.infn.it",
                                                                      "srm-cms-disk.gridpp.rl.ac.uk"])

        # initialize the alert framework (if available - config.Alert present)
        #    self.sendAlert will be then be available
        self.initAlerts(compName = "PhEDExInjector")

        self.blocksToRecover = []
Ejemplo n.º 15
0
 def __init__(self, config):
     """
     Initialize config
     """
     BaseWorkerThread.__init__(self)
     # set the workqueue service for REST call
     self.config = config
Ejemplo n.º 16
0
    def __init__(self, config):
        """
        ___init___

        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.dbsUrl = config.DBSInterface.globalDBSUrl
        self.phedexGroup = config.PhEDExInjector.phedexGroup

        self.pollCounter = 0
        self.subFrequency = None
        if getattr(config.PhEDExInjector, "subscribeDatasets", False):
            pollInterval = config.PhEDExInjector.pollInterval
            subInterval = config.PhEDExInjector.subscribeInterval
            self.subFrequency = max(1, int(round(subInterval / pollInterval)))
            logging.info("SubscribeDataset and deleteBlocks will run every %d polling cycles", self.subFrequency)
            # subscribe on first cycle
            self.pollCounter = self.subFrequency - 1

        # retrieving the node mappings is fickle and can fail quite often
        self.phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl},
                             "json", dbsUrl=self.dbsUrl)
        try:
            nodeMappings = self.phedex.getNodeMap()
        except:
            time.sleep(2)
            try:
                nodeMappings = self.phedex.getNodeMap()
            except:
                time.sleep(4)
                nodeMappings = self.phedex.getNodeMap()

        # This will be used to map SE names which are stored in the DBSBuffer to
        # PhEDEx node names.  The first key will be the "kind" which consists
        # of one of the following: MSS, Disk, Buffer.  The next key will be the
        # SE name.
        self.seMap = {}
        self.nodeNames = []
        for node in nodeMappings["phedex"]["node"]:
            if node["kind"] not in self.seMap:
                self.seMap[node["kind"]] = {}
            logging.info("Adding mapping %s -> %s", node["se"], node["name"])
            self.seMap[node["kind"]][node["se"]] = node["name"]
            self.nodeNames.append(node["name"])

        self.phedexNodes = {'MSS': [], 'Disk': []}
        for node in nodeMappings["phedex"]["node"]:
            if node["kind"] in ["MSS", "Disk"]:
                self.phedexNodes[node["kind"]].append(node["name"])

        self.blocksToRecover = []

        # X-component configuration is BAD! But it will only be here during the
        # Rucio commissioning within WM
        self.listTiersToSkip = config.RucioInjector.listTiersToInject
        logging.info("Component configured to skip data injection for data tiers: %s",
                     self.listTiersToSkip)

        return
Ejemplo n.º 17
0
    def __init__(self, config):
        """
        ___init___

        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl}, "json")
        self.siteDB = SiteDBJSON()
        self.dbsUrl = config.DBSInterface.globalDBSUrl
        self.group = getattr(config.PhEDExInjector, "group", "DataOps")
        self.safeMode = getattr(config.PhEDExInjector, "safeOperationMode", False)

        # Subscribed state in the DBSBuffer table for datasets
        self.terminalSubscriptionState = 1
        if self.safeMode:
            self.terminalSubscriptionState = 2

        # We will map node names to CMS names, that what the spec will have.
        # If a CMS name is associated to many PhEDEx node then choose the MSS option
        self.cmsToPhedexMap = {}

        # initialize the alert framework (if available - config.Alert present)
        #    self.sendAlert will be then be available    
        self.initAlerts(compName = "PhEDExInjector")
Ejemplo n.º 18
0
    def __init__(self, config):

        BaseWorkerThread.__init__(self)

        self.config = config.AsyncTransfer

        # self.logger is set up by the BaseWorkerThread, we just set it's level
        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging
            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)

        self.logger.debug('Configuration loaded')

        # Set up a factory for loading plugins
        self.factory = WMFactory(self.config.pluginDir, namespace = self.config.pluginDir)

        # Asynch db
        server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
        self.db = server.connectDatabase(self.config.files_database)
        self.logger.debug('Connected to CouchDB')

        return
Ejemplo n.º 19
0
    def __init__(self, config):
        BaseWorkerThread.__init__(self)
        self.config = config
        self.accountantWorkSize = getattr(self.config.JobAccountant,
                                          'accountantWorkSize', 100)

        return
Ejemplo n.º 20
0
    def __init__(self, config):
        """
        initialize properties specified from config
        """
        BaseWorkerThread.__init__(self)
        # set the workqueue service for REST call
        self.config = config
        # need to get campaign, user, owner info
        self.agentInfo = initAgentInfo(self.config)
        self.summaryLevel = config.AnalyticsDataCollector.summaryLevel

        proxyArgs = {'logger': logging.getLogger(), 'cleanEnvironment': True}
        self.proxy = Proxy(proxyArgs)
        self.proxyFile = self.proxy.getProxyFilename()  # X509_USER_PROXY
        self.userCertFile = self.proxy.getUserCertFilename()  # X509_USER_CERT
        # credential lifetime warning/error thresholds, in days
        self.credThresholds = {'proxy': {'error': 3, 'warning': 5},
                               'certificate': {'error': 10, 'warning': 20}}

        # Monitoring setup
        self.userAMQ = getattr(config.AgentStatusWatcher, "userAMQ", None)
        self.passAMQ = getattr(config.AgentStatusWatcher, "passAMQ", None)
        self.postToAMQ = getattr(config.AgentStatusWatcher, "enableAMQ", False)
        self.topicAMQ = getattr(config.AgentStatusWatcher, "topicAMQ", None)
        self.hostPortAMQ = getattr(config.AgentStatusWatcher, "hostPortAMQ", [('cms-mb.cern.ch', 61313)])

        # T0 doesn't have WorkQueue, so some monitoring/replication code has to be skipped here
        if hasattr(self.config, "Tier0Feeder"):
            self.isT0agent = True
            self.producer = "tier0wmagent"
        else:
            self.isT0agent = False
            self.producer = "wmagent"
            localWQUrl = config.AnalyticsDataCollector.localQueueURL
            self.workqueueDS = WorkQueueDS(localWQUrl)
Ejemplo n.º 21
0
 def __init__(self, resources):
     """
     Initialise class members
     """
     BaseWorkerThread.__init__(self)
     self.resources = resources
     self.wq = localQueue()
Ejemplo n.º 22
0
    def __init__(self, config):
        """
        Initialise class members
        """
        #Need a better way to test this without turning off this next line
        BaseWorkerThread.__init__(self)
        #logging.basicConfig(format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',datefmt = '%m-%d %H:%M')
        #self.logger = logging.getLogger()
        # self.logger is set up by the BaseWorkerThread, we just set it's level

        self.config = config.AsyncTransfer
        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging
            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)
        self.pool = Pool(processes=self.config.pool_size)
        # Set up a factory for loading plugins
        self.factory = WMFactory(self.config.schedAlgoDir, namespace = self.config.schedAlgoDir)
        self.dropbox_dir = '%s/dropbox/inputs' % self.config.componentDir
        if not os.path.isdir(self.dropbox_dir):
            try:
                os.makedirs(self.dropbox_dir)
            except OSError, e:
                if e.errno == errno.EEXIST:
                    pass
                else:
                    self.logger.error('Unknown error in mkdir' % e.errno)
                    raise
Ejemplo n.º 23
0
    def __init__(self, config):
        """
        ___init___

        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.config = config
        self.phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl}, "json")
        self.dbsUrl = config.DBSInterface.globalDBSUrl
        self.group = getattr(config.PhEDExInjector, "group", "DataOps")

        # This will be used to map SE names which are stored in the DBSBuffer to
        # PhEDEx node names.  The first key will be the "kind" which consists
        # of one of the following: MSS, Disk, Buffer.  The next key will be the
        # SE name.
        self.seMap = {}
        self.nodeNames = []

        self.diskSites = getattr(config.PhEDExInjector, "diskSites", ["storm-fe-cms.cr.cnaf.infn.it",
                                                                      "srm-cms-disk.gridpp.rl.ac.uk"])

        # initialize the alert framework (if available - config.Alert present)
        #    self.sendAlert will be then be available
        self.initAlerts(compName = "PhEDExInjector")

        self.filesToRecover = None
Ejemplo n.º 24
0
    def __init__(self, config):
        BaseWorkerThread.__init__(self)
        self.config = config.Statistics

        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging
            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)

        self.logger.debug('Configuration loaded')

        server = CouchServer(
            dburl=self.config.couch_instance,
            ckey=self.config.opsProxy,
            cert=self.config.opsProxy)
        self.db = server.connectDatabase(self.config.files_database)
        config_server = CouchServer(
            dburl=self.config.config_couch_instance,
            ckey=self.config.opsProxy,
            cert=self.config.opsProxy)
        self.config_db = config_server.connectDatabase(
            self.config.config_database)
        self.logger.debug('Connected to CouchDB')

        statserver = CouchServer(self.config.couch_statinstance)
        self.statdb = statserver.connectDatabase(
            self.config.statitics_database)
        self.logger.debug('Connected to Stat CouchDB')

        self.iteration_docs = []
        self.exptime = None
Ejemplo n.º 25
0
    def __init__(self, config):
        BaseWorkerThread.__init__(self)
        self.config = config.FilesCleaner
        self.logger.debug('Configuration loaded')

        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging
            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)
        self.logger.debug('Configuration loaded')
        config_server = CouchServer(dburl=self.config.config_couch_instance)
        self.config_db = config_server.connectDatabase(self.config.config_database)
        self.logger.debug('Connected to files DB')
        self.phedex = PhEDEx(responseType='xml')
        self.log_dir = '%s/logs/%s/%s/%s' % ( self.config.componentDir, \
 str(datetime.datetime.now().month), str(datetime.datetime.now().year), "Ops")
        try:
            os.makedirs(self.log_dir)
        except OSError, e:
            if e.errno == errno.EEXIST:
                pass
            else:
                self.logger.error('Unknown error in mkdir' % e.errno)
                raise
Ejemplo n.º 26
0
    def __init__(self, config):
        """
        ___init___

        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl},
                             "json")
        self.siteDB = SiteDBJSON()
        self.dbsUrl = config.DBSInterface.globalDBSUrl
        self.group = getattr(config.PhEDExInjector, "group", "DataOps")
        self.safeMode = getattr(config.PhEDExInjector, "safeOperationMode",
                                False)
        self.replicaOnly = getattr(config.PhEDExInjector, "replicaOnly", False)

        # Subscribed state in the DBSBuffer table for datasets
        self.terminalSubscriptionState = 1
        if self.safeMode:
            self.terminalSubscriptionState = 2

        # We will map node names to CMS names, that what the spec will have.
        # If a CMS name is associated to many PhEDEx node then choose the MSS option
        self.cmsToPhedexMap = {}

        # initialize the alert framework (if available - config.Alert present)
        #    self.sendAlert will be then be available

        self.initAlerts(compName="PhEDExInjector")
Ejemplo n.º 27
0
    def __init__(self, config):
        BaseWorkerThread.__init__(self)
        self.config = config.Statistics

        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging
            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)

        self.logger.debug('Configuration loaded')

        server = CouchServer(dburl=self.config.couch_instance,
                             ckey=self.config.opsProxy,
                             cert=self.config.opsProxy)
        self.db = server.connectDatabase(self.config.files_database)

        config_server = CouchServer(dburl=self.config.config_couch_instance,
                                    ckey=self.config.opsProxy,
                                    cert=self.config.opsProxy)
        self.config_db = config_server.connectDatabase(
            self.config.config_database)
        self.mon_db = server.connectDatabase(self.config.mon_database)
        self.logger.debug('Connected to CouchDB')

        statserver = CouchServer(self.config.couch_statinstance)
        self.statdb = statserver.connectDatabase(
            self.config.statitics_database)
        self.logger.debug('Connected to Stat CouchDB')

        self.iteration_docs = []
        self.exptime = None
 def __init__(self, queue, config):
     """
     Initialise class members
     """
     BaseWorkerThread.__init__(self)
     self.queue = queue
     self.config = config
Ejemplo n.º 29
0
    def __init__(self, config):
        """
        Initialise class members
        """
        #Need a better way to test this without turning off this next line
        BaseWorkerThread.__init__(self)
        #logging.basicConfig(format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',datefmt = '%m-%d %H:%M')
        #self.logger = logging.getLogger()
        # self.logger is set up by the BaseWorkerThread, we just set it's level

        self.config = config.AsyncTransfer
        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging
            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)

        self.logger.debug('Configuration loaded')
        server = CouchServer(dburl=self.config.couch_instance,
                             ckey=self.config.opsProxy,
                             cert=self.config.opsProxy)
        self.db = server.connectDatabase(self.config.files_database)
        config_server = CouchServer(dburl=self.config.config_couch_instance)
        self.config_db = config_server.connectDatabase(
            self.config.config_database)
        self.logger.debug('Connected to CouchDB')
        self.pool = Pool(processes=self.config.pool_size)
        try:
            self.phedex = PhEDEx(responseType='xml')
        except Exception, e:
            self.logger.exception('PhEDEx exception: %s' % e)
Ejemplo n.º 30
0
    def __init__(self, config):
        BaseWorkerThread.__init__(self)
        self.config = config.FilesCleaner
        self.logger.debug('Configuration loaded')

        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging
            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)
        self.logger.debug('Configuration loaded')
        config_server = CouchServer(dburl=self.config.config_couch_instance)
        self.config_db = config_server.connectDatabase(
            self.config.config_database)
        self.logger.debug('Connected to files DB')
        self.phedex = PhEDEx(responseType='xml')
        self.log_dir = '%s/logs/%s/%s/%s' % ( self.config.componentDir, \
 str(datetime.datetime.now().month), str(datetime.datetime.now().year), "Ops")
        try:
            os.makedirs(self.log_dir)
        except OSError, e:
            if e.errno == errno.EEXIST:
                pass
            else:
                self.logger.error('Unknown error in mkdir' % e.errno)
                raise
Ejemplo n.º 31
0
    def __init__(self, config):
        """
        initialize properties specified from config
        """
        BaseWorkerThread.__init__(self)
        # set the workqueue service for REST call
        self.config = config
        # need to get campaign, user, owner info
        self.agentInfo = initAgentInfo(self.config)
        self.summaryLevel = config.AnalyticsDataCollector.summaryLevel

        proxyArgs = {'logger': logging.getLogger()}
        self.proxy = Proxy(proxyArgs)
        self.proxyFile = self.proxy.getProxyFilename()  # X509_USER_PROXY
        self.userCertFile = self.proxy.getUserCertFilename()  # X509_USER_CERT
        # credential lifetime warning/error thresholds, in days
        self.credThresholds = {'proxy': {'error': 3, 'warning': 5},
                               'certificate': {'error': 10, 'warning': 20}}

        # Monitoring setup
        self.userAMQ = getattr(config.AgentStatusWatcher, "userAMQ", None)
        self.passAMQ = getattr(config.AgentStatusWatcher, "passAMQ", None)
        self.postToAMQ = getattr(config.AgentStatusWatcher, "enableAMQ", False)
        self.topicAMQ = getattr(config.AgentStatusWatcher, "topicAMQ", None)
        self.hostPortAMQ = getattr(config.AgentStatusWatcher, "hostPortAMQ", [('cms-mb.cern.ch', 61313)])

        # T0 doesn't have WorkQueue, so some monitoring/replication code has to be skipped here
        if hasattr(self.config, "Tier0Feeder"):
            self.isT0agent = True
            self.producer = "tier0wmagent"
        else:
            self.isT0agent = False
            self.producer = "wmagent"
            localWQUrl = config.AnalyticsDataCollector.localQueueURL
            self.workqueueDS = WorkQueueDS(localWQUrl)
Ejemplo n.º 32
0
 def __init__(self, config):
     """
     initialize properties specified from config
     """
     BaseWorkerThread.__init__(self)
     self.config = config
     self.drainAPI = DrainStatusAPI()
Ejemplo n.º 33
0
    def __init__(self, config):
        """
        ___init___

        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.dbsUrl = config.DBSInterface.globalDBSUrl

        self.pollCounter = 0
        self.subFrequency = None
        if getattr(config.PhEDExInjector, "subscribeDatasets", False):
            pollInterval = config.PhEDExInjector.pollInterval
            subInterval = config.PhEDExInjector.subscribeInterval
            self.subFrequency = max(1, int(round(subInterval / pollInterval)))
            logging.info(
                "SubscribeDataset and deleteBlocks will run every %d polling cycles",
                self.subFrequency)
            # subscribe on first cycle
            self.pollCounter = self.subFrequency - 1

        # retrieving the node mappings is fickle and can fail quite often
        self.phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl},
                             "json")
        try:
            nodeMappings = self.phedex.getNodeMap()
        except:
            time.sleep(2)
            try:
                nodeMappings = self.phedex.getNodeMap()
            except:
                time.sleep(4)
                nodeMappings = self.phedex.getNodeMap()

        # This will be used to map SE names which are stored in the DBSBuffer to
        # PhEDEx node names.  The first key will be the "kind" which consists
        # of one of the following: MSS, Disk, Buffer.  The next key will be the
        # SE name.
        self.seMap = {}
        self.nodeNames = []
        for node in nodeMappings["phedex"]["node"]:
            if node["kind"] not in self.seMap:
                self.seMap[node["kind"]] = {}
            logging.info("Adding mapping %s -> %s", node["se"], node["name"])
            self.seMap[node["kind"]][node["se"]] = node["name"]
            self.nodeNames.append(node["name"])

        self.phedexNodes = {'MSS': [], 'Disk': []}
        for node in nodeMappings["phedex"]["node"]:
            if node["kind"] in ["MSS", "Disk"]:
                self.phedexNodes[node["kind"]].append(node["name"])

        # initialize the alert framework (if available - config.Alert present)
        #    self.sendAlert will be then be available
        self.initAlerts(compName="PhEDExInjector")

        self.blocksToRecover = []

        return
Ejemplo n.º 34
0
    def __init__(self, config):
        """
        _init_

        """
        BaseWorkerThread.__init__(self)

        myThread = threading.currentThread()

        self.daoFactory = DAOFactory(package="T0.WMBS", logger=logging, dbinterface=myThread.dbi)

        self.tier0ConfigFile = config.Tier0Feeder.tier0ConfigFile
        self.specDirectory = config.Tier0Feeder.specDirectory
        self.dropboxuser = getattr(config.Tier0Feeder, "dropboxuser", None)
        self.dropboxpass = getattr(config.Tier0Feeder, "dropboxpass", None)

        self.transferSystemBaseDir = getattr(config.Tier0Feeder, "transferSystemBaseDir", None)
        if self.transferSystemBaseDir != None:
            if not os.path.exists(self.transferSystemBaseDir):
                self.transferSystemBaseDir = None

        self.dqmUploadProxy = getattr(config.Tier0Feeder, "dqmUploadProxy", None)
        self.serviceProxy = getattr(config.Tier0Feeder, "serviceProxy", None)

        self.localRequestCouchDB = RequestDBWriter(
            config.AnalyticsDataCollector.localT0RequestDBURL, couchapp=config.AnalyticsDataCollector.RequestCouchApp
        )

        hltConfConnectUrl = config.HLTConfDatabase.connectUrl
        dbFactoryHltConf = DBFactory(logging, dburl=hltConfConnectUrl, options={})
        dbInterfaceHltConf = dbFactoryHltConf.connect()
        daoFactoryHltConf = DAOFactory(package="T0.WMBS", logger=logging, dbinterface=dbInterfaceHltConf)
        self.getHLTConfigDAO = daoFactoryHltConf(classname="RunConfig.GetHLTConfig")

        storageManagerConnectUrl = config.StorageManagerDatabase.connectUrl
        dbFactoryStorageManager = DBFactory(logging, dburl=storageManagerConnectUrl, options={})
        self.dbInterfaceStorageManager = dbFactoryStorageManager.connect()

        self.getExpressReadyRunsDAO = None
        if hasattr(config, "PopConLogDatabase"):
            popConLogConnectUrl = getattr(config.PopConLogDatabase, "connectUrl", None)
            if popConLogConnectUrl != None:
                dbFactoryPopConLog = DBFactory(logging, dburl=popConLogConnectUrl, options={})
                dbInterfacePopConLog = dbFactoryPopConLog.connect()
                daoFactoryPopConLog = DAOFactory(package="T0.WMBS", logger=logging, dbinterface=dbInterfacePopConLog)
                self.getExpressReadyRunsDAO = daoFactoryPopConLog(classname="Tier0Feeder.GetExpressReadyRuns")

        self.haveT0DataSvc = False
        if hasattr(config, "T0DataSvcDatabase"):
            t0datasvcConnectUrl = getattr(config.T0DataSvcDatabase, "connectUrl", None)
            if t0datasvcConnectUrl != None:
                self.haveT0DataSvc = True
                dbFactoryT0DataSvc = DBFactory(logging, dburl=t0datasvcConnectUrl, options={})
                dbInterfaceT0DataSvc = dbFactoryT0DataSvc.connect()
                self.daoFactoryT0DataSvc = DAOFactory(
                    package="T0.WMBS", logger=logging, dbinterface=dbInterfaceT0DataSvc
                )

        return
Ejemplo n.º 35
0
    def __init__(self, config):
        BaseWorkerThread.__init__(self)
        myThread = threading.currentThread()

        #DAO factory for WMBS objects
        self.daoFactory = DAOFactory(package = "WMCore.WMBS", \
                                     logger = logging,
                                     dbinterface = myThread.dbi)

        self.config = config

        #Libraries
        self.resourceControl = ResourceControl()



        self.changeState = ChangeState(self.config)
        self.repollCount = getattr(self.config.JobSubmitter, 'repollCount', 10000)

        # BossAir
        self.bossAir = BossAirAPI(config = self.config)

        # Additions for caching-based JobSubmitter
        self.workflowTimestamps = {}
        self.cachedJobIDs       = set()
        self.cachedJobs         = {}
        self.jobDataCache       = {}
        self.jobsToPackage      = {}
        self.sandboxPackage     = {}
        self.siteKeys           = {}
        self.locationDict       = {}
        self.cmsNames           = {}
        self.drainSites         = []
        self.sortedSites        = []
        self.packageSize        = getattr(self.config.JobSubmitter, 'packageSize', 500)
        self.collSize           = getattr(self.config.JobSubmitter, 'collectionSize',
                                          self.packageSize * 1000)

        # initialize the alert framework (if available)
        self.initAlerts(compName = "JobSubmitter")

        try:
            if not getattr(self.config.JobSubmitter, 'submitDir', None):
                self.config.JobSubmitter.submitDir = self.config.JobSubmitter.componentDir
            self.packageDir = os.path.join(self.config.JobSubmitter.submitDir, 'packages')

            if not os.path.exists(self.packageDir):
                os.makedirs(self.packageDir)
        except Exception, ex:
            msg =  "Error while trying to create packageDir %s\n!"
            msg += str(ex)
            logging.error(msg)
            self.sendAlert(6, msg = msg)
            try:
                logging.debug("PackageDir: %s" % self.packageDir)
                logging.debug("Config: %s" % config)
            except:
                pass
            raise JobSubmitterPollerException(msg)
Ejemplo n.º 36
0
    def __init__(self, config):
        """
        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.config      = config
        self.changeState = ChangeState(self.config)

        myThread        = threading.currentThread()
        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        self.loadAction = self.daoFactory(classname = "Jobs.LoadFromIDWithWorkflow")


        # Variables
        self.numberOfJobsToCluster = getattr(self.config.JobArchiver,
                                             "numberOfJobsToCluster", 1000)

        # initialize the alert framework (if available)
        self.initAlerts(compName = "JobArchiver")

        try:
            self.uploadPublishDir = getattr(self.config.JobArchiver, 'uploadPublishDir',
                                  os.path.join(config.JobArchiver.componentDir, 'publishDir'))
            self.logDir = getattr(config.JobArchiver, 'logDir',
                                  os.path.join(config.JobArchiver.componentDir, 'logDir'))
            if not os.path.isdir(self.logDir):
                os.makedirs(self.logDir)
            if not os.path.isdir(self.uploadPublishDir):
                os.makedirs(self.uploadPublishDir)
        except Exception as ex:
            msg =  "Unhandled exception while setting up logDir and/or uploadPublishDir!\n"
            msg += str(ex)
            logging.error(msg)
            self.sendAlert(6, msg = msg)
            try:
                logging.debug("Directory: %s" % self.logDir)
                logging.debug("Config: %s" % config)
            except:
                pass
            raise JobArchiverPollerException(msg)

        try:
            from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
            self.workQueue = queueFromConfig(self.config)
        except Exception as ex:
            msg =  "Could not load workQueue"
            msg += str(ex)
            logging.error(msg)
            #raise JobArchiverPollerException(msg)

        self.uploadPublishInfo = getattr(self.config.JobArchiver, 'uploadPublishInfo', False)
        self.userFileCacheURL = getattr(self.config.JobArchiver, 'userFileCacheURL', None)
        self.handleWorkflowInjection = getattr(self.config.JobArchiver,
                                               'handleInjected', True)

        return
Ejemplo n.º 37
0
    def __init__(self, config):
        """
        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.config = config
        self.changeState = ChangeState(self.config)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package="WMCore.WMBS",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)
        self.loadAction = self.daoFactory(classname="Jobs.LoadFromIDWithWorkflow")


        # Variables
        self.numberOfJobsToCluster = getattr(self.config.JobArchiver,
                                             "numberOfJobsToCluster", 1000)

        # initialize the alert framework (if available)
        self.initAlerts(compName="JobArchiver")

        try:
            self.uploadPublishDir = getattr(self.config.JobArchiver, 'uploadPublishDir',
                                            os.path.join(config.JobArchiver.componentDir, 'publishDir'))
            self.logDir = getattr(config.JobArchiver, 'logDir',
                                  os.path.join(config.JobArchiver.componentDir, 'logDir'))
            if not os.path.isdir(self.logDir):
                os.makedirs(self.logDir)
            if not os.path.isdir(self.uploadPublishDir):
                os.makedirs(self.uploadPublishDir)
        except Exception as ex:
            msg = "Unhandled exception while setting up logDir and/or uploadPublishDir!\n"
            msg += str(ex)
            logging.error(msg)
            self.sendAlert(6, msg=msg)
            try:
                logging.debug("Directory: %s", self.logDir)
                logging.debug("Config: %s", config)
            except:
                pass
            raise JobArchiverPollerException(msg)

        try:
            from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
            self.workQueue = queueFromConfig(self.config)
        except Exception as ex:
            msg = "Could not load workQueue"
            msg += str(ex)
            logging.error(msg)
            # raise JobArchiverPollerException(msg)

        self.uploadPublishInfo = getattr(self.config.JobArchiver, 'uploadPublishInfo', False)
        self.userFileCacheURL = getattr(self.config.JobArchiver, 'userFileCacheURL', None)
        self.handleWorkflowInjection = getattr(self.config.JobArchiver,
                                               'handleInjected', True)

        return
Ejemplo n.º 38
0
 def __init__(self, threads=8):
     """
     Initialise class members
     """
     BaseWorkerThread.__init__(self)
     self.fileset_watch = {}
     self.last_poll_time = time.time() / 60
     self.workq = ThreadPool \
           ([self.pollExternal for _ in range(threads)])
Ejemplo n.º 39
0
 def __init__(self, threads=8):
     """
     Initialise class members
     """
     BaseWorkerThread.__init__(self)
     self.fileset_watch = {}
     self.last_poll_time = time.time()/60
     self.workq = ThreadPool \
           ([self.pollExternal for _ in range(threads)])
Ejemplo n.º 40
0
    def __init__(self, config):
        """
        Initialize
        """
        BaseWorkerThread.__init__(self)
        self.config = config

        self.ssb2AgentStatus = {
            'enabled': 'Normal',
            'drain': 'Draining',
            'disabled': 'Down',
            'test': 'Draining',
            'unknown': None
        }
        self.tasksCPU = ['Processing', 'Production']
        self.tasksIO = ['Merge', 'Cleanup', 'Harvesting', 'LogCollect', 'Skim']
        self.minCPUSlots = 50
        self.minIOSlots = 25

        # get dashboard url, set metric columns from config
        _token = config.AgentStatusWatcher.grafanaToken
        self.grafanaURL = config.AgentStatusWatcher.grafanaURL
        self.grafanaAPIName = config.AgentStatusWatcher.grafanaSSB
        self.grafana = Grafana(_token,
                               configDict={"endpoint": self.grafanaURL})

        # set pending percentages from config
        self.pendingSlotsSitePercent = config.AgentStatusWatcher.pendingSlotsSitePercent
        self.pendingSlotsTaskPercent = config.AgentStatusWatcher.pendingSlotsTaskPercent
        self.runningExpressPercent = config.AgentStatusWatcher.runningExpressPercent
        self.runningRepackPercent = config.AgentStatusWatcher.runningRepackPercent

        # sites forced to down
        self.forceSiteDown = getattr(config.AgentStatusWatcher,
                                     'forceSiteDown', [])

        # agent team (for dynamic threshold) and queueParams (drain mode)
        self.teamName = config.Agent.teamName
        self.agentsNumByTeam = getattr(config.AgentStatusWatcher,
                                       'defaultAgentsNumByTeam', 5)

        # only SSB sites
        self.onlySSB = config.AgentStatusWatcher.onlySSB

        # tier mode
        self.tier0Mode = hasattr(config, "Tier0Feeder")
        self.t1SitesCores = config.AgentStatusWatcher.t1SitesCores

        # switch this component on/off
        self.enabled = getattr(config.AgentStatusWatcher, 'enabled', True)

        # set resource control
        self.resourceControl = ResourceControl(config=self.config)

        # wmstats connection
        self.centralCouchDBReader = WMStatsReader(
            self.config.General.centralWMStatsURL)
Ejemplo n.º 41
0
    def __init__(self, config):
        BaseWorkerThread.__init__(self)
        self.config = config
        self.accountantWorkSize = getattr(self.config.JobAccountant, 'accountantWorkSize', 100)
        # initialize the alert framework (if available - config.Alert present)
        #    self.sendAlert will be then be available
        self.initAlerts(compName = "JobAccountant")

        return
    def __init__(self, queue):
        """
        Initialise class members
        """
        BaseWorkerThread.__init__(self)

        self.queue = queue

        self.previousWorkList = []
    def __init__(self, queue):
        """
        Initialise class members
        """
        BaseWorkerThread.__init__(self)

        self.queue = queue

        self.previousWorkList = []
Ejemplo n.º 44
0
    def __init__(self, config):
        """
        ___init___

        Initialise class members
        """
        BaseWorkerThread.__init__(self)

        self.enabled = config.RucioInjector.enabled
        # dataset rule creation has a larger polling cycle
        self.pollRules = config.RucioInjector.pollIntervalRules
        self.lastRulesExecTime = 0
        self.createBlockRules = config.RucioInjector.createBlockRules
        self.containerDiskRuleParams = config.RucioInjector.containerDiskRuleParams
        self.containerDiskRuleRSEExpr = config.RucioInjector.containerDiskRuleRSEExpr
        self.skipRulesForTiers = config.RucioInjector.skipRulesForTiers
        self.listTiersToInject = config.RucioInjector.listTiersToInject
        if config.RucioInjector.metaDIDProject not in RUCIO_VALID_PROJECT:
            msg = "Component configured with an invalid 'project' DID: %s"
            raise RucioInjectorException(msg % config.RucioInjector.metaDIDProject)
        self.metaDIDProject = dict(project=config.RucioInjector.metaDIDProject)

        # setup cache for container and blocks (containers can be much longer, make 6 days now)
        self.containersCache = MemoryCache(config.RucioInjector.cacheExpiration * 3, set())
        self.blocksCache = MemoryCache(config.RucioInjector.cacheExpiration, set())

        self.scope = getattr(config.RucioInjector, "scope", "cms")
        self.rucioAcct = config.RucioInjector.rucioAccount
        self.rucio = Rucio(acct=self.rucioAcct,
                           hostUrl=config.RucioInjector.rucioUrl,
                           authUrl=config.RucioInjector.rucioAuthUrl,
                           configDict={'logger': self.logger})

        # metadata dictionary information to be added to block/container rules
        # cannot be a python dictionary, but a JSON string instead
        self.metaData = json.dumps(dict(agentHost=config.Agent.hostName,
                                        userAgent=config.Agent.agentName))

        self.testRSEs = config.RucioInjector.RSEPostfix
        self.filesToRecover = []

        # output data placement has a different behaviour between T0 and Production agents
        if hasattr(config, "Tier0Feeder"):
            logging.info("RucioInjector running on a T0 WMAgent")
            self.isT0agent = True
        else:
            self.isT0agent = False

        if not self.listTiersToInject:
            logging.info("Component configured to inject all the data tiers")
        else:
            logging.info("Component configured to only inject data for data tiers: %s",
                         self.listTiersToInject)
        logging.info("Component configured to skip container rule creation for data tiers: %s",
                     self.skipRulesForTiers)
        logging.info("Component configured to create block rules: %s", self.createBlockRules)
    def __init__(self, config):
        """
        ___init___

        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.dbsUrl = config.DBSInterface.globalDBSUrl

        self.pollCounter = 0
        self.subFrequency = None
        if getattr(config.PhEDExInjector, "subscribeDatasets", False):
            pollInterval = config.PhEDExInjector.pollInterval
            subInterval = config.PhEDExInjector.subscribeInterval
            self.subFrequency = max(1, int(round(subInterval/pollInterval)))
            logging.info("SubscribeDataset and deleteBlocks will run every %d polling cycles", self.subFrequency)
            # subscribe on first cycle
            self.pollCounter = self.subFrequency - 1

        # retrieving the node mappings is fickle and can fail quite often
        self.phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl}, "json")
        try:
            nodeMappings = self.phedex.getNodeMap()
        except:
            time.sleep(2)
            try:
                nodeMappings = self.phedex.getNodeMap()
            except:
                time.sleep(4)
                nodeMappings = self.phedex.getNodeMap()

        # This will be used to map SE names which are stored in the DBSBuffer to
        # PhEDEx node names.  The first key will be the "kind" which consists
        # of one of the following: MSS, Disk, Buffer.  The next key will be the
        # SE name.
        self.seMap = {}
        self.nodeNames = []
        for node in nodeMappings["phedex"]["node"]:
            if node["kind"] not in self.seMap:
                self.seMap[node["kind"]] = {}
            logging.info("Adding mapping %s -> %s", node["se"], node["name"])
            self.seMap[node["kind"]][node["se"]] = node["name"]
            self.nodeNames.append(node["name"])

        self.phedexNodes = {'MSS': [], 'Disk': []}
        for node in nodeMappings["phedex"]["node"]:
            if node["kind"] in ["MSS", "Disk"]:
                self.phedexNodes[node["kind"]].append(node["name"])

        # initialize the alert framework (if available - config.Alert present)
        #    self.sendAlert will be then be available
        self.initAlerts(compName = "PhEDExInjector")

        self.blocksToRecover = []

        return
Ejemplo n.º 46
0
    def __init__(self, config):
        """
        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.logger = logging.getLogger()

        self.config = config.AsyncTransferTest
        self.logger.setLevel(self.config.log_level)
        self.logger.debug('Configuration loaded')
 def __init__(self, queue, config):
     """
     Initialise class members
     """
     BaseWorkerThread.__init__(self)
     self.queue = queue
     self.config = config
     self.reqmgr2Svc = ReqMgr(self.config.TaskArchiver.ReqMgr2ServiceURL)
     # state lists which shouldn't be populated in wmbs. (To prevent creating work before WQE status updated)
     self.abortedAndForceCompleteWorkflowCache = self.reqmgr2Svc.getAbortedAndForceCompleteRequestsFromMemoryCache(expire=120)
Ejemplo n.º 48
0
 def __init__(self, config):
     """
     initialize properties specified from config
     """
     BaseWorkerThread.__init__(self)
     # set the workqueue service for REST call
     self.config = config
     # need to get campaign, user, owner info
     self.agentInfo = initAgentInfo(self.config)
     self.summaryLevel = (config.AnalyticsDataCollector.summaryLevel).lower()
Ejemplo n.º 49
0
    def __init__(self, config):
        """
        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.logger = logging.getLogger()

        self.config = config.AsyncTransferTest
        self.logger.setLevel(self.config.log_level)
        self.logger.debug('Configuration loaded')
Ejemplo n.º 50
0
    def __init__(self, config):
        BaseWorkerThread.__init__(self)
        self.config = config
        self.accountantWorkSize = getattr(self.config.JobAccountant,
                                          'accountantWorkSize', 100)
        # initialize the alert framework (if available - config.Alert present)
        #    self.sendAlert will be then be available
        self.initAlerts(compName = "JobAccountant")

        return
Ejemplo n.º 51
0
 def __init__(self, config):
     """
     initialize properties specified from config
     """
     BaseWorkerThread.__init__(self)
     # set the workqueue service for REST call
     self.config = config
     # need to get campaign, user, owner info
     self.agentInfo = initAgentInfo(self.config)
     self.summaryLevel = (config.AnalyticsDataCollector.summaryLevel).lower()
Ejemplo n.º 52
0
 def __init__(self, queue, config):
     """
     Initialise class members
     """
     BaseWorkerThread.__init__(self)
     self.queue = queue
     self.config = config
     self.reqmgr2Svc = ReqMgr(self.config.TaskArchiver.ReqMgr2ServiceURL)
     # state lists which shouldn't be populated in wmbs. (To prevent creating work before WQE status updated)
     self.abortedAndForceCompleteWorkflowCache = self.reqmgr2Svc.getAbortedAndForceCompleteRequestsFromMemoryCache(
     )
Ejemplo n.º 53
0
    def __init__(self, config):
        """
        Initialise class members
        """
        BaseWorkerThread.__init__(self)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package="WMCore.WMBS",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)

        self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
                                        logger=myThread.logger,
                                        dbinterface=myThread.dbi)

        self.config = config
        self.jobCacheDir = self.config.JobCreator.jobCacheDir

        if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
            # Get workqueue setup from config unless overridden
            if hasattr(self.config.TaskArchiver, 'WorkQueueParams'):
                self.workQueue = localQueue(
                    **self.config.TaskArchiver.WorkQueueParams)
            else:
                from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
                self.workQueue = queueFromConfig(self.config)
        else:
            self.workQueue = None

        self.timeout = getattr(self.config.TaskArchiver, "timeOut", None)
        self.useReqMgrForCompletionCheck = getattr(
            self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)

        if not self.useReqMgrForCompletionCheck:
            #sets the local monitor summary couch db
            self.requestLocalCouchDB = RequestDBWriter(
                self.config.AnalyticsDataCollector.localT0RequestDBURL,
                couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
            self.centralCouchDBWriter = self.requestLocalCouchDB
        else:
            self.centralCouchDBWriter = RequestDBWriter(
                self.config.AnalyticsDataCollector.centralRequestDBURL)

            self.reqmgr2Svc = ReqMgr(
                self.config.TaskArchiver.ReqMgr2ServiceURL)
            #TODO: remove this when reqmgr2 replace reqmgr completely (reqmgr2Only)
            self.reqmgrSvc = RequestManager(
                {'endpoint': self.config.TaskArchiver.ReqMgrServiceURL})

        #Load the cleanout state ID and save it
        stateIDDAO = self.daoFactory(classname="Jobs.GetStateID")
        self.stateID = stateIDDAO.execute("cleanout")

        return
Ejemplo n.º 54
0
    def __init__(self, config):
        """
        Initialise class members
        """
        BaseWorkerThread.__init__(self)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)

        self.config      = config
        self.jobCacheDir = self.config.JobCreator.jobCacheDir
        
        if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
            # Get workqueue setup from config unless overridden
            if hasattr(self.config.TaskArchiver, 'WorkQueueParams'):
                self.workQueue = localQueue(**self.config.TaskArchiver.WorkQueueParams)
            else:
                from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
                self.workQueue = queueFromConfig(self.config)
        else:
            self.workQueue = None

        self.timeout    = getattr(self.config.TaskArchiver, "timeOut", 0)
        self.nOffenders = getattr(self.config.TaskArchiver, 'nOffenders', 3)

        # Set up optional histograms
        self.histogramKeys  = getattr(self.config.TaskArchiver, "histogramKeys", [])
        self.histogramBins  = getattr(self.config.TaskArchiver, "histogramBins", 10)
        self.histogramLimit = getattr(self.config.TaskArchiver, "histogramLimit", 5.0)

        # Start a couch server for getting job info
        # from the FWJRs for committal to archive
        try:
            self.dbname       = getattr(self.config.TaskArchiver, 'workloadSummaryCouchURL',
                                        self.config.JobStateMachine.couchDBName)
            self.couchdb      = CouchServer(self.config.JobStateMachine.couchurl)
            self.summarydb    = getattr(self.config.TaskArchiver, "summaryDBName", self.dbname)
            self.jobsdatabase = self.couchdb.connectDatabase("%s/jobs" % self.dbname)
            self.fwjrdatabase = self.couchdb.connectDatabase("%s/fwjrs" % self.dbname)
            self.workdatabase = self.couchdb.connectDatabase(self.dbname)
            logging.debug("Using url %s" % self.config.JobStateMachine.couchurl)
            logging.debug("Writing to %s" % self.dbname)
            self.requireCouch = getattr(self.config.TaskArchiver, 'requireCouch', False)
        except Exception, ex:
            msg =  "Error in connecting to couch.\n"
            msg += str(ex)
            logging.error(msg)
            self.jobsdatabase = None
            self.fwjrdatabase = None
            if getattr(self.config.TaskArchiver, 'requireCouch', False):
                raise TaskArchiverPollerException(msg)
 def __init__(self, queue, config, reqMgr = None):
     """
     Initialise class members
     """
     BaseWorkerThread.__init__(self)
     self.wq = queue
     self.config = config
     if reqMgr:
         self.reqMgr = reqMgr
     else:
         self.reqMgr = WorkQueueReqMgrInterface(**self.config)
     self.previousState = {}
Ejemplo n.º 56
0
 def __init__(self, queue, config, reqMgr=None):
     """
     Initialise class members
     """
     BaseWorkerThread.__init__(self)
     self.wq = queue
     self.config = config
     if reqMgr:
         self.reqMgr = reqMgr
     else:
         self.reqMgr = WorkQueueReqMgrInterface(**self.config)
     self.previousState = {}
Ejemplo n.º 57
0
 def __init__(self):
     """
     Get a callback to the test object used to access "thread doing stuff"
     data
     """
     BaseWorkerThread.__init__(self)
     myThread = threading.currentThread()
     self.dummySetupCallback = myThread.dummySetupCallback
     self.dummyAlgoCallback = myThread.dummyAlgoCallback
     self.dummyTerminateCallback = myThread.dummyTerminateCallback
     if 'workerThreadManager' in dir(myThread):
         self.workerThreadManager = myThread.workerThreadManager
Ejemplo n.º 58
0
 def __init__(self):
     """
     Get a callback to the test object used to access "thread doing stuff"
     data
     """
     BaseWorkerThread.__init__(self)
     myThread = threading.currentThread()
     self.dummySetupCallback = myThread.dummySetupCallback
     self.dummyAlgoCallback = myThread.dummyAlgoCallback
     self.dummyTerminateCallback = myThread.dummyTerminateCallback
     if 'workerThreadManager' in dir(myThread):
         self.workerThreadManager = myThread.workerThreadManager
Ejemplo n.º 59
0
    def __init__(self, config):
        """
        init jobCreator
        """

        BaseWorkerThread.__init__(self)

        myThread = threading.currentThread()

        # DAO factory for WMBS objects
        self.daoFactory = DAOFactory(package="WMCore.WMBS",
                                     logger=logging,
                                     dbinterface=myThread.dbi)

        self.setBulkCache = self.daoFactory(classname="Jobs.SetCache")
        self.countJobs = self.daoFactory(
            classname="Jobs.GetNumberOfJobsPerWorkflow")
        self.subscriptionList = self.daoFactory(
            classname="Subscriptions.ListIncomplete")
        self.setFWJRPath = self.daoFactory(classname="Jobs.SetFWJRPath")

        # information
        self.config = config

        # Variables
        self.defaultJobType = config.JobCreator.defaultJobType
        self.limit = getattr(config.JobCreator, 'fileLoadLimit', 500)
        self.agentNumber = int(getattr(config.Agent, 'agentNumber', 0))
        self.agentName = getattr(config.Agent, 'hostName', '')
        self.glideinLimits = getattr(config.JobCreator, 'GlideInRestriction',
                                     None)

        # initialize the alert framework (if available - config.Alert present)
        #    self.sendAlert will be then be available
        self.initAlerts(compName="JobCreator")

        try:
            self.jobCacheDir = getattr(
                config.JobCreator, 'jobCacheDir',
                os.path.join(config.JobCreator.componentDir, 'jobCacheDir'))
            self.check()
        except WMException:
            raise
        except Exception as ex:
            msg = "Unhandled exception while setting up jobCacheDir!\n"
            msg += str(ex)
            logging.error(msg)
            self.sendAlert(6, msg=msg)
            raise JobCreatorException(msg)

        self.changeState = ChangeState(self.config)

        return