def __init__(self, *args, **kwargs): """Constructor.""" AgentModule.__init__(self, *args, **kwargs) self.rsClient = None self.months = 36
def __init__(self, *args, **kwargs): AgentModule.__init__(self, *args, **kwargs) self.xmlPath = None self.dryRun = False
def __init__( self, *args, **kwargs ): """ c'tor """ AgentModule.__init__( self, *args, **kwargs ) self.queueDict = {} self.queueCECache = {} self.queueSlots = {} self.failedQueues = defaultdict( int ) self.firstPass = True self.maxJobsInFillMode = MAX_JOBS_IN_FILLMODE self.maxPilotsToSubmit = MAX_PILOTS_TO_SUBMIT self.gridEnv = '' self.vo = '' self.group = '' # self.voGroups contain all the eligible user groups for pilots submitted by this SiteDirector self.voGroups = [] self.pilotDN = '' self.pilotGroup = '' self.platforms = [] self.sites = [] self.proxy = None self.updateStatus = True self.getOutput = False self.sendAccounting = True
def __init__(self, *args, **kwargs): """c'tor""" AgentModule.__init__(self, *args, **kwargs) self.jobDB = None self.pilotDB = None self.diracadmin = None
def __init__(self, *args, **kwargs): AgentModule.__init__(self, *args, **kwargs) self.diracAdmin = None self.default_value = None self.rsClient = ResourceStatusClient()
def __init__(self, *args, **kwargs): """ c'tor Always call this in the extension agent """ AgentModule.__init__(self, *args, **kwargs) TransformationAgentsUtilities.__init__(self) self.transClient = None self.jobManagerClient = None self.transType = [] self.tasksPerLoop = 50 self.maxParametricJobs = 20 # will be updated in execute() # credentials self.shifterProxy = None self.credentials = None self.credTuple = (None, None, None) self.pluginLocation = '' self.bulkSubmissionFlag = False # for the threading self.transQueue = Queue() self.transInQueue = [] self.transInThread = {}
def __init__( self, *args, **kwargs ): ''' c'tor ''' AgentModule.__init__( self, *args, **kwargs ) # # replica manager self.replicaManager = ReplicaManager() # # transformation client self.transClient = TransformationClient() # # wms client self.wmsClient = WMSClient() # # request client self.requestClient = RequestClient() # # file catalog clinet self.metadataClient = FileCatalogClient() # # placeholders for CS options # # transformations types self.transformationTypes = None # # directory locations self.directoryLocations = None # # transformation metadata self.transfidmeta = None # # archive periof in days self.archiveAfter = None # # active SEs self.activeStorages = None # # transformation log SEs self.logSE = None # # enable/disable execution self.enableFlag = None
def __init__(self, *args, **kwargs): ''' c'tor ''' AgentModule.__init__(self, *args, **kwargs) self.rsClient = None
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) self.shifterProxy = None # # transformation client self.transClient = None # # wms client self.wmsClient = None # # request client self.reqClient = None # # file catalog client self.metadataClient = None # # transformations types self.transformationTypes = None # # directory locations self.directoryLocations = ['TransformationDB', 'MetadataCatalog'] # # transformation metadata self.transfidmeta = 'TransformationID' # # archive periof in days self.archiveAfter = 7 # # transformation log SEs self.logSE = 'LogSE' # # enable/disable execution self.enableFlag = 'True' self.dataProcTTypes = ['MCSimulation', 'Merge'] self.dataManipTTypes = ['Replication', 'Removal']
def __init__( self, *args, **kwargs ): ''' c'tor ''' AgentModule.__init__( self, *args, **kwargs ) self.rsClient = None
def __init__(self, *args, **kwargs): """Initialize the agent, clients, default values.""" AgentModule.__init__(self, *args, **kwargs) self.name = 'MonitorAgents' self.setup = "Production" self.enabled = False self.restartAgents = False self.restartExecutors = False self.restartServices = False self.controlComponents = False self.commitURLs = False self.diracLocation = "/opt/dirac/pro" self.sysAdminClient = SystemAdministratorClient(socket.gethostname()) self.jobMonClient = JobMonitoringClient() self.nClient = NotificationClient() self.csAPI = None self.agents = dict() self.executors = dict() self.services = dict() self.errors = list() self.accounting = defaultdict(dict) self.addressTo = ["*****@*****.**"] self.addressFrom = "*****@*****.**" self.emailSubject = "MonitorAgents on %s" % socket.gethostname()
def __init__(self, *args, **kwargs): """Initialize the agent, clients, default values.""" AgentModule.__init__(self, *args, **kwargs) self.name = "ComponentSupervisionAgent" self.setup = "DIRAC-Production" self.enabled = False self.restartAgents = False self.restartExecutors = False self.restartServices = False self.controlComponents = False self.commitURLs = False self.doNotRestartInstancePattern = ["RequestExecutingAgent"] self.diracLocation = rootPath self.sysAdminClient = SystemAdministratorClient(socket.getfqdn()) self.jobMonClient = JobMonitoringClient() self.nClient = NotificationClient() self.csAPI = None self.agents = dict() self.executors = dict() self.services = dict() self._tornadoPort = "8443" self.errors = list() self.accounting = defaultdict(dict) self.addressTo = [] self.addressFrom = "" self.emailSubject = "ComponentSupervisionAgent on %s" % socket.getfqdn( )
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) self.bkClient = None self.prodReq = None
def __init__( self, *args, **kwargs ): """ c'tor """ AgentModule.__init__( self, *args, **kwargs ) # # data manager self.dm = None # # transformation client self.transClient = None # # wms client self.wmsClient = None # # request client self.reqClient = None # # file catalog client self.metadataClient = None # # transformations types self.transformationTypes = None # # directory locations self.directoryLocations = None # # transformation metadata self.transfidmeta = None # # archive periof in days self.archiveAfter = None # # active SEs self.activeStorages = None # # transformation log SEs self.logSE = None # # enable/disable execution self.enableFlag = None
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) TransformationAgentsUtilities.__init__(self) self.bkQueriesToBeChecked = Queue.Queue() self.bkQueriesInCheck = [] self.fullUpdatePeriod = 86400 self.bkUpdateLatency = 7200 self.debug = False self.transInThread = {} self.pickleFile = 'BookkeepingWatchAgent.pkl' self.chunkSize = 1000 self.pluginsWithNoRunInfo = ['LHCbStandard', 'ReplicateDataset', 'ArchiveDataset', 'LHCbMCDSTBroadcastRandom', 'ReplicateToLocalSE', 'RemoveReplicas', 'RemoveReplicasWhenProcessed', 'RemoveReplicasWithAncestors', 'ReplicateWithAncestors', 'ReduceReplicas', 'RemoveDatasetFromDisk', 'DestroyDataset', 'DestroyDatasetWhenProcessed', 'BySize', 'Standard'] self.timeLog = {} self.fullTimeLog = {} self.bkQueries = {} self.transClient = None self.bkClient = None
def __init__(self, *args, **kwargs): """c'tor""" AgentModule.__init__(self, *args, **kwargs) TransformationAgentsUtilities.__init__(self) # few parameters self.pluginLocation = "" self.transformationStatus = [] self.maxFiles = 0 self.transformationTypes = [] # clients (out of the threads) self.transfClient = None # parameters for caching self.workDirectory = "" self.cacheFile = "" self.controlDirectory = "" self.lastFileOffset = {} # Validity of the cache self.replicaCache = None self.replicaCacheValidity = None self.writingCache = False self.removedFromCache = 0 self.noUnusedDelay = 0 self.unusedFiles = {} self.unusedTimeStamp = {} self.debug = False self.pluginTimeout = {}
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) self.vmTypeDict = {} self.vmTypeCECache = {} self.vmTypeSlots = {} self.failedVMTypes = defaultdict(int) self.firstPass = True self.vo = '' self.group = '' # self.voGroups contain all the eligible user groups for clouds submitted by this SiteDirector self.voGroups = [] self.cloudDN = '' self.cloudGroup = '' self.platforms = [] self.sites = [] self.wmsClient = WMSAdministratorClient() self.proxy = None self.updateStatus = True self.getOutput = False self.sendAccounting = True
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) self.jobDB = None self.pilotDB = None self.diracadmin = None
def __init__( self, agentName, baseAgentName, properties ): AgentModule.__init__( self, agentName, baseAgentName, properties ) self.addressTo = '' self.addressFrom = '' self.voName = '' self.csAPI = CSAPI() self.subject = "CE2CSAgent" self.alternativeBDIIs = []
def __init__( self, *args, **kwargs ): """ Constructor. """ AgentModule.__init__( self, *args, **kwargs ) self.rsClient = None
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) self.rawIntegrityDB = None self.fileCatalog = None self.onlineRequestMgr = None
def __init__( self, agentName, loadName, baseAgentName=False, properties=dict() ): """ c'tor :param self: self reference :param str agentName: name of agent :param bool baseAgentName: whatever :param dict properties: whatever else """ AgentModule.__init__( self, agentName, loadName, baseAgentName, properties )
def __init__(self, *args, **kwargs): AgentModule.__init__(self, *args, **kwargs) # ElementType, to be defined among Site, Resource or Node self.sitesToBeChecked = None self.threadPool = None self.siteClient = None self.clients = {}
def __init__(self, *args, **kwargs): AgentModule.__init__(self, *args, **kwargs) self.commands = {} self.clients = {} self.cCaller = None self.rmClient = None
def __init__( self, *args, **kwargs ): AgentModule.__init__( self, *args, **kwargs ) # ElementType, to be defined among Site, Resource or Node self.sitesToBeChecked = None self.threadPool = None self.siteClient = None self.clients = {}
def __init__(self, *args, **kwargs): """c'tor""" AgentModule.__init__(self, *args, **kwargs) # ElementType, to be defined among Resource or Node self.elementType = "Resource" self.rsClient = None self.clients = {}
def __init__( self, *args, **kwargs ): ''' c'tor ''' AgentModule.__init__( self, *args, **kwargs ) self.taskManager = None self.shifterProxy = '' self.transClient = TransformationClient() self.transType = []
def __init__( self, *args, **kwargs ): AgentModule.__init__( self, *args, **kwargs ) self.commands = {} self.clients = {} self.cCaller = None self.rmClient = None
def __init__(self, agentName, baseAgentName=False, properties=dict()): """ c'tor :param self: self reference :param str agentName: name of agent :param bool baseAgentName: whatever :param dict properties: whatever else """ AgentModule.__init__(self, agentName, baseAgentName, properties)
def __init__( self, *args, **kwargs ): """ Constructor """ AgentModule.__init__( self, *args, **kwargs ) self.opHelper = None self.stopAgentPath = '' self.cfgToUpdate = ''
def __init__( self, *args, **kwargs ): AgentModule.__init__( self, *args, **kwargs ) self.diracAdmin = None self.default_value = None if 'DIRAC' in os.environ: self.cacheFile = os.path.join( os.getenv('DIRAC'), 'work/ResourceStatus/cache.db' ) else: self.cacheFile = os.path.realpath('cache.db')
def __init__( self, *args ): """ Initialize the base class and define some extra data members """ AgentModule.__init__( self, *args ) self.requestDBClient = None self.replicaManager = None self.maxNumberOfThreads = 4 self.maxRequestsInQueue = 100 self.threadPool = None
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) self.transClient = None self.bkClient = None self.notifyClient = None self.operations = None self.failedTransIDs = []
def __init__(self, *args, **kwargs): """c'tor""" AgentModule.__init__(self, *args, **kwargs) # clients self.jobDB = None self.maxJobsAtOnce = 500 self.prodTypes = [] self.removeStatusDelay = {} self.removeStatusDelayHB = {}
def __init__(self, *args, **kwargs): """c'tor""" AgentModule.__init__(self, *args, **kwargs) self.notifyHours = 12 self.adminMail = "" self.rsClient = None self.tokenDict = {} self.diracAdmin = None
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) self.jobDB = None self.logDB = None self.startingMinorStatus = None self.startingMajorStatus = JobStatus.CHECKING self.failedStatus = None self.requiredJobInfo = 'jdl' self._initResult = None
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) self.jobDB = None self.logDB = None self.matchedTime = 7200 self.rescheduledTime = 600 self.completedTime = 86400 self.submittingTime = 300
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) self.jobDB = None self.logDB = None self.startingMinorStatus = None self.startingMajorStatus = "Checking" self.failedStatus = None self.requiredJobInfo = 'jdl' self._initResult = None
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) self.jobDB = None self.logDB = None self.matchedTime = 7200 self.rescheduledTime = 600 self.submittingTime = 300 self.stalledJobsTolerantSites = []
def __init__( self, *args, **kwargs ): """ c'tor """ AgentModule.__init__( self, *args, **kwargs ) self.queueDict = {} self.queueCECache = {} self.queueSlots = {} self.failedQueues = defaultdict( int ) self.firstPass = True self.maxJobsInFillMode = MAX_JOBS_IN_FILLMODE self.maxPilotsToSubmit = MAX_PILOTS_TO_SUBMIT
def __init__( self, *args, **kwargs ): ''' c'tor ''' AgentModule.__init__( self, *args, **kwargs ) self.maxCacheLifetime = self.__maxCacheLifetime self.maxHistoryLifetime = self.__maxHistoryLifetime self.maxLogLifetime = self.__maxLogLifetime self.rsClient = None self.rmClient = None
def __init__( self, *args, **kwargs ): ''' c'tor ''' AgentModule.__init__( self, *args, **kwargs ) self.notifyHours = 12 self.adminMail = '' self.rsClient = None self.tokenDict = {} self.diracAdmin = None
def __init__( self, *args, **kwargs ): """ c'tor :param self: self reference :param str agentName: name of agent :param bool baseAgentName: whatever :param dict properties: whatever else """ AgentModule.__init__( self, *args, **kwargs ) self.transClient = TransformationClient() self.transfStatuses = self.am_getOption( 'TransformationStatuses', ['Active', 'Stopped'] )
def __init__( self, *args, **kwargs ): """ c'tor """ AgentModule.__init__( self, *args, **kwargs ) # ElementType, to be defined among Site, Resource or Node self.elementType = '' self.elementsToBeChecked = None self.threadPool = None self.rsClient = None self.clients = {}
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) self.notifyHours = self.__notifyHours self.adminMail = self.__adminMail self.rsClient = None self.tokenDict = None self.diracAdmin = None
def __init__(self, *args): """ Initialize the base class and define some extra data members """ AgentModule.__init__(self, *args) self.requestDBClient = None self.replicaManager = None self.maxNumberOfThreads = 4 self.maxRequestsInQueue = 100 self.threadPool = None self.timeOutCounter = 0 self.pendingRequests = True
def __init__( self, *args, **kwargs ): ''' Constructor ''' AgentModule.__init__( self, *args, **kwargs ) self.name = "TarTheProdLogsAgent" self.log = gLogger self.basepath = "" self.baselogpath = "" self.transclient = None self.ops = None self.storageElement = None self.baselfn = ""
def __init__( self, agentName, loadName, baseAgentName, properties = {} ): ''' c'tor ''' AgentModule.__init__( self, agentName, loadName, baseAgentName, properties ) self.transClient = TransformationClient() agentTSTypes = self.am_getOption( 'TransformationTypes', [] ) if agentTSTypes: self.transformationTypes = sortList( agentTSTypes ) else: self.transformationTypes = sortList( Operations().getValue( 'Transformations/ExtendableTransfTypes', ['MCSimulation', 'Simulation'] ) ) self.maxIterationTasks = self.am_getOption( 'TasksPerIteration', 50 ) self.maxFailRate = self.am_getOption( 'MaxFailureRate', 30 ) self.maxWaitingJobs = self.am_getOption( 'MaxWaitingJobs', 1000 )
def __init__( self, *args, **kwargs ): """ c'tor :param self: self reference :param str agentName: name of agent :param str loadName: name of module :param bool baseAgentName: whatever :param dict properties: whatever else """ AgentModule.__init__( self, *args, **kwargs ) agentName = args[0] ## save config path self.__configPath = PathFinder.getAgentSection( agentName ) self.log.info( "Will use %s config path" % self.__configPath ) ## ProcessPool related stuff self.__requestsPerCycle = self.am_getOption( "RequestsPerCycle", 10 ) self.log.info("requests/cycle = %d" % self.__requestsPerCycle ) self.__minProcess = self.am_getOption( "MinProcess", 1 ) self.log.info("ProcessPool min process = %d" % self.__minProcess ) self.__maxProcess = self.am_getOption( "MaxProcess", 4 ) self.log.info("ProcessPool max process = %d" % self.__maxProcess ) self.__queueSize = self.am_getOption( "ProcessPoolQueueSize", 10 ) self.log.info("ProcessPool queue size = %d" % self.__queueSize ) self.__poolTimeout = int( self.am_getOption( "ProcessPoolTimeout", 300 ) ) self.log.info("ProcessPool timeout = %d seconds" % self.__poolTimeout ) self.__taskTimeout = int( self.am_getOption( "ProcessTaskTimeout", 300 ) ) self.log.info("ProcessTask timeout = %d seconds" % self.__taskTimeout ) ## request type self.__requestType = self.am_getOption( "RequestType", self.__requestType ) self.log.info( "Will process '%s' request type." % str( self.__requestType ) ) ## shifter proxy self.am_setOption( "shifterProxy", "DataManager" ) self.log.info( "Will use DataManager proxy by default." ) ## common monitor activity self.monitor.registerActivity( "Iteration", "Agent Loops", self.__class__.__name__, "Loops/min", gMonitor.OP_SUM ) self.monitor.registerActivity( "Execute", "Request Processed", self.__class__.__name__, "Requests/min", gMonitor.OP_SUM ) self.monitor.registerActivity( "Done", "Request Completed", self.__class__.__name__, "Requests/min", gMonitor.OP_SUM ) ## create request dict self.__requestHolder = dict()
def initialize( self ): self.RequestDBClient = RequestClient() backend = self.am_getOption( 'Backend', '' ) self.RequestDB = False if backend == 'mysql': from DIRAC.RequestManagementSystem.DB.RequestDBMySQL import RequestDBMySQL requestDB = RequestDBMySQL() if requestDB._connected: self.RequestDB = requestDB gMonitor.registerActivity( "Iteration", "Agent Loops", "DISETForwardingAgent", "Loops/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "Attempted", "Request Processed", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "Successful", "Request Forward Successful", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "Failed", "Request Forward Failed", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM ) self.local = PathFinder.getServiceURL( "RequestManagement/localURL" ) if not self.local: self.local = AgentModule.am_getOption( self, 'localURL', '' ) if not self.local: errStr = 'The RequestManagement/localURL option must be defined.' gLogger.fatal( errStr ) return S_ERROR( errStr ) return S_OK()
def __init__( self, *args, **kwargs ): """ c'tor """ AgentModule.__init__( self, *args, **kwargs ) #clients # FIXME: shouldn't we avoid using the DBs directly, and instead go through the service? self.jobDB = None self.taskQueueDB = None self.jobLoggingDB = None self.maxJobsAtOnce = 100 self.jobByJob = False self.throttlingPeriod = 0. self.removeStatusDelay = {}