import pickle from DIRAC import S_OK, S_ERROR from DIRAC.Core.Base.AgentModule import AgentModule from DIRAC.Core.Utilities.ThreadPool import ThreadPool from DIRAC.Core.Utilities.ThreadSafe import Synchronizer from DIRAC.Core.Utilities.List import breakListIntoChunks, randomize from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient from DIRAC.TransformationSystem.Agent.TransformationAgentsUtilities import TransformationAgentsUtilities from DIRAC.DataManagementSystem.Client.DataManager import DataManager __RCSID__ = "$Id$" AGENT_NAME = 'Transformation/TransformationAgent' gSynchro = Synchronizer() class TransformationAgent(AgentModule, TransformationAgentsUtilities): """ Usually subclass of AgentModule """ def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) TransformationAgentsUtilities.__init__(self) # few parameters self.pluginLocation = '' self.transformationStatus = [] self.maxFiles = 0
__RCSID__ = "$Id$" import types import threading from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR from DIRAC.Core.Utilities.ThreadSafe import Synchronizer from DIRAC.FrameworkSystem.Client.Logger import gLogger gEventSync = Synchronizer() class EventDispatcher(object): def __init__(self): self.__events = {} self.__processingEvents = set() @gEventSync def registerEvent(self, eventName): if eventName in self.__events: return self.__events[eventName] = [] @gEventSync def addListener(self, eventName, functor): if eventName not in self.__events: return S_ERROR("Event %s is not registered" % eventName) if functor in self.__events[eventName]: return S_OK() self.__events[eventName].append(functor) return S_OK()
""" a scheduler of threads, of course! """ import hashlib import threading import time from DIRAC import S_ERROR, S_OK, gLogger from DIRAC.Core.Utilities.ThreadSafe import Synchronizer gSchedulerLock = Synchronizer() class ThreadScheduler(object): def __init__(self, enableReactorThread=True, minPeriod=60): self.__thId = False self.__minPeriod = minPeriod self.__taskDict = {} self.__hood = [] self.__createReactorThread = enableReactorThread self.__nowEpoch = time.time self.__sleeper = time.sleep self.__min = min def setMinValidPeriod(self, period): self.__minPeriod = period def disableCreateReactorThread(self): self.__createReactorThread = False def addPeriodicTask(self, period, taskFunc, taskArgs=(), executions=0, elapsedTime=0): if not callable(taskFunc):
__RCSID__ = "$Id$" import types import random from hashlib import md5 from DIRAC.Core.Utilities.ThreadSafe import Synchronizer from DIRAC.Core.DISET.private.BaseClient import BaseClient from DIRAC.Core.DISET.private.MessageBroker import getGlobalMessageBroker from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR, isReturnStructure from DIRAC.Core.Utilities import Network, Time from DIRAC.FrameworkSystem.Client.Logger import gLogger gMsgSync = Synchronizer() class MessageClient(BaseClient): class MSGException(Exception): pass def _initialize(self): self.__trid = False self.__transport = None self.__uniqueName = self.__generateUniqueClientName() self.__msgBroker = getGlobalMessageBroker() self.__callbacks = {} self.__connectExtraParams = {} self.__specialCallbacks = {'drop': [], 'msg': []} def __generateUniqueClientName(self):
# $HeadURL$ __RCSID__ = "$Id$" import time import random from DIRAC import S_OK, S_ERROR, gLogger, gConfig from DIRAC.Core.DISET.RPCClient import RPCClient from DIRAC.Core.Utilities.ThreadSafe import Synchronizer from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer from DIRAC.RequestManagementSystem.Client.RequestClient import RequestClient gAccountingSynchro = Synchronizer() random.seed() class DataStoreClient: """ Class providing front end access to DIRAC Accounting DataStore Service - It allows to reduce the interactions with the server by building and list of pending Registers to be sent that are sent in a bundle using the commit method. - In case the DataStore is down Registers are sent as DISET requests. """ def __init__( self, setup = False, retryGraceTime = 0 ): self.__setup = setup self.__maxRecordsInABundle = 1000 self.__registersList = [] self.__maxTimeRetrying = retryGraceTime self.__lastSuccessfulCommit = time.time() self.__failoverEnabled = not gConfig.getValue( '/LocalSite/DisableFailover', False ) def setRetryGraceTime( self, retryGraceTime ): """
import os, shutil, time from DIRAC import gConfig, S_OK, S_ERROR, gLogger from DIRAC.Core.Utilities.ThreadPool import ThreadPool from DIRAC.Core.Utilities.ThreadSafe import Synchronizer from DIRAC.Core.Utilities.Subprocess import pythonCall from DIRAC.Core.Utilities import List from DIRAC.DataManagementSystem.Client.ReplicaManager import ReplicaManager from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.Resources.Storage.StorageElement import StorageElement transferSync = Synchronizer() class OutputDataExecutor: def __init__(self, csPath=""): self.log = gLogger.getSubLogger("OutputDataExecutor") if not csPath: vo = gConfig.getValue("/DIRAC/VirtualOrganization", "") self.__transfersCSPath = '/Operations/%s/OutputData' % vo else: self.__transfersCSPath = csPath self.log.verbose("Reading transfer paths from %s" % self.__transfersCSPath) self.__requiredCSOptions = [ 'InputPath', 'InputFC', 'OutputPath', 'OutputFC', 'OutputSE' ] self.__threadPool = ThreadPool( gConfig.getValue("%s/MinTransfers" % self.__transfersCSPath, 1), gConfig.getValue("%s/MaxTransfers" % self.__transfersCSPath, 4), gConfig.getValue("%s/MaxQueuedTransfers" % self.__transfersCSPath,