import pickle
from DIRAC import S_OK, S_ERROR
from DIRAC.Core.Base.AgentModule import AgentModule
from DIRAC.Core.Utilities.ThreadPool import ThreadPool
from DIRAC.Core.Utilities.ThreadSafe import Synchronizer
from DIRAC.Core.Utilities.List import breakListIntoChunks, randomize
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
from DIRAC.TransformationSystem.Agent.TransformationAgentsUtilities import TransformationAgentsUtilities
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
__RCSID__ = "$Id$"
AGENT_NAME = 'Transformation/TransformationAgent'
gSynchro = Synchronizer()
class TransformationAgent(AgentModule, TransformationAgentsUtilities):
""" Usually subclass of AgentModule
"""
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
TransformationAgentsUtilities.__init__(self)
# few parameters
self.pluginLocation = ''
self.transformationStatus = []
self.maxFiles = 0
__RCSID__ = "$Id$"
import types
import threading
from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR
from DIRAC.Core.Utilities.ThreadSafe import Synchronizer
from DIRAC.FrameworkSystem.Client.Logger import gLogger
gEventSync = Synchronizer()
class EventDispatcher(object):
def __init__(self):
self.__events = {}
self.__processingEvents = set()
@gEventSync
def registerEvent(self, eventName):
if eventName in self.__events:
return
self.__events[eventName] = []
@gEventSync
def addListener(self, eventName, functor):
if eventName not in self.__events:
return S_ERROR("Event %s is not registered" % eventName)
if functor in self.__events[eventName]:
return S_OK()
self.__events[eventName].append(functor)
return S_OK()
""" a scheduler of threads, of course!
"""
import hashlib
import threading
import time
from DIRAC import S_ERROR, S_OK, gLogger
from DIRAC.Core.Utilities.ThreadSafe import Synchronizer
gSchedulerLock = Synchronizer()
class ThreadScheduler(object):
def __init__(self, enableReactorThread=True, minPeriod=60):
self.__thId = False
self.__minPeriod = minPeriod
self.__taskDict = {}
self.__hood = []
self.__createReactorThread = enableReactorThread
self.__nowEpoch = time.time
self.__sleeper = time.sleep
self.__min = min
def setMinValidPeriod(self, period):
self.__minPeriod = period
def disableCreateReactorThread(self):
self.__createReactorThread = False
def addPeriodicTask(self, period, taskFunc, taskArgs=(), executions=0, elapsedTime=0):
if not callable(taskFunc):
__RCSID__ = "$Id$"
import types
import random
from hashlib import md5
from DIRAC.Core.Utilities.ThreadSafe import Synchronizer
from DIRAC.Core.DISET.private.BaseClient import BaseClient
from DIRAC.Core.DISET.private.MessageBroker import getGlobalMessageBroker
from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR, isReturnStructure
from DIRAC.Core.Utilities import Network, Time
from DIRAC.FrameworkSystem.Client.Logger import gLogger
gMsgSync = Synchronizer()
class MessageClient(BaseClient):
class MSGException(Exception):
pass
def _initialize(self):
self.__trid = False
self.__transport = None
self.__uniqueName = self.__generateUniqueClientName()
self.__msgBroker = getGlobalMessageBroker()
self.__callbacks = {}
self.__connectExtraParams = {}
self.__specialCallbacks = {'drop': [], 'msg': []}
def __generateUniqueClientName(self):
# $HeadURL$
__RCSID__ = "$Id$"
import time
import random
from DIRAC import S_OK, S_ERROR, gLogger, gConfig
from DIRAC.Core.DISET.RPCClient import RPCClient
from DIRAC.Core.Utilities.ThreadSafe import Synchronizer
from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer
from DIRAC.RequestManagementSystem.Client.RequestClient import RequestClient
gAccountingSynchro = Synchronizer()
random.seed()
class DataStoreClient:
"""
Class providing front end access to DIRAC Accounting DataStore Service
- It allows to reduce the interactions with the server by building and list of
pending Registers to be sent that are sent in a bundle using the commit method.
- In case the DataStore is down Registers are sent as DISET requests.
"""
def __init__( self, setup = False, retryGraceTime = 0 ):
self.__setup = setup
self.__maxRecordsInABundle = 1000
self.__registersList = []
self.__maxTimeRetrying = retryGraceTime
self.__lastSuccessfulCommit = time.time()
self.__failoverEnabled = not gConfig.getValue( '/LocalSite/DisableFailover', False )
def setRetryGraceTime( self, retryGraceTime ):
"""
import os, shutil, time
from DIRAC import gConfig, S_OK, S_ERROR, gLogger
from DIRAC.Core.Utilities.ThreadPool import ThreadPool
from DIRAC.Core.Utilities.ThreadSafe import Synchronizer
from DIRAC.Core.Utilities.Subprocess import pythonCall
from DIRAC.Core.Utilities import List
from DIRAC.DataManagementSystem.Client.ReplicaManager import ReplicaManager
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.Resources.Storage.StorageElement import StorageElement
transferSync = Synchronizer()
class OutputDataExecutor:
def __init__(self, csPath=""):
self.log = gLogger.getSubLogger("OutputDataExecutor")
if not csPath:
vo = gConfig.getValue("/DIRAC/VirtualOrganization", "")
self.__transfersCSPath = '/Operations/%s/OutputData' % vo
else:
self.__transfersCSPath = csPath
self.log.verbose("Reading transfer paths from %s" %
self.__transfersCSPath)
self.__requiredCSOptions = [
'InputPath', 'InputFC', 'OutputPath', 'OutputFC', 'OutputSE'
]
self.__threadPool = ThreadPool(
gConfig.getValue("%s/MinTransfers" % self.__transfersCSPath, 1),
gConfig.getValue("%s/MaxTransfers" % self.__transfersCSPath, 4),
gConfig.getValue("%s/MaxQueuedTransfers" % self.__transfersCSPath,
