def __init__(self, *args, **kwargs): """ c'tor Always call this in the extension agent """ AgentModule.__init__(self, *args, **kwargs) TransformationAgentsUtilities.__init__(self) self.transClient = None self.jobManagerClient = None self.transType = [] self.tasksPerLoop = 50 self.maxParametricJobs = 20 # will be updated in execute() # credentials self.shifterProxy = None self.credentials = None self.credTuple = (None, None, None) self.pluginLocation = '' self.bulkSubmissionFlag = False # for the threading self.transQueue = Queue() self.transInQueue = [] self.transInThread = {}
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) TransformationAgentsUtilities.__init__(self) self.bkQueriesToBeChecked = Queue.Queue() self.bkQueriesInCheck = [] self.fullUpdatePeriod = 86400 self.bkUpdateLatency = 7200 self.debug = False self.transInThread = {} self.pickleFile = 'BookkeepingWatchAgent.pkl' self.chunkSize = 1000 self.pluginsWithNoRunInfo = ['LHCbStandard', 'ReplicateDataset', 'ArchiveDataset', 'LHCbMCDSTBroadcastRandom', 'ReplicateToLocalSE', 'RemoveReplicas', 'RemoveReplicasWhenProcessed', 'RemoveReplicasWithAncestors', 'ReplicateWithAncestors', 'ReduceReplicas', 'RemoveDatasetFromDisk', 'DestroyDataset', 'DestroyDatasetWhenProcessed', 'BySize', 'Standard'] self.timeLog = {} self.fullTimeLog = {} self.bkQueries = {} self.transClient = None self.bkClient = None
def __init__(self, *args, **kwargs): """c'tor""" AgentModule.__init__(self, *args, **kwargs) TransformationAgentsUtilities.__init__(self) # few parameters self.pluginLocation = "" self.transformationStatus = [] self.maxFiles = 0 self.transformationTypes = [] # clients (out of the threads) self.transfClient = None # parameters for caching self.workDirectory = "" self.cacheFile = "" self.controlDirectory = "" self.lastFileOffset = {} # Validity of the cache self.replicaCache = None self.replicaCacheValidity = None self.writingCache = False self.removedFromCache = 0 self.noUnusedDelay = 0 self.unusedFiles = {} self.unusedTimeStamp = {} self.debug = False self.pluginTimeout = {}
def __init__(self, *args, **kwargs): """ c'tor Always call this in the extension agent """ AgentModule.__init__(self, *args, **kwargs) TransformationAgentsUtilities.__init__(self) self.transClient = None self.jobManagerClient = None self.transType = [] self.tasksPerLoop = 50 self.maxParametricJobs = 20 # will be updated in execute() # credentials self.shifterProxy = None self.credentials = None self.credTuple = (None, None, None) self.pluginLocation = '' self.bulkSubmissionFlag = False # for the threading self.transQueue = Queue() self.transInQueue = [] self.transInThread = {}
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) TransformationAgentsUtilities.__init__(self) #few parameters self.pluginLocation = self.am_getOption( 'PluginLocation', 'DIRAC.TransformationSystem.Agent.TransformationPlugin') self.transformationStatus = self.am_getOption( 'transformationStatus', ['Active', 'Completing', 'Flush']) self.maxFiles = self.am_getOption('MaxFiles', 5000) agentTSTypes = self.am_getOption('TransformationTypes', []) if agentTSTypes: self.transformationTypes = sortList(agentTSTypes) else: dataProc = Operations().getValue('Transformations/DataProcessing', ['MCSimulation', 'Merge']) dataManip = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal']) self.transformationTypes = sortList(dataProc + dataManip) #clients self.transfClient = TransformationClient() #for the threading self.transQueue = Queue.Queue() self.transInQueue = [] #for caching using a pickle file self.workDirectory = self.am_getWorkDirectory() self.cacheFile = os.path.join(self.workDirectory, 'ReplicaCache.pkl') self.dateWriteCache = datetime.datetime.utcnow() # Validity of the cache self.replicaCache = None self.replicaCacheValidity = self.am_getOption('ReplicaCacheValidity', 2) self.writingCache = False self.noUnusedDelay = self.am_getOption('NoUnusedDelay', 6) self.unusedFiles = {} self.unusedTimeStamp = {} self.debug = False self.transInThread = {}
def __init__( self, *args, **kwargs ): """ c'tor """ AgentModule.__init__( self, *args, **kwargs ) TransformationAgentsUtilities.__init__( self ) # few parameters self.pluginLocation = self.am_getOption( 'PluginLocation', 'DIRAC.TransformationSystem.Agent.TransformationPlugin' ) self.transformationStatus = self.am_getOption( 'transformationStatus', ['Active', 'Completing', 'Flush'] ) self.maxFiles = self.am_getOption( 'MaxFiles', 5000 ) agentTSTypes = self.am_getOption( 'TransformationTypes', [] ) if agentTSTypes: self.transformationTypes = sortList( agentTSTypes ) else: dataProc = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] ) dataManip = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'] ) self.transformationTypes = sortList( dataProc + dataManip ) # clients self.transfClient = TransformationClient() # for the threading self.transQueue = Queue.Queue() self.transInQueue = [] # for caching using a pickle file self.workDirectory = self.am_getWorkDirectory() self.cacheFile = os.path.join( self.workDirectory, 'ReplicaCache.pkl' ) self.controlDirectory = self.am_getControlDirectory() self.dateWriteCache = datetime.datetime.utcnow() # Validity of the cache self.replicaCache = None self.replicaCacheValidity = self.am_getOption( 'ReplicaCacheValidity', 2 ) self.writingCache = False self.noUnusedDelay = self.am_getOption( 'NoUnusedDelay', 6 ) self.unusedFiles = {} self.unusedTimeStamp = {} self.debug = False self.transInThread = {}
def __init__(self, *args, **kwargs): """ c'tor Always call this in the extension agent """ AgentModule.__init__(self, *args, **kwargs) TransformationAgentsUtilities.__init__(self) self.transClient = None self.transType = [] self.tasksPerLoop = 50 self.owner = '' self.ownerGroup = '' self.ownerDN = '' # for the threading self.transQueue = Queue() self.transInQueue = [] self.transInThread = {}
def __init__( self, *args, **kwargs ): """ c'tor Always call this in the extension agent """ AgentModule.__init__( self, *args, **kwargs ) TransformationAgentsUtilities.__init__( self ) self.transClient = None self.transType = [] self.tasksPerLoop = 50 self.owner = '' self.ownerGroup = '' self.ownerDN = '' # for the threading self.transQueue = Queue() self.transInQueue = [] self.transInThread = {}
def __init__( self, *args, **kwargs ): """ c'tor """ AgentModule.__init__( self, *args, **kwargs ) TransformationAgentsUtilities.__init__( self ) # few parameters self.pluginLocation = '' self.transformationStatus = [] self.maxFiles = 0 self.transformationTypes = [] # clients (out of the threads) self.transfClient = None # parameters for the threading self.transQueue = Queue.Queue() self.transInQueue = [] # parameters for caching self.workDirectory = '' self.cacheFile = '' self.controlDirectory = '' self.lastFileOffset = None # Validity of the cache self.replicaCache = None self.replicaCacheValidity = None self.writingCache = False self.removedFromCache = 0 self.noUnusedDelay = 0 self.unusedFiles = {} self.unusedTimeStamp = {} self.debug = False self.transInThread = {} self.pluginTimeout = {}