def __init__(self, *args, **kwargs): """ c'tor """ DiracTCAgent.__init__(self, *args, **kwargs) self.directoryLocations = ['TransformationDB', 'StorageUsage'] self.archiveAfter = 7 self.fileTypesToKeep = ['GAUSSHIST'] self.bkClient = None self.transClient = None self.storageUsageClient = None
def initialize(self): """ Standard initialize method for agents """ DiracTCAgent.initialize(self) self.directoryLocations = sorted( self.am_getOption('DirectoryLocations', self.directoryLocations)) self.archiveAfter = self.am_getOption('ArchiveAfter', self.archiveAfter) # days self.fileTypesToKeep = Operations().getValue( 'Transformations/FileTypesToKeep', self.fileTypesToKeep) self.bkClient = BookkeepingClient() self.transClient = TransformationClient() self.storageUsageClient = StorageUsageClient() return S_OK()
def getTransformationDirectories(self, transID): """ get the directories for the supplied transformation from the transformation system :param self: self reference :param int transID: transformation ID """ res = DiracTCAgent.getTransformationDirectories(self, transID) if not res['OK']: return res directories = res['Value'] if isinstance(directories, basestring): # Check for (stupid) formats directories = ast.literal_eval(directories) if not isinstance(directories, list): return S_ERROR("Wrong format of output directories") if 'StorageUsage' in self.directoryLocations: res = self.storageUsageClient.getStorageDirectories( '', '', transID, []) if not res['OK']: self.log.error("Failed to obtain storage usage directories", res['Message']) return res transDirectories = res['Value'] directories = self._addDirs(transID, transDirectories, directories) if not directories: self.log.info("No output directories found") # We should be removing from the list of directories # those directories created for file types that are part of those: # - uploaded (as output files) # - not merged by subsequent steps # but this is pretty difficult to identify at run time, so we better remove the "RemovingFiles" production status # and replace it with a flush (this applies only to MC). # So we just have a created list. fileTypesToKeepDirs = [] for fileTypeToKeep in self.fileTypesToKeep: fileTypesToKeepDirs.extend( [x for x in directories if fileTypeToKeep in x]) directories = list( set(directories).difference(set(fileTypesToKeepDirs))) directories = sorted(directories) return S_OK(directories)
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(["transID: transformation ID"]) _, args = Script.parseCommandLine() transIDs = [int(arg) for arg in args] from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent agent = TransformationCleaningAgent( "Transformation/TransformationCleaningAgent", "Transformation/TransformationCleaningAgent", "dirac-transformation-remove-output", ) agent.initialize() for transID in transIDs: agent.removeTransformationOutput(transID)
def main(): Script.parseCommandLine() args = Script.getPositionalArgs() if not args: Script.showHelp() transIDs = [int(arg) for arg in args] from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient agent = TransformationCleaningAgent( 'Transformation/TransformationCleaningAgent', 'Transformation/TransformationCleaningAgent', 'dirac-transformation-remove-output') agent.initialize() client = TransformationClient() for transID in transIDs: agent.removeTransformationOutput(transID)
__RCSID__ = "$Id$" import sys if len(sys.argv) < 2: print 'Usage: dirac-transformation-remove-output transID [transID] [transID]' sys.exit() else: transIDs = [int(arg) for arg in sys.argv[1:]] from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient from DIRAC import gLogger import DIRAC agent = TransformationCleaningAgent( 'Transformation/TransformationCleaningAgent', 'dirac-transformation-remove-output') agent.initialize() client = TransformationClient() for transID in transIDs: res = client.getTransformationParameters(transID, ['Status']) if not res['OK']: gLogger.error("Failed to determine transformation status") gLogger.error(res['Message']) continue status = res['Value'] if not status in [ 'RemovingFiles', 'RemovingOutput', 'ValidatingInput', 'Active' ]: gLogger.error(
__RCSID__ = "$Id$" import sys if len( sys.argv ) < 2: print 'Usage: dirac-transformation-clean transID [transID] [transID]' sys.exit() else: transIDs = [int( arg ) for arg in sys.argv[1:]] from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient from DIRAC import gLogger import DIRAC agent = TransformationCleaningAgent( 'Transformation/TransformationCleaningAgent', 'dirac-transformation-clean' ) agent.initialize() client = TransformationClient() for transID in transIDs: res = client.getTransformationParameters( transID, ['Status'] ) if not res['OK']: gLogger.error( "Failed to determine transformation status" ) gLogger.error( res['Message'] ) continue status = res['Value'] if not status in ['Deleted', 'Cleaning', 'Archived', 'Completed']: gLogger.error( "The transformation is in %s status and can not be cleaned" % status ) continue agent.cleanTransformation( transID )
#!/usr/bin/env python """ Archive a transformation """ from __future__ import print_function import sys from DIRAC.Core.Base.Script import parseCommandLine, getPositionalArgs parseCommandLine() if not getPositionalArgs(): print('Usage: dirac-transformation-archive transID [transID] [transID]') sys.exit() else: transIDs = [int(arg) for arg in getPositionalArgs()] from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient agent = TransformationCleaningAgent('Transformation/TransformationCleaningAgent', 'Transformation/TransformationCleaningAgent', 'dirac-transformation-archive') agent.initialize() client = TransformationClient() for transID in transIDs: agent.archiveTransformation(transID)
import sys if len( sys.argv ) < 2: print 'Usage: dirac-transformation-clean transID [transID] [transID]' sys.exit() else: transIDs = [int( arg ) for arg in sys.argv[1:]] from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient from DIRAC import gLogger import DIRAC agent = TransformationCleaningAgent( 'Transformation/TransformationCleaningAgent', 'Transformation/TransformationCleaningAgent', 'dirac-transformation-clean' ) agent.initialize() client = TransformationClient() for transID in transIDs: res = client.getTransformationParameters( transID, ['Status'] ) if not res['OK']: gLogger.error( "Failed to determine transformation status" ) gLogger.error( res['Message'] ) continue status = res['Value'] if not status in ['Deleted', 'Cleaning', 'Archived', 'Completed']: gLogger.error( "The transformation is in %s status and can not be cleaned" % status ) continue agent.cleanTransformation( transID )
#!/usr/bin/env python """ Remove the outputs produced by a transformation """ import sys from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() if len( sys.argv ) < 2: print 'Usage: dirac-transformation-remove-output transID [transID] [transID]' sys.exit() else: transIDs = [int( arg ) for arg in sys.argv[1:]] from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient agent = TransformationCleaningAgent( 'Transformation/TransformationCleaningAgent', 'Transformation/TransformationCleaningAgent', 'dirac-transformation-remove-output' ) agent.initialize() client = TransformationClient() for transID in transIDs: agent.removeTransformationOutput( transID )
__RCSID__ = "$Id$" import sys if len( sys.argv ) < 2: print 'Usage: dirac-transformation-archive transID [transID] [transID]' sys.exit() else: transIDs = [int( arg ) for arg in sys.argv[1:]] from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient from DIRAC import gLogger import DIRAC agent = TransformationCleaningAgent( 'Transformation/TransformationCleaningAgent', 'dirac-transformation-archive' ) agent.initialize() client = TransformationClient() for transID in transIDs: res = client.getTransformationParameters( transID, ['Status'] ) if not res['OK']: gLogger.error( "Failed to determine transformation status" ) gLogger.error( res['Message'] ) continue status = res['Value'] if not status in ['Completed']: gLogger.error( "The transformation is in %s status and can not be archived" % status ) continue agent.archiveTransformation( transID )