def __init__(self, *args, **kwargs):
""" c'tor
"""
DiracTCAgent.__init__(self, *args, **kwargs)
self.directoryLocations = ['TransformationDB', 'StorageUsage']
self.archiveAfter = 7
self.fileTypesToKeep = ['GAUSSHIST']
self.bkClient = None
self.transClient = None
self.storageUsageClient = None
def initialize(self):
""" Standard initialize method for agents
"""
DiracTCAgent.initialize(self)
self.directoryLocations = sorted(
self.am_getOption('DirectoryLocations', self.directoryLocations))
self.archiveAfter = self.am_getOption('ArchiveAfter',
self.archiveAfter) # days
self.fileTypesToKeep = Operations().getValue(
'Transformations/FileTypesToKeep', self.fileTypesToKeep)
self.bkClient = BookkeepingClient()
self.transClient = TransformationClient()
self.storageUsageClient = StorageUsageClient()
return S_OK()
def getTransformationDirectories(self, transID):
""" get the directories for the supplied transformation from the transformation system
:param self: self reference
:param int transID: transformation ID
"""
res = DiracTCAgent.getTransformationDirectories(self, transID)
if not res['OK']:
return res
directories = res['Value']
if isinstance(directories, basestring): # Check for (stupid) formats
directories = ast.literal_eval(directories)
if not isinstance(directories, list):
return S_ERROR("Wrong format of output directories")
if 'StorageUsage' in self.directoryLocations:
res = self.storageUsageClient.getStorageDirectories(
'', '', transID, [])
if not res['OK']:
self.log.error("Failed to obtain storage usage directories",
res['Message'])
return res
transDirectories = res['Value']
directories = self._addDirs(transID, transDirectories, directories)
if not directories:
self.log.info("No output directories found")
# We should be removing from the list of directories
# those directories created for file types that are part of those:
# - uploaded (as output files)
# - not merged by subsequent steps
# but this is pretty difficult to identify at run time, so we better remove the "RemovingFiles" production status
# and replace it with a flush (this applies only to MC).
# So we just have a created list.
fileTypesToKeepDirs = []
for fileTypeToKeep in self.fileTypesToKeep:
fileTypesToKeepDirs.extend(
[x for x in directories if fileTypeToKeep in x])
directories = list(
set(directories).difference(set(fileTypesToKeepDirs)))
directories = sorted(directories)
return S_OK(directories)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(["transID: transformation ID"])
_, args = Script.parseCommandLine()
transIDs = [int(arg) for arg in args]
from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent
agent = TransformationCleaningAgent(
"Transformation/TransformationCleaningAgent",
"Transformation/TransformationCleaningAgent",
"dirac-transformation-remove-output",
)
agent.initialize()
for transID in transIDs:
agent.removeTransformationOutput(transID)
def main():
Script.parseCommandLine()
args = Script.getPositionalArgs()
if not args:
Script.showHelp()
transIDs = [int(arg) for arg in args]
from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
agent = TransformationCleaningAgent(
'Transformation/TransformationCleaningAgent',
'Transformation/TransformationCleaningAgent',
'dirac-transformation-remove-output')
agent.initialize()
client = TransformationClient()
for transID in transIDs:
agent.removeTransformationOutput(transID)
__RCSID__ = "$Id$"
import sys
if len(sys.argv) < 2:
print 'Usage: dirac-transformation-remove-output transID [transID] [transID]'
sys.exit()
else:
transIDs = [int(arg) for arg in sys.argv[1:]]
from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
from DIRAC import gLogger
import DIRAC
agent = TransformationCleaningAgent(
'Transformation/TransformationCleaningAgent',
'dirac-transformation-remove-output')
agent.initialize()
client = TransformationClient()
for transID in transIDs:
res = client.getTransformationParameters(transID, ['Status'])
if not res['OK']:
gLogger.error("Failed to determine transformation status")
gLogger.error(res['Message'])
continue
status = res['Value']
if not status in [
'RemovingFiles', 'RemovingOutput', 'ValidatingInput', 'Active'
]:
gLogger.error(
__RCSID__ = "$Id$"
import sys
if len( sys.argv ) < 2:
print 'Usage: dirac-transformation-clean transID [transID] [transID]'
sys.exit()
else:
transIDs = [int( arg ) for arg in sys.argv[1:]]
from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
from DIRAC import gLogger
import DIRAC
agent = TransformationCleaningAgent( 'Transformation/TransformationCleaningAgent', 'dirac-transformation-clean' )
agent.initialize()
client = TransformationClient()
for transID in transIDs:
res = client.getTransformationParameters( transID, ['Status'] )
if not res['OK']:
gLogger.error( "Failed to determine transformation status" )
gLogger.error( res['Message'] )
continue
status = res['Value']
if not status in ['Deleted', 'Cleaning', 'Archived', 'Completed']:
gLogger.error( "The transformation is in %s status and can not be cleaned" % status )
continue
agent.cleanTransformation( transID )
#!/usr/bin/env python
""" Archive a transformation
"""
from __future__ import print_function
import sys
from DIRAC.Core.Base.Script import parseCommandLine, getPositionalArgs
parseCommandLine()
if not getPositionalArgs():
print('Usage: dirac-transformation-archive transID [transID] [transID]')
sys.exit()
else:
transIDs = [int(arg) for arg in getPositionalArgs()]
from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
agent = TransformationCleaningAgent('Transformation/TransformationCleaningAgent',
'Transformation/TransformationCleaningAgent',
'dirac-transformation-archive')
agent.initialize()
client = TransformationClient()
for transID in transIDs:
agent.archiveTransformation(transID)
import sys
if len( sys.argv ) < 2:
print 'Usage: dirac-transformation-clean transID [transID] [transID]'
sys.exit()
else:
transIDs = [int( arg ) for arg in sys.argv[1:]]
from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
from DIRAC import gLogger
import DIRAC
agent = TransformationCleaningAgent( 'Transformation/TransformationCleaningAgent',
'Transformation/TransformationCleaningAgent',
'dirac-transformation-clean' )
agent.initialize()
client = TransformationClient()
for transID in transIDs:
res = client.getTransformationParameters( transID, ['Status'] )
if not res['OK']:
gLogger.error( "Failed to determine transformation status" )
gLogger.error( res['Message'] )
continue
status = res['Value']
if not status in ['Deleted', 'Cleaning', 'Archived', 'Completed']:
gLogger.error( "The transformation is in %s status and can not be cleaned" % status )
continue
agent.cleanTransformation( transID )
#!/usr/bin/env python
""" Remove the outputs produced by a transformation
"""
import sys
from DIRAC.Core.Base.Script import parseCommandLine
parseCommandLine()
if len( sys.argv ) < 2:
print 'Usage: dirac-transformation-remove-output transID [transID] [transID]'
sys.exit()
else:
transIDs = [int( arg ) for arg in sys.argv[1:]]
from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
agent = TransformationCleaningAgent( 'Transformation/TransformationCleaningAgent',
'Transformation/TransformationCleaningAgent',
'dirac-transformation-remove-output' )
agent.initialize()
client = TransformationClient()
for transID in transIDs:
agent.removeTransformationOutput( transID )
__RCSID__ = "$Id$"
import sys
if len( sys.argv ) < 2:
print 'Usage: dirac-transformation-archive transID [transID] [transID]'
sys.exit()
else:
transIDs = [int( arg ) for arg in sys.argv[1:]]
from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
from DIRAC import gLogger
import DIRAC
agent = TransformationCleaningAgent( 'Transformation/TransformationCleaningAgent', 'dirac-transformation-archive' )
agent.initialize()
client = TransformationClient()
for transID in transIDs:
res = client.getTransformationParameters( transID, ['Status'] )
if not res['OK']:
gLogger.error( "Failed to determine transformation status" )
gLogger.error( res['Message'] )
continue
status = res['Value']
if not status in ['Completed']:
gLogger.error( "The transformation is in %s status and can not be archived" % status )
continue
agent.archiveTransformation( transID )
