class Transformation(API):
#############################################################################
def __init__(self, transID=0, transClient=None):
""" c'tor
"""
super(Transformation, self).__init__()
self.paramTypes = {
'TransformationID': [types.IntType, types.LongType],
'TransformationName': types.StringTypes,
'Status': types.StringTypes,
'Description': types.StringTypes,
'LongDescription': types.StringTypes,
'Type': types.StringTypes,
'Plugin': types.StringTypes,
'AgentType': types.StringTypes,
'FileMask': types.StringTypes,
'TransformationGroup': types.StringTypes,
'GroupSize': [types.IntType, types.LongType, types.FloatType],
'InheritedFrom': [types.IntType, types.LongType],
'Body': types.StringTypes,
'MaxNumberOfTasks': [types.IntType, types.LongType],
'EventsPerTask': [types.IntType, types.LongType]
}
self.paramValues = {
'TransformationID': 0,
'TransformationName': '',
'Status': 'New',
'Description': '',
'LongDescription': '',
'Type': '',
'Plugin': 'Standard',
'AgentType': 'Manual',
'FileMask': '',
'TransformationGroup': 'General',
'GroupSize': 1,
'InheritedFrom': 0,
'Body': '',
'MaxNumberOfTasks': 0,
'EventsPerTask': 0
}
self.ops = Operations()
self.supportedPlugins = self.ops.getValue(
'Transformations/AllowedPlugins',
['Broadcast', 'Standard', 'BySize', 'ByShare'])
if not transClient:
self.transClient = TransformationClient()
else:
self.transClient = transClient
self.serverURL = self.transClient.getServer()
self.exists = False
if transID:
self.paramValues['TransformationID'] = transID
res = self.getTransformation()
if res['OK']:
self.exists = True
elif res['Message'] == 'Transformation does not exist':
raise AttributeError('TransformationID %d does not exist' %
transID)
else:
self.paramValues['TransformationID'] = 0
gLogger.fatal(
"Failed to get transformation from database",
"%s @ %s" % (transID, self.transClient.serverURL))
def setServer(self, server):
self.serverURL = server
self.transClient.setServer(self.serverURL)
def getServer(self):
return self.serverURL
def reset(self, transID=0):
self.__init__(transID)
self.transClient.setServer(self.serverURL)
return S_OK()
def setTargetSE(self, seList):
return self.__setSE('TargetSE', seList)
def setSourceSE(self, seList):
return self.__setSE('SourceSE', seList)
def setBody(self, body):
""" check that the body is a string, or using the proper syntax for multiple operations
:param body: transformation body, for example
.. code :: python
body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }),
( "RemoveReplica", { "TargetSE":"FOO-SRM" } ),
]
:type body: string or list of tuples (or lists) of string and dictionaries
:raises TypeError: If the structure is not as expected
:raises ValueError: If unknown attribute for the :class:`~DIRAC.RequestManagementSystem.Client.Operation.Operation` is used
:returns: S_OK, S_ERROR
"""
self.item_called = "Body"
if isinstance(body, basestring):
return self.__setParam(body)
if not isinstance(body, (list, tuple)):
raise TypeError("Expected list or string, but %r is %s" %
(body, type(body)))
for tup in body:
if not isinstance(tup, (tuple, list)):
raise TypeError("Expected tuple or list, but %r is %s" %
(tup, type(tup)))
if len(tup) != 2:
raise TypeError("Expected 2-tuple, but %r is length %d" %
(tup, len(tup)))
if not isinstance(tup[0], basestring):
raise TypeError(
"Expected string, but first entry in tuple %r is %s" %
(tup, type(tup[0])))
if not isinstance(tup[1], dict):
raise TypeError(
"Expected dictionary, but second entry in tuple %r is %s" %
(tup, type(tup[0])))
for par, val in tup[1].iteritems():
if not isinstance(par, basestring):
raise TypeError(
"Expected string, but key in dictionary %r is %s" %
(par, type(par)))
if not par in Operation.ATTRIBUTE_NAMES:
raise ValueError("Unknown attribute for Operation: %s" %
par)
if not isinstance(
val,
(basestring, int, long, float, list, tuple, dict)):
raise TypeError("Cannot encode %r, in json" % (val))
return self.__setParam(json.dumps(body))
def __setSE(self, seParam, seList):
if isinstance(seList, basestring):
try:
seList = eval(seList)
except BaseException:
seList = seList.split(',')
elif isinstance(seList, (list, dict, tuple)):
seList = list(seList)
else:
return S_ERROR("Bad argument type")
res = self.__checkSEs(seList)
if not res['OK']:
return res
self.item_called = seParam
return self.__setParam(seList)
def __getattr__(self, name):
if name.find('get') == 0:
item = name[3:]
self.item_called = item
return self.__getParam
if name.find('set') == 0:
item = name[3:]
self.item_called = item
return self.__setParam
raise AttributeError(name)
def __getParam(self):
if self.item_called == 'Available':
return S_OK(self.paramTypes.keys())
if self.item_called == 'Parameters':
return S_OK(self.paramValues)
if self.item_called in self.paramValues:
return S_OK(self.paramValues[self.item_called])
raise AttributeError("Unknown parameter for transformation: %s" %
self.item_called)
def __setParam(self, value):
change = False
if self.item_called in self.paramTypes:
if self.paramValues[self.item_called] != value:
if type(value) in self.paramTypes[self.item_called]:
change = True
else:
raise TypeError("%s %s %s expected one of %s" %
(self.item_called, value, type(value),
self.paramTypes[self.item_called]))
else:
if self.item_called not in self.paramValues:
change = True
else:
if self.paramValues[self.item_called] != value:
change = True
if not change:
gLogger.verbose("No change of parameter %s required" %
self.item_called)
else:
gLogger.verbose("Parameter %s to be changed" % self.item_called)
transID = self.paramValues['TransformationID']
if self.exists and transID:
res = self.transClient.setTransformationParameter(
transID, self.item_called, value)
if not res['OK']:
return res
self.paramValues[self.item_called] = value
return S_OK()
def getTransformation(self, printOutput=False):
transID = self.paramValues['TransformationID']
if not transID:
gLogger.fatal("No TransformationID known")
return S_ERROR()
res = self.transClient.getTransformation(transID, extraParams=True)
if not res['OK']:
if printOutput:
self._prettyPrint(res)
return res
transParams = res['Value']
for paramName, paramValue in transParams.items():
setter = None
setterName = "set%s" % paramName
if hasattr(self, setterName) and callable(getattr(
self, setterName)):
setter = getattr(self, setterName)
if not setterName:
gLogger.error(
"Unable to invoke setter %s, it isn't a member function" %
setterName)
continue
setter(paramValue)
if printOutput:
gLogger.info("No printing available yet")
return S_OK(transParams)
def getTransformationLogging(self, printOutput=False):
transID = self.paramValues['TransformationID']
if not transID:
gLogger.fatal("No TransformationID known")
return S_ERROR()
res = self.transClient.getTransformationLogging(transID)
if not res['OK']:
if printOutput:
self._prettyPrint(res)
return res
loggingList = res['Value']
if printOutput:
self._printFormattedDictList(
loggingList, ['Message', 'MessageDate', 'AuthorDN'],
'MessageDate', 'MessageDate')
return S_OK(loggingList)
def extendTransformation(self, nTasks, printOutput=False):
return self.__executeOperation('extendTransformation',
nTasks,
printOutput=printOutput)
def cleanTransformation(self, printOutput=False):
res = self.__executeOperation('cleanTransformation',
printOutput=printOutput)
if res['OK']:
self.paramValues['Status'] = 'Cleaned'
return res
def deleteTransformation(self, printOutput=False):
res = self.__executeOperation('deleteTransformation',
printOutput=printOutput)
if res['OK']:
self.reset()
return res
def addFilesToTransformation(self, lfns, printOutput=False):
return self.__executeOperation('addFilesToTransformation',
lfns,
printOutput=printOutput)
def setFileStatusForTransformation(self, status, lfns, printOutput=False):
return self.__executeOperation('setFileStatusForTransformation',
status,
lfns,
printOutput=printOutput)
def getTransformationTaskStats(self, printOutput=False):
return self.__executeOperation('getTransformationTaskStats',
printOutput=printOutput)
def getTransformationStats(self, printOutput=False):
return self.__executeOperation('getTransformationStats',
printOutput=printOutput)
def deleteTasks(self, taskMin, taskMax, printOutput=False):
return self.__executeOperation('deleteTasks',
taskMin,
taskMax,
printOutput=printOutput)
def addTaskForTransformation(self,
lfns=[],
se='Unknown',
printOutput=False):
return self.__executeOperation('addTaskForTransformation',
lfns,
se,
printOutput=printOutput)
def setTaskStatus(self, taskID, status, printOutput=False):
return self.__executeOperation('setTaskStatus',
taskID,
status,
printOutput=printOutput)
def __executeOperation(self, operation, *parms, **kwds):
transID = self.paramValues['TransformationID']
if not transID:
gLogger.fatal("No TransformationID known")
return S_ERROR()
printOutput = kwds.pop('printOutput')
fcn = None
if hasattr(self.transClient, operation) and callable(
getattr(self.transClient, operation)):
fcn = getattr(self.transClient, operation)
if not fcn:
return S_ERROR(
"Unable to invoke %s, it isn't a member funtion of TransformationClient"
)
res = fcn(transID, *parms, **kwds)
if printOutput:
self._prettyPrint(res)
return res
def getTransformationFiles(self,
fileStatus=[],
lfns=[],
outputFields=[
'FileID', 'LFN', 'Status', 'TaskID',
'TargetSE', 'UsedSE', 'ErrorCount',
'InsertedTime', 'LastUpdate'
],
orderBy='FileID',
printOutput=False):
condDict = {'TransformationID': self.paramValues['TransformationID']}
if fileStatus:
condDict['Status'] = fileStatus
if lfns:
condDict['LFN'] = lfns
res = self.transClient.getTransformationFiles(condDict=condDict)
if not res['OK']:
if printOutput:
self._prettyPrint(res)
return res
if printOutput:
if not outputFields:
gLogger.info("Available fields are: %s" %
res['ParameterNames'].join(' '))
elif not res['Value']:
gLogger.info("No tasks found for selection")
else:
self._printFormattedDictList(res['Value'], outputFields,
'FileID', orderBy)
return res
def getTransformationTasks(self,
taskStatus=[],
taskIDs=[],
outputFields=[
'TransformationID', 'TaskID',
'ExternalStatus', 'ExternalID', 'TargetSE',
'CreationTime', 'LastUpdateTime'
],
orderBy='TaskID',
printOutput=False):
condDict = {'TransformationID': self.paramValues['TransformationID']}
if taskStatus:
condDict['ExternalStatus'] = taskStatus
if taskIDs:
condDict['TaskID'] = taskIDs
res = self.transClient.getTransformationTasks(condDict=condDict)
if not res['OK']:
if printOutput:
self._prettyPrint(res)
return res
if printOutput:
if not outputFields:
gLogger.info("Available fields are: %s" %
res['ParameterNames'].join(' '))
elif not res['Value']:
gLogger.info("No tasks found for selection")
else:
self._printFormattedDictList(res['Value'], outputFields,
'TaskID', orderBy)
return res
#############################################################################
def getTransformations(self,
transID=[],
transStatus=[],
outputFields=[
'TransformationID', 'Status', 'AgentType',
'TransformationName', 'CreationDate'
],
orderBy='TransformationID',
printOutput=False):
condDict = {}
if transID:
condDict['TransformationID'] = transID
if transStatus:
condDict['Status'] = transStatus
res = self.transClient.getTransformations(condDict=condDict)
if not res['OK']:
if printOutput:
self._prettyPrint(res)
return res
if printOutput:
if not outputFields:
gLogger.info("Available fields are: %s" %
res['ParameterNames'].join(' '))
elif not res['Value']:
gLogger.info("No tasks found for selection")
else:
self._printFormattedDictList(res['Value'], outputFields,
'TransformationID', orderBy)
return res
#############################################################################
def getAuthorDNfromProxy(self):
""" gets the AuthorDN and username of the transformation from the uploaded proxy
"""
username = ""
author = ""
res = getProxyInfo()
if res['OK']:
author = res['Value']['identity']
username = res['Value']['username']
else:
gLogger.error("Unable to get uploaded proxy Info %s " %
res['Message'])
return S_ERROR(res['Message'])
res = {'username': username, 'authorDN': author}
return S_OK(res)
#############################################################################
def getTransformationsByUser(self,
authorDN="",
userName="",
transID=[],
transStatus=[],
outputFields=[
'TransformationID', 'Status', 'AgentType',
'TransformationName', 'CreationDate',
'AuthorDN'
],
orderBy='TransformationID',
printOutput=False):
condDict = {}
if authorDN == "":
res = self.getAuthorDNfromProxy()
if not res['OK']:
gLogger.error(res['Message'])
return S_ERROR(res['Message'])
else:
foundUserName = res['Value']['username']
foundAuthor = res['Value']['authorDN']
# If the username whom created the uploaded proxy is different than the provided username report error and exit
if not (userName == "" or userName == foundUserName):
gLogger.error(
"Couldn't resolve the authorDN for user '%s' from the uploaded proxy (proxy created by '%s')"
% (userName, foundUserName))
return S_ERROR(
"Couldn't resolve the authorDN for user '%s' from the uploaded proxy (proxy created by '%s')"
% (userName, foundUserName))
userName = foundUserName
authorDN = foundAuthor
gLogger.info(
"Will list transformations created by user '%s' with status '%s'"
% (userName, ', '.join(transStatus)))
else:
gLogger.info(
"Will list transformations created by '%s' with status '%s'" %
(authorDN, ', '.join(transStatus)))
condDict['AuthorDN'] = authorDN
if transID:
condDict['TransformationID'] = transID
if transStatus:
condDict['Status'] = transStatus
res = self.transClient.getTransformations(condDict=condDict)
if not res['OK']:
if printOutput:
self._prettyPrint(res)
return res
if printOutput:
if not outputFields:
gLogger.info("Available fields are: %s" %
res['ParameterNames'].join(' '))
elif not res['Value']:
gLogger.info("No tasks found for selection")
else:
self._printFormattedDictList(res['Value'], outputFields,
'TransformationID', orderBy)
return res
#############################################################################
def getSummaryTransformations(self, transID=[]):
"""Show the summary for a list of Transformations
Fields starting with 'F' ('J') refers to files (jobs).
Proc. stand for processed.
"""
condDict = {'TransformationID': transID}
orderby = []
start = 0
maxitems = len(transID)
paramShowNames = ['TransformationID', 'Type', 'Status', 'Files_Total', 'Files_PercentProcessed', \
'Files_Processed', 'Files_Unused', 'Jobs_TotalCreated', 'Jobs_Waiting', \
'Jobs_Running', 'Jobs_Done', 'Jobs_Failed', 'Jobs_Stalled']
# Below, the header used for each field in the printing: short to fit in one line
paramShowNamesShort = ['TransID', 'Type', 'Status', 'F_Total', 'F_Proc.(%)', 'F_Proc.', \
'F_Unused', 'J_Created', 'J_Wait', 'J_Run', 'J_Done', 'J_Fail', 'J_Stalled']
dictList = []
result = self.transClient.getTransformationSummaryWeb(
condDict, orderby, start, maxitems)
if not result['OK']:
self._prettyPrint(result)
return result
if result['Value']['TotalRecords'] > 0:
try:
paramNames = result['Value']['ParameterNames']
for paramValues in result['Value']['Records']:
paramShowValues = map(
lambda pname: paramValues[paramNames.index(pname)],
paramShowNames)
showDict = dict(zip(paramShowNamesShort, paramShowValues))
dictList.append(showDict)
except Exception as x:
print 'Exception %s ' % str(x)
if not len(dictList) > 0:
gLogger.error(
'No found transformations satisfying input condition')
return S_ERROR(
'No found transformations satisfying input condition')
else:
print self._printFormattedDictList(dictList, paramShowNamesShort,
paramShowNamesShort[0],
paramShowNamesShort[0])
return S_OK(dictList)
#############################################################################
def addTransformation(self, addFiles=True, printOutput=False):
res = self._checkCreation()
if not res['OK']:
return self._errorReport(res, 'Failed transformation sanity check')
if printOutput:
gLogger.info(
"Will attempt to create transformation with the following parameters"
)
self._prettyPrint(self.paramValues)
res = self.transClient.addTransformation(
self.paramValues['TransformationName'],
self.paramValues['Description'],
self.paramValues['LongDescription'],
self.paramValues['Type'],
self.paramValues['Plugin'],
self.paramValues['AgentType'],
self.paramValues['FileMask'],
transformationGroup=self.paramValues['TransformationGroup'],
groupSize=self.paramValues['GroupSize'],
inheritedFrom=self.paramValues['InheritedFrom'],
body=self.paramValues['Body'],
maxTasks=self.paramValues['MaxNumberOfTasks'],
eventsPerTask=self.paramValues['EventsPerTask'],
addFiles=addFiles)
if not res['OK']:
if printOutput:
self._prettyPrint(res)
return res
transID = res['Value']
self.exists = True
self.setTransformationID(transID)
gLogger.notice("Created transformation %d" % transID)
for paramName, paramValue in self.paramValues.items():
if paramName not in self.paramTypes:
res = self.transClient.setTransformationParameter(
transID, paramName, paramValue)
if not res['OK']:
gLogger.error("Failed to add parameter",
"%s %s" % (paramName, res['Message']))
gLogger.notice(
"To add this parameter later please execute the following."
)
gLogger.notice("oTransformation = Transformation(%d)" %
transID)
gLogger.notice("oTransformation.set%s(...)" % paramName)
return S_OK(transID)
def _checkCreation(self):
""" Few checks
"""
if self.paramValues['TransformationID']:
gLogger.info(
"You are currently working with an active transformation definition."
)
gLogger.info(
"If you wish to create a new transformation reset the TransformationID."
)
gLogger.info("oTransformation.reset()")
return S_ERROR()
requiredParameters = [
'TransformationName', 'Description', 'LongDescription', 'Type'
]
for parameter in requiredParameters:
if not self.paramValues[parameter]:
gLogger.info(
"%s is not defined for this transformation. This is required..."
% parameter)
self.paramValues[parameter] = raw_input(
"Please enter the value of " + parameter + " ")
plugin = self.paramValues['Plugin']
if plugin:
if not plugin in self.supportedPlugins:
gLogger.info(
"The selected Plugin (%s) is not known to the transformation agent."
% plugin)
res = self.__promptForParameter('Plugin',
choices=self.supportedPlugins,
default='Standard')
if not res['OK']:
return res
self.paramValues['Plugin'] = res['Value']
plugin = self.paramValues['Plugin']
return S_OK()
def _checkBySizePlugin(self):
return self._checkStandardPlugin()
def _checkBySharePlugin(self):
return self._checkStandardPlugin()
def _checkStandardPlugin(self):
groupSize = self.paramValues['GroupSize']
if groupSize <= 0:
gLogger.info(
"The GroupSize was found to be less than zero. It has been set to 1."
)
res = self.setGroupSize(1)
if not res['OK']:
return res
return S_OK()
def _checkBroadcastPlugin(self):
gLogger.info(
"The Broadcast plugin requires the following parameters be set: %s"
% (', '.join(['SourceSE', 'TargetSE'])))
requiredParams = ['SourceSE', 'TargetSE']
for requiredParam in requiredParams:
if not self.paramValues.get(requiredParam):
paramValue = raw_input("Please enter " + requiredParam + " ")
setter = None
setterName = "set%s" % requiredParam
if hasattr(self, setterName) and callable(
getattr(self, setterName)):
setter = getattr(self, setterName)
if not setter:
return S_ERROR(
"Unable to invoke %s, this function hasn't been implemented."
% setterName)
ses = paramValue.replace(',', ' ').split()
res = setter(ses)
if not res['OK']:
return res
return S_OK()
def __checkSEs(self, seList):
res = gConfig.getSections('/Resources/StorageElements')
if not res['OK']:
return self._errorReport(res,
'Failed to get possible StorageElements')
missing = set(seList) - set(res['Value'])
if missing:
for se in missing:
gLogger.error("StorageElement %s is not known" % se)
return S_ERROR("%d StorageElements not known" % len(missing))
return S_OK()
def __promptForParameter(self,
parameter,
choices=[],
default='',
insert=True):
res = promptUser("Please enter %s" % parameter,
choices=choices,
default=default)
if not res['OK']:
return self._errorReport(res)
gLogger.notice("%s will be set to '%s'" % (parameter, res['Value']))
paramValue = res['Value']
if insert:
setter = None
setterName = "set%s" % parameter
if hasattr(self, setterName) and callable(getattr(
self, setterName)):
setter = getattr(self, setterName)
if not setter:
return S_ERROR(
"Unable to invoke %s, it isn't a member function of Transformation!"
)
res = setter(paramValue)
if not res['OK']:
return res
return S_OK(paramValue)
class TaskManagerAgentBase(AgentModule):
#############################################################################
def initialize(self):
self.section = self.am_getOption("section")
gMonitor.registerActivity("SubmittedTasks","Automatically submitted tasks","Transformation Monitoring","Tasks", gMonitor.OP_ACUM)
self.transClient = TransformationClient()
return S_OK()
#############################################################################
def execute(self):
""" The TaskManagerBase execution method. """
# Determine whether the task status is to be monitored and updated
enableTaskMonitor = self.am_getOption('MonitorTasks','')
if not enableTaskMonitor:
gLogger.info("execute: Monitoring of tasks is disabled.")
gLogger.info("execute: To enable create the 'MonitorTasks' option")
else:
res = self.updateTaskStatus()
if not res['OK']:
gLogger.warn('execute: Failed to update task states', res['Message'])
# Determine whether the task files status is to be monitored and updated
enableFileMonitor = self.am_getOption('MonitorFiles','')
if not enableFileMonitor:
gLogger.info("execute: Monitoring of files is disabled.")
gLogger.info("execute: To enable create the 'MonitorFiles' option")
else:
res = self.updateFileStatus()
if not res['OK']:
gLogger.warn('execute: Failed to update file states', res['Message'])
# Determine whether the checking of reserved tasks is to be performed
enableCheckReserved = self.am_getOption('CheckReserved','')
if not enableCheckReserved:
gLogger.info("execute: Checking of reserved tasks is disabled.")
gLogger.info("execute: To enable create the 'CheckReserved' option")
else:
res = self.checkReservedTasks()
if not res['OK']:
gLogger.warn('execute: Failed to checked reserved tasks',res['Message'])
# Determine whether the submission of tasks is to be executed
enableSubmission = self.am_getOption('SubmitTasks','')
if not enableSubmission:
gLogger.info("execute: Submission of tasks is disabled.")
gLogger.info("execute: To enable create the 'SubmitTasks' option")
else:
res = self.submitTasks()
if not res['OK']:
gLogger.warn('execute: Failed to submit created tasks', res['Message'])
return S_OK()
def _selectTransformations(self,transType=[],status=['Active','Completing'],agentType=['Automatic']):
selectCond = {}
if status:
selectCond['Status'] = status
if transType:
selectCond['Type'] = transType
if agentType:
selectCond['AgentType'] = agentType
res = self.transClient.getTransformations(condDict=selectCond)
if not res['OK']:
gLogger.error("_selectTransformations: Failed to get transformations for selection.",res['Message'])
elif not res['Value']:
gLogger.info("_selectTransformations: No transformations found for selection.")
else:
gLogger.info("_selectTransformations: Obtained %d transformations for selection" % len(res['Value']))
return res
def updateTaskStatus(self):
gLogger.info("updateTaskStatus: Updating the Status of tasks")
# Get the transformations to be updated
status = self.am_getOption('UpdateTasksStatus',['Active','Completing','Stopped'])
res = self._selectTransformations(transType=self.transType,status=status,agentType=[])
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
# Get the tasks which are in a UPDATE state
updateStatus = self.am_getOption('TaskUpdateStatus',['Checking','Deleted','Killed','Staging','Stalled','Matched','Rescheduled','Completed','Submitted','Received','Waiting','Running'])
condDict = {"TransformationID":transID,"ExternalStatus":updateStatus}
timeStamp = str(datetime.datetime.utcnow() - datetime.timedelta(minutes=10))
res = self.transClient.getTransformationTasks(condDict=condDict,older=timeStamp, timeStamp='LastUpdateTime')
if not res['OK']:
gLogger.error("updateTaskStatus: Failed to get tasks to update for transformation", "%s %s" % (transID,res['Message']))
continue
if not res['Value']:
gLogger.verbose("updateTaskStatus: No tasks found to update for transformation %s" % transID)
continue
res = self.getSubmittedTaskStatus(res['Value'])
if not res['OK']:
gLogger.error("updateTaskStatus: Failed to get updated task statuses for transformation", "%s %s" % (transID,res['Message']))
continue
statusDict = res['Value']
for status in sortList(statusDict.keys()):
taskIDs = statusDict[status]
gLogger.info("updateTaskStatus: Updating %d task(s) from transformation %d to %s" % (len(taskIDs),transID,status))
res = self.transClient.setTaskStatus(transID,taskIDs,status)
if not res['OK']:
gLogger.error("updateTaskStatus: Failed to update task status for transformation", "%s %s" % (transID,res['Message']))
gLogger.info("updateTaskStatus: Transformation task status update complete")
return S_OK()
def updateFileStatus(self):
gLogger.info("updateFileStatus: Updating Status of task files")
#Get the transformations to be updated
status = self.am_getOption('UpdateFilesStatus',['Active','Completing','Stopped'])
res = self._selectTransformations(transType=self.transType,status=status,agentType=[])
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
# Get the files which are in a UPDATE state
updateStatus = self.am_getOption('FileUpdateStatus',['Submitted','Received','Waiting','Running'])
timeStamp = str(datetime.datetime.utcnow() - datetime.timedelta(minutes=10))
condDict = {'TransformationID' : transID, 'Status' : ['Assigned']}
res = self.transClient.getTransformationFiles(condDict=condDict,older=timeStamp, timeStamp='LastUpdate')
if not res['OK']:
gLogger.error("updateFileStatus: Failed to get transformation files to update.",res['Message'])
continue
if not res['Value']:
gLogger.info("updateFileStatus: No files to be updated for transformation %s." % transID)
continue
res = self.getSubmittedFileStatus(res['Value'])
if not res['OK']:
gLogger.error("updateFileStatus: Failed to get updated file statuses for transformation","%s %s" % (transID,res['Message']))
continue
statusDict = res['Value']
if not statusDict:
gLogger.info("updateFileStatus: No file statuses to be updated for transformation %s." % transID)
continue
fileReport = FileReport(server=self.transClient.getServer())
for lfn,status in statusDict.items():
fileReport.setFileStatus(int(transID),lfn,status)
res = fileReport.commit()
if not res['OK']:
gLogger.error("updateFileStatus: Failed to update file status for transformation", "%s %s" % (transID, res['Message']))
else:
for status,update in res['Value'].items():
gLogger.info("updateFileStatus: Updated %s files for %s to %s." % (update, transID, status))
gLogger.info("updateFileStatus: Transformation file status update complete")
return S_OK()
def checkReservedTasks(self):
gLogger.info("checkReservedTasks: Checking Reserved tasks")
# Get the transformations which should be checked
status = self.am_getOption('CheckReservedStatus',['Active','Completing','Stopped'])
res = self._selectTransformations(transType=self.transType,status=status,agentType=[])
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
# Select the tasks which have been in Reserved status for more than 1 hour for selected transformations
condDict = {"TransformationID":transID,"ExternalStatus":'Reserved'}
time_stamp_older = str(datetime.datetime.utcnow() - datetime.timedelta(hours=1))
time_stamp_newer = str(datetime.datetime.utcnow() - datetime.timedelta(days=7))
res = self.transClient.getTransformationTasks(condDict=condDict,older=time_stamp_older,newer=time_stamp_newer, timeStamp='LastUpdateTime')
if not res['OK']:
gLogger.error("checkReservedTasks: Failed to get Reserved tasks for transformation", "%s %s" % (transID,res['Message']))
continue
if not res['Value']:
gLogger.verbose("checkReservedTasks: No Reserved tasks found for transformation %s" % transID)
continue
res = self.updateTransformationReservedTasks(res['Value'])
if not res['OK']:
gLogger.info("checkReservedTasks: No Reserved tasks found for transformation %s" % transID)
continue
noTasks = res['Value']['NoTasks']
taskNameIDs = res['Value']['TaskNameIDs']
# For the tasks with no associated request found re-set the status of the task in the transformationDB
for taskName in noTasks:
transID,taskID = taskName.split('_')
gLogger.info("checkReservedTasks: Resetting status of %s to Created as no associated task found" % (taskName))
res = self.transClient.setTaskStatus(int(transID),int(taskID),'Created')
if not res['OK']:
gLogger.warn("checkReservedTasks: Failed to update task status and ID after recovery", "%s %s" % (taskName,res['Message']))
# For the tasks for which an associated request was found update the task details in the transformationDB
for taskName,extTaskID in taskNameIDs.items():
transID,taskID = taskName.split('_')
gLogger.info("checkReservedTasks: Resetting status of %s to Created with ID %s" % (taskName,extTaskID))
res = self.transClient.setTaskStatusAndWmsID(int(transID),int(taskID),'Submitted',str(extTaskID))
if not res['OK']:
gLogger.warn("checkReservedTasks: Failed to update task status and ID after recovery", "%s %s" % (taskName,res['Message']))
gLogger.info("checkReservedTasks: Updating of reserved tasks complete")
return S_OK()
def submitTasks(self):
gLogger.info("submitTasks: Submitting tasks for transformations")
res = getProxyInfo(False,False)
if not res['OK']:
gLogger.error("submitTasks: Failed to determine credentials for submission",res['Message'])
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
gLogger.info("submitTasks: Tasks will be submitted with the credentials %s:%s" % (owner,ownerGroup))
# Get the transformations which should be submitted
tasksPerLoop = self.am_getOption('TasksPerLoop',50)
status = self.am_getOption('SubmitStatus',['Active','Completing'])
res = self._selectTransformations(transType=self.transType,status=status)
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
transBody = transformation['Body']
res = self.transClient.getTasksToSubmit(transID,tasksPerLoop)
if not res['OK']:
gLogger.error("submitTasks: Failed to obtain tasks for transformation", "%s %s" % (transID,res['Message']))
continue
tasks = res['Value']['JobDictionary']
if not tasks:
gLogger.verbose("submitTasks: No tasks found for submission for transformation %s" % transID)
continue
gLogger.info("submitTasks: Obtained %d tasks for submission for transformation %s" % (len(tasks),transID))
res = self.prepareTransformationTasks(transBody,tasks,owner,ownerGroup)
if not res['OK']:
gLogger.error("submitTasks: Failed to prepare tasks for transformation", "%s %s" % (transID,res['Message']))
continue
res = self.submitTransformationTasks(res['Value'])
if not res['OK']:
gLogger.error("submitTasks: Failed to submit prepared tasks for transformation", "%s %s" % (transID,res['Message']))
continue
res = self.updateDBAfterTaskSubmission(res['Value'])
if not res['OK']:
gLogger.error("submitTasks: Failed to update DB after task submission for transformation", "%s %s" % (transID,res['Message']))
continue
gLogger.info("submitTasks: Submission of transformation tasks complete")
return S_OK()
class Transformation(API):
#############################################################################
def __init__(self, transID=0, transClient=None):
""" c'tor
"""
super(Transformation, self).__init__()
self.paramTypes = {
'TransformationID': [types.IntType, types.LongType],
'TransformationName': types.StringTypes,
'Status': types.StringTypes,
'Description': types.StringTypes,
'LongDescription': types.StringTypes,
'Type': types.StringTypes,
'Plugin': types.StringTypes,
'AgentType': types.StringTypes,
'FileMask': types.StringTypes,
'TransformationGroup': types.StringTypes,
'GroupSize': [types.IntType, types.LongType, types.FloatType],
'InheritedFrom': [types.IntType, types.LongType],
'Body': types.StringTypes,
'MaxNumberOfTasks': [types.IntType, types.LongType],
'EventsPerTask': [types.IntType, types.LongType]
}
self.paramValues = {
'TransformationID': 0,
'TransformationName': '',
'Status': 'New',
'Description': '',
'LongDescription': '',
'Type': '',
'Plugin': 'Standard',
'AgentType': 'Manual',
'FileMask': '',
'TransformationGroup': 'General',
'GroupSize': 1,
'InheritedFrom': 0,
'Body': '',
'MaxNumberOfTasks': 0,
'EventsPerTask': 0
}
self.ops = Operations()
self.supportedPlugins = self.ops.getValue(
'Transformations/AllowedPlugins',
['Broadcast', 'Standard', 'BySize', 'ByShare'])
if not transClient:
self.transClient = TransformationClient()
else:
self.transClient = transClient
self.serverURL = self.transClient.getServer()
self.exists = False
if transID:
self.paramValues['TransformationID'] = transID
res = self.getTransformation()
if res['OK']:
self.exists = True
elif res['Message'] == 'Transformation does not exist':
raise AttributeError, 'TransformationID %d does not exist' % transID
else:
self.paramValues['TransformationID'] = 0
gLogger.fatal(
"Failed to get transformation from database",
"%s @ %s" % (transID, self.transClient.serverURL))
def setServer(self, server):
self.serverURL = server
self.transClient.setServer(self.serverURL)
def getServer(self):
return self.serverURL
def reset(self, transID=0):
self.__init__(transID)
self.transClient.setServer(self.serverURL)
return S_OK()
def setTargetSE(self, seList):
return self.__setSE('TargetSE', seList)
def setSourceSE(self, seList):
return self.__setSE('SourceSE', seList)
def __setSE(self, se, seList):
if type(seList) in types.StringTypes:
try:
seList = eval(seList)
except:
seList = seList.replace(',', ' ').split()
res = self.__checkSEs(seList)
if not res['OK']:
return res
self.item_called = se
return self.__setParam(seList)
def __getattr__(self, name):
if name.find('get') == 0:
item = name[3:]
self.item_called = item
return self.__getParam
if name.find('set') == 0:
item = name[3:]
self.item_called = item
return self.__setParam
raise AttributeError, name
def __getParam(self):
if self.item_called == 'Available':
return S_OK(self.paramTypes.keys())
if self.item_called == 'Parameters':
return S_OK(self.paramValues)
if self.item_called in self.paramValues:
return S_OK(self.paramValues[self.item_called])
raise AttributeError, "Unknown parameter for transformation: %s" % self.item_called
def __setParam(self, value):
change = False
if self.item_called in self.paramTypes:
oldValue = self.paramValues[self.item_called]
if oldValue != value:
if type(value) in self.paramTypes[self.item_called]:
change = True
else:
raise TypeError, "%s %s %s expected one of %s" % (
self.item_called, value, type(value),
self.paramTypes[self.item_called])
if not self.item_called in self.paramTypes.keys():
if not self.paramValues.has_key(self.item_called):
change = True
else:
oldValue = self.paramValues[self.item_called]
if oldValue != value:
change = True
if not change:
gLogger.verbose("No change of parameter %s required" %
self.item_called)
else:
gLogger.verbose("Parameter %s to be changed" % self.item_called)
transID = self.paramValues['TransformationID']
if self.exists and transID:
res = self.transClient.setTransformationParameter(
transID, self.item_called, value)
if not res['OK']:
return res
self.paramValues[self.item_called] = value
return S_OK()
def getTransformation(self, printOutput=False):
transID = self.paramValues['TransformationID']
if not transID:
gLogger.fatal("No TransformationID known")
return S_ERROR()
res = self.transClient.getTransformation(transID, extraParams=True)
if not res['OK']:
if printOutput:
self._prettyPrint(res)
return res
transParams = res['Value']
for paramName, paramValue in transParams.items():
setter = None
setterName = "set%s" % paramName
if hasattr(self, setterName) and callable(getattr(
self, setterName)):
setter = getattr(self, setterName)
if not setterName:
gLogger.error(
"Unable to invoke setter %s, it isn't a member function" %
setterName)
continue
setter(paramValue)
if printOutput:
gLogger.info("No printing available yet")
return S_OK(transParams)
def getTransformationLogging(self, printOutput=False):
transID = self.paramValues['TransformationID']
if not transID:
gLogger.fatal("No TransformationID known")
return S_ERROR()
res = self.transClient.getTransformationLogging(transID)
if not res['OK']:
if printOutput:
self._prettyPrint(res)
return res
loggingList = res['Value']
if printOutput:
self._printFormattedDictList(
loggingList, ['Message', 'MessageDate', 'AuthorDN'],
'MessageDate', 'MessageDate')
return S_OK(loggingList)
def extendTransformation(self, nTasks, printOutput=False):
return self.__executeOperation('extendTransformation',
nTasks,
printOutput=printOutput)
def cleanTransformation(self, printOutput=False):
res = self.__executeOperation('cleanTransformation',
printOutput=printOutput)
if res['OK']:
self.paramValues['Status'] = 'Cleaned'
return res
def deleteTransformation(self, printOutput=False):
res = self.__executeOperation('deleteTransformation',
printOutput=printOutput)
if res['OK']:
self.reset()
return res
def addFilesToTransformation(self, lfns, printOutput=False):
return self.__executeOperation('addFilesToTransformation',
lfns,
printOutput=printOutput)
def setFileStatusForTransformation(self, status, lfns, printOutput=False):
return self.__executeOperation('setFileStatusForTransformation',
status,
lfns,
printOutput=printOutput)
def getTransformationTaskStats(self, printOutput=False):
return self.__executeOperation('getTransformationTaskStats',
printOutput=printOutput)
def getTransformationStats(self, printOutput=False):
return self.__executeOperation('getTransformationStats',
printOutput=printOutput)
def deleteTasks(self, taskMin, taskMax, printOutput=False):
return self.__executeOperation('deleteTasks',
taskMin,
taskMax,
printOutput=printOutput)
def addTaskForTransformation(self,
lfns=[],
se='Unknown',
printOutput=False):
return self.__executeOperation('addTaskForTransformation',
lfns,
se,
printOutput=printOutput)
def setTaskStatus(self, taskID, status, printOutput=False):
return self.__executeOperation('setTaskStatus',
taskID,
status,
printOutput=printOutput)
def __executeOperation(self, operation, *parms, **kwds):
transID = self.paramValues['TransformationID']
if not transID:
gLogger.fatal("No TransformationID known")
return S_ERROR()
printOutput = kwds.pop('printOutput')
fcn = None
if hasattr(self.transClient, operation) and callable(
getattr(self.transClient, operation)):
fcn = getattr(self.transClient, operation)
if not fcn:
return S_ERROR(
"Unable to invoke %s, it isn't a member funtion of TransformationClient"
)
res = fcn(transID, *parms, **kwds)
if printOutput:
self._prettyPrint(res)
return res
def getTransformationFiles(self,
fileStatus=[],
lfns=[],
outputFields=[
'FileID', 'LFN', 'Status', 'TaskID',
'TargetSE', 'UsedSE', 'ErrorCount',
'InsertedTime', 'LastUpdate'
],
orderBy='FileID',
printOutput=False):
condDict = {'TransformationID': self.paramValues['TransformationID']}
if fileStatus:
condDict['Status'] = fileStatus
if lfns:
condDict['LFN'] = lfns
res = self.transClient.getTransformationFiles(condDict=condDict)
if not res['OK']:
if printOutput:
self._prettyPrint(res)
return res
if printOutput:
if not outputFields:
gLogger.info("Available fields are: %s" %
res['ParameterNames'].join(' '))
elif not res['Value']:
gLogger.info("No tasks found for selection")
else:
self._printFormattedDictList(res['Value'], outputFields,
'FileID', orderBy)
return res
def getTransformationTasks(self,
taskStatus=[],
taskIDs=[],
outputFields=[
'TransformationID', 'TaskID',
'ExternalStatus', 'ExternalID', 'TargetSE',
'CreationTime', 'LastUpdateTime'
],
orderBy='TaskID',
printOutput=False):
condDict = {'TransformationID': self.paramValues['TransformationID']}
if taskStatus:
condDict['ExternalStatus'] = taskStatus
if taskIDs:
condDict['TaskID'] = taskIDs
res = self.transClient.getTransformationTasks(condDict=condDict)
if not res['OK']:
if printOutput:
self._prettyPrint(res)
return res
if printOutput:
if not outputFields:
gLogger.info("Available fields are: %s" %
res['ParameterNames'].join(' '))
elif not res['Value']:
gLogger.info("No tasks found for selection")
else:
self._printFormattedDictList(res['Value'], outputFields,
'TaskID', orderBy)
return res
#############################################################################
def getTransformations(self,
transID=[],
transStatus=[],
outputFields=[
'TransformationID', 'Status', 'AgentType',
'TransformationName', 'CreationDate'
],
orderBy='TransformationID',
printOutput=False):
condDict = {}
if transID:
condDict['TransformationID'] = transID
if transStatus:
condDict['Status'] = transStatus
res = self.transClient.getTransformations(condDict=condDict)
if not res['OK']:
if printOutput:
self._prettyPrint(res)
return res
if printOutput:
if not outputFields:
gLogger.info("Available fields are: %s" %
res['ParameterNames'].join(' '))
elif not res['Value']:
gLogger.info("No tasks found for selection")
else:
self._printFormattedDictList(res['Value'], outputFields,
'TransformationID', orderBy)
return res
#############################################################################
def addTransformation(self, addFiles=True, printOutput=False):
res = self._checkCreation()
if not res['OK']:
return self._errorReport(res, 'Failed transformation sanity check')
if printOutput:
gLogger.info(
"Will attempt to create transformation with the following parameters"
)
self._prettyPrint(self.paramValues)
res = self.transClient.addTransformation(
self.paramValues['TransformationName'],
self.paramValues['Description'],
self.paramValues['LongDescription'],
self.paramValues['Type'],
self.paramValues['Plugin'],
self.paramValues['AgentType'],
self.paramValues['FileMask'],
transformationGroup=self.paramValues['TransformationGroup'],
groupSize=self.paramValues['GroupSize'],
inheritedFrom=self.paramValues['InheritedFrom'],
body=self.paramValues['Body'],
maxTasks=self.paramValues['MaxNumberOfTasks'],
eventsPerTask=self.paramValues['EventsPerTask'],
addFiles=addFiles)
if not res['OK']:
if printOutput:
self._prettyPrint(res)
return res
transID = res['Value']
self.exists = True
self.setTransformationID(transID)
gLogger.notice("Created transformation %d" % transID)
for paramName, paramValue in self.paramValues.items():
if not self.paramTypes.has_key(paramName):
res = self.transClient.setTransformationParameter(
transID, paramName, paramValue)
if not res['OK']:
gLogger.error("Failed to add parameter",
"%s %s" % (paramName, res['Message']))
gLogger.notice(
"To add this parameter later please execute the following."
)
gLogger.notice("oTransformation = Transformation(%d)" %
transID)
gLogger.notice("oTransformation.set%s(...)" % paramName)
return S_OK(transID)
def _checkCreation(self):
if self.paramValues['TransformationID']:
gLogger.info(
"You are currently working with an active transformation definition."
)
gLogger.info(
"If you wish to create a new transformation reset the TransformationID."
)
gLogger.info("oTransformation.reset()")
return S_ERROR()
requiredParameters = [
'TransformationName', 'Description', 'LongDescription', 'Type'
]
for parameter in requiredParameters:
if not self.paramValues[parameter]:
gLogger.info(
"%s is not defined for this transformation. This is required..."
% parameter)
self.paramValues[parameter] = raw_input(
"Please enter the value of " + parameter + " ")
plugin = self.paramValues['Plugin']
if not plugin in self.supportedPlugins:
gLogger.info(
"The selected Plugin (%s) is not known to the transformation agent."
% plugin)
res = self.__promptForParameter('Plugin',
choices=self.supportedPlugins,
default='Standard')
if not res['OK']:
return res
self.paramValues['Plugin'] = res['Value']
plugin = self.paramValues['Plugin']
#checkPlugin = "_check%sPlugin" % plugin
#fcn = None
#if hasattr( self, checkPlugin ) and callable( getattr( self, checkPlugin ) ):
# fcn = getattr( self, checkPlugin )
#if not fcn:
# return S_ERROR( "Unable to invoke %s, it isn't a member function" % checkPlugin )
#res = fcn()
return S_OK()
def _checkBySizePlugin(self):
return self._checkStandardPlugin()
def _checkBySharePlugin(self):
return self._checkStandardPlugin()
def _checkStandardPlugin(self):
groupSize = self.paramValues['GroupSize']
if (groupSize <= 0):
gLogger.info(
"The GroupSize was found to be less than zero. It has been set to 1."
)
res = self.setGroupSize(1)
if not res['OK']:
return res
return S_OK()
def _checkBroadcastPlugin(self):
gLogger.info(
"The Broadcast plugin requires the following parameters be set: %s"
% (', '.join(['SourceSE', 'TargetSE'])))
requiredParams = ['SourceSE', 'TargetSE']
for requiredParam in requiredParams:
if (not self.paramValues.has_key(requiredParam)) or (
not self.paramValues[requiredParam]):
paramValue = raw_input("Please enter " + requiredParam + " ")
setter = None
setterName = "set%s" % requiredParam
if hasattr(self, setterName) and callable(
getattr(self, setterName)):
setter = getattr(self, setterName)
if not setter:
return S_ERROR(
"Unable to invoke %s, this function hasn't been implemented."
% setterName)
ses = paramValue.replace(',', ' ').split()
res = setter(ses)
if not res['OK']:
return res
return S_OK()
def __checkSEs(self, seList):
res = gConfig.getSections('/Resources/StorageElements')
if not res['OK']:
return self._errorReport(res,
'Failed to get possible StorageElements')
missing = []
for se in seList:
if not se in res['Value']:
gLogger.error("StorageElement %s is not known" % se)
missing.append(se)
if missing:
return S_ERROR("%d StorageElements not known" % len(missing))
return S_OK()
def __promptForParameter(self,
parameter,
choices=[],
default='',
insert=True):
res = promptUser("Please enter %s" % parameter,
choices=choices,
default=default)
if not res['OK']:
return self._errorReport(res)
gLogger.notice("%s will be set to '%s'" % (parameter, res['Value']))
paramValue = res['Value']
if insert:
setter = None
setterName = "set%s" % parameter
if hasattr(self, setterName) and callable(getattr(
self, setterName)):
setter = getattr(self, setterName)
if not setter:
return S_ERROR(
"Unable to invoke %s, it isn't a member function of Transformation!"
)
res = setter(paramValue)
if not res['OK']:
return res
return S_OK(paramValue)
class Transformation( API ):
#############################################################################
def __init__( self, transID = 0, transClient = None ):
""" c'tor
"""
super( Transformation, self ).__init__()
self.paramTypes = { 'TransformationID' : [types.IntType, types.LongType],
'TransformationName' : types.StringTypes,
'Status' : types.StringTypes,
'Description' : types.StringTypes,
'LongDescription' : types.StringTypes,
'Type' : types.StringTypes,
'Plugin' : types.StringTypes,
'AgentType' : types.StringTypes,
'FileMask' : types.StringTypes,
'TransformationGroup' : types.StringTypes,
'GroupSize' : [types.IntType, types.LongType, types.FloatType],
'InheritedFrom' : [types.IntType, types.LongType],
'Body' : types.StringTypes,
'MaxNumberOfTasks' : [types.IntType, types.LongType],
'EventsPerTask' : [types.IntType, types.LongType]}
self.paramValues = { 'TransformationID' : 0,
'TransformationName' : '',
'Status' : 'New',
'Description' : '',
'LongDescription' : '',
'Type' : '',
'Plugin' : 'Standard',
'AgentType' : 'Manual',
'FileMask' : '',
'TransformationGroup' : 'General',
'GroupSize' : 1,
'InheritedFrom' : 0,
'Body' : '',
'MaxNumberOfTasks' : 0,
'EventsPerTask' : 0}
self.supportedPlugins = ['Broadcast', 'Standard', 'BySize', 'ByShare']
if not transClient:
self.transClient = TransformationClient()
else:
self.transClient = transClient
self.serverURL = self.transClient.getServer()
self.exists = False
if transID:
self.paramValues['TransformationID'] = transID
res = self.getTransformation()
if res['OK']:
self.exists = True
elif res['Message'] == 'Transformation does not exist':
raise AttributeError, 'TransformationID %d does not exist' % transID
else:
self.paramValues['TransformationID'] = 0
gLogger.fatal( "Failed to get transformation from database", "%s @ %s" % ( transID, self.transClient.serverURL ) )
def setServer( self, server ):
self.serverURL = server
self.transClient.setServer( self.serverURL )
def getServer( self ):
return self.serverURL
def reset( self, transID = 0 ):
self.__init__( transID )
self.transClient.setServer( self.serverURL )
return S_OK()
def setTargetSE( self, seList ):
return self.__setSE( 'TargetSE', seList )
def setSourceSE( self, seList ):
return self.__setSE( 'SourceSE', seList )
def __setSE( self, se, seList ):
if type( seList ) in types.StringTypes:
try:
seList = eval( seList )
except:
seList = seList.replace( ',', ' ' ).split()
res = self.__checkSEs( seList )
if not res['OK']:
return res
self.item_called = se
return self.__setParam( seList )
def __getattr__( self, name ):
if name.find( 'get' ) == 0:
item = name[3:]
self.item_called = item
return self.__getParam
if name.find( 'set' ) == 0:
item = name[3:]
self.item_called = item
return self.__setParam
raise AttributeError, name
def __getParam( self ):
if self.item_called == 'Available':
return S_OK( self.paramTypes.keys() )
if self.item_called == 'Parameters':
return S_OK( self.paramValues )
if self.item_called in self.paramValues:
return S_OK( self.paramValues[self.item_called] )
raise AttributeError, "Unknown parameter for transformation: %s" % self.item_called
def __setParam( self, value ):
change = False
if self.item_called in self.paramTypes:
oldValue = self.paramValues[self.item_called]
if oldValue != value:
if type( value ) in self.paramTypes[self.item_called]:
change = True
else:
raise TypeError, "%s %s %s expected one of %s" % ( self.item_called, value, type( value ), self.paramTypes[self.item_called] )
if not self.item_called in self.paramTypes.keys():
if not self.paramValues.has_key( self.item_called ):
change = True
else:
oldValue = self.paramValues[self.item_called]
if oldValue != value:
change = True
if not change:
gLogger.verbose( "No change of parameter %s required" % self.item_called )
else:
gLogger.verbose( "Parameter %s to be changed" % self.item_called )
transID = self.paramValues['TransformationID']
if self.exists and transID:
res = self.transClient.setTransformationParameter( transID, self.item_called, value )
if not res['OK']:
return res
self.paramValues[self.item_called] = value
return S_OK()
def getTransformation( self, printOutput = False ):
transID = self.paramValues['TransformationID']
if not transID:
gLogger.fatal( "No TransformationID known" )
return S_ERROR()
res = self.transClient.getTransformation( transID, extraParams = True )
if not res['OK']:
if printOutput:
self._prettyPrint( res )
return res
transParams = res['Value']
for paramName, paramValue in transParams.items():
setter = None
setterName = "set%s" % paramName
if hasattr( self, setterName ) and callable( getattr( self, setterName ) ):
setter = getattr( self, setterName )
if not setterName:
gLogger.error( "Unable to invoke setter %s, it isn't a member function" % setterName )
continue
setter( paramValue )
if printOutput:
gLogger.info( "No printing available yet" )
return S_OK( transParams )
def getTransformationLogging( self, printOutput = False ):
transID = self.paramValues['TransformationID']
if not transID:
gLogger.fatal( "No TransformationID known" )
return S_ERROR()
res = self.transClient.getTransformationLogging( transID )
if not res['OK']:
if printOutput:
self._prettyPrint( res )
return res
loggingList = res['Value']
if printOutput:
self._printFormattedDictList( loggingList, ['Message', 'MessageDate', 'AuthorDN'], 'MessageDate', 'MessageDate' )
return S_OK( loggingList )
def extendTransformation( self, nTasks, printOutput = False ):
return self.__executeOperation( 'extendTransformation', nTasks, printOutput = printOutput )
def cleanTransformation( self, printOutput = False ):
res = self.__executeOperation( 'cleanTransformation', printOutput = printOutput )
if res['OK']:
self.paramValues['Status'] = 'Cleaned'
return res
def deleteTransformation( self, printOutput = False ):
res = self.__executeOperation( 'deleteTransformation', printOutput = printOutput )
if res['OK']:
self.reset()
return res
def addFilesToTransformation( self, lfns, printOutput = False ):
return self.__executeOperation( 'addFilesToTransformation', lfns, printOutput = printOutput )
def setFileStatusForTransformation( self, status, lfns, printOutput = False ):
return self.__executeOperation( 'setFileStatusForTransformation', status, lfns, printOutput = printOutput )
def getTransformationTaskStats( self, printOutput = False ):
return self.__executeOperation( 'getTransformationTaskStats', printOutput = printOutput )
def getTransformationStats( self, printOutput = False ):
return self.__executeOperation( 'getTransformationStats', printOutput = printOutput )
def deleteTasks( self, taskMin, taskMax, printOutput = False ):
return self.__executeOperation( 'deleteTasks', taskMin, taskMax, printOutput = printOutput )
def addTaskForTransformation( self, lfns = [], se = 'Unknown', printOutput = False ):
return self.__executeOperation( 'addTaskForTransformation', lfns, se, printOutput = printOutput )
def setTaskStatus( self, taskID, status, printOutput = False ):
return self.__executeOperation( 'setTaskStatus', taskID, status, printOutput = printOutput )
def __executeOperation( self, operation, *parms, **kwds ):
transID = self.paramValues['TransformationID']
if not transID:
gLogger.fatal( "No TransformationID known" )
return S_ERROR()
printOutput = kwds.pop( 'printOutput' )
fcn = None
if hasattr( self.transClient, operation ) and callable( getattr( self.transClient, operation ) ):
fcn = getattr( self.transClient, operation )
if not fcn:
return S_ERROR( "Unable to invoke %s, it isn't a member funtion of TransformationClient" )
res = fcn( transID, *parms, **kwds )
if printOutput:
self._prettyPrint( res )
return res
def getTransformationFiles( self, fileStatus = [], lfns = [], outputFields = ['FileID', 'LFN', 'Status', 'TaskID', 'TargetSE', 'UsedSE', 'ErrorCount', 'InsertedTime', 'LastUpdate'], orderBy = 'FileID', printOutput = False ):
condDict = {'TransformationID':self.paramValues['TransformationID']}
if fileStatus:
condDict['Status'] = fileStatus
if lfns:
condDict['LFN'] = lfns
res = self.transClient.getTransformationFiles( condDict = condDict )
if not res['OK']:
if printOutput:
self._prettyPrint( res )
return res
if printOutput:
if not outputFields:
gLogger.info( "Available fields are: %s" % string.join( res['ParameterNames'] ) )
elif not res['Value']:
gLogger.info( "No tasks found for selection" )
else:
self._printFormattedDictList( res['Value'], outputFields, 'FileID', orderBy )
return res
def getTransformationTasks( self, taskStatus = [], taskIDs = [], outputFields = ['TransformationID', 'TaskID', 'ExternalStatus', 'ExternalID', 'TargetSE', 'CreationTime', 'LastUpdateTime'], orderBy = 'TaskID', printOutput = False ):
condDict = {'TransformationID':self.paramValues['TransformationID']}
if taskStatus:
condDict['ExternalStatus'] = taskStatus
if taskIDs:
condDict['TaskID'] = taskIDs
res = self.transClient.getTransformationTasks( condDict = condDict )
if not res['OK']:
if printOutput:
self._prettyPrint( res )
return res
if printOutput:
if not outputFields:
gLogger.info( "Available fields are: %s" % string.join( res['ParameterNames'] ) )
elif not res['Value']:
gLogger.info( "No tasks found for selection" )
else:
self._printFormattedDictList( res['Value'], outputFields, 'TaskID', orderBy )
return res
#############################################################################
def getTransformations( self, transID = [], transStatus = [], outputFields = ['TransformationID', 'Status', 'AgentType', 'TransformationName', 'CreationDate'], orderBy = 'TransformationID', printOutput = False ):
condDict = {}
if transID:
condDict['TransformationID'] = transID
if transStatus:
condDict['Status'] = transStatus
res = self.transClient.getTransformations( condDict = condDict )
if not res['OK']:
if printOutput:
self._prettyPrint( res )
return res
if printOutput:
if not outputFields:
gLogger.info( "Available fields are: %s" % string.join( res['ParameterNames'] ) )
elif not res['Value']:
gLogger.info( "No tasks found for selection" )
else:
self._printFormattedDictList( res['Value'], outputFields, 'TransformationID', orderBy )
return res
#############################################################################
def addTransformation( self, addFiles = True, printOutput = False ):
res = self._checkCreation()
if not res['OK']:
return self._errorReport( res, 'Failed transformation sanity check' )
if printOutput:
gLogger.info( "Will attempt to create transformation with the following parameters" )
self._prettyPrint( self.paramValues )
res = self.transClient.addTransformation( self.paramValues['TransformationName'],
self.paramValues['Description'],
self.paramValues['LongDescription'],
self.paramValues['Type'],
self.paramValues['Plugin'],
self.paramValues['AgentType'],
self.paramValues['FileMask'],
transformationGroup = self.paramValues['TransformationGroup'],
groupSize = self.paramValues['GroupSize'],
inheritedFrom = self.paramValues['InheritedFrom'],
body = self.paramValues['Body'],
maxTasks = self.paramValues['MaxNumberOfTasks'],
eventsPerTask = self.paramValues['EventsPerTask'],
addFiles = addFiles )
if not res['OK']:
if printOutput:
self._prettyPrint( res )
return res
transID = res['Value']
self.exists = True
self.setTransformationID( transID )
gLogger.info( "Created transformation %d" % transID )
for paramName, paramValue in self.paramValues.items():
if not self.paramTypes.has_key( paramName ):
res = self.transClient.setTransformationParameter( transID, paramName, paramValue )
if not res['OK']:
gLogger.error( "Failed to add parameter", "%s %s" % ( paramName, res['Message'] ) )
gLogger.info( "To add this parameter later please execute the following." )
gLogger.info( "oTransformation = Transformation(%d)" % transID )
gLogger.info( "oTransformation.set%s(...)" % paramName )
return S_OK( transID )
def _checkCreation( self ):
if self.paramValues['TransformationID']:
gLogger.info( "You are currently working with an active transformation definition." )
gLogger.info( "If you wish to create a new transformation reset the TransformationID." )
gLogger.info( "oTransformation.reset()" )
return S_ERROR()
requiredParameters = ['TransformationName', 'Description' , 'LongDescription', 'Type']
for parameter in requiredParameters:
if not self.paramValues[parameter]:
gLogger.info( "%s is not defined for this transformation. This is required..." % parameter )
res = self.__promptForParameter( parameter )
if not res['OK']:
return res
plugin = self.paramValues['Plugin']
if not plugin in self.supportedPlugins:
gLogger.info( "The selected Plugin (%s) is not known to the transformation agent." % plugin )
res = self.__promptForParameter( 'Plugin', choices = self.supportedPlugins, default = 'Standard' )
if not res['OK']:
return res
plugin = self.paramValues['Plugin']
checkPlugin = "_check%sPlugin" % plugin
fcn = None
if hasattr( self, checkPlugin ) and callable( getattr( self, checkPlugin ) ):
fcn = getattr( self, checkPlugin )
if not fcn:
return S_ERROR( "Unable to invoke %s, it isn't a member function" % checkPlugin )
res = fcn()
return res
def _checkBySizePlugin( self ):
return self._checkStandardPlugin()
def _checkBySharePlugin( self ):
return self._checkStandardPlugin()
def _checkStandardPlugin( self ):
groupSize = self.paramValues['GroupSize']
if ( groupSize <= 0 ):
gLogger.info( "The GroupSize was found to be less than zero. It has been set to 1." )
res = self.setGroupSize( 1 )
if not res['OK']:
return res
return S_OK()
def _checkBroadcastPlugin( self ):
gLogger.info( "The Broadcast plugin requires the following parameters be set: %s" % ( string.join( ['SourceSE', 'TargetSE'], ', ' ) ) )
requiredParams = ['SourceSE', 'TargetSE']
for requiredParam in requiredParams:
if ( not self.paramValues.has_key( requiredParam ) ) or ( not self.paramValues[requiredParam] ):
res = self.__promptForParameter( requiredParam, insert = False )
if not res['OK']:
return res
paramValue = res['Value']
setter = None
setterName = "set%s" % requiredParam
if hasattr( self, setterName ) and callable( getattr( self, setterName ) ):
setter = getattr( self, setterName )
if not setter:
return S_ERROR( "Unable to invoke %s, this function hasn't been implemented." % setterName )
ses = paramValue.replace( ',', ' ' ).split()
res = setter( ses )
if not res['OK']:
return res
return S_OK()
def __checkSEs( self, seList ):
res = gConfig.getSections( '/Resources/StorageElements' )
if not res['OK']:
return self._errorReport( res, 'Failed to get possible StorageElements' )
missing = []
for se in seList:
if not se in res['Value']:
gLogger.error( "StorageElement %s is not known" % se )
missing.append( se )
if missing:
return S_ERROR( "%d StorageElements not known" % len( missing ) )
return S_OK()
def __promptForParameter( self, parameter, choices = [], default = '', insert = True ):
res = promptUser( "Please enter %s" % parameter, choices = choices, default = default )
if not res['OK']:
return self._errorReport( res )
gLogger.info( "%s will be set to '%s'" % ( parameter, res['Value'] ) )
paramValue = res['Value']
if insert:
setter = None
setterName = "set%s" % parameter
if hasattr( self, setterName ) and callable( getattr( self, setterName ) ):
setter = getattr( self, setterName )
if not setter:
return S_ERROR( "Unable to invoke %s, it isn't a member function of Transformation!" )
res = setter( paramValue )
if not res['OK']:
return res
return S_OK( paramValue )
class Transformation( API ):
#############################################################################
def __init__( self, transID = 0, transClient = None ):
""" c'tor
"""
super( Transformation, self ).__init__()
self.paramTypes = { 'TransformationID' : [types.IntType, types.LongType],
'TransformationName' : types.StringTypes,
'Status' : types.StringTypes,
'Description' : types.StringTypes,
'LongDescription' : types.StringTypes,
'Type' : types.StringTypes,
'Plugin' : types.StringTypes,
'AgentType' : types.StringTypes,
'FileMask' : types.StringTypes,
'TransformationGroup' : types.StringTypes,
'GroupSize' : [types.IntType, types.LongType, types.FloatType],
'InheritedFrom' : [types.IntType, types.LongType],
'Body' : types.StringTypes,
'MaxNumberOfTasks' : [types.IntType, types.LongType],
'EventsPerTask' : [types.IntType, types.LongType]}
self.paramValues = { 'TransformationID' : 0,
'TransformationName' : '',
'Status' : 'New',
'Description' : '',
'LongDescription' : '',
'Type' : '',
'Plugin' : 'Standard',
'AgentType' : 'Manual',
'FileMask' : '',
'TransformationGroup' : 'General',
'GroupSize' : 1,
'InheritedFrom' : 0,
'Body' : '',
'MaxNumberOfTasks' : 0,
'EventsPerTask' : 0}
self.ops = Operations()
self.supportedPlugins = self.ops.getValue( 'Transformations/AllowedPlugins',
['Broadcast', 'Standard', 'BySize', 'ByShare'] )
if not transClient:
self.transClient = TransformationClient()
else:
self.transClient = transClient
self.serverURL = self.transClient.getServer()
self.exists = False
if transID:
self.paramValues['TransformationID'] = transID
res = self.getTransformation()
if res['OK']:
self.exists = True
elif res['Message'] == 'Transformation does not exist':
raise AttributeError( 'TransformationID %d does not exist' % transID )
else:
self.paramValues['TransformationID'] = 0
gLogger.fatal( "Failed to get transformation from database", "%s @ %s" % ( transID,
self.transClient.serverURL ) )
def setServer( self, server ):
self.serverURL = server
self.transClient.setServer( self.serverURL )
def getServer( self ):
return self.serverURL
def reset( self, transID = 0 ):
self.__init__( transID )
self.transClient.setServer( self.serverURL )
return S_OK()
def setTargetSE( self, seList ):
return self.__setSE( 'TargetSE', seList )
def setSourceSE( self, seList ):
return self.__setSE( 'SourceSE', seList )
def setBody( self, body ):
""" check that the body is a string, or using the proper syntax for multiple operations
:param body: transformation body, for example
.. code :: python
body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }),
( "RemoveReplica", { "TargetSE":"FOO-SRM" } ),
]
:type body: string or list of tuples (or lists) of string and dictionaries
:raises TypeError: If the structure is not as expected
:raises ValueError: If unknown attribute for the :class:`~DIRAC.RequestManagementSystem.Client.Operation.Operation` is used
:returns: S_OK, S_ERROR
"""
self.item_called = "Body"
if isinstance( body, basestring ):
return self.__setParam( body )
if not isinstance( body, ( list, tuple ) ):
raise TypeError( "Expected list or string, but %r is %s" % ( body, type( body ) ) )
for tup in body:
if not isinstance( tup, ( tuple, list ) ):
raise TypeError( "Expected tuple or list, but %r is %s" % ( tup, type( tup ) ) )
if len( tup ) != 2:
raise TypeError( "Expected 2-tuple, but %r is length %d" % ( tup, len( tup ) ) )
if not isinstance( tup[0], basestring ):
raise TypeError( "Expected string, but first entry in tuple %r is %s" % ( tup, type( tup[0] ) ) )
if not isinstance( tup[1], dict ):
raise TypeError( "Expected dictionary, but second entry in tuple %r is %s" % ( tup, type( tup[0] ) ) )
for par, val in tup[1].iteritems():
if not isinstance( par, basestring ):
raise TypeError( "Expected string, but key in dictionary %r is %s" % ( par, type( par ) ) )
if not par in Operation.ATTRIBUTE_NAMES:
raise ValueError( "Unknown attribute for Operation: %s" % par )
if not isinstance( val, ( basestring, int, long, float, list, tuple, dict ) ):
raise TypeError( "Cannot encode %r, in json" % ( val ) )
return self.__setParam( json.dumps( body ) )
def __setSE( self, seParam, seList ):
if isinstance( seList, basestring ):
try:
seList = eval( seList )
except:
seList = seList.split( ',' )
elif isinstance( seList, ( list, dict, tuple ) ):
seList = list( seList )
else:
return S_ERROR( "Bad argument type" )
res = self.__checkSEs( seList )
if not res['OK']:
return res
self.item_called = seParam
return self.__setParam( seList )
def __getattr__( self, name ):
if name.find( 'get' ) == 0:
item = name[3:]
self.item_called = item
return self.__getParam
if name.find( 'set' ) == 0:
item = name[3:]
self.item_called = item
return self.__setParam
raise AttributeError( name )
def __getParam( self ):
if self.item_called == 'Available':
return S_OK( self.paramTypes.keys() )
if self.item_called == 'Parameters':
return S_OK( self.paramValues )
if self.item_called in self.paramValues:
return S_OK( self.paramValues[self.item_called] )
raise AttributeError( "Unknown parameter for transformation: %s" % self.item_called )
def __setParam( self, value ):
change = False
if self.item_called in self.paramTypes:
if self.paramValues[self.item_called] != value:
if type( value ) in self.paramTypes[self.item_called]:
change = True
else:
raise TypeError( "%s %s %s expected one of %s" % ( self.item_called, value, type( value ),
self.paramTypes[self.item_called] ) )
else:
if self.item_called not in self.paramValues:
change = True
else:
if self.paramValues[self.item_called] != value:
change = True
if not change:
gLogger.verbose( "No change of parameter %s required" % self.item_called )
else:
gLogger.verbose( "Parameter %s to be changed" % self.item_called )
transID = self.paramValues['TransformationID']
if self.exists and transID:
res = self.transClient.setTransformationParameter( transID, self.item_called, value )
if not res['OK']:
return res
self.paramValues[self.item_called] = value
return S_OK()
def getTransformation( self, printOutput = False ):
transID = self.paramValues['TransformationID']
if not transID:
gLogger.fatal( "No TransformationID known" )
return S_ERROR()
res = self.transClient.getTransformation( transID, extraParams = True )
if not res['OK']:
if printOutput:
self._prettyPrint( res )
return res
transParams = res['Value']
for paramName, paramValue in transParams.items():
setter = None
setterName = "set%s" % paramName
if hasattr( self, setterName ) and callable( getattr( self, setterName ) ):
setter = getattr( self, setterName )
if not setterName:
gLogger.error( "Unable to invoke setter %s, it isn't a member function" % setterName )
continue
setter( paramValue )
if printOutput:
gLogger.info( "No printing available yet" )
return S_OK( transParams )
def getTransformationLogging( self, printOutput = False ):
transID = self.paramValues['TransformationID']
if not transID:
gLogger.fatal( "No TransformationID known" )
return S_ERROR()
res = self.transClient.getTransformationLogging( transID )
if not res['OK']:
if printOutput:
self._prettyPrint( res )
return res
loggingList = res['Value']
if printOutput:
self._printFormattedDictList( loggingList, ['Message', 'MessageDate', 'AuthorDN'], 'MessageDate', 'MessageDate' )
return S_OK( loggingList )
def extendTransformation( self, nTasks, printOutput = False ):
return self.__executeOperation( 'extendTransformation', nTasks, printOutput = printOutput )
def cleanTransformation( self, printOutput = False ):
res = self.__executeOperation( 'cleanTransformation', printOutput = printOutput )
if res['OK']:
self.paramValues['Status'] = 'Cleaned'
return res
def deleteTransformation( self, printOutput = False ):
res = self.__executeOperation( 'deleteTransformation', printOutput = printOutput )
if res['OK']:
self.reset()
return res
def addFilesToTransformation( self, lfns, printOutput = False ):
return self.__executeOperation( 'addFilesToTransformation', lfns, printOutput = printOutput )
def setFileStatusForTransformation( self, status, lfns, printOutput = False ):
return self.__executeOperation( 'setFileStatusForTransformation', status, lfns, printOutput = printOutput )
def getTransformationTaskStats( self, printOutput = False ):
return self.__executeOperation( 'getTransformationTaskStats', printOutput = printOutput )
def getTransformationStats( self, printOutput = False ):
return self.__executeOperation( 'getTransformationStats', printOutput = printOutput )
def deleteTasks( self, taskMin, taskMax, printOutput = False ):
return self.__executeOperation( 'deleteTasks', taskMin, taskMax, printOutput = printOutput )
def addTaskForTransformation( self, lfns = [], se = 'Unknown', printOutput = False ):
return self.__executeOperation( 'addTaskForTransformation', lfns, se, printOutput = printOutput )
def setTaskStatus( self, taskID, status, printOutput = False ):
return self.__executeOperation( 'setTaskStatus', taskID, status, printOutput = printOutput )
def __executeOperation( self, operation, *parms, **kwds ):
transID = self.paramValues['TransformationID']
if not transID:
gLogger.fatal( "No TransformationID known" )
return S_ERROR()
printOutput = kwds.pop( 'printOutput' )
fcn = None
if hasattr( self.transClient, operation ) and callable( getattr( self.transClient, operation ) ):
fcn = getattr( self.transClient, operation )
if not fcn:
return S_ERROR( "Unable to invoke %s, it isn't a member funtion of TransformationClient" )
res = fcn( transID, *parms, **kwds )
if printOutput:
self._prettyPrint( res )
return res
def getTransformationFiles( self, fileStatus = [], lfns = [], outputFields = ['FileID', 'LFN', 'Status', 'TaskID',
'TargetSE', 'UsedSE', 'ErrorCount',
'InsertedTime', 'LastUpdate'],
orderBy = 'FileID', printOutput = False ):
condDict = {'TransformationID':self.paramValues['TransformationID']}
if fileStatus:
condDict['Status'] = fileStatus
if lfns:
condDict['LFN'] = lfns
res = self.transClient.getTransformationFiles( condDict = condDict )
if not res['OK']:
if printOutput:
self._prettyPrint( res )
return res
if printOutput:
if not outputFields:
gLogger.info( "Available fields are: %s" % res['ParameterNames'].join( ' ' ) )
elif not res['Value']:
gLogger.info( "No tasks found for selection" )
else:
self._printFormattedDictList( res['Value'], outputFields, 'FileID', orderBy )
return res
def getTransformationTasks( self, taskStatus = [], taskIDs = [], outputFields = ['TransformationID', 'TaskID',
'ExternalStatus', 'ExternalID',
'TargetSE', 'CreationTime',
'LastUpdateTime'],
orderBy = 'TaskID', printOutput = False ):
condDict = {'TransformationID':self.paramValues['TransformationID']}
if taskStatus:
condDict['ExternalStatus'] = taskStatus
if taskIDs:
condDict['TaskID'] = taskIDs
res = self.transClient.getTransformationTasks( condDict = condDict )
if not res['OK']:
if printOutput:
self._prettyPrint( res )
return res
if printOutput:
if not outputFields:
gLogger.info( "Available fields are: %s" % res['ParameterNames'].join( ' ' ) )
elif not res['Value']:
gLogger.info( "No tasks found for selection" )
else:
self._printFormattedDictList( res['Value'], outputFields, 'TaskID', orderBy )
return res
#############################################################################
def getTransformations( self, transID = [], transStatus = [], outputFields = ['TransformationID', 'Status',
'AgentType', 'TransformationName',
'CreationDate'],
orderBy = 'TransformationID', printOutput = False ):
condDict = {}
if transID:
condDict['TransformationID'] = transID
if transStatus:
condDict['Status'] = transStatus
res = self.transClient.getTransformations( condDict = condDict )
if not res['OK']:
if printOutput:
self._prettyPrint( res )
return res
if printOutput:
if not outputFields:
gLogger.info( "Available fields are: %s" % res['ParameterNames'].join( ' ' ) )
elif not res['Value']:
gLogger.info( "No tasks found for selection" )
else:
self._printFormattedDictList( res['Value'], outputFields, 'TransformationID', orderBy )
return res
#############################################################################
def getAuthorDNfromProxy( self ):
""" gets the AuthorDN and username of the transformation from the uploaded proxy
"""
username = ""
author = ""
res = getProxyInfo()
if res['OK']:
author = res['Value']['identity']
username = res['Value']['username']
else:
gLogger.error( "Unable to get uploaded proxy Info %s " % res['Message'] )
return S_ERROR( res['Message'] )
res = {'username' : username, 'authorDN' : author }
return S_OK( res )
#############################################################################
def getTransformationsByUser( self, authorDN = "", userName = "", transID = [], transStatus = [],
outputFields = ['TransformationID', 'Status',
'AgentType', 'TransformationName',
'CreationDate', 'AuthorDN'],
orderBy = 'TransformationID', printOutput = False ):
condDict = {}
if authorDN == "":
res = self.getAuthorDNfromProxy()
if not res['OK']:
gLogger.error( res['Message'] )
return S_ERROR( res['Message'] )
else:
foundUserName = res['Value']['username']
foundAuthor = res['Value']['authorDN']
# If the username whom created the uploaded proxy is different than the provided username report error and exit
if not ( userName == "" or userName == foundUserName ):
gLogger.error( "Couldn't resolve the authorDN for user '%s' from the uploaded proxy (proxy created by '%s')" % ( userName, foundUserName ) )
return S_ERROR( "Couldn't resolve the authorDN for user '%s' from the uploaded proxy (proxy created by '%s')" % ( userName, foundUserName ) )
userName = foundUserName
authorDN = foundAuthor
gLogger.info( "Will list transformations created by user '%s' with status '%s'" % ( userName, ', '.join( transStatus ) ) )
else:
gLogger.info( "Will list transformations created by '%s' with status '%s'" % ( authorDN, ', '.join( transStatus ) ) )
condDict['AuthorDN'] = authorDN
if transID:
condDict['TransformationID'] = transID
if transStatus:
condDict['Status'] = transStatus
res = self.transClient.getTransformations( condDict = condDict )
if not res['OK']:
if printOutput:
self._prettyPrint( res )
return res
if printOutput:
if not outputFields:
gLogger.info( "Available fields are: %s" % res['ParameterNames'].join( ' ' ) )
elif not res['Value']:
gLogger.info( "No tasks found for selection" )
else:
self._printFormattedDictList( res['Value'], outputFields, 'TransformationID', orderBy )
return res
#############################################################################
def getSummaryTransformations( self , transID = [] ):
"""Show the summary for a list of Transformations
Fields starting with 'F' ('J') refers to files (jobs).
Proc. stand for processed.
"""
condDict = { 'TransformationID' : transID }
orderby = []
start = 0
maxitems = len( transID )
paramShowNames = ['TransformationID', 'Type', 'Status', 'Files_Total', 'Files_PercentProcessed', \
'Files_Processed', 'Files_Unused', 'Jobs_TotalCreated', 'Jobs_Waiting', \
'Jobs_Running', 'Jobs_Done', 'Jobs_Failed', 'Jobs_Stalled']
# Below, the header used for each field in the printing: short to fit in one line
paramShowNamesShort = ['TransID', 'Type', 'Status', 'F_Total', 'F_Proc.(%)', 'F_Proc.', \
'F_Unused', 'J_Created', 'J_Wait', 'J_Run', 'J_Done', 'J_Fail', 'J_Stalled']
dictList = []
result = self.transClient.getTransformationSummaryWeb( condDict, orderby, start, maxitems )
if not result['OK']:
self._prettyPrint( result )
return result
if result['Value']['TotalRecords'] > 0:
try:
paramNames = result['Value']['ParameterNames']
for paramValues in result['Value']['Records']:
paramShowValues = map( lambda pname: paramValues[ paramNames.index( pname ) ], paramShowNames )
showDict = dict( zip( paramShowNamesShort, paramShowValues ) )
dictList.append( showDict )
except Exception as x:
print 'Exception %s ' % str( x )
if not len( dictList ) > 0:
gLogger.error( 'No found transformations satisfying input condition' )
return S_ERROR( 'No found transformations satisfying input condition' )
else:
print self._printFormattedDictList( dictList, paramShowNamesShort, paramShowNamesShort[0], paramShowNamesShort[0] )
return S_OK( dictList )
#############################################################################
def addTransformation( self, addFiles = True, printOutput = False ):
res = self._checkCreation()
if not res['OK']:
return self._errorReport( res, 'Failed transformation sanity check' )
if printOutput:
gLogger.info( "Will attempt to create transformation with the following parameters" )
self._prettyPrint( self.paramValues )
res = self.transClient.addTransformation( self.paramValues['TransformationName'],
self.paramValues['Description'],
self.paramValues['LongDescription'],
self.paramValues['Type'],
self.paramValues['Plugin'],
self.paramValues['AgentType'],
self.paramValues['FileMask'],
transformationGroup = self.paramValues['TransformationGroup'],
groupSize = self.paramValues['GroupSize'],
inheritedFrom = self.paramValues['InheritedFrom'],
body = self.paramValues['Body'],
maxTasks = self.paramValues['MaxNumberOfTasks'],
eventsPerTask = self.paramValues['EventsPerTask'],
addFiles = addFiles )
if not res['OK']:
if printOutput:
self._prettyPrint( res )
return res
transID = res['Value']
self.exists = True
self.setTransformationID( transID )
gLogger.notice( "Created transformation %d" % transID )
for paramName, paramValue in self.paramValues.items():
if paramName not in self.paramTypes:
res = self.transClient.setTransformationParameter( transID, paramName, paramValue )
if not res['OK']:
gLogger.error( "Failed to add parameter", "%s %s" % ( paramName, res['Message'] ) )
gLogger.notice( "To add this parameter later please execute the following." )
gLogger.notice( "oTransformation = Transformation(%d)" % transID )
gLogger.notice( "oTransformation.set%s(...)" % paramName )
return S_OK( transID )
def _checkCreation( self ):
""" Few checks
"""
if self.paramValues['TransformationID']:
gLogger.info( "You are currently working with an active transformation definition." )
gLogger.info( "If you wish to create a new transformation reset the TransformationID." )
gLogger.info( "oTransformation.reset()" )
return S_ERROR()
requiredParameters = ['TransformationName', 'Description' , 'LongDescription', 'Type']
for parameter in requiredParameters:
if not self.paramValues[parameter]:
gLogger.info( "%s is not defined for this transformation. This is required..." % parameter )
self.paramValues[parameter] = raw_input( "Please enter the value of " + parameter + " " )
plugin = self.paramValues['Plugin']
if plugin:
if not plugin in self.supportedPlugins:
gLogger.info( "The selected Plugin (%s) is not known to the transformation agent." % plugin )
res = self.__promptForParameter( 'Plugin', choices = self.supportedPlugins, default = 'Standard' )
if not res['OK']:
return res
self.paramValues['Plugin'] = res['Value']
plugin = self.paramValues['Plugin']
return S_OK()
def _checkBySizePlugin( self ):
return self._checkStandardPlugin()
def _checkBySharePlugin( self ):
return self._checkStandardPlugin()
def _checkStandardPlugin( self ):
groupSize = self.paramValues['GroupSize']
if groupSize <= 0:
gLogger.info( "The GroupSize was found to be less than zero. It has been set to 1." )
res = self.setGroupSize( 1 )
if not res['OK']:
return res
return S_OK()
def _checkBroadcastPlugin( self ):
gLogger.info( "The Broadcast plugin requires the following parameters be set: %s" % ( ', '.join( ['SourceSE',
'TargetSE'] ) ) )
requiredParams = ['SourceSE', 'TargetSE']
for requiredParam in requiredParams:
if not self.paramValues.get( requiredParam ):
paramValue = raw_input( "Please enter " + requiredParam + " " )
setter = None
setterName = "set%s" % requiredParam
if hasattr( self, setterName ) and callable( getattr( self, setterName ) ):
setter = getattr( self, setterName )
if not setter:
return S_ERROR( "Unable to invoke %s, this function hasn't been implemented." % setterName )
ses = paramValue.replace( ',', ' ' ).split()
res = setter( ses )
if not res['OK']:
return res
return S_OK()
def __checkSEs( self, seList ):
res = gConfig.getSections( '/Resources/StorageElements' )
if not res['OK']:
return self._errorReport( res, 'Failed to get possible StorageElements' )
missing = set( seList ) - set( res['Value'] )
if missing:
for se in missing:
gLogger.error( "StorageElement %s is not known" % se )
return S_ERROR( "%d StorageElements not known" % len( missing ) )
return S_OK()
def __promptForParameter( self, parameter, choices = [], default = '', insert = True ):
res = promptUser( "Please enter %s" % parameter, choices = choices, default = default )
if not res['OK']:
return self._errorReport( res )
gLogger.notice( "%s will be set to '%s'" % ( parameter, res['Value'] ) )
paramValue = res['Value']
if insert:
setter = None
setterName = "set%s" % parameter
if hasattr( self, setterName ) and callable( getattr( self, setterName ) ):
setter = getattr( self, setterName )
if not setter:
return S_ERROR( "Unable to invoke %s, it isn't a member function of Transformation!" )
res = setter( paramValue )
if not res['OK']:
return res
return S_OK( paramValue )
class TaskManagerAgentBase( AgentModule ):
''' To be extended. The extension needs to:
- provide a taskManager object as data member
- provide a shifterProxy (string) as data member
- provide a transType (list of strings) as data member
'''
def __init__( self, *args, **kwargs ):
''' c'tor
'''
AgentModule.__init__( self, *args, **kwargs )
self.taskManager = None
self.shifterProxy = ''
self.transClient = TransformationClient()
self.transType = []
#############################################################################
def initialize( self ):
''' agent initialization
'''
if not self.taskManager:
return S_ERROR( 'No task manager provided!' )
if not self.shifterProxy:
return S_ERROR( 'No shifter proxy provided!' )
self.am_setOption( 'shifterProxy', self.shifterProxy )
if not self.transType:
return S_ERROR( 'No transformation types to look for!' )
gLogger.info( "Looking for %s" % self.transType )
gMonitor.registerActivity( "SubmittedTasks", "Automatically submitted tasks", "Transformation Monitoring", "Tasks",
gMonitor.OP_ACUM )
return S_OK()
#############################################################################
def execute( self ):
''' The TaskManagerBase execution method.
'''
# Determine whether the task status is to be monitored and updated
enableTaskMonitor = self.am_getOption( 'MonitorTasks', '' )
if not enableTaskMonitor:
gLogger.info( "execute: Monitoring of tasks is disabled." )
gLogger.info( "execute: To enable create the 'MonitorTasks' option" )
else:
res = self.updateTaskStatus()
if not res['OK']:
gLogger.warn( 'execute: Failed to update task states', res['Message'] )
# Determine whether the task files status is to be monitored and updated
enableFileMonitor = self.am_getOption( 'MonitorFiles', '' )
if not enableFileMonitor:
gLogger.info( "execute: Monitoring of files is disabled." )
gLogger.info( "execute: To enable create the 'MonitorFiles' option" )
else:
res = self.updateFileStatus()
if not res['OK']:
gLogger.warn( 'execute: Failed to update file states', res['Message'] )
# Determine whether the checking of reserved tasks is to be performed
enableCheckReserved = self.am_getOption( 'CheckReserved', '' )
if not enableCheckReserved:
gLogger.info( "execute: Checking of reserved tasks is disabled." )
gLogger.info( "execute: To enable create the 'CheckReserved' option" )
else:
res = self.checkReservedTasks()
if not res['OK']:
gLogger.warn( 'execute: Failed to checked reserved tasks', res['Message'] )
# Determine whether the submission of tasks is to be executed
enableSubmission = self.am_getOption( 'SubmitTasks', '' )
if not enableSubmission:
gLogger.info( "execute: Submission of tasks is disabled." )
gLogger.info( "execute: To enable create the 'SubmitTasks' option" )
else:
res = self.submitTasks()
if not res['OK']:
gLogger.warn( 'execute: Failed to submit created tasks', res['Message'] )
return S_OK()
def _selectTransformations( self, transType = [], status = ['Active', 'Completing'], agentType = ['Automatic'] ):
''' get the transformations
'''
selectCond = {}
if status:
selectCond['Status'] = status
if transType:
selectCond['Type'] = transType
if agentType:
selectCond['AgentType'] = agentType
res = self.transClient.getTransformations( condDict = selectCond )
if not res['OK']:
gLogger.error( "_selectTransformations: Failed to get transformations for selection.", res['Message'] )
elif not res['Value']:
gLogger.info( "_selectTransformations: No transformations found for selection." )
else:
gLogger.info( "_selectTransformations: Obtained %d transformations for selection" % len( res['Value'] ) )
return res
def updateTaskStatus( self ):
''' Updates the task status
'''
gLogger.info( "updateTaskStatus: Updating the Status of tasks" )
# Get the transformations to be updated
status = self.am_getOption( 'UpdateTasksStatus', ['Active', 'Completing', 'Stopped'] )
res = self._selectTransformations( transType = self.transType, status = status, agentType = [] )
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
# Get the tasks which are in a UPDATE state
updateStatus = self.am_getOption( 'TaskUpdateStatus', ['Checking', 'Deleted', 'Killed', 'Staging', 'Stalled',
'Matched', 'Rescheduled', 'Completed', 'Submitted',
'Received', 'Waiting', 'Running'] )
condDict = {"TransformationID":transID, "ExternalStatus":updateStatus}
timeStamp = str( datetime.datetime.utcnow() - datetime.timedelta( minutes = 10 ) )
res = self.transClient.getTransformationTasks( condDict = condDict,
older = timeStamp,
timeStamp = 'LastUpdateTime' )
if not res['OK']:
gLogger.error( "updateTaskStatus: Failed to get tasks to update for transformation", "%s %s" % ( transID,
res['Message'] ) )
continue
if not res['Value']:
gLogger.verbose( "updateTaskStatus: No tasks found to update for transformation %s" % transID )
continue
res = self.taskManager.getSubmittedTaskStatus( res['Value'] )
if not res['OK']:
gLogger.error( "updateTaskStatus: Failed to get updated task statuses for transformation", "%s %s" % ( transID,
res['Message'] ) )
continue
statusDict = res['Value']
for status in sortList( statusDict.keys() ):
taskIDs = statusDict[status]
gLogger.info( "updateTaskStatus: Updating %d task(s) from transformation %d to %s" % ( len( taskIDs ),
transID, status ) )
res = self.transClient.setTaskStatus( transID, taskIDs, status )
if not res['OK']:
gLogger.error( "updateTaskStatus: Failed to update task status for transformation", "%s %s" % ( transID,
res['Message'] ) )
gLogger.info( "updateTaskStatus: Transformation task status update complete" )
return S_OK()
def updateFileStatus( self ):
''' Update the files status
'''
gLogger.info( "updateFileStatus: Updating Status of task files" )
#Get the transformations to be updated
status = self.am_getOption( 'UpdateFilesStatus', ['Active', 'Completing', 'Stopped'] )
res = self._selectTransformations( transType = self.transType, status = status, agentType = [] )
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
timeStamp = str( datetime.datetime.utcnow() - datetime.timedelta( minutes = 10 ) )
condDict = {'TransformationID' : transID, 'Status' : ['Assigned']}
res = self.transClient.getTransformationFiles( condDict = condDict, older = timeStamp, timeStamp = 'LastUpdate' )
if not res['OK']:
gLogger.error( "updateFileStatus: Failed to get transformation files to update.", res['Message'] )
continue
if not res['Value']:
gLogger.info( "updateFileStatus: No files to be updated for transformation %s." % transID )
continue
res = self.taskManager.getSubmittedFileStatus( res['Value'] )
if not res['OK']:
gLogger.error( "updateFileStatus: Failed to get updated file statuses for transformation", "%s %s" % ( transID,
res['Message'] ) )
continue
statusDict = res['Value']
if not statusDict:
gLogger.info( "updateFileStatus: No file statuses to be updated for transformation %s." % transID )
continue
fileReport = FileReport( server = self.transClient.getServer() )
for lfn, status in statusDict.items():
fileReport.setFileStatus( int( transID ), lfn, status )
res = fileReport.commit()
if not res['OK']:
gLogger.error( "updateFileStatus: Failed to update file status for transformation", "%s %s" % ( transID,
res['Message'] ) )
else:
for status, update in res['Value'].items():
gLogger.info( "updateFileStatus: Updated %s files for %s to %s." % ( update, transID, status ) )
gLogger.info( "updateFileStatus: Transformation file status update complete" )
return S_OK()
def checkReservedTasks( self ):
gLogger.info( "checkReservedTasks: Checking Reserved tasks" )
# Get the transformations which should be checked
status = self.am_getOption( 'CheckReservedStatus', ['Active', 'Completing', 'Stopped'] )
res = self._selectTransformations( transType = self.transType, status = status, agentType = [] )
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
# Select the tasks which have been in Reserved status for more than 1 hour for selected transformations
condDict = {"TransformationID":transID, "ExternalStatus":'Reserved'}
time_stamp_older = str( datetime.datetime.utcnow() - datetime.timedelta( hours = 1 ) )
time_stamp_newer = str( datetime.datetime.utcnow() - datetime.timedelta( days = 7 ) )
res = self.transClient.getTransformationTasks( condDict = condDict, older = time_stamp_older,
newer = time_stamp_newer )
if not res['OK']:
gLogger.error( "checkReservedTasks: Failed to get Reserved tasks for transformation", "%s %s" % ( transID,
res['Message'] ) )
continue
if not res['Value']:
gLogger.verbose( "checkReservedTasks: No Reserved tasks found for transformation %s" % transID )
continue
res = self.taskManager.updateTransformationReservedTasks( res['Value'] )
if not res['OK']:
gLogger.info( "checkReservedTasks: No Reserved tasks found for transformation %s" % transID )
continue
noTasks = res['Value']['NoTasks']
taskNameIDs = res['Value']['TaskNameIDs']
# For the tasks with no associated request found re-set the status of the task in the transformationDB
for taskName in noTasks:
transID, taskID = taskName.split( '_' )
gLogger.info( "checkReservedTasks: Resetting status of %s to Created as no associated task found" % ( taskName ) )
res = self.transClient.setTaskStatus( int( transID ), int( taskID ), 'Created' )
if not res['OK']:
gLogger.warn( "checkReservedTasks: Failed to update task status and ID after recovery", "%s %s" % ( taskName,
res['Message'] ) )
# For the tasks for which an associated request was found update the task details in the transformationDB
for taskName, extTaskID in taskNameIDs.items():
transID, taskID = taskName.split( '_' )
gLogger.info( "checkReservedTasks: Resetting status of %s to Created with ID %s" % ( taskName, extTaskID ) )
res = self.transClient.setTaskStatusAndWmsID( int( transID ), int( taskID ), 'Submitted', str( extTaskID ) )
if not res['OK']:
gLogger.warn( "checkReservedTasks: Failed to update task status and ID after recovery", "%s %s" % ( taskName,
res['Message'] ) )
gLogger.info( "checkReservedTasks: Updating of reserved tasks complete" )
return S_OK()
def submitTasks( self ):
""" Submit the tasks to an external system, using the taskManager provided
"""
gLogger.info( "submitTasks: Submitting tasks for transformations" )
res = getProxyInfo( False, False )
if not res['OK']:
gLogger.error( "submitTasks: Failed to determine credentials for submission", res['Message'] )
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
gLogger.info( "submitTasks: Tasks will be submitted with the credentials %s:%s" % ( owner, ownerGroup ) )
# Get the transformations which should be submitted
tasksPerLoop = self.am_getOption( 'TasksPerLoop', 50 )
status = self.am_getOption( 'SubmitStatus', ['Active', 'Completing'] )
res = self._selectTransformations( transType = self.transType, status = status )
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
transBody = transformation['Body']
res = self.transClient.getTasksToSubmit( transID, tasksPerLoop )
if not res['OK']:
gLogger.error( "submitTasks: Failed to obtain tasks for transformation", "%s %s" % ( transID, res['Message'] ) )
continue
tasks = res['Value']['JobDictionary']
if not tasks:
gLogger.verbose( "submitTasks: No tasks found for submission for transformation %s" % transID )
continue
gLogger.info( "submitTasks: Obtained %d tasks for submission for transformation %s" % ( len( tasks ), transID ) )
res = self.taskManager.prepareTransformationTasks( transBody, tasks, owner, ownerGroup )
if not res['OK']:
gLogger.error( "submitTasks: Failed to prepare tasks for transformation", "%s %s" % ( transID,
res['Message'] ) )
continue
res = self.taskManager.submitTransformationTasks( res['Value'] )
if not res['OK']:
gLogger.error( "submitTasks: Failed to submit prepared tasks for transformation", "%s %s" % ( transID,
res['Message'] ) )
continue
res = self.taskManager.updateDBAfterTaskSubmission( res['Value'] )
if not res['OK']:
gLogger.error( "submitTasks: Failed to update DB after task submission for transformation", "%s %s" % ( transID,
res['Message'] ) )
continue
gLogger.info( "submitTasks: Submission of transformation tasks complete" )
return S_OK()
class Transformation(API):
#############################################################################
def __init__(self, transID=0, transClient=None):
""" c'tor
"""
super(Transformation, self).__init__()
self.paramTypes = {
"TransformationID": [types.IntType, types.LongType],
"TransformationName": types.StringTypes,
"Status": types.StringTypes,
"Description": types.StringTypes,
"LongDescription": types.StringTypes,
"Type": types.StringTypes,
"Plugin": types.StringTypes,
"AgentType": types.StringTypes,
"FileMask": types.StringTypes,
"TransformationGroup": types.StringTypes,
"GroupSize": [types.IntType, types.LongType, types.FloatType],
"InheritedFrom": [types.IntType, types.LongType],
"Body": types.StringTypes,
"MaxNumberOfTasks": [types.IntType, types.LongType],
"EventsPerTask": [types.IntType, types.LongType],
}
self.paramValues = {
"TransformationID": 0,
"TransformationName": "",
"Status": "New",
"Description": "",
"LongDescription": "",
"Type": "",
"Plugin": "Standard",
"AgentType": "Manual",
"FileMask": "",
"TransformationGroup": "General",
"GroupSize": 1,
"InheritedFrom": 0,
"Body": "",
"MaxNumberOfTasks": 0,
"EventsPerTask": 0,
}
self.ops = Operations()
self.supportedPlugins = self.ops.getValue(
"Transformations/AllowedPlugins", ["Broadcast", "Standard", "BySize", "ByShare"]
)
if not transClient:
self.transClient = TransformationClient()
else:
self.transClient = transClient
self.serverURL = self.transClient.getServer()
self.exists = False
if transID:
self.paramValues["TransformationID"] = transID
res = self.getTransformation()
if res["OK"]:
self.exists = True
elif res["Message"] == "Transformation does not exist":
raise AttributeError, "TransformationID %d does not exist" % transID
else:
self.paramValues["TransformationID"] = 0
gLogger.fatal(
"Failed to get transformation from database", "%s @ %s" % (transID, self.transClient.serverURL)
)
def setServer(self, server):
self.serverURL = server
self.transClient.setServer(self.serverURL)
def getServer(self):
return self.serverURL
def reset(self, transID=0):
self.__init__(transID)
self.transClient.setServer(self.serverURL)
return S_OK()
def setTargetSE(self, seList):
return self.__setSE("TargetSE", seList)
def setSourceSE(self, seList):
return self.__setSE("SourceSE", seList)
def __setSE(self, se, seList):
if type(seList) in types.StringTypes:
try:
seList = eval(seList)
except:
seList = seList.replace(",", " ").split()
res = self.__checkSEs(seList)
if not res["OK"]:
return res
self.item_called = se
return self.__setParam(seList)
def __getattr__(self, name):
if name.find("get") == 0:
item = name[3:]
self.item_called = item
return self.__getParam
if name.find("set") == 0:
item = name[3:]
self.item_called = item
return self.__setParam
raise AttributeError, name
def __getParam(self):
if self.item_called == "Available":
return S_OK(self.paramTypes.keys())
if self.item_called == "Parameters":
return S_OK(self.paramValues)
if self.item_called in self.paramValues:
return S_OK(self.paramValues[self.item_called])
raise AttributeError, "Unknown parameter for transformation: %s" % self.item_called
def __setParam(self, value):
change = False
if self.item_called in self.paramTypes:
oldValue = self.paramValues[self.item_called]
if oldValue != value:
if type(value) in self.paramTypes[self.item_called]:
change = True
else:
raise TypeError, "%s %s %s expected one of %s" % (
self.item_called,
value,
type(value),
self.paramTypes[self.item_called],
)
if not self.item_called in self.paramTypes.keys():
if not self.paramValues.has_key(self.item_called):
change = True
else:
oldValue = self.paramValues[self.item_called]
if oldValue != value:
change = True
if not change:
gLogger.verbose("No change of parameter %s required" % self.item_called)
else:
gLogger.verbose("Parameter %s to be changed" % self.item_called)
transID = self.paramValues["TransformationID"]
if self.exists and transID:
res = self.transClient.setTransformationParameter(transID, self.item_called, value)
if not res["OK"]:
return res
self.paramValues[self.item_called] = value
return S_OK()
def getTransformation(self, printOutput=False):
transID = self.paramValues["TransformationID"]
if not transID:
gLogger.fatal("No TransformationID known")
return S_ERROR()
res = self.transClient.getTransformation(transID, extraParams=True)
if not res["OK"]:
if printOutput:
self._prettyPrint(res)
return res
transParams = res["Value"]
for paramName, paramValue in transParams.items():
setter = None
setterName = "set%s" % paramName
if hasattr(self, setterName) and callable(getattr(self, setterName)):
setter = getattr(self, setterName)
if not setterName:
gLogger.error("Unable to invoke setter %s, it isn't a member function" % setterName)
continue
setter(paramValue)
if printOutput:
gLogger.info("No printing available yet")
return S_OK(transParams)
def getTransformationLogging(self, printOutput=False):
transID = self.paramValues["TransformationID"]
if not transID:
gLogger.fatal("No TransformationID known")
return S_ERROR()
res = self.transClient.getTransformationLogging(transID)
if not res["OK"]:
if printOutput:
self._prettyPrint(res)
return res
loggingList = res["Value"]
if printOutput:
self._printFormattedDictList(
loggingList, ["Message", "MessageDate", "AuthorDN"], "MessageDate", "MessageDate"
)
return S_OK(loggingList)
def extendTransformation(self, nTasks, printOutput=False):
return self.__executeOperation("extendTransformation", nTasks, printOutput=printOutput)
def cleanTransformation(self, printOutput=False):
res = self.__executeOperation("cleanTransformation", printOutput=printOutput)
if res["OK"]:
self.paramValues["Status"] = "Cleaned"
return res
def deleteTransformation(self, printOutput=False):
res = self.__executeOperation("deleteTransformation", printOutput=printOutput)
if res["OK"]:
self.reset()
return res
def addFilesToTransformation(self, lfns, printOutput=False):
return self.__executeOperation("addFilesToTransformation", lfns, printOutput=printOutput)
def setFileStatusForTransformation(self, status, lfns, printOutput=False):
return self.__executeOperation("setFileStatusForTransformation", status, lfns, printOutput=printOutput)
def getTransformationTaskStats(self, printOutput=False):
return self.__executeOperation("getTransformationTaskStats", printOutput=printOutput)
def getTransformationStats(self, printOutput=False):
return self.__executeOperation("getTransformationStats", printOutput=printOutput)
def deleteTasks(self, taskMin, taskMax, printOutput=False):
return self.__executeOperation("deleteTasks", taskMin, taskMax, printOutput=printOutput)
def addTaskForTransformation(self, lfns=[], se="Unknown", printOutput=False):
return self.__executeOperation("addTaskForTransformation", lfns, se, printOutput=printOutput)
def setTaskStatus(self, taskID, status, printOutput=False):
return self.__executeOperation("setTaskStatus", taskID, status, printOutput=printOutput)
def __executeOperation(self, operation, *parms, **kwds):
transID = self.paramValues["TransformationID"]
if not transID:
gLogger.fatal("No TransformationID known")
return S_ERROR()
printOutput = kwds.pop("printOutput")
fcn = None
if hasattr(self.transClient, operation) and callable(getattr(self.transClient, operation)):
fcn = getattr(self.transClient, operation)
if not fcn:
return S_ERROR("Unable to invoke %s, it isn't a member funtion of TransformationClient")
res = fcn(transID, *parms, **kwds)
if printOutput:
self._prettyPrint(res)
return res
def getTransformationFiles(
self,
fileStatus=[],
lfns=[],
outputFields=[
"FileID",
"LFN",
"Status",
"TaskID",
"TargetSE",
"UsedSE",
"ErrorCount",
"InsertedTime",
"LastUpdate",
],
orderBy="FileID",
printOutput=False,
):
condDict = {"TransformationID": self.paramValues["TransformationID"]}
if fileStatus:
condDict["Status"] = fileStatus
if lfns:
condDict["LFN"] = lfns
res = self.transClient.getTransformationFiles(condDict=condDict)
if not res["OK"]:
if printOutput:
self._prettyPrint(res)
return res
if printOutput:
if not outputFields:
gLogger.info("Available fields are: %s" % res["ParameterNames"].join(" "))
elif not res["Value"]:
gLogger.info("No tasks found for selection")
else:
self._printFormattedDictList(res["Value"], outputFields, "FileID", orderBy)
return res
def getTransformationTasks(
self,
taskStatus=[],
taskIDs=[],
outputFields=[
"TransformationID",
"TaskID",
"ExternalStatus",
"ExternalID",
"TargetSE",
"CreationTime",
"LastUpdateTime",
],
orderBy="TaskID",
printOutput=False,
):
condDict = {"TransformationID": self.paramValues["TransformationID"]}
if taskStatus:
condDict["ExternalStatus"] = taskStatus
if taskIDs:
condDict["TaskID"] = taskIDs
res = self.transClient.getTransformationTasks(condDict=condDict)
if not res["OK"]:
if printOutput:
self._prettyPrint(res)
return res
if printOutput:
if not outputFields:
gLogger.info("Available fields are: %s" % res["ParameterNames"].join(" "))
elif not res["Value"]:
gLogger.info("No tasks found for selection")
else:
self._printFormattedDictList(res["Value"], outputFields, "TaskID", orderBy)
return res
#############################################################################
def getTransformations(
self,
transID=[],
transStatus=[],
outputFields=["TransformationID", "Status", "AgentType", "TransformationName", "CreationDate"],
orderBy="TransformationID",
printOutput=False,
):
condDict = {}
if transID:
condDict["TransformationID"] = transID
if transStatus:
condDict["Status"] = transStatus
res = self.transClient.getTransformations(condDict=condDict)
if not res["OK"]:
if printOutput:
self._prettyPrint(res)
return res
if printOutput:
if not outputFields:
gLogger.info("Available fields are: %s" % res["ParameterNames"].join(" "))
elif not res["Value"]:
gLogger.info("No tasks found for selection")
else:
self._printFormattedDictList(res["Value"], outputFields, "TransformationID", orderBy)
return res
#############################################################################
def addTransformation(self, addFiles=True, printOutput=False):
res = self._checkCreation()
if not res["OK"]:
return self._errorReport(res, "Failed transformation sanity check")
if printOutput:
gLogger.info("Will attempt to create transformation with the following parameters")
self._prettyPrint(self.paramValues)
res = self.transClient.addTransformation(
self.paramValues["TransformationName"],
self.paramValues["Description"],
self.paramValues["LongDescription"],
self.paramValues["Type"],
self.paramValues["Plugin"],
self.paramValues["AgentType"],
self.paramValues["FileMask"],
transformationGroup=self.paramValues["TransformationGroup"],
groupSize=self.paramValues["GroupSize"],
inheritedFrom=self.paramValues["InheritedFrom"],
body=self.paramValues["Body"],
maxTasks=self.paramValues["MaxNumberOfTasks"],
eventsPerTask=self.paramValues["EventsPerTask"],
addFiles=addFiles,
)
if not res["OK"]:
if printOutput:
self._prettyPrint(res)
return res
transID = res["Value"]
self.exists = True
self.setTransformationID(transID)
gLogger.notice("Created transformation %d" % transID)
for paramName, paramValue in self.paramValues.items():
if not self.paramTypes.has_key(paramName):
res = self.transClient.setTransformationParameter(transID, paramName, paramValue)
if not res["OK"]:
gLogger.error("Failed to add parameter", "%s %s" % (paramName, res["Message"]))
gLogger.notice("To add this parameter later please execute the following.")
gLogger.notice("oTransformation = Transformation(%d)" % transID)
gLogger.notice("oTransformation.set%s(...)" % paramName)
return S_OK(transID)
def _checkCreation(self):
""" Few checks
"""
if self.paramValues["TransformationID"]:
gLogger.info("You are currently working with an active transformation definition.")
gLogger.info("If you wish to create a new transformation reset the TransformationID.")
gLogger.info("oTransformation.reset()")
return S_ERROR()
requiredParameters = ["TransformationName", "Description", "LongDescription", "Type"]
for parameter in requiredParameters:
if not self.paramValues[parameter]:
gLogger.info("%s is not defined for this transformation. This is required..." % parameter)
self.paramValues[parameter] = raw_input("Please enter the value of " + parameter + " ")
plugin = self.paramValues["Plugin"]
if plugin:
if not plugin in self.supportedPlugins:
gLogger.info("The selected Plugin (%s) is not known to the transformation agent." % plugin)
res = self.__promptForParameter("Plugin", choices=self.supportedPlugins, default="Standard")
if not res["OK"]:
return res
self.paramValues["Plugin"] = res["Value"]
plugin = self.paramValues["Plugin"]
return S_OK()
def _checkBySizePlugin(self):
return self._checkStandardPlugin()
def _checkBySharePlugin(self):
return self._checkStandardPlugin()
def _checkStandardPlugin(self):
groupSize = self.paramValues["GroupSize"]
if groupSize <= 0:
gLogger.info("The GroupSize was found to be less than zero. It has been set to 1.")
res = self.setGroupSize(1)
if not res["OK"]:
return res
return S_OK()
def _checkBroadcastPlugin(self):
gLogger.info(
"The Broadcast plugin requires the following parameters be set: %s" % (", ".join(["SourceSE", "TargetSE"]))
)
requiredParams = ["SourceSE", "TargetSE"]
for requiredParam in requiredParams:
if (not self.paramValues.has_key(requiredParam)) or (not self.paramValues[requiredParam]):
paramValue = raw_input("Please enter " + requiredParam + " ")
setter = None
setterName = "set%s" % requiredParam
if hasattr(self, setterName) and callable(getattr(self, setterName)):
setter = getattr(self, setterName)
if not setter:
return S_ERROR("Unable to invoke %s, this function hasn't been implemented." % setterName)
ses = paramValue.replace(",", " ").split()
res = setter(ses)
if not res["OK"]:
return res
return S_OK()
def __checkSEs(self, seList):
res = gConfig.getSections("/Resources/StorageElements")
if not res["OK"]:
return self._errorReport(res, "Failed to get possible StorageElements")
missing = []
for se in seList:
if not se in res["Value"]:
gLogger.error("StorageElement %s is not known" % se)
missing.append(se)
if missing:
return S_ERROR("%d StorageElements not known" % len(missing))
return S_OK()
def __promptForParameter(self, parameter, choices=[], default="", insert=True):
res = promptUser("Please enter %s" % parameter, choices=choices, default=default)
if not res["OK"]:
return self._errorReport(res)
gLogger.notice("%s will be set to '%s'" % (parameter, res["Value"]))
paramValue = res["Value"]
if insert:
setter = None
setterName = "set%s" % parameter
if hasattr(self, setterName) and callable(getattr(self, setterName)):
setter = getattr(self, setterName)
if not setter:
return S_ERROR("Unable to invoke %s, it isn't a member function of Transformation!")
res = setter(paramValue)
if not res["OK"]:
return res
return S_OK(paramValue)
class Transformation(API):
#############################################################################
def __init__(self, transID=0, transClient=None):
"""c'tor"""
super(Transformation, self).__init__()
self.paramTypes = {
"TransformationID": six.integer_types,
"TransformationName": six.string_types,
"Status": six.string_types,
"Description": six.string_types,
"LongDescription": six.string_types,
"Type": six.string_types,
"Plugin": six.string_types,
"AgentType": six.string_types,
"FileMask": six.string_types,
"TransformationGroup": six.string_types,
"GroupSize": six.integer_types + (float, ),
"InheritedFrom": six.integer_types,
"Body": six.string_types,
"MaxNumberOfTasks": six.integer_types,
"EventsPerTask": six.integer_types,
}
self.paramValues = {
"TransformationID": 0,
"TransformationName": "",
"Status": "New",
"Description": "",
"LongDescription": "",
"Type": "",
"Plugin": "Standard",
"AgentType": "Manual",
"FileMask": "",
"TransformationGroup": "General",
"GroupSize": 1,
"InheritedFrom": 0,
"Body": "",
"MaxNumberOfTasks": 0,
"EventsPerTask": 0,
}
# the metaquery parameters are neither part of the transformation parameters nor the additional parameters, so
# special treatment is necessary
self.inputMetaQuery = None
self.outputMetaQuery = None
self.ops = Operations()
self.supportedPlugins = self.ops.getValue(
"Transformations/AllowedPlugins",
["Broadcast", "Standard", "BySize", "ByShare"])
if not transClient:
self.transClient = TransformationClient()
else:
self.transClient = transClient
self.serverURL = self.transClient.getServer()
self.exists = False
if transID:
self.paramValues["TransformationID"] = transID
res = self.getTransformation()
if res["OK"]:
self.exists = True
elif res["Message"] == "Transformation does not exist":
raise AttributeError("TransformationID %d does not exist" %
transID)
else:
self.paramValues["TransformationID"] = 0
gLogger.fatal(
"Failed to get transformation from database",
"%s @ %s" % (transID, self.transClient.serverURL))
def getServer(self):
return self.serverURL
def reset(self, transID=0):
self.__init__(transID)
self.transClient.setServer(self.serverURL)
return S_OK()
def setTargetSE(self, seList):
return self.__setSE("TargetSE", seList)
def setSourceSE(self, seList):
return self.__setSE("SourceSE", seList)
def setBody(self, body):
"""check that the body is a string, or using the proper syntax for multiple operations,
or is a BodyPlugin object
:param body: transformation body, for example
.. code :: python
body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }),
( "RemoveReplica", { "TargetSE":"FOO-SRM" } ),
]
:type body: string or list of tuples (or lists) of string and dictionaries or a Body plugin (:py:class:`DIRAC.TransformationSystem.Client.BodyPlugin.BaseBody.BaseBody`)
:raises TypeError: If the structure is not as expected
:raises ValueError: If unknown attribute for the :class:`~DIRAC.RequestManagementSystem.Client.Operation.Operation`
is used
:returns: S_OK, S_ERROR
"""
self.item_called = "Body"
# Simple single operation body case
if isinstance(body, six.string_types):
return self.__setParam(body)
# BodyPlugin case
elif isinstance(body, BaseBody):
return self.__setParam(encode(body))
if not isinstance(body, (list, tuple)):
raise TypeError("Expected list or string, but %r is %s" %
(body, type(body)))
# MultiOperation body case
for tup in body:
if not isinstance(tup, (tuple, list)):
raise TypeError("Expected tuple or list, but %r is %s" %
(tup, type(tup)))
if len(tup) != 2:
raise TypeError("Expected 2-tuple, but %r is length %d" %
(tup, len(tup)))
if not isinstance(tup[0], six.string_types):
raise TypeError(
"Expected string, but first entry in tuple %r is %s" %
(tup, type(tup[0])))
if not isinstance(tup[1], dict):
raise TypeError(
"Expected dictionary, but second entry in tuple %r is %s" %
(tup, type(tup[0])))
for par, val in tup[1].items():
if not isinstance(par, six.string_types):
raise TypeError(
"Expected string, but key in dictionary %r is %s" %
(par, type(par)))
if par not in Operation.ATTRIBUTE_NAMES:
raise ValueError("Unknown attribute for Operation: %s" %
par)
if not isinstance(
val, six.string_types + six.integer_types +
(float, list, tuple, dict)):
raise TypeError("Cannot encode %r, in json" % (val))
return self.__setParam(json.dumps(body))
def setInputMetaQuery(self, query):
"""Set the input meta query.
:param dict query: dictionary to use for input meta query
"""
self.inputMetaQuery = query
return S_OK()
def setOutputMetaQuery(self, query):
"""Set the output meta query.
:param dict query: dictionary to use for output meta query
"""
self.outputMetaQuery = query
return S_OK()
def __setSE(self, seParam, seList):
if isinstance(seList, six.string_types):
try:
seList = eval(seList)
except Exception:
seList = seList.split(",")
elif isinstance(seList, (list, dict, tuple)):
seList = list(seList)
else:
return S_ERROR("Bad argument type")
res = self.__checkSEs(seList)
if not res["OK"]:
return res
self.item_called = seParam
return self.__setParam(seList)
def __getattr__(self, name):
if name.find("get") == 0:
item = name[3:]
self.item_called = item
return self.__getParam
if name.find("set") == 0:
item = name[3:]
self.item_called = item
return self.__setParam
raise AttributeError(name)
def __getParam(self):
if self.item_called == "Available":
return S_OK(list(self.paramTypes))
if self.item_called == "Parameters":
return S_OK(self.paramValues)
if self.item_called in self.paramValues:
return S_OK(self.paramValues[self.item_called])
raise AttributeError("Unknown parameter for transformation: %s" %
self.item_called)
def __setParam(self, value):
change = False
if self.item_called in self.paramTypes:
if self.paramValues[self.item_called] != value:
if isinstance(value, self.paramTypes[self.item_called]):
change = True
else:
raise TypeError("%s %s %s expected one of %s" %
(self.item_called, value, type(value),
self.paramTypes[self.item_called]))
else:
if self.item_called not in self.paramValues:
change = True
else:
if self.paramValues[self.item_called] != value:
change = True
if not change:
gLogger.verbose("No change of parameter %s required" %
self.item_called)
else:
gLogger.verbose("Parameter %s to be changed" % self.item_called)
transID = self.paramValues["TransformationID"]
if self.exists and transID:
res = self.transClient.setTransformationParameter(
transID, self.item_called, value)
if not res["OK"]:
return res
self.paramValues[self.item_called] = value
return S_OK()
def getTransformation(self, printOutput=False):
transID = self.paramValues["TransformationID"]
if not transID:
gLogger.fatal("No TransformationID known")
return S_ERROR()
res = self.transClient.getTransformation(transID, extraParams=True)
if not res["OK"]:
if printOutput:
self._prettyPrint(res)
return res
transParams = res["Value"]
for paramName, paramValue in transParams.items():
setter = None
setterName = "set%s" % paramName
if hasattr(self, setterName) and callable(getattr(
self, setterName)):
setter = getattr(self, setterName)
if not setterName:
gLogger.error(
"Unable to invoke setter %s, it isn't a member function" %
setterName)
continue
setter(paramValue)
if printOutput:
gLogger.info("No printing available yet")
return S_OK(transParams)
def getTransformationLogging(self, printOutput=False):
transID = self.paramValues["TransformationID"]
if not transID:
gLogger.fatal("No TransformationID known")
return S_ERROR()
res = self.transClient.getTransformationLogging(transID)
if not res["OK"]:
if printOutput:
self._prettyPrint(res)
return res
loggingList = res["Value"]
if printOutput:
self._printFormattedDictList(
loggingList, ["Message", "MessageDate", "AuthorDN"],
"MessageDate", "MessageDate")
return S_OK(loggingList)
def extendTransformation(self, nTasks, printOutput=False):
return self.__executeOperation("extendTransformation",
nTasks,
printOutput=printOutput)
def cleanTransformation(self, printOutput=False):
res = self.__executeOperation("cleanTransformation",
printOutput=printOutput)
if res["OK"]:
self.paramValues["Status"] = "Cleaned"
return res
def deleteTransformation(self, printOutput=False):
res = self.__executeOperation("deleteTransformation",
printOutput=printOutput)
if res["OK"]:
self.reset()
return res
def addFilesToTransformation(self, lfns, printOutput=False):
return self.__executeOperation("addFilesToTransformation",
lfns,
printOutput=printOutput)
def setFileStatusForTransformation(self, status, lfns, printOutput=False):
return self.__executeOperation("setFileStatusForTransformation",
status,
lfns,
printOutput=printOutput)
def getTransformationTaskStats(self, printOutput=False):
return self.__executeOperation("getTransformationTaskStats",
printOutput=printOutput)
def getTransformationStats(self, printOutput=False):
return self.__executeOperation("getTransformationStats",
printOutput=printOutput)
def deleteTasks(self, taskMin, taskMax, printOutput=False):
return self.__executeOperation("deleteTasks",
taskMin,
taskMax,
printOutput=printOutput)
def addTaskForTransformation(self,
lfns=[],
se="Unknown",
printOutput=False):
return self.__executeOperation("addTaskForTransformation",
lfns,
se,
printOutput=printOutput)
def setTaskStatus(self, taskID, status, printOutput=False):
return self.__executeOperation("setTaskStatus",
taskID,
status,
printOutput=printOutput)
def __executeOperation(self, operation, *parms, **kwds):
transID = self.paramValues["TransformationID"]
if not transID:
gLogger.fatal("No TransformationID known")
return S_ERROR()
printOutput = kwds.pop("printOutput")
fcn = None
if hasattr(self.transClient, operation) and callable(
getattr(self.transClient, operation)):
fcn = getattr(self.transClient, operation)
if not fcn:
return S_ERROR(
"Unable to invoke %s, it isn't a member funtion of TransformationClient"
)
res = fcn(transID, *parms, **kwds)
if printOutput:
self._prettyPrint(res)
return res
def getTransformationFiles(
self,
fileStatus=[],
lfns=[],
outputFields=[
"FileID",
"LFN",
"Status",
"TaskID",
"TargetSE",
"UsedSE",
"ErrorCount",
"InsertedTime",
"LastUpdate",
],
orderBy="FileID",
printOutput=False,
):
condDict = {"TransformationID": self.paramValues["TransformationID"]}
if fileStatus:
condDict["Status"] = fileStatus
if lfns:
condDict["LFN"] = lfns
res = self.transClient.getTransformationFiles(condDict=condDict)
if not res["OK"]:
if printOutput:
self._prettyPrint(res)
return res
if printOutput:
if not outputFields:
gLogger.info("Available fields are: %s" %
res["ParameterNames"].join(" "))
elif not res["Value"]:
gLogger.info("No tasks found for selection")
else:
self._printFormattedDictList(res["Value"], outputFields,
"FileID", orderBy)
return res
def getTransformationTasks(
self,
taskStatus=[],
taskIDs=[],
outputFields=[
"TransformationID",
"TaskID",
"ExternalStatus",
"ExternalID",
"TargetSE",
"CreationTime",
"LastUpdateTime",
],
orderBy="TaskID",
printOutput=False,
):
condDict = {"TransformationID": self.paramValues["TransformationID"]}
if taskStatus:
condDict["ExternalStatus"] = taskStatus
if taskIDs:
condDict["TaskID"] = taskIDs
res = self.transClient.getTransformationTasks(condDict=condDict)
if not res["OK"]:
if printOutput:
self._prettyPrint(res)
return res
if printOutput:
if not outputFields:
gLogger.info("Available fields are: %s" %
res["ParameterNames"].join(" "))
elif not res["Value"]:
gLogger.info("No tasks found for selection")
else:
self._printFormattedDictList(res["Value"], outputFields,
"TaskID", orderBy)
return res
#############################################################################
def getTransformations(
self,
transID=[],
transStatus=[],
outputFields=[
"TransformationID", "Status", "AgentType", "TransformationName",
"CreationDate"
],
orderBy="TransformationID",
printOutput=False,
):
condDict = {}
if transID:
condDict["TransformationID"] = transID
if transStatus:
condDict["Status"] = transStatus
res = self.transClient.getTransformations(condDict=condDict)
if not res["OK"]:
if printOutput:
self._prettyPrint(res)
return res
if printOutput:
if not outputFields:
gLogger.info("Available fields are: %s" %
res["ParameterNames"].join(" "))
elif not res["Value"]:
gLogger.info("No tasks found for selection")
else:
self._printFormattedDictList(res["Value"], outputFields,
"TransformationID", orderBy)
return res
#############################################################################
def getAuthorDNfromProxy(self):
"""gets the AuthorDN and username of the transformation from the uploaded proxy"""
username = ""
author = ""
res = getProxyInfo()
if res["OK"]:
author = res["Value"]["identity"]
username = res["Value"]["username"]
else:
gLogger.error("Unable to get uploaded proxy Info %s " %
res["Message"])
return S_ERROR(res["Message"])
res = {"username": username, "authorDN": author}
return S_OK(res)
#############################################################################
def getTransformationsByUser(
self,
authorDN="",
userName="",
transID=[],
transStatus=[],
outputFields=[
"TransformationID", "Status", "AgentType", "TransformationName",
"CreationDate", "AuthorDN"
],
orderBy="TransformationID",
printOutput=False,
):
condDict = {}
if authorDN == "":
res = self.getAuthorDNfromProxy()
if not res["OK"]:
gLogger.error(res["Message"])
return S_ERROR(res["Message"])
else:
foundUserName = res["Value"]["username"]
foundAuthor = res["Value"]["authorDN"]
# If the username whom created the uploaded proxy is different than the provided username report error and exit
if not (userName == "" or userName == foundUserName):
gLogger.error(
"Couldn't resolve the authorDN for user '%s' from the uploaded proxy (proxy created by '%s')"
% (userName, foundUserName))
return S_ERROR(
"Couldn't resolve the authorDN for user '%s' from the uploaded proxy (proxy created by '%s')"
% (userName, foundUserName))
userName = foundUserName
authorDN = foundAuthor
gLogger.info(
"Will list transformations created by user '%s' with status '%s'"
% (userName, ", ".join(transStatus)))
else:
gLogger.info(
"Will list transformations created by '%s' with status '%s'" %
(authorDN, ", ".join(transStatus)))
condDict["AuthorDN"] = authorDN
if transID:
condDict["TransformationID"] = transID
if transStatus:
condDict["Status"] = transStatus
res = self.transClient.getTransformations(condDict=condDict)
if not res["OK"]:
if printOutput:
self._prettyPrint(res)
return res
if printOutput:
if not outputFields:
gLogger.info("Available fields are: %s" %
res["ParameterNames"].join(" "))
elif not res["Value"]:
gLogger.info("No tasks found for selection")
else:
self._printFormattedDictList(res["Value"], outputFields,
"TransformationID", orderBy)
return res
#############################################################################
def getSummaryTransformations(self, transID=[]):
"""Show the summary for a list of Transformations
Fields starting with 'F' ('J') refers to files (jobs).
Proc. stand for processed.
"""
condDict = {"TransformationID": transID}
orderby = []
start = 0
maxitems = len(transID)
paramShowNames = [
"TransformationID",
"Type",
"Status",
"Files_Total",
"Files_PercentProcessed",
"Files_Processed",
"Files_Unused",
"Jobs_TotalCreated",
"Jobs_Waiting",
"Jobs_Running",
"Jobs_Done",
"Jobs_Failed",
"Jobs_Stalled",
]
# Below, the header used for each field in the printing: short to fit in one line
paramShowNamesShort = [
"TransID",
"Type",
"Status",
"F_Total",
"F_Proc.(%)",
"F_Proc.",
"F_Unused",
"J_Created",
"J_Wait",
"J_Run",
"J_Done",
"J_Fail",
"J_Stalled",
]
dictList = []
result = self.transClient.getTransformationSummaryWeb(
condDict, orderby, start, maxitems)
if not result["OK"]:
self._prettyPrint(result)
return result
if result["Value"]["TotalRecords"] > 0:
try:
paramNames = result["Value"]["ParameterNames"]
for paramValues in result["Value"]["Records"]:
paramShowValues = map(
lambda pname: paramValues[paramNames.index(pname)],
paramShowNames)
showDict = dict(zip(paramShowNamesShort, paramShowValues))
dictList.append(showDict)
except Exception as x:
print("Exception %s " % str(x))
if not len(dictList) > 0:
gLogger.error(
"No found transformations satisfying input condition")
return S_ERROR(
"No found transformations satisfying input condition")
else:
print(
self._printFormattedDictList(dictList, paramShowNamesShort,
paramShowNamesShort[0],
paramShowNamesShort[0]))
return S_OK(dictList)
#############################################################################
def addTransformation(self, addFiles=True, printOutput=False):
"""Add transformation to the transformation system.
Sets all parameters currently assigned to the transformation.
:param bool addFiles: if True, immediately perform input data query
:param bool printOutput: if True, print information about transformation
"""
res = self._checkCreation()
if not res["OK"]:
return self._errorReport(res, "Failed transformation sanity check")
if printOutput:
gLogger.info(
"Will attempt to create transformation with the following parameters"
)
self._prettyPrint(self.paramValues)
res = self.transClient.addTransformation(
self.paramValues["TransformationName"],
self.paramValues["Description"],
self.paramValues["LongDescription"],
self.paramValues["Type"],
self.paramValues["Plugin"],
self.paramValues["AgentType"],
self.paramValues["FileMask"],
transformationGroup=self.paramValues["TransformationGroup"],
groupSize=self.paramValues["GroupSize"],
inheritedFrom=self.paramValues["InheritedFrom"],
body=self.paramValues["Body"],
maxTasks=self.paramValues["MaxNumberOfTasks"],
eventsPerTask=self.paramValues["EventsPerTask"],
addFiles=addFiles,
inputMetaQuery=self.inputMetaQuery,
outputMetaQuery=self.outputMetaQuery,
)
if not res["OK"]:
if printOutput:
self._prettyPrint(res)
return res
transID = res["Value"]
self.exists = True
self.setTransformationID(transID)
gLogger.notice("Created transformation %d" % transID)
for paramName, paramValue in self.paramValues.items():
if paramName not in self.paramTypes:
res = self.transClient.setTransformationParameter(
transID, paramName, paramValue)
if not res["OK"]:
gLogger.error("Failed to add parameter",
"%s %s" % (paramName, res["Message"]))
gLogger.notice(
"To add this parameter later please execute the following."
)
gLogger.notice("oTransformation = Transformation(%d)" %
transID)
gLogger.notice("oTransformation.set%s(...)" % paramName)
return S_OK(transID)
def _checkCreation(self):
"""Few checks"""
if self.paramValues["TransformationID"]:
gLogger.info(
"You are currently working with an active transformation definition."
)
gLogger.info(
"If you wish to create a new transformation reset the TransformationID."
)
gLogger.info("oTransformation.reset()")
return S_ERROR()
requiredParameters = [
"TransformationName", "Description", "LongDescription", "Type"
]
for parameter in requiredParameters:
if not self.paramValues[parameter]:
gLogger.info(
"%s is not defined for this transformation. This is required..."
% parameter)
self.paramValues[parameter] = six.moves.input(
"Please enter the value of " + parameter + " ")
plugin = self.paramValues["Plugin"]
if plugin:
if plugin not in self.supportedPlugins:
gLogger.info(
"The selected Plugin (%s) is not known to the transformation agent."
% plugin)
res = self.__promptForParameter("Plugin",
choices=self.supportedPlugins,
default="Standard")
if not res["OK"]:
return res
self.paramValues["Plugin"] = res["Value"]
plugin = self.paramValues["Plugin"]
return S_OK()
def _checkBySizePlugin(self):
return self._checkStandardPlugin()
def _checkBySharePlugin(self):
return self._checkStandardPlugin()
def _checkStandardPlugin(self):
groupSize = self.paramValues["GroupSize"]
if groupSize <= 0:
gLogger.info(
"The GroupSize was found to be less than zero. It has been set to 1."
)
res = self.setGroupSize(1)
if not res["OK"]:
return res
return S_OK()
def _checkBroadcastPlugin(self):
gLogger.info(
"The Broadcast plugin requires the following parameters be set: %s"
% (", ".join(["SourceSE", "TargetSE"])))
requiredParams = ["SourceSE", "TargetSE"]
for requiredParam in requiredParams:
if not self.paramValues.get(requiredParam):
paramValue = six.moves.input("Please enter " + requiredParam +
" ")
setter = None
setterName = "set%s" % requiredParam
if hasattr(self, setterName) and callable(
getattr(self, setterName)):
setter = getattr(self, setterName)
if not setter:
return S_ERROR(
"Unable to invoke %s, this function hasn't been implemented."
% setterName)
ses = paramValue.replace(",", " ").split()
res = setter(ses)
if not res["OK"]:
return res
return S_OK()
def __checkSEs(self, seList):
res = gConfig.getSections("/Resources/StorageElements")
if not res["OK"]:
return self._errorReport(res,
"Failed to get possible StorageElements")
missing = set(seList) - set(res["Value"])
if missing:
for se in missing:
gLogger.error("StorageElement %s is not known" % se)
return S_ERROR("%d StorageElements not known" % len(missing))
return S_OK()
def __promptForParameter(self,
parameter,
choices=[],
default="",
insert=True):
res = promptUser("Please enter %s" % parameter,
choices=choices,
default=default)
if not res["OK"]:
return self._errorReport(res)
gLogger.notice("%s will be set to '%s'" % (parameter, res["Value"]))
paramValue = res["Value"]
if insert:
setter = None
setterName = "set%s" % parameter
if hasattr(self, setterName) and callable(getattr(
self, setterName)):
setter = getattr(self, setterName)
if not setter:
return S_ERROR(
"Unable to invoke %s, it isn't a member function of Transformation!"
)
res = setter(paramValue)
if not res["OK"]:
return res
return S_OK(paramValue)
class Transformation(API):
#############################################################################
def __init__(self, transID=0, transClient=None):
""" c'tor
"""
super(Transformation, self).__init__()
self.paramTypes = {
'TransformationID': [types.IntType, types.LongType],
'TransformationName': types.StringTypes,
'Status': types.StringTypes,
'Description': types.StringTypes,
'LongDescription': types.StringTypes,
'Type': types.StringTypes,
'Plugin': types.StringTypes,
'AgentType': types.StringTypes,
'FileMask': types.StringTypes,
'TransformationGroup': types.StringTypes,
'GroupSize': [types.IntType, types.LongType, types.FloatType],
'InheritedFrom': [types.IntType, types.LongType],
'Body': types.StringTypes,
'MaxNumberOfTasks': [types.IntType, types.LongType],
'EventsPerTask': [types.IntType, types.LongType]
}
self.paramValues = {
'TransformationID': 0,
'TransformationName': '',
'Status': 'New',
'Description': '',
'LongDescription': '',
'Type': '',
'Plugin': 'Standard',
'AgentType': 'Manual',
'FileMask': '',
'TransformationGroup': 'General',
'GroupSize': 1,
'InheritedFrom': 0,
'Body': '',
'MaxNumberOfTasks': 0,
'EventsPerTask': 0
}
self.ops = Operations()
self.supportedPlugins = self.ops.getValue(
'Transformations/AllowedPlugins',
['Broadcast', 'Standard', 'BySize', 'ByShare'])
if not transClient:
self.transClient = TransformationClient()
else:
self.transClient = transClient
self.serverURL = self.transClient.getServer()
self.exists = False
if transID:
self.paramValues['TransformationID'] = transID
res = self.getTransformation()
if res['OK']:
self.exists = True
elif res['Message'] == 'Transformation does not exist':
raise AttributeError('TransformationID %d does not exist' %
transID)
else:
self.paramValues['TransformationID'] = 0
gLogger.fatal(
"Failed to get transformation from database",
"%s @ %s" % (transID, self.transClient.serverURL))
def setServer(self, server):
self.serverURL = server
self.transClient.setServer(self.serverURL)
def getServer(self):
return self.serverURL
def reset(self, transID=0):
self.__init__(transID)
self.transClient.setServer(self.serverURL)
return S_OK()
def setTargetSE(self, seList):
return self.__setSE('TargetSE', seList)
def setSourceSE(self, seList):
return self.__setSE('SourceSE', seList)
def __setSE(self, seParam, seList):
if isinstance(seList, basestring):
try:
seList = eval(seList)
except:
seList = seList.split(',')
elif isinstance(seList, (list, dict, tuple)):
seList = list(seList)
else:
return S_ERROR("Bad argument type")
res = self.__checkSEs(seList)
if not res['OK']:
return res
self.item_called = seParam
return self.__setParam(seList)
def __getattr__(self, name):
if name.find('get') == 0:
item = name[3:]
self.item_called = item
return self.__getParam
if name.find('set') == 0:
item = name[3:]
self.item_called = item
return self.__setParam
raise AttributeError(name)
def __getParam(self):
if self.item_called == 'Available':
return S_OK(self.paramTypes.keys())
if self.item_called == 'Parameters':
return S_OK(self.paramValues)
if self.item_called in self.paramValues:
return S_OK(self.paramValues[self.item_called])
raise AttributeError("Unknown parameter for transformation: %s" %
self.item_called)
def __setParam(self, value):
change = False
if self.item_called in self.paramTypes:
if self.paramValues[self.item_called] != value:
if type(value) in self.paramTypes[self.item_called]:
change = True
else:
raise TypeError("%s %s %s expected one of %s" %
(self.item_called, value, type(value),
self.paramTypes[self.item_called]))
else:
if self.item_called not in self.paramValues:
change = True
else:
if self.paramValues[self.item_called] != value:
change = True
if not change:
gLogger.verbose("No change of parameter %s required" %
self.item_called)
else:
gLogger.verbose("Parameter %s to be changed" % self.item_called)
transID = self.paramValues['TransformationID']
if self.exists and transID:
res = self.transClient.setTransformationParameter(
transID, self.item_called, value)
if not res['OK']:
return res
self.paramValues[self.item_called] = value
return S_OK()
def getTransformation(self, printOutput=False):
transID = self.paramValues['TransformationID']
if not transID:
gLogger.fatal("No TransformationID known")
return S_ERROR()
res = self.transClient.getTransformation(transID, extraParams=True)
if not res['OK']:
if printOutput:
self._prettyPrint(res)
return res
transParams = res['Value']
for paramName, paramValue in transParams.items():
setter = None
setterName = "set%s" % paramName
if hasattr(self, setterName) and callable(getattr(
self, setterName)):
setter = getattr(self, setterName)
if not setterName:
gLogger.error(
"Unable to invoke setter %s, it isn't a member function" %
setterName)
continue
setter(paramValue)
if printOutput:
gLogger.info("No printing available yet")
return S_OK(transParams)
def getTransformationLogging(self, printOutput=False):
transID = self.paramValues['TransformationID']
if not transID:
gLogger.fatal("No TransformationID known")
return S_ERROR()
res = self.transClient.getTransformationLogging(transID)
if not res['OK']:
if printOutput:
self._prettyPrint(res)
return res
loggingList = res['Value']
if printOutput:
self._printFormattedDictList(
loggingList, ['Message', 'MessageDate', 'AuthorDN'],
'MessageDate', 'MessageDate')
return S_OK(loggingList)
def extendTransformation(self, nTasks, printOutput=False):
return self.__executeOperation('extendTransformation',
nTasks,
printOutput=printOutput)
def cleanTransformation(self, printOutput=False):
res = self.__executeOperation('cleanTransformation',
printOutput=printOutput)
if res['OK']:
self.paramValues['Status'] = 'Cleaned'
return res
def deleteTransformation(self, printOutput=False):
res = self.__executeOperation('deleteTransformation',
printOutput=printOutput)
if res['OK']:
self.reset()
return res
def addFilesToTransformation(self, lfns, printOutput=False):
return self.__executeOperation('addFilesToTransformation',
lfns,
printOutput=printOutput)
def setFileStatusForTransformation(self, status, lfns, printOutput=False):
return self.__executeOperation('setFileStatusForTransformation',
status,
lfns,
printOutput=printOutput)
def getTransformationTaskStats(self, printOutput=False):
return self.__executeOperation('getTransformationTaskStats',
printOutput=printOutput)
def getTransformationStats(self, printOutput=False):
return self.__executeOperation('getTransformationStats',
printOutput=printOutput)
def deleteTasks(self, taskMin, taskMax, printOutput=False):
return self.__executeOperation('deleteTasks',
taskMin,
taskMax,
printOutput=printOutput)
def addTaskForTransformation(self,
lfns=[],
se='Unknown',
printOutput=False):
return self.__executeOperation('addTaskForTransformation',
lfns,
se,
printOutput=printOutput)
def setTaskStatus(self, taskID, status, printOutput=False):
return self.__executeOperation('setTaskStatus',
taskID,
status,
printOutput=printOutput)
def __executeOperation(self, operation, *parms, **kwds):
transID = self.paramValues['TransformationID']
if not transID:
gLogger.fatal("No TransformationID known")
return S_ERROR()
printOutput = kwds.pop('printOutput')
fcn = None
if hasattr(self.transClient, operation) and callable(
getattr(self.transClient, operation)):
fcn = getattr(self.transClient, operation)
if not fcn:
return S_ERROR(
"Unable to invoke %s, it isn't a member funtion of TransformationClient"
)
res = fcn(transID, *parms, **kwds)
if printOutput:
self._prettyPrint(res)
return res
def getTransformationFiles(self,
fileStatus=[],
lfns=[],
outputFields=[
'FileID', 'LFN', 'Status', 'TaskID',
'TargetSE', 'UsedSE', 'ErrorCount',
'InsertedTime', 'LastUpdate'
],
orderBy='FileID',
printOutput=False):
condDict = {'TransformationID': self.paramValues['TransformationID']}
if fileStatus:
condDict['Status'] = fileStatus
if lfns:
condDict['LFN'] = lfns
res = self.transClient.getTransformationFiles(condDict=condDict)
if not res['OK']:
if printOutput:
self._prettyPrint(res)
return res
if printOutput:
if not outputFields:
gLogger.info("Available fields are: %s" %
res['ParameterNames'].join(' '))
elif not res['Value']:
gLogger.info("No tasks found for selection")
else:
self._printFormattedDictList(res['Value'], outputFields,
'FileID', orderBy)
return res
def getTransformationTasks(self,
taskStatus=[],
taskIDs=[],
outputFields=[
'TransformationID', 'TaskID',
'ExternalStatus', 'ExternalID', 'TargetSE',
'CreationTime', 'LastUpdateTime'
],
orderBy='TaskID',
printOutput=False):
condDict = {'TransformationID': self.paramValues['TransformationID']}
if taskStatus:
condDict['ExternalStatus'] = taskStatus
if taskIDs:
condDict['TaskID'] = taskIDs
res = self.transClient.getTransformationTasks(condDict=condDict)
if not res['OK']:
if printOutput:
self._prettyPrint(res)
return res
if printOutput:
if not outputFields:
gLogger.info("Available fields are: %s" %
res['ParameterNames'].join(' '))
elif not res['Value']:
gLogger.info("No tasks found for selection")
else:
self._printFormattedDictList(res['Value'], outputFields,
'TaskID', orderBy)
return res
#############################################################################
def getTransformations(self,
transID=[],
transStatus=[],
outputFields=[
'TransformationID', 'Status', 'AgentType',
'TransformationName', 'CreationDate'
],
orderBy='TransformationID',
printOutput=False):
condDict = {}
if transID:
condDict['TransformationID'] = transID
if transStatus:
condDict['Status'] = transStatus
res = self.transClient.getTransformations(condDict=condDict)
if not res['OK']:
if printOutput:
self._prettyPrint(res)
return res
if printOutput:
if not outputFields:
gLogger.info("Available fields are: %s" %
res['ParameterNames'].join(' '))
elif not res['Value']:
gLogger.info("No tasks found for selection")
else:
self._printFormattedDictList(res['Value'], outputFields,
'TransformationID', orderBy)
return res
#############################################################################
def getAuthorDNfromProxy(self):
""" gets the AuthorDN and username of the transformation from the uploaded proxy
"""
username = ""
author = ""
res = getProxyInfo()
if res['OK']:
author = res['Value']['identity']
username = res['Value']['username']
else:
gLogger.error("Unable to get uploaded proxy Info %s " %
res['Message'])
return S_ERROR(res['Message'])
res = {'username': username, 'authorDN': author}
return S_OK(res)
#############################################################################
def getTransformationsByUser(self,
authorDN="",
userName="",
transID=[],
transStatus=[],
outputFields=[
'TransformationID', 'Status', 'AgentType',
'TransformationName', 'CreationDate',
'AuthorDN'
],
orderBy='TransformationID',
printOutput=False):
condDict = {}
if authorDN == "":
res = self.getAuthorDNfromProxy()
if not res['OK']:
gLogger.error(res['Message'])
return S_ERROR(res['Message'])
else:
foundUserName = res['Value']['username']
foundAuthor = res['Value']['authorDN']
# If the username whom created the uploaded proxy is different than the provided username report error and exit
if not (userName == "" or userName == foundUserName):
gLogger.error(
"Couldn't resolve the authorDN for user '%s' from the uploaded proxy (proxy created by '%s')"
% (userName, foundUserName))
return S_ERROR(
"Couldn't resolve the authorDN for user '%s' from the uploaded proxy (proxy created by '%s')"
% (userName, foundUserName))
userName = foundUserName
authorDN = foundAuthor
gLogger.info(
"Will list transformations created by user '%s' with status '%s'"
% (userName, ', '.join(transStatus)))
else:
gLogger.info(
"Will list transformations created by '%s' with status '%s'" %
(authorDN, ', '.join(transStatus)))
condDict['AuthorDN'] = authorDN
if transID:
condDict['TransformationID'] = transID
if transStatus:
condDict['Status'] = transStatus
res = self.transClient.getTransformations(condDict=condDict)
if not res['OK']:
if printOutput:
self._prettyPrint(res)
return res
if printOutput:
if not outputFields:
gLogger.info("Available fields are: %s" %
res['ParameterNames'].join(' '))
elif not res['Value']:
gLogger.info("No tasks found for selection")
else:
self._printFormattedDictList(res['Value'], outputFields,
'TransformationID', orderBy)
return res
#############################################################################
def getSummaryTransformations(self, transID=[]):
"""Show the summary for a list of Transformations
Fields starting with 'F' ('J') refers to files (jobs).
Proc. stand for processed.
"""
condDict = {'TransformationID': transID}
orderby = []
start = 0
maxitems = len(transID)
paramShowNames = ['TransformationID','Type','Status','Files_Total','Files_PercentProcessed',\
'Files_Processed','Files_Unused','Jobs_TotalCreated','Jobs_Waiting',\
'Jobs_Running','Jobs_Done','Jobs_Failed','Jobs_Stalled']
# Below, the header used for each field in the printing: short to fit in one line
paramShowNamesShort = ['TransID','Type','Status','F_Total','F_Proc.(%)','F_Proc.',\
'F_Unused','J_Created','J_Wait','J_Run','J_Done','J_Fail','J_Stalled']
dictList = []
result = self.transClient.getTransformationSummaryWeb(
condDict, orderby, start, maxitems)
if not result['OK']:
self._prettyPrint(result)
return result
if result['Value']['TotalRecords'] > 0:
try:
paramNames = result['Value']['ParameterNames']
for paramValues in result['Value']['Records']:
paramShowValues = map(
lambda pname: paramValues[paramNames.index(pname)],
paramShowNames)
showDict = dict(zip(paramShowNamesShort, paramShowValues))
dictList.append(showDict)
except Exception, x:
print 'Exception %s ' % str(x)
if not len(dictList) > 0:
gLogger.error(
'No found transformations satisfying input condition')
return S_ERROR(
'No found transformations satisfying input condition')
else:
print self._printFormattedDictList(dictList, paramShowNamesShort,
paramShowNamesShort[0],
paramShowNamesShort[0])
return S_OK(dictList)
class Transformation( API ):
#############################################################################
def __init__( self, transID = 0, transClient = None ):
""" c'tor
"""
super( Transformation, self ).__init__()
self.paramTypes = { 'TransformationID' : [types.IntType, types.LongType],
'TransformationName' : types.StringTypes,
'Status' : types.StringTypes,
'Description' : types.StringTypes,
'LongDescription' : types.StringTypes,
'Type' : types.StringTypes,
'Plugin' : types.StringTypes,
'AgentType' : types.StringTypes,
'FileMask' : types.StringTypes,
'TransformationGroup' : types.StringTypes,
'GroupSize' : [types.IntType, types.LongType, types.FloatType],
'InheritedFrom' : [types.IntType, types.LongType],
'Body' : types.StringTypes,
'MaxNumberOfTasks' : [types.IntType, types.LongType],
'EventsPerTask' : [types.IntType, types.LongType]}
self.paramValues = { 'TransformationID' : 0,
'TransformationName' : '',
'Status' : 'New',
'Description' : '',
'LongDescription' : '',
'Type' : '',
'Plugin' : 'Standard',
'AgentType' : 'Manual',
'FileMask' : '',
'TransformationGroup' : 'General',
'GroupSize' : 1,
'InheritedFrom' : 0,
'Body' : '',
'MaxNumberOfTasks' : 0,
'EventsPerTask' : 0}
self.ops = Operations()
self.supportedPlugins = self.ops.getValue( 'Transformations/AllowedPlugins',
['Broadcast', 'Standard', 'BySize', 'ByShare'] )
if not transClient:
self.transClient = TransformationClient()
else:
self.transClient = transClient
self.serverURL = self.transClient.getServer()
self.exists = False
if transID:
self.paramValues['TransformationID'] = transID
res = self.getTransformation()
if res['OK']:
self.exists = True
elif res['Message'] == 'Transformation does not exist':
raise AttributeError( 'TransformationID %d does not exist' % transID )
else:
self.paramValues['TransformationID'] = 0
gLogger.fatal( "Failed to get transformation from database", "%s @ %s" % ( transID,
self.transClient.serverURL ) )
def setServer( self, server ):
self.serverURL = server
self.transClient.setServer( self.serverURL )
def getServer( self ):
return self.serverURL
def reset( self, transID = 0 ):
self.__init__( transID )
self.transClient.setServer( self.serverURL )
return S_OK()
def setTargetSE( self, seList ):
return self.__setSE( 'TargetSE', seList )
def setSourceSE( self, seList ):
return self.__setSE( 'SourceSE', seList )
def __setSE( self, seParam, seList ):
if isinstance( seList, basestring ):
try:
seList = eval( seList )
except:
seList = seList.split( ',' )
elif isinstance( seList, ( list, dict, tuple ) ):
seList = list( seList )
else:
return S_ERROR( "Bad argument type" )
res = self.__checkSEs( seList )
if not res['OK']:
return res
self.item_called = seParam
return self.__setParam( seList )
def __getattr__( self, name ):
if name.find( 'get' ) == 0:
item = name[3:]
self.item_called = item
return self.__getParam
if name.find( 'set' ) == 0:
item = name[3:]
self.item_called = item
return self.__setParam
raise AttributeError( name )
def __getParam( self ):
if self.item_called == 'Available':
return S_OK( self.paramTypes.keys() )
if self.item_called == 'Parameters':
return S_OK( self.paramValues )
if self.item_called in self.paramValues:
return S_OK( self.paramValues[self.item_called] )
raise AttributeError( "Unknown parameter for transformation: %s" % self.item_called )
def __setParam( self, value ):
change = False
if self.item_called in self.paramTypes:
if self.paramValues[self.item_called] != value:
if type( value ) in self.paramTypes[self.item_called]:
change = True
else:
raise TypeError( "%s %s %s expected one of %s" % ( self.item_called, value, type( value ),
self.paramTypes[self.item_called] ) )
else:
if self.item_called not in self.paramValues:
change = True
else:
if self.paramValues[self.item_called] != value:
change = True
if not change:
gLogger.verbose( "No change of parameter %s required" % self.item_called )
else:
gLogger.verbose( "Parameter %s to be changed" % self.item_called )
transID = self.paramValues['TransformationID']
if self.exists and transID:
res = self.transClient.setTransformationParameter( transID, self.item_called, value )
if not res['OK']:
return res
self.paramValues[self.item_called] = value
return S_OK()
def getTransformation( self, printOutput = False ):
transID = self.paramValues['TransformationID']
if not transID:
gLogger.fatal( "No TransformationID known" )
return S_ERROR()
res = self.transClient.getTransformation( transID, extraParams = True )
if not res['OK']:
if printOutput:
self._prettyPrint( res )
return res
transParams = res['Value']
for paramName, paramValue in transParams.items():
setter = None
setterName = "set%s" % paramName
if hasattr( self, setterName ) and callable( getattr( self, setterName ) ):
setter = getattr( self, setterName )
if not setterName:
gLogger.error( "Unable to invoke setter %s, it isn't a member function" % setterName )
continue
setter( paramValue )
if printOutput:
gLogger.info( "No printing available yet" )
return S_OK( transParams )
def getTransformationLogging( self, printOutput = False ):
transID = self.paramValues['TransformationID']
if not transID:
gLogger.fatal( "No TransformationID known" )
return S_ERROR()
res = self.transClient.getTransformationLogging( transID )
if not res['OK']:
if printOutput:
self._prettyPrint( res )
return res
loggingList = res['Value']
if printOutput:
self._printFormattedDictList( loggingList, ['Message', 'MessageDate', 'AuthorDN'], 'MessageDate', 'MessageDate' )
return S_OK( loggingList )
def extendTransformation( self, nTasks, printOutput = False ):
return self.__executeOperation( 'extendTransformation', nTasks, printOutput = printOutput )
def cleanTransformation( self, printOutput = False ):
res = self.__executeOperation( 'cleanTransformation', printOutput = printOutput )
if res['OK']:
self.paramValues['Status'] = 'Cleaned'
return res
def deleteTransformation( self, printOutput = False ):
res = self.__executeOperation( 'deleteTransformation', printOutput = printOutput )
if res['OK']:
self.reset()
return res
def addFilesToTransformation( self, lfns, printOutput = False ):
return self.__executeOperation( 'addFilesToTransformation', lfns, printOutput = printOutput )
def setFileStatusForTransformation( self, status, lfns, printOutput = False ):
return self.__executeOperation( 'setFileStatusForTransformation', status, lfns, printOutput = printOutput )
def getTransformationTaskStats( self, printOutput = False ):
return self.__executeOperation( 'getTransformationTaskStats', printOutput = printOutput )
def getTransformationStats( self, printOutput = False ):
return self.__executeOperation( 'getTransformationStats', printOutput = printOutput )
def deleteTasks( self, taskMin, taskMax, printOutput = False ):
return self.__executeOperation( 'deleteTasks', taskMin, taskMax, printOutput = printOutput )
def addTaskForTransformation( self, lfns = [], se = 'Unknown', printOutput = False ):
return self.__executeOperation( 'addTaskForTransformation', lfns, se, printOutput = printOutput )
def setTaskStatus( self, taskID, status, printOutput = False ):
return self.__executeOperation( 'setTaskStatus', taskID, status, printOutput = printOutput )
def __executeOperation( self, operation, *parms, **kwds ):
transID = self.paramValues['TransformationID']
if not transID:
gLogger.fatal( "No TransformationID known" )
return S_ERROR()
printOutput = kwds.pop( 'printOutput' )
fcn = None
if hasattr( self.transClient, operation ) and callable( getattr( self.transClient, operation ) ):
fcn = getattr( self.transClient, operation )
if not fcn:
return S_ERROR( "Unable to invoke %s, it isn't a member funtion of TransformationClient" )
res = fcn( transID, *parms, **kwds )
if printOutput:
self._prettyPrint( res )
return res
def getTransformationFiles( self, fileStatus = [], lfns = [], outputFields = ['FileID', 'LFN', 'Status', 'TaskID',
'TargetSE', 'UsedSE', 'ErrorCount',
'InsertedTime', 'LastUpdate'],
orderBy = 'FileID', printOutput = False ):
condDict = {'TransformationID':self.paramValues['TransformationID']}
if fileStatus:
condDict['Status'] = fileStatus
if lfns:
condDict['LFN'] = lfns
res = self.transClient.getTransformationFiles( condDict = condDict )
if not res['OK']:
if printOutput:
self._prettyPrint( res )
return res
if printOutput:
if not outputFields:
gLogger.info( "Available fields are: %s" % res['ParameterNames'].join( ' ' ) )
elif not res['Value']:
gLogger.info( "No tasks found for selection" )
else:
self._printFormattedDictList( res['Value'], outputFields, 'FileID', orderBy )
return res
def getTransformationTasks( self, taskStatus = [], taskIDs = [], outputFields = ['TransformationID', 'TaskID',
'ExternalStatus', 'ExternalID',
'TargetSE', 'CreationTime',
'LastUpdateTime'],
orderBy = 'TaskID', printOutput = False ):
condDict = {'TransformationID':self.paramValues['TransformationID']}
if taskStatus:
condDict['ExternalStatus'] = taskStatus
if taskIDs:
condDict['TaskID'] = taskIDs
res = self.transClient.getTransformationTasks( condDict = condDict )
if not res['OK']:
if printOutput:
self._prettyPrint( res )
return res
if printOutput:
if not outputFields:
gLogger.info( "Available fields are: %s" % res['ParameterNames'].join( ' ' ) )
elif not res['Value']:
gLogger.info( "No tasks found for selection" )
else:
self._printFormattedDictList( res['Value'], outputFields, 'TaskID', orderBy )
return res
#############################################################################
def getTransformations( self, transID = [], transStatus = [], outputFields = ['TransformationID', 'Status',
'AgentType', 'TransformationName',
'CreationDate'],
orderBy = 'TransformationID', printOutput = False ):
condDict = {}
if transID:
condDict['TransformationID'] = transID
if transStatus:
condDict['Status'] = transStatus
res = self.transClient.getTransformations( condDict = condDict )
if not res['OK']:
if printOutput:
self._prettyPrint( res )
return res
if printOutput:
if not outputFields:
gLogger.info( "Available fields are: %s" % res['ParameterNames'].join( ' ' ) )
elif not res['Value']:
gLogger.info( "No tasks found for selection" )
else:
self._printFormattedDictList( res['Value'], outputFields, 'TransformationID', orderBy )
return res
#############################################################################
def getAuthorDNfromProxy( self ):
""" gets the AuthorDN and username of the transformation from the uploaded proxy
"""
username = ""
author = ""
res = getProxyInfo()
if res['OK']:
author = res['Value']['identity']
username = res['Value']['username']
else:
gLogger.error( "Unable to get uploaded proxy Info %s " %res['Message'] )
return S_ERROR( res['Message'] )
res = {'username' : username, 'authorDN' : author }
return S_OK( res )
#############################################################################
def getTransformationsByUser( self, authorDN = "", userName = "", transID = [], transStatus = [], outputFields = ['TransformationID', 'Status',
'AgentType', 'TransformationName',
'CreationDate', 'AuthorDN'],
orderBy = 'TransformationID', printOutput = False ):
condDict = {}
if authorDN == "":
res = self.getAuthorDNfromProxy()
if not res['OK']:
gLogger.error( res['Message'] )
return S_ERROR( res['Message'] )
else:
foundUserName = res['Value']['username']
foundAuthor = res['Value']['authorDN']
# If the username whom created the uploaded proxy is different than the provided username report error and exit
if not ( userName == "" or userName == foundUserName ):
gLogger.error("Couldn't resolve the authorDN for user '%s' from the uploaded proxy (proxy created by '%s')" %(userName, foundUserName))
return S_ERROR("Couldn't resolve the authorDN for user '%s' from the uploaded proxy (proxy created by '%s')" %(userName, foundUserName))
userName = foundUserName
authorDN = foundAuthor
gLogger.info("Will list transformations created by user '%s' with status '%s'" %(userName, ', '.join( transStatus )))
else:
gLogger.info("Will list transformations created by '%s' with status '%s'" %(authorDN, ', '.join( transStatus )))
condDict['AuthorDN'] = authorDN
if transID:
condDict['TransformationID'] = transID
if transStatus:
condDict['Status'] = transStatus
res = self.transClient.getTransformations( condDict = condDict )
if not res['OK']:
if printOutput:
self._prettyPrint( res )
return res
if printOutput:
if not outputFields:
gLogger.info( "Available fields are: %s" % res['ParameterNames'].join( ' ' ) )
elif not res['Value']:
gLogger.info( "No tasks found for selection" )
else:
self._printFormattedDictList( res['Value'], outputFields, 'TransformationID', orderBy )
return res
#############################################################################
def getSummaryTransformations( self , transID = []):
"""Show the summary for a list of Transformations
Fields starting with 'F' ('J') refers to files (jobs).
Proc. stand for processed.
"""
condDict = { 'TransformationID' : transID }
orderby = []
start = 0
maxitems = len(transID)
paramShowNames = ['TransformationID','Type','Status','Files_Total','Files_PercentProcessed',\
'Files_Processed','Files_Unused','Jobs_TotalCreated','Jobs_Waiting',\
'Jobs_Running','Jobs_Done','Jobs_Failed','Jobs_Stalled']
# Below, the header used for each field in the printing: short to fit in one line
paramShowNamesShort = ['TransID','Type','Status','F_Total','F_Proc.(%)','F_Proc.',\
'F_Unused','J_Created','J_Wait','J_Run','J_Done','J_Fail','J_Stalled']
dictList = []
result = self.transClient.getTransformationSummaryWeb( condDict, orderby, start, maxitems )
if not result['OK']:
self._prettyPrint( result )
return result
if result['Value']['TotalRecords'] > 0:
try:
paramNames = result['Value']['ParameterNames']
for paramValues in result['Value']['Records']:
paramShowValues = map(lambda pname: paramValues[ paramNames.index(pname) ], paramShowNames)
showDict = dict(zip( paramShowNamesShort, paramShowValues ))
dictList.append( showDict )
except Exception, x:
print 'Exception %s ' %str(x)
if not len(dictList) > 0:
gLogger.error( 'No found transformations satisfying input condition')
return S_ERROR( 'No found transformations satisfying input condition')
else:
print self._printFormattedDictList( dictList, paramShowNamesShort, paramShowNamesShort[0], paramShowNamesShort[0] )
return S_OK( dictList )