class Request( Record ):
"""
.. class:: Request
:param int RequestID: requestID
:param str Name: request' name
:param str OwnerDN: request's owner DN
:param str OwnerGroup: request owner group
:param str Setup: DIRAC setup
:param str SourceComponent: whatever
:param int JobID: jobID
:param datetime.datetime CreationTime: UTC datetime
:param datetime.datetime SubmissionTime: UTC datetime
:param datetime.datetime LastUpdate: UTC datetime
:param str Status: request's status
:param TypedList operations: list of operations
"""
ALL_STATES = ( "Waiting", "Failed", "Done", "Scheduled", "Assigned", "Canceled" )
FINAL_STATES = ( "Done", "Failed", "Canceled" )
def __init__( self, fromDict = None ):
"""c'tor
:param self: self reference
"""
Record.__init__( self )
self.__waiting = None
now = datetime.datetime.utcnow().replace( microsecond = 0 )
self.__data__["CreationTime"] = now
self.__data__["SubmitTime"] = now
self.__data__["LastUpdate"] = now
self.__data__["Status"] = "Done"
self.__data__["JobID"] = 0
self.__data__["RequestID"] = 0
proxyInfo = getProxyInfo()
if proxyInfo["OK"]:
proxyInfo = proxyInfo["Value"]
if proxyInfo["validGroup"] and proxyInfo["validDN"]:
self.OwnerDN = proxyInfo["identity"]
self.OwnerGroup = proxyInfo["group"]
self.__dirty = []
self.__operations__ = TypedList( allowedTypes = Operation )
fromDict = fromDict if fromDict else {}
self.__dirty = fromDict.get( "__dirty", [] )
if "__dirty" in fromDict:
del fromDict["__dirty"]
for opDict in fromDict.get( "Operations", [] ):
self +=Operation( opDict )
if "Operations" in fromDict:
del fromDict["Operations"]
for key, value in fromDict.items():
if key not in self.__data__:
raise AttributeError( "Unknown Request attribute '%s'" % key )
if value:
setattr( self, key, value )
self._notify()
@staticmethod
def tableDesc():
""" get table desc """
return { "Fields" :
{ "RequestID" : "INTEGER NOT NULL AUTO_INCREMENT",
"RequestName" : "VARCHAR(255) NOT NULL",
"OwnerDN" : "VARCHAR(255)",
"OwnerGroup" : "VARCHAR(32)",
"Status" : "ENUM('Waiting', 'Assigned', 'Done', 'Failed', 'Canceled', 'Scheduled') DEFAULT 'Waiting'",
"Error" : "VARCHAR(255)",
"DIRACSetup" : "VARCHAR(32)",
"SourceComponent" : "BLOB",
"JobID" : "INTEGER DEFAULT 0",
"CreationTime" : "DATETIME",
"SubmitTime" : "DATETIME",
"LastUpdate" : "DATETIME" },
"PrimaryKey" : [ "RequestID" ],
'UniqueIndexes': {'RequestName' : [ 'RequestName'] }
}
def _notify( self ):
""" simple state machine for sub request statuses """
self.__waiting = None
# # update operations statuses
rStatus = "Waiting"
opStatusList = [ ( op.Status, op ) for op in self ]
self.__waiting = None
while opStatusList:
# # Scan all status in order!
opStatus, op = opStatusList.pop( 0 )
# # Failed -> Failed
if opStatus == "Failed":
rStatus = "Failed"
break
# Scheduled -> Scheduled
if opStatus == "Scheduled":
if self.__waiting == None:
self.__waiting = op
rStatus = "Scheduled"
# # First operation Queued becomes Waiting if no Waiting/Scheduled before
elif opStatus == "Queued":
if self.__waiting == None:
self.__waiting = op
op._setWaiting( self )
rStatus = "Waiting"
# # First operation Waiting is next to execute, others are queued
elif opStatus == "Waiting":
rStatus = "Waiting"
if self.__waiting == None:
self.__waiting = op
else:
op._setQueued( self )
# # All operations Done -> Done
elif opStatus == "Done" and self.__waiting == None:
rStatus = "Done"
self.__data__['Error'] = ''
self.Status = rStatus
def getWaiting( self ):
""" get waiting operation if any """
# # update states
self._notify()
return S_OK( self.__waiting )
# # Operation arithmetics
def __contains__( self, operation ):
""" in operator
:param self: self reference
:param Operation subRequest: a subRequest
"""
return bool( operation in self.__operations__ )
def __iadd__( self, operation ):
""" += operator for subRequest
:param self: self reference
:param Operation operation: sub-request to add
"""
if operation not in self:
self.__operations__.append( operation )
operation._parent = self
self._notify()
return self
def insertBefore( self, newOperation, existingOperation ):
""" insert :newOperation: just before :existingOperation:
:param self: self reference
:param Operation newOperation: Operation to be inserted
:param Operation existingOperation: previous Operation sibling
"""
if existingOperation not in self:
return S_ERROR( "%s is not in" % existingOperation )
if newOperation in self:
return S_ERROR( "%s is already in" % newOperation )
self.__operations__.insert( self.__operations__.index( existingOperation ), newOperation )
newOperation._parent = self
self._notify()
return S_OK()
def insertAfter( self, newOperation, existingOperation ):
""" insert :newOperation: just after :existingOperation:
:param self: self reference
:param Operation newOperation: Operation to be inserted
:param Operation existingOperation: next Operation sibling
"""
if existingOperation not in self:
return S_ERROR( "%s is not in" % existingOperation )
if newOperation in self:
return S_ERROR( "%s is already in" % newOperation )
self.__operations__.insert( self.__operations__.index( existingOperation ) + 1, newOperation )
newOperation._parent = self
self._notify()
return S_OK()
def addOperation( self, operation ):
""" add :operation: to list of Operations
:param self: self reference
:param Operation operation: Operation to be inserted
"""
if operation in self:
return S_ERROR( "This operation is already in!!!" )
self +=operation
return S_OK()
def isEmpty( self ):
""" Evaluate if the request is empty
"""
return len( self.__operations__ ) == 0
def __iter__( self ):
""" iterator for sub-request """
return self.__operations__.__iter__()
def __getitem__( self, i ):
""" [] op for sub requests """
return self.__operations__.__getitem__( i )
def __setitem__( self, i, value ):
""" self[i] = val """
self.__operations__._typeCheck( value )
if self[i].OperationID:
self.__dirty.append( self[i].OperationID )
self.__operations__.__setitem__( i, value )
value._parent = self
self._notify()
def __delitem__( self, i ):
""" del self[i]"""
if not self.RequestID:
self.__operations__.__delitem__( i )
else:
opId = self[i].OperationID
if opId:
self.__dirty.append( opId )
self.__operations__.__delitem__( i )
self._notify()
def indexOf( self, subReq ):
""" return index of subReq (execution order) """
return self.__operations__.index( subReq ) if subReq in self else -1
def __nonzero__( self ):
""" for comparisons
"""
return True
def __len__( self ):
""" nb of subRequests """
return len( self.__operations__ )
def __str__( self ):
""" str operator """
return str( self.toJSON()["Value"] )
def subStatusList( self ):
""" list of statuses for all operations """
return [ subReq.Status for subReq in self ]
# # properties
@property
def RequestID( self ):
""" request ID getter """
return self.__data__["RequestID"]
@RequestID.setter
def RequestID( self, value ):
""" requestID setter (shouldn't be RO???) """
self.__data__["RequestID"] = long( value ) if value else 0
@property
def RequestName( self ):
""" request's name getter """
return self.__data__["RequestName"]
@RequestName.setter
def RequestName( self, value ):
""" request name setter """
if type( value ) != str:
raise TypeError( "RequestName should be a string" )
self.__data__["RequestName"] = value[:128]
@property
def OwnerDN( self ):
""" request owner DN getter """
return self.__data__["OwnerDN"]
@OwnerDN.setter
def OwnerDN( self, value ):
""" request owner DN setter """
if type( value ) != str:
raise TypeError( "OwnerDN should be a string!" )
self.__data__["OwnerDN"] = value
@property
def OwnerGroup( self ):
""" request owner group getter """
return self.__data__["OwnerGroup"]
@OwnerGroup.setter
def OwnerGroup( self, value ):
""" request owner group setter """
if type( value ) != str:
raise TypeError( "OwnerGroup should be a string!" )
self.__data__["OwnerGroup"] = value
@property
def DIRACSetup( self ):
""" DIRAC setup getter """
return self.__data__["DIRACSetup"]
@DIRACSetup.setter
def DIRACSetup( self, value ):
""" DIRAC setup setter """
if type( value ) != str:
raise TypeError( "setup should be a string!" )
self.__data__["DIRACSetup"] = value
@property
def SourceComponent( self ):
""" source component getter """
return self.__data__["SourceComponent"]
@SourceComponent.setter
def SourceComponent( self, value ):
""" source component setter """
if type( value ) != str:
raise TypeError( "Setup should be a string!" )
self.__data__["SourceComponent"] = value
@property
def JobID( self ):
""" jobID getter """
return self.__data__["JobID"]
@JobID.setter
def JobID( self, value = 0 ):
""" jobID setter """
self.__data__["JobID"] = long( value ) if value else 0
@property
def CreationTime( self ):
""" creation time getter """
return self.__data__["CreationTime"]
@CreationTime.setter
def CreationTime( self, value = None ):
""" creation time setter """
if type( value ) not in ( datetime.datetime, str ) :
raise TypeError( "CreationTime should be a datetime.datetime!" )
if type( value ) == str:
value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' )
self.__data__["CreationTime"] = value
@property
def SubmitTime( self ):
""" request's submission time getter """
return self.__data__["SubmitTime"]
@SubmitTime.setter
def SubmitTime( self, value = None ):
""" submission time setter """
if type( value ) not in ( datetime.datetime, str ):
raise TypeError( "SubmitTime should be a datetime.datetime!" )
if type( value ) == str:
value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' )
self.__data__["SubmitTime"] = value
@property
def LastUpdate( self ):
""" last update getter """
return self.__data__["LastUpdate"]
@LastUpdate.setter
def LastUpdate( self, value = None ):
""" last update setter """
if type( value ) not in ( datetime.datetime, str ):
raise TypeError( "LastUpdate should be a datetime.datetime!" )
if type( value ) == str:
value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' )
self.__data__["LastUpdate"] = value
@property
def Status( self ):
""" status getter """
self._notify()
return self.__data__["Status"]
@Status.setter
def Status( self, value ):
""" status setter """
if value not in Request.ALL_STATES:
raise ValueError( "Unknown status: %s" % str( value ) )
if value == 'Done':
self.__data__['Error'] = ''
self.__data__["Status"] = value
@property
def Order( self ):
""" ro execution order getter """
self._notify()
opStatuses = [ op.Status for op in self.__operations__ ]
return opStatuses.index( "Waiting" ) if "Waiting" in opStatuses else len( opStatuses )
@property
def Error( self ):
""" error getter """
return self.__data__["Error"]
@Error.setter
def Error( self, value ):
""" error setter """
if type( value ) != str:
raise TypeError( "Error has to be a string!" )
self.__data__["Error"] = self._escapeStr( value, 255 )
def toSQL( self ):
""" prepare SQL INSERT or UPDATE statement """
colVals = [ ( "`%s`" % column, "'%s'" % value
if type( value ) in ( str, datetime.datetime ) else str( value ) if value != None else "NULL" )
for column, value in self.__data__.items()
if ( column == 'Error' or value ) and column not in ( "RequestID", "LastUpdate" ) ]
colVals.append( ( "`LastUpdate`", "UTC_TIMESTAMP()" ) )
query = []
if self.RequestID:
query.append( "UPDATE `Request` SET " )
query.append( ", ".join( [ "%s=%s" % item for item in colVals ] ) )
query.append( " WHERE `RequestID`=%d;\n" % self.RequestID )
else:
query.append( "INSERT INTO `Request` " )
columns = "(%s)" % ",".join( [ column for column, value in colVals ] )
values = "(%s)" % ",".join( [ value for column, value in colVals ] )
query.append( columns )
query.append( " VALUES %s;" % values )
return S_OK( "".join( query ) )
def cleanUpSQL( self ):
""" delete query for dirty operations """
query = []
if self.RequestID and self.__dirty:
opIDs = ",".join( [ str( opID ) for opID in self.__dirty ] )
query.append( "DELETE FROM `Operation` WHERE `RequestID`=%s AND `OperationID` IN (%s);\n" % ( self.RequestID,
opIDs ) )
for opID in self.__dirty:
query.append( "DELETE FROM `File` WHERE `OperationID`=%s;\n" % opID )
return query
# # digest
def toJSON( self ):
""" serialize to JSON format """
digest = dict( [( key, str( getattr( self, key ) ) if getattr( self, key ) else "" ) for key in self.__data__] )
digest["RequestID"] = self.RequestID
digest["__dirty"] = self.__dirty
digest["Operations"] = [op.toJSON()['Value'] for op in self]
return S_OK( digest )
def getDigest( self ):
""" return digest for request """
digest = ['Name:' + self.RequestName]
for op in self:
opDigest = [ str( item ) for item in ( op.Type, op.Type, op.Status, op.Order ) ]
if op.TargetSE:
opDigest.append( op.TargetSE )
if op.Catalog:
opDigest.append( op.Catalog )
if len( op ):
opFile = op[0]
opDigest.append( opFile.LFN )
opDigest.append( ",...<%d files>" % len( op ) )
digest.append( ":".join( opDigest ) )
return S_OK( "\n".join( digest ) )
class Operation(Record):
"""
.. class:: Operation
:param long OperationID: OperationID as read from DB backend
:param long RequestID: parent RequestID
:param str Status: execution status
:param str Type: operation to perform
:param str Arguments: additional arguments
:param str SourceSE: source SE name
:param str TargetSE: target SE names as comma separated list
:param str Catalog: catalog to use as comma separated list
:param str Error: error string if any
:param Request parent: parent Request instance
"""
# # max files in a single operation
MAX_FILES = 10000
# # all states
ALL_STATES = ("Queued", "Waiting", "Scheduled", "Assigned", "Failed",
"Done", "Canceled")
# # final states
FINAL_STATES = ("Failed", "Done", "Canceled")
def __init__(self, fromDict=None):
""" c'tor
:param self: self reference
:param dict fromDict: attributes dictionary
"""
Record.__init__(self)
self._parent = None
# # sub-request attributes
# self.__data__ = dict.fromkeys( self.tableDesc()["Fields"].keys(), None )
now = datetime.datetime.utcnow().replace(microsecond=0)
self.__data__["SubmitTime"] = now
self.__data__["LastUpdate"] = now
self.__data__["CreationTime"] = now
self.__data__["OperationID"] = 0
self.__data__["RequestID"] = 0
self.__data__["Status"] = "Queued"
# # operation files
self.__files__ = TypedList(allowedTypes=File)
# # dirty fileIDs
self.__dirty = []
# # init from dict
fromDict = fromDict if fromDict else {}
self.__dirty = fromDict.get("__dirty", [])
if "__dirty" in fromDict:
del fromDict["__dirty"]
for fileDict in fromDict.get("Files", []):
self.addFile(File(fileDict))
if "Files" in fromDict:
del fromDict["Files"]
for key, value in fromDict.items():
if key not in self.__data__:
raise AttributeError("Unknown Operation attribute '%s'" % key)
if key != "Order" and value:
setattr(self, key, value)
@staticmethod
def tableDesc():
""" get table desc """
return { "Fields" :
{ "OperationID" : "INTEGER NOT NULL AUTO_INCREMENT",
"RequestID" : "INTEGER NOT NULL",
"Type" : "VARCHAR(64) NOT NULL",
"Status" : "ENUM('Waiting', 'Assigned', 'Queued', 'Done', 'Failed', 'Canceled', 'Scheduled') "\
"DEFAULT 'Queued'",
"Arguments" : "MEDIUMBLOB",
"Order" : "INTEGER NOT NULL",
"SourceSE" : "VARCHAR(255)",
"TargetSE" : "VARCHAR(255)",
"Catalog" : "VARCHAR(255)",
"Error": "VARCHAR(255)",
"CreationTime" : "DATETIME",
"SubmitTime" : "DATETIME",
"LastUpdate" : "DATETIME" },
'ForeignKeys': {'RequestID': 'Request.RequestID' },
"PrimaryKey" : "OperationID" }
# # protected methods for parent only
def _notify(self):
""" notify self about file status change """
fStatus = set(self.fileStatusList())
if fStatus == set(['Failed']):
# All files Failed -> Failed
newStatus = 'Failed'
elif 'Scheduled' in fStatus:
newStatus = 'Scheduled'
elif "Waiting" in fStatus:
newStatus = 'Queued'
elif 'Failed' in fStatus:
newStatus = 'Failed'
else:
self.__data__['Error'] = ''
newStatus = 'Done'
# If the status moved to Failed or Done, update the lastUpdate time
if newStatus in ('Failed', 'Done', 'Scheduled'):
if self.__data__["Status"] != newStatus:
self.LastUpdate = datetime.datetime.utcnow().replace(
microsecond=0)
self.__data__["Status"] = newStatus
if self._parent:
self._parent._notify()
def _setQueued(self, caller):
""" don't touch """
if caller == self._parent:
self.__data__["Status"] = "Queued"
def _setWaiting(self, caller):
""" don't touch as well """
if caller == self._parent:
self.__data__["Status"] = "Waiting"
# # Files arithmetics
def __contains__(self, opFile):
""" in operator """
return opFile in self.__files__
def __iadd__(self, opFile):
""" += operator """
if len(self) >= Operation.MAX_FILES:
raise RuntimeError("too many Files in a single Operation")
self.addFile(opFile)
return self
def addFile(self, opFile):
""" add :opFile: to operation """
if len(self) >= Operation.MAX_FILES:
raise RuntimeError("too many Files in a single Operation")
if opFile not in self:
self.__files__.append(opFile)
opFile._parent = self
self._notify()
# # helpers for looping
def __iter__(self):
""" files iterator """
return self.__files__.__iter__()
def __getitem__(self, i):
""" [] op for opFiles """
return self.__files__.__getitem__(i)
def __delitem__(self, i):
""" remove file from op, only if OperationID is NOT set """
if not self.OperationID:
self.__files__.__delitem__(i)
else:
if self[i].FileID:
self.__dirty.append(self[i].FileID)
self.__files__.__delitem__(i)
self._notify()
def __setitem__(self, i, opFile):
""" overwrite opFile """
self.__files__._typeCheck(opFile)
toDelete = self[i]
if toDelete.FileID:
self.__dirty.append(toDelete.FileID)
self.__files__.__setitem__(i, opFile)
opFile._parent = self
self._notify()
def fileStatusList(self):
""" get list of files statuses """
return [subFile.Status for subFile in self]
def __nonzero__(self):
""" for comparisons
"""
return True
def __len__(self):
""" nb of subFiles """
return len(self.__files__)
# # properties
@property
def RequestID(self):
""" RequestID getter (RO) """
return self._parent.RequestID if self._parent else -1
@RequestID.setter
def RequestID(self, value):
""" can't set RequestID by hand """
self.__data__[
"RequestID"] = self._parent.RequestID if self._parent else -1
@property
def OperationID(self):
""" OperationID getter """
return self.__data__["OperationID"]
@OperationID.setter
def OperationID(self, value):
""" OperationID setter """
self.__data__["OperationID"] = long(value) if value else 0
@property
def Type(self):
""" operation type prop """
return self.__data__["Type"]
@Type.setter
def Type(self, value):
""" operation type setter """
self.__data__["Type"] = str(value)
@property
def Arguments(self):
""" arguments getter """
return self.__data__["Arguments"]
@Arguments.setter
def Arguments(self, value):
""" arguments setter """
self.__data__["Arguments"] = value if value else ""
@property
def SourceSE(self):
""" source SE prop """
return self.__data__["SourceSE"] if self.__data__["SourceSE"] else ""
@SourceSE.setter
def SourceSE(self, value):
""" source SE setter """
value = ",".join(self._uniqueList(value))
if len(value) > 256:
raise ValueError("SourceSE list too long")
self.__data__["SourceSE"] = str(value)[:255] if value else ""
@property
def sourceSEList(self):
""" helper property returning source SEs as a list"""
return self.SourceSE.split(",")
@property
def TargetSE(self):
""" target SE prop """
return self.__data__["TargetSE"] if self.__data__["TargetSE"] else ""
@TargetSE.setter
def TargetSE(self, value):
""" target SE setter """
value = ",".join(self._uniqueList(value))
if len(value) > 256:
raise ValueError("TargetSE list too long")
self.__data__["TargetSE"] = value[:255] if value else ""
@property
def targetSEList(self):
""" helper property returning target SEs as a list"""
return self.TargetSE.split(",")
@property
def Catalog(self):
""" catalog prop """
return self.__data__["Catalog"]
@Catalog.setter
def Catalog(self, value):
""" catalog setter """
# FIXME
######### THIS IS A TEMPORARY HOT FIX MEANT TO SMOOTH THE LFC->DFC MIGRATION
if value == "LcgFileCatalogCombined":
value = "FileCatalog,LcgFileCatalogCombined"
###########################################################################
value = ",".join(self._uniqueList(value))
if len(value) > 255:
raise ValueError("Catalog list too long")
self.__data__["Catalog"] = value if value else ""
@property
def catalogList(self):
""" helper property returning catalogs as list """
return self.__data__["Catalog"].split(",")
@property
def Error(self):
""" error prop """
return self.__data__["Error"]
@Error.setter
def Error(self, value):
""" error setter """
if type(value) != str:
raise TypeError("Error has to be a string!")
self.__data__["Error"] = self._escapeStr(value[:240], 255)
@property
def Status(self):
""" Status prop """
return self.__data__["Status"]
@Status.setter
def Status(self, value):
""" Status setter """
if value not in Operation.ALL_STATES:
raise ValueError("unknown Status '%s'" % str(value))
if self.__files__:
self._notify()
else:
# If the status moved to Failed or Done, update the lastUpdate time
if value in ('Failed', 'Done'):
if self.__data__["Status"] != value:
self.LastUpdate = datetime.datetime.utcnow().replace(
microsecond=0)
self.__data__["Status"] = value
if self._parent:
self._parent._notify()
if self.__data__['Status'] == 'Done':
self.__data__['Error'] = ''
@property
def Order(self):
""" order prop """
if self._parent:
self.__data__["Order"] = self._parent.indexOf(
self) if self._parent else -1
return self.__data__["Order"]
@property
def CreationTime(self):
""" operation creation time prop """
return self.__data__["CreationTime"]
@CreationTime.setter
def CreationTime(self, value=None):
""" creation time setter """
if type(value) not in (datetime.datetime, str):
raise TypeError("CreationTime should be a datetime.datetime!")
if type(value) == str:
value = datetime.datetime.strptime(
value.split(".")[0], '%Y-%m-%d %H:%M:%S')
self.__data__["CreationTime"] = value
@property
def SubmitTime(self):
""" subrequest's submit time prop """
return self.__data__["SubmitTime"]
@SubmitTime.setter
def SubmitTime(self, value=None):
""" submit time setter """
if type(value) not in (datetime.datetime, str):
raise TypeError("SubmitTime should be a datetime.datetime!")
if type(value) == str:
value = datetime.datetime.strptime(
value.split(".")[0], '%Y-%m-%d %H:%M:%S')
self.__data__["SubmitTime"] = value
@property
def LastUpdate(self):
""" last update prop """
return self.__data__["LastUpdate"]
@LastUpdate.setter
def LastUpdate(self, value=None):
""" last update setter """
if type(value) not in (datetime.datetime, str):
raise TypeError("LastUpdate should be a datetime.datetime!")
if type(value) == str:
value = datetime.datetime.strptime(
value.split(".")[0], '%Y-%m-%d %H:%M:%S')
self.__data__["LastUpdate"] = value
if self._parent:
self._parent.LastUpdate = value
def __str__(self):
""" str operator """
return str(self.toJSON()["Value"])
def toSQL(self):
""" get SQL INSERT or UPDATE statement """
if not getattr(self, "RequestID"):
raise AttributeError("RequestID not set")
colVals = [
("`%s`" % column,
"'%s'" % getattr(self, column) if type(getattr(self, column))
in (str, datetime.datetime) else str(getattr(self, column))
if getattr(self, column) != None else "NULL")
for column in self.__data__
if (column == 'Error' or getattr(self, column)) and column not in (
"OperationID", "LastUpdate", "Order")
]
colVals.append(("`LastUpdate`", "UTC_TIMESTAMP()"))
colVals.append(("`Order`", str(self.Order)))
# colVals.append( ( "`Status`", "'%s'" % str(self.Status) ) )
query = []
if self.OperationID:
query.append("UPDATE `Operation` SET ")
query.append(", ".join(["%s=%s" % item for item in colVals]))
query.append(" WHERE `OperationID`=%d;\n" % self.OperationID)
else:
query.append("INSERT INTO `Operation` ")
columns = "(%s)" % ",".join([column for column, value in colVals])
values = "(%s)" % ",".join([value for column, value in colVals])
query.append(columns)
query.append(" VALUES %s;\n" % values)
return S_OK("".join(query))
def cleanUpSQL(self):
""" query deleting dirty records from File table """
if self.OperationID and self.__dirty:
fIDs = ",".join([str(fid) for fid in self.__dirty])
return "DELETE FROM `File` WHERE `OperationID` = %s AND `FileID` IN (%s);\n" % (
self.OperationID, fIDs)
def toJSON(self):
""" get json digest """
digest = dict([(key,
str(getattr(self, key)) if getattr(self, key) else "")
for key in self.__data__])
digest["RequestID"] = str(self.RequestID)
digest["Order"] = str(self.Order)
if self.__dirty:
digest["__dirty"] = self.__dirty
digest["Files"] = [opFile.toJSON()['Value'] for opFile in self]
return S_OK(digest)
class Request( Record ):
"""
.. class:: Request
:param int RequestID: requestID
:param str Name: request' name
:param str OwnerDN: request's owner DN
:param str OwnerGroup: request owner group
:param str Setup: DIRAC setup
:param str SourceComponent: whatever
:param int JobID: jobID
:param datetime.datetime CreationTime: UTC datetime
:param datetime.datetime SubmissionTime: UTC datetime
:param datetime.datetime LastUpdate: UTC datetime
:param str Status: request's status
:param TypedList operations: list of operations
"""
ALL_STATES = ( "Waiting", "Failed", "Done", "Scheduled", "Assigned", "Canceled" )
FINAL_STATES = ( "Done", "Failed", "Canceled" )
def __init__( self, fromDict = None ):
"""c'tor
:param self: self reference
"""
Record.__init__( self )
self.__waiting = None
now = datetime.datetime.utcnow().replace( microsecond = 0 )
self.__data__["CreationTime"] = now
self.__data__["SubmitTime"] = now
self.__data__["LastUpdate"] = now
self.__data__["Status"] = "Done"
self.__data__["JobID"] = 0
self.__data__["RequestID"] = 0
proxyInfo = getProxyInfo()
if proxyInfo["OK"]:
proxyInfo = proxyInfo["Value"]
if proxyInfo["validGroup"] and proxyInfo["validDN"]:
self.OwnerDN = proxyInfo["identity"]
self.OwnerGroup = proxyInfo["group"]
self.__dirty = []
self.__operations__ = TypedList( allowedTypes = Operation )
fromDict = fromDict if fromDict else {}
self.__dirty = fromDict.get( "__dirty", [] )
if "__dirty" in fromDict:
del fromDict["__dirty"]
for opDict in fromDict.get( "Operations", [] ):
self +=Operation( opDict )
if "Operations" in fromDict:
del fromDict["Operations"]
for key, value in fromDict.items():
if key not in self.__data__:
raise AttributeError( "Unknown Request attribute '%s'" % key )
if value:
setattr( self, key, value )
self._notify()
@staticmethod
def tableDesc():
""" get table desc """
return { "Fields" :
{ "RequestID" : "INTEGER NOT NULL AUTO_INCREMENT",
"RequestName" : "VARCHAR(255) NOT NULL",
"OwnerDN" : "VARCHAR(255)",
"OwnerGroup" : "VARCHAR(32)",
"Status" : "ENUM('Waiting', 'Assigned', 'Done', 'Failed', 'Canceled', 'Scheduled') DEFAULT 'Waiting'",
"Error" : "VARCHAR(255)",
"DIRACSetup" : "VARCHAR(32)",
"SourceComponent" : "BLOB",
"JobID" : "INTEGER DEFAULT 0",
"CreationTime" : "DATETIME",
"SubmitTime" : "DATETIME",
"LastUpdate" : "DATETIME" },
"PrimaryKey" : [ "RequestID" ],
'UniqueIndexes': {'RequestName' : [ 'RequestName'] }
}
def _notify( self ):
""" simple state machine for sub request statuses """
self.__waiting = None
# # update operations statuses
rStatus = "Waiting"
opStatusList = [ ( op.Status, op ) for op in self ]
self.__waiting = None
while opStatusList:
# # Scan all status in order!
opStatus, op = opStatusList.pop( 0 )
# # Failed -> Failed
if opStatus == "Failed":
rStatus = "Failed"
break
# Scheduled -> Scheduled
if opStatus == "Scheduled":
if self.__waiting == None:
self.__waiting = op
rStatus = "Scheduled"
# # First operation Queued becomes Waiting if no Waiting/Scheduled before
elif opStatus == "Queued":
if self.__waiting == None:
self.__waiting = op
op._setWaiting( self )
rStatus = "Waiting"
# # First operation Waiting is next to execute, others are queued
elif opStatus == "Waiting":
rStatus = "Waiting"
if self.__waiting == None:
self.__waiting = op
else:
op._setQueued( self )
# # All operations Done -> Done
elif opStatus == "Done" and self.__waiting == None:
rStatus = "Done"
self.__data__['Error'] = ''
self.Status = rStatus
def getWaiting( self ):
""" get waiting operation if any """
# # update states
self._notify()
return S_OK( self.__waiting )
# # Operation arithmetics
def __contains__( self, operation ):
""" in operator
:param self: self reference
:param Operation subRequest: a subRequest
"""
return bool( operation in self.__operations__ )
def __iadd__( self, operation ):
""" += operator for subRequest
:param self: self reference
:param Operation operation: sub-request to add
"""
if operation not in self:
self.__operations__.append( operation )
operation._parent = self
self._notify()
return self
def insertBefore( self, newOperation, existingOperation ):
""" insert :newOperation: just before :existingOperation:
:param self: self reference
:param Operation newOperation: Operation to be inserted
:param Operation existingOperation: previous Operation sibling
"""
if existingOperation not in self:
return S_ERROR( "%s is not in" % existingOperation )
if newOperation in self:
return S_ERROR( "%s is already in" % newOperation )
self.__operations__.insert( self.__operations__.index( existingOperation ), newOperation )
newOperation._parent = self
self._notify()
return S_OK()
def insertAfter( self, newOperation, existingOperation ):
""" insert :newOperation: just after :existingOperation:
:param self: self reference
:param Operation newOperation: Operation to be inserted
:param Operation existingOperation: next Operation sibling
"""
if existingOperation not in self:
return S_ERROR( "%s is not in" % existingOperation )
if newOperation in self:
return S_ERROR( "%s is already in" % newOperation )
self.__operations__.insert( self.__operations__.index( existingOperation ) + 1, newOperation )
newOperation._parent = self
self._notify()
return S_OK()
def addOperation( self, operation ):
""" add :operation: to list of Operations
:param self: self reference
:param Operation operation: Operation to be inserted
"""
if operation in self:
return S_ERROR( "This operation is already in!!!" )
self +=operation
return S_OK()
def isEmpty( self ):
""" Evaluate if the request is empty
"""
return len( self.__operations__ ) == 0
def __iter__( self ):
""" iterator for sub-request """
return self.__operations__.__iter__()
def __getitem__( self, i ):
""" [] op for sub requests """
return self.__operations__.__getitem__( i )
def __setitem__( self, i, value ):
""" self[i] = val """
self.__operations__._typeCheck( value )
if self[i].OperationID:
self.__dirty.append( self[i].OperationID )
self.__operations__.__setitem__( i, value )
value._parent = self
self._notify()
def __delitem__( self, i ):
""" del self[i]"""
if not self.RequestID:
self.__operations__.__delitem__( i )
else:
opId = self[i].OperationID
if opId:
self.__dirty.append( opId )
self.__operations__.__delitem__( i )
self._notify()
def indexOf( self, subReq ):
""" return index of subReq (execution order) """
return self.__operations__.index( subReq ) if subReq in self else -1
def __nonzero__( self ):
""" for comparisons
"""
return True
def __len__( self ):
""" nb of subRequests """
return len( self.__operations__ )
def __str__( self ):
""" str operator """
return str( self.toJSON()["Value"] )
def subStatusList( self ):
""" list of statuses for all operations """
return [ subReq.Status for subReq in self ]
# # properties
@property
def RequestID( self ):
""" request ID getter """
return self.__data__["RequestID"]
@RequestID.setter
def RequestID( self, value ):
""" requestID setter (shouldn't be RO???) """
self.__data__["RequestID"] = long( value ) if value else 0
@property
def RequestName( self ):
""" request's name getter """
return self.__data__["RequestName"]
@RequestName.setter
def RequestName( self, value ):
""" request name setter """
if type( value ) != str:
raise TypeError( "RequestName should be a string" )
self.__data__["RequestName"] = value[:128]
@property
def OwnerDN( self ):
""" request owner DN getter """
return self.__data__["OwnerDN"]
@OwnerDN.setter
def OwnerDN( self, value ):
""" request owner DN setter """
if type( value ) != str:
raise TypeError( "OwnerDN should be a string!" )
self.__data__["OwnerDN"] = value
@property
def OwnerGroup( self ):
""" request owner group getter """
return self.__data__["OwnerGroup"]
@OwnerGroup.setter
def OwnerGroup( self, value ):
""" request owner group setter """
if type( value ) != str:
raise TypeError( "OwnerGroup should be a string!" )
self.__data__["OwnerGroup"] = value
@property
def DIRACSetup( self ):
""" DIRAC setup getter """
return self.__data__["DIRACSetup"]
@DIRACSetup.setter
def DIRACSetup( self, value ):
""" DIRAC setup setter """
if type( value ) != str:
raise TypeError( "setup should be a string!" )
self.__data__["DIRACSetup"] = value
@property
def SourceComponent( self ):
""" source component getter """
return self.__data__["SourceComponent"]
@SourceComponent.setter
def SourceComponent( self, value ):
""" source component setter """
if type( value ) != str:
raise TypeError( "Setup should be a string!" )
self.__data__["SourceComponent"] = value
@property
def JobID( self ):
""" jobID getter """
return self.__data__["JobID"]
@JobID.setter
def JobID( self, value = 0 ):
""" jobID setter """
self.__data__["JobID"] = long( value ) if value else 0
@property
def CreationTime( self ):
""" creation time getter """
return self.__data__["CreationTime"]
@CreationTime.setter
def CreationTime( self, value = None ):
""" creation time setter """
if type( value ) not in ( datetime.datetime, str ) :
raise TypeError( "CreationTime should be a datetime.datetime!" )
if type( value ) == str:
value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' )
self.__data__["CreationTime"] = value
@property
def SubmitTime( self ):
""" request's submission time getter """
return self.__data__["SubmitTime"]
@SubmitTime.setter
def SubmitTime( self, value = None ):
""" submission time setter """
if type( value ) not in ( datetime.datetime, str ):
raise TypeError( "SubmitTime should be a datetime.datetime!" )
if type( value ) == str:
value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' )
self.__data__["SubmitTime"] = value
@property
def LastUpdate( self ):
""" last update getter """
return self.__data__["LastUpdate"]
@LastUpdate.setter
def LastUpdate( self, value = None ):
""" last update setter """
if type( value ) not in ( datetime.datetime, str ):
raise TypeError( "LastUpdate should be a datetime.datetime!" )
if type( value ) == str:
value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' )
self.__data__["LastUpdate"] = value
@property
def Status( self ):
""" status getter """
self._notify()
return self.__data__["Status"]
@Status.setter
def Status( self, value ):
""" status setter """
if value not in Request.ALL_STATES:
raise ValueError( "Unknown status: %s" % str( value ) )
# If the status moved to Failed or Done, update the lastUpdate time
if value in ( 'Done', 'Failed' ):
if value != self.__data__["Status"]:
self.LastUpdate = datetime.datetime.utcnow().replace( microsecond = 0 )
if value == 'Done':
self.__data__['Error'] = ''
self.__data__["Status"] = value
@property
def Order( self ):
""" ro execution order getter """
self._notify()
opStatuses = [ op.Status for op in self.__operations__ ]
return opStatuses.index( "Waiting" ) if "Waiting" in opStatuses else len( opStatuses )
@property
def Error( self ):
""" error getter """
return self.__data__["Error"]
@Error.setter
def Error( self, value ):
""" error setter """
if type( value ) != str:
raise TypeError( "Error has to be a string!" )
self.__data__["Error"] = self._escapeStr( value[:240], 255 )
def toSQL( self ):
""" prepare SQL INSERT or UPDATE statement """
colVals = [ ( "`%s`" % column, "'%s'" % value
if type( value ) in ( str, datetime.datetime ) else str( value ) if value != None else "NULL" )
for column, value in self.__data__.items()
if ( column == 'Error' or value ) and column not in ( "RequestID", "LastUpdate" ) ]
colVals.append( ( "`LastUpdate`", "UTC_TIMESTAMP()" ) )
query = []
if self.RequestID:
query.append( "UPDATE `Request` SET " )
query.append( ", ".join( [ "%s=%s" % item for item in colVals ] ) )
query.append( " WHERE `RequestID`=%d;\n" % self.RequestID )
else:
query.append( "INSERT INTO `Request` " )
columns = "(%s)" % ",".join( [ column for column, value in colVals ] )
values = "(%s)" % ",".join( [ value for column, value in colVals ] )
query.append( columns )
query.append( " VALUES %s;" % values )
return S_OK( "".join( query ) )
def cleanUpSQL( self ):
""" delete query for dirty operations """
query = []
if self.RequestID and self.__dirty:
opIDs = ",".join( [ str( opID ) for opID in self.__dirty ] )
query.append( "DELETE FROM `Operation` WHERE `RequestID`=%s AND `OperationID` IN (%s);\n" % ( self.RequestID,
opIDs ) )
for opID in self.__dirty:
query.append( "DELETE FROM `File` WHERE `OperationID`=%s;\n" % opID )
return query
# # digest
def toJSON( self ):
""" serialize to JSON format """
digest = dict( [( key, str( getattr( self, key ) ) if getattr( self, key ) else "" ) for key in self.__data__] )
digest["RequestID"] = self.RequestID
digest["__dirty"] = self.__dirty
digest["Operations"] = [op.toJSON()['Value'] for op in self]
return S_OK( digest )
def getDigest( self ):
""" return digest for request """
digest = ['Name:' + self.RequestName]
for op in self:
opDigest = [ str( item ) for item in ( op.Type, op.Type, op.Status, op.Order ) ]
if op.TargetSE:
opDigest.append( op.TargetSE )
if op.Catalog:
opDigest.append( op.Catalog )
if len( op ):
opFile = op[0]
opDigest.append( opFile.LFN )
opDigest.append( ",...<%d files>" % len( op ) )
digest.append( ":".join( opDigest ) )
return S_OK( "\n".join( digest ) )
def optimize( self ):
""" Merges together the operations that can be merged. They need to have the following arguments equal:
* Type
* Arguments
* SourceSE
* TargetSE
* Catalog
It also makes sure that the maximum number of Files in an Operation is never overcome.
CAUTION: this method is meant to be called before inserting into the DB.
So if the RequestId is not 0, we don't touch
:return S_ERROR if the Request should not be optimized (because already in the DB
S_OK(True) if a optimization was carried out
S_OK(False) if no optimization were carried out
"""
# If the RequestID is not the default one (0), it probably means
# the Request is already in the DB, so we don't touch anything
if self.RequestID:
return S_ERROR( "Cannot optimize because Request seems to be already in the DB (RequestID %s)" % self.RequestID )
# Set to True if the request could be optimized
optimized = False
# Recognise Failover request series
repAndRegList = []
removeRepList = []
i = 0
while i < len( self.__operations__ ):
insertNow = True
if i < len( self.__operations__ ) - 1:
op1 = self.__operations__[i]
op2 = self.__operations__[i + 1]
if getattr( op1, 'Type' ) == 'ReplicateAndRegister' and \
getattr( op2, 'Type' ) == 'RemoveReplica':
fileSetA = set( list( f.LFN for f in op1 ) )
fileSetB = set( list( f.LFN for f in op2 ) )
if fileSetA == fileSetB:
# Source is useless if failover
if 'FAILOVER' in op1.SourceSE:
op1.SourceSE = ''
repAndRegList.append( ( op1.TargetSE, op1 ) )
removeRepList.append( ( op2.TargetSE, op2 ) )
del self.__operations__[i]
del self.__operations__[i]
# If we are at the end of the request, we must insert the new operations
insertNow = ( i == len( self.__operations__ ) )
# print i, self.__operations__[i].Type if i < len( self.__operations__ ) else None, len( repAndRegList ), insertNow
if insertNow:
if repAndRegList:
# We must insert the new operations there
# i.e. insert before operation i (if it exists)
# Replication first, removeReplica next
optimized = True
insertBefore = self.__operations__[i] if i < len( self.__operations__ ) else None
# print 'Insert new operations before', insertBefore
for op in \
[op for _targetSE, op in sorted( repAndRegList )] + \
[op for _targetSE, op in sorted( removeRepList )]:
_res = self.insertBefore( op, insertBefore ) if insertBefore else self.addOperation( op )
# Skip the newly inserted operation
i += 1
repAndRegList = []
removeRepList = []
else:
# Skip current operation
i += 1
else:
# Just to show that in that case we don't increment i
pass
# List of attributes that must be equal for operations to be merged
attrList = ["Type", "Arguments", "SourceSE", "TargetSE", "Catalog" ]
i = 0
while i < len( self.__operations__ ):
while i < len( self.__operations__ ) - 1:
# Some attributes need to be the same
attrMismatch = False
for attr in attrList:
if getattr( self.__operations__[i], attr ) != getattr( self.__operations__[i + 1], attr ):
attrMismatch = True
break
if attrMismatch:
break
# We do not do the merge if there are common files in the operations
fileSetA = set( list( f.LFN for f in self.__operations__[i] ) )
fileSetB = set( list( f.LFN for f in self.__operations__[i + 1] ) )
if fileSetA & fileSetB:
break
# There is a maximum number of files one can add into an operation
try:
while len( self.__operations__[i + 1] ):
self.__operations__[i] += self.__operations__[i + 1][0]
del self.__operations__[i + 1][0]
optimized = True
del self.__operations__[i + 1]
except RuntimeError:
i += 1
i += 1
return S_OK( optimized )
class Request(Record):
"""
.. class:: Request
:param int RequestID: requestID
:param str Name: request' name
:param str OwnerDN: request's owner DN
:param str OwnerGroup: request owner group
:param str Setup: DIRAC setup
:param str SourceComponent: whatever
:param int JobID: jobID
:param datetime.datetime CreationTime: UTC datetime
:param datetime.datetime SubmissionTime: UTC datetime
:param datetime.datetime LastUpdate: UTC datetime
:param str Status: request's status
:param TypedList operations: list of operations
"""
ALL_STATES = ("Waiting", "Failed", "Done", "Scheduled", "Assigned",
"Canceled")
FINAL_STATES = ("Done", "Failed", "Canceled")
def __init__(self, fromDict=None):
"""c'tor
:param self: self reference
"""
Record.__init__(self)
self.__waiting = None
now = datetime.datetime.utcnow().replace(microsecond=0)
self.__data__["CreationTime"] = now
self.__data__["SubmitTime"] = now
self.__data__["LastUpdate"] = now
self.__data__["Status"] = "Done"
self.__data__["JobID"] = 0
self.__data__["RequestID"] = 0
proxyInfo = getProxyInfo()
if proxyInfo["OK"]:
proxyInfo = proxyInfo["Value"]
if proxyInfo["validGroup"] and proxyInfo["validDN"]:
self.OwnerDN = proxyInfo["identity"]
self.OwnerGroup = proxyInfo["group"]
self.__dirty = []
self.__operations__ = TypedList(allowedTypes=Operation)
fromDict = fromDict if fromDict else {}
self.__dirty = fromDict.get("__dirty", [])
if "__dirty" in fromDict:
del fromDict["__dirty"]
for opDict in fromDict.get("Operations", []):
self += Operation(opDict)
if "Operations" in fromDict:
del fromDict["Operations"]
for key, value in fromDict.items():
if key not in self.__data__:
raise AttributeError("Unknown Request attribute '%s'" % key)
if value:
setattr(self, key, value)
self._notify()
@staticmethod
def tableDesc():
""" get table desc """
return {
"Fields": {
"RequestID": "INTEGER NOT NULL AUTO_INCREMENT",
"RequestName": "VARCHAR(255) NOT NULL",
"OwnerDN": "VARCHAR(255)",
"OwnerGroup": "VARCHAR(32)",
"Status":
"ENUM('Waiting', 'Assigned', 'Done', 'Failed', 'Canceled', 'Scheduled') DEFAULT 'Waiting'",
"Error": "VARCHAR(255)",
"DIRACSetup": "VARCHAR(32)",
"SourceComponent": "BLOB",
"JobID": "INTEGER DEFAULT 0",
"CreationTime": "DATETIME",
"SubmitTime": "DATETIME",
"LastUpdate": "DATETIME"
},
"PrimaryKey": ["RequestID"],
'UniqueIndexes': {
'RequestName': ['RequestName']
}
}
def _notify(self):
""" simple state machine for sub request statuses """
self.__waiting = None
# # update operations statuses
rStatus = "Waiting"
opStatusList = [(op.Status, op) for op in self]
self.__waiting = None
while opStatusList:
# # Scan all status in order!
opStatus, op = opStatusList.pop(0)
# # Failed -> Failed
if opStatus == "Failed":
rStatus = "Failed"
break
# Scheduled -> Scheduled
if opStatus == "Scheduled":
if self.__waiting == None:
self.__waiting = op
rStatus = "Scheduled"
# # First operation Queued becomes Waiting if no Waiting/Scheduled before
elif opStatus == "Queued":
if self.__waiting == None:
self.__waiting = op
op._setWaiting(self)
rStatus = "Waiting"
# # First operation Waiting is next to execute, others are queued
elif opStatus == "Waiting":
rStatus = "Waiting"
if self.__waiting == None:
self.__waiting = op
else:
op._setQueued(self)
# # All operations Done -> Done
elif opStatus == "Done" and self.__waiting == None:
rStatus = "Done"
self.__data__['Error'] = ''
self.Status = rStatus
def getWaiting(self):
""" get waiting operation if any """
# # update states
self._notify()
return S_OK(self.__waiting)
# # Operation arithmetics
def __contains__(self, operation):
""" in operator
:param self: self reference
:param Operation subRequest: a subRequest
"""
return bool(operation in self.__operations__)
def __iadd__(self, operation):
""" += operator for subRequest
:param self: self reference
:param Operation operation: sub-request to add
"""
if operation not in self:
self.__operations__.append(operation)
operation._parent = self
self._notify()
return self
def insertBefore(self, newOperation, existingOperation):
""" insert :newOperation: just before :existingOperation:
:param self: self reference
:param Operation newOperation: Operation to be inserted
:param Operation existingOperation: previous Operation sibling
"""
if existingOperation not in self:
return S_ERROR("%s is not in" % existingOperation)
if newOperation in self:
return S_ERROR("%s is already in" % newOperation)
self.__operations__.insert(
self.__operations__.index(existingOperation), newOperation)
newOperation._parent = self
self._notify()
return S_OK()
def insertAfter(self, newOperation, existingOperation):
""" insert :newOperation: just after :existingOperation:
:param self: self reference
:param Operation newOperation: Operation to be inserted
:param Operation existingOperation: next Operation sibling
"""
if existingOperation not in self:
return S_ERROR("%s is not in" % existingOperation)
if newOperation in self:
return S_ERROR("%s is already in" % newOperation)
self.__operations__.insert(
self.__operations__.index(existingOperation) + 1, newOperation)
newOperation._parent = self
self._notify()
return S_OK()
def addOperation(self, operation):
""" add :operation: to list of Operations
:param self: self reference
:param Operation operation: Operation to be inserted
"""
if operation in self:
return S_ERROR("This operation is already in!!!")
self += operation
return S_OK()
def isEmpty(self):
""" Evaluate if the request is empty
"""
return len(self.__operations__) == 0
def __iter__(self):
""" iterator for sub-request """
return self.__operations__.__iter__()
def __getitem__(self, i):
""" [] op for sub requests """
return self.__operations__.__getitem__(i)
def __setitem__(self, i, value):
""" self[i] = val """
self.__operations__._typeCheck(value)
if self[i].OperationID:
self.__dirty.append(self[i].OperationID)
self.__operations__.__setitem__(i, value)
value._parent = self
self._notify()
def __delitem__(self, i):
""" del self[i]"""
if not self.RequestID:
self.__operations__.__delitem__(i)
else:
opId = self[i].OperationID
if opId:
self.__dirty.append(opId)
self.__operations__.__delitem__(i)
self._notify()
def indexOf(self, subReq):
""" return index of subReq (execution order) """
return self.__operations__.index(subReq) if subReq in self else -1
def __nonzero__(self):
""" for comparisons
"""
return True
def __len__(self):
""" nb of subRequests """
return len(self.__operations__)
def __str__(self):
""" str operator """
return str(self.toJSON()["Value"])
def subStatusList(self):
""" list of statuses for all operations """
return [subReq.Status for subReq in self]
# # properties
@property
def RequestID(self):
""" request ID getter """
return self.__data__["RequestID"]
@RequestID.setter
def RequestID(self, value):
""" requestID setter (shouldn't be RO???) """
self.__data__["RequestID"] = long(value) if value else 0
@property
def RequestName(self):
""" request's name getter """
return self.__data__["RequestName"]
@RequestName.setter
def RequestName(self, value):
""" request name setter """
if type(value) != str:
raise TypeError("RequestName should be a string")
self.__data__["RequestName"] = value[:128]
@property
def OwnerDN(self):
""" request owner DN getter """
return self.__data__["OwnerDN"]
@OwnerDN.setter
def OwnerDN(self, value):
""" request owner DN setter """
if type(value) != str:
raise TypeError("OwnerDN should be a string!")
self.__data__["OwnerDN"] = value
@property
def OwnerGroup(self):
""" request owner group getter """
return self.__data__["OwnerGroup"]
@OwnerGroup.setter
def OwnerGroup(self, value):
""" request owner group setter """
if type(value) != str:
raise TypeError("OwnerGroup should be a string!")
self.__data__["OwnerGroup"] = value
@property
def DIRACSetup(self):
""" DIRAC setup getter """
return self.__data__["DIRACSetup"]
@DIRACSetup.setter
def DIRACSetup(self, value):
""" DIRAC setup setter """
if type(value) != str:
raise TypeError("setup should be a string!")
self.__data__["DIRACSetup"] = value
@property
def SourceComponent(self):
""" source component getter """
return self.__data__["SourceComponent"]
@SourceComponent.setter
def SourceComponent(self, value):
""" source component setter """
if type(value) != str:
raise TypeError("Setup should be a string!")
self.__data__["SourceComponent"] = value
@property
def JobID(self):
""" jobID getter """
return self.__data__["JobID"]
@JobID.setter
def JobID(self, value=0):
""" jobID setter """
self.__data__["JobID"] = long(value) if value else 0
@property
def CreationTime(self):
""" creation time getter """
return self.__data__["CreationTime"]
@CreationTime.setter
def CreationTime(self, value=None):
""" creation time setter """
if type(value) not in (datetime.datetime, str):
raise TypeError("CreationTime should be a datetime.datetime!")
if type(value) == str:
value = datetime.datetime.strptime(
value.split(".")[0], '%Y-%m-%d %H:%M:%S')
self.__data__["CreationTime"] = value
@property
def SubmitTime(self):
""" request's submission time getter """
return self.__data__["SubmitTime"]
@SubmitTime.setter
def SubmitTime(self, value=None):
""" submission time setter """
if type(value) not in (datetime.datetime, str):
raise TypeError("SubmitTime should be a datetime.datetime!")
if type(value) == str:
value = datetime.datetime.strptime(
value.split(".")[0], '%Y-%m-%d %H:%M:%S')
self.__data__["SubmitTime"] = value
@property
def LastUpdate(self):
""" last update getter """
return self.__data__["LastUpdate"]
@LastUpdate.setter
def LastUpdate(self, value=None):
""" last update setter """
if type(value) not in (datetime.datetime, str):
raise TypeError("LastUpdate should be a datetime.datetime!")
if type(value) == str:
value = datetime.datetime.strptime(
value.split(".")[0], '%Y-%m-%d %H:%M:%S')
self.__data__["LastUpdate"] = value
@property
def Status(self):
""" status getter """
self._notify()
return self.__data__["Status"]
@Status.setter
def Status(self, value):
""" status setter """
if value not in Request.ALL_STATES:
raise ValueError("Unknown status: %s" % str(value))
# If the status moved to Failed or Done, update the lastUpdate time
if value in ('Done', 'Failed'):
if value != self.__data__["Status"]:
self.LastUpdate = datetime.datetime.utcnow().replace(
microsecond=0)
if value == 'Done':
self.__data__['Error'] = ''
self.__data__["Status"] = value
@property
def Order(self):
""" ro execution order getter """
self._notify()
opStatuses = [op.Status for op in self.__operations__]
return opStatuses.index("Waiting") if "Waiting" in opStatuses else len(
opStatuses)
@property
def Error(self):
""" error getter """
return self.__data__["Error"]
@Error.setter
def Error(self, value):
""" error setter """
if type(value) != str:
raise TypeError("Error has to be a string!")
self.__data__["Error"] = self._escapeStr(value, 255)
def toSQL(self):
""" prepare SQL INSERT or UPDATE statement """
colVals = [
("`%s`" % column,
"'%s'" % value if type(value) in (str, datetime.datetime) else
str(value) if value != None else "NULL")
for column, value in self.__data__.items()
if (column == 'Error' or value) and column not in ("RequestID",
"LastUpdate")
]
colVals.append(("`LastUpdate`", "UTC_TIMESTAMP()"))
query = []
if self.RequestID:
query.append("UPDATE `Request` SET ")
query.append(", ".join(["%s=%s" % item for item in colVals]))
query.append(" WHERE `RequestID`=%d;\n" % self.RequestID)
else:
query.append("INSERT INTO `Request` ")
columns = "(%s)" % ",".join([column for column, value in colVals])
values = "(%s)" % ",".join([value for column, value in colVals])
query.append(columns)
query.append(" VALUES %s;" % values)
return S_OK("".join(query))
def cleanUpSQL(self):
""" delete query for dirty operations """
query = []
if self.RequestID and self.__dirty:
opIDs = ",".join([str(opID) for opID in self.__dirty])
query.append(
"DELETE FROM `Operation` WHERE `RequestID`=%s AND `OperationID` IN (%s);\n"
% (self.RequestID, opIDs))
for opID in self.__dirty:
query.append("DELETE FROM `File` WHERE `OperationID`=%s;\n" %
opID)
return query
# # digest
def toJSON(self):
""" serialize to JSON format """
digest = dict([(key,
str(getattr(self, key)) if getattr(self, key) else "")
for key in self.__data__])
digest["RequestID"] = self.RequestID
digest["__dirty"] = self.__dirty
digest["Operations"] = [op.toJSON()['Value'] for op in self]
return S_OK(digest)
def getDigest(self):
""" return digest for request """
digest = ['Name:' + self.RequestName]
for op in self:
opDigest = [
str(item) for item in (op.Type, op.Type, op.Status, op.Order)
]
if op.TargetSE:
opDigest.append(op.TargetSE)
if op.Catalog:
opDigest.append(op.Catalog)
if len(op):
opFile = op[0]
opDigest.append(opFile.LFN)
opDigest.append(",...<%d files>" % len(op))
digest.append(":".join(opDigest))
return S_OK("\n".join(digest))
def optimize(self):
""" Merges together the operations that can be merged. They need to have the following arguments equal:
* Type
* Arguments
* SourceSE
* TargetSE
* Catalog
It also makes sure that the maximum number of Files in an Operation is never overcome.
CAUTION: this method is meant to be called before inserting into the DB.
So if the RequestId is not 0, we don't touch
:return S_ERROR if the Request should not be optimized (because already in the DB
S_OK(True) if a optimization was carried out
S_OK(False) if no optimization were carried out
"""
# Set to True if the request could be optimized
optimized = False
# List of attributes that must be equal for operations to be merged
attrList = ["Type", "Arguments", "SourceSE", "TargetSE", "Catalog"]
i = 0
# If the RequestID is not the default one (0), it probably means
# the Request is already in the DB, so we don't touch anything
if self.RequestID != 0:
return S_ERROR(
"Cannot optimize because Request seems to be already in the DB (RequestID %s)"
% self.RequestID)
# We could do it with a single loop (the 2nd one), but by doing this,
# we can replace
# i += 1
# continue
#
# with
# break
#
# which is nicer in my opinion
while i < len(self.__operations__):
while (i + 1) < len(self.__operations__):
# Some attributes need to be the same
attrMismatch = False
for attr in attrList:
if getattr(self.__operations__[i], attr) != getattr(
self.__operations__[i + 1], attr):
attrMismatch = True
break
if attrMismatch:
break
# We do not do the merge if there are common files in the operations
fileSetA = set(list(f.LFN for f in self.__operations__[i]))
fileSetB = set(list(f.LFN for f in self.__operations__[i + 1]))
if len(fileSetA & fileSetB):
break
# There is a maximum number of files one can add into an operation
try:
while len(self.__operations__[i + 1]):
self.__operations__[i] += self.__operations__[i + 1][0]
del self.__operations__[i + 1][0]
optimized = True
del self.__operations__[i + 1]
except RuntimeError:
i += 1
i += 1
return S_OK(optimized)
class Operation( Record ):
"""
.. class:: Operation
:param long OperationID: OperationID as read from DB backend
:param long RequestID: parent RequestID
:param str Status: execution status
:param str Type: operation to perform
:param str Arguments: additional arguments
:param str SourceSE: source SE name
:param str TargetSE: target SE names as comma separated list
:param str Catalog: catalog to use as comma separated list
:param str Error: error string if any
:param Request parent: parent Request instance
"""
# # max files in a single operation
MAX_FILES = 10000
# # all states
ALL_STATES = ( "Queued", "Waiting", "Scheduled", "Assigned", "Failed", "Done", "Canceled" )
# # final states
FINAL_STATES = ( "Failed", "Done", "Canceled" )
def __init__( self, fromDict = None ):
""" c'tor
:param self: self reference
:param dict fromDict: attributes dictionary
"""
Record.__init__( self )
self._parent = None
# # sub-request attributes
# self.__data__ = dict.fromkeys( self.tableDesc()["Fields"].keys(), None )
now = datetime.datetime.utcnow().replace( microsecond = 0 )
self.__data__["SubmitTime"] = now
self.__data__["LastUpdate"] = now
self.__data__["CreationTime"] = now
self.__data__["OperationID"] = 0
self.__data__["RequestID"] = 0
self.__data__["Status"] = "Queued"
# # operation files
self.__files__ = TypedList( allowedTypes = File )
# # dirty fileIDs
self.__dirty = []
# # init from dict
fromDict = fromDict if fromDict else {}
self.__dirty = fromDict.get( "__dirty", [] )
if "__dirty" in fromDict:
del fromDict["__dirty"]
for fileDict in fromDict.get( "Files", [] ):
self.addFile( File( fileDict ) )
if "Files" in fromDict:
del fromDict["Files"]
for key, value in fromDict.items():
if key not in self.__data__:
raise AttributeError( "Unknown Operation attribute '%s'" % key )
if key != "Order" and value:
setattr( self, key, value )
@staticmethod
def tableDesc():
""" get table desc """
return { "Fields" :
{ "OperationID" : "INTEGER NOT NULL AUTO_INCREMENT",
"RequestID" : "INTEGER NOT NULL",
"Type" : "VARCHAR(64) NOT NULL",
"Status" : "ENUM('Waiting', 'Assigned', 'Queued', 'Done', 'Failed', 'Canceled', 'Scheduled') "\
"DEFAULT 'Queued'",
"Arguments" : "MEDIUMBLOB",
"Order" : "INTEGER NOT NULL",
"SourceSE" : "VARCHAR(255)",
"TargetSE" : "VARCHAR(255)",
"Catalog" : "VARCHAR(255)",
"Error": "VARCHAR(255)",
"CreationTime" : "DATETIME",
"SubmitTime" : "DATETIME",
"LastUpdate" : "DATETIME" },
'ForeignKeys': {'RequestID': 'Request.RequestID' },
"PrimaryKey" : "OperationID" }
# # protected methods for parent only
def _notify( self ):
""" notify self about file status change """
fStatus = set( self.fileStatusList() )
if fStatus == set( ['Failed'] ):
# All files Failed -> Failed
newStatus = 'Failed'
elif 'Scheduled' in fStatus:
newStatus = 'Scheduled'
elif "Waiting" in fStatus:
newStatus = 'Queued'
elif 'Failed' in fStatus:
newStatus = 'Failed'
else:
self.__data__['Error'] = ''
newStatus = 'Done'
# If the status moved to Failed or Done, update the lastUpdate time
if newStatus in ( 'Failed', 'Done', 'Scheduled' ):
if self.__data__["Status"] != newStatus:
self.LastUpdate = datetime.datetime.utcnow().replace( microsecond = 0 )
self.__data__["Status"] = newStatus
if self._parent:
self._parent._notify()
def _setQueued( self, caller ):
""" don't touch """
if caller == self._parent:
self.__data__["Status"] = "Queued"
def _setWaiting( self, caller ):
""" don't touch as well """
if caller == self._parent:
self.__data__["Status"] = "Waiting"
# # Files arithmetics
def __contains__( self, opFile ):
""" in operator """
return opFile in self.__files__
def __iadd__( self, opFile ):
""" += operator """
if len( self ) >= Operation.MAX_FILES:
raise RuntimeError( "too many Files in a single Operation" )
self.addFile( opFile )
return self
def addFile( self, opFile ):
""" add :opFile: to operation """
if len( self ) >= Operation.MAX_FILES:
raise RuntimeError( "too many Files in a single Operation" )
if opFile not in self:
self.__files__.append( opFile )
opFile._parent = self
self._notify()
# # helpers for looping
def __iter__( self ):
""" files iterator """
return self.__files__.__iter__()
def __getitem__( self, i ):
""" [] op for opFiles """
return self.__files__.__getitem__( i )
def __delitem__( self, i ):
""" remove file from op, only if OperationID is NOT set """
if not self.OperationID:
self.__files__.__delitem__( i )
else:
if self[i].FileID:
self.__dirty.append( self[i].FileID )
self.__files__.__delitem__( i )
self._notify()
def __setitem__( self, i, opFile ):
""" overwrite opFile """
self.__files__._typeCheck( opFile )
toDelete = self[i]
if toDelete.FileID:
self.__dirty.append( toDelete.FileID )
self.__files__.__setitem__( i, opFile )
opFile._parent = self
self._notify()
def fileStatusList( self ):
""" get list of files statuses """
return [ subFile.Status for subFile in self ]
def __nonzero__( self ):
""" for comparisons
"""
return True
def __len__( self ):
""" nb of subFiles """
return len( self.__files__ )
# # properties
@property
def RequestID( self ):
""" RequestID getter (RO) """
return self._parent.RequestID if self._parent else -1
@RequestID.setter
def RequestID( self, value ):
""" can't set RequestID by hand """
self.__data__["RequestID"] = self._parent.RequestID if self._parent else -1
@property
def OperationID( self ):
""" OperationID getter """
return self.__data__["OperationID"]
@OperationID.setter
def OperationID( self, value ):
""" OperationID setter """
self.__data__["OperationID"] = long( value ) if value else 0
@property
def Type( self ):
""" operation type prop """
return self.__data__["Type"]
@Type.setter
def Type( self, value ):
""" operation type setter """
self.__data__["Type"] = str( value )
@property
def Arguments( self ):
""" arguments getter """
return self.__data__["Arguments"]
@Arguments.setter
def Arguments( self, value ):
""" arguments setter """
self.__data__["Arguments"] = value if value else ""
@property
def SourceSE( self ):
""" source SE prop """
return self.__data__["SourceSE"] if self.__data__["SourceSE"] else ""
@SourceSE.setter
def SourceSE( self, value ):
""" source SE setter """
value = ",".join( self._uniqueList( value ) )
if len( value ) > 256:
raise ValueError( "SourceSE list too long" )
self.__data__["SourceSE"] = str( value )[:255] if value else ""
@property
def sourceSEList( self ):
""" helper property returning source SEs as a list"""
return self.SourceSE.split( "," )
@property
def TargetSE( self ):
""" target SE prop """
return self.__data__["TargetSE"] if self.__data__["TargetSE"] else ""
@TargetSE.setter
def TargetSE( self, value ):
""" target SE setter """
value = ",".join( self._uniqueList( value ) )
if len( value ) > 256:
raise ValueError( "TargetSE list too long" )
self.__data__["TargetSE"] = value[:255] if value else ""
@property
def targetSEList( self ):
""" helper property returning target SEs as a list"""
return self.TargetSE.split( "," )
@property
def Catalog( self ):
""" catalog prop """
return self.__data__["Catalog"]
@Catalog.setter
def Catalog( self, value ):
""" catalog setter """
# FIXME
######### THIS IS A TEMPORARY HOT FIX MEANT TO SMOOTH THE LFC->DFC MIGRATION
if value == "LcgFileCatalogCombined":
value = "FileCatalog,LcgFileCatalogCombined"
###########################################################################
value = ",".join( self._uniqueList( value ) )
if len( value ) > 255:
raise ValueError( "Catalog list too long" )
self.__data__["Catalog"] = value if value else ""
@property
def catalogList( self ):
""" helper property returning catalogs as list """
return self.__data__["Catalog"].split( "," )
@property
def Error( self ):
""" error prop """
return self.__data__["Error"]
@Error.setter
def Error( self, value ):
""" error setter """
if type( value ) != str:
raise TypeError( "Error has to be a string!" )
self.__data__["Error"] = self._escapeStr( value[:240], 255 )
@property
def Status( self ):
""" Status prop """
return self.__data__["Status"]
@Status.setter
def Status( self, value ):
""" Status setter """
if value not in Operation.ALL_STATES:
raise ValueError( "unknown Status '%s'" % str( value ) )
if self.__files__:
self._notify()
else:
# If the status moved to Failed or Done, update the lastUpdate time
if value in ( 'Failed', 'Done' ):
if self.__data__["Status"] != value:
self.LastUpdate = datetime.datetime.utcnow().replace( microsecond = 0 )
self.__data__["Status"] = value
if self._parent:
self._parent._notify()
if self.__data__['Status'] == 'Done':
self.__data__['Error'] = ''
@property
def Order( self ):
""" order prop """
if self._parent:
self.__data__["Order"] = self._parent.indexOf( self ) if self._parent else -1
return self.__data__["Order"]
@property
def CreationTime( self ):
""" operation creation time prop """
return self.__data__["CreationTime"]
@CreationTime.setter
def CreationTime( self, value = None ):
""" creation time setter """
if type( value ) not in ( datetime.datetime, str ):
raise TypeError( "CreationTime should be a datetime.datetime!" )
if type( value ) == str:
value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' )
self.__data__["CreationTime"] = value
@property
def SubmitTime( self ):
""" subrequest's submit time prop """
return self.__data__["SubmitTime"]
@SubmitTime.setter
def SubmitTime( self, value = None ):
""" submit time setter """
if type( value ) not in ( datetime.datetime, str ):
raise TypeError( "SubmitTime should be a datetime.datetime!" )
if type( value ) == str:
value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' )
self.__data__["SubmitTime"] = value
@property
def LastUpdate( self ):
""" last update prop """
return self.__data__["LastUpdate"]
@LastUpdate.setter
def LastUpdate( self, value = None ):
""" last update setter """
if type( value ) not in ( datetime.datetime, str ):
raise TypeError( "LastUpdate should be a datetime.datetime!" )
if type( value ) == str:
value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' )
self.__data__["LastUpdate"] = value
if self._parent:
self._parent.LastUpdate = value
def __str__( self ):
""" str operator """
return str( self.toJSON()["Value"] )
def toSQL( self ):
""" get SQL INSERT or UPDATE statement """
if not getattr( self, "RequestID" ):
raise AttributeError( "RequestID not set" )
colVals = [ ( "`%s`" % column, "'%s'" % getattr( self, column )
if type( getattr( self, column ) ) in ( str, datetime.datetime )
else str( getattr( self, column ) ) if getattr( self, column ) != None else "NULL" )
for column in self.__data__
if ( column == 'Error' or getattr( self, column ) ) and column not in ( "OperationID", "LastUpdate", "Order" ) ]
colVals.append( ( "`LastUpdate`", "UTC_TIMESTAMP()" ) )
colVals.append( ( "`Order`", str( self.Order ) ) )
# colVals.append( ( "`Status`", "'%s'" % str(self.Status) ) )
query = []
if self.OperationID:
query.append( "UPDATE `Operation` SET " )
query.append( ", ".join( [ "%s=%s" % item for item in colVals ] ) )
query.append( " WHERE `OperationID`=%d;\n" % self.OperationID )
else:
query.append( "INSERT INTO `Operation` " )
columns = "(%s)" % ",".join( [ column for column, value in colVals ] )
values = "(%s)" % ",".join( [ value for column, value in colVals ] )
query.append( columns )
query.append( " VALUES %s;\n" % values )
return S_OK( "".join( query ) )
def cleanUpSQL( self ):
""" query deleting dirty records from File table """
if self.OperationID and self.__dirty:
fIDs = ",".join( [ str( fid ) for fid in self.__dirty ] )
return "DELETE FROM `File` WHERE `OperationID` = %s AND `FileID` IN (%s);\n" % ( self.OperationID, fIDs )
def toJSON( self ):
""" get json digest """
digest = dict( [( key, str( getattr( self, key ) ) if getattr( self, key ) else "" ) for key in self.__data__] )
digest["RequestID"] = str( self.RequestID )
digest["Order"] = str( self.Order )
if self.__dirty:
digest["__dirty"] = self.__dirty
digest["Files"] = [opFile.toJSON()['Value'] for opFile in self]
return S_OK( digest )
