class Request( Record ): """ .. class:: Request :param int RequestID: requestID :param str Name: request' name :param str OwnerDN: request's owner DN :param str OwnerGroup: request owner group :param str Setup: DIRAC setup :param str SourceComponent: whatever :param int JobID: jobID :param datetime.datetime CreationTime: UTC datetime :param datetime.datetime SubmissionTime: UTC datetime :param datetime.datetime LastUpdate: UTC datetime :param str Status: request's status :param TypedList operations: list of operations """ ALL_STATES = ( "Waiting", "Failed", "Done", "Scheduled", "Assigned", "Canceled" ) FINAL_STATES = ( "Done", "Failed", "Canceled" ) def __init__( self, fromDict = None ): """c'tor :param self: self reference """ Record.__init__( self ) self.__waiting = None now = datetime.datetime.utcnow().replace( microsecond = 0 ) self.__data__["CreationTime"] = now self.__data__["SubmitTime"] = now self.__data__["LastUpdate"] = now self.__data__["Status"] = "Done" self.__data__["JobID"] = 0 self.__data__["RequestID"] = 0 proxyInfo = getProxyInfo() if proxyInfo["OK"]: proxyInfo = proxyInfo["Value"] if proxyInfo["validGroup"] and proxyInfo["validDN"]: self.OwnerDN = proxyInfo["identity"] self.OwnerGroup = proxyInfo["group"] self.__dirty = [] self.__operations__ = TypedList( allowedTypes = Operation ) fromDict = fromDict if fromDict else {} self.__dirty = fromDict.get( "__dirty", [] ) if "__dirty" in fromDict: del fromDict["__dirty"] for opDict in fromDict.get( "Operations", [] ): self +=Operation( opDict ) if "Operations" in fromDict: del fromDict["Operations"] for key, value in fromDict.items(): if key not in self.__data__: raise AttributeError( "Unknown Request attribute '%s'" % key ) if value: setattr( self, key, value ) self._notify() @staticmethod def tableDesc(): """ get table desc """ return { "Fields" : { "RequestID" : "INTEGER NOT NULL AUTO_INCREMENT", "RequestName" : "VARCHAR(255) NOT NULL", "OwnerDN" : "VARCHAR(255)", "OwnerGroup" : "VARCHAR(32)", "Status" : "ENUM('Waiting', 'Assigned', 'Done', 'Failed', 'Canceled', 'Scheduled') DEFAULT 'Waiting'", "Error" : "VARCHAR(255)", "DIRACSetup" : "VARCHAR(32)", "SourceComponent" : "BLOB", "JobID" : "INTEGER DEFAULT 0", "CreationTime" : "DATETIME", "SubmitTime" : "DATETIME", "LastUpdate" : "DATETIME" }, "PrimaryKey" : [ "RequestID" ], 'UniqueIndexes': {'RequestName' : [ 'RequestName'] } } def _notify( self ): """ simple state machine for sub request statuses """ self.__waiting = None # # update operations statuses rStatus = "Waiting" opStatusList = [ ( op.Status, op ) for op in self ] self.__waiting = None while opStatusList: # # Scan all status in order! opStatus, op = opStatusList.pop( 0 ) # # Failed -> Failed if opStatus == "Failed": rStatus = "Failed" break # Scheduled -> Scheduled if opStatus == "Scheduled": if self.__waiting == None: self.__waiting = op rStatus = "Scheduled" # # First operation Queued becomes Waiting if no Waiting/Scheduled before elif opStatus == "Queued": if self.__waiting == None: self.__waiting = op op._setWaiting( self ) rStatus = "Waiting" # # First operation Waiting is next to execute, others are queued elif opStatus == "Waiting": rStatus = "Waiting" if self.__waiting == None: self.__waiting = op else: op._setQueued( self ) # # All operations Done -> Done elif opStatus == "Done" and self.__waiting == None: rStatus = "Done" self.__data__['Error'] = '' self.Status = rStatus def getWaiting( self ): """ get waiting operation if any """ # # update states self._notify() return S_OK( self.__waiting ) # # Operation arithmetics def __contains__( self, operation ): """ in operator :param self: self reference :param Operation subRequest: a subRequest """ return bool( operation in self.__operations__ ) def __iadd__( self, operation ): """ += operator for subRequest :param self: self reference :param Operation operation: sub-request to add """ if operation not in self: self.__operations__.append( operation ) operation._parent = self self._notify() return self def insertBefore( self, newOperation, existingOperation ): """ insert :newOperation: just before :existingOperation: :param self: self reference :param Operation newOperation: Operation to be inserted :param Operation existingOperation: previous Operation sibling """ if existingOperation not in self: return S_ERROR( "%s is not in" % existingOperation ) if newOperation in self: return S_ERROR( "%s is already in" % newOperation ) self.__operations__.insert( self.__operations__.index( existingOperation ), newOperation ) newOperation._parent = self self._notify() return S_OK() def insertAfter( self, newOperation, existingOperation ): """ insert :newOperation: just after :existingOperation: :param self: self reference :param Operation newOperation: Operation to be inserted :param Operation existingOperation: next Operation sibling """ if existingOperation not in self: return S_ERROR( "%s is not in" % existingOperation ) if newOperation in self: return S_ERROR( "%s is already in" % newOperation ) self.__operations__.insert( self.__operations__.index( existingOperation ) + 1, newOperation ) newOperation._parent = self self._notify() return S_OK() def addOperation( self, operation ): """ add :operation: to list of Operations :param self: self reference :param Operation operation: Operation to be inserted """ if operation in self: return S_ERROR( "This operation is already in!!!" ) self +=operation return S_OK() def isEmpty( self ): """ Evaluate if the request is empty """ return len( self.__operations__ ) == 0 def __iter__( self ): """ iterator for sub-request """ return self.__operations__.__iter__() def __getitem__( self, i ): """ [] op for sub requests """ return self.__operations__.__getitem__( i ) def __setitem__( self, i, value ): """ self[i] = val """ self.__operations__._typeCheck( value ) if self[i].OperationID: self.__dirty.append( self[i].OperationID ) self.__operations__.__setitem__( i, value ) value._parent = self self._notify() def __delitem__( self, i ): """ del self[i]""" if not self.RequestID: self.__operations__.__delitem__( i ) else: opId = self[i].OperationID if opId: self.__dirty.append( opId ) self.__operations__.__delitem__( i ) self._notify() def indexOf( self, subReq ): """ return index of subReq (execution order) """ return self.__operations__.index( subReq ) if subReq in self else -1 def __nonzero__( self ): """ for comparisons """ return True def __len__( self ): """ nb of subRequests """ return len( self.__operations__ ) def __str__( self ): """ str operator """ return str( self.toJSON()["Value"] ) def subStatusList( self ): """ list of statuses for all operations """ return [ subReq.Status for subReq in self ] # # properties @property def RequestID( self ): """ request ID getter """ return self.__data__["RequestID"] @RequestID.setter def RequestID( self, value ): """ requestID setter (shouldn't be RO???) """ self.__data__["RequestID"] = long( value ) if value else 0 @property def RequestName( self ): """ request's name getter """ return self.__data__["RequestName"] @RequestName.setter def RequestName( self, value ): """ request name setter """ if type( value ) != str: raise TypeError( "RequestName should be a string" ) self.__data__["RequestName"] = value[:128] @property def OwnerDN( self ): """ request owner DN getter """ return self.__data__["OwnerDN"] @OwnerDN.setter def OwnerDN( self, value ): """ request owner DN setter """ if type( value ) != str: raise TypeError( "OwnerDN should be a string!" ) self.__data__["OwnerDN"] = value @property def OwnerGroup( self ): """ request owner group getter """ return self.__data__["OwnerGroup"] @OwnerGroup.setter def OwnerGroup( self, value ): """ request owner group setter """ if type( value ) != str: raise TypeError( "OwnerGroup should be a string!" ) self.__data__["OwnerGroup"] = value @property def DIRACSetup( self ): """ DIRAC setup getter """ return self.__data__["DIRACSetup"] @DIRACSetup.setter def DIRACSetup( self, value ): """ DIRAC setup setter """ if type( value ) != str: raise TypeError( "setup should be a string!" ) self.__data__["DIRACSetup"] = value @property def SourceComponent( self ): """ source component getter """ return self.__data__["SourceComponent"] @SourceComponent.setter def SourceComponent( self, value ): """ source component setter """ if type( value ) != str: raise TypeError( "Setup should be a string!" ) self.__data__["SourceComponent"] = value @property def JobID( self ): """ jobID getter """ return self.__data__["JobID"] @JobID.setter def JobID( self, value = 0 ): """ jobID setter """ self.__data__["JobID"] = long( value ) if value else 0 @property def CreationTime( self ): """ creation time getter """ return self.__data__["CreationTime"] @CreationTime.setter def CreationTime( self, value = None ): """ creation time setter """ if type( value ) not in ( datetime.datetime, str ) : raise TypeError( "CreationTime should be a datetime.datetime!" ) if type( value ) == str: value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' ) self.__data__["CreationTime"] = value @property def SubmitTime( self ): """ request's submission time getter """ return self.__data__["SubmitTime"] @SubmitTime.setter def SubmitTime( self, value = None ): """ submission time setter """ if type( value ) not in ( datetime.datetime, str ): raise TypeError( "SubmitTime should be a datetime.datetime!" ) if type( value ) == str: value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' ) self.__data__["SubmitTime"] = value @property def LastUpdate( self ): """ last update getter """ return self.__data__["LastUpdate"] @LastUpdate.setter def LastUpdate( self, value = None ): """ last update setter """ if type( value ) not in ( datetime.datetime, str ): raise TypeError( "LastUpdate should be a datetime.datetime!" ) if type( value ) == str: value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' ) self.__data__["LastUpdate"] = value @property def Status( self ): """ status getter """ self._notify() return self.__data__["Status"] @Status.setter def Status( self, value ): """ status setter """ if value not in Request.ALL_STATES: raise ValueError( "Unknown status: %s" % str( value ) ) if value == 'Done': self.__data__['Error'] = '' self.__data__["Status"] = value @property def Order( self ): """ ro execution order getter """ self._notify() opStatuses = [ op.Status for op in self.__operations__ ] return opStatuses.index( "Waiting" ) if "Waiting" in opStatuses else len( opStatuses ) @property def Error( self ): """ error getter """ return self.__data__["Error"] @Error.setter def Error( self, value ): """ error setter """ if type( value ) != str: raise TypeError( "Error has to be a string!" ) self.__data__["Error"] = self._escapeStr( value, 255 ) def toSQL( self ): """ prepare SQL INSERT or UPDATE statement """ colVals = [ ( "`%s`" % column, "'%s'" % value if type( value ) in ( str, datetime.datetime ) else str( value ) if value != None else "NULL" ) for column, value in self.__data__.items() if ( column == 'Error' or value ) and column not in ( "RequestID", "LastUpdate" ) ] colVals.append( ( "`LastUpdate`", "UTC_TIMESTAMP()" ) ) query = [] if self.RequestID: query.append( "UPDATE `Request` SET " ) query.append( ", ".join( [ "%s=%s" % item for item in colVals ] ) ) query.append( " WHERE `RequestID`=%d;\n" % self.RequestID ) else: query.append( "INSERT INTO `Request` " ) columns = "(%s)" % ",".join( [ column for column, value in colVals ] ) values = "(%s)" % ",".join( [ value for column, value in colVals ] ) query.append( columns ) query.append( " VALUES %s;" % values ) return S_OK( "".join( query ) ) def cleanUpSQL( self ): """ delete query for dirty operations """ query = [] if self.RequestID and self.__dirty: opIDs = ",".join( [ str( opID ) for opID in self.__dirty ] ) query.append( "DELETE FROM `Operation` WHERE `RequestID`=%s AND `OperationID` IN (%s);\n" % ( self.RequestID, opIDs ) ) for opID in self.__dirty: query.append( "DELETE FROM `File` WHERE `OperationID`=%s;\n" % opID ) return query # # digest def toJSON( self ): """ serialize to JSON format """ digest = dict( [( key, str( getattr( self, key ) ) if getattr( self, key ) else "" ) for key in self.__data__] ) digest["RequestID"] = self.RequestID digest["__dirty"] = self.__dirty digest["Operations"] = [op.toJSON()['Value'] for op in self] return S_OK( digest ) def getDigest( self ): """ return digest for request """ digest = ['Name:' + self.RequestName] for op in self: opDigest = [ str( item ) for item in ( op.Type, op.Type, op.Status, op.Order ) ] if op.TargetSE: opDigest.append( op.TargetSE ) if op.Catalog: opDigest.append( op.Catalog ) if len( op ): opFile = op[0] opDigest.append( opFile.LFN ) opDigest.append( ",...<%d files>" % len( op ) ) digest.append( ":".join( opDigest ) ) return S_OK( "\n".join( digest ) )
class Operation(Record): """ .. class:: Operation :param long OperationID: OperationID as read from DB backend :param long RequestID: parent RequestID :param str Status: execution status :param str Type: operation to perform :param str Arguments: additional arguments :param str SourceSE: source SE name :param str TargetSE: target SE names as comma separated list :param str Catalog: catalog to use as comma separated list :param str Error: error string if any :param Request parent: parent Request instance """ # # max files in a single operation MAX_FILES = 10000 # # all states ALL_STATES = ("Queued", "Waiting", "Scheduled", "Assigned", "Failed", "Done", "Canceled") # # final states FINAL_STATES = ("Failed", "Done", "Canceled") def __init__(self, fromDict=None): """ c'tor :param self: self reference :param dict fromDict: attributes dictionary """ Record.__init__(self) self._parent = None # # sub-request attributes # self.__data__ = dict.fromkeys( self.tableDesc()["Fields"].keys(), None ) now = datetime.datetime.utcnow().replace(microsecond=0) self.__data__["SubmitTime"] = now self.__data__["LastUpdate"] = now self.__data__["CreationTime"] = now self.__data__["OperationID"] = 0 self.__data__["RequestID"] = 0 self.__data__["Status"] = "Queued" # # operation files self.__files__ = TypedList(allowedTypes=File) # # dirty fileIDs self.__dirty = [] # # init from dict fromDict = fromDict if fromDict else {} self.__dirty = fromDict.get("__dirty", []) if "__dirty" in fromDict: del fromDict["__dirty"] for fileDict in fromDict.get("Files", []): self.addFile(File(fileDict)) if "Files" in fromDict: del fromDict["Files"] for key, value in fromDict.items(): if key not in self.__data__: raise AttributeError("Unknown Operation attribute '%s'" % key) if key != "Order" and value: setattr(self, key, value) @staticmethod def tableDesc(): """ get table desc """ return { "Fields" : { "OperationID" : "INTEGER NOT NULL AUTO_INCREMENT", "RequestID" : "INTEGER NOT NULL", "Type" : "VARCHAR(64) NOT NULL", "Status" : "ENUM('Waiting', 'Assigned', 'Queued', 'Done', 'Failed', 'Canceled', 'Scheduled') "\ "DEFAULT 'Queued'", "Arguments" : "MEDIUMBLOB", "Order" : "INTEGER NOT NULL", "SourceSE" : "VARCHAR(255)", "TargetSE" : "VARCHAR(255)", "Catalog" : "VARCHAR(255)", "Error": "VARCHAR(255)", "CreationTime" : "DATETIME", "SubmitTime" : "DATETIME", "LastUpdate" : "DATETIME" }, 'ForeignKeys': {'RequestID': 'Request.RequestID' }, "PrimaryKey" : "OperationID" } # # protected methods for parent only def _notify(self): """ notify self about file status change """ fStatus = set(self.fileStatusList()) if fStatus == set(['Failed']): # All files Failed -> Failed newStatus = 'Failed' elif 'Scheduled' in fStatus: newStatus = 'Scheduled' elif "Waiting" in fStatus: newStatus = 'Queued' elif 'Failed' in fStatus: newStatus = 'Failed' else: self.__data__['Error'] = '' newStatus = 'Done' # If the status moved to Failed or Done, update the lastUpdate time if newStatus in ('Failed', 'Done', 'Scheduled'): if self.__data__["Status"] != newStatus: self.LastUpdate = datetime.datetime.utcnow().replace( microsecond=0) self.__data__["Status"] = newStatus if self._parent: self._parent._notify() def _setQueued(self, caller): """ don't touch """ if caller == self._parent: self.__data__["Status"] = "Queued" def _setWaiting(self, caller): """ don't touch as well """ if caller == self._parent: self.__data__["Status"] = "Waiting" # # Files arithmetics def __contains__(self, opFile): """ in operator """ return opFile in self.__files__ def __iadd__(self, opFile): """ += operator """ if len(self) >= Operation.MAX_FILES: raise RuntimeError("too many Files in a single Operation") self.addFile(opFile) return self def addFile(self, opFile): """ add :opFile: to operation """ if len(self) >= Operation.MAX_FILES: raise RuntimeError("too many Files in a single Operation") if opFile not in self: self.__files__.append(opFile) opFile._parent = self self._notify() # # helpers for looping def __iter__(self): """ files iterator """ return self.__files__.__iter__() def __getitem__(self, i): """ [] op for opFiles """ return self.__files__.__getitem__(i) def __delitem__(self, i): """ remove file from op, only if OperationID is NOT set """ if not self.OperationID: self.__files__.__delitem__(i) else: if self[i].FileID: self.__dirty.append(self[i].FileID) self.__files__.__delitem__(i) self._notify() def __setitem__(self, i, opFile): """ overwrite opFile """ self.__files__._typeCheck(opFile) toDelete = self[i] if toDelete.FileID: self.__dirty.append(toDelete.FileID) self.__files__.__setitem__(i, opFile) opFile._parent = self self._notify() def fileStatusList(self): """ get list of files statuses """ return [subFile.Status for subFile in self] def __nonzero__(self): """ for comparisons """ return True def __len__(self): """ nb of subFiles """ return len(self.__files__) # # properties @property def RequestID(self): """ RequestID getter (RO) """ return self._parent.RequestID if self._parent else -1 @RequestID.setter def RequestID(self, value): """ can't set RequestID by hand """ self.__data__[ "RequestID"] = self._parent.RequestID if self._parent else -1 @property def OperationID(self): """ OperationID getter """ return self.__data__["OperationID"] @OperationID.setter def OperationID(self, value): """ OperationID setter """ self.__data__["OperationID"] = long(value) if value else 0 @property def Type(self): """ operation type prop """ return self.__data__["Type"] @Type.setter def Type(self, value): """ operation type setter """ self.__data__["Type"] = str(value) @property def Arguments(self): """ arguments getter """ return self.__data__["Arguments"] @Arguments.setter def Arguments(self, value): """ arguments setter """ self.__data__["Arguments"] = value if value else "" @property def SourceSE(self): """ source SE prop """ return self.__data__["SourceSE"] if self.__data__["SourceSE"] else "" @SourceSE.setter def SourceSE(self, value): """ source SE setter """ value = ",".join(self._uniqueList(value)) if len(value) > 256: raise ValueError("SourceSE list too long") self.__data__["SourceSE"] = str(value)[:255] if value else "" @property def sourceSEList(self): """ helper property returning source SEs as a list""" return self.SourceSE.split(",") @property def TargetSE(self): """ target SE prop """ return self.__data__["TargetSE"] if self.__data__["TargetSE"] else "" @TargetSE.setter def TargetSE(self, value): """ target SE setter """ value = ",".join(self._uniqueList(value)) if len(value) > 256: raise ValueError("TargetSE list too long") self.__data__["TargetSE"] = value[:255] if value else "" @property def targetSEList(self): """ helper property returning target SEs as a list""" return self.TargetSE.split(",") @property def Catalog(self): """ catalog prop """ return self.__data__["Catalog"] @Catalog.setter def Catalog(self, value): """ catalog setter """ # FIXME ######### THIS IS A TEMPORARY HOT FIX MEANT TO SMOOTH THE LFC->DFC MIGRATION if value == "LcgFileCatalogCombined": value = "FileCatalog,LcgFileCatalogCombined" ########################################################################### value = ",".join(self._uniqueList(value)) if len(value) > 255: raise ValueError("Catalog list too long") self.__data__["Catalog"] = value if value else "" @property def catalogList(self): """ helper property returning catalogs as list """ return self.__data__["Catalog"].split(",") @property def Error(self): """ error prop """ return self.__data__["Error"] @Error.setter def Error(self, value): """ error setter """ if type(value) != str: raise TypeError("Error has to be a string!") self.__data__["Error"] = self._escapeStr(value[:240], 255) @property def Status(self): """ Status prop """ return self.__data__["Status"] @Status.setter def Status(self, value): """ Status setter """ if value not in Operation.ALL_STATES: raise ValueError("unknown Status '%s'" % str(value)) if self.__files__: self._notify() else: # If the status moved to Failed or Done, update the lastUpdate time if value in ('Failed', 'Done'): if self.__data__["Status"] != value: self.LastUpdate = datetime.datetime.utcnow().replace( microsecond=0) self.__data__["Status"] = value if self._parent: self._parent._notify() if self.__data__['Status'] == 'Done': self.__data__['Error'] = '' @property def Order(self): """ order prop """ if self._parent: self.__data__["Order"] = self._parent.indexOf( self) if self._parent else -1 return self.__data__["Order"] @property def CreationTime(self): """ operation creation time prop """ return self.__data__["CreationTime"] @CreationTime.setter def CreationTime(self, value=None): """ creation time setter """ if type(value) not in (datetime.datetime, str): raise TypeError("CreationTime should be a datetime.datetime!") if type(value) == str: value = datetime.datetime.strptime( value.split(".")[0], '%Y-%m-%d %H:%M:%S') self.__data__["CreationTime"] = value @property def SubmitTime(self): """ subrequest's submit time prop """ return self.__data__["SubmitTime"] @SubmitTime.setter def SubmitTime(self, value=None): """ submit time setter """ if type(value) not in (datetime.datetime, str): raise TypeError("SubmitTime should be a datetime.datetime!") if type(value) == str: value = datetime.datetime.strptime( value.split(".")[0], '%Y-%m-%d %H:%M:%S') self.__data__["SubmitTime"] = value @property def LastUpdate(self): """ last update prop """ return self.__data__["LastUpdate"] @LastUpdate.setter def LastUpdate(self, value=None): """ last update setter """ if type(value) not in (datetime.datetime, str): raise TypeError("LastUpdate should be a datetime.datetime!") if type(value) == str: value = datetime.datetime.strptime( value.split(".")[0], '%Y-%m-%d %H:%M:%S') self.__data__["LastUpdate"] = value if self._parent: self._parent.LastUpdate = value def __str__(self): """ str operator """ return str(self.toJSON()["Value"]) def toSQL(self): """ get SQL INSERT or UPDATE statement """ if not getattr(self, "RequestID"): raise AttributeError("RequestID not set") colVals = [ ("`%s`" % column, "'%s'" % getattr(self, column) if type(getattr(self, column)) in (str, datetime.datetime) else str(getattr(self, column)) if getattr(self, column) != None else "NULL") for column in self.__data__ if (column == 'Error' or getattr(self, column)) and column not in ( "OperationID", "LastUpdate", "Order") ] colVals.append(("`LastUpdate`", "UTC_TIMESTAMP()")) colVals.append(("`Order`", str(self.Order))) # colVals.append( ( "`Status`", "'%s'" % str(self.Status) ) ) query = [] if self.OperationID: query.append("UPDATE `Operation` SET ") query.append(", ".join(["%s=%s" % item for item in colVals])) query.append(" WHERE `OperationID`=%d;\n" % self.OperationID) else: query.append("INSERT INTO `Operation` ") columns = "(%s)" % ",".join([column for column, value in colVals]) values = "(%s)" % ",".join([value for column, value in colVals]) query.append(columns) query.append(" VALUES %s;\n" % values) return S_OK("".join(query)) def cleanUpSQL(self): """ query deleting dirty records from File table """ if self.OperationID and self.__dirty: fIDs = ",".join([str(fid) for fid in self.__dirty]) return "DELETE FROM `File` WHERE `OperationID` = %s AND `FileID` IN (%s);\n" % ( self.OperationID, fIDs) def toJSON(self): """ get json digest """ digest = dict([(key, str(getattr(self, key)) if getattr(self, key) else "") for key in self.__data__]) digest["RequestID"] = str(self.RequestID) digest["Order"] = str(self.Order) if self.__dirty: digest["__dirty"] = self.__dirty digest["Files"] = [opFile.toJSON()['Value'] for opFile in self] return S_OK(digest)
class Request( Record ): """ .. class:: Request :param int RequestID: requestID :param str Name: request' name :param str OwnerDN: request's owner DN :param str OwnerGroup: request owner group :param str Setup: DIRAC setup :param str SourceComponent: whatever :param int JobID: jobID :param datetime.datetime CreationTime: UTC datetime :param datetime.datetime SubmissionTime: UTC datetime :param datetime.datetime LastUpdate: UTC datetime :param str Status: request's status :param TypedList operations: list of operations """ ALL_STATES = ( "Waiting", "Failed", "Done", "Scheduled", "Assigned", "Canceled" ) FINAL_STATES = ( "Done", "Failed", "Canceled" ) def __init__( self, fromDict = None ): """c'tor :param self: self reference """ Record.__init__( self ) self.__waiting = None now = datetime.datetime.utcnow().replace( microsecond = 0 ) self.__data__["CreationTime"] = now self.__data__["SubmitTime"] = now self.__data__["LastUpdate"] = now self.__data__["Status"] = "Done" self.__data__["JobID"] = 0 self.__data__["RequestID"] = 0 proxyInfo = getProxyInfo() if proxyInfo["OK"]: proxyInfo = proxyInfo["Value"] if proxyInfo["validGroup"] and proxyInfo["validDN"]: self.OwnerDN = proxyInfo["identity"] self.OwnerGroup = proxyInfo["group"] self.__dirty = [] self.__operations__ = TypedList( allowedTypes = Operation ) fromDict = fromDict if fromDict else {} self.__dirty = fromDict.get( "__dirty", [] ) if "__dirty" in fromDict: del fromDict["__dirty"] for opDict in fromDict.get( "Operations", [] ): self +=Operation( opDict ) if "Operations" in fromDict: del fromDict["Operations"] for key, value in fromDict.items(): if key not in self.__data__: raise AttributeError( "Unknown Request attribute '%s'" % key ) if value: setattr( self, key, value ) self._notify() @staticmethod def tableDesc(): """ get table desc """ return { "Fields" : { "RequestID" : "INTEGER NOT NULL AUTO_INCREMENT", "RequestName" : "VARCHAR(255) NOT NULL", "OwnerDN" : "VARCHAR(255)", "OwnerGroup" : "VARCHAR(32)", "Status" : "ENUM('Waiting', 'Assigned', 'Done', 'Failed', 'Canceled', 'Scheduled') DEFAULT 'Waiting'", "Error" : "VARCHAR(255)", "DIRACSetup" : "VARCHAR(32)", "SourceComponent" : "BLOB", "JobID" : "INTEGER DEFAULT 0", "CreationTime" : "DATETIME", "SubmitTime" : "DATETIME", "LastUpdate" : "DATETIME" }, "PrimaryKey" : [ "RequestID" ], 'UniqueIndexes': {'RequestName' : [ 'RequestName'] } } def _notify( self ): """ simple state machine for sub request statuses """ self.__waiting = None # # update operations statuses rStatus = "Waiting" opStatusList = [ ( op.Status, op ) for op in self ] self.__waiting = None while opStatusList: # # Scan all status in order! opStatus, op = opStatusList.pop( 0 ) # # Failed -> Failed if opStatus == "Failed": rStatus = "Failed" break # Scheduled -> Scheduled if opStatus == "Scheduled": if self.__waiting == None: self.__waiting = op rStatus = "Scheduled" # # First operation Queued becomes Waiting if no Waiting/Scheduled before elif opStatus == "Queued": if self.__waiting == None: self.__waiting = op op._setWaiting( self ) rStatus = "Waiting" # # First operation Waiting is next to execute, others are queued elif opStatus == "Waiting": rStatus = "Waiting" if self.__waiting == None: self.__waiting = op else: op._setQueued( self ) # # All operations Done -> Done elif opStatus == "Done" and self.__waiting == None: rStatus = "Done" self.__data__['Error'] = '' self.Status = rStatus def getWaiting( self ): """ get waiting operation if any """ # # update states self._notify() return S_OK( self.__waiting ) # # Operation arithmetics def __contains__( self, operation ): """ in operator :param self: self reference :param Operation subRequest: a subRequest """ return bool( operation in self.__operations__ ) def __iadd__( self, operation ): """ += operator for subRequest :param self: self reference :param Operation operation: sub-request to add """ if operation not in self: self.__operations__.append( operation ) operation._parent = self self._notify() return self def insertBefore( self, newOperation, existingOperation ): """ insert :newOperation: just before :existingOperation: :param self: self reference :param Operation newOperation: Operation to be inserted :param Operation existingOperation: previous Operation sibling """ if existingOperation not in self: return S_ERROR( "%s is not in" % existingOperation ) if newOperation in self: return S_ERROR( "%s is already in" % newOperation ) self.__operations__.insert( self.__operations__.index( existingOperation ), newOperation ) newOperation._parent = self self._notify() return S_OK() def insertAfter( self, newOperation, existingOperation ): """ insert :newOperation: just after :existingOperation: :param self: self reference :param Operation newOperation: Operation to be inserted :param Operation existingOperation: next Operation sibling """ if existingOperation not in self: return S_ERROR( "%s is not in" % existingOperation ) if newOperation in self: return S_ERROR( "%s is already in" % newOperation ) self.__operations__.insert( self.__operations__.index( existingOperation ) + 1, newOperation ) newOperation._parent = self self._notify() return S_OK() def addOperation( self, operation ): """ add :operation: to list of Operations :param self: self reference :param Operation operation: Operation to be inserted """ if operation in self: return S_ERROR( "This operation is already in!!!" ) self +=operation return S_OK() def isEmpty( self ): """ Evaluate if the request is empty """ return len( self.__operations__ ) == 0 def __iter__( self ): """ iterator for sub-request """ return self.__operations__.__iter__() def __getitem__( self, i ): """ [] op for sub requests """ return self.__operations__.__getitem__( i ) def __setitem__( self, i, value ): """ self[i] = val """ self.__operations__._typeCheck( value ) if self[i].OperationID: self.__dirty.append( self[i].OperationID ) self.__operations__.__setitem__( i, value ) value._parent = self self._notify() def __delitem__( self, i ): """ del self[i]""" if not self.RequestID: self.__operations__.__delitem__( i ) else: opId = self[i].OperationID if opId: self.__dirty.append( opId ) self.__operations__.__delitem__( i ) self._notify() def indexOf( self, subReq ): """ return index of subReq (execution order) """ return self.__operations__.index( subReq ) if subReq in self else -1 def __nonzero__( self ): """ for comparisons """ return True def __len__( self ): """ nb of subRequests """ return len( self.__operations__ ) def __str__( self ): """ str operator """ return str( self.toJSON()["Value"] ) def subStatusList( self ): """ list of statuses for all operations """ return [ subReq.Status for subReq in self ] # # properties @property def RequestID( self ): """ request ID getter """ return self.__data__["RequestID"] @RequestID.setter def RequestID( self, value ): """ requestID setter (shouldn't be RO???) """ self.__data__["RequestID"] = long( value ) if value else 0 @property def RequestName( self ): """ request's name getter """ return self.__data__["RequestName"] @RequestName.setter def RequestName( self, value ): """ request name setter """ if type( value ) != str: raise TypeError( "RequestName should be a string" ) self.__data__["RequestName"] = value[:128] @property def OwnerDN( self ): """ request owner DN getter """ return self.__data__["OwnerDN"] @OwnerDN.setter def OwnerDN( self, value ): """ request owner DN setter """ if type( value ) != str: raise TypeError( "OwnerDN should be a string!" ) self.__data__["OwnerDN"] = value @property def OwnerGroup( self ): """ request owner group getter """ return self.__data__["OwnerGroup"] @OwnerGroup.setter def OwnerGroup( self, value ): """ request owner group setter """ if type( value ) != str: raise TypeError( "OwnerGroup should be a string!" ) self.__data__["OwnerGroup"] = value @property def DIRACSetup( self ): """ DIRAC setup getter """ return self.__data__["DIRACSetup"] @DIRACSetup.setter def DIRACSetup( self, value ): """ DIRAC setup setter """ if type( value ) != str: raise TypeError( "setup should be a string!" ) self.__data__["DIRACSetup"] = value @property def SourceComponent( self ): """ source component getter """ return self.__data__["SourceComponent"] @SourceComponent.setter def SourceComponent( self, value ): """ source component setter """ if type( value ) != str: raise TypeError( "Setup should be a string!" ) self.__data__["SourceComponent"] = value @property def JobID( self ): """ jobID getter """ return self.__data__["JobID"] @JobID.setter def JobID( self, value = 0 ): """ jobID setter """ self.__data__["JobID"] = long( value ) if value else 0 @property def CreationTime( self ): """ creation time getter """ return self.__data__["CreationTime"] @CreationTime.setter def CreationTime( self, value = None ): """ creation time setter """ if type( value ) not in ( datetime.datetime, str ) : raise TypeError( "CreationTime should be a datetime.datetime!" ) if type( value ) == str: value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' ) self.__data__["CreationTime"] = value @property def SubmitTime( self ): """ request's submission time getter """ return self.__data__["SubmitTime"] @SubmitTime.setter def SubmitTime( self, value = None ): """ submission time setter """ if type( value ) not in ( datetime.datetime, str ): raise TypeError( "SubmitTime should be a datetime.datetime!" ) if type( value ) == str: value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' ) self.__data__["SubmitTime"] = value @property def LastUpdate( self ): """ last update getter """ return self.__data__["LastUpdate"] @LastUpdate.setter def LastUpdate( self, value = None ): """ last update setter """ if type( value ) not in ( datetime.datetime, str ): raise TypeError( "LastUpdate should be a datetime.datetime!" ) if type( value ) == str: value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' ) self.__data__["LastUpdate"] = value @property def Status( self ): """ status getter """ self._notify() return self.__data__["Status"] @Status.setter def Status( self, value ): """ status setter """ if value not in Request.ALL_STATES: raise ValueError( "Unknown status: %s" % str( value ) ) # If the status moved to Failed or Done, update the lastUpdate time if value in ( 'Done', 'Failed' ): if value != self.__data__["Status"]: self.LastUpdate = datetime.datetime.utcnow().replace( microsecond = 0 ) if value == 'Done': self.__data__['Error'] = '' self.__data__["Status"] = value @property def Order( self ): """ ro execution order getter """ self._notify() opStatuses = [ op.Status for op in self.__operations__ ] return opStatuses.index( "Waiting" ) if "Waiting" in opStatuses else len( opStatuses ) @property def Error( self ): """ error getter """ return self.__data__["Error"] @Error.setter def Error( self, value ): """ error setter """ if type( value ) != str: raise TypeError( "Error has to be a string!" ) self.__data__["Error"] = self._escapeStr( value[:240], 255 ) def toSQL( self ): """ prepare SQL INSERT or UPDATE statement """ colVals = [ ( "`%s`" % column, "'%s'" % value if type( value ) in ( str, datetime.datetime ) else str( value ) if value != None else "NULL" ) for column, value in self.__data__.items() if ( column == 'Error' or value ) and column not in ( "RequestID", "LastUpdate" ) ] colVals.append( ( "`LastUpdate`", "UTC_TIMESTAMP()" ) ) query = [] if self.RequestID: query.append( "UPDATE `Request` SET " ) query.append( ", ".join( [ "%s=%s" % item for item in colVals ] ) ) query.append( " WHERE `RequestID`=%d;\n" % self.RequestID ) else: query.append( "INSERT INTO `Request` " ) columns = "(%s)" % ",".join( [ column for column, value in colVals ] ) values = "(%s)" % ",".join( [ value for column, value in colVals ] ) query.append( columns ) query.append( " VALUES %s;" % values ) return S_OK( "".join( query ) ) def cleanUpSQL( self ): """ delete query for dirty operations """ query = [] if self.RequestID and self.__dirty: opIDs = ",".join( [ str( opID ) for opID in self.__dirty ] ) query.append( "DELETE FROM `Operation` WHERE `RequestID`=%s AND `OperationID` IN (%s);\n" % ( self.RequestID, opIDs ) ) for opID in self.__dirty: query.append( "DELETE FROM `File` WHERE `OperationID`=%s;\n" % opID ) return query # # digest def toJSON( self ): """ serialize to JSON format """ digest = dict( [( key, str( getattr( self, key ) ) if getattr( self, key ) else "" ) for key in self.__data__] ) digest["RequestID"] = self.RequestID digest["__dirty"] = self.__dirty digest["Operations"] = [op.toJSON()['Value'] for op in self] return S_OK( digest ) def getDigest( self ): """ return digest for request """ digest = ['Name:' + self.RequestName] for op in self: opDigest = [ str( item ) for item in ( op.Type, op.Type, op.Status, op.Order ) ] if op.TargetSE: opDigest.append( op.TargetSE ) if op.Catalog: opDigest.append( op.Catalog ) if len( op ): opFile = op[0] opDigest.append( opFile.LFN ) opDigest.append( ",...<%d files>" % len( op ) ) digest.append( ":".join( opDigest ) ) return S_OK( "\n".join( digest ) ) def optimize( self ): """ Merges together the operations that can be merged. They need to have the following arguments equal: * Type * Arguments * SourceSE * TargetSE * Catalog It also makes sure that the maximum number of Files in an Operation is never overcome. CAUTION: this method is meant to be called before inserting into the DB. So if the RequestId is not 0, we don't touch :return S_ERROR if the Request should not be optimized (because already in the DB S_OK(True) if a optimization was carried out S_OK(False) if no optimization were carried out """ # If the RequestID is not the default one (0), it probably means # the Request is already in the DB, so we don't touch anything if self.RequestID: return S_ERROR( "Cannot optimize because Request seems to be already in the DB (RequestID %s)" % self.RequestID ) # Set to True if the request could be optimized optimized = False # Recognise Failover request series repAndRegList = [] removeRepList = [] i = 0 while i < len( self.__operations__ ): insertNow = True if i < len( self.__operations__ ) - 1: op1 = self.__operations__[i] op2 = self.__operations__[i + 1] if getattr( op1, 'Type' ) == 'ReplicateAndRegister' and \ getattr( op2, 'Type' ) == 'RemoveReplica': fileSetA = set( list( f.LFN for f in op1 ) ) fileSetB = set( list( f.LFN for f in op2 ) ) if fileSetA == fileSetB: # Source is useless if failover if 'FAILOVER' in op1.SourceSE: op1.SourceSE = '' repAndRegList.append( ( op1.TargetSE, op1 ) ) removeRepList.append( ( op2.TargetSE, op2 ) ) del self.__operations__[i] del self.__operations__[i] # If we are at the end of the request, we must insert the new operations insertNow = ( i == len( self.__operations__ ) ) # print i, self.__operations__[i].Type if i < len( self.__operations__ ) else None, len( repAndRegList ), insertNow if insertNow: if repAndRegList: # We must insert the new operations there # i.e. insert before operation i (if it exists) # Replication first, removeReplica next optimized = True insertBefore = self.__operations__[i] if i < len( self.__operations__ ) else None # print 'Insert new operations before', insertBefore for op in \ [op for _targetSE, op in sorted( repAndRegList )] + \ [op for _targetSE, op in sorted( removeRepList )]: _res = self.insertBefore( op, insertBefore ) if insertBefore else self.addOperation( op ) # Skip the newly inserted operation i += 1 repAndRegList = [] removeRepList = [] else: # Skip current operation i += 1 else: # Just to show that in that case we don't increment i pass # List of attributes that must be equal for operations to be merged attrList = ["Type", "Arguments", "SourceSE", "TargetSE", "Catalog" ] i = 0 while i < len( self.__operations__ ): while i < len( self.__operations__ ) - 1: # Some attributes need to be the same attrMismatch = False for attr in attrList: if getattr( self.__operations__[i], attr ) != getattr( self.__operations__[i + 1], attr ): attrMismatch = True break if attrMismatch: break # We do not do the merge if there are common files in the operations fileSetA = set( list( f.LFN for f in self.__operations__[i] ) ) fileSetB = set( list( f.LFN for f in self.__operations__[i + 1] ) ) if fileSetA & fileSetB: break # There is a maximum number of files one can add into an operation try: while len( self.__operations__[i + 1] ): self.__operations__[i] += self.__operations__[i + 1][0] del self.__operations__[i + 1][0] optimized = True del self.__operations__[i + 1] except RuntimeError: i += 1 i += 1 return S_OK( optimized )
class Request(Record): """ .. class:: Request :param int RequestID: requestID :param str Name: request' name :param str OwnerDN: request's owner DN :param str OwnerGroup: request owner group :param str Setup: DIRAC setup :param str SourceComponent: whatever :param int JobID: jobID :param datetime.datetime CreationTime: UTC datetime :param datetime.datetime SubmissionTime: UTC datetime :param datetime.datetime LastUpdate: UTC datetime :param str Status: request's status :param TypedList operations: list of operations """ ALL_STATES = ("Waiting", "Failed", "Done", "Scheduled", "Assigned", "Canceled") FINAL_STATES = ("Done", "Failed", "Canceled") def __init__(self, fromDict=None): """c'tor :param self: self reference """ Record.__init__(self) self.__waiting = None now = datetime.datetime.utcnow().replace(microsecond=0) self.__data__["CreationTime"] = now self.__data__["SubmitTime"] = now self.__data__["LastUpdate"] = now self.__data__["Status"] = "Done" self.__data__["JobID"] = 0 self.__data__["RequestID"] = 0 proxyInfo = getProxyInfo() if proxyInfo["OK"]: proxyInfo = proxyInfo["Value"] if proxyInfo["validGroup"] and proxyInfo["validDN"]: self.OwnerDN = proxyInfo["identity"] self.OwnerGroup = proxyInfo["group"] self.__dirty = [] self.__operations__ = TypedList(allowedTypes=Operation) fromDict = fromDict if fromDict else {} self.__dirty = fromDict.get("__dirty", []) if "__dirty" in fromDict: del fromDict["__dirty"] for opDict in fromDict.get("Operations", []): self += Operation(opDict) if "Operations" in fromDict: del fromDict["Operations"] for key, value in fromDict.items(): if key not in self.__data__: raise AttributeError("Unknown Request attribute '%s'" % key) if value: setattr(self, key, value) self._notify() @staticmethod def tableDesc(): """ get table desc """ return { "Fields": { "RequestID": "INTEGER NOT NULL AUTO_INCREMENT", "RequestName": "VARCHAR(255) NOT NULL", "OwnerDN": "VARCHAR(255)", "OwnerGroup": "VARCHAR(32)", "Status": "ENUM('Waiting', 'Assigned', 'Done', 'Failed', 'Canceled', 'Scheduled') DEFAULT 'Waiting'", "Error": "VARCHAR(255)", "DIRACSetup": "VARCHAR(32)", "SourceComponent": "BLOB", "JobID": "INTEGER DEFAULT 0", "CreationTime": "DATETIME", "SubmitTime": "DATETIME", "LastUpdate": "DATETIME" }, "PrimaryKey": ["RequestID"], 'UniqueIndexes': { 'RequestName': ['RequestName'] } } def _notify(self): """ simple state machine for sub request statuses """ self.__waiting = None # # update operations statuses rStatus = "Waiting" opStatusList = [(op.Status, op) for op in self] self.__waiting = None while opStatusList: # # Scan all status in order! opStatus, op = opStatusList.pop(0) # # Failed -> Failed if opStatus == "Failed": rStatus = "Failed" break # Scheduled -> Scheduled if opStatus == "Scheduled": if self.__waiting == None: self.__waiting = op rStatus = "Scheduled" # # First operation Queued becomes Waiting if no Waiting/Scheduled before elif opStatus == "Queued": if self.__waiting == None: self.__waiting = op op._setWaiting(self) rStatus = "Waiting" # # First operation Waiting is next to execute, others are queued elif opStatus == "Waiting": rStatus = "Waiting" if self.__waiting == None: self.__waiting = op else: op._setQueued(self) # # All operations Done -> Done elif opStatus == "Done" and self.__waiting == None: rStatus = "Done" self.__data__['Error'] = '' self.Status = rStatus def getWaiting(self): """ get waiting operation if any """ # # update states self._notify() return S_OK(self.__waiting) # # Operation arithmetics def __contains__(self, operation): """ in operator :param self: self reference :param Operation subRequest: a subRequest """ return bool(operation in self.__operations__) def __iadd__(self, operation): """ += operator for subRequest :param self: self reference :param Operation operation: sub-request to add """ if operation not in self: self.__operations__.append(operation) operation._parent = self self._notify() return self def insertBefore(self, newOperation, existingOperation): """ insert :newOperation: just before :existingOperation: :param self: self reference :param Operation newOperation: Operation to be inserted :param Operation existingOperation: previous Operation sibling """ if existingOperation not in self: return S_ERROR("%s is not in" % existingOperation) if newOperation in self: return S_ERROR("%s is already in" % newOperation) self.__operations__.insert( self.__operations__.index(existingOperation), newOperation) newOperation._parent = self self._notify() return S_OK() def insertAfter(self, newOperation, existingOperation): """ insert :newOperation: just after :existingOperation: :param self: self reference :param Operation newOperation: Operation to be inserted :param Operation existingOperation: next Operation sibling """ if existingOperation not in self: return S_ERROR("%s is not in" % existingOperation) if newOperation in self: return S_ERROR("%s is already in" % newOperation) self.__operations__.insert( self.__operations__.index(existingOperation) + 1, newOperation) newOperation._parent = self self._notify() return S_OK() def addOperation(self, operation): """ add :operation: to list of Operations :param self: self reference :param Operation operation: Operation to be inserted """ if operation in self: return S_ERROR("This operation is already in!!!") self += operation return S_OK() def isEmpty(self): """ Evaluate if the request is empty """ return len(self.__operations__) == 0 def __iter__(self): """ iterator for sub-request """ return self.__operations__.__iter__() def __getitem__(self, i): """ [] op for sub requests """ return self.__operations__.__getitem__(i) def __setitem__(self, i, value): """ self[i] = val """ self.__operations__._typeCheck(value) if self[i].OperationID: self.__dirty.append(self[i].OperationID) self.__operations__.__setitem__(i, value) value._parent = self self._notify() def __delitem__(self, i): """ del self[i]""" if not self.RequestID: self.__operations__.__delitem__(i) else: opId = self[i].OperationID if opId: self.__dirty.append(opId) self.__operations__.__delitem__(i) self._notify() def indexOf(self, subReq): """ return index of subReq (execution order) """ return self.__operations__.index(subReq) if subReq in self else -1 def __nonzero__(self): """ for comparisons """ return True def __len__(self): """ nb of subRequests """ return len(self.__operations__) def __str__(self): """ str operator """ return str(self.toJSON()["Value"]) def subStatusList(self): """ list of statuses for all operations """ return [subReq.Status for subReq in self] # # properties @property def RequestID(self): """ request ID getter """ return self.__data__["RequestID"] @RequestID.setter def RequestID(self, value): """ requestID setter (shouldn't be RO???) """ self.__data__["RequestID"] = long(value) if value else 0 @property def RequestName(self): """ request's name getter """ return self.__data__["RequestName"] @RequestName.setter def RequestName(self, value): """ request name setter """ if type(value) != str: raise TypeError("RequestName should be a string") self.__data__["RequestName"] = value[:128] @property def OwnerDN(self): """ request owner DN getter """ return self.__data__["OwnerDN"] @OwnerDN.setter def OwnerDN(self, value): """ request owner DN setter """ if type(value) != str: raise TypeError("OwnerDN should be a string!") self.__data__["OwnerDN"] = value @property def OwnerGroup(self): """ request owner group getter """ return self.__data__["OwnerGroup"] @OwnerGroup.setter def OwnerGroup(self, value): """ request owner group setter """ if type(value) != str: raise TypeError("OwnerGroup should be a string!") self.__data__["OwnerGroup"] = value @property def DIRACSetup(self): """ DIRAC setup getter """ return self.__data__["DIRACSetup"] @DIRACSetup.setter def DIRACSetup(self, value): """ DIRAC setup setter """ if type(value) != str: raise TypeError("setup should be a string!") self.__data__["DIRACSetup"] = value @property def SourceComponent(self): """ source component getter """ return self.__data__["SourceComponent"] @SourceComponent.setter def SourceComponent(self, value): """ source component setter """ if type(value) != str: raise TypeError("Setup should be a string!") self.__data__["SourceComponent"] = value @property def JobID(self): """ jobID getter """ return self.__data__["JobID"] @JobID.setter def JobID(self, value=0): """ jobID setter """ self.__data__["JobID"] = long(value) if value else 0 @property def CreationTime(self): """ creation time getter """ return self.__data__["CreationTime"] @CreationTime.setter def CreationTime(self, value=None): """ creation time setter """ if type(value) not in (datetime.datetime, str): raise TypeError("CreationTime should be a datetime.datetime!") if type(value) == str: value = datetime.datetime.strptime( value.split(".")[0], '%Y-%m-%d %H:%M:%S') self.__data__["CreationTime"] = value @property def SubmitTime(self): """ request's submission time getter """ return self.__data__["SubmitTime"] @SubmitTime.setter def SubmitTime(self, value=None): """ submission time setter """ if type(value) not in (datetime.datetime, str): raise TypeError("SubmitTime should be a datetime.datetime!") if type(value) == str: value = datetime.datetime.strptime( value.split(".")[0], '%Y-%m-%d %H:%M:%S') self.__data__["SubmitTime"] = value @property def LastUpdate(self): """ last update getter """ return self.__data__["LastUpdate"] @LastUpdate.setter def LastUpdate(self, value=None): """ last update setter """ if type(value) not in (datetime.datetime, str): raise TypeError("LastUpdate should be a datetime.datetime!") if type(value) == str: value = datetime.datetime.strptime( value.split(".")[0], '%Y-%m-%d %H:%M:%S') self.__data__["LastUpdate"] = value @property def Status(self): """ status getter """ self._notify() return self.__data__["Status"] @Status.setter def Status(self, value): """ status setter """ if value not in Request.ALL_STATES: raise ValueError("Unknown status: %s" % str(value)) # If the status moved to Failed or Done, update the lastUpdate time if value in ('Done', 'Failed'): if value != self.__data__["Status"]: self.LastUpdate = datetime.datetime.utcnow().replace( microsecond=0) if value == 'Done': self.__data__['Error'] = '' self.__data__["Status"] = value @property def Order(self): """ ro execution order getter """ self._notify() opStatuses = [op.Status for op in self.__operations__] return opStatuses.index("Waiting") if "Waiting" in opStatuses else len( opStatuses) @property def Error(self): """ error getter """ return self.__data__["Error"] @Error.setter def Error(self, value): """ error setter """ if type(value) != str: raise TypeError("Error has to be a string!") self.__data__["Error"] = self._escapeStr(value, 255) def toSQL(self): """ prepare SQL INSERT or UPDATE statement """ colVals = [ ("`%s`" % column, "'%s'" % value if type(value) in (str, datetime.datetime) else str(value) if value != None else "NULL") for column, value in self.__data__.items() if (column == 'Error' or value) and column not in ("RequestID", "LastUpdate") ] colVals.append(("`LastUpdate`", "UTC_TIMESTAMP()")) query = [] if self.RequestID: query.append("UPDATE `Request` SET ") query.append(", ".join(["%s=%s" % item for item in colVals])) query.append(" WHERE `RequestID`=%d;\n" % self.RequestID) else: query.append("INSERT INTO `Request` ") columns = "(%s)" % ",".join([column for column, value in colVals]) values = "(%s)" % ",".join([value for column, value in colVals]) query.append(columns) query.append(" VALUES %s;" % values) return S_OK("".join(query)) def cleanUpSQL(self): """ delete query for dirty operations """ query = [] if self.RequestID and self.__dirty: opIDs = ",".join([str(opID) for opID in self.__dirty]) query.append( "DELETE FROM `Operation` WHERE `RequestID`=%s AND `OperationID` IN (%s);\n" % (self.RequestID, opIDs)) for opID in self.__dirty: query.append("DELETE FROM `File` WHERE `OperationID`=%s;\n" % opID) return query # # digest def toJSON(self): """ serialize to JSON format """ digest = dict([(key, str(getattr(self, key)) if getattr(self, key) else "") for key in self.__data__]) digest["RequestID"] = self.RequestID digest["__dirty"] = self.__dirty digest["Operations"] = [op.toJSON()['Value'] for op in self] return S_OK(digest) def getDigest(self): """ return digest for request """ digest = ['Name:' + self.RequestName] for op in self: opDigest = [ str(item) for item in (op.Type, op.Type, op.Status, op.Order) ] if op.TargetSE: opDigest.append(op.TargetSE) if op.Catalog: opDigest.append(op.Catalog) if len(op): opFile = op[0] opDigest.append(opFile.LFN) opDigest.append(",...<%d files>" % len(op)) digest.append(":".join(opDigest)) return S_OK("\n".join(digest)) def optimize(self): """ Merges together the operations that can be merged. They need to have the following arguments equal: * Type * Arguments * SourceSE * TargetSE * Catalog It also makes sure that the maximum number of Files in an Operation is never overcome. CAUTION: this method is meant to be called before inserting into the DB. So if the RequestId is not 0, we don't touch :return S_ERROR if the Request should not be optimized (because already in the DB S_OK(True) if a optimization was carried out S_OK(False) if no optimization were carried out """ # Set to True if the request could be optimized optimized = False # List of attributes that must be equal for operations to be merged attrList = ["Type", "Arguments", "SourceSE", "TargetSE", "Catalog"] i = 0 # If the RequestID is not the default one (0), it probably means # the Request is already in the DB, so we don't touch anything if self.RequestID != 0: return S_ERROR( "Cannot optimize because Request seems to be already in the DB (RequestID %s)" % self.RequestID) # We could do it with a single loop (the 2nd one), but by doing this, # we can replace # i += 1 # continue # # with # break # # which is nicer in my opinion while i < len(self.__operations__): while (i + 1) < len(self.__operations__): # Some attributes need to be the same attrMismatch = False for attr in attrList: if getattr(self.__operations__[i], attr) != getattr( self.__operations__[i + 1], attr): attrMismatch = True break if attrMismatch: break # We do not do the merge if there are common files in the operations fileSetA = set(list(f.LFN for f in self.__operations__[i])) fileSetB = set(list(f.LFN for f in self.__operations__[i + 1])) if len(fileSetA & fileSetB): break # There is a maximum number of files one can add into an operation try: while len(self.__operations__[i + 1]): self.__operations__[i] += self.__operations__[i + 1][0] del self.__operations__[i + 1][0] optimized = True del self.__operations__[i + 1] except RuntimeError: i += 1 i += 1 return S_OK(optimized)
class Operation( Record ): """ .. class:: Operation :param long OperationID: OperationID as read from DB backend :param long RequestID: parent RequestID :param str Status: execution status :param str Type: operation to perform :param str Arguments: additional arguments :param str SourceSE: source SE name :param str TargetSE: target SE names as comma separated list :param str Catalog: catalog to use as comma separated list :param str Error: error string if any :param Request parent: parent Request instance """ # # max files in a single operation MAX_FILES = 10000 # # all states ALL_STATES = ( "Queued", "Waiting", "Scheduled", "Assigned", "Failed", "Done", "Canceled" ) # # final states FINAL_STATES = ( "Failed", "Done", "Canceled" ) def __init__( self, fromDict = None ): """ c'tor :param self: self reference :param dict fromDict: attributes dictionary """ Record.__init__( self ) self._parent = None # # sub-request attributes # self.__data__ = dict.fromkeys( self.tableDesc()["Fields"].keys(), None ) now = datetime.datetime.utcnow().replace( microsecond = 0 ) self.__data__["SubmitTime"] = now self.__data__["LastUpdate"] = now self.__data__["CreationTime"] = now self.__data__["OperationID"] = 0 self.__data__["RequestID"] = 0 self.__data__["Status"] = "Queued" # # operation files self.__files__ = TypedList( allowedTypes = File ) # # dirty fileIDs self.__dirty = [] # # init from dict fromDict = fromDict if fromDict else {} self.__dirty = fromDict.get( "__dirty", [] ) if "__dirty" in fromDict: del fromDict["__dirty"] for fileDict in fromDict.get( "Files", [] ): self.addFile( File( fileDict ) ) if "Files" in fromDict: del fromDict["Files"] for key, value in fromDict.items(): if key not in self.__data__: raise AttributeError( "Unknown Operation attribute '%s'" % key ) if key != "Order" and value: setattr( self, key, value ) @staticmethod def tableDesc(): """ get table desc """ return { "Fields" : { "OperationID" : "INTEGER NOT NULL AUTO_INCREMENT", "RequestID" : "INTEGER NOT NULL", "Type" : "VARCHAR(64) NOT NULL", "Status" : "ENUM('Waiting', 'Assigned', 'Queued', 'Done', 'Failed', 'Canceled', 'Scheduled') "\ "DEFAULT 'Queued'", "Arguments" : "MEDIUMBLOB", "Order" : "INTEGER NOT NULL", "SourceSE" : "VARCHAR(255)", "TargetSE" : "VARCHAR(255)", "Catalog" : "VARCHAR(255)", "Error": "VARCHAR(255)", "CreationTime" : "DATETIME", "SubmitTime" : "DATETIME", "LastUpdate" : "DATETIME" }, 'ForeignKeys': {'RequestID': 'Request.RequestID' }, "PrimaryKey" : "OperationID" } # # protected methods for parent only def _notify( self ): """ notify self about file status change """ fStatus = set( self.fileStatusList() ) if fStatus == set( ['Failed'] ): # All files Failed -> Failed newStatus = 'Failed' elif 'Scheduled' in fStatus: newStatus = 'Scheduled' elif "Waiting" in fStatus: newStatus = 'Queued' elif 'Failed' in fStatus: newStatus = 'Failed' else: self.__data__['Error'] = '' newStatus = 'Done' # If the status moved to Failed or Done, update the lastUpdate time if newStatus in ( 'Failed', 'Done', 'Scheduled' ): if self.__data__["Status"] != newStatus: self.LastUpdate = datetime.datetime.utcnow().replace( microsecond = 0 ) self.__data__["Status"] = newStatus if self._parent: self._parent._notify() def _setQueued( self, caller ): """ don't touch """ if caller == self._parent: self.__data__["Status"] = "Queued" def _setWaiting( self, caller ): """ don't touch as well """ if caller == self._parent: self.__data__["Status"] = "Waiting" # # Files arithmetics def __contains__( self, opFile ): """ in operator """ return opFile in self.__files__ def __iadd__( self, opFile ): """ += operator """ if len( self ) >= Operation.MAX_FILES: raise RuntimeError( "too many Files in a single Operation" ) self.addFile( opFile ) return self def addFile( self, opFile ): """ add :opFile: to operation """ if len( self ) >= Operation.MAX_FILES: raise RuntimeError( "too many Files in a single Operation" ) if opFile not in self: self.__files__.append( opFile ) opFile._parent = self self._notify() # # helpers for looping def __iter__( self ): """ files iterator """ return self.__files__.__iter__() def __getitem__( self, i ): """ [] op for opFiles """ return self.__files__.__getitem__( i ) def __delitem__( self, i ): """ remove file from op, only if OperationID is NOT set """ if not self.OperationID: self.__files__.__delitem__( i ) else: if self[i].FileID: self.__dirty.append( self[i].FileID ) self.__files__.__delitem__( i ) self._notify() def __setitem__( self, i, opFile ): """ overwrite opFile """ self.__files__._typeCheck( opFile ) toDelete = self[i] if toDelete.FileID: self.__dirty.append( toDelete.FileID ) self.__files__.__setitem__( i, opFile ) opFile._parent = self self._notify() def fileStatusList( self ): """ get list of files statuses """ return [ subFile.Status for subFile in self ] def __nonzero__( self ): """ for comparisons """ return True def __len__( self ): """ nb of subFiles """ return len( self.__files__ ) # # properties @property def RequestID( self ): """ RequestID getter (RO) """ return self._parent.RequestID if self._parent else -1 @RequestID.setter def RequestID( self, value ): """ can't set RequestID by hand """ self.__data__["RequestID"] = self._parent.RequestID if self._parent else -1 @property def OperationID( self ): """ OperationID getter """ return self.__data__["OperationID"] @OperationID.setter def OperationID( self, value ): """ OperationID setter """ self.__data__["OperationID"] = long( value ) if value else 0 @property def Type( self ): """ operation type prop """ return self.__data__["Type"] @Type.setter def Type( self, value ): """ operation type setter """ self.__data__["Type"] = str( value ) @property def Arguments( self ): """ arguments getter """ return self.__data__["Arguments"] @Arguments.setter def Arguments( self, value ): """ arguments setter """ self.__data__["Arguments"] = value if value else "" @property def SourceSE( self ): """ source SE prop """ return self.__data__["SourceSE"] if self.__data__["SourceSE"] else "" @SourceSE.setter def SourceSE( self, value ): """ source SE setter """ value = ",".join( self._uniqueList( value ) ) if len( value ) > 256: raise ValueError( "SourceSE list too long" ) self.__data__["SourceSE"] = str( value )[:255] if value else "" @property def sourceSEList( self ): """ helper property returning source SEs as a list""" return self.SourceSE.split( "," ) @property def TargetSE( self ): """ target SE prop """ return self.__data__["TargetSE"] if self.__data__["TargetSE"] else "" @TargetSE.setter def TargetSE( self, value ): """ target SE setter """ value = ",".join( self._uniqueList( value ) ) if len( value ) > 256: raise ValueError( "TargetSE list too long" ) self.__data__["TargetSE"] = value[:255] if value else "" @property def targetSEList( self ): """ helper property returning target SEs as a list""" return self.TargetSE.split( "," ) @property def Catalog( self ): """ catalog prop """ return self.__data__["Catalog"] @Catalog.setter def Catalog( self, value ): """ catalog setter """ # FIXME ######### THIS IS A TEMPORARY HOT FIX MEANT TO SMOOTH THE LFC->DFC MIGRATION if value == "LcgFileCatalogCombined": value = "FileCatalog,LcgFileCatalogCombined" ########################################################################### value = ",".join( self._uniqueList( value ) ) if len( value ) > 255: raise ValueError( "Catalog list too long" ) self.__data__["Catalog"] = value if value else "" @property def catalogList( self ): """ helper property returning catalogs as list """ return self.__data__["Catalog"].split( "," ) @property def Error( self ): """ error prop """ return self.__data__["Error"] @Error.setter def Error( self, value ): """ error setter """ if type( value ) != str: raise TypeError( "Error has to be a string!" ) self.__data__["Error"] = self._escapeStr( value[:240], 255 ) @property def Status( self ): """ Status prop """ return self.__data__["Status"] @Status.setter def Status( self, value ): """ Status setter """ if value not in Operation.ALL_STATES: raise ValueError( "unknown Status '%s'" % str( value ) ) if self.__files__: self._notify() else: # If the status moved to Failed or Done, update the lastUpdate time if value in ( 'Failed', 'Done' ): if self.__data__["Status"] != value: self.LastUpdate = datetime.datetime.utcnow().replace( microsecond = 0 ) self.__data__["Status"] = value if self._parent: self._parent._notify() if self.__data__['Status'] == 'Done': self.__data__['Error'] = '' @property def Order( self ): """ order prop """ if self._parent: self.__data__["Order"] = self._parent.indexOf( self ) if self._parent else -1 return self.__data__["Order"] @property def CreationTime( self ): """ operation creation time prop """ return self.__data__["CreationTime"] @CreationTime.setter def CreationTime( self, value = None ): """ creation time setter """ if type( value ) not in ( datetime.datetime, str ): raise TypeError( "CreationTime should be a datetime.datetime!" ) if type( value ) == str: value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' ) self.__data__["CreationTime"] = value @property def SubmitTime( self ): """ subrequest's submit time prop """ return self.__data__["SubmitTime"] @SubmitTime.setter def SubmitTime( self, value = None ): """ submit time setter """ if type( value ) not in ( datetime.datetime, str ): raise TypeError( "SubmitTime should be a datetime.datetime!" ) if type( value ) == str: value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' ) self.__data__["SubmitTime"] = value @property def LastUpdate( self ): """ last update prop """ return self.__data__["LastUpdate"] @LastUpdate.setter def LastUpdate( self, value = None ): """ last update setter """ if type( value ) not in ( datetime.datetime, str ): raise TypeError( "LastUpdate should be a datetime.datetime!" ) if type( value ) == str: value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' ) self.__data__["LastUpdate"] = value if self._parent: self._parent.LastUpdate = value def __str__( self ): """ str operator """ return str( self.toJSON()["Value"] ) def toSQL( self ): """ get SQL INSERT or UPDATE statement """ if not getattr( self, "RequestID" ): raise AttributeError( "RequestID not set" ) colVals = [ ( "`%s`" % column, "'%s'" % getattr( self, column ) if type( getattr( self, column ) ) in ( str, datetime.datetime ) else str( getattr( self, column ) ) if getattr( self, column ) != None else "NULL" ) for column in self.__data__ if ( column == 'Error' or getattr( self, column ) ) and column not in ( "OperationID", "LastUpdate", "Order" ) ] colVals.append( ( "`LastUpdate`", "UTC_TIMESTAMP()" ) ) colVals.append( ( "`Order`", str( self.Order ) ) ) # colVals.append( ( "`Status`", "'%s'" % str(self.Status) ) ) query = [] if self.OperationID: query.append( "UPDATE `Operation` SET " ) query.append( ", ".join( [ "%s=%s" % item for item in colVals ] ) ) query.append( " WHERE `OperationID`=%d;\n" % self.OperationID ) else: query.append( "INSERT INTO `Operation` " ) columns = "(%s)" % ",".join( [ column for column, value in colVals ] ) values = "(%s)" % ",".join( [ value for column, value in colVals ] ) query.append( columns ) query.append( " VALUES %s;\n" % values ) return S_OK( "".join( query ) ) def cleanUpSQL( self ): """ query deleting dirty records from File table """ if self.OperationID and self.__dirty: fIDs = ",".join( [ str( fid ) for fid in self.__dirty ] ) return "DELETE FROM `File` WHERE `OperationID` = %s AND `FileID` IN (%s);\n" % ( self.OperationID, fIDs ) def toJSON( self ): """ get json digest """ digest = dict( [( key, str( getattr( self, key ) ) if getattr( self, key ) else "" ) for key in self.__data__] ) digest["RequestID"] = str( self.RequestID ) digest["Order"] = str( self.Order ) if self.__dirty: digest["__dirty"] = self.__dirty digest["Files"] = [opFile.toJSON()['Value'] for opFile in self] return S_OK( digest )