def getRequest():
"""
helper fcn to build requestContainer
"""
requestContainer = RequestContainer(init=False)
## get request
requestContainer.setRequestName("00009423_00000118")
requestContainer.setJobID(0)
requestContainer.setOwnerDN("")
requestContainer.setOwnerGroup("")
requestContainer.setDIRACSetup("")
requestContainer.setSourceComponent(None)
requestContainer.setCreationTime("0000-00-00 00:00:00")
requestContainer.setLastUpdate("2011-02-19 04:57:02")
requestContainer.setStatus("Waiting")
## get subrequest
requestContainer.initiateSubRequest("transfer")
subRequestDict = {
"Status": "Waiting",
"SubRequestID": 2259916,
"Operation": "replicateAndRegister",
"Arguments": None,
"ExecutionOrder": 0,
"SourceSE": None,
"TargetSE": "GRIDKA_MC-DST,GRIDKA_MC_M-DST",
"Catalogue": None,
"CreationTime": "2011-02-19 04:57:02",
"SubmissionTime": "2011-02-19 04:57:02",
"LastUpdate": "2011-08-18 20:14:22"
}
requestContainer.setSubRequestAttributes(0, "transfer", subRequestDict)
## get subrequest files
files = [{
"FileID": 1610538,
"LFN":
"/lhcb/MC/MC10/ALLSTREAMS.DST/00009422/0000/00009422_00000171_1.allstreams.dst",
"Size": None,
"PFN": None,
"GUID": None,
"Md5": None,
"Addler": None,
"Attempt": 1,
"Status": "Scheduled"
}]
requestContainer.setSubRequestFiles(0, "transfer", files)
return {
"OK": True,
"Value": {
"RequestName": "00009423_00000118",
"RequestString": requestContainer.toXML()["Value"],
"JobID": 0,
"RequestContainer": requestContainer
}
}
def getRegisterRequest():
""" helper fcn to build request
"""
requestContainer = RequestContainer(init=False)
requestContainer.setJobID(11889410)
#requestContainer.setOwnerDN( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" )
requestContainer.setOwnerGroup("lhcb_user")
requestContainer.setDIRACSetup("LHCb-Production")
requestContainer.setSourceComponent(None)
requestContainer.setCreationTime("0000-00-00 00:00:00")
requestContainer.setLastUpdate("2011-02-19 04:57:02")
requestContainer.setStatus("Waiting")
requestContainer.initiateSubRequest("register")
subRequestDict = {
"Status": "Waiting",
"SubRequestID": 2259916,
"Operation": "registerFile",
"Arguments": None,
"ExecutionOrder": 0,
"SourceSE": None,
"TargetSE": "RAL-USER",
"Catalogue": "LcgFileCatalogCombined",
"CreationTime": "2011-02-19 04:57:02",
"SubmissionTime": "2011-02-19 04:57:02",
"LastUpdate": "2011-08-18 20:14:22"
}
requestContainer.setSubRequestAttributes(0, "register", subRequestDict)
files = [{
"FileID": 1610538,
"LFN": "/lhcb/user/c/cblanks/11889/11889410/LDSB.rsQrRL",
"Size": 153961749,
"PFN":
"srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cblanks/11889/11889410/LDSB.rsQrRL",
"GUID": "5911A19C-7CDF-7F2A-36ED-089CD410F98A",
"Md5": None,
"Addler": "92b85e26",
"Attempt": 1,
"Status": "Waiting"
}]
requestContainer.setSubRequestFiles(0, "register", files)
return {
"requestName":
"11889410.xml",
"requestString":
requestContainer.toXML()["Value"],
"jobID":
11889410,
"executionOrder":
0,
"sourceServer":
"foobarserver",
"configPath":
"/Systems/DataManagement/Development/Agents/RegistrationAgent"
}
def getRequest(operation):
""" fake requestDict
:param str operation: sub-request operation attribute
"""
requestContainer = RequestContainer(init=False)
requestContainer.setJobID(1)
#requestContainer.setOwnerDN( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" )
requestContainer.setOwnerGroup("lhcb_user")
requestContainer.setDIRACSetup("LHCb-Production")
requestContainer.setSourceComponent(None)
requestContainer.setCreationTime("0000-00-00 00:00:00")
requestContainer.setLastUpdate("2011-12-01 04:57:02")
requestContainer.setStatus("Waiting")
requestContainer.setAttribute("RequestID", 123456789)
requestContainer.initiateSubRequest("transfer")
subRequestDict = {
"Status": "Waiting",
"SubRequestID": 2222222,
"Operation": operation,
"Arguments": None,
"ExecutionOrder": 0,
"SourceSE": None,
"TargetSE": "CERN-USER",
"Catalogue": "LcgFileCatalogCombined",
"CreationTime": "2011-12-01 04:57:02",
"SubmissionTime": "2011-12-01 04:57:02",
"LastUpdate": "2011-12-01 20:14:22"
}
requestContainer.setSubRequestAttributes(0, "transfer", subRequestDict)
files = [{
"FileID": 3333333,
"LFN": "/lhcb/user/c/cibak/11889/11889410/test.zzz",
"Size": 44444444,
"PFN":
"srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cibak/11889/11889410/test.zzz",
"GUID": "5P13RD4L-4J5L-3D21-U5P1-3RD4L4J5P13R",
"Md5": None,
"Addler": "92b85e26",
"Attempt": 1,
"Status": "Waiting"
}]
requestContainer.setSubRequestFiles(0, "transfer", files)
return {
"OK": True,
"Value": {
"requestName": "%s.xml" % operation,
"requestString": requestContainer.toXML_new()["Value"],
"requestObj": requestContainer,
"jobID": 1,
"executionOrder": 0,
"sourceServer": "foobarserver"
}
}
def getRegisterRequest():
""" helper fcn to build request
"""
requestContainer = RequestContainer(init=False)
requestContainer.setJobID(11889410)
# requestContainer.setOwnerDN( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" )
requestContainer.setOwnerGroup("lhcb_user")
requestContainer.setDIRACSetup("LHCb-Production")
requestContainer.setSourceComponent(None)
requestContainer.setCreationTime("0000-00-00 00:00:00")
requestContainer.setLastUpdate("2011-02-19 04:57:02")
requestContainer.setStatus("Waiting")
requestContainer.initiateSubRequest("register")
subRequestDict = {
"Status": "Waiting",
"SubRequestID": 2259916,
"Operation": "registerFile",
"Arguments": None,
"ExecutionOrder": 0,
"SourceSE": None,
"TargetSE": "RAL-USER",
"Catalogue": "LcgFileCatalogCombined",
"CreationTime": "2011-02-19 04:57:02",
"SubmissionTime": "2011-02-19 04:57:02",
"LastUpdate": "2011-08-18 20:14:22",
}
requestContainer.setSubRequestAttributes(0, "register", subRequestDict)
files = [
{
"FileID": 1610538,
"LFN": "/lhcb/user/c/cblanks/11889/11889410/LDSB.rsQrRL",
"Size": 153961749,
"PFN": "srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cblanks/11889/11889410/LDSB.rsQrRL",
"GUID": "5911A19C-7CDF-7F2A-36ED-089CD410F98A",
"Md5": None,
"Addler": "92b85e26",
"Attempt": 1,
"Status": "Waiting",
}
]
requestContainer.setSubRequestFiles(0, "register", files)
return {
"requestName": "11889410.xml",
"requestString": requestContainer.toXML()["Value"],
"jobID": 11889410,
"executionOrder": 0,
"sourceServer": "foobarserver",
"configPath": "/Systems/DataManagement/Development/Agents/RegistrationAgent",
}
def getRequest():
"""
helper fcn to build requestContainer
"""
requestContainer = RequestContainer( init = False )
## get request
requestContainer.setRequestName( "00009423_00000118" )
requestContainer.setJobID( 0 )
requestContainer.setOwnerDN( "" )
requestContainer.setOwnerGroup( "" )
requestContainer.setDIRACSetup( "" )
requestContainer.setSourceComponent( None )
requestContainer.setCreationTime( "0000-00-00 00:00:00" )
requestContainer.setLastUpdate( "2011-02-19 04:57:02" )
requestContainer.setStatus( "Waiting" )
## get subrequest
requestContainer.initiateSubRequest( "transfer" )
subRequestDict = { "Status" : "Waiting",
"SubRequestID" : 2259916,
"Operation" : "replicateAndRegister",
"Arguments" : None,
"ExecutionOrder" : 0,
"SourceSE" : None,
"TargetSE" : "GRIDKA_MC-DST,GRIDKA_MC_M-DST",
"Catalogue" : None,
"CreationTime" : "2011-02-19 04:57:02",
"SubmissionTime" : "2011-02-19 04:57:02",
"LastUpdate" : "2011-08-18 20:14:22" }
requestContainer.setSubRequestAttributes( 0, "transfer", subRequestDict )
## get subrequest files
files = [ { "FileID" : 1610538,
"LFN" : "/lhcb/MC/MC10/ALLSTREAMS.DST/00009422/0000/00009422_00000171_1.allstreams.dst",
"Size" : None,
"PFN" : None,
"GUID" : None,
"Md5" : None,
"Addler" : None,
"Attempt" : 1,
"Status" : "Scheduled" } ]
requestContainer.setSubRequestFiles( 0, "transfer", files )
return { "OK" : True,
"Value" : { "RequestName" : "00009423_00000118",
"RequestString" : requestContainer.toXML()["Value"],
"JobID" : 0,
"RequestContainer" : requestContainer } }
def getKwargsRemoveFile():
""" helper fcn to build request """
requestContainer = RequestContainer(init=False)
requestContainer.setJobID(11111111)
#requestContainer.setOwnerDN( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" )
requestContainer.setOwnerGroup("lhcb_user")
requestContainer.setDIRACSetup("LHCb-Production")
requestContainer.setSourceComponent(None)
requestContainer.setCreationTime("0000-00-00 00:00:00")
requestContainer.setLastUpdate("2011-12-01 04:57:02")
requestContainer.setStatus("Waiting")
requestContainer.initiateSubRequest("removal")
subRequestDict = {
"Status": "Waiting",
"SubRequestID": 2222222,
"Operation": "removeFile",
"Arguments": None,
"ExecutionOrder": 0,
"SourceSE": None,
"TargetSE": "RAL-USER",
"Catalogue": "LcgFileCatalogCombined",
"CreationTime": "2011-12-01 04:57:02",
"SubmissionTime": "2011-12-01 04:57:02",
"LastUpdate": "2011-12-01 20:14:22"
}
requestContainer.setSubRequestAttributes(0, "removal", subRequestDict)
files = [{
"FileID": 3333333,
"LFN": "/lhcb/user/c/cibak/11889/11889410/test.zzz",
"Size": 44444444,
"PFN":
"srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cibak/11889/11889410/test.zzz",
"GUID": "5P13RD4L-4J5L-3D21-U5P1-3RD4L4J5P13R",
"Md5": None,
"Addler": "92b85e26",
"Attempt": 1,
"Status": "Waiting"
}]
requestContainer.setSubRequestFiles(0, "removal", files)
return {
"requestName": "00000001.xml",
"requestString": requestContainer.toXML()["Value"],
"jobID": 1,
"executionOrder": 0,
"sourceServer": "foobarserver",
"configPath": "/Systems/DataManagement/Development/Agents/RemovalAgent"
}
def getRequest(operation):
""" fake requestDict
:param str operation: sub-request operation attribute
"""
requestContainer = RequestContainer(init=False)
requestContainer.setJobID(1)
requestContainer.setOwnerDN("/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba")
requestContainer.setOwnerGroup("lhcb_user")
requestContainer.setDIRACSetup("LHCb-Production")
requestContainer.setSourceComponent(None)
requestContainer.setCreationTime("0000-00-00 00:00:00")
requestContainer.setLastUpdate("2011-12-01 04:57:02")
requestContainer.setStatus("Waiting")
requestContainer.setAttribute("RequestID", 123456789)
requestContainer.initiateSubRequest("transfer")
subRequestDict = {
"Status": "Waiting",
"SubRequestID": 2222222,
"Operation": operation,
"Arguments": None,
"ExecutionOrder": 0,
"SourceSE": None,
"TargetSE": "CERN-USER,PIC-USER",
"Catalogue": "LcgFileCatalogCombined",
"CreationTime": "2011-12-01 04:57:02",
"SubmissionTime": "2011-12-01 04:57:02",
"LastUpdate": "2011-12-01 20:14:22",
}
requestContainer.setSubRequestAttributes(0, "transfer", subRequestDict)
files = [
{
"FileID": 3333333,
"LFN": "/lhcb/user/c/cibak/11889/11889410/test.zzz",
"Size": 44444444,
"PFN": "srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cibak/11889/11889410/test.zzz",
"GUID": "5P13RD4L-4J5L-3D21-U5P1-3RD4L4J5P13R",
"Md5": None,
"Addler": "92b85e26",
"Attempt": 1,
"Status": "Waiting",
}
]
requestContainer.setSubRequestFiles(0, "transfer", files)
return requestContainer
def getKwargsRemoveFile():
""" helper fcn to build request """
requestContainer = RequestContainer( init = False )
requestContainer.setJobID( 11111111 )
#requestContainer.setOwnerDN( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" )
requestContainer.setOwnerGroup( "lhcb_user" )
requestContainer.setDIRACSetup( "LHCb-Production" )
requestContainer.setSourceComponent( None )
requestContainer.setCreationTime( "0000-00-00 00:00:00" )
requestContainer.setLastUpdate( "2011-12-01 04:57:02" )
requestContainer.setStatus( "Waiting" )
requestContainer.initiateSubRequest( "removal" )
subRequestDict = { "Status" : "Waiting",
"SubRequestID" : 2222222,
"Operation" : "removeFile",
"Arguments" : None,
"ExecutionOrder" : 0,
"SourceSE" : None,
"TargetSE" : "RAL-USER",
"Catalogue" : "LcgFileCatalogCombined",
"CreationTime" : "2011-12-01 04:57:02",
"SubmissionTime" : "2011-12-01 04:57:02",
"LastUpdate" : "2011-12-01 20:14:22" }
requestContainer.setSubRequestAttributes( 0, "removal", subRequestDict )
files = [ { "FileID" : 3333333,
"LFN" : "/lhcb/user/c/cibak/11889/11889410/test.zzz",
"Size" : 44444444,
"PFN" : "srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cibak/11889/11889410/test.zzz",
"GUID" : "5P13RD4L-4J5L-3D21-U5P1-3RD4L4J5P13R",
"Md5" : None,
"Addler" : "92b85e26",
"Attempt" : 1,
"Status" : "Waiting" } ]
requestContainer.setSubRequestFiles( 0, "removal", files )
return { "requestName" : "00000001.xml",
"requestString" : requestContainer.toXML()["Value"],
"jobID" : 1,
"executionOrder" : 0,
"sourceServer" : "foobarserver",
"configPath" : "/Systems/DataManagement/Development/Agents/RemovalAgent" }
def getRequest( self, requestType ):
""" Get a request of a given type eligible for execution
"""
# RG: What if requestType is not given?
# the first query will return nothing.
# KC: maybe returning S_ERROR would be enough?
# alternatively we should check if requestType is known (in 'transfer', 'removal', 'register' and 'diset')
if not requestType or type( requestType ) not in types.StringTypes:
return S_ERROR( "Request type not given." )
myRequestType = self._escapeString( requestType )
if not myRequestType:
return myRequestType
myRequestType = myRequestType['Value']
start = time.time()
dmRequest = RequestContainer( init = False )
requestID = 0
subIDList = []
fields = ['RequestID', 'SubRequestID', 'Operation', 'Arguments',
'ExecutionOrder', 'SourceSE', 'TargetSE', 'Catalogue',
'CreationTime', 'SubmissionTime', 'LastUpdate', 'Status', 'RequestType']
# get the pending SubRequest sorted by ExecutionOrder and LastUpdate
req = "SELECT `RequestID`,`ExecutionOrder`,`Status`,`RequestType`,`LastUpdate` FROM `SubRequests` "\
"WHERE `Status` IN ( 'Waiting', 'Assigned' ) ORDER BY `ExecutionOrder`,`LastUpdate`"
# now get sorted list of RequestID (according to the above)
req = "SELECT * FROM ( %s ) as T1 GROUP BY `RequestID`" % req
# and get the 100 oldest ones of Type requestType
req = "SELECT `RequestID`,`ExecutionOrder` FROM ( %s ) as T2 WHERE `RequestType`=%s "\
"ORDER BY `LastUpdate` LIMIT 100" % ( req, myRequestType )
# and now get all waiting SubRequest for the selected RequestID and ExecutionOrder
req = "SELECT A.%s FROM SubRequests AS A, ( %s ) AS B WHERE " % ( ', A.'.join( fields ), req )
req = "%s A.RequestID=B.RequestID AND A.ExecutionOrder=B.ExecutionOrder" % ( req )
result = self._query( req )
if not result['OK']:
err = 'RequestDB._getRequest: Failed to retrieve Requests'
return S_ERROR( '%s\n%s' % ( err, result['Message'] ) )
if not result['Value']:
return S_OK()
# We get up to 10 Request candidates, to add some randomness
reqDict = {}
for row in result['Value']:
if ('"%s"' % row[-1]) != myRequestType:
continue
if row[-2] != 'Waiting':
continue
reqDict.setdefault( row[0], [] )
reqDict[row[0]].append( row[1:-2] )
reqIDList = reqDict.keys()
random.shuffle( reqIDList )
for reqID in reqIDList:
sidList = [ x[0] for x in reqDict[reqID] ]
for subID in sidList:
req = "UPDATE SubRequests SET Status='Assigned' WHERE RequestID=%s AND SubRequestID=%s;" % ( reqID, subID )
resAssigned = self._update( req )
if not resAssigned['OK']:
if subIDList:
self.__releaseSubRequests( reqID, subIDList )
return S_ERROR( 'Failed to assign subrequests: %s' % resAssigned['Message'] )
if resAssigned['Value'] == 0:
# Somebody has assigned this request
gLogger.warn( 'Already assigned subrequest %d of request %d' % ( subID, reqID ) )
else:
subIDList.append( subID )
if subIDList:
# We managed to get some requests, can continue now
requestID = reqID
break
# Haven't succeeded to get any request
if not requestID:
return S_OK()
dmRequest.setRequestID( requestID )
fields = ['FileID', 'LFN', 'Size', 'PFN', 'GUID', 'Md5', 'Addler', 'Attempt', 'Status' ]
for subRequestID, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, \
creationTime, submissionTime, lastUpdate in reqDict[requestID]:
if not subRequestID in subIDList: continue
res = dmRequest.initiateSubRequest( requestType )
ind = res['Value']
subRequestDict = {
'Status' : 'Waiting',
'SubRequestID' : subRequestID,
'Operation' : operation,
'Arguments' : arguments,
'ExecutionOrder': int( executionOrder ),
'SourceSE' : sourceSE,
'TargetSE' : targetSE,
'Catalogue' : catalogue,
'CreationTime' : creationTime,
'SubmissionTime': submissionTime,
'LastUpdate' : lastUpdate
}
res = dmRequest.setSubRequestAttributes( ind, requestType, subRequestDict )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set subRequest attributes for RequestID %s' % requestID
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
req = "SELECT %s FROM `Files` WHERE `SubRequestID`=%s ORDER BY `FileID`;" % ( ', '.join( fields ),
subRequestID )
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to get File attributes for RequestID %s.%s' % ( requestID,
subRequestID )
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
files = []
for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res['Value']:
fileDict = {'FileID':fileID, 'LFN':lfn, 'Size':size, 'PFN':pfn, 'GUID':guid,
'Md5':md5, 'Addler':addler, 'Attempt':attempt, 'Status':status}
files.append( fileDict )
res = dmRequest.setSubRequestFiles( ind, requestType, files )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set files into Request for RequestID %s.%s' % ( requestID,
subRequestID )
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
req = "SELECT Dataset,Status FROM Datasets WHERE SubRequestID = %s;" % subRequestID
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to get Datasets for RequestID %s.%s' % ( requestID, subRequestID )
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
datasets = []
for dataset, status in res['Value']:
datasets.append( dataset )
res = dmRequest.setSubRequestDatasets( ind, requestType, datasets )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set datasets into Request for RequestID %s.%s' % ( requestID,
subRequestID )
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
fields = ['RequestName', 'JobID', 'OwnerDN', 'OwnerGroup',
'DIRACSetup', 'SourceComponent', 'CreationTime',
'SubmissionTime', 'LastUpdate']
req = "SELECT %s FROM `Requests` WHERE `RequestID`=%s;" % ( ', '.join( fields ), requestID )
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to retrieve max RequestID'
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, \
creationTime, submissionTime, lastUpdate = res['Value'][0]
dmRequest.setRequestName( requestName )
dmRequest.setJobID( jobID )
dmRequest.setOwnerDN( ownerDN )
dmRequest.setOwnerGroup( ownerGroup )
dmRequest.setDIRACSetup( diracSetup )
dmRequest.setSourceComponent( sourceComponent )
dmRequest.setCreationTime( str( creationTime ) )
dmRequest.setLastUpdate( str( lastUpdate ) )
res = dmRequest.toXML()
if not res['OK']:
err = 'RequestDB._getRequest: Failed to create XML for RequestID %s' % ( requestID )
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
requestString = res['Value']
#still have to manage the status of the dataset properly
resultDict = {}
resultDict['RequestName'] = requestName
resultDict['RequestString'] = requestString
resultDict['JobID'] = jobID
return S_OK( resultDict )
def getRequest(self, requestType):
""" Get a request of a given type eligible for execution
"""
# RG: What if requestType is not given?
# the first query will return nothing.
# KC: maybe returning S_ERROR would be enough?
# alternatively we should check if requestType is known (in 'transfer', 'removal', 'register' and 'diset')
if not requestType or type(requestType) not in types.StringTypes:
return S_ERROR("Request type not given.")
myRequestType = self._escapeString(requestType)
if not myRequestType:
return myRequestType
myRequestType = myRequestType['Value']
start = time.time()
dmRequest = RequestContainer(init=False)
requestID = 0
subIDList = []
fields = [
'RequestID', 'SubRequestID', 'Operation', 'Arguments',
'ExecutionOrder', 'SourceSE', 'TargetSE', 'Catalogue',
'CreationTime', 'SubmissionTime', 'LastUpdate'
]
# get the pending SubRequest sorted by ExecutionOrder and LastUpdate
req = "SELECT RequestID, ExecutionOrder, Status, RequestType, LastUpdate from SubRequests WHERE Status IN ( 'Waiting', 'Assigned' ) ORDER BY ExecutionOrder, LastUpdate"
# now get sorted list of RequestID (according to the above)
req = "SELECT * from ( %s ) as T1 GROUP BY RequestID" % req
# and get the 100 oldest ones of Type requestType
req = "SELECT RequestID, ExecutionOrder FROM ( %s ) as T2 WHERE RequestType = %s ORDER BY LastUpdate limit 100" % (
req, myRequestType)
# and now get all waiting SubRequest for the selected RequestID and ExecutionOrder
req = "SELECT A.%s FROM SubRequests AS A, ( %s ) AS B WHERE " % (
', A.'.join(fields), req)
req = "%s A.RequestID = B.RequestID AND A.ExecutionOrder = B.ExecutionOrder AND A.Status = 'Waiting' AND A.RequestType = %s;" % (
req, myRequestType)
result = self._query(req)
if not result['OK']:
err = 'RequestDB._getRequest: Failed to retrieve Requests'
return S_ERROR('%s\n%s' % (err, result['Message']))
if not result['Value']:
return S_OK()
# We get up to 10 Request candidates, to add some randomness
reqDict = {}
for row in result['Value']:
reqDict.setdefault(row[0], [])
reqDict[row[0]].append(row[1:])
reqIDList = reqDict.keys()
random.shuffle(reqIDList)
for reqID in reqIDList:
sidList = [x[0] for x in reqDict[reqID]]
for subID in sidList:
req = "UPDATE SubRequests SET Status='Assigned' WHERE RequestID=%s AND SubRequestID=%s;" % (
reqID, subID)
resAssigned = self._update(req)
if not resAssigned['OK']:
if subIDList:
self.__releaseSubRequests(reqID, subIDList)
return S_ERROR('Failed to assign subrequests: %s' %
resAssigned['Message'])
if resAssigned['Value'] == 0:
# Somebody has assigned this request
gLogger.warn(
'Already assigned subrequest %d of request %d' %
(subID, reqID))
else:
subIDList.append(subID)
if subIDList:
# We managed to get some requests, can continue now
requestID = reqID
break
# Haven't succeeded to get any request
if not requestID:
return S_OK()
dmRequest.setRequestID(requestID)
fields = [
'FileID', 'LFN', 'Size', 'PFN', 'GUID', 'Md5', 'Addler', 'Attempt',
'Status'
]
for subRequestID, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, creationTime, submissionTime, lastUpdate in reqDict[
requestID]:
if not subRequestID in subIDList: continue
res = dmRequest.initiateSubRequest(requestType)
ind = res['Value']
subRequestDict = {
'Status': 'Waiting',
'SubRequestID': subRequestID,
'Operation': operation,
'Arguments': arguments,
'ExecutionOrder': int(executionOrder),
'SourceSE': sourceSE,
'TargetSE': targetSE,
'Catalogue': catalogue,
'CreationTime': creationTime,
'SubmissionTime': submissionTime,
'LastUpdate': lastUpdate
}
res = dmRequest.setSubRequestAttributes(ind, requestType,
subRequestDict)
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set subRequest attributes for RequestID %s' % requestID
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
req = "SELECT %s FROM Files WHERE SubRequestID = %s ORDER BY FileID;" % (
', '.join(fields), subRequestID)
res = self._query(req)
if not res['OK']:
err = 'RequestDB._getRequest: Failed to get File attributes for RequestID %s.%s' % (
requestID, subRequestID)
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
files = []
for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res[
'Value']:
fileDict = {
'FileID': fileID,
'LFN': lfn,
'Size': size,
'PFN': pfn,
'GUID': guid,
'Md5': md5,
'Addler': addler,
'Attempt': attempt,
'Status': status
}
files.append(fileDict)
res = dmRequest.setSubRequestFiles(ind, requestType, files)
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set files into Request for RequestID %s.%s' % (
requestID, subRequestID)
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
req = "SELECT Dataset,Status FROM Datasets WHERE SubRequestID = %s;" % subRequestID
res = self._query(req)
if not res['OK']:
err = 'RequestDB._getRequest: Failed to get Datasets for RequestID %s.%s' % (
requestID, subRequestID)
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
datasets = []
for dataset, status in res['Value']:
datasets.append(dataset)
res = dmRequest.setSubRequestDatasets(ind, requestType, datasets)
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set datasets into Request for RequestID %s.%s' % (
requestID, subRequestID)
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
fields = [
'RequestName', 'JobID', 'OwnerDN', 'OwnerGroup', 'DIRACSetup',
'SourceComponent', 'CreationTime', 'SubmissionTime', 'LastUpdate'
]
req = "SELECT %s from Requests WHERE RequestID = %s;" % (
', '.join(fields), requestID)
res = self._query(req)
if not res['OK']:
err = 'RequestDB._getRequest: Failed to retrieve max RequestID'
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, creationTime, submissionTime, lastUpdate = res[
'Value'][0]
dmRequest.setRequestName(requestName)
dmRequest.setJobID(jobID)
dmRequest.setOwnerDN(ownerDN)
dmRequest.setOwnerGroup(ownerGroup)
dmRequest.setDIRACSetup(diracSetup)
dmRequest.setSourceComponent(sourceComponent)
dmRequest.setCreationTime(str(creationTime))
dmRequest.setLastUpdate(str(lastUpdate))
res = dmRequest.toXML()
if not res['OK']:
err = 'RequestDB._getRequest: Failed to create XML for RequestID %s' % (
requestID)
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
requestString = res['Value']
#still have to manage the status of the dataset properly
resultDict = {}
resultDict['RequestName'] = requestName
resultDict['RequestString'] = requestString
resultDict['JobID'] = jobID
return S_OK(resultDict)
def getRequestForSubRequest(self, itself, subRequestID):
"""
Select Request given SubRequestID.
:param self: plugin reference
:param itself: patient reference for injection
:param int subRequestID: SubRequests.SubRequestID
:warn: function has to be injected to RequestDBMySQL instance
"""
## get RequestID
requestID = "SELECT RequestID FROM SubRequests WHERE SubRequestID = %s;" % str(
subRequestID)
requestID = self._query(requestID)
if not requestID["OK"]:
return requestID
requestID = requestID["Value"][0]
## create RequestContainer
requestContainer = RequestContainer(init=False)
requestContainer.setRequestID(requestID)
## put some basic infos in
requestInfo = "SELECT RequestName, JobID, OwnerDN, OwnerGroup, DIRACSetup, SourceComponent, CreationTime, SubmissionTime, LastUpdate, Status "
requestInfo += "FROM Requests WHERE RequestID = %d;" % requestID
requestInfo = self._query(requestInfo)
if not requestInfo["OK"]:
return requestInfo
requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, creationTime, submissionTime, lastUpdate, status = requestInfo[
'Value'][0]
requestContainer.setRequestName(requestName)
requestContainer.setJobID(jobID)
requestContainer.setOwnerDN(ownerDN)
requestContainer.setOwnerGroup(ownerGroup)
requestContainer.setDIRACSetup(diracSetup)
requestContainer.setSourceComponent(sourceComponent)
requestContainer.setCreationTime(str(creationTime))
requestContainer.setLastUpdate(str(lastUpdate))
requestContainer.setStatus(status)
## get sub-requests
subRequests = "SELECT SubRequestID, Status, RequestType, Operation, Arguments, ExecutionOrder, SourceSE, "
subRequests += "TargetSE, Catalogue, CreationTime, SubmissionTime, LastUpdate FROM SubRequests WHERE RequestID=%s;" % requestID
subRequests = self._query(subRequests)
if not subRequests["OK"]:
return subRequests
## loop over sub requests
for subRequestID, status, requestType, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, creationTime, submissionTime, lastUpdate in subRequests[
"Value"]:
res = requestContainer.initiateSubRequest(requestType)
ind = res["Value"]
subRequestDict = {
"Status": status,
"SubRequestID": subRequestID,
"Operation": operation,
"Arguments": arguments,
"ExecutionOrder": int(executionOrder),
"SourceSE": sourceSE,
"TargetSE": targetSE,
"Catalogue": catalogue,
"CreationTime": creationTime,
"SubmissionTime": submissionTime,
"LastUpdate": lastUpdate
}
res = requestContainer.setSubRequestAttributes(
ind, requestType, subRequestDict)
if not res["OK"]:
return res
## get files for this subrequest
req = "SELECT FileID, LFN, Size, PFN, GUID, Md5, Addler, Attempt, Status FROM Files WHERE SubRequestID = %s ORDER BY FileID;" % str(
subRequestID)
res = self._query(req)
if not res["OK"]:
return res
files = []
for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res[
"Value"]:
fileDict = {
"FileID": fileID,
"LFN": lfn,
"Size": size,
"PFN": pfn,
"GUID": guid,
"Md5": md5,
"Addler": addler,
"Attempt": attempt,
"Status": status
}
files.append(fileDict)
res = requestContainer.setSubRequestFiles(ind, requestType, files)
if not res["OK"]:
return res
## dump request to XML
res = requestContainer.toXML()
if not res["OK"]:
return res
requestString = res["Value"]
## return dictonary with all info in at least
return S_OK({
"RequestName": requestName,
"RequestString": requestString,
"JobID": jobID,
"RequestContainer": requestContainer
})
def getRequest( self, requestType = '' ):
""" Get a request of a given type
"""
# RG: What if requestType is not given?
# the first query will return nothing.
start = time.time()
dmRequest = RequestContainer( init = False )
requestID = 0
req = "SELECT RequestID,SubRequestID FROM SubRequests WHERE Status = 'Waiting' AND RequestType = '%s' ORDER BY LastUpdate ASC LIMIT 50;" % requestType
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to retrieve max RequestID'
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
if not res['Value']:
return S_OK()
reqIDList = [ x[0] for x in res['Value'] ]
random.shuffle( reqIDList )
count = 0
for reqID in reqIDList:
count += 1
if requestType:
req = "SELECT SubRequestID,Operation,Arguments,ExecutionOrder,SourceSE,TargetSE,Catalogue,CreationTime,SubmissionTime,LastUpdate \
from SubRequests WHERE RequestID=%s AND RequestType='%s' AND Status='%s'" % ( reqID, requestType, 'Waiting' )
else:
# RG: What if requestType is not given?
# we should never get there, and it misses the "AND Status='Waiting'"
req = "SELECT SubRequestID,Operation,Arguments,ExecutionOrder,SourceSE,TargetSE,Catalogue,CreationTime,SubmissionTime,LastUpdate \
from SubRequests WHERE RequestID=%s" % reqID
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to retrieve SubRequests for RequestID %s' % reqID
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
subIDList = []
for tuple in res['Value']:
subID = tuple[0]
# RG: We should set the condition "AND Status='Waiting'"
# if the subrequest has got assigned it will failed
req = "UPDATE SubRequests SET Status='Assigned' WHERE RequestID=%s AND SubRequestID=%s;" % ( reqID, subID )
resAssigned = self._update( req )
if not resAssigned['OK']:
if subIDList:
self.__releaseSubRequests( reqID, subIDList )
return S_ERROR( 'Failed to assign subrequests: %s' % resAssigned['Message'] )
if resAssigned['Value'] == 0:
# Somebody has assigned this request
gLogger.warn( 'Already assigned subrequest %d of request %d' % ( subID, reqID ) )
else:
subIDList.append( subID )
# RG: We need to check that all subRequest with smaller ExecutionOrder are "Done"
if subIDList:
# We managed to get some requests, can continue now
requestID = reqID
break
# Haven't succeeded to get any request
if not requestID:
return S_OK()
dmRequest.setRequestID( requestID )
# RG: We have this list in subIDList, can different queries get part of the subrequets of the same type?
subRequestIDs = []
for subRequestID, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, creationTime, submissionTime, lastUpdate in res['Value']:
if not subRequestID in subIDList: continue
subRequestIDs.append( subRequestID )
# RG: res['Value'] is the range of the loop and it gets redefined here !!!!!!
res = dmRequest.initiateSubRequest( requestType )
ind = res['Value']
subRequestDict = {
'Status' : 'Waiting',
'SubRequestID' : subRequestID,
'Operation' : operation,
'Arguments' : arguments,
'ExecutionOrder': int( executionOrder ),
'SourceSE' : sourceSE,
'TargetSE' : targetSE,
'Catalogue' : catalogue,
'CreationTime' : creationTime,
'SubmissionTime': submissionTime,
'LastUpdate' : lastUpdate
}
res = dmRequest.setSubRequestAttributes( ind, requestType, subRequestDict )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set subRequest attributes for RequestID %s' % requestID
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
req = "SELECT FileID,LFN,Size,PFN,GUID,Md5,Addler,Attempt,Status \
from Files WHERE SubRequestID = %s ORDER BY FileID;" % subRequestID
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to get File attributes for RequestID %s.%s' % ( requestID, subRequestID )
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
files = []
for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res['Value']:
fileDict = {'FileID':fileID, 'LFN':lfn, 'Size':size, 'PFN':pfn, 'GUID':guid, 'Md5':md5, 'Addler':addler, 'Attempt':attempt, 'Status':status}
files.append( fileDict )
res = dmRequest.setSubRequestFiles( ind, requestType, files )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set files into Request for RequestID %s.%s' % ( requestID, subRequestID )
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
req = "SELECT Dataset,Status FROM Datasets WHERE SubRequestID = %s;" % subRequestID
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to get Datasets for RequestID %s.%s' % ( requestID, subRequestID )
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
datasets = []
for dataset, status in res['Value']:
datasets.append( dataset )
res = dmRequest.setSubRequestDatasets( ind, requestType, datasets )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set datasets into Request for RequestID %s.%s' % ( requestID, subRequestID )
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
req = "SELECT RequestName,JobID,OwnerDN,OwnerGroup,DIRACSetup,SourceComponent,CreationTime,SubmissionTime,LastUpdate from Requests WHERE RequestID = %s;" % requestID
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to retrieve max RequestID'
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, creationTime, submissionTime, lastUpdate = res['Value'][0]
dmRequest.setRequestName( requestName )
dmRequest.setJobID( jobID )
dmRequest.setOwnerDN( ownerDN )
dmRequest.setOwnerGroup( ownerGroup )
dmRequest.setDIRACSetup( diracSetup )
dmRequest.setSourceComponent( sourceComponent )
dmRequest.setCreationTime( str( creationTime ) )
dmRequest.setLastUpdate( str( lastUpdate ) )
res = dmRequest.toXML()
if not res['OK']:
err = 'RequestDB._getRequest: Failed to create XML for RequestID %s' % ( requestID )
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
requestString = res['Value']
#still have to manage the status of the dataset properly
resultDict = {}
resultDict['RequestName'] = requestName
resultDict['RequestString'] = requestString
resultDict['JobID'] = jobID
return S_OK( resultDict )
def getRequestForSubRequest(self, itself, subRequestID ):
"""
Select Request given SubRequestID.
:param self: plugin reference
:param itself: patient reference for injection
:param int subRequestID: SubRequests.SubRequestID
:warn: function has to be injected to RequestDBMySQL instance
"""
## get RequestID
requestID = "SELECT RequestID FROM SubRequests WHERE SubRequestID = %s;" % str(subRequestID)
requestID = self._query( requestID )
if not requestID["OK"]:
return requestID
requestID = requestID["Value"][0]
## create RequestContainer
requestContainer = RequestContainer( init = False )
requestContainer.setRequestID( requestID )
## put some basic infos in
requestInfo = "SELECT RequestName, JobID, OwnerDN, OwnerGroup, DIRACSetup, SourceComponent, CreationTime, SubmissionTime, LastUpdate, Status "
requestInfo += "FROM Requests WHERE RequestID = %d;" % requestID
requestInfo = self._query( requestInfo )
if not requestInfo["OK"]:
return requestInfo
requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, creationTime, submissionTime, lastUpdate, status = requestInfo['Value'][0]
requestContainer.setRequestName( requestName )
requestContainer.setJobID( jobID )
requestContainer.setOwnerDN( ownerDN )
requestContainer.setOwnerGroup( ownerGroup )
requestContainer.setDIRACSetup( diracSetup )
requestContainer.setSourceComponent( sourceComponent )
requestContainer.setCreationTime( str( creationTime ) )
requestContainer.setLastUpdate( str( lastUpdate ) )
requestContainer.setStatus( status )
## get sub-requests
subRequests = "SELECT SubRequestID, Status, RequestType, Operation, Arguments, ExecutionOrder, SourceSE, "
subRequests += "TargetSE, Catalogue, CreationTime, SubmissionTime, LastUpdate FROM SubRequests WHERE RequestID=%s;" % requestID
subRequests = self._query( subRequests )
if not subRequests["OK"]:
return subRequests
## loop over sub requests
for subRequestID, status, requestType, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, creationTime, submissionTime, lastUpdate in subRequests["Value"]:
res = requestContainer.initiateSubRequest( requestType )
ind = res["Value"]
subRequestDict = { "Status" : status, "SubRequestID" : subRequestID, "Operation" : operation, "Arguments" : arguments,
"ExecutionOrder" : int( executionOrder ), "SourceSE" : sourceSE, "TargetSE" : targetSE,
"Catalogue" : catalogue, "CreationTime" : creationTime, "SubmissionTime" : submissionTime,
"LastUpdate" : lastUpdate }
res = requestContainer.setSubRequestAttributes( ind, requestType, subRequestDict )
if not res["OK"]:
return res
## get files for this subrequest
req = "SELECT FileID, LFN, Size, PFN, GUID, Md5, Addler, Attempt, Status FROM Files WHERE SubRequestID = %s ORDER BY FileID;" % str(subRequestID)
res = self._query( req )
if not res["OK"]:
return res
files = []
for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res["Value"]:
fileDict = { "FileID" : fileID, "LFN" : lfn, "Size" : size,
"PFN" : pfn, "GUID" : guid, "Md5" : md5,
"Addler" : addler, "Attempt" : attempt,
"Status" : status }
files.append( fileDict )
res = requestContainer.setSubRequestFiles( ind, requestType, files )
if not res["OK"]:
return res
## dump request to XML
res = requestContainer.toXML()
if not res["OK"]:
return res
requestString = res["Value"]
## return dictonary with all info in at least
return S_OK( {
"RequestName" : requestName,
"RequestString" : requestString,
"JobID" : jobID,
"RequestContainer" : requestContainer
} )
