break
# # operation status check
if operation.Status == "Done" and pluginName:
gMonitor.addMark("%s%s" % (pluginName, "OK"), 1)
elif operation.Status == "Failed" and pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
elif operation.Status in ("Waiting", "Scheduled"):
# # no update for waiting or all files scheduled
break
# # not a shifter at all? delete temp proxy file
if not shifter:
os.unlink(proxyFile)
gMonitor.flush()
if error:
return S_ERROR(error)
# # request done?
if self.request.Status == "Done":
# # update request to the RequestDB
self.log.info('updating request with status %s' %
self.request.Status)
update = self.updateRequest()
if not update["OK"]:
self.log.error(update["Message"])
return update
self.log.info("request '%s' is done" % self.request.RequestName)
gMonitor.addMark("RequestOK", 1)
def __call__(self):
""" request processing """
self.log.debug("about to execute request")
gMonitor.addMark("RequestAtt", 1)
# # setup proxy for request owner
setupProxy = self.setupProxy()
if not setupProxy["OK"]:
self.request.Error = setupProxy["Message"]
if 'has no proxy registered' in setupProxy["Message"]:
self.log.error('Request set to Failed:', setupProxy["Message"])
# If user is no longer registered, fail the request
for operation in self.request:
for opFile in operation:
opFile.Status = 'Failed'
operation.Status = 'Failed'
else:
self.log.error(setupProxy["Message"])
return S_OK(self.request)
shifter = setupProxy["Value"]["Shifter"]
proxyFile = setupProxy["Value"]["ProxyFile"]
error = None
while self.request.Status == "Waiting":
# # get waiting operation
operation = self.request.getWaiting()
if not operation["OK"]:
self.log.error(operation["Message"])
return operation
operation = operation["Value"]
self.log.info("executing operation #%s '%s'" %
(operation.Order, operation.Type))
# # and handler for it
handler = self.getHandler(operation)
if not handler["OK"]:
self.log.error("unable to process operation %s: %s" %
(operation.Type, handler["Message"]))
# gMonitor.addMark( "%s%s" % ( operation.Type, "Fail" ), 1 )
operation.Error = handler["Message"]
break
handler = handler["Value"]
# # set shifters list in the handler
handler.shifter = shifter
# # and execute
pluginName = self.getPluginName(
self.handlersDict.get(operation.Type))
if self.standalone:
useServerCertificate = gConfig.useServerCertificate()
else:
# Always use server certificates if executed within an agent
useServerCertificate = True
try:
if pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Att"), 1)
# Always use request owner proxy
if useServerCertificate:
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'false')
exe = handler()
if useServerCertificate:
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'true')
if not exe["OK"]:
self.log.error("unable to process operation %s: %s" %
(operation.Type, exe["Message"]))
if pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
gMonitor.addMark("RequestFail", 1)
if self.request.JobID:
# Check if the job exists
monitorServer = RPCClient(
"WorkloadManagement/JobMonitoring",
useCertificates=True)
res = monitorServer.getJobPrimarySummary(
int(self.request.JobID))
if not res["OK"]:
self.log.error(
"RequestTask: Failed to get job %d status" %
self.request.JobID)
elif not res['Value']:
self.log.warn(
"RequestTask: job %d does not exist (anymore): failed request"
% self.request.JobID)
for opFile in operation:
opFile.Status = 'Failed'
if operation.Status != 'Failed':
operation.Status = 'Failed'
self.request.Error = 'Job no longer exists'
except Exception as error:
self.log.exception("hit by exception: %s" % str(error))
if pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
gMonitor.addMark("RequestFail", 1)
if useServerCertificate:
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'true')
break
# # operation status check
if operation.Status == "Done" and pluginName:
gMonitor.addMark("%s%s" % (pluginName, "OK"), 1)
elif operation.Status == "Failed" and pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
elif operation.Status in ("Waiting", "Scheduled"):
# # no update for waiting or all files scheduled
break
gMonitor.flush()
if error:
return S_ERROR(error)
# # request done?
if self.request.Status == "Done":
# # update request to the RequestDB
self.log.info('updating request with status %s' %
self.request.Status)
update = self.updateRequest()
if not update["OK"]:
self.log.error(update["Message"])
return update
self.log.info("request '%s' is done" % self.request.RequestName)
gMonitor.addMark("RequestOK", 1)
# # and there is a job waiting for it? finalize!
if self.request.JobID:
attempts = 0
while True:
finalizeRequest = self.requestClient.finalizeRequest(
self.request.RequestID, self.request.JobID) # pylint: disable=no-member
if not finalizeRequest["OK"]:
if not attempts:
self.log.error(
"unable to finalize request %s: %s, will retry"
% (self.request.RequestName,
finalizeRequest["Message"]))
self.log.verbose("Waiting 10 seconds")
attempts += 1
if attempts == 10:
self.log.error(
"giving up finalize request after %d attempts"
% attempts)
return S_ERROR('Could not finalize request')
time.sleep(10)
else:
self.log.info("request '%s' is finalized%s" %
(self.request.RequestName,
(' after %d attempts' %
attempts) if attempts else ''))
break
# Request will be updated by the callBack method
self.log.verbose("RequestTasks exiting, request %s" %
self.request.Status)
return S_OK(self.request)
def __call__(self):
"""request processing"""
self.log.debug("about to execute request")
if not self.rmsMonitoring:
gMonitor.addMark("RequestAtt", 1)
# # setup proxy for request owner
setupProxy = self.setupProxy()
if not setupProxy["OK"]:
userSuspended = "User is currently suspended"
self.request.Error = setupProxy["Message"]
# In case the user does not have proxy
if DErrno.cmpError(setupProxy, DErrno.EPROXYFIND):
self.log.error("Error setting proxy. Request set to Failed:",
setupProxy["Message"])
# If user is no longer registered, fail the request
for operation in self.request:
for opFile in operation:
opFile.Status = "Failed"
operation.Status = "Failed"
elif userSuspended in setupProxy["Message"]:
# If user is suspended, wait for a long time
self.request.delayNextExecution(6 * 60)
self.request.Error = userSuspended
self.log.error("Error setting proxy: " + userSuspended,
self.request.OwnerDN)
else:
self.log.error("Error setting proxy", setupProxy["Message"])
return S_OK(self.request)
shifter = setupProxy["Value"]["Shifter"]
error = None
while self.request.Status == "Waiting":
# # get waiting operation
operation = self.request.getWaiting()
if not operation["OK"]:
self.log.error("Cannot get waiting operation",
operation["Message"])
return operation
operation = operation["Value"]
self.log.info("executing operation", "%s" % operation.Type)
# # and handler for it
handler = self.getHandler(operation)
if not handler["OK"]:
self.log.error("Unable to process operation",
"%s: %s" % (operation.Type, handler["Message"]))
# gMonitor.addMark( "%s%s" % ( operation.Type, "Fail" ), 1 )
operation.Error = handler["Message"]
break
handler = handler["Value"]
# # set shifters list in the handler
handler.shifter = shifter
# set rmsMonitoring flag for the RequestOperation
handler.rmsMonitoring = self.rmsMonitoring
# # and execute
pluginName = self.getPluginName(
self.handlersDict.get(operation.Type))
if self.standalone:
useServerCertificate = gConfig.useServerCertificate()
else:
# Always use server certificates if executed within an agent
useServerCertificate = True
try:
if pluginName:
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord({
"timestamp":
int(Time.toEpoch()),
"host":
Network.getFQDN(),
"objectType":
"Operation",
"operationType":
pluginName,
"objectID":
operation.OperationID,
"parentID":
operation.RequestID,
"status":
"Attempted",
"nbObject":
1,
})
else:
gMonitor.addMark("%s%s" % (pluginName, "Att"), 1)
# Always use request owner proxy
if useServerCertificate:
gConfigurationData.setOptionInCFG(
"/DIRAC/Security/UseServerCertificate", "false")
exe = handler()
if useServerCertificate:
gConfigurationData.setOptionInCFG(
"/DIRAC/Security/UseServerCertificate", "true")
if not exe["OK"]:
self.log.error("unable to process operation",
"%s: %s" % (operation.Type, exe["Message"]))
if pluginName:
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord({
"timestamp":
int(Time.toEpoch()),
"host":
Network.getFQDN(),
"objectType":
"Operation",
"operationType":
pluginName,
"objectID":
operation.OperationID,
"parentID":
operation.RequestID,
"status":
"Failed",
"nbObject":
1,
})
else:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord({
"timestamp":
int(Time.toEpoch()),
"host":
Network.getFQDN(),
"objectType":
"Request",
"objectID":
operation.RequestID,
"status":
"Failed",
"nbObject":
1,
})
else:
gMonitor.addMark("RequestFail", 1)
if self.request.JobID:
# Check if the job exists
monitorServer = JobMonitoringClient(
useCertificates=True)
res = monitorServer.getJobSummary(
int(self.request.JobID))
if not res["OK"]:
self.log.error(
"RequestTask: Failed to get job status",
"%d" % self.request.JobID)
elif not res["Value"]:
self.log.warn(
"RequestTask: job does not exist (anymore): failed request",
"JobID: %d" % self.request.JobID,
)
for opFile in operation:
opFile.Status = "Failed"
if operation.Status != "Failed":
operation.Status = "Failed"
self.request.Error = "Job no longer exists"
except Exception as e:
error = str(e)
self.log.exception("hit by exception:", "%s" % error)
if pluginName:
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord({
"timestamp":
int(Time.toEpoch()),
"host":
Network.getFQDN(),
"objectType":
"Operation",
"operationType":
pluginName,
"objectID":
operation.OperationID,
"parentID":
operation.RequestID,
"status":
"Failed",
"nbObject":
1,
})
else:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord({
"timestamp":
int(Time.toEpoch()),
"host":
Network.getFQDN(),
"objectType":
"Request",
"objectID":
operation.RequestID,
"status":
"Failed",
"nbObject":
1,
})
else:
gMonitor.addMark("RequestFail", 1)
if useServerCertificate:
gConfigurationData.setOptionInCFG(
"/DIRAC/Security/UseServerCertificate", "true")
break
# # operation status check
if operation.Status == "Done" and pluginName:
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord({
"timestamp":
int(Time.toEpoch()),
"host":
Network.getFQDN(),
"objectType":
"Operation",
"operationType":
pluginName,
"objectID":
operation.OperationID,
"parentID":
operation.RequestID,
"status":
"Successful",
"nbObject":
1,
})
else:
gMonitor.addMark("%s%s" % (pluginName, "OK"), 1)
elif operation.Status == "Failed" and pluginName:
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord({
"timestamp":
int(Time.toEpoch()),
"host":
Network.getFQDN(),
"objectType":
"Operation",
"operationType":
pluginName,
"objectID":
operation.OperationID,
"parentID":
operation.RequestID,
"status":
"Failed",
"nbObject":
1,
})
else:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
elif operation.Status in ("Waiting", "Scheduled"):
# # no update for waiting or all files scheduled
break
if not self.rmsMonitoring:
gMonitor.flush()
if error:
return S_ERROR(error)
# # request done?
if self.request.Status == "Done":
# # update request to the RequestDB
self.log.info("Updating request status:",
"%s" % self.request.Status)
update = self.updateRequest()
if not update["OK"]:
self.log.error("Cannot update request status",
update["Message"])
return update
self.log.info("request is done", "%s" % self.request.RequestName)
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord({
"timestamp":
int(Time.toEpoch()),
"host":
Network.getFQDN(),
"objectType":
"Request",
"objectID":
getattr(self.request, "RequestID", 0),
"status":
"Successful",
"nbObject":
1,
})
else:
gMonitor.addMark("RequestOK", 1)
# # and there is a job waiting for it? finalize!
if self.request.JobID:
attempts = 0
while True:
finalizeRequest = self.requestClient.finalizeRequest(
self.request.RequestID,
self.request.JobID # pylint: disable=no-member
)
if not finalizeRequest["OK"]:
if not attempts:
self.log.error(
"unable to finalize request, will retry",
"ReqName %s:%s" % (self.request.RequestName,
finalizeRequest["Message"]),
)
self.log.debug("Waiting 10 seconds")
attempts += 1
if attempts == 10:
self.log.error("Giving up finalize request")
return S_ERROR("Could not finalize request")
time.sleep(10)
else:
self.log.info(
"request is finalized",
"ReqName %s %s" % (self.request.RequestName,
(" after %d attempts" %
attempts) if attempts else ""),
)
break
# Commit all the data to the ES Backend
if self.rmsMonitoring:
self.rmsMonitoringReporter.commit()
# Request will be updated by the callBack method
self.log.verbose("RequestTasks exiting",
"request %s" % self.request.Status)
return S_OK(self.request)
def __call__(self):
""" request processing """
self.log.debug("about to execute request")
gMonitor.addMark("RequestAtt", 1)
# # setup proxy for request owner
setupProxy = self.setupProxy()
if not setupProxy["OK"]:
self.request.Error = setupProxy["Message"]
if 'has no proxy registered' in setupProxy["Message"]:
self.log.error('Request set to Failed:', setupProxy["Message"])
# If user is no longer registered, fail the request
for operation in self.request:
for opFile in operation:
opFile.Status = 'Failed'
operation.Status = 'Failed'
else:
self.log.error(setupProxy["Message"])
return S_OK(self.request)
shifter = setupProxy["Value"]["Shifter"]
proxyFile = setupProxy["Value"]["ProxyFile"]
error = None
while self.request.Status == "Waiting":
# # get waiting operation
operation = self.request.getWaiting()
if not operation["OK"]:
self.log.error(operation["Message"])
return operation
operation = operation["Value"]
self.log.info("executing operation #%s '%s'" % (operation.Order, operation.Type))
# # and handler for it
handler = self.getHandler(operation)
if not handler["OK"]:
self.log.error("unable to process operation %s: %s" % (operation.Type, handler["Message"]))
# gMonitor.addMark( "%s%s" % ( operation.Type, "Fail" ), 1 )
operation.Error = handler["Message"]
break
handler = handler["Value"]
# # set shifters list in the handler
handler.shifter = shifter
# # and execute
pluginName = self.getPluginName(self.handlersDict.get(operation.Type))
if self.standalone:
useServerCertificate = gConfig.useServerCertificate()
else:
# Always use server certificates if executed within an agent
useServerCertificate = True
try:
if pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Att"), 1)
# Always use request owner proxy
if useServerCertificate:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'false')
exe = handler()
if useServerCertificate:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true')
if not exe["OK"]:
self.log.error("unable to process operation %s: %s" % (operation.Type, exe["Message"]))
if pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
gMonitor.addMark("RequestFail", 1)
if self.request.JobID:
# Check if the job exists
monitorServer = RPCClient("WorkloadManagement/JobMonitoring", useCertificates=True)
res = monitorServer.getJobPrimarySummary(int(self.request.JobID))
if not res["OK"]:
self.log.error("RequestTask: Failed to get job %d status" % self.request.JobID)
elif not res['Value']:
self.log.warn(
"RequestTask: job %d does not exist (anymore): failed request" %
self.request.JobID)
for opFile in operation:
opFile.Status = 'Failed'
if operation.Status != 'Failed':
operation.Status = 'Failed'
self.request.Error = 'Job no longer exists'
except Exception as error:
self.log.exception("hit by exception: %s" % str(error))
if pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
gMonitor.addMark("RequestFail", 1)
if useServerCertificate:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true')
break
# # operation status check
if operation.Status == "Done" and pluginName:
gMonitor.addMark("%s%s" % (pluginName, "OK"), 1)
elif operation.Status == "Failed" and pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
elif operation.Status in ("Waiting", "Scheduled"):
# # no update for waiting or all files scheduled
break
gMonitor.flush()
if error:
return S_ERROR(error)
# # request done?
if self.request.Status == "Done":
# # update request to the RequestDB
self.log.info('updating request with status %s' % self.request.Status)
update = self.updateRequest()
if not update["OK"]:
self.log.error(update["Message"])
return update
self.log.info("request '%s' is done" % self.request.RequestName)
gMonitor.addMark("RequestOK", 1)
# # and there is a job waiting for it? finalize!
if self.request.JobID:
attempts = 0
while True:
finalizeRequest = self.requestClient.finalizeRequest(
self.request.RequestID, self.request.JobID) # pylint: disable=no-member
if not finalizeRequest["OK"]:
if not attempts:
self.log.error(
"unable to finalize request %s: %s, will retry" %
(self.request.RequestName, finalizeRequest["Message"]))
self.log.verbose("Waiting 10 seconds")
attempts += 1
if attempts == 10:
self.log.error("giving up finalize request after %d attempts" % attempts)
return S_ERROR('Could not finalize request')
time.sleep(10)
else:
self.log.info(
"request '%s' is finalized%s" %
(self.request.RequestName,
(' after %d attempts' %
attempts) if attempts else ''))
break
# Request will be updated by the callBack method
self.log.verbose("RequestTasks exiting, request %s" % self.request.Status)
return S_OK(self.request)
break
# # operation status check
if operation.Status == "Done" and pluginName:
gMonitor.addMark( "%s%s" % ( pluginName, "OK" ), 1 )
elif operation.Status == "Failed" and pluginName:
gMonitor.addMark( "%s%s" % ( pluginName, "Fail" ), 1 )
elif operation.Status in ( "Waiting", "Scheduled" ):
# # no update for waiting or all files scheduled
break
# # not a shifter at all? delete temp proxy file
if not shifter:
os.unlink( proxyFile )
gMonitor.flush()
if error:
return S_ERROR( error )
# # request done?
if self.request.Status == "Done":
# # update request to the RequestDB
self.log.info( 'updating request with status %s' % self.request.Status )
update = self.updateRequest()
if not update["OK"]:
self.log.error( update["Message"] )
return update
self.log.info( "request '%s' is done" % self.request.RequestName )
gMonitor.addMark( "RequestOK", 1 )
# # and there is a job waiting for it? finalize!