def __call__(self):
"""execute RPC stub"""
# # decode arguments
try:
decode, length = DEncode.decode(self.operation.Arguments)
self.log.debug("decoded len=%s val=%s" % (length, decode))
except ValueError as error:
self.log.exception(error)
self.operation.Error = str(error)
self.operation.Status = "Failed"
return S_ERROR(str(error))
# Ensure the forwarded request is done on behalf of the request owner
decode[0][1]["delegatedDN"] = self.request.OwnerDN
decode[0][1]["delegatedGroup"] = self.request.OwnerGroup
# ForwardDiset is supposed to be used with a host certificate
useServerCertificate = gConfig.useServerCertificate()
gConfigurationData.setOptionInCFG(
"/DIRAC/Security/UseServerCertificate", "true")
forward = executeRPCStub(decode)
if not useServerCertificate:
gConfigurationData.setOptionInCFG(
"/DIRAC/Security/UseServerCertificate", "false")
if not forward["OK"]:
self.log.error(
"unable to execute operation",
"'%s' : %s" % (self.operation.Type, forward["Message"]))
self.operation.Error = forward["Message"]
return forward
self.log.info("DISET forwarding done")
self.operation.Status = "Done"
return S_OK()
def test_rpcStubs_are_equals(UseServerCertificate):
"""
Test if Clients returns the same rpcStubs
Navigating through array is a bit complicated in this test...
repDirac and repTornado may have the same structure:
repDirac dict{
OK: True
rpcStub: tuple{
ServiceName: str
kwargs: dict{
*** kwargs used to instanciate client ***
}
methodName: str
arguments: list
}
Value: dict { # NOT USED IN THIS TEST
*** Credentials dictionnary extracted by server ***
}
}
"""
gConfigurationData.setOptionInCFG("/DIRAC/Security/UseServerCertificate", UseServerCertificate)
(repTornado, repDirac) = get_all_returnedValues()
# Explicitly removed in Tornado
del repDirac["rpcStub"][0][1]["keepAliveLapse"]
# rep['rpcStub'] is at form (rpcStub, method, args) where rpcStub is tuple with (serviceName, kwargs)
assert repTornado["rpcStub"][0][0] != repDirac["rpcStub"][0][0] # Services name are different
assert repTornado["rpcStub"][0][1] == repDirac["rpcStub"][0][1] # Check kwargs returned by rpcStub
assert repTornado["rpcStub"][1:] != repDirac["rpcStub"][1:] # Check method/args
def cleanMetadataCatalogFiles(self, transID):
""" wipe out files from catalog """
res = self.metadataClient.findFilesByMetadata(
{self.transfidmeta: transID})
if not res['OK']:
return res
fileToRemove = res['Value']
if not fileToRemove:
self.log.info('No files found for transID %s' % transID)
return S_OK()
# Executing with shifter proxy
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'false')
res = DataManager().removeFile(fileToRemove, force=True)
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'true')
if not res['OK']:
return res
for lfn, reason in res['Value']['Failed'].items():
self.log.error("Failed to remove file found in metadata catalog",
"%s %s" % (lfn, reason))
if res['Value']['Failed']:
return S_ERROR(
"Failed to remove all files found in the metadata catalog")
self.log.info("Successfully removed all files found in the BK")
return S_OK()
def loadUserData( self ):
"""
This is the magic method that reads the command line and processes it
It is used by the Script Base class and the dirac-service and dirac-agent scripts
Before being called:
- any additional switches to be processed
- mandatory and default configuration configuration options must be defined.
"""
if self.initialized:
return S_OK()
self.initialized = True
try:
retVal = self.__addUserDataToConfiguration()
for optionTuple in self.optionalEntryList:
optionPath = self.__getAbsolutePath( optionTuple[0] )
if not gConfigurationData.extractOptionFromCFG( optionPath ):
gConfigurationData.setOptionInCFG( optionPath, optionTuple[1] )
self.__initLogger( self.componentName, self.loggingSection )
if not retVal[ 'OK' ]:
return retVal
retVal = self.__checkMandatoryOptions()
if not retVal[ 'OK' ]:
return retVal
except Exception as e:
gLogger.exception()
return S_ERROR( str( e ) )
return S_OK()
def loadUserData(self):
"""
This is the magic method that reads the command line and processes it
It is used by the Script Base class and the dirac-service and dirac-agent scripts
Before being called:
- any additional switches to be processed
- mandatory and default configuration configuration options must be defined.
"""
if self.initialized:
return S_OK()
self.initialized = True
try:
retVal = self.__addUserDataToConfiguration()
for optionTuple in self.optionalEntryList:
optionPath = self.__getAbsolutePath(optionTuple[0])
if not gConfigurationData.extractOptionFromCFG(optionPath):
gConfigurationData.setOptionInCFG(optionPath,
optionTuple[1])
self.__initLogger(self.componentName, self.loggingSection)
if not retVal['OK']:
return retVal
retVal = self.__checkMandatoryOptions()
if not retVal['OK']:
return retVal
except Exception as e:
gLogger.exception()
return S_ERROR(str(e))
return S_OK()
def setOptionValue(self, optionPath, value):
""" Set a value in the local configuration
:param str optionPath: option path
:param str value: value
"""
gConfigurationData.setOptionInCFG(optionPath, value)
def wrapped_fcn(*args, **kwargs):
# Get the lock and acquire it
executionLock = LockRing().getLock("_UseUserProxy_", recursive=True)
executionLock.acquire()
# Check if the caller is executing with the host certificate
useServerCertificate = gConfig.useServerCertificate()
if useServerCertificate:
gConfigurationData.setOptionInCFG(
"/DIRAC/Security/UseServerCertificate", "false")
try:
return fcn(*args, **kwargs)
except Exception as lException: # pylint: disable=broad-except
value = ",".join([str(arg) for arg in lException.args])
exceptType = lException.__class__.__name__
return S_ERROR("Exception - %s: %s" % (exceptType, value))
finally:
# Restore the default host certificate usage if necessary
if useServerCertificate:
gConfigurationData.setOptionInCFG(
"/DIRAC/Security/UseServerCertificate", "true")
# release the lock
executionLock.release()
def cleanContent(self, directory):
""" wipe out everything from catalog under folder :directory:
:param self: self reference
:params str directory: folder name
"""
self.log.verbose("Cleaning Catalog contents")
res = self.__getCatalogDirectoryContents([directory])
if not res['OK']:
return res
filesFound = res['Value']
if not filesFound:
self.log.info("No files are registered in the catalog directory %s" % directory)
return S_OK()
self.log.info("Attempting to remove %d possible remnants from the catalog and storage" % len(filesFound))
# Executing with shifter proxy
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'false')
res = DataManager().removeFile(filesFound, force=True)
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true')
if not res['OK']:
return res
realFailure = False
for lfn, reason in res['Value']['Failed'].items():
if "File does not exist" in str(reason):
self.log.warn("File %s not found in some catalog: " % (lfn))
else:
self.log.error("Failed to remove file found in the catalog", "%s %s" % (lfn, reason))
realFailure = True
if realFailure:
return S_ERROR("Failed to remove all files found in the catalog")
return S_OK()
def cleanCatalogContents( self, directory ):
""" wipe out everything from catalog under folder :directory:
:param self: self reference
:params str directory: folder name
"""
self.log.verbose("Cleaning Catalog contents")
res = self.__getCatalogDirectoryContents( [directory] )
if not res['OK']:
return res
filesFound = res['Value']
if not filesFound:
self.log.info( "No files are registered in the catalog directory %s" % directory )
return S_OK()
self.log.info( "Attempting to remove %d possible remnants from the catalog and storage" % len( filesFound ) )
# Executing with shifter proxy
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
res = DataManager().removeFile( filesFound, force = True )
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
if not res['OK']:
return res
realFailure = False
for lfn, reason in res['Value']['Failed'].items():
if "File does not exist" in str( reason ):
self.log.warn( "File %s not found in some catalog: " % ( lfn ) )
else:
self.log.error( "Failed to remove file found in the catalog", "%s %s" % ( lfn, reason ) )
realFailure = True
if realFailure:
return S_ERROR( "Failed to remove all files found in the catalog" )
return S_OK()
def wrapped_fcn( *args, **kwargs ):
userName = kwargs.pop( 'proxyUserName', '' )
userDN = kwargs.pop( 'proxyUserDN', '' )
userGroup = kwargs.pop( 'proxyUserGroup', '' )
vomsFlag = kwargs.pop( 'proxyWithVOMS', True )
proxyFilePath = kwargs.pop( 'proxyFilePath', False )
if ( userName or userDN ) and userGroup:
# Setup user proxy
originalUserProxy = os.environ.get( 'X509_USER_PROXY' )
if not userDN:
result = getDNForUsername( userName )
if not result[ 'OK' ]:
return result
userDN = result[ 'Value' ][0]
vomsAttr = ''
if vomsFlag:
vomsAttr = getVOMSAttributeForGroup( userGroup )
if vomsAttr:
result = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup,
requiredVOMSAttribute = vomsAttr,
filePath = proxyFilePath,
requiredTimeLeft = 3600,
cacheTime = 3600 )
else:
result = gProxyManager.downloadProxyToFile( userDN, userGroup,
filePath = proxyFilePath,
requiredTimeLeft = 3600,
cacheTime = 3600 )
if not result['OK']:
gLogger.warn( "Can't download proxy to file", result['Message'] )
return result
proxyFile = result['Value']
os.environ['X509_USER_PROXY'] = proxyFile
# Check if the caller is executing with the host certificate
useServerCertificate = gConfig.useServerCertificate()
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
try:
resultFcn = fcn( *args, **kwargs )
except Exception, x:
resultFcn = S_ERROR( "Exception: %s" % str( x ) )
# Restore the default host certificate usage if necessary
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
if originalUserProxy:
os.environ['X509_USER_PROXY'] = originalUserProxy
else:
os.environ.pop( 'X509_USER_PROXY' )
return resultFcn
def wrapped_fcn(*args, **kwargs):
userName = kwargs.pop("proxyUserName", "")
userGroup = kwargs.pop("proxyUserGroup", "")
vomsFlag = kwargs.pop("proxyWithVOMS", True)
proxyFilePath = kwargs.pop("proxyFilePath", False)
if userName and userGroup:
# Setup user proxy
originalUserProxy = os.environ.get("X509_USER_PROXY")
result = getDNForUsername(userName)
if not result["OK"]:
return result
userDN = result["Value"][0]
vomsAttr = ""
if vomsFlag:
vomsAttr = getVOMSAttributeForGroup(userGroup)
if vomsAttr:
result = gProxyManager.downloadVOMSProxyToFile(
userDN,
userGroup,
requiredVOMSAttribute=vomsAttr,
filePath=proxyFilePath,
requiredTimeLeft=3600,
cacheTime=3600,
)
else:
result = gProxyManager.downloadProxyToFile(
userDN, userGroup, filePath=proxyFilePath, requiredTimeLeft=3600, cacheTime=3600
)
if not result["OK"]:
return result
proxyFile = result["Value"]
os.environ["X509_USER_PROXY"] = proxyFile
# Check if the caller is executing with the host certificate
useServerCertificate = gConfig.useServerCertificate()
if useServerCertificate:
gConfigurationData.setOptionInCFG("/DIRAC/Security/UseServerCertificate", "false")
try:
resultFcn = fcn(*args, **kwargs)
except Exception, x:
resultFcn = S_ERROR("Exception: %s" % str(x))
# Restore the default host certificate usage if necessary
if useServerCertificate:
gConfigurationData.setOptionInCFG("/DIRAC/Security/UseServerCertificate", "true")
if originalUserProxy:
os.environ["X509_USER_PROXY"] = originalUserProxy
else:
os.environ.pop("X509_USER_PROXY")
return resultFcn
def wrapped_fcn(*args, **kwargs):
userName = kwargs.pop('proxyUserName', '')
userDN = kwargs.pop('proxyUserDN', '')
userGroup = kwargs.pop('proxyUserGroup', '')
vomsFlag = kwargs.pop('proxyWithVOMS', True)
proxyFilePath = kwargs.pop('proxyFilePath', False)
if (userName or userDN) and userGroup:
# Setup user proxy
originalUserProxy = os.environ.get('X509_USER_PROXY')
if userDN:
userDNs = [userDN]
else:
result = getDNForUsername(userName)
if not result['OK']:
return result
userDNs = result[
'Value'] # a same user may have more than one DN
vomsAttr = ''
if vomsFlag:
vomsAttr = getVOMSAttributeForGroup(userGroup)
result = getProxy(userDNs, userGroup, vomsAttr, proxyFilePath)
if not result['OK']:
return result
proxyFile = result['Value']
os.environ['X509_USER_PROXY'] = proxyFile
# Check if the caller is executing with the host certificate
useServerCertificate = gConfig.useServerCertificate()
if useServerCertificate:
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'false')
try:
return fcn(*args, **kwargs)
except Exception as lException: #pylint: disable=broad-except
value = ','.join([str(arg) for arg in lException.args])
exceptType = lException.__class__.__name__
return S_ERROR("Exception - %s: %s" % (exceptType, value))
finally:
# Restore the default host certificate usage if necessary
if useServerCertificate:
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'true')
if originalUserProxy:
os.environ['X509_USER_PROXY'] = originalUserProxy
else:
os.environ.pop('X509_USER_PROXY')
else:
# No proxy substitution requested
return fcn(*args, **kwargs)
def wrapped_fcn( *args, **kwargs ):
userName = kwargs.pop( 'proxyUserName', '' )
userGroup = kwargs.pop( 'proxyUserGroup', '' )
vomsFlag = kwargs.pop( 'proxyWithVOMS', True )
proxyFilePath = kwargs.pop( 'proxyFilePath', False )
if userName and userGroup:
# Setup user proxy
originalUserProxy = os.environ.get( 'X509_USER_PROXY' )
result = getDNForUsername( userName )
if not result[ 'OK' ]:
return result
userDN = result[ 'Value' ][0]
vomsAttr = ''
if vomsFlag:
vomsAttr = getVOMSAttributeForGroup( userGroup )
if vomsAttr:
result = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup,
requiredVOMSAttribute = vomsAttr,
filePath = proxyFilePath,
requiredTimeLeft = 3600,
cacheTime = 3600 )
else:
result = gProxyManager.downloadProxyToFile( userDN, userGroup,
filePath = proxyFilePath,
requiredTimeLeft = 3600,
cacheTime = 3600 )
if not result['OK']:
return result
proxyFile = result['Value']
os.environ['X509_USER_PROXY'] = proxyFile
# Check if the caller is executing with the host certificate
useServerCertificate = gConfig.useServerCertificate()
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
try:
resultFcn = fcn( *args, **kwargs )
except Exception, x:
resultFcn = S_ERROR( "Exception: %s" % str( x ) )
# Restore the default host certificate usage if necessary
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
if originalUserProxy:
os.environ['X509_USER_PROXY'] = originalUserProxy
else:
os.environ.pop( 'X509_USER_PROXY' )
return resultFcn
def execute( self ):
""" one cycle execution """
# Don't use the server certificate otherwise the DFC wont let us write
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
log = gLogger.getSubLogger( "execute" )
# # reset FTSPlacement if expired
now = datetime.datetime.now()
if now > self.__ftsPlacementValidStamp:
log.info( "resetting expired FTS placement..." )
resetFTSPlacement = self.resetFTSPlacement()
if not resetFTSPlacement["OK"]:
log.error( "FTSPlacement recreation error:" , resetFTSPlacement["Message"] )
return resetFTSPlacement
self.__ftsPlacementValidStamp = now + datetime.timedelta( seconds = self.FTSPLACEMENT_REFRESH )
requestIDs = self.requestClient().getRequestIDsList( statusList = [ "Scheduled" ], limit = self.MAX_REQUESTS )
if not requestIDs["OK"]:
log.error( "unable to read scheduled request ids" , requestIDs["Message"] )
return requestIDs
if not requestIDs["Value"]:
requestIDs = []
else:
requestIDs = [ req[0] for req in requestIDs["Value"] if req[0] not in self.__reqCache ]
requestIDs += self.__reqCache.keys()
if not requestIDs:
log.info( "no 'Scheduled' requests to process" )
return S_OK()
log.info( "found %s requests to process:" % len( requestIDs ) )
log.info( " => from internal cache: %s" % ( len( self.__reqCache ) ) )
log.info( " => new read from RMS: %s" % ( len( requestIDs ) - len( self.__reqCache ) ) )
for requestID in requestIDs:
request = self.getRequest( requestID )
if not request["OK"]:
log.error( "Error getting request", "%s: %s" % ( requestID, request["Message"] ) )
continue
request = request["Value"]
sTJId = request.RequestID
while True:
queue = self.threadPool().generateJobAndQueueIt( self.processRequest,
args = ( request, ),
sTJId = sTJId )
if queue["OK"]:
log.info( "Request enqueued for execution", sTJId )
gMonitor.addMark( "RequestsAtt", 1 )
break
time.sleep( 1 )
# # process all results
self.threadPool().processAllResults()
return S_OK()
def execute( self ):
""" one cycle execution """
# Don't use the server certificate otherwise the DFC wont let us write
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
log = gLogger.getSubLogger( "execute" )
# # reset FTSPlacement if expired
now = datetime.datetime.now()
if now > self.__ftsPlacementValidStamp:
log.info( "resetting expired FTS placement..." )
resetFTSPlacement = self.resetFTSPlacement()
if not resetFTSPlacement["OK"]:
log.error( "FTSPlacement recreation error:" , resetFTSPlacement["Message"] )
return resetFTSPlacement
self.__ftsPlacementValidStamp = now + datetime.timedelta( seconds = self.FTSPLACEMENT_REFRESH )
requestIDs = self.requestClient().getRequestIDsList( statusList = [ "Scheduled" ], limit = self.MAX_REQUESTS )
if not requestIDs["OK"]:
log.error( "unable to read scheduled request ids" , requestIDs["Message"] )
return requestIDs
if not requestIDs["Value"]:
requestIDs = []
else:
requestIDs = [ req[0] for req in requestIDs["Value"] if req[0] not in self.__reqCache ]
requestIDs += self.__reqCache.keys()
if not requestIDs:
log.info( "no 'Scheduled' requests to process" )
return S_OK()
log.info( "found %s requests to process:" % len( requestIDs ) )
log.info( " => from internal cache: %s" % ( len( self.__reqCache ) ) )
log.info( " => new read from RMS: %s" % ( len( requestIDs ) - len( self.__reqCache ) ) )
for requestID in requestIDs:
request = self.getRequest( requestID )
if not request["OK"]:
log.error( "Error getting request", "%s: %s" % ( requestID, request["Message"] ) )
continue
request = request["Value"]
sTJId = request.RequestID
while True:
queue = self.threadPool().generateJobAndQueueIt( self.processRequest,
args = ( request, ),
sTJId = sTJId )
if queue["OK"]:
log.info( "Request enqueued for execution", sTJId )
gMonitor.addMark( "RequestsAtt", 1 )
break
time.sleep( 1 )
# # process all results
self.threadPool().processAllResults()
return S_OK()
def test_return_credential_are_equals(UseServerCertificate):
"""
Check if certificates sended AND extraction have same comportement is DISET and HTTPS
"""
gConfigurationData.setOptionInCFG("/DIRAC/Security/UseServerCertificate", UseServerCertificate)
(repTornado, repDirac) = get_all_returnedValues()
# Service returns credentials
assert repDirac["Value"] == repTornado["Value"]
def wrapped_fcn( *args, **kwargs ):
userName = kwargs.pop( 'proxyUserName', '' )
userDN = kwargs.pop( 'proxyUserDN', '' )
userGroup = kwargs.pop( 'proxyUserGroup', '' )
vomsFlag = kwargs.pop( 'proxyWithVOMS', True )
proxyFilePath = kwargs.pop( 'proxyFilePath', False )
if ( userName or userDN ) and userGroup:
# Setup user proxy
originalUserProxy = os.environ.get( 'X509_USER_PROXY' )
if userDN:
userDNs = [userDN]
else:
result = getDNForUsername( userName )
if not result[ 'OK' ]:
return result
userDNs = result['Value'] # a same user may have more than one DN
vomsAttr = ''
if vomsFlag:
vomsAttr = getVOMSAttributeForGroup( userGroup )
result = getProxy(userDNs, userGroup, vomsAttr, proxyFilePath)
if not result['OK']:
return result
proxyFile = result['Value']
os.environ['X509_USER_PROXY'] = proxyFile
# Check if the caller is executing with the host certificate
useServerCertificate = gConfig.useServerCertificate()
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
try:
return fcn( *args, **kwargs )
except Exception as lException:
value = ','.join( [str( arg ) for arg in lException.args] )
exceptType = lException.__class__.__name__
return S_ERROR( "Exception - %s: %s" % ( exceptType, value ) )
finally:
# Restore the default host certificate usage if necessary
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
if originalUserProxy:
os.environ['X509_USER_PROXY'] = originalUserProxy
else:
os.environ.pop( 'X509_USER_PROXY' )
else:
# No proxy substitution requested
return fcn( *args, **kwargs )
def main():
with open(os.devnull, 'w') as redirectStdout, open(os.devnull, 'w') as redirectStderr:
from DIRAC import gLogger
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true')
gLogger.setLevel('FATAL')
from DIRAC.Core.DISET.RPCClient import RPCClient
rpc = RPCClient('dips://localhost:%s' % sys.argv[1])
res = rpc.ping()
time.sleep(0.1)
sys.exit(0 if res['OK'] else 1)
def cleanMetadataCatalogFiles(self, transID):
""" clean the metadata using BKK and Data Manager. This method is a replacement of the one from base class
:param self: self reference
:param int transID: transformation ID
"""
res = self.bkClient.getProductionFiles(transID, 'ALL', 'Yes')
if not res['OK']:
return res
bkMetadata = res['Value']
fileToRemove = []
yesReplica = []
self.log.info(
"Found a total of %d files in the BK for transformation %d" %
(len(bkMetadata), transID))
for lfn, metadata in bkMetadata.iteritems():
if metadata['FileType'] != 'LOG':
fileToRemove.append(lfn)
if metadata['GotReplica'] == 'Yes':
yesReplica.append(lfn)
if fileToRemove:
self.log.info(
"Attempting to remove %d possible remnants from the catalog and storage"
% len(fileToRemove))
# Executing with shifter proxy
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'false')
res = DataManager().removeFile(fileToRemove, force=True)
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'true')
if not res['OK']:
return res
for lfn, reason in res['Value']['Failed'].iteritems():
self.log.error("Failed to remove file found in BK",
"%s %s" % (lfn, reason))
if res['Value']['Failed']:
return S_ERROR("Failed to remove all files found in the BK")
if yesReplica:
self.log.info(
"Ensuring that %d files are removed from the BK" %
(len(yesReplica)))
res = FileCatalog(
catalogs=['BookkeepingDB']).removeFile(yesReplica)
if not res['OK']:
return res
for lfn, reason in res['Value']['Failed'].iteritems():
self.log.error("Failed to remove file from BK",
"%s %s" % (lfn, reason))
if res['Value']['Failed']:
return S_ERROR("Failed to remove all files from the BK")
self.log.info("Successfully removed all files found in the BK")
return S_OK()
def _restoreProxyState(originalUserProxy=None, useServerCertificate=None, executionLock=None):
"""Restore the default host certificate and proxy state if necessary.
Parameters like the output from `_putProxy`.
"""
if useServerCertificate:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true')
if originalUserProxy:
os.environ['X509_USER_PROXY'] = originalUserProxy
else:
os.environ.pop('X509_USER_PROXY', None)
if executionLock:
executionLock.release()
def _restoreProxyState(originalUserProxy=None, useServerCertificate=None, executionLock=None):
"""Restore the default host certificate and proxy state if necessary.
Parameters like the output from `_putProxy`.
"""
if useServerCertificate:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true')
if originalUserProxy:
os.environ['X509_USER_PROXY'] = originalUserProxy
else:
os.environ.pop('X509_USER_PROXY', None)
if executionLock:
executionLock.release()
def main():
with open(os.devnull, "w") as redirectStdout, open(os.devnull, "w") as redirectStderr:
from DIRAC import gLogger
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
gConfigurationData.setOptionInCFG("/DIRAC/Security/UseServerCertificate", "true")
gLogger.setLevel("FATAL")
from DIRAC.Core.Base.Client import Client
rpc = Client(url="dips://localhost:%s" % sys.argv[1])
res = rpc.ping()
time.sleep(0.1)
sys.exit(0 if res["OK"] else 1)
def cleanOutputs(self, jobInfo):
"""remove all job outputs"""
if len(jobInfo.outputFiles) == 0:
return
descendants = self.__findAllDescendants(jobInfo.outputFiles)
existingOutputFiles = [
lfn for lfn, status in izip_longest(jobInfo.outputFiles,
jobInfo.outputFileStatus)
if status == "Exists"
]
filesToDelete = existingOutputFiles + descendants
if not filesToDelete:
return
if not self.enabled:
self.log.notice("Would have removed these files: \n +++ %s " %
"\n +++ ".join(filesToDelete))
return
self.log.notice("Remove these files: \n +++ %s " %
"\n +++ ".join(filesToDelete))
errorReasons = {}
successfullyRemoved = 0
for lfnList in breakListIntoChunks(filesToDelete, 200):
## this is needed to remove the file with the Shifter credentials and not with the server credentials
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'false')
result = DataManager().removeFile(lfnList)
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'true')
if not result['OK']:
self.log.error("Failed to remove LFNs", result['Message'])
raise RuntimeError("Failed to remove LFNs: %s" %
result['Message'])
for lfn, err in result['Value']['Failed'].items():
reason = str(err)
if reason not in errorReasons.keys():
errorReasons[reason] = []
errorReasons[reason].append(lfn)
successfullyRemoved += len(result['Value']['Successful'].keys())
for reason, lfns in errorReasons.items():
self.log.error("Failed to remove %d files with error: %s" %
(len(lfns), reason))
self.log.notice("Successfully removed %d files" % successfullyRemoved)
class ForwardDISET(OperationHandlerBase):
"""
.. class:: ForwardDISET
functor forwarding DISET operations
"""
def __init__(self, operation=None, csPath=None):
""" c'tor
:param Operation operation: an Operation instance
:param str csPath: CS path for this handler
"""
# # call base class c'tor
OperationHandlerBase.__init__(self, operation, csPath)
def __call__(self):
""" execute RPC stub """
# # decode arguments
try:
decode, length = DEncode.decode(self.operation.Arguments)
self.log.debug("decoded len=%s val=%s" % (length, decode))
except ValueError, error:
self.log.exception(error)
self.operation.Error = str(error)
self.operation.Status = "Failed"
return S_ERROR(str(error))
# ForwardDiset is supposed to be used with a host certificate
useServerCertificate = gConfig.useServerCertificate()
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'true')
forward = executeRPCStub(decode)
if not useServerCertificate:
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'false')
if not forward["OK"]:
self.log.error(
"unable to execute operation",
"'%s' : %s" % (self.operation.Type, forward["Message"]))
self.operation.Error = forward["Message"]
return forward
self.log.info("DISET forwarding done")
self.operation.Status = "Done"
return S_OK()
def __removeEmptyDir(self, dirPath, useOwnerProxy=True):
''' unlink empty folder :dirPath: '''
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
if len(List.fromChar(dirPath, "/")) < self.__keepDirLevels:
return S_OK()
if useOwnerProxy:
result = self.__getOwnerProxy(dirPath)
if not result['OK']:
if 'Proxy not available' not in result['Message']:
self.log.error(result['Message'])
return result
upFile = result['Value']
prevProxyEnv = os.environ['X509_USER_PROXY']
os.environ['X509_USER_PROXY'] = upFile
try:
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'false')
# res = self.catalog.removeDirectory( dirPath )
res = self.catalog.writeCatalogs[0][1].removeDirectory(dirPath)
if not res['OK']:
self.log.error(
"Error removing empty directory from File Catalog.",
res['Message'])
return res
elif dirPath in res['Value']['Failed']:
self.log.error(
"Failed to remove empty directory from File Catalog.",
res['Value']['Failed'][dirPath])
self.log.debug(str(res))
return S_ERROR(res['Value']['Failed'][dirPath])
res = self.storageUsage.removeDirectory(dirPath)
if not res['OK']:
self.log.error(
"Failed to remove empty directory from Storage Usage database.",
res['Message'])
return res
return S_OK()
finally:
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'true')
if useOwnerProxy:
os.environ['X509_USER_PROXY'] = prevProxyEnv
def _putProxy(userDN=None,
userName=None,
userGroup=None,
vomsFlag=None,
proxyFilePath=None,
executionLockFlag=False):
"""Download proxy, place in a file and populate X509_USER_PROXY environment variable.
Parameters like `userProxy` or `executeWithUserProxy`.
:returns: Tuple of originalUserProxy, useServerCertificate, executionLock
"""
# Setup user proxy
if userDN:
userDNs = [userDN]
else:
result = getDNForUsername(userName)
if not result["OK"]:
return result
userDNs = result["Value"] # a same user may have more than one DN
vomsAttr = ""
if vomsFlag:
vomsAttr = getVOMSAttributeForGroup(userGroup)
result = getProxy(userDNs, userGroup, vomsAttr, proxyFilePath)
if not result["OK"]:
return result
executionLock = LockRing().getLock(
"_UseUserProxy_", recursive=True) if executionLockFlag else None
if executionLockFlag:
executionLock.acquire()
os.environ["X509_USER_PROXY"], originalUserProxy = result[
"Value"], os.environ.get("X509_USER_PROXY")
# Check if the caller is executing with the host certificate
useServerCertificate = gConfig.useServerCertificate()
if useServerCertificate:
gConfigurationData.setOptionInCFG(
"/DIRAC/Security/UseServerCertificate", "false")
return S_OK((originalUserProxy, useServerCertificate, executionLock))
def wrapped_fcn( *args, **kwargs ):
# Get the lock and acquire it
executionLock = LockRing().getLock( '_UseUserProxy_', recursive = True )
executionLock.acquire()
# Check if the caller is executing with the host certificate
useServerCertificate = gConfig.useServerCertificate()
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
try:
return fcn( *args, **kwargs )
except Exception as lException: # pylint: disable=broad-except
value = ','.join( [str( arg ) for arg in lException.args] )
exceptType = lException.__class__.__name__
return S_ERROR( "Exception - %s: %s" % ( exceptType, value ) )
finally:
# Restore the default host certificate usage if necessary
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
# release the lock
executionLock.release()
def cleanMetadataCatalogFiles(self, transID):
""" wipe out files from catalog """
res = self.metadataClient.findFilesByMetadata({self.transfidmeta: transID})
if not res['OK']:
return res
fileToRemove = res['Value']
if not fileToRemove:
self.log.info('No files found for transID %s' % transID)
return S_OK()
# Executing with shifter proxy
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'false')
res = DataManager().removeFile(fileToRemove, force=True)
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true')
if not res['OK']:
return res
for lfn, reason in res['Value']['Failed'].items():
self.log.error("Failed to remove file found in metadata catalog", "%s %s" % (lfn, reason))
if res['Value']['Failed']:
return S_ERROR("Failed to remove all files found in the metadata catalog")
self.log.info("Successfully removed all files found in the BK")
return S_OK()
def _putProxy(userDN=None, userName=None, userGroup=None, vomsFlag=None, proxyFilePath=None, executionLockFlag=False):
"""Download proxy, place in a file and populate X509_USER_PROXY environment variable.
Parameters like `userProxy` or `executeWithUserProxy`.
:returns: Tuple of originalUserProxy, useServerCertificate, executionLock
"""
# Setup user proxy
if userDN:
userDNs = [userDN]
else:
result = getDNForUsername(userName)
if not result['OK']:
return result
userDNs = result['Value'] # a same user may have more than one DN
vomsAttr = ''
if vomsFlag:
vomsAttr = getVOMSAttributeForGroup(userGroup)
result = getProxy(userDNs, userGroup, vomsAttr, proxyFilePath)
if not result['OK']:
return result
executionLock = LockRing().getLock('_UseUserProxy_', recursive=True) if executionLockFlag else None
if executionLockFlag:
executionLock.acquire()
os.environ['X509_USER_PROXY'], originalUserProxy = result['Value'], os.environ.get('X509_USER_PROXY')
# Check if the caller is executing with the host certificate
useServerCertificate = gConfig.useServerCertificate()
if useServerCertificate:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'false')
return S_OK((originalUserProxy, useServerCertificate, executionLock))
def create_serverAndClient(request):
"""This function starts a server, and closes it after
The server will use the parametrized transport type
"""
# Reinitialize the configuration.
# We do it here rather than at the start of the module
# to accommodate for pytest when going through all the DIRAC tests
gConfigurationData.localCFG = CFG()
gConfigurationData.remoteCFG = CFG()
gConfigurationData.mergedCFG = CFG()
gConfigurationData.generateNewVersion()
gConfigurationData.setOptionInCFG("/DIRAC/Security/CALocation", caLocation)
gConfigurationData.setOptionInCFG("/DIRAC/Security/CertFile",
hostCertLocation)
gConfigurationData.setOptionInCFG("/DIRAC/Security/KeyFile",
hostKeyLocation)
testStr = request.param
serverName, clientName = testStr.split("-")
serverClass = transportByName(serverName)
clientClass = transportByName(clientName)
sr = DummyServiceReactor(serverClass, PORT_NUMBER)
server_thread = threading.Thread(target=sr.serve)
sr.prepare()
server_thread.start()
# Create the client
clientOptions = {
"clientMode": True,
"proxyLocation": proxyFile,
}
clientTransport = clientClass(("localhost", PORT_NUMBER),
bServerMode=False,
**clientOptions)
res = clientTransport.initAsClient()
assert res["OK"], res
yield sr, clientTransport
clientTransport.close()
sr.closeListeningConnections()
server_thread.join()
# Clean the config
gConfigurationData.localCFG = CFG()
gConfigurationData.remoteCFG = CFG()
gConfigurationData.mergedCFG = CFG()
gConfigurationData.generateNewVersion()
def __call__( self ):
""" request processing """
self.log.debug( "about to execute request" )
gMonitor.addMark( "RequestAtt", 1 )
# # setup proxy for request owner
setupProxy = self.setupProxy()
if not setupProxy["OK"]:
self.log.error( setupProxy["Message"] )
self.request.Error = setupProxy["Message"]
return S_ERROR( 'Change proxy error' )
shifter = setupProxy["Value"]["Shifter"]
proxyFile = setupProxy["Value"]["ProxyFile"]
error = None
while self.request.Status == "Waiting":
# # get waiting operation
operation = self.request.getWaiting()
if not operation["OK"]:
self.log.error( operation["Message"] )
return operation
operation = operation["Value"]
self.log.info( "executing operation #%s '%s'" % ( operation.Order, operation.Type ) )
# # and handler for it
handler = self.getHandler( operation )
if not handler["OK"]:
self.log.error( "unable to process operation %s: %s" % ( operation.Type, handler["Message"] ) )
# gMonitor.addMark( "%s%s" % ( operation.Type, "Fail" ), 1 )
operation.Error = handler["Message"]
break
handler = handler["Value"]
# # set shifters list in the handler
handler.shifter = shifter
# # and execute
pluginName = self.getPluginName( self.handlersDict.get( operation.Type ) )
if self.standalone:
useServerCertificate = gConfig.useServerCertificate()
else:
# Always use server certificates if executed within an agent
useServerCertificate = True
try:
if pluginName:
gMonitor.addMark( "%s%s" % ( pluginName, "Att" ), 1 )
# Always use request owner proxy
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
exe = handler()
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
if not exe["OK"]:
self.log.error( "unable to process operation %s: %s" % ( operation.Type, exe["Message"] ) )
if pluginName:
gMonitor.addMark( "%s%s" % ( pluginName, "Fail" ), 1 )
gMonitor.addMark( "RequestFail", 1 )
if self.request.JobID:
# Check if the job exists
monitorServer = RPCClient( "WorkloadManagement/JobMonitoring", useCertificates = True )
res = monitorServer.getJobPrimarySummary( int( self.request.JobID ) )
if not res["OK"]:
self.log.error( "RequestTask: Failed to get job %d status" % self.request.JobID )
elif not res['Value']:
self.log.warn( "RequestTask: job %d does not exist (anymore): failed request" % self.request.JobID )
for opFile in operation:
opFile.Status = 'Failed'
if operation.Status != 'Failed':
operation.Status = 'Failed'
self.request.Error = 'Job no longer exists'
except Exception, error:
self.log.exception( "hit by exception: %s" % str( error ) )
if pluginName:
gMonitor.addMark( "%s%s" % ( pluginName, "Fail" ), 1 )
gMonitor.addMark( "RequestFail", 1 )
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
break
# # operation status check
if operation.Status == "Done" and pluginName:
gMonitor.addMark( "%s%s" % ( pluginName, "OK" ), 1 )
elif operation.Status == "Failed" and pluginName:
gMonitor.addMark( "%s%s" % ( pluginName, "Fail" ), 1 )
elif operation.Status in ( "Waiting", "Scheduled" ):
# # no update for waiting or all files scheduled
break
def __call__(self):
""" request processing """
self.log.debug("about to execute request")
gMonitor.addMark("RequestAtt", 1)
# # setup proxy for request owner
setupProxy = self.setupProxy()
if not setupProxy["OK"]:
self.request.Error = setupProxy["Message"]
if 'has no proxy registered' in setupProxy["Message"]:
self.log.error('Request set to Failed:', setupProxy["Message"])
# If user is no longer registered, fail the request
for operation in self.request:
for opFile in operation:
opFile.Status = 'Failed'
operation.Status = 'Failed'
else:
self.log.error(setupProxy["Message"])
return S_OK(self.request)
shifter = setupProxy["Value"]["Shifter"]
proxyFile = setupProxy["Value"]["ProxyFile"]
error = None
while self.request.Status == "Waiting":
# # get waiting operation
operation = self.request.getWaiting()
if not operation["OK"]:
self.log.error(operation["Message"])
return operation
operation = operation["Value"]
self.log.info("executing operation #%s '%s'" %
(operation.Order, operation.Type))
# # and handler for it
handler = self.getHandler(operation)
if not handler["OK"]:
self.log.error("unable to process operation %s: %s" %
(operation.Type, handler["Message"]))
# gMonitor.addMark( "%s%s" % ( operation.Type, "Fail" ), 1 )
operation.Error = handler["Message"]
break
handler = handler["Value"]
# # set shifters list in the handler
handler.shifter = shifter
# # and execute
pluginName = self.getPluginName(
self.handlersDict.get(operation.Type))
if self.standalone:
useServerCertificate = gConfig.useServerCertificate()
else:
# Always use server certificates if executed within an agent
useServerCertificate = True
try:
if pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Att"), 1)
# Always use request owner proxy
if useServerCertificate:
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'false')
exe = handler()
if useServerCertificate:
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'true')
if not exe["OK"]:
self.log.error("unable to process operation %s: %s" %
(operation.Type, exe["Message"]))
if pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
gMonitor.addMark("RequestFail", 1)
if self.request.JobID:
# Check if the job exists
monitorServer = RPCClient(
"WorkloadManagement/JobMonitoring",
useCertificates=True)
res = monitorServer.getJobPrimarySummary(
int(self.request.JobID))
if not res["OK"]:
self.log.error(
"RequestTask: Failed to get job %d status" %
self.request.JobID)
elif not res['Value']:
self.log.warn(
"RequestTask: job %d does not exist (anymore): failed request"
% self.request.JobID)
for opFile in operation:
opFile.Status = 'Failed'
if operation.Status != 'Failed':
operation.Status = 'Failed'
self.request.Error = 'Job no longer exists'
except Exception, error:
self.log.exception("hit by exception: %s" % str(error))
if pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
gMonitor.addMark("RequestFail", 1)
if useServerCertificate:
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'true')
break
# # operation status check
if operation.Status == "Done" and pluginName:
gMonitor.addMark("%s%s" % (pluginName, "OK"), 1)
elif operation.Status == "Failed" and pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
elif operation.Status in ("Waiting", "Scheduled"):
# # no update for waiting or all files scheduled
break
def execute(self):
""" execution in one cycle
"""
# Don't use the server certificate otherwise the DFC wont let us write
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'false')
gMonitor.addMark("Iteration", 1)
############################################################
#
# Obtain the files which have not yet been migrated
#
self.log.info("Obtaining un-migrated files.")
res = self.rawIntegrityDB.getUnmigratedFiles()
if not res['OK']:
errStr = "Failed to obtain un-migrated files."
self.log.error(errStr, res['Message'])
return S_OK()
# allUnmigratedFilesMeta contain all the files that are not yet
# migrated (not copied, not registered, not removed), as well as their metadata
allUnmigratedFilesMeta = res['Value']
self.log.info("Obtained %s un-migrated files." % len(allUnmigratedFilesMeta))
if not allUnmigratedFilesMeta:
return S_OK()
# activeFiles contains the files that are not yet copied
activeFiles = {}
# copiedFiles contains files that are copied but not yet registered
copiedFiles = {}
# registeredFiles contains files that are copied, registered, but not removed from source
registeredFiles = {}
# Assign them
for lfn, lfnMetadata in allUnmigratedFilesMeta.iteritems():
status = lfnMetadata.pop('Status')
if status == 'Active':
activeFiles[lfn] = lfnMetadata
elif status == 'Copied':
copiedFiles[lfn] = lfnMetadata
elif status == 'Registered':
registeredFiles[lfn] = lfnMetadata
gMonitor.addMark("WaitingFiles", len(activeFiles))
totalSize = 0
for lfn, fileDict in activeFiles.iteritems():
totalSize += int(fileDict['Size'])
# gMonitor.addMark("TimeInQueue", (fileDict['WaitTime'] / 60))
gMonitor.addMark("WaitSize", (totalSize / (1024 * 1024 * 1024.0)))
############################################################
#
# Checking newly copied files
#
# Get the list of lfns properly copied and not copied
filesNewlyCopied, filesNotYetCopied = self.getNewlyCopiedFiles(activeFiles)
####################################################
#
# Registering copied files
#
####################################################
filesNewlyRegistered = self.registerCopiedFiles(filesNewlyCopied, copiedFiles,
allUnmigratedFilesMeta)
####################################################
#
# Performing the removal from the online storage
#
####################################################
filesNewlyRemoved = self.removeRegisteredFiles(filesNewlyRegistered, registeredFiles,
allUnmigratedFilesMeta)
# Doing some accounting
migratedSize = sum(allUnmigratedFilesMeta[lfn]['Size'] for lfn in filesNewlyRemoved)
# The number of files that we failed at migrating
# is the number of files at the beginning, minus the one we processed completely
# minus those that are not yet copied
failedMigrated = len(allUnmigratedFilesMeta) - len(filesNewlyRemoved) - len(filesNotYetCopied)
res = self.rawIntegrityDB.setLastMonitorTime()
migratedSizeGB = migratedSize / (1024 * 1024 * 1024.0)
gMonitor.addMark("TotMigratedSize", migratedSizeGB)
gMonitor.addMark("NewlyMigrated", len(filesNewlyRemoved))
gMonitor.addMark("TotMigrated", len(filesNewlyRemoved))
gMonitor.addMark("FailedMigrated", failedMigrated)
gMonitor.addMark("TotFailMigrated", failedMigrated)
return S_OK()
def __setOptionValue( self, optionPath, value ):
gConfigurationData.setOptionInCFG( self.__getAbsolutePath( optionPath ),
str( value ) )
def __call__(self):
"""request processing"""
self.log.debug("about to execute request")
if not self.rmsMonitoring:
gMonitor.addMark("RequestAtt", 1)
# # setup proxy for request owner
setupProxy = self.setupProxy()
if not setupProxy["OK"]:
userSuspended = "User is currently suspended"
self.request.Error = setupProxy["Message"]
# In case the user does not have proxy
if DErrno.cmpError(setupProxy, DErrno.EPROXYFIND):
self.log.error("Error setting proxy. Request set to Failed:",
setupProxy["Message"])
# If user is no longer registered, fail the request
for operation in self.request:
for opFile in operation:
opFile.Status = "Failed"
operation.Status = "Failed"
elif userSuspended in setupProxy["Message"]:
# If user is suspended, wait for a long time
self.request.delayNextExecution(6 * 60)
self.request.Error = userSuspended
self.log.error("Error setting proxy: " + userSuspended,
self.request.OwnerDN)
else:
self.log.error("Error setting proxy", setupProxy["Message"])
return S_OK(self.request)
shifter = setupProxy["Value"]["Shifter"]
error = None
while self.request.Status == "Waiting":
# # get waiting operation
operation = self.request.getWaiting()
if not operation["OK"]:
self.log.error("Cannot get waiting operation",
operation["Message"])
return operation
operation = operation["Value"]
self.log.info("executing operation", "%s" % operation.Type)
# # and handler for it
handler = self.getHandler(operation)
if not handler["OK"]:
self.log.error("Unable to process operation",
"%s: %s" % (operation.Type, handler["Message"]))
# gMonitor.addMark( "%s%s" % ( operation.Type, "Fail" ), 1 )
operation.Error = handler["Message"]
break
handler = handler["Value"]
# # set shifters list in the handler
handler.shifter = shifter
# set rmsMonitoring flag for the RequestOperation
handler.rmsMonitoring = self.rmsMonitoring
# # and execute
pluginName = self.getPluginName(
self.handlersDict.get(operation.Type))
if self.standalone:
useServerCertificate = gConfig.useServerCertificate()
else:
# Always use server certificates if executed within an agent
useServerCertificate = True
try:
if pluginName:
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord({
"timestamp":
int(Time.toEpoch()),
"host":
Network.getFQDN(),
"objectType":
"Operation",
"operationType":
pluginName,
"objectID":
operation.OperationID,
"parentID":
operation.RequestID,
"status":
"Attempted",
"nbObject":
1,
})
else:
gMonitor.addMark("%s%s" % (pluginName, "Att"), 1)
# Always use request owner proxy
if useServerCertificate:
gConfigurationData.setOptionInCFG(
"/DIRAC/Security/UseServerCertificate", "false")
exe = handler()
if useServerCertificate:
gConfigurationData.setOptionInCFG(
"/DIRAC/Security/UseServerCertificate", "true")
if not exe["OK"]:
self.log.error("unable to process operation",
"%s: %s" % (operation.Type, exe["Message"]))
if pluginName:
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord({
"timestamp":
int(Time.toEpoch()),
"host":
Network.getFQDN(),
"objectType":
"Operation",
"operationType":
pluginName,
"objectID":
operation.OperationID,
"parentID":
operation.RequestID,
"status":
"Failed",
"nbObject":
1,
})
else:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord({
"timestamp":
int(Time.toEpoch()),
"host":
Network.getFQDN(),
"objectType":
"Request",
"objectID":
operation.RequestID,
"status":
"Failed",
"nbObject":
1,
})
else:
gMonitor.addMark("RequestFail", 1)
if self.request.JobID:
# Check if the job exists
monitorServer = JobMonitoringClient(
useCertificates=True)
res = monitorServer.getJobSummary(
int(self.request.JobID))
if not res["OK"]:
self.log.error(
"RequestTask: Failed to get job status",
"%d" % self.request.JobID)
elif not res["Value"]:
self.log.warn(
"RequestTask: job does not exist (anymore): failed request",
"JobID: %d" % self.request.JobID,
)
for opFile in operation:
opFile.Status = "Failed"
if operation.Status != "Failed":
operation.Status = "Failed"
self.request.Error = "Job no longer exists"
except Exception as e:
error = str(e)
self.log.exception("hit by exception:", "%s" % error)
if pluginName:
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord({
"timestamp":
int(Time.toEpoch()),
"host":
Network.getFQDN(),
"objectType":
"Operation",
"operationType":
pluginName,
"objectID":
operation.OperationID,
"parentID":
operation.RequestID,
"status":
"Failed",
"nbObject":
1,
})
else:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord({
"timestamp":
int(Time.toEpoch()),
"host":
Network.getFQDN(),
"objectType":
"Request",
"objectID":
operation.RequestID,
"status":
"Failed",
"nbObject":
1,
})
else:
gMonitor.addMark("RequestFail", 1)
if useServerCertificate:
gConfigurationData.setOptionInCFG(
"/DIRAC/Security/UseServerCertificate", "true")
break
# # operation status check
if operation.Status == "Done" and pluginName:
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord({
"timestamp":
int(Time.toEpoch()),
"host":
Network.getFQDN(),
"objectType":
"Operation",
"operationType":
pluginName,
"objectID":
operation.OperationID,
"parentID":
operation.RequestID,
"status":
"Successful",
"nbObject":
1,
})
else:
gMonitor.addMark("%s%s" % (pluginName, "OK"), 1)
elif operation.Status == "Failed" and pluginName:
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord({
"timestamp":
int(Time.toEpoch()),
"host":
Network.getFQDN(),
"objectType":
"Operation",
"operationType":
pluginName,
"objectID":
operation.OperationID,
"parentID":
operation.RequestID,
"status":
"Failed",
"nbObject":
1,
})
else:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
elif operation.Status in ("Waiting", "Scheduled"):
# # no update for waiting or all files scheduled
break
if not self.rmsMonitoring:
gMonitor.flush()
if error:
return S_ERROR(error)
# # request done?
if self.request.Status == "Done":
# # update request to the RequestDB
self.log.info("Updating request status:",
"%s" % self.request.Status)
update = self.updateRequest()
if not update["OK"]:
self.log.error("Cannot update request status",
update["Message"])
return update
self.log.info("request is done", "%s" % self.request.RequestName)
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord({
"timestamp":
int(Time.toEpoch()),
"host":
Network.getFQDN(),
"objectType":
"Request",
"objectID":
getattr(self.request, "RequestID", 0),
"status":
"Successful",
"nbObject":
1,
})
else:
gMonitor.addMark("RequestOK", 1)
# # and there is a job waiting for it? finalize!
if self.request.JobID:
attempts = 0
while True:
finalizeRequest = self.requestClient.finalizeRequest(
self.request.RequestID,
self.request.JobID # pylint: disable=no-member
)
if not finalizeRequest["OK"]:
if not attempts:
self.log.error(
"unable to finalize request, will retry",
"ReqName %s:%s" % (self.request.RequestName,
finalizeRequest["Message"]),
)
self.log.debug("Waiting 10 seconds")
attempts += 1
if attempts == 10:
self.log.error("Giving up finalize request")
return S_ERROR("Could not finalize request")
time.sleep(10)
else:
self.log.info(
"request is finalized",
"ReqName %s %s" % (self.request.RequestName,
(" after %d attempts" %
attempts) if attempts else ""),
)
break
# Commit all the data to the ES Backend
if self.rmsMonitoring:
self.rmsMonitoringReporter.commit()
# Request will be updated by the callBack method
self.log.verbose("RequestTasks exiting",
"request %s" % self.request.Status)
return S_OK(self.request)
def wrapped_fcn( *args, **kwargs ):
userName = kwargs.pop( 'proxyUserName', '' )
userDN = kwargs.pop( 'proxyUserDN', '' )
userGroup = kwargs.pop( 'proxyUserGroup', '' )
vomsFlag = kwargs.pop( 'proxyWithVOMS', True )
proxyFilePath = kwargs.pop( 'proxyFilePath', False )
if ( userName or userDN ) and userGroup:
# Setup user proxy
originalUserProxy = os.environ.get( 'X509_USER_PROXY' )
if not userDN:
result = getDNForUsername( userName )
if not result[ 'OK' ]:
return result
userDN = result[ 'Value' ][0]
vomsAttr = ''
if vomsFlag:
vomsAttr = getVOMSAttributeForGroup( userGroup )
if vomsAttr:
result = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup,
requiredVOMSAttribute = vomsAttr,
filePath = proxyFilePath,
requiredTimeLeft = 3600,
cacheTime = 3600 )
else:
result = gProxyManager.downloadProxyToFile( userDN, userGroup,
filePath = proxyFilePath,
requiredTimeLeft = 3600,
cacheTime = 3600 )
if not result['OK']:
gLogger.warn( "Can't download proxy to file", result['Message'] )
return result
proxyFile = result['Value']
os.environ['X509_USER_PROXY'] = proxyFile
# Check if the caller is executing with the host certificate
useServerCertificate = gConfig.useServerCertificate()
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
try:
return fcn( *args, **kwargs )
except Exception as lException:
value = ','.join( [str( arg ) for arg in lException.args] )
exceptType = lException.__class__.__name__
return S_ERROR( "Exception - %s: %s" % ( exceptType, value ) )
finally:
# Restore the default host certificate usage if necessary
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
if originalUserProxy:
os.environ['X509_USER_PROXY'] = originalUserProxy
else:
os.environ.pop( 'X509_USER_PROXY' )
else:
# No proxy substitution requested
return fcn( *args, **kwargs )
def __call__( self ):
""" generic function to process one Request of a type requestType
This method could be run in a thread.
:param self: self reference
:param str requestType: request type
:return: S_OK/S_ERROR
"""
self.always("executing request %s" % self.requestName )
################################################################
## get ownerDN and ownerGroup
ownerDN = self.requestObj.getAttribute( "OwnerDN" )
if not ownerDN["OK"]:
return ownerDN
ownerDN = ownerDN["Value"]
ownerGroup = self.requestObj.getAttribute( "OwnerGroup" )
if not ownerGroup["OK"]:
return ownerGroup
ownerGroup = ownerGroup["Value"]
## save request owner
self.requestOwnerDN = ownerDN if ownerDN else ""
self.requestOwnerGroup = ownerGroup if ownerGroup else ""
#################################################################
## change proxy
ownerProxyFile = None
if ownerDN and ownerGroup:
ownerProxyFile = self.changeProxy( ownerDN, ownerGroup )
if not ownerProxyFile["OK"]:
self.error( "handleReuqest: unable to get proxy for '%s'@'%s': %s" % ( ownerDN,
ownerGroup,
ownerProxyFile["Message"] ) )
update = self.putBackRequest( self.requestName, self.requestString )
if not update["OK"]:
self.error( "handleRequest: error when updating request: %s" % update["Message"] )
return update
return ownerProxyFile
ownerProxyFile = ownerProxyFile["Value"]
#self.ownerProxyFile = ownerProxyFile
self.info( "Will execute request for '%s'@'%s' using proxy file %s" % ( ownerDN, ownerGroup, ownerProxyFile ) )
else:
self.info( "Will execute request for DataManager using her/his proxy")
#################################################################
## execute handlers
ret = { "OK" : False, "Message" : "" }
useServerCert = gConfig.useServerCertificate()
try:
# Execute task with the owner proxy even for contacting DIRAC services
if useServerCert:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate','false')
ret = self.handleRequest()
finally:
if useServerCert:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate','true')
## delete owner proxy
if self.__dataManagerProxy:
os.environ["X509_USER_PROXY"] = self.__dataManagerProxy
if ownerProxyFile and os.path.exists( ownerProxyFile ):
os.unlink( ownerProxyFile )
if not ret["OK"]:
self.error( "handleRequest: error during request processing: %s" % ret["Message"] )
self.error( "handleRequest: will put original request back" )
update = self.putBackRequest( self.requestName, self.requestString )
if not update["OK"]:
self.error( "handleRequest: error when putting back request: %s" % update["Message"] )
## return at least
return ret
#!/usr/bin/env python
""" Performs a DIPS ping on a given target and exit with the return code.
It uses the local host certificate
The target is specified as ""<port>/System/Service"
The script does not print anything, and just exists with 0 in case of success,
or 1 in case of error """
__RCSID__ = "$Id$"
import sys
import os
import time
with open(os.devnull, 'w') as redirectStdout, open(os.devnull, 'w') as redirectStderr:
from DIRAC import gLogger
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
gLogger.setLevel('FATAL')
from DIRAC.Core.DISET.RPCClient import RPCClient
rpc = RPCClient('dips://localhost:%s'%sys.argv[1])
res = rpc.ping()
time.sleep(0.1)
if not res['OK']:
sys.exit(1)
else:
sys.exit(0)
def setOptionValue( self, optionPath, value ): """ Set a value in the local configuration """ gConfigurationData.setOptionInCFG( optionPath, value )
def setOptionValue(self, optionPath, value):
"""
Set a value in the local configuration
"""
gConfigurationData.setOptionInCFG(optionPath, value)
def __call__( self ):
""" request processing """
self.log.debug( "about to execute request" )
gMonitor.addMark( "RequestAtt", 1 )
# # setup proxy for request owner
setupProxy = self.setupProxy()
if not setupProxy["OK"]:
self.log.error( setupProxy["Message"] )
self.request.Error = setupProxy["Message"]
return S_ERROR( 'Change proxy error' )
shifter = setupProxy["Value"]["Shifter"]
proxyFile = setupProxy["Value"]["ProxyFile"]
error = None
while self.request.Status == "Waiting":
# # get waiting operation
operation = self.request.getWaiting()
if not operation["OK"]:
self.log.error( operation["Message"] )
return operation
operation = operation["Value"]
self.log.info( "executing operation #%s '%s'" % ( operation.Order, operation.Type ) )
# # and handler for it
handler = self.getHandler( operation )
if not handler["OK"]:
self.log.error( "unable to process operation %s: %s" % ( operation.Type, handler["Message"] ) )
# gMonitor.addMark( "%s%s" % ( operation.Type, "Fail" ), 1 )
operation.Error = handler["Message"]
break
handler = handler["Value"]
# # set shifters list in the handler
handler.shifter = shifter
# # and execute
pluginName = self.getPluginName( self.handlersDict.get( operation.Type ) )
useServerCertificate = gConfig.useServerCertificate()
try:
if pluginName:
gMonitor.addMark( "%s%s" % ( pluginName, "Att" ), 1 )
# Always use request owner proxy
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
exe = handler()
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
if not exe["OK"]:
self.log.error( "unable to process operation %s: %s" % ( operation.Type, exe["Message"] ) )
if pluginName:
gMonitor.addMark( "%s%s" % ( pluginName, "Fail" ), 1 )
gMonitor.addMark( "RequestFail", 1 )
except Exception, error:
self.log.exception( "hit by exception: %s" % str( error ) )
if pluginName:
gMonitor.addMark( "%s%s" % ( pluginName, "Fail" ), 1 )
gMonitor.addMark( "RequestFail", 1 )
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
break
# # operation status check
if operation.Status == "Done" and pluginName:
gMonitor.addMark( "%s%s" % ( pluginName, "OK" ), 1 )
elif operation.Status == "Failed" and pluginName:
gMonitor.addMark( "%s%s" % ( pluginName, "Fail" ), 1 )
elif operation.Status in ( "Waiting", "Scheduled" ):
# # no update for waiting or all files scheduled
break
def __call__( self ):
""" generic function to process one Request of a type requestType
This method could be run in a thread.
:param self: self reference
:param str requestType: request type
:return: S_OK/S_ERROR
"""
self.always("executing request %s" % self.requestName )
################################################################
## get ownerDN and ownerGroup
ownerDN = self.requestObj.getAttribute( "OwnerDN" )
if not ownerDN["OK"]:
return ownerDN
ownerDN = ownerDN["Value"]
ownerGroup = self.requestObj.getAttribute( "OwnerGroup" )
if not ownerGroup["OK"]:
return ownerGroup
ownerGroup = ownerGroup["Value"]
## save request owner
self.requestOwnerDN = ownerDN if ownerDN else ""
self.requestOwnerGroup = ownerGroup if ownerGroup else ""
#################################################################
## change proxy
ownerProxyFile = None
if ownerDN and ownerGroup:
ownerProxyFile = self.changeProxy( ownerDN, ownerGroup )
if not ownerProxyFile["OK"]:
self.error( "handleReuqest: unable to get proxy for '%s'@'%s': %s" % ( ownerDN,
ownerGroup,
ownerProxyFile["Message"] ) )
update = self.putBackRequest( self.requestName, self.requestString )
if not update["OK"]:
self.error( "handleRequest: error when updating request: %s" % update["Message"] )
return update
return ownerProxyFile
ownerProxyFile = ownerProxyFile["Value"]
#self.ownerProxyFile = ownerProxyFile
self.info( "Will execute request for '%s'@'%s' using proxy file %s" % ( ownerDN, ownerGroup, ownerProxyFile ) )
else:
self.info( "Will execute request for DataManager using her/his proxy")
#################################################################
## execute handlers
ret = { "OK" : False, "Message" : "" }
useServerCert = gConfig.useServerCertificate()
try:
# Execute task with the owner proxy even for contacting DIRAC services
if useServerCert:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate','false')
ret = self.handleRequest()
finally:
if useServerCert:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate','true')
## delete owner proxy
if self.__dataManagerProxy:
os.environ["X509_USER_PROXY"] = self.__dataManagerProxy
if ownerProxyFile and os.path.exists( ownerProxyFile ):
os.unlink( ownerProxyFile )
if not ret["OK"]:
self.error( "handleRequest: error during request processing: %s" % ret["Message"] )
self.error( "handleRequest: will put original request back" )
update = self.putBackRequest( self.requestName, self.requestString )
if not update["OK"]:
self.error( "handleRequest: error when putting back request: %s" % update["Message"] )
## return at least
return ret
def __setOptionValue(self, optionPath, value):
gConfigurationData.setOptionInCFG(self.__getAbsolutePath(optionPath),
str(value))
def __call__(self):
""" request processing """
self.log.debug("about to execute request")
gMonitor.addMark("RequestAtt", 1)
# # setup proxy for request owner
setupProxy = self.setupProxy()
if not setupProxy["OK"]:
self.request.Error = setupProxy["Message"]
if 'has no proxy registered' in setupProxy["Message"]:
self.log.error('Request set to Failed:', setupProxy["Message"])
# If user is no longer registered, fail the request
for operation in self.request:
for opFile in operation:
opFile.Status = 'Failed'
operation.Status = 'Failed'
else:
self.log.error(setupProxy["Message"])
return S_OK(self.request)
shifter = setupProxy["Value"]["Shifter"]
proxyFile = setupProxy["Value"]["ProxyFile"]
error = None
while self.request.Status == "Waiting":
# # get waiting operation
operation = self.request.getWaiting()
if not operation["OK"]:
self.log.error(operation["Message"])
return operation
operation = operation["Value"]
self.log.info("executing operation #%s '%s'" % (operation.Order, operation.Type))
# # and handler for it
handler = self.getHandler(operation)
if not handler["OK"]:
self.log.error("unable to process operation %s: %s" % (operation.Type, handler["Message"]))
# gMonitor.addMark( "%s%s" % ( operation.Type, "Fail" ), 1 )
operation.Error = handler["Message"]
break
handler = handler["Value"]
# # set shifters list in the handler
handler.shifter = shifter
# # and execute
pluginName = self.getPluginName(self.handlersDict.get(operation.Type))
if self.standalone:
useServerCertificate = gConfig.useServerCertificate()
else:
# Always use server certificates if executed within an agent
useServerCertificate = True
try:
if pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Att"), 1)
# Always use request owner proxy
if useServerCertificate:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'false')
exe = handler()
if useServerCertificate:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true')
if not exe["OK"]:
self.log.error("unable to process operation %s: %s" % (operation.Type, exe["Message"]))
if pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
gMonitor.addMark("RequestFail", 1)
if self.request.JobID:
# Check if the job exists
monitorServer = RPCClient("WorkloadManagement/JobMonitoring", useCertificates=True)
res = monitorServer.getJobPrimarySummary(int(self.request.JobID))
if not res["OK"]:
self.log.error("RequestTask: Failed to get job %d status" % self.request.JobID)
elif not res['Value']:
self.log.warn(
"RequestTask: job %d does not exist (anymore): failed request" %
self.request.JobID)
for opFile in operation:
opFile.Status = 'Failed'
if operation.Status != 'Failed':
operation.Status = 'Failed'
self.request.Error = 'Job no longer exists'
except Exception as error:
self.log.exception("hit by exception: %s" % str(error))
if pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
gMonitor.addMark("RequestFail", 1)
if useServerCertificate:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true')
break
# # operation status check
if operation.Status == "Done" and pluginName:
gMonitor.addMark("%s%s" % (pluginName, "OK"), 1)
elif operation.Status == "Failed" and pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
elif operation.Status in ("Waiting", "Scheduled"):
# # no update for waiting or all files scheduled
break
gMonitor.flush()
if error:
return S_ERROR(error)
# # request done?
if self.request.Status == "Done":
# # update request to the RequestDB
self.log.info('updating request with status %s' % self.request.Status)
update = self.updateRequest()
if not update["OK"]:
self.log.error(update["Message"])
return update
self.log.info("request '%s' is done" % self.request.RequestName)
gMonitor.addMark("RequestOK", 1)
# # and there is a job waiting for it? finalize!
if self.request.JobID:
attempts = 0
while True:
finalizeRequest = self.requestClient.finalizeRequest(
self.request.RequestID, self.request.JobID) # pylint: disable=no-member
if not finalizeRequest["OK"]:
if not attempts:
self.log.error(
"unable to finalize request %s: %s, will retry" %
(self.request.RequestName, finalizeRequest["Message"]))
self.log.verbose("Waiting 10 seconds")
attempts += 1
if attempts == 10:
self.log.error("giving up finalize request after %d attempts" % attempts)
return S_ERROR('Could not finalize request')
time.sleep(10)
else:
self.log.info(
"request '%s' is finalized%s" %
(self.request.RequestName,
(' after %d attempts' %
attempts) if attempts else ''))
break
# Request will be updated by the callBack method
self.log.verbose("RequestTasks exiting, request %s" % self.request.Status)
return S_OK(self.request)
import sys
from urlparse import urlparse
from DIRAC import gLogger
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate',
'true')
gLogger.setLevel('FATAL')
from DIRAC.Core.DISET.RPCClient import RPCClient
import SimpleHTTPServer
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
import SocketServer
class MyRequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path == '/self':
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
# Send the html message
self.wfile.write("I am good thanks")
return
if not self.path.startswith('/ping'):
self.send_error(404)
return
outputText = "Nothing to report"
query = urlparse(self.path).query
def __call__(self):
""" request processing """
self.log.debug("about to execute request")
gMonitor.addMark("RequestAtt", 1)
# # setup proxy for request owner
setupProxy = self.setupProxy()
if not setupProxy["OK"]:
self.request.Error = setupProxy["Message"]
if 'has no proxy registered' in setupProxy["Message"]:
self.log.error('Request set to Failed:', setupProxy["Message"])
# If user is no longer registered, fail the request
for operation in self.request:
for opFile in operation:
opFile.Status = 'Failed'
operation.Status = 'Failed'
else:
self.log.error(setupProxy["Message"])
return S_OK(self.request)
shifter = setupProxy["Value"]["Shifter"]
proxyFile = setupProxy["Value"]["ProxyFile"]
error = None
while self.request.Status == "Waiting":
# # get waiting operation
operation = self.request.getWaiting()
if not operation["OK"]:
self.log.error(operation["Message"])
return operation
operation = operation["Value"]
self.log.info("executing operation #%s '%s'" %
(operation.Order, operation.Type))
# # and handler for it
handler = self.getHandler(operation)
if not handler["OK"]:
self.log.error("unable to process operation %s: %s" %
(operation.Type, handler["Message"]))
# gMonitor.addMark( "%s%s" % ( operation.Type, "Fail" ), 1 )
operation.Error = handler["Message"]
break
handler = handler["Value"]
# # set shifters list in the handler
handler.shifter = shifter
# # and execute
pluginName = self.getPluginName(
self.handlersDict.get(operation.Type))
if self.standalone:
useServerCertificate = gConfig.useServerCertificate()
else:
# Always use server certificates if executed within an agent
useServerCertificate = True
try:
if pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Att"), 1)
# Always use request owner proxy
if useServerCertificate:
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'false')
exe = handler()
if useServerCertificate:
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'true')
if not exe["OK"]:
self.log.error("unable to process operation %s: %s" %
(operation.Type, exe["Message"]))
if pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
gMonitor.addMark("RequestFail", 1)
if self.request.JobID:
# Check if the job exists
monitorServer = RPCClient(
"WorkloadManagement/JobMonitoring",
useCertificates=True)
res = monitorServer.getJobPrimarySummary(
int(self.request.JobID))
if not res["OK"]:
self.log.error(
"RequestTask: Failed to get job %d status" %
self.request.JobID)
elif not res['Value']:
self.log.warn(
"RequestTask: job %d does not exist (anymore): failed request"
% self.request.JobID)
for opFile in operation:
opFile.Status = 'Failed'
if operation.Status != 'Failed':
operation.Status = 'Failed'
self.request.Error = 'Job no longer exists'
except Exception as error:
self.log.exception("hit by exception: %s" % str(error))
if pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
gMonitor.addMark("RequestFail", 1)
if useServerCertificate:
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'true')
break
# # operation status check
if operation.Status == "Done" and pluginName:
gMonitor.addMark("%s%s" % (pluginName, "OK"), 1)
elif operation.Status == "Failed" and pluginName:
gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
elif operation.Status in ("Waiting", "Scheduled"):
# # no update for waiting or all files scheduled
break
gMonitor.flush()
if error:
return S_ERROR(error)
# # request done?
if self.request.Status == "Done":
# # update request to the RequestDB
self.log.info('updating request with status %s' %
self.request.Status)
update = self.updateRequest()
if not update["OK"]:
self.log.error(update["Message"])
return update
self.log.info("request '%s' is done" % self.request.RequestName)
gMonitor.addMark("RequestOK", 1)
# # and there is a job waiting for it? finalize!
if self.request.JobID:
attempts = 0
while True:
finalizeRequest = self.requestClient.finalizeRequest(
self.request.RequestID, self.request.JobID) # pylint: disable=no-member
if not finalizeRequest["OK"]:
if not attempts:
self.log.error(
"unable to finalize request %s: %s, will retry"
% (self.request.RequestName,
finalizeRequest["Message"]))
self.log.verbose("Waiting 10 seconds")
attempts += 1
if attempts == 10:
self.log.error(
"giving up finalize request after %d attempts"
% attempts)
return S_ERROR('Could not finalize request')
time.sleep(10)
else:
self.log.info("request '%s' is finalized%s" %
(self.request.RequestName,
(' after %d attempts' %
attempts) if attempts else ''))
break
# Request will be updated by the callBack method
self.log.verbose("RequestTasks exiting, request %s" %
self.request.Status)
return S_OK(self.request)
