示例#1
0
    def __call__(self):
        """execute RPC stub"""
        # # decode arguments
        try:
            decode, length = DEncode.decode(self.operation.Arguments)
            self.log.debug("decoded len=%s val=%s" % (length, decode))
        except ValueError as error:
            self.log.exception(error)
            self.operation.Error = str(error)
            self.operation.Status = "Failed"
            return S_ERROR(str(error))

        # Ensure the forwarded request is done on behalf of the request owner
        decode[0][1]["delegatedDN"] = self.request.OwnerDN
        decode[0][1]["delegatedGroup"] = self.request.OwnerGroup

        # ForwardDiset is supposed to be used with a host certificate
        useServerCertificate = gConfig.useServerCertificate()
        gConfigurationData.setOptionInCFG(
            "/DIRAC/Security/UseServerCertificate", "true")
        forward = executeRPCStub(decode)
        if not useServerCertificate:
            gConfigurationData.setOptionInCFG(
                "/DIRAC/Security/UseServerCertificate", "false")

        if not forward["OK"]:
            self.log.error(
                "unable to execute operation",
                "'%s' : %s" % (self.operation.Type, forward["Message"]))
            self.operation.Error = forward["Message"]
            return forward
        self.log.info("DISET forwarding done")
        self.operation.Status = "Done"
        return S_OK()
示例#2
0
def test_rpcStubs_are_equals(UseServerCertificate):
    """
    Test if Clients returns the same rpcStubs

    Navigating through array is a bit complicated in this test...
    repDirac and repTornado may have the same structure:

    repDirac dict{
      OK: True
      rpcStub: tuple{
        ServiceName: str
        kwargs: dict{
          *** kwargs used to instanciate client ***
        }
        methodName: str
        arguments: list
      }
      Value: dict { # NOT USED IN THIS TEST
        *** Credentials dictionnary extracted by server ***
      }
    }
    """

    gConfigurationData.setOptionInCFG("/DIRAC/Security/UseServerCertificate", UseServerCertificate)
    (repTornado, repDirac) = get_all_returnedValues()

    # Explicitly removed in Tornado
    del repDirac["rpcStub"][0][1]["keepAliveLapse"]

    # rep['rpcStub'] is at form (rpcStub, method, args) where rpcStub is tuple with (serviceName, kwargs)
    assert repTornado["rpcStub"][0][0] != repDirac["rpcStub"][0][0]  # Services name are different
    assert repTornado["rpcStub"][0][1] == repDirac["rpcStub"][0][1]  # Check kwargs returned by rpcStub
    assert repTornado["rpcStub"][1:] != repDirac["rpcStub"][1:]  # Check method/args
    def cleanMetadataCatalogFiles(self, transID):
        """ wipe out files from catalog """
        res = self.metadataClient.findFilesByMetadata(
            {self.transfidmeta: transID})
        if not res['OK']:
            return res
        fileToRemove = res['Value']
        if not fileToRemove:
            self.log.info('No files found for transID %s' % transID)
            return S_OK()

        # Executing with shifter proxy
        gConfigurationData.setOptionInCFG(
            '/DIRAC/Security/UseServerCertificate', 'false')
        res = DataManager().removeFile(fileToRemove, force=True)
        gConfigurationData.setOptionInCFG(
            '/DIRAC/Security/UseServerCertificate', 'true')

        if not res['OK']:
            return res
        for lfn, reason in res['Value']['Failed'].items():
            self.log.error("Failed to remove file found in metadata catalog",
                           "%s %s" % (lfn, reason))
        if res['Value']['Failed']:
            return S_ERROR(
                "Failed to remove all files found in the metadata catalog")
        self.log.info("Successfully removed all files found in the BK")
        return S_OK()
  def loadUserData( self ):
    """
    This is the magic method that reads the command line and processes it
    It is used by the Script Base class and the dirac-service and dirac-agent scripts
    Before being called:
    - any additional switches to be processed
    - mandatory and default configuration configuration options must be defined.

    """
    if self.initialized:
      return S_OK()
    self.initialized = True
    try:
      retVal = self.__addUserDataToConfiguration()

      for optionTuple in self.optionalEntryList:
        optionPath = self.__getAbsolutePath( optionTuple[0] )
        if not gConfigurationData.extractOptionFromCFG( optionPath ):
          gConfigurationData.setOptionInCFG( optionPath, optionTuple[1] )

      self.__initLogger( self.componentName, self.loggingSection )
      if not retVal[ 'OK' ]:
        return retVal

      retVal = self.__checkMandatoryOptions()
      if not retVal[ 'OK' ]:
        return retVal

    except Exception as e:
      gLogger.exception()
      return S_ERROR( str( e ) )
    return S_OK()
示例#5
0
    def loadUserData(self):
        """
    This is the magic method that reads the command line and processes it
    It is used by the Script Base class and the dirac-service and dirac-agent scripts
    Before being called:
    - any additional switches to be processed
    - mandatory and default configuration configuration options must be defined.

    """
        if self.initialized:
            return S_OK()
        self.initialized = True
        try:
            retVal = self.__addUserDataToConfiguration()

            for optionTuple in self.optionalEntryList:
                optionPath = self.__getAbsolutePath(optionTuple[0])
                if not gConfigurationData.extractOptionFromCFG(optionPath):
                    gConfigurationData.setOptionInCFG(optionPath,
                                                      optionTuple[1])

            self.__initLogger(self.componentName, self.loggingSection)
            if not retVal['OK']:
                return retVal

            retVal = self.__checkMandatoryOptions()
            if not retVal['OK']:
                return retVal

        except Exception as e:
            gLogger.exception()
            return S_ERROR(str(e))
        return S_OK()
示例#6
0
    def setOptionValue(self, optionPath, value):
        """ Set a value in the local configuration

        :param str optionPath: option path
        :param str value: value
    """
        gConfigurationData.setOptionInCFG(optionPath, value)
示例#7
0
    def wrapped_fcn(*args, **kwargs):

        # Get the lock and acquire it
        executionLock = LockRing().getLock("_UseUserProxy_", recursive=True)
        executionLock.acquire()

        # Check if the caller is executing with the host certificate
        useServerCertificate = gConfig.useServerCertificate()
        if useServerCertificate:
            gConfigurationData.setOptionInCFG(
                "/DIRAC/Security/UseServerCertificate", "false")

        try:
            return fcn(*args, **kwargs)
        except Exception as lException:  # pylint: disable=broad-except
            value = ",".join([str(arg) for arg in lException.args])
            exceptType = lException.__class__.__name__
            return S_ERROR("Exception - %s: %s" % (exceptType, value))
        finally:
            # Restore the default host certificate usage if necessary
            if useServerCertificate:
                gConfigurationData.setOptionInCFG(
                    "/DIRAC/Security/UseServerCertificate", "true")
            # release the lock
            executionLock.release()
  def cleanContent(self, directory):
    """ wipe out everything from catalog under folder :directory:

    :param self: self reference
    :params str directory: folder name
    """
    self.log.verbose("Cleaning Catalog contents")
    res = self.__getCatalogDirectoryContents([directory])
    if not res['OK']:
      return res
    filesFound = res['Value']
    if not filesFound:
      self.log.info("No files are registered in the catalog directory %s" % directory)
      return S_OK()
    self.log.info("Attempting to remove %d possible remnants from the catalog and storage" % len(filesFound))

    # Executing with shifter proxy
    gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'false')
    res = DataManager().removeFile(filesFound, force=True)
    gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true')

    if not res['OK']:
      return res
    realFailure = False
    for lfn, reason in res['Value']['Failed'].items():
      if "File does not exist" in str(reason):
        self.log.warn("File %s not found in some catalog: " % (lfn))
      else:
        self.log.error("Failed to remove file found in the catalog", "%s %s" % (lfn, reason))
        realFailure = True
    if realFailure:
      return S_ERROR("Failed to remove all files found in the catalog")
    return S_OK()
示例#9
0
  def cleanCatalogContents( self, directory ):
    """ wipe out everything from catalog under folder :directory:

    :param self: self reference
    :params str directory: folder name
    """
    self.log.verbose("Cleaning Catalog contents")
    res = self.__getCatalogDirectoryContents( [directory] )
    if not res['OK']:
      return res
    filesFound = res['Value']
    if not filesFound:
      self.log.info( "No files are registered in the catalog directory %s" % directory )
      return S_OK()
    self.log.info( "Attempting to remove %d possible remnants from the catalog and storage" % len( filesFound ) )

    # Executing with shifter proxy
    gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
    res = DataManager().removeFile( filesFound, force = True )
    gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )

    if not res['OK']:
      return res
    realFailure = False
    for lfn, reason in res['Value']['Failed'].items():
      if "File does not exist" in str( reason ):
        self.log.warn( "File %s not found in some catalog: " % ( lfn ) )
      else:
        self.log.error( "Failed to remove file found in the catalog", "%s %s" % ( lfn, reason ) )
        realFailure = True
    if realFailure:
      return S_ERROR( "Failed to remove all files found in the catalog" )
    return S_OK()
示例#10
0
文件: Proxy.py 项目: petricm/DIRAC
  def wrapped_fcn( *args, **kwargs ):

    userName = kwargs.pop( 'proxyUserName', '' )
    userDN = kwargs.pop( 'proxyUserDN', '' )
    userGroup = kwargs.pop( 'proxyUserGroup', '' )
    vomsFlag = kwargs.pop( 'proxyWithVOMS', True )
    proxyFilePath = kwargs.pop( 'proxyFilePath', False )

    if ( userName or userDN ) and userGroup:

      # Setup user proxy
      originalUserProxy = os.environ.get( 'X509_USER_PROXY' )
      if not userDN:
        result = getDNForUsername( userName )
        if not result[ 'OK' ]:
          return result
        userDN = result[ 'Value' ][0]
      vomsAttr = ''
      if vomsFlag:
        vomsAttr = getVOMSAttributeForGroup( userGroup )

      if vomsAttr:
        result = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup,
                                                        requiredVOMSAttribute = vomsAttr,
                                                        filePath = proxyFilePath,
                                                        requiredTimeLeft = 3600,
                                                        cacheTime =  3600 )
      else:
        result = gProxyManager.downloadProxyToFile( userDN, userGroup,
                                                    filePath = proxyFilePath,
                                                    requiredTimeLeft = 3600,
                                                    cacheTime =  3600 )

      if not result['OK']:
        gLogger.warn( "Can't download proxy to file", result['Message'] )
        return result

      proxyFile = result['Value']
      os.environ['X509_USER_PROXY'] = proxyFile

      # Check if the caller is executing with the host certificate
      useServerCertificate = gConfig.useServerCertificate()
      if useServerCertificate:
        gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )

      try:
        resultFcn = fcn( *args, **kwargs )
      except Exception, x:
        resultFcn = S_ERROR( "Exception: %s" % str( x ) )

      # Restore the default host certificate usage if necessary
      if useServerCertificate:
        gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
      if originalUserProxy:
        os.environ['X509_USER_PROXY'] = originalUserProxy
      else:
        os.environ.pop( 'X509_USER_PROXY' )

      return resultFcn
示例#11
0
文件: Proxy.py 项目: yujikato/DIRAC
    def wrapped_fcn(*args, **kwargs):

        userName = kwargs.pop("proxyUserName", "")
        userGroup = kwargs.pop("proxyUserGroup", "")
        vomsFlag = kwargs.pop("proxyWithVOMS", True)
        proxyFilePath = kwargs.pop("proxyFilePath", False)

        if userName and userGroup:

            # Setup user proxy
            originalUserProxy = os.environ.get("X509_USER_PROXY")
            result = getDNForUsername(userName)
            if not result["OK"]:
                return result
            userDN = result["Value"][0]
            vomsAttr = ""
            if vomsFlag:
                vomsAttr = getVOMSAttributeForGroup(userGroup)

            if vomsAttr:
                result = gProxyManager.downloadVOMSProxyToFile(
                    userDN,
                    userGroup,
                    requiredVOMSAttribute=vomsAttr,
                    filePath=proxyFilePath,
                    requiredTimeLeft=3600,
                    cacheTime=3600,
                )
            else:
                result = gProxyManager.downloadProxyToFile(
                    userDN, userGroup, filePath=proxyFilePath, requiredTimeLeft=3600, cacheTime=3600
                )

            if not result["OK"]:
                return result

            proxyFile = result["Value"]
            os.environ["X509_USER_PROXY"] = proxyFile

            # Check if the caller is executing with the host certificate
            useServerCertificate = gConfig.useServerCertificate()
            if useServerCertificate:
                gConfigurationData.setOptionInCFG("/DIRAC/Security/UseServerCertificate", "false")

            try:
                resultFcn = fcn(*args, **kwargs)
            except Exception, x:
                resultFcn = S_ERROR("Exception: %s" % str(x))

            # Restore the default host certificate usage if necessary
            if useServerCertificate:
                gConfigurationData.setOptionInCFG("/DIRAC/Security/UseServerCertificate", "true")
            if originalUserProxy:
                os.environ["X509_USER_PROXY"] = originalUserProxy
            else:
                os.environ.pop("X509_USER_PROXY")

            return resultFcn
示例#12
0
    def wrapped_fcn(*args, **kwargs):

        userName = kwargs.pop('proxyUserName', '')
        userDN = kwargs.pop('proxyUserDN', '')
        userGroup = kwargs.pop('proxyUserGroup', '')
        vomsFlag = kwargs.pop('proxyWithVOMS', True)
        proxyFilePath = kwargs.pop('proxyFilePath', False)

        if (userName or userDN) and userGroup:

            # Setup user proxy
            originalUserProxy = os.environ.get('X509_USER_PROXY')
            if userDN:
                userDNs = [userDN]
            else:
                result = getDNForUsername(userName)
                if not result['OK']:
                    return result
                userDNs = result[
                    'Value']  # a same user may have more than one DN
            vomsAttr = ''
            if vomsFlag:
                vomsAttr = getVOMSAttributeForGroup(userGroup)

            result = getProxy(userDNs, userGroup, vomsAttr, proxyFilePath)

            if not result['OK']:
                return result

            proxyFile = result['Value']
            os.environ['X509_USER_PROXY'] = proxyFile

            # Check if the caller is executing with the host certificate
            useServerCertificate = gConfig.useServerCertificate()
            if useServerCertificate:
                gConfigurationData.setOptionInCFG(
                    '/DIRAC/Security/UseServerCertificate', 'false')

            try:
                return fcn(*args, **kwargs)
            except Exception as lException:  #pylint: disable=broad-except
                value = ','.join([str(arg) for arg in lException.args])
                exceptType = lException.__class__.__name__
                return S_ERROR("Exception - %s: %s" % (exceptType, value))
            finally:
                # Restore the default host certificate usage if necessary
                if useServerCertificate:
                    gConfigurationData.setOptionInCFG(
                        '/DIRAC/Security/UseServerCertificate', 'true')
                if originalUserProxy:
                    os.environ['X509_USER_PROXY'] = originalUserProxy
                else:
                    os.environ.pop('X509_USER_PROXY')

        else:
            # No proxy substitution requested
            return fcn(*args, **kwargs)
示例#13
0
  def wrapped_fcn( *args, **kwargs ):

    userName = kwargs.pop( 'proxyUserName', '' )
    userGroup = kwargs.pop( 'proxyUserGroup', '' )
    vomsFlag = kwargs.pop( 'proxyWithVOMS', True )
    proxyFilePath = kwargs.pop( 'proxyFilePath', False )

    if userName and userGroup:

      # Setup user proxy
      originalUserProxy = os.environ.get( 'X509_USER_PROXY' )
      result = getDNForUsername( userName )
      if not result[ 'OK' ]:
        return result
      userDN = result[ 'Value' ][0]
      vomsAttr = ''
      if vomsFlag:
        vomsAttr = getVOMSAttributeForGroup( userGroup )

      if vomsAttr:
        result = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup,
                                                        requiredVOMSAttribute = vomsAttr,
                                                        filePath = proxyFilePath,
                                                        requiredTimeLeft = 3600,
                                                        cacheTime =  3600 )
      else:
        result = gProxyManager.downloadProxyToFile( userDN, userGroup,
                                                    filePath = proxyFilePath,
                                                    requiredTimeLeft = 3600,
                                                    cacheTime =  3600 )

      if not result['OK']:
        return result

      proxyFile = result['Value']
      os.environ['X509_USER_PROXY'] = proxyFile

      # Check if the caller is executing with the host certificate
      useServerCertificate = gConfig.useServerCertificate()
      if useServerCertificate:
        gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )

      try:
        resultFcn = fcn( *args, **kwargs )
      except Exception, x:
        resultFcn = S_ERROR( "Exception: %s" % str( x ) )

      # Restore the default host certificate usage if necessary
      if useServerCertificate:
        gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
      if originalUserProxy:
        os.environ['X509_USER_PROXY'] = originalUserProxy
      else:
        os.environ.pop( 'X509_USER_PROXY' )

      return resultFcn
示例#14
0
  def execute( self ):
    """ one cycle execution """

    # Don't use the server certificate otherwise the DFC wont let us write
    gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )


    log = gLogger.getSubLogger( "execute" )
    # # reset FTSPlacement if expired
    now = datetime.datetime.now()
    if now > self.__ftsPlacementValidStamp:
      log.info( "resetting expired FTS placement..." )
      resetFTSPlacement = self.resetFTSPlacement()
      if not resetFTSPlacement["OK"]:
        log.error( "FTSPlacement recreation error:" , resetFTSPlacement["Message"] )
        return resetFTSPlacement
      self.__ftsPlacementValidStamp = now + datetime.timedelta( seconds = self.FTSPLACEMENT_REFRESH )

    requestIDs = self.requestClient().getRequestIDsList( statusList = [ "Scheduled" ], limit = self.MAX_REQUESTS )
    if not requestIDs["OK"]:
      log.error( "unable to read scheduled request ids" , requestIDs["Message"] )
      return requestIDs
    if not requestIDs["Value"]:
      requestIDs = []
    else:
      requestIDs = [ req[0] for req in requestIDs["Value"] if req[0] not in self.__reqCache ]
    requestIDs += self.__reqCache.keys()

    if not requestIDs:
      log.info( "no 'Scheduled' requests to process" )
      return S_OK()

    log.info( "found %s requests to process:" % len( requestIDs ) )
    log.info( " => from internal cache: %s" % ( len( self.__reqCache ) ) )
    log.info( " =>   new read from RMS: %s" % ( len( requestIDs ) - len( self.__reqCache ) ) )

    for requestID in requestIDs:
      request = self.getRequest( requestID )
      if not request["OK"]:
        log.error( "Error getting request", "%s: %s" % ( requestID, request["Message"] ) )
        continue
      request = request["Value"]
      sTJId = request.RequestID
      while True:
        queue = self.threadPool().generateJobAndQueueIt( self.processRequest,
                                                         args = ( request, ),
                                                         sTJId = sTJId )
        if queue["OK"]:
          log.info( "Request enqueued for execution", sTJId )
          gMonitor.addMark( "RequestsAtt", 1 )
          break
        time.sleep( 1 )

    # # process all results
    self.threadPool().processAllResults()
    return S_OK()
示例#15
0
  def execute( self ):
    """ one cycle execution """

    # Don't use the server certificate otherwise the DFC wont let us write
    gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )


    log = gLogger.getSubLogger( "execute" )
    # # reset FTSPlacement if expired
    now = datetime.datetime.now()
    if now > self.__ftsPlacementValidStamp:
      log.info( "resetting expired FTS placement..." )
      resetFTSPlacement = self.resetFTSPlacement()
      if not resetFTSPlacement["OK"]:
        log.error( "FTSPlacement recreation error:" , resetFTSPlacement["Message"] )
        return resetFTSPlacement
      self.__ftsPlacementValidStamp = now + datetime.timedelta( seconds = self.FTSPLACEMENT_REFRESH )

    requestIDs = self.requestClient().getRequestIDsList( statusList = [ "Scheduled" ], limit = self.MAX_REQUESTS )
    if not requestIDs["OK"]:
      log.error( "unable to read scheduled request ids" , requestIDs["Message"] )
      return requestIDs
    if not requestIDs["Value"]:
      requestIDs = []
    else:
      requestIDs = [ req[0] for req in requestIDs["Value"] if req[0] not in self.__reqCache ]
    requestIDs += self.__reqCache.keys()

    if not requestIDs:
      log.info( "no 'Scheduled' requests to process" )
      return S_OK()

    log.info( "found %s requests to process:" % len( requestIDs ) )
    log.info( " => from internal cache: %s" % ( len( self.__reqCache ) ) )
    log.info( " =>   new read from RMS: %s" % ( len( requestIDs ) - len( self.__reqCache ) ) )

    for requestID in requestIDs:
      request = self.getRequest( requestID )
      if not request["OK"]:
        log.error( "Error getting request", "%s: %s" % ( requestID, request["Message"] ) )
        continue
      request = request["Value"]
      sTJId = request.RequestID
      while True:
        queue = self.threadPool().generateJobAndQueueIt( self.processRequest,
                                                         args = ( request, ),
                                                         sTJId = sTJId )
        if queue["OK"]:
          log.info( "Request enqueued for execution", sTJId )
          gMonitor.addMark( "RequestsAtt", 1 )
          break
        time.sleep( 1 )

    # # process all results
    self.threadPool().processAllResults()
    return S_OK()
示例#16
0
def test_return_credential_are_equals(UseServerCertificate):
    """
    Check if certificates sended AND extraction have same comportement is DISET and HTTPS
    """
    gConfigurationData.setOptionInCFG("/DIRAC/Security/UseServerCertificate", UseServerCertificate)

    (repTornado, repDirac) = get_all_returnedValues()

    # Service returns credentials
    assert repDirac["Value"] == repTornado["Value"]
示例#17
0
文件: Proxy.py 项目: ahaupt/DIRAC
  def wrapped_fcn( *args, **kwargs ):

    userName = kwargs.pop( 'proxyUserName', '' )
    userDN = kwargs.pop( 'proxyUserDN', '' )
    userGroup = kwargs.pop( 'proxyUserGroup', '' )
    vomsFlag = kwargs.pop( 'proxyWithVOMS', True )
    proxyFilePath = kwargs.pop( 'proxyFilePath', False )

    if ( userName or userDN ) and userGroup:

      # Setup user proxy
      originalUserProxy = os.environ.get( 'X509_USER_PROXY' )
      if userDN:
        userDNs = [userDN]
      else:
        result = getDNForUsername( userName )
        if not result[ 'OK' ]:
          return result
        userDNs = result['Value'] # a same user may have more than one DN
      vomsAttr = ''
      if vomsFlag:
        vomsAttr = getVOMSAttributeForGroup( userGroup )

      result = getProxy(userDNs, userGroup, vomsAttr, proxyFilePath)

      if not result['OK']:
        return result

      proxyFile = result['Value']
      os.environ['X509_USER_PROXY'] = proxyFile

      # Check if the caller is executing with the host certificate
      useServerCertificate = gConfig.useServerCertificate()
      if useServerCertificate:
        gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )

      try:
        return fcn( *args, **kwargs )
      except Exception as lException:
        value = ','.join( [str( arg ) for arg in lException.args] )
        exceptType = lException.__class__.__name__
        return S_ERROR( "Exception - %s: %s" % ( exceptType, value ) )
      finally:
        # Restore the default host certificate usage if necessary
        if useServerCertificate:
          gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
        if originalUserProxy:
          os.environ['X509_USER_PROXY'] = originalUserProxy
        else:
          os.environ.pop( 'X509_USER_PROXY' )

    else:
      # No proxy substitution requested
      return fcn( *args, **kwargs )
示例#18
0
def main():
  with open(os.devnull, 'w') as redirectStdout, open(os.devnull, 'w') as redirectStderr:
    from DIRAC import gLogger
    from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
    gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true')
    gLogger.setLevel('FATAL')
    from DIRAC.Core.DISET.RPCClient import RPCClient

    rpc = RPCClient('dips://localhost:%s' % sys.argv[1])
    res = rpc.ping()
    time.sleep(0.1)
    sys.exit(0 if res['OK'] else 1)
示例#19
0
    def cleanMetadataCatalogFiles(self, transID):
        """ clean the metadata using BKK and Data Manager. This method is a replacement of the one from base class

    :param self: self reference
    :param int transID: transformation ID
    """
        res = self.bkClient.getProductionFiles(transID, 'ALL', 'Yes')
        if not res['OK']:
            return res
        bkMetadata = res['Value']
        fileToRemove = []
        yesReplica = []
        self.log.info(
            "Found a total of %d files in the BK for transformation %d" %
            (len(bkMetadata), transID))
        for lfn, metadata in bkMetadata.iteritems():
            if metadata['FileType'] != 'LOG':
                fileToRemove.append(lfn)
                if metadata['GotReplica'] == 'Yes':
                    yesReplica.append(lfn)
        if fileToRemove:
            self.log.info(
                "Attempting to remove %d possible remnants from the catalog and storage"
                % len(fileToRemove))
            # Executing with shifter proxy
            gConfigurationData.setOptionInCFG(
                '/DIRAC/Security/UseServerCertificate', 'false')
            res = DataManager().removeFile(fileToRemove, force=True)
            gConfigurationData.setOptionInCFG(
                '/DIRAC/Security/UseServerCertificate', 'true')
            if not res['OK']:
                return res
            for lfn, reason in res['Value']['Failed'].iteritems():
                self.log.error("Failed to remove file found in BK",
                               "%s %s" % (lfn, reason))
            if res['Value']['Failed']:
                return S_ERROR("Failed to remove all files found in the BK")
            if yesReplica:
                self.log.info(
                    "Ensuring that %d files are removed from the BK" %
                    (len(yesReplica)))
                res = FileCatalog(
                    catalogs=['BookkeepingDB']).removeFile(yesReplica)
                if not res['OK']:
                    return res
                for lfn, reason in res['Value']['Failed'].iteritems():
                    self.log.error("Failed to remove file from BK",
                                   "%s %s" % (lfn, reason))
                if res['Value']['Failed']:
                    return S_ERROR("Failed to remove all files from the BK")
        self.log.info("Successfully removed all files found in the BK")
        return S_OK()
示例#20
0
def _restoreProxyState(originalUserProxy=None, useServerCertificate=None, executionLock=None):
  """Restore the default host certificate and proxy state if necessary.

  Parameters like the output from `_putProxy`.
  """
  if useServerCertificate:
    gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true')
  if originalUserProxy:
    os.environ['X509_USER_PROXY'] = originalUserProxy
  else:
    os.environ.pop('X509_USER_PROXY', None)
  if executionLock:
    executionLock.release()
示例#21
0
文件: Proxy.py 项目: DIRACGrid/DIRAC
def _restoreProxyState(originalUserProxy=None, useServerCertificate=None, executionLock=None):
  """Restore the default host certificate and proxy state if necessary.

  Parameters like the output from `_putProxy`.
  """
  if useServerCertificate:
    gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true')
  if originalUserProxy:
    os.environ['X509_USER_PROXY'] = originalUserProxy
  else:
    os.environ.pop('X509_USER_PROXY', None)
  if executionLock:
    executionLock.release()
def main():
    with open(os.devnull, "w") as redirectStdout, open(os.devnull, "w") as redirectStderr:
        from DIRAC import gLogger
        from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData

        gConfigurationData.setOptionInCFG("/DIRAC/Security/UseServerCertificate", "true")
        gLogger.setLevel("FATAL")
        from DIRAC.Core.Base.Client import Client

        rpc = Client(url="dips://localhost:%s" % sys.argv[1])
        res = rpc.ping()
        time.sleep(0.1)
        sys.exit(0 if res["OK"] else 1)
示例#23
0
    def cleanOutputs(self, jobInfo):
        """remove all job outputs"""
        if len(jobInfo.outputFiles) == 0:
            return
        descendants = self.__findAllDescendants(jobInfo.outputFiles)
        existingOutputFiles = [
            lfn for lfn, status in izip_longest(jobInfo.outputFiles,
                                                jobInfo.outputFileStatus)
            if status == "Exists"
        ]
        filesToDelete = existingOutputFiles + descendants

        if not filesToDelete:
            return

        if not self.enabled:
            self.log.notice("Would have removed these files: \n +++ %s " %
                            "\n +++ ".join(filesToDelete))
            return
        self.log.notice("Remove these files: \n +++ %s " %
                        "\n +++ ".join(filesToDelete))

        errorReasons = {}
        successfullyRemoved = 0

        for lfnList in breakListIntoChunks(filesToDelete, 200):
            ## this is needed to remove the file with the Shifter credentials and not with the server credentials
            gConfigurationData.setOptionInCFG(
                '/DIRAC/Security/UseServerCertificate', 'false')
            result = DataManager().removeFile(lfnList)
            gConfigurationData.setOptionInCFG(
                '/DIRAC/Security/UseServerCertificate', 'true')
            if not result['OK']:
                self.log.error("Failed to remove LFNs", result['Message'])
                raise RuntimeError("Failed to remove LFNs: %s" %
                                   result['Message'])
            for lfn, err in result['Value']['Failed'].items():
                reason = str(err)
                if reason not in errorReasons.keys():
                    errorReasons[reason] = []
                errorReasons[reason].append(lfn)
            successfullyRemoved += len(result['Value']['Successful'].keys())

        for reason, lfns in errorReasons.items():
            self.log.error("Failed to remove %d files with error: %s" %
                           (len(lfns), reason))
        self.log.notice("Successfully removed %d files" % successfullyRemoved)
示例#24
0
class ForwardDISET(OperationHandlerBase):
    """
  .. class:: ForwardDISET

  functor forwarding DISET operations
  """
    def __init__(self, operation=None, csPath=None):
        """ c'tor

    :param Operation operation: an Operation instance
    :param str csPath: CS path for this handler
    """
        # # call base class c'tor
        OperationHandlerBase.__init__(self, operation, csPath)

    def __call__(self):
        """ execute RPC stub """
        # # decode arguments
        try:
            decode, length = DEncode.decode(self.operation.Arguments)
            self.log.debug("decoded len=%s val=%s" % (length, decode))
        except ValueError, error:
            self.log.exception(error)
            self.operation.Error = str(error)
            self.operation.Status = "Failed"
            return S_ERROR(str(error))

        # ForwardDiset is supposed to be used with a host certificate
        useServerCertificate = gConfig.useServerCertificate()
        gConfigurationData.setOptionInCFG(
            '/DIRAC/Security/UseServerCertificate', 'true')
        forward = executeRPCStub(decode)
        if not useServerCertificate:
            gConfigurationData.setOptionInCFG(
                '/DIRAC/Security/UseServerCertificate', 'false')

        if not forward["OK"]:
            self.log.error(
                "unable to execute operation",
                "'%s' : %s" % (self.operation.Type, forward["Message"]))
            self.operation.Error = forward["Message"]
            return forward
        self.log.info("DISET forwarding done")
        self.operation.Status = "Done"
        return S_OK()
示例#25
0
    def __removeEmptyDir(self, dirPath, useOwnerProxy=True):
        ''' unlink empty folder :dirPath: '''
        from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
        if len(List.fromChar(dirPath, "/")) < self.__keepDirLevels:
            return S_OK()

        if useOwnerProxy:
            result = self.__getOwnerProxy(dirPath)
            if not result['OK']:
                if 'Proxy not available' not in result['Message']:
                    self.log.error(result['Message'])
                return result

            upFile = result['Value']
            prevProxyEnv = os.environ['X509_USER_PROXY']
            os.environ['X509_USER_PROXY'] = upFile
        try:
            gConfigurationData.setOptionInCFG(
                '/DIRAC/Security/UseServerCertificate', 'false')
            # res = self.catalog.removeDirectory( dirPath )
            res = self.catalog.writeCatalogs[0][1].removeDirectory(dirPath)
            if not res['OK']:
                self.log.error(
                    "Error removing empty directory from File Catalog.",
                    res['Message'])
                return res
            elif dirPath in res['Value']['Failed']:
                self.log.error(
                    "Failed to remove empty directory from File Catalog.",
                    res['Value']['Failed'][dirPath])
                self.log.debug(str(res))
                return S_ERROR(res['Value']['Failed'][dirPath])
            res = self.storageUsage.removeDirectory(dirPath)
            if not res['OK']:
                self.log.error(
                    "Failed to remove empty directory from Storage Usage database.",
                    res['Message'])
                return res
            return S_OK()
        finally:
            gConfigurationData.setOptionInCFG(
                '/DIRAC/Security/UseServerCertificate', 'true')
            if useOwnerProxy:
                os.environ['X509_USER_PROXY'] = prevProxyEnv
示例#26
0
def _putProxy(userDN=None,
              userName=None,
              userGroup=None,
              vomsFlag=None,
              proxyFilePath=None,
              executionLockFlag=False):
    """Download proxy, place in a file and populate X509_USER_PROXY environment variable.

    Parameters like `userProxy` or `executeWithUserProxy`.
    :returns: Tuple of originalUserProxy, useServerCertificate, executionLock
    """
    # Setup user proxy
    if userDN:
        userDNs = [userDN]
    else:
        result = getDNForUsername(userName)
        if not result["OK"]:
            return result
        userDNs = result["Value"]  # a same user may have more than one DN

    vomsAttr = ""
    if vomsFlag:
        vomsAttr = getVOMSAttributeForGroup(userGroup)

    result = getProxy(userDNs, userGroup, vomsAttr, proxyFilePath)

    if not result["OK"]:
        return result

    executionLock = LockRing().getLock(
        "_UseUserProxy_", recursive=True) if executionLockFlag else None
    if executionLockFlag:
        executionLock.acquire()

    os.environ["X509_USER_PROXY"], originalUserProxy = result[
        "Value"], os.environ.get("X509_USER_PROXY")

    # Check if the caller is executing with the host certificate
    useServerCertificate = gConfig.useServerCertificate()
    if useServerCertificate:
        gConfigurationData.setOptionInCFG(
            "/DIRAC/Security/UseServerCertificate", "false")

    return S_OK((originalUserProxy, useServerCertificate, executionLock))
示例#27
0
文件: Proxy.py 项目: DIRACGrid/DIRAC
  def wrapped_fcn( *args, **kwargs ):

    # Get the lock and acquire it
    executionLock = LockRing().getLock( '_UseUserProxy_', recursive = True )
    executionLock.acquire()

    # Check if the caller is executing with the host certificate
    useServerCertificate = gConfig.useServerCertificate()
    if useServerCertificate:
      gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )

    try:
      return fcn( *args, **kwargs )
    except Exception as lException:  # pylint: disable=broad-except
      value = ','.join( [str( arg ) for arg in lException.args] )
      exceptType = lException.__class__.__name__
      return S_ERROR( "Exception - %s: %s" % ( exceptType, value ) )
    finally:
      # Restore the default host certificate usage if necessary
      if useServerCertificate:
        gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
      # release the lock
      executionLock.release()
  def cleanMetadataCatalogFiles(self, transID):
    """ wipe out files from catalog """
    res = self.metadataClient.findFilesByMetadata({self.transfidmeta: transID})
    if not res['OK']:
      return res
    fileToRemove = res['Value']
    if not fileToRemove:
      self.log.info('No files found for transID %s' % transID)
      return S_OK()

    # Executing with shifter proxy
    gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'false')
    res = DataManager().removeFile(fileToRemove, force=True)
    gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true')

    if not res['OK']:
      return res
    for lfn, reason in res['Value']['Failed'].items():
      self.log.error("Failed to remove file found in metadata catalog", "%s %s" % (lfn, reason))
    if res['Value']['Failed']:
      return S_ERROR("Failed to remove all files found in the metadata catalog")
    self.log.info("Successfully removed all files found in the BK")
    return S_OK()
示例#29
0
文件: Proxy.py 项目: DIRACGrid/DIRAC
def _putProxy(userDN=None, userName=None, userGroup=None, vomsFlag=None, proxyFilePath=None, executionLockFlag=False):
  """Download proxy, place in a file and populate X509_USER_PROXY environment variable.

  Parameters like `userProxy` or `executeWithUserProxy`.
  :returns: Tuple of originalUserProxy, useServerCertificate, executionLock
  """
  # Setup user proxy
  if userDN:
    userDNs = [userDN]
  else:
    result = getDNForUsername(userName)
    if not result['OK']:
      return result
    userDNs = result['Value']  # a same user may have more than one DN

  vomsAttr = ''
  if vomsFlag:
    vomsAttr = getVOMSAttributeForGroup(userGroup)

  result = getProxy(userDNs, userGroup, vomsAttr, proxyFilePath)

  if not result['OK']:
    return result

  executionLock = LockRing().getLock('_UseUserProxy_', recursive=True) if executionLockFlag else None
  if executionLockFlag:
    executionLock.acquire()

  os.environ['X509_USER_PROXY'], originalUserProxy = result['Value'], os.environ.get('X509_USER_PROXY')

  # Check if the caller is executing with the host certificate
  useServerCertificate = gConfig.useServerCertificate()
  if useServerCertificate:
    gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'false')

  return S_OK((originalUserProxy, useServerCertificate, executionLock))
示例#30
0
def create_serverAndClient(request):
    """This function starts a server, and closes it after
    The server will use the parametrized transport type
    """

    # Reinitialize the configuration.
    # We do it here rather than at the start of the module
    # to accommodate for pytest when going through all the DIRAC tests

    gConfigurationData.localCFG = CFG()
    gConfigurationData.remoteCFG = CFG()
    gConfigurationData.mergedCFG = CFG()
    gConfigurationData.generateNewVersion()
    gConfigurationData.setOptionInCFG("/DIRAC/Security/CALocation", caLocation)
    gConfigurationData.setOptionInCFG("/DIRAC/Security/CertFile",
                                      hostCertLocation)
    gConfigurationData.setOptionInCFG("/DIRAC/Security/KeyFile",
                                      hostKeyLocation)

    testStr = request.param
    serverName, clientName = testStr.split("-")
    serverClass = transportByName(serverName)
    clientClass = transportByName(clientName)

    sr = DummyServiceReactor(serverClass, PORT_NUMBER)
    server_thread = threading.Thread(target=sr.serve)
    sr.prepare()
    server_thread.start()

    # Create the client
    clientOptions = {
        "clientMode": True,
        "proxyLocation": proxyFile,
    }
    clientTransport = clientClass(("localhost", PORT_NUMBER),
                                  bServerMode=False,
                                  **clientOptions)
    res = clientTransport.initAsClient()
    assert res["OK"], res

    yield sr, clientTransport

    clientTransport.close()
    sr.closeListeningConnections()
    server_thread.join()

    # Clean the config
    gConfigurationData.localCFG = CFG()
    gConfigurationData.remoteCFG = CFG()
    gConfigurationData.mergedCFG = CFG()
    gConfigurationData.generateNewVersion()
示例#31
0
  def __call__( self ):
    """ request processing """

    self.log.debug( "about to execute request" )
    gMonitor.addMark( "RequestAtt", 1 )

    # # setup proxy for request owner
    setupProxy = self.setupProxy()
    if not setupProxy["OK"]:
      self.log.error( setupProxy["Message"] )
      self.request.Error = setupProxy["Message"]
      return S_ERROR( 'Change proxy error' )
    shifter = setupProxy["Value"]["Shifter"]
    proxyFile = setupProxy["Value"]["ProxyFile"]

    error = None
    while self.request.Status == "Waiting":

      # # get waiting operation
      operation = self.request.getWaiting()
      if not operation["OK"]:
        self.log.error( operation["Message"] )
        return operation
      operation = operation["Value"]
      self.log.info( "executing operation #%s '%s'" % ( operation.Order, operation.Type ) )

      # # and handler for it
      handler = self.getHandler( operation )
      if not handler["OK"]:
        self.log.error( "unable to process operation %s: %s" % ( operation.Type, handler["Message"] ) )
        # gMonitor.addMark( "%s%s" % ( operation.Type, "Fail" ), 1 )
        operation.Error = handler["Message"]
        break

      handler = handler["Value"]
      # # set shifters list in the handler
      handler.shifter = shifter
      # # and execute
      pluginName = self.getPluginName( self.handlersDict.get( operation.Type ) )
      if self.standalone:
        useServerCertificate = gConfig.useServerCertificate()
      else:
        # Always use server certificates if executed within an agent
        useServerCertificate = True
      try:
        if pluginName:
          gMonitor.addMark( "%s%s" % ( pluginName, "Att" ), 1 )
        # Always use request owner proxy
        if useServerCertificate:
          gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
        exe = handler()
        if useServerCertificate:
          gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
        if not exe["OK"]:
          self.log.error( "unable to process operation %s: %s" % ( operation.Type, exe["Message"] ) )
          if pluginName:
            gMonitor.addMark( "%s%s" % ( pluginName, "Fail" ), 1 )
          gMonitor.addMark( "RequestFail", 1 )
          if self.request.JobID:
            # Check if the job exists
            monitorServer = RPCClient( "WorkloadManagement/JobMonitoring", useCertificates = True )
            res = monitorServer.getJobPrimarySummary( int( self.request.JobID ) )
            if not res["OK"]:
              self.log.error( "RequestTask: Failed to get job %d status" % self.request.JobID )
            elif not res['Value']:
              self.log.warn( "RequestTask: job %d does not exist (anymore): failed request" % self.request.JobID )
              for opFile in operation:
                opFile.Status = 'Failed'
              if operation.Status != 'Failed':
                operation.Status = 'Failed'
              self.request.Error = 'Job no longer exists'
      except Exception, error:
        self.log.exception( "hit by exception: %s" % str( error ) )
        if pluginName:
          gMonitor.addMark( "%s%s" % ( pluginName, "Fail" ), 1 )
        gMonitor.addMark( "RequestFail", 1 )
        if useServerCertificate:
          gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
        break

      # # operation status check
      if operation.Status == "Done" and pluginName:
        gMonitor.addMark( "%s%s" % ( pluginName, "OK" ), 1 )
      elif operation.Status == "Failed" and pluginName:
        gMonitor.addMark( "%s%s" % ( pluginName, "Fail" ), 1 )
      elif operation.Status in ( "Waiting", "Scheduled" ):
        # # no update for waiting or all files scheduled
        break
示例#32
0
    def __call__(self):
        """ request processing """

        self.log.debug("about to execute request")
        gMonitor.addMark("RequestAtt", 1)

        # # setup proxy for request owner
        setupProxy = self.setupProxy()
        if not setupProxy["OK"]:
            self.request.Error = setupProxy["Message"]
            if 'has no proxy registered' in setupProxy["Message"]:
                self.log.error('Request set to Failed:', setupProxy["Message"])
                # If user is no longer registered, fail the request
                for operation in self.request:
                    for opFile in operation:
                        opFile.Status = 'Failed'
                    operation.Status = 'Failed'
            else:
                self.log.error(setupProxy["Message"])
            return S_OK(self.request)
        shifter = setupProxy["Value"]["Shifter"]
        proxyFile = setupProxy["Value"]["ProxyFile"]

        error = None
        while self.request.Status == "Waiting":

            # # get waiting operation
            operation = self.request.getWaiting()
            if not operation["OK"]:
                self.log.error(operation["Message"])
                return operation
            operation = operation["Value"]
            self.log.info("executing operation #%s '%s'" %
                          (operation.Order, operation.Type))

            # # and handler for it
            handler = self.getHandler(operation)
            if not handler["OK"]:
                self.log.error("unable to process operation %s: %s" %
                               (operation.Type, handler["Message"]))
                # gMonitor.addMark( "%s%s" % ( operation.Type, "Fail" ), 1 )
                operation.Error = handler["Message"]
                break

            handler = handler["Value"]
            # # set shifters list in the handler
            handler.shifter = shifter
            # # and execute
            pluginName = self.getPluginName(
                self.handlersDict.get(operation.Type))
            if self.standalone:
                useServerCertificate = gConfig.useServerCertificate()
            else:
                # Always use server certificates if executed within an agent
                useServerCertificate = True
            try:
                if pluginName:
                    gMonitor.addMark("%s%s" % (pluginName, "Att"), 1)
                # Always use request owner proxy
                if useServerCertificate:
                    gConfigurationData.setOptionInCFG(
                        '/DIRAC/Security/UseServerCertificate', 'false')
                exe = handler()
                if useServerCertificate:
                    gConfigurationData.setOptionInCFG(
                        '/DIRAC/Security/UseServerCertificate', 'true')
                if not exe["OK"]:
                    self.log.error("unable to process operation %s: %s" %
                                   (operation.Type, exe["Message"]))
                    if pluginName:
                        gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
                    gMonitor.addMark("RequestFail", 1)
                    if self.request.JobID:
                        # Check if the job exists
                        monitorServer = RPCClient(
                            "WorkloadManagement/JobMonitoring",
                            useCertificates=True)
                        res = monitorServer.getJobPrimarySummary(
                            int(self.request.JobID))
                        if not res["OK"]:
                            self.log.error(
                                "RequestTask: Failed to get job %d status" %
                                self.request.JobID)
                        elif not res['Value']:
                            self.log.warn(
                                "RequestTask: job %d does not exist (anymore): failed request"
                                % self.request.JobID)
                            for opFile in operation:
                                opFile.Status = 'Failed'
                            if operation.Status != 'Failed':
                                operation.Status = 'Failed'
                            self.request.Error = 'Job no longer exists'
            except Exception, error:
                self.log.exception("hit by exception: %s" % str(error))
                if pluginName:
                    gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
                gMonitor.addMark("RequestFail", 1)
                if useServerCertificate:
                    gConfigurationData.setOptionInCFG(
                        '/DIRAC/Security/UseServerCertificate', 'true')
                break

            # # operation status check
            if operation.Status == "Done" and pluginName:
                gMonitor.addMark("%s%s" % (pluginName, "OK"), 1)
            elif operation.Status == "Failed" and pluginName:
                gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
            elif operation.Status in ("Waiting", "Scheduled"):
                # # no update for waiting or all files scheduled
                break
示例#33
0
  def execute(self):
    """ execution in one cycle

    """

    # Don't use the server certificate otherwise the DFC wont let us write
    gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'false')

    gMonitor.addMark("Iteration", 1)

    ############################################################
    #
    # Obtain the files which have not yet been migrated
    #
    self.log.info("Obtaining un-migrated files.")
    res = self.rawIntegrityDB.getUnmigratedFiles()
    if not res['OK']:
      errStr = "Failed to obtain un-migrated files."
      self.log.error(errStr, res['Message'])
      return S_OK()

    # allUnmigratedFilesMeta contain all the files that are not yet
    # migrated (not copied, not registered, not removed), as well as their metadata
    allUnmigratedFilesMeta = res['Value']
    self.log.info("Obtained %s un-migrated files." % len(allUnmigratedFilesMeta))
    if not allUnmigratedFilesMeta:
      return S_OK()

    # activeFiles contains the files that are not yet copied
    activeFiles = {}
    # copiedFiles contains files that are copied but not yet registered
    copiedFiles = {}
    # registeredFiles contains files that are copied, registered, but not removed from source
    registeredFiles = {}

    # Assign them
    for lfn, lfnMetadata in allUnmigratedFilesMeta.iteritems():
      status = lfnMetadata.pop('Status')
      if status == 'Active':
        activeFiles[lfn] = lfnMetadata
      elif status == 'Copied':
        copiedFiles[lfn] = lfnMetadata
      elif status == 'Registered':
        registeredFiles[lfn] = lfnMetadata

    gMonitor.addMark("WaitingFiles", len(activeFiles))
    totalSize = 0
    for lfn, fileDict in activeFiles.iteritems():
      totalSize += int(fileDict['Size'])
      # gMonitor.addMark("TimeInQueue", (fileDict['WaitTime'] / 60))
    gMonitor.addMark("WaitSize", (totalSize / (1024 * 1024 * 1024.0)))

    ############################################################
    #
    # Checking newly copied files
    #

    # Get the list of lfns properly copied and not copied
    filesNewlyCopied, filesNotYetCopied = self.getNewlyCopiedFiles(activeFiles)

    ####################################################
    #
    # Registering copied files
    #
    ####################################################

    filesNewlyRegistered = self.registerCopiedFiles(filesNewlyCopied, copiedFiles,
                                                    allUnmigratedFilesMeta)

    ####################################################
    #
    # Performing the removal from the online storage
    #
    ####################################################
    filesNewlyRemoved = self.removeRegisteredFiles(filesNewlyRegistered, registeredFiles,
                                                   allUnmigratedFilesMeta)

    # Doing some accounting

    migratedSize = sum(allUnmigratedFilesMeta[lfn]['Size'] for lfn in filesNewlyRemoved)

    # The number of files that we failed at migrating
    # is the number of files at the beginning, minus the one we processed completely
    # minus those that are not yet copied
    failedMigrated = len(allUnmigratedFilesMeta) - len(filesNewlyRemoved) - len(filesNotYetCopied)

    res = self.rawIntegrityDB.setLastMonitorTime()
    migratedSizeGB = migratedSize / (1024 * 1024 * 1024.0)
    gMonitor.addMark("TotMigratedSize", migratedSizeGB)
    gMonitor.addMark("NewlyMigrated", len(filesNewlyRemoved))
    gMonitor.addMark("TotMigrated", len(filesNewlyRemoved))
    gMonitor.addMark("FailedMigrated", failedMigrated)
    gMonitor.addMark("TotFailMigrated", failedMigrated)

    return S_OK()
示例#34
0
 def __setOptionValue( self, optionPath, value ):
   gConfigurationData.setOptionInCFG( self.__getAbsolutePath( optionPath ),
                                      str( value ) )
示例#35
0
    def __call__(self):
        """request processing"""

        self.log.debug("about to execute request")
        if not self.rmsMonitoring:
            gMonitor.addMark("RequestAtt", 1)

        # # setup proxy for request owner
        setupProxy = self.setupProxy()
        if not setupProxy["OK"]:
            userSuspended = "User is currently suspended"
            self.request.Error = setupProxy["Message"]
            # In case the user does not have proxy
            if DErrno.cmpError(setupProxy, DErrno.EPROXYFIND):
                self.log.error("Error setting proxy. Request set to Failed:",
                               setupProxy["Message"])
                # If user is no longer registered, fail the request
                for operation in self.request:
                    for opFile in operation:
                        opFile.Status = "Failed"
                    operation.Status = "Failed"
            elif userSuspended in setupProxy["Message"]:
                # If user is suspended, wait for a long time
                self.request.delayNextExecution(6 * 60)
                self.request.Error = userSuspended
                self.log.error("Error setting proxy: " + userSuspended,
                               self.request.OwnerDN)
            else:
                self.log.error("Error setting proxy", setupProxy["Message"])
            return S_OK(self.request)
        shifter = setupProxy["Value"]["Shifter"]

        error = None

        while self.request.Status == "Waiting":

            # # get waiting operation
            operation = self.request.getWaiting()
            if not operation["OK"]:
                self.log.error("Cannot get waiting operation",
                               operation["Message"])
                return operation
            operation = operation["Value"]
            self.log.info("executing operation", "%s" % operation.Type)

            # # and handler for it
            handler = self.getHandler(operation)
            if not handler["OK"]:
                self.log.error("Unable to process operation",
                               "%s: %s" % (operation.Type, handler["Message"]))
                # gMonitor.addMark( "%s%s" % ( operation.Type, "Fail" ), 1 )
                operation.Error = handler["Message"]
                break

            handler = handler["Value"]
            # # set shifters list in the handler
            handler.shifter = shifter
            # set rmsMonitoring flag for the RequestOperation
            handler.rmsMonitoring = self.rmsMonitoring
            # # and execute
            pluginName = self.getPluginName(
                self.handlersDict.get(operation.Type))
            if self.standalone:
                useServerCertificate = gConfig.useServerCertificate()
            else:
                # Always use server certificates if executed within an agent
                useServerCertificate = True
            try:
                if pluginName:
                    if self.rmsMonitoring:
                        self.rmsMonitoringReporter.addRecord({
                            "timestamp":
                            int(Time.toEpoch()),
                            "host":
                            Network.getFQDN(),
                            "objectType":
                            "Operation",
                            "operationType":
                            pluginName,
                            "objectID":
                            operation.OperationID,
                            "parentID":
                            operation.RequestID,
                            "status":
                            "Attempted",
                            "nbObject":
                            1,
                        })
                    else:
                        gMonitor.addMark("%s%s" % (pluginName, "Att"), 1)
                # Always use request owner proxy
                if useServerCertificate:
                    gConfigurationData.setOptionInCFG(
                        "/DIRAC/Security/UseServerCertificate", "false")
                exe = handler()
                if useServerCertificate:
                    gConfigurationData.setOptionInCFG(
                        "/DIRAC/Security/UseServerCertificate", "true")
                if not exe["OK"]:
                    self.log.error("unable to process operation",
                                   "%s: %s" % (operation.Type, exe["Message"]))
                    if pluginName:
                        if self.rmsMonitoring:
                            self.rmsMonitoringReporter.addRecord({
                                "timestamp":
                                int(Time.toEpoch()),
                                "host":
                                Network.getFQDN(),
                                "objectType":
                                "Operation",
                                "operationType":
                                pluginName,
                                "objectID":
                                operation.OperationID,
                                "parentID":
                                operation.RequestID,
                                "status":
                                "Failed",
                                "nbObject":
                                1,
                            })
                        else:
                            gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
                    if self.rmsMonitoring:
                        self.rmsMonitoringReporter.addRecord({
                            "timestamp":
                            int(Time.toEpoch()),
                            "host":
                            Network.getFQDN(),
                            "objectType":
                            "Request",
                            "objectID":
                            operation.RequestID,
                            "status":
                            "Failed",
                            "nbObject":
                            1,
                        })
                    else:
                        gMonitor.addMark("RequestFail", 1)

                    if self.request.JobID:
                        # Check if the job exists
                        monitorServer = JobMonitoringClient(
                            useCertificates=True)
                        res = monitorServer.getJobSummary(
                            int(self.request.JobID))
                        if not res["OK"]:
                            self.log.error(
                                "RequestTask: Failed to get job status",
                                "%d" % self.request.JobID)
                        elif not res["Value"]:
                            self.log.warn(
                                "RequestTask: job does not exist (anymore): failed request",
                                "JobID: %d" % self.request.JobID,
                            )
                            for opFile in operation:
                                opFile.Status = "Failed"
                            if operation.Status != "Failed":
                                operation.Status = "Failed"
                            self.request.Error = "Job no longer exists"
            except Exception as e:
                error = str(e)
                self.log.exception("hit by exception:", "%s" % error)
                if pluginName:
                    if self.rmsMonitoring:
                        self.rmsMonitoringReporter.addRecord({
                            "timestamp":
                            int(Time.toEpoch()),
                            "host":
                            Network.getFQDN(),
                            "objectType":
                            "Operation",
                            "operationType":
                            pluginName,
                            "objectID":
                            operation.OperationID,
                            "parentID":
                            operation.RequestID,
                            "status":
                            "Failed",
                            "nbObject":
                            1,
                        })
                    else:
                        gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
                if self.rmsMonitoring:
                    self.rmsMonitoringReporter.addRecord({
                        "timestamp":
                        int(Time.toEpoch()),
                        "host":
                        Network.getFQDN(),
                        "objectType":
                        "Request",
                        "objectID":
                        operation.RequestID,
                        "status":
                        "Failed",
                        "nbObject":
                        1,
                    })
                else:
                    gMonitor.addMark("RequestFail", 1)

                if useServerCertificate:
                    gConfigurationData.setOptionInCFG(
                        "/DIRAC/Security/UseServerCertificate", "true")
                break

            # # operation status check
            if operation.Status == "Done" and pluginName:
                if self.rmsMonitoring:
                    self.rmsMonitoringReporter.addRecord({
                        "timestamp":
                        int(Time.toEpoch()),
                        "host":
                        Network.getFQDN(),
                        "objectType":
                        "Operation",
                        "operationType":
                        pluginName,
                        "objectID":
                        operation.OperationID,
                        "parentID":
                        operation.RequestID,
                        "status":
                        "Successful",
                        "nbObject":
                        1,
                    })
                else:
                    gMonitor.addMark("%s%s" % (pluginName, "OK"), 1)
            elif operation.Status == "Failed" and pluginName:
                if self.rmsMonitoring:
                    self.rmsMonitoringReporter.addRecord({
                        "timestamp":
                        int(Time.toEpoch()),
                        "host":
                        Network.getFQDN(),
                        "objectType":
                        "Operation",
                        "operationType":
                        pluginName,
                        "objectID":
                        operation.OperationID,
                        "parentID":
                        operation.RequestID,
                        "status":
                        "Failed",
                        "nbObject":
                        1,
                    })
                else:
                    gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
            elif operation.Status in ("Waiting", "Scheduled"):
                # # no update for waiting or all files scheduled
                break

        if not self.rmsMonitoring:
            gMonitor.flush()

        if error:
            return S_ERROR(error)

        # # request done?
        if self.request.Status == "Done":
            # # update request to the RequestDB
            self.log.info("Updating request status:",
                          "%s" % self.request.Status)
            update = self.updateRequest()
            if not update["OK"]:
                self.log.error("Cannot update request status",
                               update["Message"])
                return update
            self.log.info("request is done", "%s" % self.request.RequestName)
            if self.rmsMonitoring:
                self.rmsMonitoringReporter.addRecord({
                    "timestamp":
                    int(Time.toEpoch()),
                    "host":
                    Network.getFQDN(),
                    "objectType":
                    "Request",
                    "objectID":
                    getattr(self.request, "RequestID", 0),
                    "status":
                    "Successful",
                    "nbObject":
                    1,
                })
            else:
                gMonitor.addMark("RequestOK", 1)
            # # and there is a job waiting for it? finalize!
            if self.request.JobID:
                attempts = 0
                while True:
                    finalizeRequest = self.requestClient.finalizeRequest(
                        self.request.RequestID,
                        self.request.JobID  # pylint: disable=no-member
                    )
                    if not finalizeRequest["OK"]:
                        if not attempts:
                            self.log.error(
                                "unable to finalize request, will retry",
                                "ReqName %s:%s" % (self.request.RequestName,
                                                   finalizeRequest["Message"]),
                            )
                        self.log.debug("Waiting 10 seconds")
                        attempts += 1
                        if attempts == 10:
                            self.log.error("Giving up finalize request")
                            return S_ERROR("Could not finalize request")

                        time.sleep(10)

                    else:
                        self.log.info(
                            "request is finalized",
                            "ReqName %s %s" % (self.request.RequestName,
                                               (" after %d attempts" %
                                                attempts) if attempts else ""),
                        )
                        break

        # Commit all the data to the ES Backend
        if self.rmsMonitoring:
            self.rmsMonitoringReporter.commit()
        # Request will be updated by the callBack method
        self.log.verbose("RequestTasks exiting",
                         "request %s" % self.request.Status)
        return S_OK(self.request)
示例#36
0
  def wrapped_fcn( *args, **kwargs ):

    userName = kwargs.pop( 'proxyUserName', '' )
    userDN = kwargs.pop( 'proxyUserDN', '' )
    userGroup = kwargs.pop( 'proxyUserGroup', '' )
    vomsFlag = kwargs.pop( 'proxyWithVOMS', True )
    proxyFilePath = kwargs.pop( 'proxyFilePath', False )

    if ( userName or userDN ) and userGroup:

      # Setup user proxy
      originalUserProxy = os.environ.get( 'X509_USER_PROXY' )
      if not userDN:
        result = getDNForUsername( userName )
        if not result[ 'OK' ]:
          return result
        userDN = result[ 'Value' ][0]
      vomsAttr = ''
      if vomsFlag:
        vomsAttr = getVOMSAttributeForGroup( userGroup )

      if vomsAttr:
        result = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup,
                                                        requiredVOMSAttribute = vomsAttr,
                                                        filePath = proxyFilePath,
                                                        requiredTimeLeft = 3600,
                                                        cacheTime = 3600 )
      else:
        result = gProxyManager.downloadProxyToFile( userDN, userGroup,
                                                    filePath = proxyFilePath,
                                                    requiredTimeLeft = 3600,
                                                    cacheTime = 3600 )

      if not result['OK']:
        gLogger.warn( "Can't download proxy to file", result['Message'] )
        return result

      proxyFile = result['Value']
      os.environ['X509_USER_PROXY'] = proxyFile

      # Check if the caller is executing with the host certificate
      useServerCertificate = gConfig.useServerCertificate()
      if useServerCertificate:
        gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )

      try:
        return fcn( *args, **kwargs )
      except Exception as lException:
        value = ','.join( [str( arg ) for arg in lException.args] )
        exceptType = lException.__class__.__name__
        return S_ERROR( "Exception - %s: %s" % ( exceptType, value ) )
      finally:
        # Restore the default host certificate usage if necessary
        if useServerCertificate:
          gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
        if originalUserProxy:
          os.environ['X509_USER_PROXY'] = originalUserProxy
        else:
          os.environ.pop( 'X509_USER_PROXY' )

    else:
      # No proxy substitution requested
      return fcn( *args, **kwargs )
示例#37
0
  def __call__( self ):
    """ generic function to process one Request of a type requestType

    This method could be run in a thread.

    :param self: self reference
    :param str requestType: request type 
    :return: S_OK/S_ERROR
    """
    self.always("executing request %s" % self.requestName )

    ################################################################
    ## get ownerDN and ownerGroup
    ownerDN = self.requestObj.getAttribute( "OwnerDN" )
    if not ownerDN["OK"]:
      return ownerDN
    ownerDN = ownerDN["Value"]
    ownerGroup = self.requestObj.getAttribute( "OwnerGroup" )
    if not ownerGroup["OK"]:
      return ownerGroup
    ownerGroup = ownerGroup["Value"]
    
    ## save request owner
    self.requestOwnerDN = ownerDN if ownerDN else ""
    self.requestOwnerGroup = ownerGroup if ownerGroup else ""

    #################################################################
    ## change proxy
    ownerProxyFile = None
    if ownerDN and ownerGroup:
      ownerProxyFile = self.changeProxy( ownerDN, ownerGroup )
      if not ownerProxyFile["OK"]:
        self.error( "handleReuqest: unable to get proxy for '%s'@'%s': %s" % ( ownerDN, 
                                                                               ownerGroup, 
                                                                               ownerProxyFile["Message"] ) )
        update = self.putBackRequest( self.requestName, self.requestString )
        if not update["OK"]:
          self.error( "handleRequest: error when updating request: %s" % update["Message"] )
          return update
        return ownerProxyFile
      ownerProxyFile = ownerProxyFile["Value"]
      #self.ownerProxyFile = ownerProxyFile
      self.info( "Will execute request for '%s'@'%s' using proxy file %s" % ( ownerDN, ownerGroup, ownerProxyFile ) )
    else:
      self.info( "Will execute request for DataManager using her/his proxy")

    #################################################################
    ## execute handlers
    ret = { "OK" : False, "Message" : "" }
    useServerCert = gConfig.useServerCertificate()
    try:
      # Execute task with the owner proxy even for contacting DIRAC services
      if useServerCert:
        gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate','false')
      ret = self.handleRequest()
    finally:
      if useServerCert:
        gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate','true') 
      ## delete owner proxy
      if self.__dataManagerProxy:
        os.environ["X509_USER_PROXY"] = self.__dataManagerProxy
        if ownerProxyFile and os.path.exists( ownerProxyFile ):
          os.unlink( ownerProxyFile )
    if not ret["OK"]:
      self.error( "handleRequest: error during request processing: %s" % ret["Message"] )
      self.error( "handleRequest: will put original request back" )
      update = self.putBackRequest( self.requestName, self.requestString )
      if not update["OK"]:
        self.error( "handleRequest: error when putting back request: %s" % update["Message"] )
    ## return at least
    return ret
#!/usr/bin/env python

""" Performs a DIPS ping on a given target and exit with the return code.
It uses the local host certificate
The target is specified as ""<port>/System/Service"
The script does not print anything, and just exists with 0 in case of success,
or 1 in case of error """

__RCSID__ = "$Id$"


import sys
import os
import time

with open(os.devnull, 'w') as redirectStdout, open(os.devnull, 'w') as redirectStderr:
  from DIRAC import gLogger
  from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
  gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
  gLogger.setLevel('FATAL')
  from DIRAC.Core.DISET.RPCClient import RPCClient

  rpc = RPCClient('dips://localhost:%s'%sys.argv[1])
  res = rpc.ping()
  time.sleep(0.1)
  if not res['OK']:
    sys.exit(1)
  else:
    sys.exit(0)
示例#39
0
 def setOptionValue( self, optionPath, value ):
   """
   Set a value in the local configuration
   """
   gConfigurationData.setOptionInCFG( optionPath, value )
示例#40
0
 def setOptionValue(self, optionPath, value):
     """
 Set a value in the local configuration
 """
     gConfigurationData.setOptionInCFG(optionPath, value)
示例#41
0
  def __call__( self ):
    """ request processing """

    self.log.debug( "about to execute request" )
    gMonitor.addMark( "RequestAtt", 1 )

    # # setup proxy for request owner
    setupProxy = self.setupProxy()
    if not setupProxy["OK"]:
      self.log.error( setupProxy["Message"] )
      self.request.Error = setupProxy["Message"]
      return S_ERROR( 'Change proxy error' )
    shifter = setupProxy["Value"]["Shifter"]
    proxyFile = setupProxy["Value"]["ProxyFile"]

    error = None
    while self.request.Status == "Waiting":

      # # get waiting operation
      operation = self.request.getWaiting()
      if not operation["OK"]:
        self.log.error( operation["Message"] )
        return operation
      operation = operation["Value"]
      self.log.info( "executing operation #%s '%s'" % ( operation.Order, operation.Type ) )

      # # and handler for it
      handler = self.getHandler( operation )
      if not handler["OK"]:
        self.log.error( "unable to process operation %s: %s" % ( operation.Type, handler["Message"] ) )
        # gMonitor.addMark( "%s%s" % ( operation.Type, "Fail" ), 1 )
        operation.Error = handler["Message"]
        break

      handler = handler["Value"]
      # # set shifters list in the handler
      handler.shifter = shifter
      # # and execute
      pluginName = self.getPluginName( self.handlersDict.get( operation.Type ) )
      useServerCertificate = gConfig.useServerCertificate()
      try:
        if pluginName:
          gMonitor.addMark( "%s%s" % ( pluginName, "Att" ), 1 )
        # Always use request owner proxy
        if useServerCertificate:
          gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
        exe = handler()
        if useServerCertificate:
          gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
        if not exe["OK"]:
          self.log.error( "unable to process operation %s: %s" % ( operation.Type, exe["Message"] ) )
          if pluginName:
            gMonitor.addMark( "%s%s" % ( pluginName, "Fail" ), 1 )
          gMonitor.addMark( "RequestFail", 1 )
      except Exception, error:
        self.log.exception( "hit by exception: %s" % str( error ) )
        if pluginName:
          gMonitor.addMark( "%s%s" % ( pluginName, "Fail" ), 1 )
        gMonitor.addMark( "RequestFail", 1 )
        if useServerCertificate:
          gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
        break

      # # operation status check
      if operation.Status == "Done" and pluginName:
        gMonitor.addMark( "%s%s" % ( pluginName, "OK" ), 1 )
      elif operation.Status == "Failed" and pluginName:
        gMonitor.addMark( "%s%s" % ( pluginName, "Fail" ), 1 )
      elif operation.Status in ( "Waiting", "Scheduled" ):
        # # no update for waiting or all files scheduled
        break
示例#42
0
  def __call__( self ):
    """ generic function to process one Request of a type requestType

    This method could be run in a thread.

    :param self: self reference
    :param str requestType: request type 
    :return: S_OK/S_ERROR
    """
    self.always("executing request %s" % self.requestName )

    ################################################################
    ## get ownerDN and ownerGroup
    ownerDN = self.requestObj.getAttribute( "OwnerDN" )
    if not ownerDN["OK"]:
      return ownerDN
    ownerDN = ownerDN["Value"]
    ownerGroup = self.requestObj.getAttribute( "OwnerGroup" )
    if not ownerGroup["OK"]:
      return ownerGroup
    ownerGroup = ownerGroup["Value"]
    
    ## save request owner
    self.requestOwnerDN = ownerDN if ownerDN else ""
    self.requestOwnerGroup = ownerGroup if ownerGroup else ""

    #################################################################
    ## change proxy
    ownerProxyFile = None
    if ownerDN and ownerGroup:
      ownerProxyFile = self.changeProxy( ownerDN, ownerGroup )
      if not ownerProxyFile["OK"]:
        self.error( "handleReuqest: unable to get proxy for '%s'@'%s': %s" % ( ownerDN, 
                                                                               ownerGroup, 
                                                                               ownerProxyFile["Message"] ) )
        update = self.putBackRequest( self.requestName, self.requestString )
        if not update["OK"]:
          self.error( "handleRequest: error when updating request: %s" % update["Message"] )
          return update
        return ownerProxyFile
      ownerProxyFile = ownerProxyFile["Value"]
      #self.ownerProxyFile = ownerProxyFile
      self.info( "Will execute request for '%s'@'%s' using proxy file %s" % ( ownerDN, ownerGroup, ownerProxyFile ) )
    else:
      self.info( "Will execute request for DataManager using her/his proxy")

    #################################################################
    ## execute handlers
    ret = { "OK" : False, "Message" : "" }
    useServerCert = gConfig.useServerCertificate()
    try:
      # Execute task with the owner proxy even for contacting DIRAC services
      if useServerCert:
        gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate','false')
      ret = self.handleRequest()
    finally:
      if useServerCert:
        gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate','true') 
      ## delete owner proxy
      if self.__dataManagerProxy:
        os.environ["X509_USER_PROXY"] = self.__dataManagerProxy
        if ownerProxyFile and os.path.exists( ownerProxyFile ):
          os.unlink( ownerProxyFile )
    if not ret["OK"]:
      self.error( "handleRequest: error during request processing: %s" % ret["Message"] )
      self.error( "handleRequest: will put original request back" )
      update = self.putBackRequest( self.requestName, self.requestString )
      if not update["OK"]:
        self.error( "handleRequest: error when putting back request: %s" % update["Message"] )
    ## return at least
    return ret
示例#43
0
 def __setOptionValue(self, optionPath, value):
     gConfigurationData.setOptionInCFG(self.__getAbsolutePath(optionPath),
                                       str(value))
示例#44
0
  def __call__(self):
    """ request processing """

    self.log.debug("about to execute request")
    gMonitor.addMark("RequestAtt", 1)

    # # setup proxy for request owner
    setupProxy = self.setupProxy()
    if not setupProxy["OK"]:
      self.request.Error = setupProxy["Message"]
      if 'has no proxy registered' in setupProxy["Message"]:
        self.log.error('Request set to Failed:', setupProxy["Message"])
        # If user is no longer registered, fail the request
        for operation in self.request:
          for opFile in operation:
            opFile.Status = 'Failed'
          operation.Status = 'Failed'
      else:
        self.log.error(setupProxy["Message"])
      return S_OK(self.request)
    shifter = setupProxy["Value"]["Shifter"]
    proxyFile = setupProxy["Value"]["ProxyFile"]

    error = None
    while self.request.Status == "Waiting":

      # # get waiting operation
      operation = self.request.getWaiting()
      if not operation["OK"]:
        self.log.error(operation["Message"])
        return operation
      operation = operation["Value"]
      self.log.info("executing operation #%s '%s'" % (operation.Order, operation.Type))

      # # and handler for it
      handler = self.getHandler(operation)
      if not handler["OK"]:
        self.log.error("unable to process operation %s: %s" % (operation.Type, handler["Message"]))
        # gMonitor.addMark( "%s%s" % ( operation.Type, "Fail" ), 1 )
        operation.Error = handler["Message"]
        break

      handler = handler["Value"]
      # # set shifters list in the handler
      handler.shifter = shifter
      # # and execute
      pluginName = self.getPluginName(self.handlersDict.get(operation.Type))
      if self.standalone:
        useServerCertificate = gConfig.useServerCertificate()
      else:
        # Always use server certificates if executed within an agent
        useServerCertificate = True
      try:
        if pluginName:
          gMonitor.addMark("%s%s" % (pluginName, "Att"), 1)
        # Always use request owner proxy
        if useServerCertificate:
          gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'false')
        exe = handler()
        if useServerCertificate:
          gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true')
        if not exe["OK"]:
          self.log.error("unable to process operation %s: %s" % (operation.Type, exe["Message"]))
          if pluginName:
            gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
          gMonitor.addMark("RequestFail", 1)
          if self.request.JobID:
            # Check if the job exists
            monitorServer = RPCClient("WorkloadManagement/JobMonitoring", useCertificates=True)
            res = monitorServer.getJobPrimarySummary(int(self.request.JobID))
            if not res["OK"]:
              self.log.error("RequestTask: Failed to get job %d status" % self.request.JobID)
            elif not res['Value']:
              self.log.warn(
                  "RequestTask: job %d does not exist (anymore): failed request" %
                  self.request.JobID)
              for opFile in operation:
                opFile.Status = 'Failed'
              if operation.Status != 'Failed':
                operation.Status = 'Failed'
              self.request.Error = 'Job no longer exists'
      except Exception as error:
        self.log.exception("hit by exception: %s" % str(error))
        if pluginName:
          gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
        gMonitor.addMark("RequestFail", 1)
        if useServerCertificate:
          gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true')
        break

      # # operation status check
      if operation.Status == "Done" and pluginName:
        gMonitor.addMark("%s%s" % (pluginName, "OK"), 1)
      elif operation.Status == "Failed" and pluginName:
        gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
      elif operation.Status in ("Waiting", "Scheduled"):
        # # no update for waiting or all files scheduled
        break

    gMonitor.flush()

    if error:
      return S_ERROR(error)

    # # request done?
    if self.request.Status == "Done":
      # # update request to the RequestDB
      self.log.info('updating request with status %s' % self.request.Status)
      update = self.updateRequest()
      if not update["OK"]:
        self.log.error(update["Message"])
        return update
      self.log.info("request '%s' is done" % self.request.RequestName)
      gMonitor.addMark("RequestOK", 1)
      # # and there is a job waiting for it? finalize!
      if self.request.JobID:
        attempts = 0
        while True:
          finalizeRequest = self.requestClient.finalizeRequest(
              self.request.RequestID, self.request.JobID)  # pylint: disable=no-member
          if not finalizeRequest["OK"]:
            if not attempts:
              self.log.error(
                  "unable to finalize request %s: %s, will retry" %
                  (self.request.RequestName, finalizeRequest["Message"]))
            self.log.verbose("Waiting 10 seconds")
            attempts += 1
            if attempts == 10:
              self.log.error("giving up finalize request after %d attempts" % attempts)
              return S_ERROR('Could not finalize request')

            time.sleep(10)

          else:
            self.log.info(
                "request '%s' is finalized%s" %
                (self.request.RequestName,
                 (' after %d attempts' %
                  attempts) if attempts else ''))
            break

    # Request will be updated by the callBack method
    self.log.verbose("RequestTasks exiting, request %s" % self.request.Status)
    return S_OK(self.request)
示例#45
0
import sys
from urlparse import urlparse
from DIRAC import gLogger
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate',
                                  'true')

gLogger.setLevel('FATAL')
from DIRAC.Core.DISET.RPCClient import RPCClient

import SimpleHTTPServer
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
import SocketServer


class MyRequestHandler(BaseHTTPRequestHandler):
    def do_GET(self):
        if self.path == '/self':
            self.send_response(200)
            self.send_header('Content-type', 'text/html')
            self.end_headers()
            # Send the html message
            self.wfile.write("I am good thanks")
            return

        if not self.path.startswith('/ping'):
            self.send_error(404)
            return

        outputText = "Nothing to report"
        query = urlparse(self.path).query
示例#46
0
    def __call__(self):
        """ request processing """

        self.log.debug("about to execute request")
        gMonitor.addMark("RequestAtt", 1)

        # # setup proxy for request owner
        setupProxy = self.setupProxy()
        if not setupProxy["OK"]:
            self.request.Error = setupProxy["Message"]
            if 'has no proxy registered' in setupProxy["Message"]:
                self.log.error('Request set to Failed:', setupProxy["Message"])
                # If user is no longer registered, fail the request
                for operation in self.request:
                    for opFile in operation:
                        opFile.Status = 'Failed'
                    operation.Status = 'Failed'
            else:
                self.log.error(setupProxy["Message"])
            return S_OK(self.request)
        shifter = setupProxy["Value"]["Shifter"]
        proxyFile = setupProxy["Value"]["ProxyFile"]

        error = None
        while self.request.Status == "Waiting":

            # # get waiting operation
            operation = self.request.getWaiting()
            if not operation["OK"]:
                self.log.error(operation["Message"])
                return operation
            operation = operation["Value"]
            self.log.info("executing operation #%s '%s'" %
                          (operation.Order, operation.Type))

            # # and handler for it
            handler = self.getHandler(operation)
            if not handler["OK"]:
                self.log.error("unable to process operation %s: %s" %
                               (operation.Type, handler["Message"]))
                # gMonitor.addMark( "%s%s" % ( operation.Type, "Fail" ), 1 )
                operation.Error = handler["Message"]
                break

            handler = handler["Value"]
            # # set shifters list in the handler
            handler.shifter = shifter
            # # and execute
            pluginName = self.getPluginName(
                self.handlersDict.get(operation.Type))
            if self.standalone:
                useServerCertificate = gConfig.useServerCertificate()
            else:
                # Always use server certificates if executed within an agent
                useServerCertificate = True
            try:
                if pluginName:
                    gMonitor.addMark("%s%s" % (pluginName, "Att"), 1)
                # Always use request owner proxy
                if useServerCertificate:
                    gConfigurationData.setOptionInCFG(
                        '/DIRAC/Security/UseServerCertificate', 'false')
                exe = handler()
                if useServerCertificate:
                    gConfigurationData.setOptionInCFG(
                        '/DIRAC/Security/UseServerCertificate', 'true')
                if not exe["OK"]:
                    self.log.error("unable to process operation %s: %s" %
                                   (operation.Type, exe["Message"]))
                    if pluginName:
                        gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
                    gMonitor.addMark("RequestFail", 1)
                    if self.request.JobID:
                        # Check if the job exists
                        monitorServer = RPCClient(
                            "WorkloadManagement/JobMonitoring",
                            useCertificates=True)
                        res = monitorServer.getJobPrimarySummary(
                            int(self.request.JobID))
                        if not res["OK"]:
                            self.log.error(
                                "RequestTask: Failed to get job %d status" %
                                self.request.JobID)
                        elif not res['Value']:
                            self.log.warn(
                                "RequestTask: job %d does not exist (anymore): failed request"
                                % self.request.JobID)
                            for opFile in operation:
                                opFile.Status = 'Failed'
                            if operation.Status != 'Failed':
                                operation.Status = 'Failed'
                            self.request.Error = 'Job no longer exists'
            except Exception as error:
                self.log.exception("hit by exception: %s" % str(error))
                if pluginName:
                    gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
                gMonitor.addMark("RequestFail", 1)
                if useServerCertificate:
                    gConfigurationData.setOptionInCFG(
                        '/DIRAC/Security/UseServerCertificate', 'true')
                break

            # # operation status check
            if operation.Status == "Done" and pluginName:
                gMonitor.addMark("%s%s" % (pluginName, "OK"), 1)
            elif operation.Status == "Failed" and pluginName:
                gMonitor.addMark("%s%s" % (pluginName, "Fail"), 1)
            elif operation.Status in ("Waiting", "Scheduled"):
                # # no update for waiting or all files scheduled
                break

        gMonitor.flush()

        if error:
            return S_ERROR(error)

        # # request done?
        if self.request.Status == "Done":
            # # update request to the RequestDB
            self.log.info('updating request with status %s' %
                          self.request.Status)
            update = self.updateRequest()
            if not update["OK"]:
                self.log.error(update["Message"])
                return update
            self.log.info("request '%s' is done" % self.request.RequestName)
            gMonitor.addMark("RequestOK", 1)
            # # and there is a job waiting for it? finalize!
            if self.request.JobID:
                attempts = 0
                while True:
                    finalizeRequest = self.requestClient.finalizeRequest(
                        self.request.RequestID, self.request.JobID)  # pylint: disable=no-member
                    if not finalizeRequest["OK"]:
                        if not attempts:
                            self.log.error(
                                "unable to finalize request %s: %s, will retry"
                                % (self.request.RequestName,
                                   finalizeRequest["Message"]))
                        self.log.verbose("Waiting 10 seconds")
                        attempts += 1
                        if attempts == 10:
                            self.log.error(
                                "giving up finalize request after %d attempts"
                                % attempts)
                            return S_ERROR('Could not finalize request')

                        time.sleep(10)

                    else:
                        self.log.info("request '%s' is finalized%s" %
                                      (self.request.RequestName,
                                       (' after %d attempts' %
                                        attempts) if attempts else ''))
                        break

        # Request will be updated by the callBack method
        self.log.verbose("RequestTasks exiting, request %s" %
                         self.request.Status)
        return S_OK(self.request)