def _runTransientErrorRetryTest(self, numErrors):

    for errorCode in _ALL_RETRIABLE_ERROR_CODES:
      clientSideEffects = [sqlalchemy.exc.OperationalError(
                            orig=MySQLdb.OperationalError(errorCode),
                            statement="err", params=None)] \
                            * numErrors + [DEFAULT]
      serverSideEffects = [sqlalchemy.exc.InternalError(
                            orig=MySQLdb.InternalError(errorCode),
                            statement="err", params=None)] \
                            * numErrors + [DEFAULT]

      if 3000 > errorCode >= 2000:
        #The error is client side. Return one operationalError, then pass
        with patch.object(Engine,
                          'execute',
                          spec_set=Engine.execute,
                          side_effect=clientSideEffects) \
            as mockExecute:
          retryOnTransientErrors(mockExecute)(Mock())
          self.assertEqual(mockExecute.call_count, numErrors + 1)

      elif errorCode >= 1000:
        #The error is server side. Return one internalError, then pass
        with patch.object(Engine,
                          'execute',
                          spec_set=Engine.execute,
                          side_effect=serverSideEffects) \
            as mockExecute:
          retryOnTransientErrors(mockExecute)(Mock())
          self.assertEqual(mockExecute.call_count, numErrors + 1)

      else:
        self.fail("Error code is neither client nor server: %s" % errorCode)
  def testTransientErrorRetryDecorator(self):
    # Setup proxy.  We'll patch config later, so we need to cache the values
    # so that the original proxy may be restarted with the original params
    config = collectorsdb.CollectorsDbConfig()

    originalHost = config.get("repository", "host")
    originalPort = config.getint("repository", "port")

    def _startProxy():
      p = startProxy(originalHost, originalPort, 6033)
      p.next()
      return p

    proxy = _startProxy()
    self.addCleanup(proxy.send, "kill")

    # Patch collectorsdb config with local proxy
    with ConfigAttributePatch(
          config.CONFIG_NAME,
          config.baseConfigDir,
          (("repository", "host", "127.0.0.1"),
           ("repository", "port", "6033"))):

      # Force refresh of engine singleton
      collectorsdb.resetEngineSingleton()
      engine = collectorsdb.engineFactory()

      # First, make sure valid query returns expected results
      res = collectorsdb.retryOnTransientErrors(engine.execute)("select 1")
      self.assertEqual(res.scalar(), 1)

      @collectorsdb.retryOnTransientErrors
      def _killProxyTryRestartProxyAndTryAgain(n=[]):
        if not n:
          # Kill the proxy on first attempt
          proxy.send("kill")
          proxy.next()
          try:
            engine.execute("select 1")
            self.fail("Proxy did not terminate as expected...")
          except sqlalchemy.exc.OperationalError:
            pass
          n.append(None)
        elif len(n) == 1:
          # Restore proxy in second attempt
          newProxy = _startProxy()
          self.addCleanup(newProxy.send, "kill")
          n.append(None)

        res = engine.execute("select 2")

        return res

      # Try again w/ retry decorator
      result = _killProxyTryRestartProxyAndTryAgain()

      # Verify that the expected value is eventually returned
      self.assertEqual(result.scalar(), 2)
    def testTransientErrorRetryDecorator(self):
        # Setup proxy.  We'll patch config later, so we need to cache the values
        # so that the original proxy may be restarted with the original params
        config = collectorsdb.CollectorsDbConfig()

        originalHost = config.get("repository", "host")
        originalPort = config.getint("repository", "port")

        def _startProxy():
            p = startProxy(originalHost, originalPort, 6033)
            p.next()
            return p

        proxy = _startProxy()
        self.addCleanup(proxy.send, "kill")

        # Patch collectorsdb config with local proxy
        with ConfigAttributePatch(config.CONFIG_NAME, config.baseConfigDir,
                                  (("repository", "host", "127.0.0.1"),
                                   ("repository", "port", "6033"))):

            # Force refresh of engine singleton
            collectorsdb.resetEngineSingleton()
            engine = collectorsdb.engineFactory()

            # First, make sure valid query returns expected results
            res = collectorsdb.retryOnTransientErrors(
                engine.execute)("select 1")
            self.assertEqual(res.scalar(), 1)

            @collectorsdb.retryOnTransientErrors
            def _killProxyTryRestartProxyAndTryAgain(n=[]):  # pylint: disable=W0102
                if not n:
                    # Kill the proxy on first attempt
                    proxy.send("kill")
                    proxy.next()
                    try:
                        engine.execute("select 1")
                        self.fail("Proxy did not terminate as expected...")
                    except sqlalchemy.exc.OperationalError:
                        pass
                    n.append(None)
                elif len(n) == 1:
                    # Restore proxy in second attempt
                    newProxy = _startProxy()
                    self.addCleanup(newProxy.send, "kill")
                    n.append(None)

                res = engine.execute("select 2")

                return res

            # Try again w/ retry decorator
            result = _killProxyTryRestartProxyAndTryAgain()

            # Verify that the expected value is eventually returned
            self.assertEqual(result.scalar(), 2)
  def testNonTransientError(self):
    # Pass MySQLdb error constant for CANT_CREATE_TABLE
    errorCode = 1005
    #The error is client side. Return an operationalError
    with patch.object(Engine,
                      'execute',
                      spec_set=Engine.execute,
                      side_effect=[sqlalchemy.exc.OperationalError(
                        orig=MySQLdb.OperationalError(errorCode),
                        statement="err", params=None)])\
        as mockExecute:

      self.assertRaises(sqlalchemy.exc.OperationalError,
                        retryOnTransientErrors(mockExecute))