def _waitForFlusherAndGarbageCollect(engineServer,
                                     engineApiKey,
                                     flusherMetricName):
  """Wait for the data path flusher metric to be created in Taurus Engine and
  also garbage-collect flushers from this and prior sessions

  :param str engineServer: dns name of ip addres of Taurus API server

  :param str engineApiKey: API Key of Taurus HTM Engine

  :param str flusherMetricName: the unique name of the flusher metric to wait
    on.

  :raises FlusherMetricNotFound: if the wait fails
  """
  flushers = [obj["name"] for obj in
              metric_utils.getAllCustomMetrics(engineServer, engineApiKey)
              if obj["name"].startswith(_DATA_PATH_FLUSHER_METRIC_PREFIX)]
  found = flusherMetricName in flushers

  # Delete flushers, including any from past attempts that failed delete
  for metric in flushers:
    g_log.info("Deleting metric data path flusher metric %s", metric)
    metric_utils.deleteMetric(host=engineServer,
                              apiKey=engineApiKey,
                              metricName=metric)

  if not found:
    raise FlusherMetricNotFound("Still waiting for data path flusher metric "
                                "{metric}".format(metric=flusherMetricName))
def _waitForFlusherAndGarbageCollect(engineServer, engineApiKey,
                                     flusherMetricName):
    """Wait for the data path flusher metric to be created in Taurus Engine and
  also garbage-collect flushers from this and prior sessions

  :param str engineServer: dns name of ip addres of Taurus API server

  :param str engineApiKey: API Key of Taurus HTM Engine

  :param str flusherMetricName: the unique name of the flusher metric to wait
    on.

  :raises FlusherMetricNotFound: if the wait fails
  """
    flushers = [
        obj["name"]
        for obj in metric_utils.getAllCustomMetrics(engineServer, engineApiKey)
        if obj["name"].startswith(_DATA_PATH_FLUSHER_METRIC_PREFIX)
    ]
    found = flusherMetricName in flushers

    # Delete flushers, including any from past attempts that failed delete
    for metric in flushers:
        g_log.info("Deleting metric data path flusher metric %s", metric)
        metric_utils.deleteMetric(host=engineServer,
                                  apiKey=engineApiKey,
                                  metricName=metric)

    if not found:
        raise FlusherMetricNotFound(
            "Still waiting for data path flusher metric "
            "{metric}".format(metric=flusherMetricName))
def _safeDeleteMetric(host, apiKey, metricName):
  """Delete metric, suppressing metric_utils.MetricNotFound exception, if any
  """
  try:
    metric_utils.deleteMetric(host=host,
                              apiKey=apiKey,
                              metricName=metricName)
  except metric_utils.MetricNotFound:
    pass
  def testDeleteMetric(self, requestsMock):
    requestsMock.delete.return_value = Mock(status_code=200)

    metric_utils.deleteMetric(host="localhost",
                              apiKey="taurus",
                              metricName="XIGNITE.FOO.VOLUME")
    requestsMock.delete.assert_called_once_with(
      "https://localhost/_metrics/custom/XIGNITE.FOO.VOLUME",
      auth=("taurus", ""), verify=ANY)
Example #5
0
    def testDeleteMetric(self, requestsMock):
        requestsMock.delete.return_value = Mock(status_code=200)

        metric_utils.deleteMetric(host="localhost",
                                  apiKey="taurus",
                                  metricName="XIGNITE.FOO.VOLUME")
        requestsMock.delete.assert_called_once_with(
            "https://localhost/_metrics/custom/XIGNITE.FOO.VOLUME",
            auth=("taurus", ""),
            verify=ANY)
  def testDeleteMetricMetricNotFound(self, requestsMock):
    requestsMock.delete.return_value = Mock(status_code=404,
                                            text="metric not found")

    with self.assertRaises(metric_utils.MetricNotFound) as errorContext:
      metric_utils.deleteMetric(host="localhost",
                                apiKey="taurus",
                                metricName="XIGNITE.FOO.VOLUME")

    requestsMock.delete.assert_called_once_with(
      "https://localhost/_metrics/custom/XIGNITE.FOO.VOLUME",
      auth=("taurus", ""), verify=ANY)

    self.assertEqual(errorContext.exception.args[0], "metric not found")
Example #7
0
    def testDeleteMetricMetricNotFound(self, requestsMock):
        requestsMock.delete.return_value = Mock(status_code=404,
                                                text="metric not found")

        with self.assertRaises(metric_utils.MetricNotFound) as errorContext:
            metric_utils.deleteMetric(host="localhost",
                                      apiKey="taurus",
                                      metricName="XIGNITE.FOO.VOLUME")

        requestsMock.delete.assert_called_once_with(
            "https://localhost/_metrics/custom/XIGNITE.FOO.VOLUME",
            auth=("taurus", ""),
            verify=ANY)

        self.assertEqual(errorContext.exception.args[0], "metric not found")
def main():
  logging_support.LoggingSupport.initTool()

  try:
    options = _parseArgs()
    g_log.info("Running %s with options=%r", sys.argv[0], options)

    for metric in metric_utils.getAllCustomMetrics(host=options["htmServer"],
                                                   apiKey=options["apiKey"]):
      if metric["name"] == options["metricName"]:
        metric_utils.deleteMetric(host=options["htmServer"],
                                  apiKey=options["apiKey"],
                                  metricName=metric["name"])

        g_log.info("Deleted metric=%s", metric["name"])
        break

  except SystemExit as e:
    if e.code != 0:
      g_log.exception("delete_metric failed")
    raise
  except Exception:
    g_log.exception("delete_metric failed")
    raise
Example #9
0
def main():
    logging_support.LoggingSupport.initTool()

    try:
        options = _parseArgs()
        g_log.info("Running %s with options=%r", sys.argv[0], options)

        for metric in metric_utils.getAllCustomMetrics(
                host=options["htmServer"], apiKey=options["apiKey"]):
            if metric["name"] == options["metricName"]:
                metric_utils.deleteMetric(host=options["htmServer"],
                                          apiKey=options["apiKey"],
                                          metricName=metric["name"])

                g_log.info("Deleted metric=%s", metric["name"])
                break

    except SystemExit as e:
        if e.code != 0:
            g_log.exception("delete_metric failed")
        raise
    except Exception:
        g_log.exception("delete_metric failed")
        raise
def deleteCompanies(tickerSymbols,
                    engineServer,
                    engineApiKey,
                    warnAboutDestructiveAction=True,
                    warningTimeout=_DEFAULT_WARNING_PROMPT_TIMEOUT_SEC):
  """Delete companies from Taurus Collector and their metrics/models from
  Taurus Engine.

  :param sequence tickerSymbols: stock ticker symbols of companies to be
    deleted

  :param str engineServer: dns name of ip addres of Taurus API server

  :param str engineApiKey: API Key of Taurus HTM Engine

  :param bool warnAboutDestructiveAction: whether to warn about destructive
    action; defaults to True.

  :param float warningTimeout: Timeout for the warning prompt; ignored if
    warnAboutDestructiveAction is False

  :raises WarningPromptTimeout: if warning prompt timed out
  :raises UserAbortedOperation: if user chose to abort the operation
  :raises FlusherMetricNotFound:
  """
  tickerSymbols = tuple(symbol.upper() for symbol in tickerSymbols)

  # Check for duplicate symbols
  repeatedSymbols = set(sym for sym in tickerSymbols
                        if tickerSymbols.count(sym) > 1)
  if repeatedSymbols:
    raise ValueError("{numRepeats} symbol(s) are present more than once in "
                     "tickerSymbols arg: {repeats}"
                     .format(numRepeats=len(repeatedSymbols),
                             repeats=repeatedSymbols))

  # Set will be handier going forward
  tickerSymbols = set(tickerSymbols)

  if warnAboutDestructiveAction:
    _warnAboutDestructiveAction(timeout=warningTimeout,
                                tickerSymbols=tickerSymbols,
                                engineServer=engineServer)

  # If any of the ticker symbols still appear in the collector's metrics config,
  # abort the operation as a precautionary measure.
  allSymbols = set(security[0].upper() for security in
                   metric_utils.getAllMetricSecurities())

  problemSymbols = tickerSymbols & allSymbols
  assert not problemSymbols, (
    "Can't delete - {numProblem} of the specified companies [{symbols}] are "
    "in active metrics configuration".format(numProblem=len(problemSymbols),
                                             symbols=problemSymbols))

  # First, we need to synchronize with Taurus Engine's metric data path.
  # If any of the data still in the pipeline is for any of the companies being
  # deleted, then the metrics may be re-created in the Engine after we delete
  # them. This is an yet unresolved subtlety with custom metrics in htmengine.
  _flushTaurusEngineMetricDataPath(engineServer, engineApiKey)

  # NOTE: We must query custom metrics after flushing the metric data path,
  # since metrics may get created as a side-effect of processing metric data.
  allMetricsMap = {
    obj["name"] : obj
    for obj in
    metric_utils.getAllCustomMetrics(host=engineServer, apiKey=engineApiKey)
  }

  allMetricNames = allMetricsMap.keys()

  for symbolNum, symbol in enumerate(tickerSymbols, 1):
    # Delete corresponding metrics from Taurus Engine
    metricNamesToDelete = metric_utils.filterCompanyMetricNamesBySymbol(
      allMetricNames,
      symbol)
    if not metricNamesToDelete:
      g_log.info("No metrics to delete for symbol=%s (%d of %d)", symbol,
                 symbolNum, len(tickerSymbols))
      continue

    g_log.info("Deleting metrics and models for ticker symbol=%s from Taurus "
               "Engine=%s (%d of %d)", symbol, engineServer,
               symbolNum, len(tickerSymbols))

    for metricName in metricNamesToDelete:
      metric_utils.deleteMetric(host=engineServer,
                                apiKey=engineApiKey,
                                metricName=metricName)
      g_log.info("Deleted metric name=%s, uid=%s", metricName,
                 allMetricsMap[metricName]["uid"])


    # Delete the symbol from xignite_security table last; this cascades to
    # delete related rows in other tables via cascading delete relationship.
    #
    # NOTE: garbage collection from other tables not tied to xiginte_security
    #  symbols presently depends on aging of the rows (e.g., twitter tables).
    #  After ENG-83, all company-specific rows from all tables will be
    # cleaned up and THIS NOTE SHOULD THEN BE REMOVED
    with collectorsdb.engineFactory().begin() as conn:
      numDeleted = (
        conn.execute(
          collectorsdb.schema.xigniteSecurity  # pylint: disable=E1120
          .delete()
          .where(collectorsdb.schema.xigniteSecurity.c.symbol == symbol))
        ).rowcount

      if numDeleted:
        g_log.info("Deleted row=%s from table=%s", symbol,
                   collectorsdb.schema.xigniteSecurity)
      else:
        g_log.warning(
          "Couldn't delete security row=%s: not found in table=%s",
          symbol, collectorsdb.schema.xigniteSecurity)
def deleteCompanies(tickerSymbols,
                    engineServer,
                    engineApiKey,
                    warnAboutDestructiveAction=True,
                    warningTimeout=_DEFAULT_WARNING_PROMPT_TIMEOUT_SEC):
    """Delete companies from Taurus Collector and their metrics/models from
  Taurus Engine.

  :param sequence tickerSymbols: stock ticker symbols of companies to be
    deleted

  :param str engineServer: dns name of ip addres of Taurus API server

  :param str engineApiKey: API Key of Taurus HTM Engine

  :param bool warnAboutDestructiveAction: whether to warn about destructive
    action; defaults to True.

  :param float warningTimeout: Timeout for the warning prompt; ignored if
    warnAboutDestructiveAction is False

  :raises WarningPromptTimeout: if warning prompt timed out
  :raises UserAbortedOperation: if user chose to abort the operation
  :raises FlusherMetricNotFound:
  """
    tickerSymbols = tuple(symbol.upper() for symbol in tickerSymbols)

    # Check for duplicate symbols
    repeatedSymbols = set(sym for sym in tickerSymbols
                          if tickerSymbols.count(sym) > 1)
    if repeatedSymbols:
        raise ValueError(
            "{numRepeats} symbol(s) are present more than once in "
            "tickerSymbols arg: {repeats}".format(
                numRepeats=len(repeatedSymbols), repeats=repeatedSymbols))

    # Set will be handier going forward
    tickerSymbols = set(tickerSymbols)

    if warnAboutDestructiveAction:
        _warnAboutDestructiveAction(timeout=warningTimeout,
                                    tickerSymbols=tickerSymbols,
                                    engineServer=engineServer)

    # If any of the ticker symbols still appear in the collector's metrics config,
    # abort the operation as a precautionary measure.
    allSymbols = set(security[0].upper()
                     for security in metric_utils.getAllMetricSecurities())

    problemSymbols = tickerSymbols & allSymbols
    assert not problemSymbols, (
        "Can't delete - {numProblem} of the specified companies [{symbols}] are "
        "in active metrics configuration".format(
            numProblem=len(problemSymbols), symbols=problemSymbols))

    # First, we need to synchronize with Taurus Engine's metric data path.
    # If any of the data still in the pipeline is for any of the companies being
    # deleted, then the metrics may be re-created in the Engine after we delete
    # them. This is an yet unresolved subtlety with custom metrics in htmengine.
    _flushTaurusEngineMetricDataPath(engineServer, engineApiKey)

    # NOTE: We must query custom metrics after flushing the metric data path,
    # since metrics may get created as a side-effect of processing metric data.
    allMetricsMap = {
        obj["name"]: obj
        for obj in metric_utils.getAllCustomMetrics(host=engineServer,
                                                    apiKey=engineApiKey)
    }

    allMetricNames = allMetricsMap.keys()

    for symbolNum, symbol in enumerate(tickerSymbols, 1):
        # Delete corresponding metrics from Taurus Engine
        metricNamesToDelete = metric_utils.filterCompanyMetricNamesBySymbol(
            allMetricNames, symbol)
        if not metricNamesToDelete:
            g_log.info("No metrics to delete for symbol=%s (%d of %d)", symbol,
                       symbolNum, len(tickerSymbols))
            continue

        g_log.info(
            "Deleting metrics and models for ticker symbol=%s from Taurus "
            "Engine=%s (%d of %d)", symbol, engineServer, symbolNum,
            len(tickerSymbols))

        for metricName in metricNamesToDelete:
            metric_utils.deleteMetric(host=engineServer,
                                      apiKey=engineApiKey,
                                      metricName=metricName)
            g_log.info("Deleted metric name=%s, uid=%s", metricName,
                       allMetricsMap[metricName]["uid"])

        # Delete the symbol from xignite_security table last; this cascades to
        # delete related rows in other tables via cascading delete relationship.
        #
        # NOTE: garbage collection from other tables not tied to xiginte_security
        #  symbols presently depends on aging of the rows (e.g., twitter tables).
        #  After ENG-83, all company-specific rows from all tables will be
        # cleaned up and THIS NOTE SHOULD THEN BE REMOVED
        with collectorsdb.engineFactory().begin() as conn:
            numDeleted = (
                conn.execute(collectorsdb.schema.xigniteSecurity  # pylint: disable=E1120
                             .delete().where(
                                 collectorsdb.schema.xigniteSecurity.c.symbol
                                 == symbol))).rowcount

            if numDeleted:
                g_log.info("Deleted row=%s from table=%s", symbol,
                           collectorsdb.schema.xigniteSecurity)
            else:
                g_log.warning(
                    "Couldn't delete security row=%s: not found in table=%s",
                    symbol, collectorsdb.schema.xigniteSecurity)