예제 #1
0
  def testFilterCompanyMetricNamesBySymbol(self):
    negatives = [
      "TWITTER.TWEET.HANDLE.ZZZ.VOLUME",
      "XIGNITE.ZZZ.CLOSINGPRICE",
      "XIGNITE.FOOBARZZZ.CLOSINGPRICE",
      "XIGNITE.ZZZFOOBAR.CLOSINGPRICE",
      "XIGNITE.FOOBAR.ZZZ.VOLUME",
      "XIGNITE.NEWS.FOOBAR.ZZZ.VOLUME",
      "FOOBAR.VOLUME",
      ".FOOBAR.CLOSINGPRICE",
      "XIGNITE.FOOBAR.",
      "FOOBARCLOSINGPRICE",
    ]

    positives = [
      "XIGNITE.FOOBAR.CLOSINGPRICE",
      "XIGNITE.FOOBAR.VOLUME",
      "TWITTER.TWEET.HANDLE.FOOBAR.VOLUME",
      "XIGNITE.NEWS.FOOBAR.VOLUME",
    ]

    # Execute
    filteredNames = metric_utils.filterCompanyMetricNamesBySymbol(
      metricNames=negatives + positives,
      tickerSymbol="FOOBAR")

    # Verify that the expected metric names were returned

    self.assertItemsEqual(positives, filteredNames)
예제 #2
0
    def testFilterCompanyMetricNamesBySymbol(self):
        negatives = [
            "TWITTER.TWEET.HANDLE.ZZZ.VOLUME",
            "XIGNITE.ZZZ.CLOSINGPRICE",
            "XIGNITE.FOOBARZZZ.CLOSINGPRICE",
            "XIGNITE.ZZZFOOBAR.CLOSINGPRICE",
            "XIGNITE.FOOBAR.ZZZ.VOLUME",
            "XIGNITE.NEWS.FOOBAR.ZZZ.VOLUME",
            "FOOBAR.VOLUME",
            ".FOOBAR.CLOSINGPRICE",
            "XIGNITE.FOOBAR.",
            "FOOBARCLOSINGPRICE",
        ]

        positives = [
            "XIGNITE.FOOBAR.CLOSINGPRICE",
            "XIGNITE.FOOBAR.VOLUME",
            "TWITTER.TWEET.HANDLE.FOOBAR.VOLUME",
            "XIGNITE.NEWS.FOOBAR.VOLUME",
        ]

        # Execute
        filteredNames = metric_utils.filterCompanyMetricNamesBySymbol(
            metricNames=negatives + positives, tickerSymbol="FOOBAR")

        # Verify that the expected metric names were returned

        self.assertItemsEqual(positives, filteredNames)
def deleteCompanies(
    tickerSymbols,
    engineServer,
    engineApiKey,
    warnAboutDestructiveAction=True,
    warningTimeout=_DEFAULT_WARNING_PROMPT_TIMEOUT_SEC,
):
    """Delete companies from Taurus Collector and their metrics/models from
  Taurus Engine.

  :param sequence tickerSymbols: stock ticker symbols of companies to be
    deleted

  :param str engineServer: dns name of ip addres of Taurus API server

  :param str engineApiKey: API Key of Taurus HTM Engine

  :param bool warnAboutDestructiveAction: whether to warn about destructive
    action; defaults to True.

  :param float warningTimeout: Timeout for the warning prompt; ignored if
    warnAboutDestructiveAction is False

  :raises WarningPromptTimeout: if warning prompt timed out
  :raises UserAbortedOperation: if user chose to abort the operation
  :raises FlusherMetricNotFound:
  """
    tickerSymbols = tuple(symbol.upper() for symbol in tickerSymbols)

    # Check for duplicate symbols
    repeatedSymbols = set(sym for sym in tickerSymbols if tickerSymbols.count(sym) > 1)
    if repeatedSymbols:
        raise ValueError(
            "{numRepeats} symbol(s) are present more than once in "
            "tickerSymbols arg: {repeats}".format(numRepeats=len(repeatedSymbols), repeats=repeatedSymbols)
        )

    # Set will be handier going forward
    tickerSymbols = set(tickerSymbols)

    if warnAboutDestructiveAction:
        _warnAboutDestructiveAction(timeout=warningTimeout, tickerSymbols=tickerSymbols, engineServer=engineServer)

    # If any of the ticker symbols still appear in the collector's metrics config,
    # abort the operation as a precautionary measure.
    allSymbols = set(security[0].upper() for security in metric_utils.getAllMetricSecurities())

    problemSymbols = tickerSymbols & allSymbols
    assert not problemSymbols, (
        "Can't delete - {numProblem} of the specified companies [{symbols}] are "
        "in active metrics configuration".format(numProblem=len(problemSymbols), symbols=problemSymbols)
    )

    # First, we need to synchronize with Taurus Engine's metric data path.
    # If any of the data still in the pipeline is for any of the companies being
    # deleted, then the metrics may be re-created in the Engine after we delete
    # them. This is an yet unresolved subtlety with custom metrics in htmengine.
    _flushTaurusEngineMetricDataPath(engineServer, engineApiKey)

    # NOTE: We must query custom metrics after flushing the metric data path,
    # since metrics may get created as a side-effect of processing metric data.
    allMetricsMap = {
        obj["name"]: obj for obj in metric_utils.getAllCustomMetrics(host=engineServer, apiKey=engineApiKey)
    }

    allMetricNames = allMetricsMap.keys()

    for symbolNum, symbol in enumerate(tickerSymbols, 1):
        # Delete corresponding metrics from Taurus Engine
        metricNamesToDelete = metric_utils.filterCompanyMetricNamesBySymbol(allMetricNames, symbol)
        if not metricNamesToDelete:
            g_log.info("No metrics to delete for symbol=%s (%d of %d)", symbol, symbolNum, len(tickerSymbols))
            continue

        g_log.info(
            "Deleting metrics and models for ticker symbol=%s from Taurus " "Engine=%s (%d of %d)",
            symbol,
            engineServer,
            symbolNum,
            len(tickerSymbols),
        )

        for metricName in metricNamesToDelete:
            metric_utils.deleteMetric(host=engineServer, apiKey=engineApiKey, metricName=metricName)
            g_log.info("Deleted metric name=%s, uid=%s", metricName, allMetricsMap[metricName]["uid"])

        # Delete the symbol from xignite_security table last; this cascades to
        # delete related rows in other tables via cascading delete relationship.
        #
        # NOTE: garbage collection from other tables not tied to xiginte_security
        #  symbols presently depends on aging of the rows (e.g., twitter tables).
        #  After ENG-83, all company-specific rows from all tables will be
        # cleaned up and THIS NOTE SHOULD THEN BE REMOVED
        with collectorsdb.engineFactory().begin() as conn:
            numDeleted = (
                conn.execute(
                    collectorsdb.schema.xigniteSecurity.delete().where(  # pylint: disable=E1120
                        collectorsdb.schema.xigniteSecurity.c.symbol == symbol
                    )
                )
            ).rowcount

            if numDeleted:
                g_log.info("Deleted row=%s from table=%s", symbol, collectorsdb.schema.xigniteSecurity)
            else:
                g_log.warning(
                    "Couldn't delete security row=%s: not found in table=%s",
                    symbol,
                    collectorsdb.schema.xigniteSecurity,
                )
def deleteCompanies(tickerSymbols,
                    engineServer,
                    engineApiKey,
                    warnAboutDestructiveAction=True,
                    warningTimeout=_DEFAULT_WARNING_PROMPT_TIMEOUT_SEC):
    """Delete companies from Taurus Collector and their metrics/models from
  Taurus Engine.

  :param sequence tickerSymbols: stock ticker symbols of companies to be
    deleted

  :param str engineServer: dns name of ip addres of Taurus API server

  :param str engineApiKey: API Key of Taurus HTM Engine

  :param bool warnAboutDestructiveAction: whether to warn about destructive
    action; defaults to True.

  :param float warningTimeout: Timeout for the warning prompt; ignored if
    warnAboutDestructiveAction is False

  :raises WarningPromptTimeout: if warning prompt timed out
  :raises UserAbortedOperation: if user chose to abort the operation
  :raises FlusherMetricNotFound:
  """
    tickerSymbols = tuple(symbol.upper() for symbol in tickerSymbols)

    # Check for duplicate symbols
    repeatedSymbols = set(sym for sym in tickerSymbols
                          if tickerSymbols.count(sym) > 1)
    if repeatedSymbols:
        raise ValueError(
            "{numRepeats} symbol(s) are present more than once in "
            "tickerSymbols arg: {repeats}".format(
                numRepeats=len(repeatedSymbols), repeats=repeatedSymbols))

    # Set will be handier going forward
    tickerSymbols = set(tickerSymbols)

    if warnAboutDestructiveAction:
        _warnAboutDestructiveAction(timeout=warningTimeout,
                                    tickerSymbols=tickerSymbols,
                                    engineServer=engineServer)

    # If any of the ticker symbols still appear in the collector's metrics config,
    # abort the operation as a precautionary measure.
    allSymbols = set(security[0].upper()
                     for security in metric_utils.getAllMetricSecurities())

    problemSymbols = tickerSymbols & allSymbols
    assert not problemSymbols, (
        "Can't delete - {numProblem} of the specified companies [{symbols}] are "
        "in active metrics configuration".format(
            numProblem=len(problemSymbols), symbols=problemSymbols))

    # First, we need to synchronize with Taurus Engine's metric data path.
    # If any of the data still in the pipeline is for any of the companies being
    # deleted, then the metrics may be re-created in the Engine after we delete
    # them. This is an yet unresolved subtlety with custom metrics in htmengine.
    _flushTaurusEngineMetricDataPath(engineServer, engineApiKey)

    # NOTE: We must query custom metrics after flushing the metric data path,
    # since metrics may get created as a side-effect of processing metric data.
    allMetricsMap = {
        obj["name"]: obj
        for obj in metric_utils.getAllCustomMetrics(host=engineServer,
                                                    apiKey=engineApiKey)
    }

    allMetricNames = allMetricsMap.keys()

    for symbolNum, symbol in enumerate(tickerSymbols, 1):
        # Delete corresponding metrics from Taurus Engine
        metricNamesToDelete = metric_utils.filterCompanyMetricNamesBySymbol(
            allMetricNames, symbol)
        if not metricNamesToDelete:
            g_log.info("No metrics to delete for symbol=%s (%d of %d)", symbol,
                       symbolNum, len(tickerSymbols))
            continue

        g_log.info(
            "Deleting metrics and models for ticker symbol=%s from Taurus "
            "Engine=%s (%d of %d)", symbol, engineServer, symbolNum,
            len(tickerSymbols))

        for metricName in metricNamesToDelete:
            metric_utils.deleteMetric(host=engineServer,
                                      apiKey=engineApiKey,
                                      metricName=metricName)
            g_log.info("Deleted metric name=%s, uid=%s", metricName,
                       allMetricsMap[metricName]["uid"])

        # Delete the symbol from xignite_security table last; this cascades to
        # delete related rows in other tables via cascading delete relationship.
        #
        # NOTE: garbage collection from other tables not tied to xiginte_security
        #  symbols presently depends on aging of the rows (e.g., twitter tables).
        #  After ENG-83, all company-specific rows from all tables will be
        # cleaned up and THIS NOTE SHOULD THEN BE REMOVED
        with collectorsdb.engineFactory().begin() as conn:
            numDeleted = (
                conn.execute(collectorsdb.schema.xigniteSecurity  # pylint: disable=E1120
                             .delete().where(
                                 collectorsdb.schema.xigniteSecurity.c.symbol
                                 == symbol))).rowcount

            if numDeleted:
                g_log.info("Deleted row=%s from table=%s", symbol,
                           collectorsdb.schema.xigniteSecurity)
            else:
                g_log.warning(
                    "Couldn't delete security row=%s: not found in table=%s",
                    symbol, collectorsdb.schema.xigniteSecurity)