コード例 #1
0
  def testCreateAllModels(self):

    host = os.environ.get("TAURUS_HTM_SERVER", "127.0.0.1")
    apikey = os.environ.get("TAURUS_APIKEY", "taurus")

    # Resize metrics down to a much smaller random sample of the original
    # so as to not overload the system under test.  We need only to test that
    # everything returned goes through the right channels.

    metrics = {
      key:value
      for (key, value)
      in random.sample(metric_utils.getMetricsConfiguration().items(), 3)
    }

    with patch("taurus.metric_collectors.metric_utils.getMetricsConfiguration",
               return_value=metrics,
               spec_set=metric_utils.getMetricsConfiguration):
      createdModels = metric_utils.createAllModels(host, apikey)

    allModels = metric_utils.getAllModels(host, apikey)

    for model in createdModels:
      self.addCleanup(requests.delete,
                      "https://%s/_metrics/custom/%s" % (host, model["name"]),
                      auth=(apikey, ""),
                      verify=False)
      remoteModel = metric_utils.getOneModel(host, apikey, model["uid"])
      self.assertDictEqual(remoteModel, model)
      self.assertIn(model, allModels)
コード例 #2
0
  def testCreateAllModelsHappyPath(self, requestsMock):
    requestsMock.post.return_value = Mock(status_code=201,
                                          text='[{"uid":"foo", "name":"bar"}]')

    totalModels = len(
      metric_utils.getMetricNamesFromConfig(
        metric_utils.getMetricsConfiguration()))

    metric_utils.createAllModels("localhost", "taurus")
    self.assertEqual(requestsMock.post.call_count, totalModels)

    for args, kwargs in requestsMock.post.call_args_list:
      self.assertEqual(args[0], "https://localhost/_models")
      self.assertIn("data", kwargs)
      data = json.loads(kwargs["data"])
      self.assertIsInstance(data, dict)
      self.assertIn("datasource", data)
      self.assertEquals(data["datasource"], "custom")
      self.assertIn("metricSpec", data)
      self.assertIn("metric", data["metricSpec"])
      self.assertIn("resource", data["metricSpec"])
      self.assertIn("userInfo", data["metricSpec"])
      self.assertIsInstance(data["metricSpec"]["userInfo"], dict)
      self.assertIn("metricType", data["metricSpec"]["userInfo"])
      self.assertIn("metricTypeName", data["metricSpec"]["userInfo"])
      self.assertIn("symbol", data["metricSpec"]["userInfo"])
      self.assertIn("modelParams", data)
コード例 #3
0
def loadMetricSpecs():
    """ Load metric specs for the xignite stock provider

  :returns: a sequence of StockMetricSpec objects

  Excerpt from metrics.json:
  {
    "Accenture": {
      "stockExchange": "NYSE",
      "symbol": "ACN",
      "metrics": {
        "XIGNITE.ACN.CLOSINGPRICE": {
          "metricTypeName": "Stock Price",
          "provider": "xignite",
          "sampleKey": "Close"
        },
        "XIGNITE.ACN.VOLUME": {
          "metricTypeName": "Stock Volume",
          "provider": "xignite",
          "sampleKey": "Volume"
        },
        . . .
      }
    },
    . . .
  }
  """
    return tuple(
        StockMetricSpec(metricName=metricName,
                        symbol=resVal["symbol"].upper(),
                        stockExchange=resVal["stockExchange"],
                        sampleKey=metricVal["sampleKey"])
        for resVal in getMetricsConfiguration().itervalues()
        for metricName, metricVal in resVal["metrics"].iteritems()
        if metricVal["provider"] == "xignite" and "sampleKey" in metricVal)
コード例 #4
0
    def testCreateAllModelsHappyPath(self, requestsMock):
        requestsMock.post.return_value = Mock(
            status_code=201, text='[{"uid":"foo", "name":"bar"}]')

        totalModels = len(
            metric_utils.getMetricNamesFromConfig(
                metric_utils.getMetricsConfiguration()))

        metric_utils.createAllModels("localhost", "taurus")
        self.assertEqual(requestsMock.post.call_count, totalModels)

        for args, kwargs in requestsMock.post.call_args_list:
            self.assertEqual(args[0], "https://localhost/_models")
            self.assertIn("data", kwargs)
            data = json.loads(kwargs["data"])
            self.assertIsInstance(data, dict)
            self.assertIn("datasource", data)
            self.assertEquals(data["datasource"], "custom")
            self.assertIn("metricSpec", data)
            self.assertIn("metric", data["metricSpec"])
            self.assertIn("resource", data["metricSpec"])
            self.assertIn("userInfo", data["metricSpec"])
            self.assertIsInstance(data["metricSpec"]["userInfo"], dict)
            self.assertIn("metricType", data["metricSpec"]["userInfo"])
            self.assertIn("metricTypeName", data["metricSpec"]["userInfo"])
            self.assertIn("symbol", data["metricSpec"]["userInfo"])
            self.assertIn("modelParams", data)
コード例 #5
0
    def testCreateAllModels(self):

        host = os.environ.get("TAURUS_HTM_SERVER", "127.0.0.1")
        apikey = os.environ.get("TAURUS_APIKEY", "taurus")

        # Resize metrics down to a much smaller random sample of the original
        # so as to not overload the system under test.  We need only to test that
        # everything returned goes through the right channels.

        metrics = {
            key: value
            for (key, value) in random.sample(
                metric_utils.getMetricsConfiguration().items(), 3)
        }

        with patch(
                "taurus.metric_collectors.metric_utils.getMetricsConfiguration",
                return_value=metrics,
                spec_set=metric_utils.getMetricsConfiguration):
            createdModels = metric_utils.createAllModels(host, apikey)

        allModels = metric_utils.getAllModels(host, apikey)

        for model in createdModels:
            self.addCleanup(requests.delete,
                            "https://%s/_metrics/custom/%s" %
                            (host, model["name"]),
                            auth=(apikey, ""),
                            verify=False)
            remoteModel = metric_utils.getOneModel(host, apikey, model["uid"])
            self.assertDictEqual(remoteModel, model)
            self.assertIn(model, allModels)
コード例 #6
0
def _loadNewsVolumeMetricSpecs():
    """ Load metric specs for our securities news volume provider

  :returns: a sequence of NewsVolumeMetricSpec objects
  """
    return tuple(
        NewsVolumeMetricSpec(metric=metricName, symbol=resVal["symbol"])
        for resVal in getMetricsConfiguration().itervalues()
        for metricName, metricVal in resVal["metrics"].iteritems()
        if metricVal["provider"] == _NEWS_VOLUME_METRIC_PROVIDER)
コード例 #7
0
def _loadNewsVolumeMetricSpecs():
  """ Load metric specs for our securities news volume provider

  :returns: a sequence of NewsVolumeMetricSpec objects
  """
  return tuple(
    NewsVolumeMetricSpec(metric=metricName,
                         symbol=resVal["symbol"])
    for resVal in getMetricsConfiguration().itervalues()
    for metricName, metricVal in resVal["metrics"].iteritems()
    if metricVal["provider"] == _NEWS_VOLUME_METRIC_PROVIDER)
コード例 #8
0
def main():
  """
  NOTE: main also serves as entry point for "console script" generated by setup
  """
  logging_support.LoggingSupport().initTool()

  try:
    options = _parseArgs()
    allSymbols = set(stockData["symbol"] for stockData in
                     metric_utils.getMetricsConfiguration().itervalues() )

    g_log.info("Verifying that agents are in hot_standby mode")
    for section in config.sections():
      assert config.get(section, "opmode") == config.OP_MODE_HOT_STANDBY

    g_log.info("Verifying that the old symbol has been removed from the "
               "metrics configuration")
    assert options.oldSymbol not in allSymbols

    g_log.info("Verifying that the new symbol is present in the metrics "
               "configuration")
    assert options.newSymbol in allSymbols

    g_log.info("Migrating BOTH twitter and xignite stock data from "
               "old-symbol=%s to new-symbol=%s",
               options.oldSymbol, options.newSymbol)

    # Rename the metrics in collectorsdb and forward new metric samples to HTM
    # Engine
    g_log.info("Modifying old metrics with new symbol")

    _resymbolTweetVolumeMetric(oldSymbol=options.oldSymbol,
                               newSymbol=options.newSymbol,
                               aggPeriod=options.aggPeriod)

    _resymbolStockMetrics(oldSymbol=options.oldSymbol,
                          newSymbol=options.newSymbol)


    # Delete metrics linked to old stock symbol from Taurus Engine
    delete_companies.deleteCompanies(
      tickerSymbols=[options.oldSymbol],
      engineServer=options.htmServer,
      engineApiKey=options.apikey,
      warnAboutDestructiveAction=False)
  except SystemExit as e:
    if e.code != 0:
      g_log.exception("Failed!")
    raise
  except Exception:
    g_log.exception("Failed!")
    raise
コード例 #9
0
def main():
    """
  NOTE: main also serves as entry point for "console script" generated by setup
  """
    logging_support.LoggingSupport().initTool()

    try:
        options = _parseArgs()
        allSymbols = set(stockData["symbol"] for stockData in
                         metric_utils.getMetricsConfiguration().itervalues())

        g_log.info("Verifying that agents are in hot_standby mode")
        for section in config.sections():
            assert config.get(section, "opmode") == config.OP_MODE_HOT_STANDBY

        g_log.info("Verifying that the old symbol has been removed from the "
                   "metrics configuration")
        assert options.oldSymbol not in allSymbols

        g_log.info("Verifying that the new symbol is present in the metrics "
                   "configuration")
        assert options.newSymbol in allSymbols

        g_log.info(
            "Migrating BOTH twitter and xignite stock data from "
            "old-symbol=%s to new-symbol=%s", options.oldSymbol,
            options.newSymbol)

        # Rename the metrics in collectorsdb and forward new metric samples to HTM
        # Engine
        g_log.info("Modifying old metrics with new symbol")

        _resymbolTweetVolumeMetric(oldSymbol=options.oldSymbol,
                                   newSymbol=options.newSymbol,
                                   aggPeriod=options.aggPeriod)

        _resymbolStockMetrics(oldSymbol=options.oldSymbol,
                              newSymbol=options.newSymbol)

        # Delete metrics linked to old stock symbol from Taurus Engine
        delete_companies.deleteCompanies(tickerSymbols=[options.oldSymbol],
                                         engineServer=options.htmServer,
                                         engineApiKey=options.apikey,
                                         warnAboutDestructiveAction=False)
    except SystemExit as e:
        if e.code != 0:
            g_log.exception("Failed!")
        raise
    except Exception:
        g_log.exception("Failed!")
        raise
コード例 #10
0
  def testCreateAllModels(self):

    host = os.environ.get("TAURUS_HTM_SERVER", "127.0.0.1")
    apikey = os.environ.get("TAURUS_APIKEY", "taurus")

    # Resize metrics down to a much smaller random sample of the original
    # so as to not overload the system under test.  We need only to test that
    # everything returned goes through the right channels.

    metricsConfig = {
      key:value
      for (key, value)
      in random.sample(metric_utils.getMetricsConfiguration().items(), 3)
    }

    expectedMetricNames = []

    for resVal in metricsConfig.itervalues():
      for metricName in resVal["metrics"]:
        expectedMetricNames.append(metricName)

        self.addCleanup(requests.delete,
                        "https://%s/_metrics/custom/%s" % (host, metricName),
                        auth=(apikey, ""),
                        verify=False)

    self.assertGreater(len(expectedMetricNames), 0)

    with patch("taurus.metric_collectors.metric_utils.getMetricsConfiguration",
               return_value=metricsConfig,
               spec_set=metric_utils.getMetricsConfiguration):
      createdModels = metric_utils.createAllModels(host, apikey)

    self.assertEqual(len(createdModels), len(expectedMetricNames))

    for model in createdModels:
      remoteModel = metric_utils.getOneModel(host, apikey, model["uid"])
      self.assertIn(remoteModel["name"], expectedMetricNames)
      # Verify that the model is either in "ACTIVE" or the transient
      # "PENDNG DATA" or "CREATE PENDING" states
      self.assertIn(remoteModel["status"], [1, 2, 8])
コード例 #11
0
    def testCreateAllModelsWithMetricNameFilter(self,
                                                createCustomHtmModelMock):
        allMetricNames = metric_utils.getMetricNamesFromConfig(
            metric_utils.getMetricsConfiguration())

        subsetOfMetricNames = allMetricNames[:(len(allMetricNames) + 1) // 2]
        self.assertGreater(len(subsetOfMetricNames), 0)

        createCustomHtmModelMock.side_effect = (lambda **kwargs: dict(
            name=kwargs["metricName"], uid=kwargs["metricName"] * 2))

        models = metric_utils.createAllModels(
            host="host", apiKey="apikey", onlyMetricNames=subsetOfMetricNames)

        self.assertEqual(createCustomHtmModelMock.call_count,
                         len(subsetOfMetricNames))

        self.assertEqual(len(models), len(subsetOfMetricNames))

        self.assertItemsEqual(subsetOfMetricNames,
                              [model["name"] for model in models])
コード例 #12
0
  def testCreateAllModelsWithMetricNameFilter(self, createCustomHtmModelMock):
    allMetricNames = metric_utils.getMetricNamesFromConfig(
      metric_utils.getMetricsConfiguration())

    subsetOfMetricNames = allMetricNames[:(len(allMetricNames) + 1) // 2]
    self.assertGreater(len(subsetOfMetricNames), 0)

    createCustomHtmModelMock.side_effect = (
      lambda **kwargs: dict(name=kwargs["metricName"],
                            uid=kwargs["metricName"] * 2))

    models = metric_utils.createAllModels(host="host",
                                          apiKey="apikey",
                                          onlyMetricNames=subsetOfMetricNames)

    self.assertEqual(createCustomHtmModelMock.call_count,
                     len(subsetOfMetricNames))

    self.assertEqual(len(models), len(subsetOfMetricNames))

    self.assertItemsEqual(subsetOfMetricNames,
                          [model["name"] for model in models])
コード例 #13
0
 def testGetMetricsConfiguration(self):
   metrics = metric_utils.getMetricsConfiguration()
   self.assertIsInstance(metrics, dict)
   self.assertTrue(metrics)
   for companyName, details in metrics.iteritems():
     self.assertIsInstance(companyName, basestring)
     self.assertIsInstance(details, dict)
     self.assertTrue(details)
     self.assertIn("metrics", details)
     self.assertIn("stockExchange", details)
     self.assertIn("symbol", details)
     for metricName, metric in details["metrics"].iteritems():
       self.assertIsInstance(metricName, basestring)
       self.assertIsInstance(metric, dict)
       self.assertTrue(metric)
       self.assertIn("metricType", metric)
       self.assertIn("metricTypeName", metric)
       self.assertIn("modelParams", metric)
       self.assertIn("provider", metric)
       if metric["provider"] == "twitter":
         self.assertIn("screenNames", metric)
       elif metric["provider"] == "xignite":
         self.assertIn("sampleKey", metric)
コード例 #14
0
def _promoteReadyMetricsToModels():
  """Promote unmonitored company metrics that reached
  _NUM_METRIC_DATA_ROWS_THRESHOLD to models
  """

  # Build a map of all configured metric names to metric/model args for
  # promoting to models
  metricsConfig = metric_utils.getMetricsConfiguration()

  readyMetricNames = _filterMetricsReadyForPromotion(
    metricsConfig=metricsConfig,
    allCustomMetrics=metric_utils.getAllCustomMetrics(
      _TAURUS_HTM_SERVER,
      _TAURUS_API_KEY))

  if not readyMetricNames:
    g_log.debug("There are no metrics that are ready for promotion at this time")
    return

  # Promote them to models
  metric_utils.createAllModels(host=_TAURUS_HTM_SERVER,
                               apiKey=_TAURUS_API_KEY,
                               onlyMetricNames=readyMetricNames)
コード例 #15
0
 def testGetMetricsConfiguration(self):
     metrics = metric_utils.getMetricsConfiguration()
     self.assertIsInstance(metrics, dict)
     self.assertTrue(metrics)
     for companyName, details in metrics.iteritems():
         self.assertIsInstance(companyName, basestring)
         self.assertIsInstance(details, dict)
         self.assertTrue(details)
         self.assertIn("metrics", details)
         self.assertIn("stockExchange", details)
         self.assertIn("symbol", details)
         for metricName, metric in details["metrics"].iteritems():
             self.assertIsInstance(metricName, basestring)
             self.assertIsInstance(metric, dict)
             self.assertTrue(metric)
             self.assertIn("metricType", metric)
             self.assertIn("metricTypeName", metric)
             self.assertIn("modelParams", metric)
             self.assertIn("provider", metric)
             if metric["provider"] == "twitter":
                 self.assertIn("screenNames", metric)
             elif metric["provider"] == "xignite":
                 self.assertIn("sampleKey", metric)
コード例 #16
0
def loadMetricSpecs():
  """ Load metric specs for the xignite stock provider

  :returns: a sequence of StockMetricSpec objects

  Excerpt from metrics.json:
  {
    "Accenture": {
      "stockExchange": "NYSE",
      "symbol": "ACN",
      "metrics": {
        "XIGNITE.ACN.CLOSINGPRICE": {
          "metricTypeName": "Stock Price",
          "provider": "xignite",
          "sampleKey": "Close"
        },
        "XIGNITE.ACN.VOLUME": {
          "metricTypeName": "Stock Volume",
          "provider": "xignite",
          "sampleKey": "Volume"
        },
        . . .
      }
    },
    . . .
  }
  """
  return tuple(
    StockMetricSpec(
      metricName=metricName,
      symbol=resVal["symbol"].upper(),
      stockExchange=resVal["stockExchange"],
      sampleKey=metricVal["sampleKey"])
    for resVal in getMetricsConfiguration().itervalues()
    for metricName, metricVal in resVal["metrics"].iteritems()
    if metricVal["provider"] == "xignite" and "sampleKey" in metricVal)
コード例 #17
0
def main():
    """
  NOTE: main also serves as entry point for "console script" generated by setup
  """
    logging_support.LoggingSupport().initTool()

    try:
        options = _parseArgs()

        g_log.info("Verifying that agents are in hot_standby mode")
        for section in config.sections():
            try:
                assert config.get(section, "opmode") == ApplicationConfig.OP_MODE_HOT_STANDBY
            except Exception, e:
                raise

        g_log.info("Verifying that the old symbol has been removed from the " "metrics configuration")
        for stockData in metric_utils.getMetricsConfiguration().itervalues():
            assert stockData["symbol"] != options.old_symbol

        if options.twitter and (not options.stocks):
            g_log.info(
                "Migrating ONLY twitter data from old-symbol=%s " "to new-symbol=%s",
                options.old_symbol,
                options.new_symbol,
            )
        elif options.stocks and (not options.twitter):
            g_log.info(
                "Migrating ONLY xignite stock data from old-symbol=%s " "to new-symbol=%s",
                options.old_symbol,
                options.new_symbol,
            )
            raise NotImplementedError
        else:
            g_log.info(
                "Migrating BOTH twitter and xignite stock data from " "old-symbol=%s to new-symbol=%s",
                options.old_symbol,
                options.new_symbol,
            )
            raise NotImplementedError

        oldSymbolTweetPrefix = "TWITTER.TWEET.HANDLE.{symbol}.".format(symbol=options.old_symbol)
        newSymbolTweetPrefix = "TWITTER.TWEET.HANDLE.{symbol}.".format(symbol=options.new_symbol)
        oldSymbolTweetMetricsList = []

        with collectorsdb.engineFactory().begin() as conn:

            g_log.info("Renaming metrics to new symbol")
            if options.twitter:
                oldSymbolTweetsQuery = sql.select([tweetSamplesSchema]).where(
                    tweetSamplesSchema.c.metric.contains(oldSymbolTweetPrefix)
                )
                oldSymbolTweets = conn.execute(oldSymbolTweetsQuery)
                for tweetSample in oldSymbolTweets:
                    newMetricName = "{newPrefix}{metric}".format(
                        newPrefix=newSymbolTweetPrefix, metric=tweetSample.metric[len(oldSymbolTweetPrefix) :]
                    )
                    if tweetSample.metric not in oldSymbolTweetMetricsList:
                        oldSymbolTweetMetricsList.append(tweetSample.metric)

                    updateSampleQuery = (
                        tweetSamplesSchema.update()
                        .where(tweetSamplesSchema.c.seq == tweetSample.seq)
                        .values(metric=newMetricName)
                    )

                    conn.execute(updateSampleQuery)

            g_log.info("Forwarding new twitter metric data to Taurus engine...")
            if options.twitter:
                oldestRecordTs = conn.execute(
                    sql.select([tweetSamplesSchema.c.agg_ts], order_by=tweetSamplesSchema.c.agg_ts.asc())
                ).first()[0]
                lastEmittedAggTime = metric_utils.establishLastEmittedSampleDatetime(
                    key=_EMITTED_TWEET_VOLUME_SAMPLE_TRACKER_KEY, aggSec=options.aggPeriod
                )
                aggOffset = (
                    math.ceil(
                        (epochFromNaiveUTCDatetime(lastEmittedAggTime) - epochFromNaiveUTCDatetime(oldestRecordTs))
                        / options.aggPeriod
                    )
                    * options.aggPeriod
                )
                aggStartDatetime = (
                    lastEmittedAggTime - timedelta(seconds=aggOffset) - timedelta(seconds=options.aggPeriod)
                )

                metric_utils.updateLastEmittedSampleDatetime(
                    key=_EMITTED_TWEET_VOLUME_SAMPLE_TRACKER_KEY, sampleDatetime=aggStartDatetime
                )

                MetricDataForwarder.runInThread(
                    metricSpecs=loadMetricSpecs(),
                    aggSec=options.aggPeriod,
                    symbolList=[options.new_symbol],
                    forwardOnlyBacklog=True,
                )

                metric_utils.updateLastEmittedSampleDatetime(
                    key=_EMITTED_TWEET_VOLUME_SAMPLE_TRACKER_KEY, sampleDatetime=lastEmittedAggTime
                )

        g_log.info("Forwarding metrics to dynamodb using new symbol...")
        if options.twitter:
            migrate_tweets_to_dynamodb.main(symbolList=[options.new_symbol])

        g_log.info("Unmonitoring and deleting existing metrics associated with " "symbol=%s", options.old_symbol)
        oldModels = metric_utils.getSymbolModels(options.htmServer, options.apikey, options.old_symbol)
        for model in oldModels:
            metric_utils.unmonitorMetric(options.htmServer, options.apikey, model.uid)
            metric_utils.deleteMetric(options.htmServer, options.apikey, model.name)