def testCreateAllModels(self):
host = os.environ.get("TAURUS_HTM_SERVER", "127.0.0.1")
apikey = os.environ.get("TAURUS_APIKEY", "taurus")
# Resize metrics down to a much smaller random sample of the original
# so as to not overload the system under test. We need only to test that
# everything returned goes through the right channels.
metrics = {
key:value
for (key, value)
in random.sample(metric_utils.getMetricsConfiguration().items(), 3)
}
with patch("taurus.metric_collectors.metric_utils.getMetricsConfiguration",
return_value=metrics,
spec_set=metric_utils.getMetricsConfiguration):
createdModels = metric_utils.createAllModels(host, apikey)
allModels = metric_utils.getAllModels(host, apikey)
for model in createdModels:
self.addCleanup(requests.delete,
"https://%s/_metrics/custom/%s" % (host, model["name"]),
auth=(apikey, ""),
verify=False)
remoteModel = metric_utils.getOneModel(host, apikey, model["uid"])
self.assertDictEqual(remoteModel, model)
self.assertIn(model, allModels)
def testCreateAllModelsHappyPath(self, requestsMock):
requestsMock.post.return_value = Mock(status_code=201,
text='[{"uid":"foo", "name":"bar"}]')
totalModels = len(
metric_utils.getMetricNamesFromConfig(
metric_utils.getMetricsConfiguration()))
metric_utils.createAllModels("localhost", "taurus")
self.assertEqual(requestsMock.post.call_count, totalModels)
for args, kwargs in requestsMock.post.call_args_list:
self.assertEqual(args[0], "https://localhost/_models")
self.assertIn("data", kwargs)
data = json.loads(kwargs["data"])
self.assertIsInstance(data, dict)
self.assertIn("datasource", data)
self.assertEquals(data["datasource"], "custom")
self.assertIn("metricSpec", data)
self.assertIn("metric", data["metricSpec"])
self.assertIn("resource", data["metricSpec"])
self.assertIn("userInfo", data["metricSpec"])
self.assertIsInstance(data["metricSpec"]["userInfo"], dict)
self.assertIn("metricType", data["metricSpec"]["userInfo"])
self.assertIn("metricTypeName", data["metricSpec"]["userInfo"])
self.assertIn("symbol", data["metricSpec"]["userInfo"])
self.assertIn("modelParams", data)
def loadMetricSpecs():
""" Load metric specs for the xignite stock provider
:returns: a sequence of StockMetricSpec objects
Excerpt from metrics.json:
{
"Accenture": {
"stockExchange": "NYSE",
"symbol": "ACN",
"metrics": {
"XIGNITE.ACN.CLOSINGPRICE": {
"metricTypeName": "Stock Price",
"provider": "xignite",
"sampleKey": "Close"
},
"XIGNITE.ACN.VOLUME": {
"metricTypeName": "Stock Volume",
"provider": "xignite",
"sampleKey": "Volume"
},
. . .
}
},
. . .
}
"""
return tuple(
StockMetricSpec(metricName=metricName,
symbol=resVal["symbol"].upper(),
stockExchange=resVal["stockExchange"],
sampleKey=metricVal["sampleKey"])
for resVal in getMetricsConfiguration().itervalues()
for metricName, metricVal in resVal["metrics"].iteritems()
if metricVal["provider"] == "xignite" and "sampleKey" in metricVal)
def testCreateAllModelsHappyPath(self, requestsMock):
requestsMock.post.return_value = Mock(
status_code=201, text='[{"uid":"foo", "name":"bar"}]')
totalModels = len(
metric_utils.getMetricNamesFromConfig(
metric_utils.getMetricsConfiguration()))
metric_utils.createAllModels("localhost", "taurus")
self.assertEqual(requestsMock.post.call_count, totalModels)
for args, kwargs in requestsMock.post.call_args_list:
self.assertEqual(args[0], "https://localhost/_models")
self.assertIn("data", kwargs)
data = json.loads(kwargs["data"])
self.assertIsInstance(data, dict)
self.assertIn("datasource", data)
self.assertEquals(data["datasource"], "custom")
self.assertIn("metricSpec", data)
self.assertIn("metric", data["metricSpec"])
self.assertIn("resource", data["metricSpec"])
self.assertIn("userInfo", data["metricSpec"])
self.assertIsInstance(data["metricSpec"]["userInfo"], dict)
self.assertIn("metricType", data["metricSpec"]["userInfo"])
self.assertIn("metricTypeName", data["metricSpec"]["userInfo"])
self.assertIn("symbol", data["metricSpec"]["userInfo"])
self.assertIn("modelParams", data)
def testCreateAllModels(self):
host = os.environ.get("TAURUS_HTM_SERVER", "127.0.0.1")
apikey = os.environ.get("TAURUS_APIKEY", "taurus")
# Resize metrics down to a much smaller random sample of the original
# so as to not overload the system under test. We need only to test that
# everything returned goes through the right channels.
metrics = {
key: value
for (key, value) in random.sample(
metric_utils.getMetricsConfiguration().items(), 3)
}
with patch(
"taurus.metric_collectors.metric_utils.getMetricsConfiguration",
return_value=metrics,
spec_set=metric_utils.getMetricsConfiguration):
createdModels = metric_utils.createAllModels(host, apikey)
allModels = metric_utils.getAllModels(host, apikey)
for model in createdModels:
self.addCleanup(requests.delete,
"https://%s/_metrics/custom/%s" %
(host, model["name"]),
auth=(apikey, ""),
verify=False)
remoteModel = metric_utils.getOneModel(host, apikey, model["uid"])
self.assertDictEqual(remoteModel, model)
self.assertIn(model, allModels)
def _loadNewsVolumeMetricSpecs():
""" Load metric specs for our securities news volume provider
:returns: a sequence of NewsVolumeMetricSpec objects
"""
return tuple(
NewsVolumeMetricSpec(metric=metricName, symbol=resVal["symbol"])
for resVal in getMetricsConfiguration().itervalues()
for metricName, metricVal in resVal["metrics"].iteritems()
if metricVal["provider"] == _NEWS_VOLUME_METRIC_PROVIDER)
def _loadNewsVolumeMetricSpecs():
""" Load metric specs for our securities news volume provider
:returns: a sequence of NewsVolumeMetricSpec objects
"""
return tuple(
NewsVolumeMetricSpec(metric=metricName,
symbol=resVal["symbol"])
for resVal in getMetricsConfiguration().itervalues()
for metricName, metricVal in resVal["metrics"].iteritems()
if metricVal["provider"] == _NEWS_VOLUME_METRIC_PROVIDER)
def main():
"""
NOTE: main also serves as entry point for "console script" generated by setup
"""
logging_support.LoggingSupport().initTool()
try:
options = _parseArgs()
allSymbols = set(stockData["symbol"] for stockData in
metric_utils.getMetricsConfiguration().itervalues() )
g_log.info("Verifying that agents are in hot_standby mode")
for section in config.sections():
assert config.get(section, "opmode") == config.OP_MODE_HOT_STANDBY
g_log.info("Verifying that the old symbol has been removed from the "
"metrics configuration")
assert options.oldSymbol not in allSymbols
g_log.info("Verifying that the new symbol is present in the metrics "
"configuration")
assert options.newSymbol in allSymbols
g_log.info("Migrating BOTH twitter and xignite stock data from "
"old-symbol=%s to new-symbol=%s",
options.oldSymbol, options.newSymbol)
# Rename the metrics in collectorsdb and forward new metric samples to HTM
# Engine
g_log.info("Modifying old metrics with new symbol")
_resymbolTweetVolumeMetric(oldSymbol=options.oldSymbol,
newSymbol=options.newSymbol,
aggPeriod=options.aggPeriod)
_resymbolStockMetrics(oldSymbol=options.oldSymbol,
newSymbol=options.newSymbol)
# Delete metrics linked to old stock symbol from Taurus Engine
delete_companies.deleteCompanies(
tickerSymbols=[options.oldSymbol],
engineServer=options.htmServer,
engineApiKey=options.apikey,
warnAboutDestructiveAction=False)
except SystemExit as e:
if e.code != 0:
g_log.exception("Failed!")
raise
except Exception:
g_log.exception("Failed!")
raise
def main():
"""
NOTE: main also serves as entry point for "console script" generated by setup
"""
logging_support.LoggingSupport().initTool()
try:
options = _parseArgs()
allSymbols = set(stockData["symbol"] for stockData in
metric_utils.getMetricsConfiguration().itervalues())
g_log.info("Verifying that agents are in hot_standby mode")
for section in config.sections():
assert config.get(section, "opmode") == config.OP_MODE_HOT_STANDBY
g_log.info("Verifying that the old symbol has been removed from the "
"metrics configuration")
assert options.oldSymbol not in allSymbols
g_log.info("Verifying that the new symbol is present in the metrics "
"configuration")
assert options.newSymbol in allSymbols
g_log.info(
"Migrating BOTH twitter and xignite stock data from "
"old-symbol=%s to new-symbol=%s", options.oldSymbol,
options.newSymbol)
# Rename the metrics in collectorsdb and forward new metric samples to HTM
# Engine
g_log.info("Modifying old metrics with new symbol")
_resymbolTweetVolumeMetric(oldSymbol=options.oldSymbol,
newSymbol=options.newSymbol,
aggPeriod=options.aggPeriod)
_resymbolStockMetrics(oldSymbol=options.oldSymbol,
newSymbol=options.newSymbol)
# Delete metrics linked to old stock symbol from Taurus Engine
delete_companies.deleteCompanies(tickerSymbols=[options.oldSymbol],
engineServer=options.htmServer,
engineApiKey=options.apikey,
warnAboutDestructiveAction=False)
except SystemExit as e:
if e.code != 0:
g_log.exception("Failed!")
raise
except Exception:
g_log.exception("Failed!")
raise
def testCreateAllModels(self):
host = os.environ.get("TAURUS_HTM_SERVER", "127.0.0.1")
apikey = os.environ.get("TAURUS_APIKEY", "taurus")
# Resize metrics down to a much smaller random sample of the original
# so as to not overload the system under test. We need only to test that
# everything returned goes through the right channels.
metricsConfig = {
key:value
for (key, value)
in random.sample(metric_utils.getMetricsConfiguration().items(), 3)
}
expectedMetricNames = []
for resVal in metricsConfig.itervalues():
for metricName in resVal["metrics"]:
expectedMetricNames.append(metricName)
self.addCleanup(requests.delete,
"https://%s/_metrics/custom/%s" % (host, metricName),
auth=(apikey, ""),
verify=False)
self.assertGreater(len(expectedMetricNames), 0)
with patch("taurus.metric_collectors.metric_utils.getMetricsConfiguration",
return_value=metricsConfig,
spec_set=metric_utils.getMetricsConfiguration):
createdModels = metric_utils.createAllModels(host, apikey)
self.assertEqual(len(createdModels), len(expectedMetricNames))
for model in createdModels:
remoteModel = metric_utils.getOneModel(host, apikey, model["uid"])
self.assertIn(remoteModel["name"], expectedMetricNames)
# Verify that the model is either in "ACTIVE" or the transient
# "PENDNG DATA" or "CREATE PENDING" states
self.assertIn(remoteModel["status"], [1, 2, 8])
def testCreateAllModelsWithMetricNameFilter(self,
createCustomHtmModelMock):
allMetricNames = metric_utils.getMetricNamesFromConfig(
metric_utils.getMetricsConfiguration())
subsetOfMetricNames = allMetricNames[:(len(allMetricNames) + 1) // 2]
self.assertGreater(len(subsetOfMetricNames), 0)
createCustomHtmModelMock.side_effect = (lambda **kwargs: dict(
name=kwargs["metricName"], uid=kwargs["metricName"] * 2))
models = metric_utils.createAllModels(
host="host", apiKey="apikey", onlyMetricNames=subsetOfMetricNames)
self.assertEqual(createCustomHtmModelMock.call_count,
len(subsetOfMetricNames))
self.assertEqual(len(models), len(subsetOfMetricNames))
self.assertItemsEqual(subsetOfMetricNames,
[model["name"] for model in models])
def testCreateAllModelsWithMetricNameFilter(self, createCustomHtmModelMock):
allMetricNames = metric_utils.getMetricNamesFromConfig(
metric_utils.getMetricsConfiguration())
subsetOfMetricNames = allMetricNames[:(len(allMetricNames) + 1) // 2]
self.assertGreater(len(subsetOfMetricNames), 0)
createCustomHtmModelMock.side_effect = (
lambda **kwargs: dict(name=kwargs["metricName"],
uid=kwargs["metricName"] * 2))
models = metric_utils.createAllModels(host="host",
apiKey="apikey",
onlyMetricNames=subsetOfMetricNames)
self.assertEqual(createCustomHtmModelMock.call_count,
len(subsetOfMetricNames))
self.assertEqual(len(models), len(subsetOfMetricNames))
self.assertItemsEqual(subsetOfMetricNames,
[model["name"] for model in models])
def testGetMetricsConfiguration(self):
metrics = metric_utils.getMetricsConfiguration()
self.assertIsInstance(metrics, dict)
self.assertTrue(metrics)
for companyName, details in metrics.iteritems():
self.assertIsInstance(companyName, basestring)
self.assertIsInstance(details, dict)
self.assertTrue(details)
self.assertIn("metrics", details)
self.assertIn("stockExchange", details)
self.assertIn("symbol", details)
for metricName, metric in details["metrics"].iteritems():
self.assertIsInstance(metricName, basestring)
self.assertIsInstance(metric, dict)
self.assertTrue(metric)
self.assertIn("metricType", metric)
self.assertIn("metricTypeName", metric)
self.assertIn("modelParams", metric)
self.assertIn("provider", metric)
if metric["provider"] == "twitter":
self.assertIn("screenNames", metric)
elif metric["provider"] == "xignite":
self.assertIn("sampleKey", metric)
def _promoteReadyMetricsToModels():
"""Promote unmonitored company metrics that reached
_NUM_METRIC_DATA_ROWS_THRESHOLD to models
"""
# Build a map of all configured metric names to metric/model args for
# promoting to models
metricsConfig = metric_utils.getMetricsConfiguration()
readyMetricNames = _filterMetricsReadyForPromotion(
metricsConfig=metricsConfig,
allCustomMetrics=metric_utils.getAllCustomMetrics(
_TAURUS_HTM_SERVER,
_TAURUS_API_KEY))
if not readyMetricNames:
g_log.debug("There are no metrics that are ready for promotion at this time")
return
# Promote them to models
metric_utils.createAllModels(host=_TAURUS_HTM_SERVER,
apiKey=_TAURUS_API_KEY,
onlyMetricNames=readyMetricNames)
def testGetMetricsConfiguration(self):
metrics = metric_utils.getMetricsConfiguration()
self.assertIsInstance(metrics, dict)
self.assertTrue(metrics)
for companyName, details in metrics.iteritems():
self.assertIsInstance(companyName, basestring)
self.assertIsInstance(details, dict)
self.assertTrue(details)
self.assertIn("metrics", details)
self.assertIn("stockExchange", details)
self.assertIn("symbol", details)
for metricName, metric in details["metrics"].iteritems():
self.assertIsInstance(metricName, basestring)
self.assertIsInstance(metric, dict)
self.assertTrue(metric)
self.assertIn("metricType", metric)
self.assertIn("metricTypeName", metric)
self.assertIn("modelParams", metric)
self.assertIn("provider", metric)
if metric["provider"] == "twitter":
self.assertIn("screenNames", metric)
elif metric["provider"] == "xignite":
self.assertIn("sampleKey", metric)
def loadMetricSpecs():
""" Load metric specs for the xignite stock provider
:returns: a sequence of StockMetricSpec objects
Excerpt from metrics.json:
{
"Accenture": {
"stockExchange": "NYSE",
"symbol": "ACN",
"metrics": {
"XIGNITE.ACN.CLOSINGPRICE": {
"metricTypeName": "Stock Price",
"provider": "xignite",
"sampleKey": "Close"
},
"XIGNITE.ACN.VOLUME": {
"metricTypeName": "Stock Volume",
"provider": "xignite",
"sampleKey": "Volume"
},
. . .
}
},
. . .
}
"""
return tuple(
StockMetricSpec(
metricName=metricName,
symbol=resVal["symbol"].upper(),
stockExchange=resVal["stockExchange"],
sampleKey=metricVal["sampleKey"])
for resVal in getMetricsConfiguration().itervalues()
for metricName, metricVal in resVal["metrics"].iteritems()
if metricVal["provider"] == "xignite" and "sampleKey" in metricVal)
def main():
"""
NOTE: main also serves as entry point for "console script" generated by setup
"""
logging_support.LoggingSupport().initTool()
try:
options = _parseArgs()
g_log.info("Verifying that agents are in hot_standby mode")
for section in config.sections():
try:
assert config.get(section, "opmode") == ApplicationConfig.OP_MODE_HOT_STANDBY
except Exception, e:
raise
g_log.info("Verifying that the old symbol has been removed from the " "metrics configuration")
for stockData in metric_utils.getMetricsConfiguration().itervalues():
assert stockData["symbol"] != options.old_symbol
if options.twitter and (not options.stocks):
g_log.info(
"Migrating ONLY twitter data from old-symbol=%s " "to new-symbol=%s",
options.old_symbol,
options.new_symbol,
)
elif options.stocks and (not options.twitter):
g_log.info(
"Migrating ONLY xignite stock data from old-symbol=%s " "to new-symbol=%s",
options.old_symbol,
options.new_symbol,
)
raise NotImplementedError
else:
g_log.info(
"Migrating BOTH twitter and xignite stock data from " "old-symbol=%s to new-symbol=%s",
options.old_symbol,
options.new_symbol,
)
raise NotImplementedError
oldSymbolTweetPrefix = "TWITTER.TWEET.HANDLE.{symbol}.".format(symbol=options.old_symbol)
newSymbolTweetPrefix = "TWITTER.TWEET.HANDLE.{symbol}.".format(symbol=options.new_symbol)
oldSymbolTweetMetricsList = []
with collectorsdb.engineFactory().begin() as conn:
g_log.info("Renaming metrics to new symbol")
if options.twitter:
oldSymbolTweetsQuery = sql.select([tweetSamplesSchema]).where(
tweetSamplesSchema.c.metric.contains(oldSymbolTweetPrefix)
)
oldSymbolTweets = conn.execute(oldSymbolTweetsQuery)
for tweetSample in oldSymbolTweets:
newMetricName = "{newPrefix}{metric}".format(
newPrefix=newSymbolTweetPrefix, metric=tweetSample.metric[len(oldSymbolTweetPrefix) :]
)
if tweetSample.metric not in oldSymbolTweetMetricsList:
oldSymbolTweetMetricsList.append(tweetSample.metric)
updateSampleQuery = (
tweetSamplesSchema.update()
.where(tweetSamplesSchema.c.seq == tweetSample.seq)
.values(metric=newMetricName)
)
conn.execute(updateSampleQuery)
g_log.info("Forwarding new twitter metric data to Taurus engine...")
if options.twitter:
oldestRecordTs = conn.execute(
sql.select([tweetSamplesSchema.c.agg_ts], order_by=tweetSamplesSchema.c.agg_ts.asc())
).first()[0]
lastEmittedAggTime = metric_utils.establishLastEmittedSampleDatetime(
key=_EMITTED_TWEET_VOLUME_SAMPLE_TRACKER_KEY, aggSec=options.aggPeriod
)
aggOffset = (
math.ceil(
(epochFromNaiveUTCDatetime(lastEmittedAggTime) - epochFromNaiveUTCDatetime(oldestRecordTs))
/ options.aggPeriod
)
* options.aggPeriod
)
aggStartDatetime = (
lastEmittedAggTime - timedelta(seconds=aggOffset) - timedelta(seconds=options.aggPeriod)
)
metric_utils.updateLastEmittedSampleDatetime(
key=_EMITTED_TWEET_VOLUME_SAMPLE_TRACKER_KEY, sampleDatetime=aggStartDatetime
)
MetricDataForwarder.runInThread(
metricSpecs=loadMetricSpecs(),
aggSec=options.aggPeriod,
symbolList=[options.new_symbol],
forwardOnlyBacklog=True,
)
metric_utils.updateLastEmittedSampleDatetime(
key=_EMITTED_TWEET_VOLUME_SAMPLE_TRACKER_KEY, sampleDatetime=lastEmittedAggTime
)
g_log.info("Forwarding metrics to dynamodb using new symbol...")
if options.twitter:
migrate_tweets_to_dynamodb.main(symbolList=[options.new_symbol])
g_log.info("Unmonitoring and deleting existing metrics associated with " "symbol=%s", options.old_symbol)
oldModels = metric_utils.getSymbolModels(options.htmServer, options.apikey, options.old_symbol)
for model in oldModels:
metric_utils.unmonitorMetric(options.htmServer, options.apikey, model.uid)
metric_utils.deleteMetric(options.htmServer, options.apikey, model.name)
