def testFilterCompanyMetricNamesBySymbol(self):
negatives = [
"TWITTER.TWEET.HANDLE.ZZZ.VOLUME",
"XIGNITE.ZZZ.CLOSINGPRICE",
"XIGNITE.FOOBARZZZ.CLOSINGPRICE",
"XIGNITE.ZZZFOOBAR.CLOSINGPRICE",
"XIGNITE.FOOBAR.ZZZ.VOLUME",
"XIGNITE.NEWS.FOOBAR.ZZZ.VOLUME",
"FOOBAR.VOLUME",
".FOOBAR.CLOSINGPRICE",
"XIGNITE.FOOBAR.",
"FOOBARCLOSINGPRICE",
]
positives = [
"XIGNITE.FOOBAR.CLOSINGPRICE",
"XIGNITE.FOOBAR.VOLUME",
"TWITTER.TWEET.HANDLE.FOOBAR.VOLUME",
"XIGNITE.NEWS.FOOBAR.VOLUME",
]
# Execute
filteredNames = metric_utils.filterCompanyMetricNamesBySymbol(
metricNames=negatives + positives,
tickerSymbol="FOOBAR")
# Verify that the expected metric names were returned
self.assertItemsEqual(positives, filteredNames)
def testFilterCompanyMetricNamesBySymbol(self):
negatives = [
"TWITTER.TWEET.HANDLE.ZZZ.VOLUME",
"XIGNITE.ZZZ.CLOSINGPRICE",
"XIGNITE.FOOBARZZZ.CLOSINGPRICE",
"XIGNITE.ZZZFOOBAR.CLOSINGPRICE",
"XIGNITE.FOOBAR.ZZZ.VOLUME",
"XIGNITE.NEWS.FOOBAR.ZZZ.VOLUME",
"FOOBAR.VOLUME",
".FOOBAR.CLOSINGPRICE",
"XIGNITE.FOOBAR.",
"FOOBARCLOSINGPRICE",
]
positives = [
"XIGNITE.FOOBAR.CLOSINGPRICE",
"XIGNITE.FOOBAR.VOLUME",
"TWITTER.TWEET.HANDLE.FOOBAR.VOLUME",
"XIGNITE.NEWS.FOOBAR.VOLUME",
]
# Execute
filteredNames = metric_utils.filterCompanyMetricNamesBySymbol(
metricNames=negatives + positives, tickerSymbol="FOOBAR")
# Verify that the expected metric names were returned
self.assertItemsEqual(positives, filteredNames)
def deleteCompanies(tickerSymbols,
engineServer,
engineApiKey,
warnAboutDestructiveAction=True,
warningTimeout=_DEFAULT_WARNING_PROMPT_TIMEOUT_SEC):
"""Delete companies from Taurus Collector and their metrics/models from
Taurus Engine.
:param sequence tickerSymbols: stock ticker symbols of companies to be
deleted
:param str engineServer: dns name of ip addres of Taurus API server
:param str engineApiKey: API Key of Taurus HTM Engine
:param bool warnAboutDestructiveAction: whether to warn about destructive
action; defaults to True.
:param float warningTimeout: Timeout for the warning prompt; ignored if
warnAboutDestructiveAction is False
:raises WarningPromptTimeout: if warning prompt timed out
:raises UserAbortedOperation: if user chose to abort the operation
:raises FlusherMetricNotFound:
"""
tickerSymbols = tuple(symbol.upper() for symbol in tickerSymbols)
# Check for duplicate symbols
repeatedSymbols = set(sym for sym in tickerSymbols
if tickerSymbols.count(sym) > 1)
if repeatedSymbols:
raise ValueError("{numRepeats} symbol(s) are present more than once in "
"tickerSymbols arg: {repeats}"
.format(numRepeats=len(repeatedSymbols),
repeats=repeatedSymbols))
# Set will be handier going forward
tickerSymbols = set(tickerSymbols)
if warnAboutDestructiveAction:
_warnAboutDestructiveAction(timeout=warningTimeout,
tickerSymbols=tickerSymbols,
engineServer=engineServer)
# If any of the ticker symbols still appear in the collector's metrics config,
# abort the operation as a precautionary measure.
allSymbols = set(security[0].upper() for security in
metric_utils.getAllMetricSecurities())
problemSymbols = tickerSymbols & allSymbols
assert not problemSymbols, (
"Can't delete - {numProblem} of the specified companies [{symbols}] are "
"in active metrics configuration".format(numProblem=len(problemSymbols),
symbols=problemSymbols))
# First, we need to synchronize with Taurus Engine's metric data path.
# If any of the data still in the pipeline is for any of the companies being
# deleted, then the metrics may be re-created in the Engine after we delete
# them. This is an yet unresolved subtlety with custom metrics in htmengine.
_flushTaurusEngineMetricDataPath(engineServer, engineApiKey)
# NOTE: We must query custom metrics after flushing the metric data path,
# since metrics may get created as a side-effect of processing metric data.
allMetricsMap = {
obj["name"] : obj
for obj in
metric_utils.getAllCustomMetrics(host=engineServer, apiKey=engineApiKey)
}
allMetricNames = allMetricsMap.keys()
for symbolNum, symbol in enumerate(tickerSymbols, 1):
# Delete corresponding metrics from Taurus Engine
metricNamesToDelete = metric_utils.filterCompanyMetricNamesBySymbol(
allMetricNames,
symbol)
if not metricNamesToDelete:
g_log.info("No metrics to delete for symbol=%s (%d of %d)", symbol,
symbolNum, len(tickerSymbols))
continue
g_log.info("Deleting metrics and models for ticker symbol=%s from Taurus "
"Engine=%s (%d of %d)", symbol, engineServer,
symbolNum, len(tickerSymbols))
for metricName in metricNamesToDelete:
metric_utils.deleteMetric(host=engineServer,
apiKey=engineApiKey,
metricName=metricName)
g_log.info("Deleted metric name=%s, uid=%s", metricName,
allMetricsMap[metricName]["uid"])
# Delete the symbol from xignite_security table last; this cascades to
# delete related rows in other tables via cascading delete relationship.
#
# NOTE: garbage collection from other tables not tied to xiginte_security
# symbols presently depends on aging of the rows (e.g., twitter tables).
# After ENG-83, all company-specific rows from all tables will be
# cleaned up and THIS NOTE SHOULD THEN BE REMOVED
with collectorsdb.engineFactory().begin() as conn:
numDeleted = (
conn.execute(
collectorsdb.schema.xigniteSecurity # pylint: disable=E1120
.delete()
.where(collectorsdb.schema.xigniteSecurity.c.symbol == symbol))
).rowcount
if numDeleted:
g_log.info("Deleted row=%s from table=%s", symbol,
collectorsdb.schema.xigniteSecurity)
else:
g_log.warning(
"Couldn't delete security row=%s: not found in table=%s",
symbol, collectorsdb.schema.xigniteSecurity)
def deleteCompanies(tickerSymbols,
engineServer,
engineApiKey,
warnAboutDestructiveAction=True,
warningTimeout=_DEFAULT_WARNING_PROMPT_TIMEOUT_SEC):
"""Delete companies from Taurus Collector and their metrics/models from
Taurus Engine.
:param sequence tickerSymbols: stock ticker symbols of companies to be
deleted
:param str engineServer: dns name of ip addres of Taurus API server
:param str engineApiKey: API Key of Taurus HTM Engine
:param bool warnAboutDestructiveAction: whether to warn about destructive
action; defaults to True.
:param float warningTimeout: Timeout for the warning prompt; ignored if
warnAboutDestructiveAction is False
:raises WarningPromptTimeout: if warning prompt timed out
:raises UserAbortedOperation: if user chose to abort the operation
:raises FlusherMetricNotFound:
"""
tickerSymbols = tuple(symbol.upper() for symbol in tickerSymbols)
# Check for duplicate symbols
repeatedSymbols = set(sym for sym in tickerSymbols
if tickerSymbols.count(sym) > 1)
if repeatedSymbols:
raise ValueError(
"{numRepeats} symbol(s) are present more than once in "
"tickerSymbols arg: {repeats}".format(
numRepeats=len(repeatedSymbols), repeats=repeatedSymbols))
# Set will be handier going forward
tickerSymbols = set(tickerSymbols)
if warnAboutDestructiveAction:
_warnAboutDestructiveAction(timeout=warningTimeout,
tickerSymbols=tickerSymbols,
engineServer=engineServer)
# If any of the ticker symbols still appear in the collector's metrics config,
# abort the operation as a precautionary measure.
allSymbols = set(security[0].upper()
for security in metric_utils.getAllMetricSecurities())
problemSymbols = tickerSymbols & allSymbols
assert not problemSymbols, (
"Can't delete - {numProblem} of the specified companies [{symbols}] are "
"in active metrics configuration".format(
numProblem=len(problemSymbols), symbols=problemSymbols))
# First, we need to synchronize with Taurus Engine's metric data path.
# If any of the data still in the pipeline is for any of the companies being
# deleted, then the metrics may be re-created in the Engine after we delete
# them. This is an yet unresolved subtlety with custom metrics in htmengine.
_flushTaurusEngineMetricDataPath(engineServer, engineApiKey)
# NOTE: We must query custom metrics after flushing the metric data path,
# since metrics may get created as a side-effect of processing metric data.
allMetricsMap = {
obj["name"]: obj
for obj in metric_utils.getAllCustomMetrics(host=engineServer,
apiKey=engineApiKey)
}
allMetricNames = allMetricsMap.keys()
for symbolNum, symbol in enumerate(tickerSymbols, 1):
# Delete corresponding metrics from Taurus Engine
metricNamesToDelete = metric_utils.filterCompanyMetricNamesBySymbol(
allMetricNames, symbol)
if not metricNamesToDelete:
g_log.info("No metrics to delete for symbol=%s (%d of %d)", symbol,
symbolNum, len(tickerSymbols))
continue
g_log.info(
"Deleting metrics and models for ticker symbol=%s from Taurus "
"Engine=%s (%d of %d)", symbol, engineServer, symbolNum,
len(tickerSymbols))
for metricName in metricNamesToDelete:
metric_utils.deleteMetric(host=engineServer,
apiKey=engineApiKey,
metricName=metricName)
g_log.info("Deleted metric name=%s, uid=%s", metricName,
allMetricsMap[metricName]["uid"])
# Delete the symbol from xignite_security table last; this cascades to
# delete related rows in other tables via cascading delete relationship.
#
# NOTE: garbage collection from other tables not tied to xiginte_security
# symbols presently depends on aging of the rows (e.g., twitter tables).
# After ENG-83, all company-specific rows from all tables will be
# cleaned up and THIS NOTE SHOULD THEN BE REMOVED
with collectorsdb.engineFactory().begin() as conn:
numDeleted = (
conn.execute(collectorsdb.schema.xigniteSecurity # pylint: disable=E1120
.delete().where(
collectorsdb.schema.xigniteSecurity.c.symbol
== symbol))).rowcount
if numDeleted:
g_log.info("Deleted row=%s from table=%s", symbol,
collectorsdb.schema.xigniteSecurity)
else:
g_log.warning(
"Couldn't delete security row=%s: not found in table=%s",
symbol, collectorsdb.schema.xigniteSecurity)
