def testFilterCompanyMetricNamesBySymbol(self): negatives = [ "TWITTER.TWEET.HANDLE.ZZZ.VOLUME", "XIGNITE.ZZZ.CLOSINGPRICE", "XIGNITE.FOOBARZZZ.CLOSINGPRICE", "XIGNITE.ZZZFOOBAR.CLOSINGPRICE", "XIGNITE.FOOBAR.ZZZ.VOLUME", "XIGNITE.NEWS.FOOBAR.ZZZ.VOLUME", "FOOBAR.VOLUME", ".FOOBAR.CLOSINGPRICE", "XIGNITE.FOOBAR.", "FOOBARCLOSINGPRICE", ] positives = [ "XIGNITE.FOOBAR.CLOSINGPRICE", "XIGNITE.FOOBAR.VOLUME", "TWITTER.TWEET.HANDLE.FOOBAR.VOLUME", "XIGNITE.NEWS.FOOBAR.VOLUME", ] # Execute filteredNames = metric_utils.filterCompanyMetricNamesBySymbol( metricNames=negatives + positives, tickerSymbol="FOOBAR") # Verify that the expected metric names were returned self.assertItemsEqual(positives, filteredNames)
def deleteCompanies(tickerSymbols, engineServer, engineApiKey, warnAboutDestructiveAction=True, warningTimeout=_DEFAULT_WARNING_PROMPT_TIMEOUT_SEC): """Delete companies from Taurus Collector and their metrics/models from Taurus Engine. :param sequence tickerSymbols: stock ticker symbols of companies to be deleted :param str engineServer: dns name of ip addres of Taurus API server :param str engineApiKey: API Key of Taurus HTM Engine :param bool warnAboutDestructiveAction: whether to warn about destructive action; defaults to True. :param float warningTimeout: Timeout for the warning prompt; ignored if warnAboutDestructiveAction is False :raises WarningPromptTimeout: if warning prompt timed out :raises UserAbortedOperation: if user chose to abort the operation :raises FlusherMetricNotFound: """ tickerSymbols = tuple(symbol.upper() for symbol in tickerSymbols) # Check for duplicate symbols repeatedSymbols = set(sym for sym in tickerSymbols if tickerSymbols.count(sym) > 1) if repeatedSymbols: raise ValueError("{numRepeats} symbol(s) are present more than once in " "tickerSymbols arg: {repeats}" .format(numRepeats=len(repeatedSymbols), repeats=repeatedSymbols)) # Set will be handier going forward tickerSymbols = set(tickerSymbols) if warnAboutDestructiveAction: _warnAboutDestructiveAction(timeout=warningTimeout, tickerSymbols=tickerSymbols, engineServer=engineServer) # If any of the ticker symbols still appear in the collector's metrics config, # abort the operation as a precautionary measure. allSymbols = set(security[0].upper() for security in metric_utils.getAllMetricSecurities()) problemSymbols = tickerSymbols & allSymbols assert not problemSymbols, ( "Can't delete - {numProblem} of the specified companies [{symbols}] are " "in active metrics configuration".format(numProblem=len(problemSymbols), symbols=problemSymbols)) # First, we need to synchronize with Taurus Engine's metric data path. # If any of the data still in the pipeline is for any of the companies being # deleted, then the metrics may be re-created in the Engine after we delete # them. This is an yet unresolved subtlety with custom metrics in htmengine. _flushTaurusEngineMetricDataPath(engineServer, engineApiKey) # NOTE: We must query custom metrics after flushing the metric data path, # since metrics may get created as a side-effect of processing metric data. allMetricsMap = { obj["name"] : obj for obj in metric_utils.getAllCustomMetrics(host=engineServer, apiKey=engineApiKey) } allMetricNames = allMetricsMap.keys() for symbolNum, symbol in enumerate(tickerSymbols, 1): # Delete corresponding metrics from Taurus Engine metricNamesToDelete = metric_utils.filterCompanyMetricNamesBySymbol( allMetricNames, symbol) if not metricNamesToDelete: g_log.info("No metrics to delete for symbol=%s (%d of %d)", symbol, symbolNum, len(tickerSymbols)) continue g_log.info("Deleting metrics and models for ticker symbol=%s from Taurus " "Engine=%s (%d of %d)", symbol, engineServer, symbolNum, len(tickerSymbols)) for metricName in metricNamesToDelete: metric_utils.deleteMetric(host=engineServer, apiKey=engineApiKey, metricName=metricName) g_log.info("Deleted metric name=%s, uid=%s", metricName, allMetricsMap[metricName]["uid"]) # Delete the symbol from xignite_security table last; this cascades to # delete related rows in other tables via cascading delete relationship. # # NOTE: garbage collection from other tables not tied to xiginte_security # symbols presently depends on aging of the rows (e.g., twitter tables). # After ENG-83, all company-specific rows from all tables will be # cleaned up and THIS NOTE SHOULD THEN BE REMOVED with collectorsdb.engineFactory().begin() as conn: numDeleted = ( conn.execute( collectorsdb.schema.xigniteSecurity # pylint: disable=E1120 .delete() .where(collectorsdb.schema.xigniteSecurity.c.symbol == symbol)) ).rowcount if numDeleted: g_log.info("Deleted row=%s from table=%s", symbol, collectorsdb.schema.xigniteSecurity) else: g_log.warning( "Couldn't delete security row=%s: not found in table=%s", symbol, collectorsdb.schema.xigniteSecurity)
def deleteCompanies(tickerSymbols, engineServer, engineApiKey, warnAboutDestructiveAction=True, warningTimeout=_DEFAULT_WARNING_PROMPT_TIMEOUT_SEC): """Delete companies from Taurus Collector and their metrics/models from Taurus Engine. :param sequence tickerSymbols: stock ticker symbols of companies to be deleted :param str engineServer: dns name of ip addres of Taurus API server :param str engineApiKey: API Key of Taurus HTM Engine :param bool warnAboutDestructiveAction: whether to warn about destructive action; defaults to True. :param float warningTimeout: Timeout for the warning prompt; ignored if warnAboutDestructiveAction is False :raises WarningPromptTimeout: if warning prompt timed out :raises UserAbortedOperation: if user chose to abort the operation :raises FlusherMetricNotFound: """ tickerSymbols = tuple(symbol.upper() for symbol in tickerSymbols) # Check for duplicate symbols repeatedSymbols = set(sym for sym in tickerSymbols if tickerSymbols.count(sym) > 1) if repeatedSymbols: raise ValueError( "{numRepeats} symbol(s) are present more than once in " "tickerSymbols arg: {repeats}".format( numRepeats=len(repeatedSymbols), repeats=repeatedSymbols)) # Set will be handier going forward tickerSymbols = set(tickerSymbols) if warnAboutDestructiveAction: _warnAboutDestructiveAction(timeout=warningTimeout, tickerSymbols=tickerSymbols, engineServer=engineServer) # If any of the ticker symbols still appear in the collector's metrics config, # abort the operation as a precautionary measure. allSymbols = set(security[0].upper() for security in metric_utils.getAllMetricSecurities()) problemSymbols = tickerSymbols & allSymbols assert not problemSymbols, ( "Can't delete - {numProblem} of the specified companies [{symbols}] are " "in active metrics configuration".format( numProblem=len(problemSymbols), symbols=problemSymbols)) # First, we need to synchronize with Taurus Engine's metric data path. # If any of the data still in the pipeline is for any of the companies being # deleted, then the metrics may be re-created in the Engine after we delete # them. This is an yet unresolved subtlety with custom metrics in htmengine. _flushTaurusEngineMetricDataPath(engineServer, engineApiKey) # NOTE: We must query custom metrics after flushing the metric data path, # since metrics may get created as a side-effect of processing metric data. allMetricsMap = { obj["name"]: obj for obj in metric_utils.getAllCustomMetrics(host=engineServer, apiKey=engineApiKey) } allMetricNames = allMetricsMap.keys() for symbolNum, symbol in enumerate(tickerSymbols, 1): # Delete corresponding metrics from Taurus Engine metricNamesToDelete = metric_utils.filterCompanyMetricNamesBySymbol( allMetricNames, symbol) if not metricNamesToDelete: g_log.info("No metrics to delete for symbol=%s (%d of %d)", symbol, symbolNum, len(tickerSymbols)) continue g_log.info( "Deleting metrics and models for ticker symbol=%s from Taurus " "Engine=%s (%d of %d)", symbol, engineServer, symbolNum, len(tickerSymbols)) for metricName in metricNamesToDelete: metric_utils.deleteMetric(host=engineServer, apiKey=engineApiKey, metricName=metricName) g_log.info("Deleted metric name=%s, uid=%s", metricName, allMetricsMap[metricName]["uid"]) # Delete the symbol from xignite_security table last; this cascades to # delete related rows in other tables via cascading delete relationship. # # NOTE: garbage collection from other tables not tied to xiginte_security # symbols presently depends on aging of the rows (e.g., twitter tables). # After ENG-83, all company-specific rows from all tables will be # cleaned up and THIS NOTE SHOULD THEN BE REMOVED with collectorsdb.engineFactory().begin() as conn: numDeleted = ( conn.execute(collectorsdb.schema.xigniteSecurity # pylint: disable=E1120 .delete().where( collectorsdb.schema.xigniteSecurity.c.symbol == symbol))).rowcount if numDeleted: g_log.info("Deleted row=%s from table=%s", symbol, collectorsdb.schema.xigniteSecurity) else: g_log.warning( "Couldn't delete security row=%s: not found in table=%s", symbol, collectorsdb.schema.xigniteSecurity)