def main():
"""
NOTE: main also serves as entry point for "console script" generated by setup
"""
logging_support.LoggingSupport().initTool()
try:
options = _parseArgs()
days = options["days"]
g_log.info("Purging records from table=%s older than numDays=%s",
collectorsdb.schema.twitterTweets, days)
twitterTweetsSchema = collectorsdb.schema.twitterTweets
query = twitterTweetsSchema.delete().where(
twitterTweetsSchema.c.created_at <
sqlalchemy.func.date_sub(
sqlalchemy.func.utc_timestamp(),
sqlalchemy.text("INTERVAL %i DAY" % (days,)))
)
with collectorsdb.engineFactory().begin() as conn:
result = conn.execute(query)
g_log.info("Purged numRows=%s from table=%s",
result.rowcount, collectorsdb.schema.twitterTweets)
except SystemExit as e:
if e.code != 0:
g_log.exception("Failed!")
raise
except Exception:
g_log.exception("Failed!")
raise
def main():
"""
NOTE: main also serves as entry point for "console script" generated by setup
"""
logging_support.LoggingSupport().initTool()
try:
_parseArgs()
# Process tweet deletions
g_log.info("Processing tweet deletions")
totalDeletedTweets = 0
tweetDeletionLimit = 100
while True:
deletedTweetIds = _purgeTweetsSlatedForDeletion(
limit=tweetDeletionLimit)
numDeletedTweets = len(deletedTweetIds)
if numDeletedTweets:
totalDeletedTweets += numDeletedTweets
g_log.info("Purged numTweets=%d: %s", numDeletedTweets,
deletedTweetIds)
if numDeletedTweets < tweetDeletionLimit:
break
g_log.info("Purged totalDeletedTweets=%d", totalDeletedTweets)
# Purge stale tweet deletion request rows
g_log.info("Purging stale tweet deletion request rows")
totalPurgedTweetDeletionRows = 0
deletionRowPurgeLimit = 100
while True:
purgedDeletionIds = _purgeStaleDeletionRecords(
limit=deletionRowPurgeLimit)
numPurgedTweetDeletionRows = len(purgedDeletionIds)
if numPurgedTweetDeletionRows:
totalPurgedTweetDeletionRows += numPurgedTweetDeletionRows
g_log.info("Purged numPurgedTweetDeletionRows=%d: %s",
numPurgedTweetDeletionRows, purgedDeletionIds)
if numPurgedTweetDeletionRows < deletionRowPurgeLimit:
break
g_log.info("Purged totalPurgedTweetDeletionRows=%d",
totalPurgedTweetDeletionRows)
except SystemExit as e:
if e.code != 0:
g_log.exception("Failed!")
raise
except Exception:
g_log.exception("Failed!")
raise
def main():
"""
NOTE: main also serves as entry point for "console script" generated by setup
"""
logging_support.LoggingSupport().initTool()
try:
options = _parseArgs()
allSymbols = set(stockData["symbol"] for stockData in
metric_utils.getMetricsConfiguration().itervalues())
g_log.info("Verifying that agents are in hot_standby mode")
for section in config.sections():
assert config.get(section, "opmode") == config.OP_MODE_HOT_STANDBY
g_log.info("Verifying that the old symbol has been removed from the "
"metrics configuration")
assert options.oldSymbol not in allSymbols
g_log.info("Verifying that the new symbol is present in the metrics "
"configuration")
assert options.newSymbol in allSymbols
g_log.info(
"Migrating BOTH twitter and xignite stock data from "
"old-symbol=%s to new-symbol=%s", options.oldSymbol,
options.newSymbol)
# Rename the metrics in collectorsdb and forward new metric samples to HTM
# Engine
g_log.info("Modifying old metrics with new symbol")
_resymbolTweetVolumeMetric(oldSymbol=options.oldSymbol,
newSymbol=options.newSymbol,
aggPeriod=options.aggPeriod)
_resymbolStockMetrics(oldSymbol=options.oldSymbol,
newSymbol=options.newSymbol)
# Delete metrics linked to old stock symbol from Taurus Engine
delete_companies.deleteCompanies(tickerSymbols=[options.oldSymbol],
engineServer=options.htmServer,
engineApiKey=options.apikey,
warnAboutDestructiveAction=False)
except SystemExit as e:
if e.code != 0:
g_log.exception("Failed!")
raise
except Exception:
g_log.exception("Failed!")
raise
def main():
"""
NOTE: main also serves as entry point for "console script" generated by setup
"""
logging_support.LoggingSupport().initTool()
try:
try:
options = _parseArgs()
except SystemExit as e:
# OptionParser uses SystemExit on option-parsing error
if e.code != 0:
g_log.exception("Failed!")
raise
purgeOldTweets(options["days"])
except Exception:
g_log.exception("Failed!")
raise
def main():
logging_support.LoggingSupport().initTool()
try:
options = _parseArgs()
host = options["host"]
user = options["user"]
password = options["password"]
config = collectorsdb.CollectorsDbConfig()
configWriter = collectorsdb.CollectorsDbConfig(
mode=Config.MODE_OVERRIDE_ONLY)
if not configWriter.has_section("repository"):
configWriter.add_section("repository")
def override(option, value):
assert config.has_option("repository", option), option
configWriter.set("repository", option, value)
override("host", host)
override("user", user)
override("password", password)
configWriter.save()
g_log.info("Override of mysql settings for %s completed successfully",
collectorsdb.CollectorsDbConfig.CONFIG_NAME)
except SystemExit as e:
if e.code != 0:
g_log.exception("Failed!")
raise
except Exception:
g_log.exception("Failed!")
raise
def main():
logging_support.LoggingSupport().initTool()
setRabbitmqLoginScriptImpl()