def main(args): sys.path.insert(0, args.python_path) from utils.TwitterDownloader import TwitterDownloader from utils.Storage import Storage from utils import twitter_analytics_helpers as tap from utils.TaggingUtils import TaggingUtils as TU from utils import email_tools as email_tools param_table = "PARAM_TWITTER_COLLECTION" parameters_list = [ "LANGUAGE", "TWEETS_PER_QUERY", "MAX_TWEETS", "CONNECTION_STRING", "DATABASE_NAME", "COLLECTION_NAME", "LOGGING", "EMAIL_USERNAME", "EMAIL_PASSWORD", "TWITTER_API_KEY", "TWITTER_API_SECRET", "BUCKET_NAME", "DESTINATION_TABLE", "LOGGING_TABLE" ] parameters = tap.get_parameters(args.param_connection_string, param_table, parameters_list) # Get dataset name common_table = "PARAM_READ_DATE" common_list = ["BQ_DATASET"] common_where = lambda x: (x["ENVIRONMENT"] == args.environment) & (x[ "STATUS"] == 'active') common_parameters = tap.get_parameters(args.param_connection_string, common_table, common_list, common_where) try: get_tweets(args) except Exception as e: print(e) q1 = """ SELECT a.constituent_name, a.downloaded_tweets, a.date, a.language FROM (SELECT constituent_name, SUM(downloaded_tweets) as downloaded_tweets, DATE(date) as date, language FROM `{0}.{1}` where language != 'StockTwits' GROUP BY constituent_name, date, language ) a, (SELECT constituent_name, MAX(DATE(date)) as date FROM `{0}.{1}` WHERE language != 'StockTwits' GROUP BY constituent_name ) b WHERE a.constituent_name = b.constituent_name AND a.date = b.date AND a.language = "en" GROUP BY a.constituent_name, a.downloaded_tweets, a.date, a.language; """.format(common_parameters["BQ_DATASET"], parameters["LOGGING_TABLE"]) q2 = """ SELECT constituent_name,count(*) FROM `{}.{}` GROUP BY constituent_name; """.format(common_parameters["BQ_DATASET"], parameters["LOGGING_TABLE"]) email_tools.send_mail(args.param_connection_string, args.google_key_path, "Twitter", "PARAM_TWITTER_COLLECTION", None, q1, q2)
def main(args): sys.path.insert(0, args.python_path) from utils.Storage import Storage from utils import twitter_analytics_helpers as tah from utils.TaggingUtils import TaggingUtils as TU from utils import email_tools as email_tools param_table = "PARAM_STOCKTWITS_COLLECTION" parameters_list = ["LOGGING", "DESTINATION_TABLE", "LOGGING_TABLE"] parameters = tah.get_parameters(args.param_connection_string, param_table, parameters_list) # Get dataset name common_table = "PARAM_READ_DATE" common_list = ["BQ_DATASET"] common_where = lambda x: (x["ENVIRONMENT"] == args.environment) & (x[ "STATUS"] == 'active') common_parameters = tah.get_parameters(args.param_connection_string, common_table, common_list, common_where) try: get_stocktwits(args) except Exception as e: print(e) q1 = """ SELECT a.constituent_name, a.downloaded_tweets, a.date, a.language FROM (SELECT constituent_name, SUM(downloaded_tweets) as downloaded_tweets, DATE(date) as date, language FROM `{0}.{1}` WHERE language = 'StockTwits' GROUP BY constituent_name, date, language ) a, (SELECT constituent_name, MAX(DATE(date)) as date FROM `{0}.{1}` WHERE language = 'StockTwits' GROUP BY constituent_name ) b WHERE a.constituent_name = b.constituent_name AND a.date = b.date GROUP BY a.constituent_name, a.downloaded_tweets, a.date, a.language; """.format(common_parameters["BQ_DATASET"], parameters["LOGGING_TABLE"]) q2 = """ SELECT constituent_name,count(*) FROM `{}.{}` WHERE source = 'StockTwits' GROUP BY constituent_name; """.format(common_parameters["BQ_DATASET"], parameters["LOGGING_TABLE"]) email_tools.send_mail(args.param_connection_string, args.google_key_path, "StockTwits", "PARAM_STOCKTWITS_COLLECTION", None, q1, q2)
def main(args): sys.path.insert(0, args.python_path) from utils.Storage import Storage from utils import twitter_analytics_helpers as tah from utils import email_tools as email_tools # Get parameters param_table = "PARAM_NEWS_COLLECTION" parameters_list = ["LOGGING", "DESTINATION_TABLE", "LOGGING_TABLE"] where = lambda x: x["SOURCE"] == 'RSS' parameters = tah.get_parameters(args.param_connection_string, param_table, parameters_list, where) # Get dataset name common_table = "PARAM_READ_DATE" common_list = ["BQ_DATASET"] common_where = lambda x: (x["ENVIRONMENT"] == args.environment) & (x[ "STATUS"] == 'active') common_parameters = tah.get_parameters(args.param_connection_string, common_table, common_list, common_where) get_rss_feed(args) q1 = """ SELECT a.constituent_name, a.downloaded_news, a.date, a.source FROM (SELECT constituent_name, SUM(downloaded_news) as downloaded_news, DATE(date) as date, source FROM `{0}.{1}` WHERE source = 'Yahoo Finance RSS' GROUP BY constituent_name, date, source ) a, (SELECT constituent_name, MAX(DATE(date)) as date FROM `{0}.{1}` WHERE source = 'Yahoo Finance RSS' GROUP BY constituent_name ) b WHERE a.constituent_name = b.constituent_name AND a.date = b.date GROUP BY a.constituent_name, a.downloaded_news, a.date, a.source; """.format(common_parameters["BQ_DATASET"], parameters["LOGGING_TABLE"]) q2 = """ SELECT constituent_name,count(*) FROM `{}.{}` WHERE news_origin = "Yahoo Finance RSS" GROUP BY constituent_name """.format(common_parameters["BQ_DATASET"], parameters["DESTINATION_TABLE"]) print("Sending email") email_tools.send_mail(args.param_connection_string, args.google_key_path, "RSS Feed", "PARAM_NEWS_COLLECTION", lambda x: x["SOURCE"] == "RSS", q1, q2)
def main(args): sys.path.insert(0, args.python_path) from utils.Storage import Storage from utils import twitter_analytics_helpers as tah from utils import email_tools as email_tools # Get parameters param_table = "PARAM_TICKER_COLLECTION" parameters_list = ["LOGGING", "DESTINATION_TABLE", "LOGGING_TABLE"] parameters = tah.get_parameters(args.param_connection_string, param_table, parameters_list) # Get dataset name common_table = "PARAM_READ_DATE" common_list = ["BQ_DATASET"] common_where = lambda x: (x["ENVIRONMENT"] == args.environment) & (x[ "STATUS"] == 'active') common_parameters = tah.get_parameters(args.param_connection_string, common_table, common_list, common_where) try: extract_ticker_data(args) except Exception as e: print(e) q1 = """ SELECT a.constituent_name, a.downloaded_ticks, a.date FROM (SELECT constituent_name, SUM(downloaded_ticks) as downloaded_ticks, DATE(date) as date FROM `{0}.{1}` GROUP BY constituent_name, date ) a, (SELECT constituent_name, MAX(DATE(date)) as date FROM `{0}.{1}` GROUP BY constituent_name ) b WHERE a.constituent_name = b.constituent_name AND a.date = b.date GROUP BY a.constituent_name, a.downloaded_ticks, a.date; """.format(common_parameters['BQ_DATASET'], parameters["LOGGING_TABLE"]) q2 = """ SELECT constituent_name,count(*) as count FROM `{}.{}` GROUP BY constituent_name """.format(common_parameters["BQ_DATASET"], parameters["DESTINATION_TABLE"]) email_tools.send_mail(args.param_connection_string, args.google_key_path, "Ticker", "PARAM_TICKER_COLLECTION", None, q1, q2)
def main(args): sys.path.insert(0, args.python_path) from utils.Storage import Storage from utils.Storage import MongoEncoder from utils.SOAPUtils import SOAPUtils from utils import twitter_analytics_helpers as tah from utils import email_tools as email_tools # Get parameters param_table = "PARAM_NEWS_COLLECTION" parameters_list = ['BVD_USERNAME', 'BVD_PASSWORD', "LOGGING", "DESTINATION_TABLE", "LOGGING_TABLE"] where = lambda x: x["SOURCE"] == 'Zephyr' parameters = tah.get_parameters(args.param_connection_string, param_table, parameters_list, where) # Get dataset name common_table = "PARAM_READ_DATE" common_list = ["BQ_DATASET"] common_where = lambda x: (x["ENVIRONMENT"] == args.environment) & (x["STATUS"] == 'active') common_parameters = tah.get_parameters(args.param_connection_string, common_table, common_list, common_where) get_daily_zephyr_ma_deals(args) q1 = """ SELECT a.constituent_name, a.downloaded_news, a.date, a.source FROM (SELECT constituent_name, SUM(downloaded_news) as downloaded_news, DATE(date) as date, source FROM `{0}.{1}` WHERE source = 'Zephyr' GROUP BY constituent_name, date, source ) a, (SELECT constituent_name, MAX(DATE(date)) as date FROM `{0}.{1}` WHERE source = 'Zephyr' GROUP BY constituent_name ) b WHERE a.constituent_name = b.constituent_name AND a.date = b.date GROUP BY a.constituent_name, a.downloaded_news, a.date, a.source; """.format(common_parameters["BQ_DATASET"],parameters["LOGGING_TABLE"]) q2 = """ SELECT constituent_name,count(*) as count FROM `{}.{}` GROUP BY constituent_name """.format(common_parameters["BQ_DATASET"],parameters["DESTINATION_TABLE"]) email_tools.send_mail(args.param_connection_string, args.google_key_path, "Zephyr", "PARAM_NEWS_COLLECTION", lambda x: x["SOURCE"] == "Zephyr", q1, q2)