def main(args):
sys.path.insert(0, args.python_path)
from utils.TwitterDownloader import TwitterDownloader
from utils.Storage import Storage
from utils import twitter_analytics_helpers as tap
from utils.TaggingUtils import TaggingUtils as TU
from utils import email_tools as email_tools
param_table = "PARAM_TWITTER_COLLECTION"
parameters_list = [
"LANGUAGE", "TWEETS_PER_QUERY", "MAX_TWEETS", "CONNECTION_STRING",
"DATABASE_NAME", "COLLECTION_NAME", "LOGGING", "EMAIL_USERNAME",
"EMAIL_PASSWORD", "TWITTER_API_KEY", "TWITTER_API_SECRET",
"BUCKET_NAME", "DESTINATION_TABLE", "LOGGING_TABLE"
]
parameters = tap.get_parameters(args.param_connection_string, param_table,
parameters_list)
# Get dataset name
common_table = "PARAM_READ_DATE"
common_list = ["BQ_DATASET"]
common_where = lambda x: (x["ENVIRONMENT"] == args.environment) & (x[
"STATUS"] == 'active')
common_parameters = tap.get_parameters(args.param_connection_string,
common_table, common_list,
common_where)
try:
get_tweets(args)
except Exception as e:
print(e)
q1 = """
SELECT a.constituent_name, a.downloaded_tweets, a.date, a.language
FROM
(SELECT constituent_name, SUM(downloaded_tweets) as downloaded_tweets, DATE(date) as date, language
FROM `{0}.{1}`
where language != 'StockTwits'
GROUP BY constituent_name, date, language
) a,
(SELECT constituent_name, MAX(DATE(date)) as date
FROM `{0}.{1}`
WHERE language != 'StockTwits'
GROUP BY constituent_name
) b
WHERE a.constituent_name = b.constituent_name AND a.date = b.date AND a.language = "en"
GROUP BY a.constituent_name, a.downloaded_tweets, a.date, a.language;
""".format(common_parameters["BQ_DATASET"],
parameters["LOGGING_TABLE"])
q2 = """
SELECT constituent_name,count(*)
FROM `{}.{}`
GROUP BY constituent_name;
""".format(common_parameters["BQ_DATASET"], parameters["LOGGING_TABLE"])
email_tools.send_mail(args.param_connection_string, args.google_key_path,
"Twitter", "PARAM_TWITTER_COLLECTION", None, q1, q2)
def main(args):
sys.path.insert(0, args.python_path)
from utils.Storage import Storage
from utils import twitter_analytics_helpers as tah
from utils.TaggingUtils import TaggingUtils as TU
from utils import email_tools as email_tools
param_table = "PARAM_STOCKTWITS_COLLECTION"
parameters_list = ["LOGGING", "DESTINATION_TABLE", "LOGGING_TABLE"]
parameters = tah.get_parameters(args.param_connection_string, param_table,
parameters_list)
# Get dataset name
common_table = "PARAM_READ_DATE"
common_list = ["BQ_DATASET"]
common_where = lambda x: (x["ENVIRONMENT"] == args.environment) & (x[
"STATUS"] == 'active')
common_parameters = tah.get_parameters(args.param_connection_string,
common_table, common_list,
common_where)
try:
get_stocktwits(args)
except Exception as e:
print(e)
q1 = """
SELECT a.constituent_name, a.downloaded_tweets, a.date, a.language
FROM
(SELECT constituent_name, SUM(downloaded_tweets) as downloaded_tweets, DATE(date) as date, language
FROM `{0}.{1}`
WHERE language = 'StockTwits'
GROUP BY constituent_name, date, language
) a,
(SELECT constituent_name, MAX(DATE(date)) as date
FROM `{0}.{1}`
WHERE language = 'StockTwits'
GROUP BY constituent_name
) b
WHERE a.constituent_name = b.constituent_name AND a.date = b.date
GROUP BY a.constituent_name, a.downloaded_tweets, a.date, a.language;
""".format(common_parameters["BQ_DATASET"],
parameters["LOGGING_TABLE"])
q2 = """
SELECT constituent_name,count(*)
FROM `{}.{}`
WHERE source = 'StockTwits'
GROUP BY constituent_name;
""".format(common_parameters["BQ_DATASET"],
parameters["LOGGING_TABLE"])
email_tools.send_mail(args.param_connection_string, args.google_key_path,
"StockTwits", "PARAM_STOCKTWITS_COLLECTION", None,
q1, q2)
def main(args):
sys.path.insert(0, args.python_path)
from utils.Storage import Storage
from utils import twitter_analytics_helpers as tah
from utils import email_tools as email_tools
# Get parameters
param_table = "PARAM_NEWS_COLLECTION"
parameters_list = ["LOGGING", "DESTINATION_TABLE", "LOGGING_TABLE"]
where = lambda x: x["SOURCE"] == 'RSS'
parameters = tah.get_parameters(args.param_connection_string, param_table,
parameters_list, where)
# Get dataset name
common_table = "PARAM_READ_DATE"
common_list = ["BQ_DATASET"]
common_where = lambda x: (x["ENVIRONMENT"] == args.environment) & (x[
"STATUS"] == 'active')
common_parameters = tah.get_parameters(args.param_connection_string,
common_table, common_list,
common_where)
get_rss_feed(args)
q1 = """
SELECT a.constituent_name, a.downloaded_news, a.date, a.source
FROM
(SELECT constituent_name, SUM(downloaded_news) as downloaded_news, DATE(date) as date, source
FROM `{0}.{1}`
WHERE source = 'Yahoo Finance RSS'
GROUP BY constituent_name, date, source
) a,
(SELECT constituent_name, MAX(DATE(date)) as date
FROM `{0}.{1}`
WHERE source = 'Yahoo Finance RSS'
GROUP BY constituent_name
) b
WHERE a.constituent_name = b.constituent_name AND a.date = b.date
GROUP BY a.constituent_name, a.downloaded_news, a.date, a.source;
""".format(common_parameters["BQ_DATASET"],
parameters["LOGGING_TABLE"])
q2 = """
SELECT constituent_name,count(*)
FROM `{}.{}`
WHERE news_origin = "Yahoo Finance RSS"
GROUP BY constituent_name
""".format(common_parameters["BQ_DATASET"],
parameters["DESTINATION_TABLE"])
print("Sending email")
email_tools.send_mail(args.param_connection_string, args.google_key_path,
"RSS Feed", "PARAM_NEWS_COLLECTION",
lambda x: x["SOURCE"] == "RSS", q1, q2)
def main(args):
sys.path.insert(0, args.python_path)
from utils.Storage import Storage
from utils import twitter_analytics_helpers as tah
from utils import email_tools as email_tools
# Get parameters
param_table = "PARAM_TICKER_COLLECTION"
parameters_list = ["LOGGING", "DESTINATION_TABLE", "LOGGING_TABLE"]
parameters = tah.get_parameters(args.param_connection_string, param_table,
parameters_list)
# Get dataset name
common_table = "PARAM_READ_DATE"
common_list = ["BQ_DATASET"]
common_where = lambda x: (x["ENVIRONMENT"] == args.environment) & (x[
"STATUS"] == 'active')
common_parameters = tah.get_parameters(args.param_connection_string,
common_table, common_list,
common_where)
try:
extract_ticker_data(args)
except Exception as e:
print(e)
q1 = """
SELECT a.constituent_name, a.downloaded_ticks, a.date
FROM
(SELECT constituent_name, SUM(downloaded_ticks) as downloaded_ticks, DATE(date) as date
FROM `{0}.{1}`
GROUP BY constituent_name, date
) a,
(SELECT constituent_name, MAX(DATE(date)) as date
FROM `{0}.{1}`
GROUP BY constituent_name
) b
WHERE a.constituent_name = b.constituent_name AND a.date = b.date
GROUP BY a.constituent_name, a.downloaded_ticks, a.date;
""".format(common_parameters['BQ_DATASET'],
parameters["LOGGING_TABLE"])
q2 = """
SELECT constituent_name,count(*) as count
FROM `{}.{}`
GROUP BY constituent_name
""".format(common_parameters["BQ_DATASET"],
parameters["DESTINATION_TABLE"])
email_tools.send_mail(args.param_connection_string, args.google_key_path,
"Ticker", "PARAM_TICKER_COLLECTION", None, q1, q2)
def main(args):
sys.path.insert(0, args.python_path)
from utils.Storage import Storage
from utils.Storage import MongoEncoder
from utils.SOAPUtils import SOAPUtils
from utils import twitter_analytics_helpers as tah
from utils import email_tools as email_tools
# Get parameters
param_table = "PARAM_NEWS_COLLECTION"
parameters_list = ['BVD_USERNAME', 'BVD_PASSWORD', "LOGGING", "DESTINATION_TABLE", "LOGGING_TABLE"]
where = lambda x: x["SOURCE"] == 'Zephyr'
parameters = tah.get_parameters(args.param_connection_string, param_table, parameters_list, where)
# Get dataset name
common_table = "PARAM_READ_DATE"
common_list = ["BQ_DATASET"]
common_where = lambda x: (x["ENVIRONMENT"] == args.environment) & (x["STATUS"] == 'active')
common_parameters = tah.get_parameters(args.param_connection_string, common_table, common_list, common_where)
get_daily_zephyr_ma_deals(args)
q1 = """
SELECT a.constituent_name, a.downloaded_news, a.date, a.source
FROM
(SELECT constituent_name, SUM(downloaded_news) as downloaded_news, DATE(date) as date, source
FROM `{0}.{1}`
WHERE source = 'Zephyr'
GROUP BY constituent_name, date, source
) a,
(SELECT constituent_name, MAX(DATE(date)) as date
FROM `{0}.{1}`
WHERE source = 'Zephyr'
GROUP BY constituent_name
) b
WHERE a.constituent_name = b.constituent_name AND a.date = b.date
GROUP BY a.constituent_name, a.downloaded_news, a.date, a.source;
""".format(common_parameters["BQ_DATASET"],parameters["LOGGING_TABLE"])
q2 = """
SELECT constituent_name,count(*) as count
FROM `{}.{}`
GROUP BY constituent_name
""".format(common_parameters["BQ_DATASET"],parameters["DESTINATION_TABLE"])
email_tools.send_mail(args.param_connection_string, args.google_key_path, "Zephyr",
"PARAM_NEWS_COLLECTION", lambda x: x["SOURCE"] == "Zephyr", q1, q2)