def _add_topic_tweet_single_day( db: DatabaseHandler, topic: dict, day: datetime.datetime, ch_class: typing.Type[AbstractCrimsonHexagon]) -> dict: """ Add a row to topic_tweet_day if it does not already exist. fetch data for new row from CH. Arguments: db - database handle topic - topic dict day - date to fetch eg '2017-12-30' ch_class - AbstractCrimsonHexagon class Return: None """ # the perl-python layer was segfaulting until I added the str() around day below -hal topic_tweet_day = db.query( "select * from topic_tweet_days where topics_id = %(a)s and day = %(b)s", { 'a': topic['topics_id'], 'b': str(day) }).hash() if topic_tweet_day is not None and topic_tweet_day['tweets_fetched']: raise McFetchTopicTweetDateFetchedException( "tweets already fetched for day " + str(day)) # if we have a ttd but had not finished fetching tweets, delete it and start over if topic_tweet_day is not None: db.delete_by_id('topic_tweet_days', topic_tweet_day['topic_tweet_days_id']) ch_posts = ch_class.fetch_posts(topic['ch_monitor_id'], day) tweet_count = ch_posts['totalPostsAvailable'] num_ch_tweets = len(ch_posts['posts']) topic_tweet_day = db.create( 'topic_tweet_days', { 'topics_id': topic['topics_id'], 'day': day, 'tweet_count': tweet_count, 'num_ch_tweets': num_ch_tweets, 'tweets_fetched': False }) topic_tweet_day['ch_posts'] = ch_posts return topic_tweet_day
def _add_topic_tweet_single_day( db: DatabaseHandler, topic: dict, day: datetime.datetime, ch_class: typing.Type[AbstractCrimsonHexagon]) -> dict: """ Add a row to topic_tweet_day if it does not already exist. fetch data for new row from CH. Arguments: db - database handle topic - topic dict day - date to fetch eg '2017-12-30' ch_class - AbstractCrimsonHexagon class Return: None """ # the perl-python layer was segfaulting until I added the str() around day below -hal topic_tweet_day = db.query( "select * from topic_tweet_days where topics_id = %(a)s and day = %(b)s", {'a': topic['topics_id'], 'b': str(day)}).hash() if topic_tweet_day is not None and topic_tweet_day['tweets_fetched']: raise McFetchTopicTweetDateFetchedException("tweets already fetched for day " + str(day)) # if we have a ttd but had not finished fetching tweets, delete it and start over if topic_tweet_day is not None: db.delete_by_id('topic_tweet_days', topic_tweet_day['topic_tweet_days_id']) ch_posts = ch_class.fetch_posts(topic['ch_monitor_id'], day) tweet_count = ch_posts['totalPostsAvailable'] num_ch_tweets = len(ch_posts['posts']) topic_tweet_day = db.create( 'topic_tweet_days', { 'topics_id': topic['topics_id'], 'day': day, 'tweet_count': tweet_count, 'num_ch_tweets': num_ch_tweets, 'tweets_fetched': False }) topic_tweet_day['ch_posts'] = ch_posts return topic_tweet_day
def _add_topic_post_single_day(db: DatabaseHandler, topic_seed_query: dict, num_posts: int, day: datetime) -> dict: """ Add a row to topic_post_day if it does not already exist. Arguments: db - database handle topic_seed_query - topic_seed_query dict day - date to fetch eg '2017-12-30' num_posts - number of posts found for that day Return: None """ # the perl-python layer was segfaulting until I added the str() around day below -hal topic_post_day = db.query( "select * from topic_post_days where topic_seed_queries_id = %(a)s and day = %(b)s", { 'a': topic_seed_query['topic_seed_queries_id'], 'b': str(day) }).hash() if topic_post_day is not None and topic_post_day['posts_fetched']: raise McFetchTopicPostsDataException( "tweets already fetched for day " + str(day)) # if we have a ttd but had not finished fetching tweets, delete it and start over if topic_post_day is not None: db.delete_by_id('topic_post_days', topic_post_day['topic_post_days_id']) topic_post_day = db.create( 'topic_post_days', { 'topic_seed_queries_id': topic_seed_query['topic_seed_queries_id'], 'day': day, 'num_posts_stored': num_posts, 'num_posts_fetched': num_posts, 'posts_fetched': False }) return topic_post_day