Beispiel #1
0
def store_tweet_payload_and_tweeter_from_twitter(payload_dicts_from_twitter, tweets):
    tweets_by_tweet_id = defaultdict(list)
    for tweet in tweets:
        tweets_by_tweet_id[tweet.tweet_id].append(tweet)

    for payload_dict in payload_dicts_from_twitter:
        tweet_id = payload_dict["id_str"]
        logger.debug("saving unsaved parts for tweet_id {tweet_id}".format(
            tweet_id=tweet_id))
        for tweet in tweets_by_tweet_id[tweet_id]:
            if not tweet.payload:
                tweet.payload = payload_dict
                logger.info(u"updated tweet payload for {tweet_id} {tiid}".format(
                    tweet_id=tweet_id, tiid=tweet.tiid))
                if "user" in payload_dict:
                    try:
                        tweet.tweeter.set_attributes_from_twitter_data(payload_dict["user"])
                    except AttributeError:
                        tweeter = Tweeter.query.get(tweet.screen_name)
                        if not tweeter:
                            tweeter = Tweeter(screen_name=tweet.screen_name)
                            db.session.add(tweeter)
                        tweeter.set_attributes_from_twitter_data(payload_dict["user"])
                        tweet.tweeter = tweeter
                        commit(db)
                    if tweet.tweeter:
                        logger.info(u"updated tweeter followers for {screen_name}".format(
                            screen_name=tweet.tweeter.screen_name))
Beispiel #2
0
def hydrate_twitter_text_and_followers(profile_id, altmetric_twitter_posts):

    logger.info(u"in hydrate_twitter_text_and_followers for profile {profile_id}".format(
        profile_id=profile_id))

    tweets_to_hydrate_from_twitter = []
    # get them all at once into the session so gets below go faster
    tweets = Tweet.query.filter(Tweet.profile_id==profile_id)
    tweet_dict = dict([((tweet.tweet_id, tweet.tiid), tweet) for tweet in tweets])

    for tiid, post_list in altmetric_twitter_posts.iteritems():
        for post in post_list:
            #### store tweet and tweeter stuff from altmetric
            tweet_id = post["tweet_id"]
            screen_name = post["author"]["id_on_source"]

            if (tweet_id, tiid) in tweet_dict.keys():
                tweet = tweet_dict[(tweet_id, tiid)]
                if not tweet.tweet_text and not tweet.is_deleted:
                    tweets_to_hydrate_from_twitter.append(tweet)
            else:
                if not Tweet.query.get((tweet_id, tiid)):
                    tweet = Tweet(tweet_id=tweet_id, tiid=tiid)
                    tweet.set_attributes_from_altmetric_post(post)
                    tweet.profile_id = profile_id
                    tweets_to_hydrate_from_twitter.append(tweet)
                    db.session.add(tweet)
                if not tweet.tweeter:
                    tweeter = Tweeter.query.get(screen_name)
                    if not tweeter:
                        tweeter = Tweeter(screen_name=screen_name)
                        db.session.add(tweeter)
                    tweeter.set_attributes_from_altmetric_post(post)
                commit(db)

    logger.info(u"before tweets_to_hydrate_from_twitter for {profile_id}".format(
        profile_id=profile_id))
    if tweets_to_hydrate_from_twitter:
        # save the altmetric stuff first
        commit(db)

        tweet_ids = [tweet.tweet_id for tweet in tweets_to_hydrate_from_twitter]
        logger.info(u"calling get_and_save_tweet_text_and_tweeter_followers for profile {profile_id}".format(
            profile_id=profile_id))

        get_and_save_tweet_text_and_tweeter_followers(tweets_to_hydrate_from_twitter)
        commit(db)

    else:
        logger.info(u"no tweets to hydrate for profile {profile_id}".format(
            profile_id=profile_id))

    return