def store_tweet_payload_and_tweeter_from_twitter(payload_dicts_from_twitter, tweets):
tweets_by_tweet_id = defaultdict(list)
for tweet in tweets:
tweets_by_tweet_id[tweet.tweet_id].append(tweet)
for payload_dict in payload_dicts_from_twitter:
tweet_id = payload_dict["id_str"]
logger.debug("saving unsaved parts for tweet_id {tweet_id}".format(
tweet_id=tweet_id))
for tweet in tweets_by_tweet_id[tweet_id]:
if not tweet.payload:
tweet.payload = payload_dict
logger.info(u"updated tweet payload for {tweet_id} {tiid}".format(
tweet_id=tweet_id, tiid=tweet.tiid))
if "user" in payload_dict:
try:
tweet.tweeter.set_attributes_from_twitter_data(payload_dict["user"])
except AttributeError:
tweeter = Tweeter.query.get(tweet.screen_name)
if not tweeter:
tweeter = Tweeter(screen_name=tweet.screen_name)
db.session.add(tweeter)
tweeter.set_attributes_from_twitter_data(payload_dict["user"])
tweet.tweeter = tweeter
commit(db)
if tweet.tweeter:
logger.info(u"updated tweeter followers for {screen_name}".format(
screen_name=tweet.tweeter.screen_name))
def hydrate_twitter_text_and_followers(profile_id, altmetric_twitter_posts):
logger.info(u"in hydrate_twitter_text_and_followers for profile {profile_id}".format(
profile_id=profile_id))
tweets_to_hydrate_from_twitter = []
# get them all at once into the session so gets below go faster
tweets = Tweet.query.filter(Tweet.profile_id==profile_id)
tweet_dict = dict([((tweet.tweet_id, tweet.tiid), tweet) for tweet in tweets])
for tiid, post_list in altmetric_twitter_posts.iteritems():
for post in post_list:
#### store tweet and tweeter stuff from altmetric
tweet_id = post["tweet_id"]
screen_name = post["author"]["id_on_source"]
if (tweet_id, tiid) in tweet_dict.keys():
tweet = tweet_dict[(tweet_id, tiid)]
if not tweet.tweet_text and not tweet.is_deleted:
tweets_to_hydrate_from_twitter.append(tweet)
else:
if not Tweet.query.get((tweet_id, tiid)):
tweet = Tweet(tweet_id=tweet_id, tiid=tiid)
tweet.set_attributes_from_altmetric_post(post)
tweet.profile_id = profile_id
tweets_to_hydrate_from_twitter.append(tweet)
db.session.add(tweet)
if not tweet.tweeter:
tweeter = Tweeter.query.get(screen_name)
if not tweeter:
tweeter = Tweeter(screen_name=screen_name)
db.session.add(tweeter)
tweeter.set_attributes_from_altmetric_post(post)
commit(db)
logger.info(u"before tweets_to_hydrate_from_twitter for {profile_id}".format(
profile_id=profile_id))
if tweets_to_hydrate_from_twitter:
# save the altmetric stuff first
commit(db)
tweet_ids = [tweet.tweet_id for tweet in tweets_to_hydrate_from_twitter]
logger.info(u"calling get_and_save_tweet_text_and_tweeter_followers for profile {profile_id}".format(
profile_id=profile_id))
get_and_save_tweet_text_and_tweeter_followers(tweets_to_hydrate_from_twitter)
commit(db)
else:
logger.info(u"no tweets to hydrate for profile {profile_id}".format(
profile_id=profile_id))
return