def store_twitter_user_data(screen_name): api = twitter_api_client() twitter_user = api.get_user(screen_name) statuses = api.user_timeline(screen_name, tweet_mode="extended", count=150) db_user = User.query.get(twitter_user.id) or User(id=twitter_user.id) db_user.screen_name = twitter_user.screen_name db_user.name = twitter_user.name db_user.location = twitter_user.location db_user.followers_count = twitter_user.followers_count db.session.add(db_user) db.session.commit() print("STATUS COUNT:", len(statuses)) basilica_api = basilica_api_client() all_tweet_texts = [status.full_text for status in statuses] embeddings = list( basilica_api.embed_sentences(all_tweet_texts, model="twitter")) print("NUMBER OF EMBEDDINGS", len(embeddings)) counter = 0 for status in statuses: print(status.full_text) print("----") db_tweet = Tweet.query.get(status.id) or Tweet(id=status.id) db_tweet.user_id = status.author.id db_tweet.full_text = status.full_text embedding = embeddings[counter] print(len(embedding)) db_tweet.embedding = embedding db.session.add(db_tweet) counter += 1 db.session.commit() return db_user, statuses
from web_app.services.basilica_service import basilica_api_clienttwitter_routes = Blueprint("twitter_routes", __name__)def store_twitter_user_data(screen_name): api = twitter_api_client() twitter_user = api.get_user(screen_name) statuses = api.user_timeline(screen_name, tweet_mode="extended", count=150, exclude_replies=True, include_rts=False) #return jsonify({"user": user._json, "tweets": [s._json for s in statuses]}) db_user = User.query.get(twitter_user.id) or User(id=twitter_user.id) db_user.screen_name = twitter_user.screen_name db_user.name = twitter_user.name db_user.location = twitter_user.location db_user.followers_count = twitter_user.followers_count db.session.add(db_user) db.session.commit() #return "OK" #breakpoint() print("STATUS COUNT:", len(statuses)) basilica_api = basilica_api_client() all_tweet_texts = [status.full_text for status in statuses] embeddings = list(basilica_api.embed_sentences(all_tweet_texts, model="twitter")) print("NUMBER OF EMBEDDINGS", len(embeddings)) # TODO: explore using the zip() function maybe... counter = 0 for status in statuses: print(status.full_text) print("----") #print(dir(status)) # Find or create database tweet: db_tweet = Tweet.query.get(status.id) or Tweet(id=status.id) db_tweet.user_id = status.author.id # or db_user.id db_tweet.full_text = status.full_text #embedding = basilica_client.embed_sentence(status.full_text, model="twitter") # todo: prefer to make a single request to basilica with all the tweet texts, instead of a request per tweet embedding = embeddings[counter] print(len(embedding)) db_tweet.embedding = embedding db.session.add(db_tweet) counter+=1 db.session.commit() return db_user, statuses@twitter_routes.route("/users")
def get_user(screen_name=None): print(screen_name) api = twitter_api_client() user = api.get_user(screen_name) statuses = api.user_timeline(screen_name, tweet_mode="extended", count=150, exclude_replies=True, include_rts=False) return jsonify({"user": user.json, "tweets": [s._json for s in statuses]})
def get_user(screen_name=None): print(screen_name) api = twitter_api_client() twitter_user = api.get_user(screen_name) statuses = api.user_timeline(screen_name, tweet_mode="extended", count=150, exclude_replies=True, include_rts=False) print("STATUSES COUNT:", len(statuses)) #return jsonify({"user": user._json, "tweets": [s._json for s in statuses]}) # get existing user from the db or initialize a new one: db_user = User.query.get(twitter_user.id) or User(id=twitter_user.id) db_user.screen_name = twitter_user.screen_name db_user.name = twitter_user.name db_user.location = twitter_user.location db_user.followers_count = twitter_user.followers_count db.session.add(db_user) db.session.commit() #breakpoint() # return "OK" #return render_template("user.html", user=db_user, tweets=statuses) # tweets=db_tweets basilica_api = basilica_api_client() all_tweet_texts = [status.full_text for status in statuses] embeddings = list( basilica_api.embed_sentences(all_tweet_texts, model="twitter")) print("NUMBER OF EMBEDDINGS", len(embeddings)) # TODO: explore using the zip() function maybe... counter = 0 for status in statuses: print(status.full_text) print("----") #print(dir(status)) # get existing tweet from the db or initialize a new one: db_tweet = Tweet.query.get(status.id) or Tweet(id=status.id) db_tweet.user_id = status.author.id # or db_user.id db_tweet.full_text = status.full_text #embedding = basilica_client.embed_sentence(status.full_text, model="twitter") # todo: prefer to make a single request to basilica with all the tweet texts, instead of a request per tweet embedding = embeddings[counter] print(len(embedding)) db_tweet.embedding = embedding db.session.add(db_tweet) counter += 1 db.session.commit() #breakpoint() return "OK"
def store_twitter_user_data(screen_name): # Get the username from the twitter api and save the # user and tweets objects api = twitter_api_client() twitter_user = api.get_user(screen_name) statuses = api.user_timeline(screen_name, tweet_mode="extended", count=150, exclude_replies=True, include_rts=False) # Check to see if the user already exists in the db user table and # if not then add it to the db user table db_user = User.query.get(twitter_user.id) or User(id=twitter_user.id) db_user.screen_name = twitter_user.screen_name db_user.name = twitter_user.name db_user.location = twitter_user.location db_user.followers_count = twitter_user.followers_count db.session.add(db_user) db.session.commit() print("STATUS COUNT:", len(statuses)) # Use the basilica api to turn the tweets (statuses) into numeric lists basilica_api = basilica_api_client() all_tweet_texts = [status.full_text for status in statuses] embeddings = list( basilica_api.embed_sentences(all_tweet_texts, model="twitter")) print("NUMBER OF EMBEDDINGS", len(embeddings)) # Put all of the tweets in the db tweet table counter = 0 for status in statuses: print(status.full_text) print("----") # Find or create database tweet: db_tweet = Tweet.query.get(status.id) or Tweet(id=status.id) db_tweet.user_id = status.author.id # or db_user.id db_tweet.full_text = status.full_text embedding = embeddings[counter] print(len(embedding)) db_tweet.embedding = embedding db.session.add(db_tweet) counter += 1 db.session.commit() return db_user, statuses
def get_user(screen_name=None): # print(screen_name) api = twitter_api_client() twitter_user = api.get_user(screen_name) statuses = api.user_timeline(screen_name, tweet_mode="extended", count=150) # print("STATUSES COUNT:", len(statuses)) # get existing user from the db or initialize a new one: db_user = User.query.get(twitter_user.id) or User(id=twitter_user.id) db_user.screen_name = twitter_user.screen_name db_user.name = twitter_user.name db_user.location = twitter_user.location db_user.followers_count = twitter_user.followers_count db.session.add(db_user) db.session.commit() basilica_api = basilica_api_client() all_tweet_texts = [status.full_text for status in statuses] embeddings = list( basilica_api.embed_sentences(all_tweet_texts, model="twitter")) # print("NUMBER OF EMBEDDINGS", len(embeddings)) counter = 0 for status in statuses: # print(status.full_text) # print("----") # get existing tweet from the db or initialize a new one: db_tweet = Tweet.query.get(status.id) or Tweet(id=status.id) db_tweet.user_id = status.author.id # or db_user.id db_tweet.full_text = status.full_text embedding = embeddings[counter] # print(len(embedding)) db_tweet.embedding = embedding db.session.add(db_tweet) counter += 1 db.session.commit() # return "OK" return render_template("user.html", user=db_user, tweets=statuses)
def seed_db(): # print(type(db)) api = twitter_api_client() for screen_name in ['elonmusk','justinbieber','s2t2']: twitter_user = api.get_user(screen_name) statuses = api.user_timeline(screen_name, tweet_mode="extended", count=150) # print("STATUSES COUNT:", len(statuses)) # get existing user from the db or initialize a new one: db_user = User.query.get(twitter_user.id) or User(id=twitter_user.id) db_user.screen_name = twitter_user.screen_name db_user.name = twitter_user.name db_user.location = twitter_user.location db_user.followers_count = twitter_user.followers_count db.session.add(db_user) db.session.commit() basilica_api = basilica_api_client() all_tweet_texts = [status.full_text for status in statuses] embeddings = list(basilica_api.embed_sentences(all_tweet_texts, model="twitter")) # print("NUMBER OF EMBEDDINGS", len(embeddings)) counter = 0 for status in statuses: # print(status.full_text) # print("----") # get existing tweet from the db or initialize a new one: db_tweet = Tweet.query.get(status.id) or Tweet(id=status.id) db_tweet.user_id = status.author.id # or db_user.id db_tweet.full_text = status.full_text embedding = embeddings[counter] # print(len(embedding)) db_tweet.embedding = embedding db.session.add(db_tweet) counter+=1 db.session.commit() return jsonify({"message": "DB SEEDED OK"})
def store_twitter_user_data(screen_name): print(screen_name) api = twitter_api_client() twitter_user = api.get_user(screen_name) statuses = api.user_timeline(screen_name, tweet_mode="extended", count=150) db_user = User.query.get(twitter_user.id) or User(id=twitter_user.id) db_user.screen_name = twitter_user.screen_name db_user.name = twitter_user.name db_user.location = twitter_user.location db_user.followers_count = twitter_user.followers_count db.session.add(db_user) db.session.commit() print("STATUS COUNT:", len(statuses)) basilica_api = basilica_api_client() all_tweet_texts = [status.full_text for status in statuses] embeddings = list( basilica_api.embed_sentences(all_tweet_texts, model="twitter")) print("NUMBER OF EMBEDDINGS", len(embeddings)) counter = 0 for status in statuses: print(status.full_text) print("----") # print(dir(status)) # Find or create database tweet: db_tweet = Tweet.query.get(status.id) or Tweet(id=status.id) db_tweet.user_id = status.author.id # or db_user.id db_tweet.full_text = status.full_text # embedding = basilica_client.embed_sentence(status.full_text, model="twitter") # todo: prefer to make a single request to basilica with all the tweet texts, instead of a request per tweet embedding = embeddings[counter] print(len(embedding)) db_tweet.embedding = embedding db.session.add(db_tweet) counter += 1 db.session.commit() return db_user, statuses