def store_twitter_user_data(screen_name):
api = twitter_api_client()
twitter_user = api.get_user(screen_name)
statuses = api.user_timeline(screen_name, tweet_mode="extended", count=150)
db_user = User.query.get(twitter_user.id) or User(id=twitter_user.id)
db_user.screen_name = twitter_user.screen_name
db_user.name = twitter_user.name
db_user.location = twitter_user.location
db_user.followers_count = twitter_user.followers_count
db.session.add(db_user)
db.session.commit()
print("STATUS COUNT:", len(statuses))
basilica_api = basilica_api_client()
all_tweet_texts = [status.full_text for status in statuses]
embeddings = list(
basilica_api.embed_sentences(all_tweet_texts, model="twitter"))
print("NUMBER OF EMBEDDINGS", len(embeddings))
counter = 0
for status in statuses:
print(status.full_text)
print("----")
db_tweet = Tweet.query.get(status.id) or Tweet(id=status.id)
db_tweet.user_id = status.author.id
db_tweet.full_text = status.full_text
embedding = embeddings[counter]
print(len(embedding))
db_tweet.embedding = embedding
db.session.add(db_tweet)
counter += 1
db.session.commit()
return db_user, statuses
from web_app.services.basilica_service import basilica_api_clienttwitter_routes = Blueprint("twitter_routes", __name__)def store_twitter_user_data(screen_name):
api = twitter_api_client()
twitter_user = api.get_user(screen_name)
statuses = api.user_timeline(screen_name, tweet_mode="extended", count=150, exclude_replies=True, include_rts=False)
#return jsonify({"user": user._json, "tweets": [s._json for s in statuses]}) db_user = User.query.get(twitter_user.id) or User(id=twitter_user.id)
db_user.screen_name = twitter_user.screen_name
db_user.name = twitter_user.name
db_user.location = twitter_user.location
db_user.followers_count = twitter_user.followers_count
db.session.add(db_user)
db.session.commit()
#return "OK"
#breakpoint() print("STATUS COUNT:", len(statuses))
basilica_api = basilica_api_client()
all_tweet_texts = [status.full_text for status in statuses]
embeddings = list(basilica_api.embed_sentences(all_tweet_texts, model="twitter"))
print("NUMBER OF EMBEDDINGS", len(embeddings)) # TODO: explore using the zip() function maybe...
counter = 0
for status in statuses:
print(status.full_text)
print("----")
#print(dir(status)) # Find or create database tweet:
db_tweet = Tweet.query.get(status.id) or Tweet(id=status.id)
db_tweet.user_id = status.author.id # or db_user.id
db_tweet.full_text = status.full_text
#embedding = basilica_client.embed_sentence(status.full_text, model="twitter") # todo: prefer to make a single request to basilica with all the tweet texts, instead of a request per tweet
embedding = embeddings[counter]
print(len(embedding))
db_tweet.embedding = embedding
db.session.add(db_tweet)
counter+=1
db.session.commit() return db_user, statuses@twitter_routes.route("/users")
def get_user(screen_name=None):
print(screen_name)
api = twitter_api_client()
user = api.get_user(screen_name)
statuses = api.user_timeline(screen_name,
tweet_mode="extended",
count=150,
exclude_replies=True,
include_rts=False)
return jsonify({"user": user.json, "tweets": [s._json for s in statuses]})
def get_user(screen_name=None):
print(screen_name)
api = twitter_api_client()
twitter_user = api.get_user(screen_name)
statuses = api.user_timeline(screen_name,
tweet_mode="extended",
count=150,
exclude_replies=True,
include_rts=False)
print("STATUSES COUNT:", len(statuses))
#return jsonify({"user": user._json, "tweets": [s._json for s in statuses]})
# get existing user from the db or initialize a new one:
db_user = User.query.get(twitter_user.id) or User(id=twitter_user.id)
db_user.screen_name = twitter_user.screen_name
db_user.name = twitter_user.name
db_user.location = twitter_user.location
db_user.followers_count = twitter_user.followers_count
db.session.add(db_user)
db.session.commit()
#breakpoint()
# return "OK"
#return render_template("user.html", user=db_user, tweets=statuses) # tweets=db_tweets
basilica_api = basilica_api_client()
all_tweet_texts = [status.full_text for status in statuses]
embeddings = list(
basilica_api.embed_sentences(all_tweet_texts, model="twitter"))
print("NUMBER OF EMBEDDINGS", len(embeddings))
# TODO: explore using the zip() function maybe...
counter = 0
for status in statuses:
print(status.full_text)
print("----")
#print(dir(status))
# get existing tweet from the db or initialize a new one:
db_tweet = Tweet.query.get(status.id) or Tweet(id=status.id)
db_tweet.user_id = status.author.id # or db_user.id
db_tweet.full_text = status.full_text
#embedding = basilica_client.embed_sentence(status.full_text, model="twitter") # todo: prefer to make a single request to basilica with all the tweet texts, instead of a request per tweet
embedding = embeddings[counter]
print(len(embedding))
db_tweet.embedding = embedding
db.session.add(db_tweet)
counter += 1
db.session.commit()
#breakpoint()
return "OK"
def store_twitter_user_data(screen_name):
# Get the username from the twitter api and save the
# user and tweets objects
api = twitter_api_client()
twitter_user = api.get_user(screen_name)
statuses = api.user_timeline(screen_name,
tweet_mode="extended",
count=150,
exclude_replies=True,
include_rts=False)
# Check to see if the user already exists in the db user table and
# if not then add it to the db user table
db_user = User.query.get(twitter_user.id) or User(id=twitter_user.id)
db_user.screen_name = twitter_user.screen_name
db_user.name = twitter_user.name
db_user.location = twitter_user.location
db_user.followers_count = twitter_user.followers_count
db.session.add(db_user)
db.session.commit()
print("STATUS COUNT:", len(statuses))
# Use the basilica api to turn the tweets (statuses) into numeric lists
basilica_api = basilica_api_client()
all_tweet_texts = [status.full_text for status in statuses]
embeddings = list(
basilica_api.embed_sentences(all_tweet_texts, model="twitter"))
print("NUMBER OF EMBEDDINGS", len(embeddings))
# Put all of the tweets in the db tweet table
counter = 0
for status in statuses:
print(status.full_text)
print("----")
# Find or create database tweet:
db_tweet = Tweet.query.get(status.id) or Tweet(id=status.id)
db_tweet.user_id = status.author.id # or db_user.id
db_tweet.full_text = status.full_text
embedding = embeddings[counter]
print(len(embedding))
db_tweet.embedding = embedding
db.session.add(db_tweet)
counter += 1
db.session.commit()
return db_user, statuses
def get_user(screen_name=None):
# print(screen_name)
api = twitter_api_client()
twitter_user = api.get_user(screen_name)
statuses = api.user_timeline(screen_name, tweet_mode="extended", count=150)
# print("STATUSES COUNT:", len(statuses))
# get existing user from the db or initialize a new one:
db_user = User.query.get(twitter_user.id) or User(id=twitter_user.id)
db_user.screen_name = twitter_user.screen_name
db_user.name = twitter_user.name
db_user.location = twitter_user.location
db_user.followers_count = twitter_user.followers_count
db.session.add(db_user)
db.session.commit()
basilica_api = basilica_api_client()
all_tweet_texts = [status.full_text for status in statuses]
embeddings = list(
basilica_api.embed_sentences(all_tweet_texts, model="twitter"))
# print("NUMBER OF EMBEDDINGS", len(embeddings))
counter = 0
for status in statuses:
# print(status.full_text)
# print("----")
# get existing tweet from the db or initialize a new one:
db_tweet = Tweet.query.get(status.id) or Tweet(id=status.id)
db_tweet.user_id = status.author.id # or db_user.id
db_tweet.full_text = status.full_text
embedding = embeddings[counter]
# print(len(embedding))
db_tweet.embedding = embedding
db.session.add(db_tweet)
counter += 1
db.session.commit()
# return "OK"
return render_template("user.html", user=db_user, tweets=statuses)
def seed_db():
# print(type(db))
api = twitter_api_client()
for screen_name in ['elonmusk','justinbieber','s2t2']:
twitter_user = api.get_user(screen_name)
statuses = api.user_timeline(screen_name, tweet_mode="extended", count=150)
# print("STATUSES COUNT:", len(statuses))
# get existing user from the db or initialize a new one:
db_user = User.query.get(twitter_user.id) or User(id=twitter_user.id)
db_user.screen_name = twitter_user.screen_name
db_user.name = twitter_user.name
db_user.location = twitter_user.location
db_user.followers_count = twitter_user.followers_count
db.session.add(db_user)
db.session.commit()
basilica_api = basilica_api_client()
all_tweet_texts = [status.full_text for status in statuses]
embeddings = list(basilica_api.embed_sentences(all_tweet_texts, model="twitter"))
# print("NUMBER OF EMBEDDINGS", len(embeddings))
counter = 0
for status in statuses:
# print(status.full_text)
# print("----")
# get existing tweet from the db or initialize a new one:
db_tweet = Tweet.query.get(status.id) or Tweet(id=status.id)
db_tweet.user_id = status.author.id # or db_user.id
db_tweet.full_text = status.full_text
embedding = embeddings[counter]
# print(len(embedding))
db_tweet.embedding = embedding
db.session.add(db_tweet)
counter+=1
db.session.commit()
return jsonify({"message": "DB SEEDED OK"})
def store_twitter_user_data(screen_name):
print(screen_name)
api = twitter_api_client()
twitter_user = api.get_user(screen_name)
statuses = api.user_timeline(screen_name, tweet_mode="extended", count=150)
db_user = User.query.get(twitter_user.id) or User(id=twitter_user.id)
db_user.screen_name = twitter_user.screen_name
db_user.name = twitter_user.name
db_user.location = twitter_user.location
db_user.followers_count = twitter_user.followers_count
db.session.add(db_user)
db.session.commit()
print("STATUS COUNT:", len(statuses))
basilica_api = basilica_api_client()
all_tweet_texts = [status.full_text for status in statuses]
embeddings = list(
basilica_api.embed_sentences(all_tweet_texts, model="twitter"))
print("NUMBER OF EMBEDDINGS", len(embeddings))
counter = 0
for status in statuses:
print(status.full_text)
print("----")
# print(dir(status))
# Find or create database tweet:
db_tweet = Tweet.query.get(status.id) or Tweet(id=status.id)
db_tweet.user_id = status.author.id # or db_user.id
db_tweet.full_text = status.full_text
# embedding = basilica_client.embed_sentence(status.full_text, model="twitter") # todo: prefer to make a single request to basilica with all the tweet texts, instead of a request per tweet
embedding = embeddings[counter]
print(len(embedding))
db_tweet.embedding = embedding
db.session.add(db_tweet)
counter += 1
db.session.commit()
return db_user, statuses