def _ingest_event(stream_event, session, twitter_api, tweet_queue, user_queue):
"""Helper function which does the individual inserts.
Used to factor inserts like retweets and quotes which may contain their substructures directly,
and thus avoid needing to become queued and get processed from a work queue.
"""
log.debug(json.dumps(stream_event))
if stream_event and stream_event.get("event"):
# This is the case of a non-message stream_event
# FIXME: see other interesting cases here:
# https://dev.twitter.com/streaming/overview/messages-types
if stream_event.get("source"):
ingest_user_object(
User.NewFromJsonDict(stream_event.get("source")), session)
if stream_event.get("target"):
ingest_user_object(
User.NewFromJsonDict(stream_event.get("target")), session)
if stream_event.get("event") in [
"favorite", "unfavorite", "quoted_tweet"
]:
# We're ingesting a tweet here
_ingest_event(stream_event.get("target_object"), session,
twitter_api, tweet_queue, user_queue)
elif stream_event.get("delete"):
# For compliance with the developer rules.
# Sadly.
event_id = stream_event.get("delete").get("status").get("id")
log.warn("Honoring delete %s", event_id)
entity = bt._tweet_or_dummy(session, event_id)
entity.tombstone = True
session.add(entity)
session.commit()
elif stream_event and "id" in stream_event and "user" in stream_event:
if "extended_tweet" in stream_event:
# This is the case of having gotten a new "extended" tweet.
# Need to munge the text representation.
#
# And by munge I just mean copy, because the twitter-python driver drops this on the floor
stream_event["text"] = stream_event["extended_tweet"]["full_text"]
ingest_tweet(stream_event, session, twitter_api, tweet_queue)
elif "friends" in stream_event:
for friend in stream_event.get("friends"):
user_queue.put(str(friend))
else:
blob = json.dumps(stream_event)
with open("mystery.log", "a") as f:
f.write(blob + "\n")
log.warn(blob)
def twit(request, page_number=1):
if 'q' in request.GET:
t = Twitter()
t.text = request.GET['q']
try:
user_name = User.objects.get(name=str(auth.get_user(request).username))
except:
user_name = User(name=str(auth.get_user(request).username))
user_name.save()
t.name = User.objects.get(name=str(auth.get_user(request).username))
t.save()
all_texts=Twitter.objects.all()
current_page = Paginator(all_texts,4)
return render_to_response('twit.html', {'text': current_page.page(page_number), 'username': auth.get_user(request).username})
def addUser(request):
if request.method=='POST':
form=addForm(request.POST)
if form.is_valid():
username=form.cleaned_data['username']
email=form.cleaned_data['email']
passwd=form.cleaned_data['passwd']
p1=User(userName=username,nikename=email,passwd=passwd)
p1.save()
form1=loginForm()
form2=addForm()
return render_to_response('mainpage.html',{'form1':form1,'form2':form2},context_instance=RequestContext(request))
def batch_processing(cls, users_attributes, tweets_attributes):
[
u_attr.update({
'created_at':
datetime.datetime.strptime(u_attr['created_at'],
cls.DATETIME_FORMAT),
'modified':
datetime.datetime.strptime(u_attr['modified'],
cls.DATETIME_FORMAT)
}) for u_attr in users_attributes
]
users = User.create_or_update(*users_attributes)
[
t_attr.update({
'created_at':
datetime.datetime.strptime(t_attr['created_at'],
cls.DATETIME_FORMAT),
"coordinates":
t_attr.get("coordinates") if t_attr.get("coordinates") else
getattr(users[k], "coordinates", [])
}) for k, t_attr in enumerate(tweets_attributes)
]
tweets = Tweet.create_or_update(*tweets_attributes)
for k, tweet in enumerate(tweets):
users[k].posts.connect(tweet)
def addUser(request):
if request.method == 'POST':
form = addForm(request.POST)
if form.is_valid():
username = form.cleaned_data['username']
email = form.cleaned_data['email']
passwd = form.cleaned_data['passwd']
p1 = User(userName=username, nikename=email, passwd=passwd)
p1.save()
form1 = loginForm()
form2 = addForm()
return render_to_response('mainpage.html', {
'form1': form1,
'form2': form2
},
context_instance=RequestContext(request))
class TwitterUserTests(unittest.TestCase):
def setUp(self):
self.twitter = Twitter()
self.user = User(name='matt', twitter=self.twitter)
self.steve = User(name='steve')
self.user.follow(self.steve)
def should_allow_users_to_follow_other_users(self):
self.assertEqual([self.steve], self.user.followed_tweeters)
def should_limit_timeline_to_followed_tweeters_tweets(self):
self.twitter.tweet(self.steve, 'hello there')
self.twitter.tweet('someone else', 'hi')
self.twitter.tweet(self.steve, 'hello again')
self.assertEqual([{'user':self.steve, 'message':'hello again'},
{'user':self.steve, 'message':'hello there'},
], list(self.user.timeline))
def should_limit_timeline_10_tweets_from_followed_users(self):
carl = User(name='Carl')
self.user.follow(carl)
for i in range(11):
self.twitter.tweet(self.steve, str(i))
self.twitter.tweet(carl, str(i))
self.twitter.tweet('asf', str(i))
self.assertEqual([{'user':carl, 'message':'10'},
{'user':self.steve, 'message':'10'},
{'user':carl, 'message':'9'},
{'user':self.steve, 'message':'9'},
{'user':carl, 'message':'8'},
{'user':self.steve, 'message':'8'},
{'user':carl, 'message':'7'},
{'user':self.steve, 'message':'7'},
{'user':carl, 'message':'6'},
{'user':self.steve, 'message':'6'},
],
list(self.user.timeline))
def get_user(user_id: int) -> User:
cache = get_cache("user", user_id)
retries = 0
if cache is None:
print("Making network request for user id %s" % user_id)
while True:
try:
data = api.GetUser(user_id)
break
except ConnectionError:
retries += 1
if retries > 1:
raise
cache = write_cache("user", data)
return User.NewFromJsonDict(cache)
def register():
form = RegistrationForm()
if current_user.is_authenticated:
return redirect(url_for('home'))
if form.validate_on_submit():
hashed_password = bcrypt.generate_password_hash(
form.password.data).decode('utf-8')
user = User(username=form.username.data,
email=form.email.data,
password=hashed_password)
db.session.add(user)
db.session.commit()
flash(f'Account created for { form.username.data }!', 'success')
return redirect(url_for('login'))
return render_template('register.html', title='Register', form=form)
def ingest_users(twitter_api, session, users):
twitter_user = None
line = None
for screen_name in users:
try:
twitter_user = twitter_api.GetUser(screen_name=screen_name)
if isinstance(twitter_user, dict):
twitter_user = User.NewFromJsonDict(twitter_user)
persona = personas.personas_by_name(session, line, one=True, exact=True)
print(bt.insert_user(session, twitter_user, persona))
except TwitterError as e:
print(line, line, e)
except AssertionError as e:
print(line, twitter_user, e)
raise e
def ingest_tweet(tweet, session, twitter_api, tweet_id_queue):
"""Actually ingest a single tweet, dealing with the required enqueuing."""
if not isinstance(tweet, Status):
tweet = Status.NewFromJsonDict(tweet)
if tweet.retweeted_status:
# We don't actually care about retweets, they aren't original content.
# Just insert the original.
ingest_tweet(tweet.retweeted_status, session, twitter_api,
tweet_id_queue)
ingest_user_object(tweet.user, session)
else:
flag = have_tweet(session, tweet.id)
t = bt.insert_tweet(session, twitter_api, tweet)
if not flag:
log.info(t)
if tweet.in_reply_to_status_id:
# This tweet is a reply. It links to some other tweet. Or possibly tweets depending on the
# link content which may link many statuses. However Twitter only considers one status to
# be the "reply" target. Create a "reply_to" relationship for the post we're inserting by
# inserting its parent post(s) (recursively!)
thread_id = str(tweet.in_reply_to_status_id)
if not have_tweet(session, thread_id):
tweet_id_queue.put(thread_id)
pass
if tweet.quoted_status:
# This is a quote tweet (possibly subtweet or snarky reply, quote tweets have different
# broadcast mechanics).
ingest_tweet(tweet.quoted_status, session, twitter_api,
tweet_id_queue)
for url in tweet.urls or []:
tweet_id = bt.tweet_id_from_url(url.expanded_url)
if tweet_id and not have_tweet(session, tweet_id):
tweet_id_queue.put(tweet_id)
pass
for user in tweet.user_mentions or []:
if not isinstance(user, User):
user = User.NewFromJsonDict(user)
ingest_user_object(user, session)
def reset_token(token):
if current_user.is_authenticated:
return redirect(url_for('home'))
user = User.verify_reset_token(token)
if user is None:
flash('That is an invalid or expired token', 'warning')
return redirect(url_for('reset_request'))
form = ResetPasswordForm()
if form.validate_on_submit():
hashed_password = bcrypt.generate_password_hash(
form.password.data).decode('utf-8')
user.password = hashed_password
db.session.commit()
flash(f'Password Updated!', 'success')
return redirect(url_for('login'))
return render_template('reset_token.html',
title='Reset Password',
form=form)
def create_user(screen_name=None):
screen_name = request.form["user"]
twitter_user = twitter_api_client.get_user(screen_name)
statuses = twitter_api_client.user_timeline(screen_name,
tweet_mode="extended",
count=150,
exclude_replies=True,
include_rts=False)
print("STATUSES COUNT:", len(statuses))
# get existing user from the db or initialize a new one:
db_user = User.query.get(twitter_user.id) or User(id=twitter_user.id)
db_user.screen_name = twitter_user.screen_name
db_user.name = twitter_user.name
db_user.location = twitter_user.location
db_user.followers_count = twitter_user.followers_count
db.session.add(db_user)
db.session.commit()
all_tweet_texts = [status.full_text for status in statuses]
embeddings = list(
basilica_api_client.embed_sentences(all_tweet_texts, model="twitter"))
print("NUMBER OF EMBEDDINGS", len(embeddings))
# TODO: explore using the zip() function maybe...
counter = 0
for status in statuses:
print(status.full_text)
print("----")
# get existing tweet from the db or initialize a new one:
db_tweet = Tweet.query.get(status.id) or Tweet(id=status.id)
db_tweet.user_id = status.author.id
db_tweet.full_text = status.full_text
embedding = embeddings[counter]
print(len(embedding))
db_tweet.embedding = embedding
db.session.add(db_tweet)
counter += 1
db.session.commit()
return render_template("user.html", user=db_user, tweets=statuses)
def batch_processing(cls, users_attributes, tweets_attributes):
[
u_attr.update({
'created_at':
datetime.datetime.strptime(u_attr['created_at'],
cls.DATETIME_FORMAT),
'modified':
datetime.datetime.strptime(u_attr['modified'],
cls.DATETIME_FORMAT)
}) for u_attr in users_attributes
]
users = User.create_or_update(*users_attributes)
[
t_attr.update({
'created_at':
datetime.datetime.strptime(t_attr['created_at'],
cls.DATETIME_FORMAT),
"coordinates":
t_attr.get("coordinates") if t_attr.get("coordinates") else
getattr(users[k], "coordinates", [])
}) for k, t_attr in enumerate(tweets_attributes)
]
tweets = Tweet.create_or_update(*tweets_attributes)
for k, tweet in enumerate(tweets):
users[k].posts.connect(tweet)
# Connect tweet with the company it mentions
# company_name = cls.find_company(tweet.text)
# company_node = Company.nodes.get(name=company_name)
# tweet.tweet_about.connect(company_node)
# for demo
company_event = Event.nodes.get(name='Autopilot rolled out')
company_event.tweet_from.connect(tweet)
def setUp(self):
self.twitter = Twitter()
self.user = User(name='matt', twitter=self.twitter)
self.steve = User(name='steve')
self.user.follow(self.steve)
class TestModel(unittest.TestCase):
dao = None
USER_IDS = (12345678901, 12345678902, 12345678903, 12345678904,
12345678905)
SCREEN_NAMES = ('user_1', 'user_2', 'user_3', 'user_4', 'user_5')
NAMES = ('name 1', '五毛外宣部副部长\U0001f1e8\U0001f1f3', 'name 3', 'name 4',
'name 5')
DESCRIPTIONS = (
'予人玫瑰 手有餘香', '北京摄影老顽童\U0001f1e8\U0001f1f3',
'spread positive energy, let the world understand China! \n再艰难，爱不会离开！',
'', '')
CREATED_ATS = ('Tue Mar 29 08:11:25 +0000 2011',
'Tue Mar 29 08:11:25 +0000 2020',
'Tue Mar 29 08:11:25 +0000 2020',
'Tue Mar 29 08:11:25 +0000 2011',
'Tue Mar 29 08:11:25 +0000 2011')
FOLLOWER_COUNTS = (1, 56, 878264, 223, 872)
FRIEND_COUNTS = (3215, 0, 782, 3295, 3)
METHODS = ('get_following_paged', 'get_followers_paged')
CURSORS = (1656974570956055733, 1656809280611943888)
USERS = [
User(id=user_id,
screen_name=screen_name,
name=name,
description=desc,
created_at=dt,
followers_count=follower_count,
friends_count=friend_count)
for user_id, screen_name, name, desc, dt, follower_count, friend_count
in zip(USER_IDS, SCREEN_NAMES, NAMES, DESCRIPTIONS, CREATED_ATS,
FOLLOWER_COUNTS, FRIEND_COUNTS)
]
@classmethod
def setUpClass(cls) -> None:
print('setUpClass started')
cls.dao = Dao()
cls.dao.reset_db()
print('setUpClass ended')
@classmethod
def tearDownClass(cls) -> None:
print('tearDownClass started')
cls.dao.reset_db()
print('tearDownClass ended')
def setUp(self):
"""save to DB, get variables of table instances:
5 tweeters, 2 tracks, 2 old wumaos, 2 new wumao
instance variables:
* tweeters
* tracks
* old_wumao_tweeter_ids
* new_wumao_tweeter_ids
* old_wumaos
* new_wumaos
:return:
"""
print('setUp started')
self.dao.bulk_save_tweeter(self.USERS)
tweeter_ids = self.dao.all_tweeter_id(self.USER_IDS)
tracks_ = [
Track(tweeter_id, method,
cursor) for tweeter_id, method, cursor in zip(
tweeter_ids, self.METHODS, self.CURSORS)
]
self.dao.bulk_save(tracks_)
self.tweeters = [self.dao.lookup_tweeter(i) for i in tweeter_ids]
self.tracks = [
self.dao.lookup_track(track.tweeter_id) for track in tracks_
]
self.old_wumao_tweeter_ids = [u.id for u in self.tweeters][:2]
self.new_wumao_tweeter_ids = [u.id for u in self.tweeters][2:4]
self.dao.bulk_save_wumao(self.old_wumao_tweeter_ids, new=False)
self.dao.bulk_save_wumao(self.new_wumao_tweeter_ids, new=True)
wumaos = [self.dao.lookup_wumao(i) for i in self.dao.all_wumao_id()]
self.old_wumaos = [w for w in wumaos if w.is_new == 0]
self.new_wumaos = [w for w in wumaos if w.is_new == 1]
print('setUp ended')
def tearDown(self):
print('tearDown started')
self.dao.reset_db()
print('tearDown ended')
def test_bulk_save(self):
"""bulk save
methods:
* dao.bulk_save
* dao.bulk_save_tweeter
:return:
"""
self.assertEqual(set(), self.dao.bulk_save_tweeter(self.USERS))
self.assertEqual(set(),
self.dao.bulk_save_wumao(self.old_wumao_tweeter_ids))
def test_tweeter(self):
"""checks DAO methods on table 'tweeter'
methods:
* dao.lookup_tweeter_id
* dao.all_tweeter_user_id
:return:
"""
set_1 = set(
self.dao.lookup_tweeter(tweeter.id).id
for tweeter in self.tweeters)
set_2 = self.dao.all_tweeter_id(TestModel.USER_IDS)
self.assertEqual(set_1, set_2)
def test_friendship(self):
"""checks many-to-many junction table 'friendship' and its 'on-delete'
constrain
friendship: 1 follows 2, 3, 4; 5 follows 1
methods:
* dao.follow
* dao.bulk_follow
* dao.bulk_attract
* dao.friend_count
* dao.follower_count
* dao.delete_tweeter
:return:
"""
tweeter_id_1, tweeter_id_2, tweeter_id_3, tweeter_id_4, tweeter_id_5 = (
u.id for u in self.tweeters)
self.dao.follow(tweeter_id_5, tweeter_id_1)
self.dao.bulk_follow(tweeter_id_1, [tweeter_id_2, tweeter_id_3])
self.dao.bulk_attract(tweeter_id_4, [tweeter_id_1])
self.assertEqual(True, self.dao.is_following(tweeter_id_1,
tweeter_id_2))
self.assertEqual(True, self.dao.is_following(tweeter_id_1,
tweeter_id_3))
self.assertEqual(True, self.dao.is_following(tweeter_id_1,
tweeter_id_4))
self.assertEqual(False,
self.dao.is_following(tweeter_id_1, tweeter_id_5))
self.assertEqual(True, self.dao.is_following(tweeter_id_5,
tweeter_id_1))
self.assertEqual(3, self.dao.friend_count(tweeter_id_1))
self.assertEqual(
2, self.dao.friend_count(tweeter_id_1,
[tweeter_id_2, tweeter_id_4]))
self.assertEqual(1, self.dao.follower_count(tweeter_id_1))
self.assertEqual(0,
self.dao.follower_count(tweeter_id_1, [tweeter_id_1]))
self.assertEqual(0, self.dao.follower_count(999))
# on-delete constrain
self.dao.delete_tweeter(tweeter_id_3)
self.assertEqual([tweeter_id_2, tweeter_id_4],
self.dao.friends_id(tweeter_id_1))
self.assertEqual([tweeter_id_5], self.dao.followers_id(tweeter_id_1))
def test_wumao(self):
"""checks DAO methods of table 'wumao' and its 'on-delete' constrain
methods
* dao.all_wumao_id
* dao.all_wumao_tweeter_id
* dao.delete_tweeter
:return:
"""
tweeter_id_1, tweeter_id_2, tweeter_id_3, tweeter_id_4, tweeter_id_5 = (
u.id for u in self.tweeters)
old_wumao_1, old_wumao_2 = self.old_wumaos
new_wumao_1, new_wumao_2 = self.new_wumaos
self.assertEqual(set(self.old_wumao_tweeter_ids),
{old_wumao_1.tweeter_id, old_wumao_2.tweeter_id})
self.assertEqual(set(self.new_wumao_tweeter_ids),
{new_wumao_1.tweeter_id, new_wumao_2.tweeter_id})
# on-delete constrain
self.dao.delete_tweeter(tweeter_id_3)
self.assertEqual({new_wumao_2.id, old_wumao_1.id, old_wumao_2.id},
self.dao.all_wumao_id())
self.assertEqual(
{
new_wumao_2.tweeter_id, old_wumao_1.tweeter_id,
old_wumao_2.tweeter_id
}, self.dao.all_wumao_tweeter_id())
# update
self.dao.upsert_wumao(new_wumao_2.tweeter_id, False)
self.assertEqual(None, self.dao.any_wumao(True))
self.dao.upsert_wumao(old_wumao_1.tweeter_id, True)
self.assertEqual(True, self.dao.lookup_wumao(old_wumao_1.id).is_new)
self.assertEqual(False, self.dao.lookup_wumao(new_wumao_2.id).is_new)
def test_track(self):
"""checks DAO methods of table 'track'
methods:
* dao.lookup_track
* dao.delete_track
* dao.update_track
:return:
"""
# initialize & preparation
track_1, track_2 = self.tracks
# delete
self.assertEqual(set(TestModel.CURSORS),
{track_1.cursor, track_2.cursor})
self.dao.delete_track(track_1.tweeter_id)
self.assertEqual(None, self.dao.lookup_track(track_1.tweeter_id))
self.assertEqual(track_2, self.dao.any_track())
self.dao.delete_track(track_2.tweeter_id)
self.assertEqual(None, self.dao.lookup_track(track_2.tweeter_id))
self.assertEqual(None, self.dao.any_track())
# multiple tracks update
# overwrite track_1 with track_2's method & cursor
self.dao.upsert_track(track_1.tweeter_id, track_1.method,
track_1.cursor)
self.dao.upsert_track(track_1.tweeter_id, track_2.method,
track_2.cursor)
self.dao.upsert_track(track_2.tweeter_id, track_2.method,
track_2.cursor)
self.dao.upsert_track(track_2.tweeter_id, track_2.method,
track_2.cursor)
self.assertEqual(track_2.cursor,
self.dao.lookup_track(track_1.tweeter_id).cursor)
self.assertEqual(track_2.method,
self.dao.lookup_track(track_1.tweeter_id).method)
self.assertEqual(track_2.cursor,
self.dao.lookup_track(track_2.tweeter_id).cursor)
def _fake_users(*_, screen_name=None): # id is the wrong type here, but we just need unique objects return [User(id=n, screen_name=n) for n in screen_name]
def insert_tweet(session, twitter_api, tweet):
"""Insert a tweet (status using the old API terminology) into the backing datastore.
This means inserting the original poster, inserting the service, inserting the post and inserting
the post distribution.
WARNING: this function does NOT recursively insert replied to tweets, or quoted tweets. It's
expected that some other system handles walking the tree of tweets to deal with all that. This is,
ultimately, to work around the garbage Twitter rate limits.
"""
_tw = insert_twitter(session)
try:
poster = tweet.user
if not isinstance(poster, User):
poster = User.NewFromJsonDict(poster)
poster = insert_user(session, poster)
assert isinstance(poster, Account)
except AssertionError as e:
print("Encountered exception", repr(e), traceback.format_exc(),
"Processing tweet", tweet)
return None
dupe = session.query(Post)\
.filter_by(external_id=twitter_external_tweet_id(tweet.id))\
.first()
# There's a dummy record in place, flesh it out. We're in a monoid here.
if dupe:
dupe.poster = poster
dupe.when = aget(
datetime.strptime(tweet.created_at, _tw_datetime_pattern))
dupe.text = _get_tweet_text(tweet)
dupe.more = tweet.AsDict()
session.add(dupe)
session.commit()
return dupe
# We're inserting a new tweet here...
else:
post = Post(service=_tw,
text=_get_tweet_text(tweet),
external_id=twitter_external_tweet_id(tweet.id_str),
poster=poster,
when=aget(
datetime.strptime(tweet.created_at,
_tw_datetime_pattern)),
more=tweet.AsDict())
session.add(post)
for user in tweet.user_mentions:
get_or_create(session,
PostDistribution,
post=post,
recipient=insert_user(session, User(id=user.id)),
rel="to")
if tweet.in_reply_to_status_id:
get_or_create(session,
PostRelationship,
left=post,
right=_tweet_or_dummy(session,
tweet.in_reply_to_status_id),
rel="reply-to")
if tweet.quoted_status_id:
get_or_create(session,
PostRelationship,
left=post,
right=_tweet_or_dummy(session,
tweet.quoted_status_id),
rel="quotes")
session.commit()
return post
from twitter.models import Status, User
MENTIONED_TWEETS = [
Status(id_str="0001_1", user=User(screen_name="user0001")),
Status(id_str="0001_2", user=User(screen_name="user0001")),
Status(id_str="0002_1", user=User(screen_name="user0002")),
Status(id_str="0003_1", user=User(screen_name="user0003")),
Status(id_str="0004_1", user=User(screen_name="user0004")),
]
SELF_TWEETS = [
Status(id_str="9999_1"),
Status(id_str="9999_2"),
Status(id_str="9999_3", in_reply_to_status_id="0003_1"),
Status(id_str="9999_4")
]
'statuses_count': [None, 0, 1, 100, 1500],
'created_at': [
None, "Wed May 02 00:47:05 +0000 2012",
"Wed May 02 00:47:05 +0000 2016", "Wed May 02 00:47:05 +0000 2018",
"Mon Sep 03 00:47:05 +0000 2018"
],
'profile.status.created_at': [
None, "Wed May 02 00:47:05 +0000 2012",
"Wed May 02 00:47:05 +0000 2016", "Wed May 02 00:47:05 +0000 2018",
"Mon Sep 03 00:47:05 +0000 2018"
]
}
profiles = []
for i in range(5):
np = User()
np.id = 'user%s' % i
np.screen_name = 'username%s' % i
for c in confs:
if c == 'description':
np.description = confs[c][i]
elif c == 'followers_count':
np.followers_count = confs[c][i]
elif c == 'location':
np.location = confs[c][i]
elif c == 'statuses_count':
np.statuses_count = confs[c][i]
elif c == 'created_at':
np.created_at = confs[c][i]
elif c == 'profile.status.created_at':
ns = Status()