def search_tweets(request):
tweet_list = []
url = TWITTER_URL + request.GET['handle']
status_list = get_tweets(url)
tweet_id = 0
for status in status_list:
regex = re.compile(r'[\n]')
tweet_text = regex.sub('<br>', status['text'])
new_tweet = Tweet(tweet_id=tweet_id,
tweet_text=tweet_text,
user_name=status['user']['name'],
screen_name=status['user']['screen_name'])
tweet_list.append(new_tweet)
tweet_id += 1
template = loader.get_template('tweet_list.html')
context = RequestContext(request, {
'tweet_list': tweet_list,
'handle': request.GET['handle'],
})
return HttpResponse(template.render(context))
def tweet(request):
if request.method == 'POST':
form = TweetForm(request.POST)
if form.is_valid():
twit = Tweet()
status = twit.tweet = form.cleaned_data['tweet']
twit.name = request.user.username
twit.save()
variables = RequestContext(request, {'status': status})
return render_to_response('tweet.html', RequestContext(request),
variables)
else:
return render_to_response('tweet.html',
{'request': RequestContext(request)})
else:
tweets = []
queryset = Tweet.objects.all()
for query in queryset:
if request.user.username == query.name:
tweets.append(query)
status = ''
for tweet in tweets:
status = tweet.tweet
form = TweetForm()
form.name = request.user
variables = RequestContext(request, {'form': form, 'status': status})
return render_to_response('tweet.html', RequestContext(request),
variables)
def database():
search_term = "bills.com"
print search_term
api = Api(consumer_key='uy3Utit3PhWPmM5284sh7w',
consumer_secret='zA6ps76R3ID2oRXXdUjQ7jsPgcmXD1rPsNYVSw',
access_token_key='61793468-PoarjfBLEa8KxtmFvNcLwFlvSGsh03U2LyNbroXye',
access_token_secret='qg6cEIa33rX725oVAqmN4CtIgG0fmuPeI7pb2Vjeyxs')
results = api.GetSearch(search_term)
tweets = []
for r in results:
user = r.GetUser()
text = r.GetText()
seconds_ago = r.GetCreatedAtInSeconds()
hours_ago = ConvertTime(seconds_ago)
source_url = r.GetSource()
screenname = user.GetScreenName()
image = user.GetProfileImageUrl()
tweet = {"text": text, "hours_ago":hours_ago, "screenname": screenname,
"img": image, "view_tweet": source_url}
tweets.append(tweet)
#Add each tweet to database.
t = Tweet( author= screenname, body = text, is_read = False )
db.session.add(t)
for t in tweets:
print t
db.session.commit()
return "Connected to db. Check db."
def add_or_update_user(username):
""" Add or update user and their tweets else error if not a Twitter User"""
try:
twitter_user = TWITTER.get_user(username)#from api
## Getting picture, follower count, etc. you get that info here.
db_user = (User.query.get(twitter_user.id) or
User(id=twitter_user.id, name=username))#add or update
#try to get the user from database first so hey does user exist already exist
#so not overrides
#if does not exist returns none and evaluates to the second line
DB.session.add(db_user)#add to database
#We want as many recent non-retweet/reply statuses as we can get
# Limit is 250
tweets = twitter_user.timeline(
count=250, exclude_replies=True, include_rts=False,
tweet_mode='extended', since_id=db_user.newest_tweet_id)#from api
#we pull tweets since the id of the latest tweet
if tweets:
db_user.newest_tweet_id = tweets[0].id#fails if there are no tweets, so if there are tweets do this
for tweet in tweets:
# Get embedding for tweet, and store in db
embedding = BASILICA.embed_sentence(tweet.full_text,
mode='twitter')#tweet returned by twitter
db_tweet = Tweet(id=tweet.id, text=tweet.full_text[:500], embedding=embedding)#500 character limit
db_user.tweets.append(db_tweet)#append tweet to user
DB.session.add(db_tweet)#add tweet to database session
except Exception as e:
print("Error processing {}: {}".format(username, e))
raise e#exception will keep going, whoever called this will also get the exeception
#so will show on the website
#raise means propagate, it casts exception but keeps going
else:
DB.session.commit()
def load_tweets(profile, api):
tweets = []
for status in tweepy.Cursor(api.user_timeline).items():
print str(status.id)
tweet = Tweet(temp=str(status.id)[-9:])
tweet.text = unicode(status.text)
tweet.created = timezone.make_aware(status.created_at, timezone.get_current_timezone())
tweet.is_retweet = status.retweeted
tweet.retweet_count = status.retweet_count
tweet.is_quote = status.is_quote_status
if status.in_reply_to_status_id is not None:
tweet.is_reply = True
for media in status.entities.get("media", [{}]):
if media.get("type", None):
tweet.media_url = media.get('media_url')
if 'video' in tweet.media_url:
tweet.is_video = True
else:
tweet.is_image = True
# tweets.append(tweet)
tweet.save()
profile.tweets.add(tweet)
# profile.tweets.bulk_create(tweets)
profile.save()
def parse_tweet(self, tweet) -> Tweet:
t = Tweet(
id=tweet.id,
text=html.unescape(tweet.text),
created_at=tweet.created_at,
user_name=tweet.user.name,
user_screen_name=tweet.user.screen_name,
)
if hasattr(tweet, 'retweeted_status'):
t.text = u'\u267B' + ' @' + tweet.retweeted_status.user.screen_name + ': ' + html.unescape(
tweet.retweeted_status.text)
if hasattr(tweet, 'quoted_status'):
t.text = re.sub(r' https://t\.co/[1-9a-zA-Z]+$', r'',
t.text) + "\n"
t.text += u'\u267B' + ' @' + tweet.quoted_status.user.screen_name + ': ' + html.unescape(
tweet.quoted_status.text)
tweet.entities.urls = []
if 'extended_entities' in tweet.quoted_status:
self.parse_tweet_media(t,
tweet.quoted_status.extended_entities)
if hasattr(tweet, 'extended_entities'):
self.parse_tweet_media(t, tweet.extended_entities)
elif tweet.entities.urls:
t.link_url = tweet.entities.urls[0].expanded_url
for url_entity in tweet.entities.urls:
expanded_url = url_entity.expanded_url
indices = url_entity.indices
display_url = tweet.text[indices[0]:indices[1]]
t.text = t.text.replace(display_url, expanded_url)
return t
def extract_tweet_and_user(self, fetched_tweet):
tweet = Tweet()
user = None
for attribute_string in dir(fetched_tweet):
if not attribute_string.startswith("__"):
fetched_tweet_field = getattr(fetched_tweet, attribute_string)
if fetched_tweet_field == "extended_tweet":
setattr(tweet, "extended_text",
fetched_tweet_field["full_text"])
try:
getattr(Tweet, attribute_string)
setattr(tweet, attribute_string, fetched_tweet_field)
except AttributeError as e:
pass
if isinstance(fetched_tweet_field, tweepy.User) and not user:
user = User()
tweet.user_id = fetched_tweet_field.id
user.twitter_user_id = fetched_tweet_field.id
user.name = fetched_tweet_field.name
user.screen_name = fetched_tweet_field.screen_name
user.statuses_count = fetched_tweet_field.statuses_count
user.followers_count = fetched_tweet_field.followers_count
user.friends_count = fetched_tweet_field.friends_count
user.location = fetched_tweet_field.location
return tweet, user
def createStreamingContext():
# Create a local StreamingContext with two working thread and batch interval of 1 second
sc = SparkContext("spark://%s:7077" % MASTER_NAME,
appName="GlutenTweet",
pyFiles=PYFILES)
ssc = StreamingContext(sc, 2)
# Create a DStream of raw data
raw = ssc.socketTextStream(MASTER_IP, 9999)
# Convert into models
tweets = raw.map(lambda r: Tweet(raw_json=r))
# Store models
tweets.foreachRDD(storeTweetsRDD)
# Sliding window analysis
window = tweets.window(20 * 60, 30)
hashtagCounts = analysisHahtagCount(window)
streamTop(hashtagCounts).pprint()
# Keyword extraction - note tweets is immutable
tweetsKeyword = tweets.map(lambda t: keywordExtraction(t))
# Update models
tweetsKeyword.foreachRDD(updateTweetsRDD)
# Sliding window analysis
window2 = tweetsKeyword.window(20 * 60, 30)
keywordCounts = analysisKeywordCount(window2)
streamTop(keywordCounts).pprint()
ssc.checkpoint(CHECKPOINT_DIR)
return ssc
def add_or_update_user(username):
"""Add or pdate a user and their tweets, or else give error"""
try:
twitter_user = TWITTER.get_user(username)
db_user = (User.query.get(twitter_user.id)
or User(id=twitter_user.id, name=username))
DB.session.add(db_user)
tweets = twitter_user.timeline(count=200,
exclude_replies=True,
include_rts=False,
tweet_mode='extended',
since_id=db_user.newest_tweet_id)
if tweets:
db_user.newest_tweet_id = tweets[0].id
for tweet in tweets:
#Calculate embedding on the full tweet
embedding = BASILICA.embed_sentence(tweet.full_text,
model='twitter')
db_tweet = Tweet(id=tweet.id,
text=tweet.full_text[:300],
embedding=embedding)
db_user.tweets.append(db_tweet)
DB.session.add(db_tweet)
except Exception as e:
print('Error processing {}: {}'.format(username, e))
raise e
else:
DB.session.commit()
def add_tweet():
new_tweet = Tweet()
new_tweet.text = request.form['text']
new_tweet.created_at = request.form['created_at']
db.session.add(new_tweet)
db.session.commit()
return tweet_schema.jsonify(new_tweet)
def saveTweet(self, username, tweetContent, publishedDate):
tweetObject = Tweet(username=username,
text=tweetContent,
date=publishedDate)
# Creates an object to be saved in the Tweet model in the database.
tweetObject.save()
def post(self):
data = request.json
validation_error = validate_user(data)
if validation_error is not None:
return validation_error
user = get_user(data['user']['username'])
if user is None:
return jsonify(status=400, message="user does not exist")
if user.password != data['user']['password']:
return jsonify(status=403, message="incorrect password")
text = data['text']
if (len(text) > 140):
return jsonify(status=400, message="tweet too long"), 400
if (len(text) == 0):
return jsonify(status=400, message="empty tweet"), 400
for word in [u"#arsenal", u"#denfølelsen"]:
if word in text.lower():
abort(418) # I am a teapot
tweet = Tweet(user, text)
db['tweets'].append(tweet)
return jsonify(tweet.serialize()), 200
async def update_db(data):
parsed = json.loads(data)
print(parsed['text'])
vote, coin = is_vote(parsed["text"])
if None in [vote, coin]:
return
print(f'New vote detected : {coin}')
async with create_engine(
user="******",
database="deviant",
host="127.0.0.1",
password="******",
port=5432,
) as engine:
mgr = SQLModelManager.instance()
mgr.database = engine
user, created = await User.objects.get_or_create(id=int(parsed['id']))
if created:
user.name = parsed["user"]["name"]
user.screen_name = parsed["user"]["screen_name"]
user.followers_count = parsed["user"]["followers_count"]
user.statuses_count = parsed["user"]["statuses_count"]
await user.save()
new_record = Tweet()
new_record.id = parsed["id"]
new_record.text = parsed["text"]
new_record.created_at = tdate_to_timestamp(parsed["created_at"])
new_record.user = user
new_record.coin = coin.upper()
await new_record.save(force_insert=True)
def add_sentiment(term):
results = get_sentiment_results(term)
tweets = []
for tweet in results['tweets']:
tweetInstance = Tweet(username=tweet['username'],
content=tweet['text'],
favorites=tweet['favorites'],
retweets=tweet['retweets'],
polarity=tweet['polarity'],
normalizedSentiment=tweet['normalizedSentiment'],
createdAt=tweet["createdAt"],
profileImage=tweet["profileImage"],
tweetId=tweet["tweetId"],
name=tweet["name"],
neg=tweet['neg'],
neu=tweet['neu'],
pos=tweet['pos']).save()
tweets.append(tweetInstance)
sentiment = Sentiment(
term=term,
averagePolarity=results['averagePolarity'],
averageWeighedPolarity=results['averageWeighedPolarity'],
positiveTweetsCount=results['positiveTweetsCount'],
negativeTweetsCount=results['negativeTweetsCount'],
neutralTweetsCount=results['neutralTweetsCount'],
tweets=tweets)
sentiment.save()
def add_or_update_user(username):
""" Add or update user and their tweets else error if not a Twitter User"""
try:
twitter_user = TWITTER.get_user(username)
## Getting picture, follower count, etc. you get that info here.
db_user = (User.query.get(twitter_user.id)
or User(id=twitter_user.id, name=username))
DB.session.add(db_user)
# Limit is 250
tweets = twitter_user.timeline(count=250,
exclude_replies=True,
include_rts=False,
tweet_mode='extended',
since_id=db_user.newest_tweet_id)
if tweets:
db_user.newest_tweet_id = tweets[0].id
for tweet in tweets:
# Get embedding for tweet, and store in db
embedding = BASILICA.embed_sentence(tweet.full_text,
model='twitter')
db_tweet = Tweet(id=tweet.id,
text=tweet.full_text[:500],
embedding=embedding)
db_user.tweets.append(db_tweet)
DB.session.add(db_tweet)
except Exception as e:
print("Error processing {}: {}".format(username, e))
raise e
else:
DB.session.commit()
def generate_movie():
"""All movies have a plot and a sideplot;
all movies have two main names, a setting name, an incongruous object, and a bad_music_choice
all movies have two bad sex scenes
depending on length, some movies have another bad sex scene or a line of dialog
"""
main1, main2 = random.sample(main_names, 2)
setting = random.choice(setting_names)
incongruous_object = random.choice(incongruous_objects)
bad_music_choice = random.choice(bad_music)
all_modifiers = {
'main1': main1,
'main2': main2,
'setting': setting,
'incongruous_object': incongruous_object,
'bad_music_choice': bad_music_choice,
}
plot_choice = random.choice(all_plots)
sideplot_choice = random.choice(all_sideplots)
sex_scenes = random.sample(bad_sex_scenes, 3)
extra_sex_scene = sex_scenes.pop()
line_of_dialog = random.choice(dialog)
if (len(plot_choice) + len(sideplot_choice)) < 10:
sex_scenes.append(random.choice([extra_sex_scene, line_of_dialog]))
raw_tweets = interleave(plot_choice, sideplot_choice, sex_scenes)
text_tweets = [tweet.format(**all_modifiers) for tweet in raw_tweets]
timestamps = generate_timestamps(len(text_tweets))
timed_text_tweets = [
" ".join([time, text])
for time, text in zip(timestamps, text_tweets)
]
plot_point_tweets = [Tweet(text) for text in timed_text_tweets]
title = random.choice(titles)
description = random.choice(descriptions)
intro_word = random.choice(['Introducing', 'Presenting'])
intro_tweet_text = "{intro_word} {title}, {description}".format(
intro_word=intro_word, title=title, description=description)
tweets = [Tweet(intro_tweet_text)]
tweets.extend(plot_point_tweets)
return Movie(tweets)
def update_tweet():
new_tweet = Tweet()
new_tweet.id = request.form['id']
new_tweet.text = request.form['text']
db.session.query(Tweet).filter_by(id=new_tweet.id).update(
{"text": new_tweet.text})
db.session.commit()
return tweet_schema.jsonify(new_tweet)
def add(request):
user = current_user(request)
# 创建微博
form = request.form()
w = Tweet(form)
w.user_id = user.id
w.save()
return redirect('/tweet/index?user_id={}'.format(user.id))
def create(self, status):
tweet = Tweet()
create = tweet.create(status)
if create is True:
# Count inserted tweet and store to variable
self.count += 1
def getTweet(self, id):
tweet = ''
try:
tweet =self.api.get_status(id,tweet_mode='extended')
tweet = Tweet(tweet,id)
except:
printError()
return tweet
def create_tweet():
#import ipdb; ipdb.set_trace()
tweet = Tweet()
tweet.text = request.json['text']
tweet.created_at = datetime.now().time()
db.session.add(tweet)
db.session.commit()
return tweet_schema.jsonify(tweet), 201
def create_tweet():
tweet = Tweet()
user = request.json.get('username')
userdb = db.session.query(User).filter(User.username == user).first()
tweet.created_by = userdb.id
tweet.text = request.json.get('text')
db.session.add(tweet)
db.session.commit()
return 'Created', 201
def user_timeline(self, user_id, count=200, **kwargs):
timeline = self.get_d("statuses/user_timeline.json",
user_id=user_id,
trim_user=1,
include_rts=1,
include_entities=1,
count=count,
**kwargs)
return [Tweet(t) for t in timeline]
def update_tweets(category, search_words, exclude_keyword, only_included_ids):
search_params = settings.DEFAULT_SEARCH_PARAMS.copy()
search_params['q'] = search_words + ' exclude:retweets'
requests = twitter.get(SEARCH_API_URL, params=search_params)
if requests.status_code != 200:
print('!!!!!ERROR!!!!! - %d' % requests.status_code)
return
for params in json.loads(requests.text)['statuses']:
user = User.query.filter(User.id == params['user']['id']).first()
user_exists = user is not None
if exclude_keyword:
screen_name = params['user']['screen_name'].lower()
text = params['text'].lower()
if re.search(r'@[^ ]*' + exclude_keyword, text) or re.search(exclude_keyword, screen_name):
if not re.search(r' [^@ ]*' + exclude_keyword, ' ' + text):
continue
if only_included_ids and not user_exists:
continue
tweet_exists = db.session.query(db.exists().where(Tweet.id == params['id'])).scalar()
if 'retweeted_status' in params or params['user']['id'] in BAN_USERS or tweet_exists:
continue
description = ''
if 'description' in params['user']:
description = params['user']['description']
if not user_exists:
new_user = User(
id=params['user']['id'],
name=params['user']['name'],
screen_name=params['user']['screen_name'],
description=description,
profile_image_url_https=params['user']['profile_image_url_https'],
tweets_count=0
)
db.session.add(new_user)
tweet = Tweet(
id=params['id'],
category=category,
user_id=params['user']['id'],
user_name=params['user']['name'],
user_screen_name=params['user']['screen_name'],
user_description=description,
user_profile_image_url_https=params['user']['profile_image_url_https'],
ban=False,
text=params['text'],
created_at=params['created_at'],
)
db.session.add(tweet)
def post_tweet():
form = TweetForm()
if form.validate():
tweet = Tweet(user_id=current_user.id, text=form.text.data, date_created=datetime.now())
db.session.add(tweet)
db.session.commit()
return redirect(url_for('timeline'))
return 'Something went wrong.'
def parse_dict(tweet_dict, event_id=None):
t = tweet_dict
u = t['user']
tweet = Tweet(
tweet_id=t['id'],
text=t['text'],
created_at=format_date(t['created_at']),
source=t['source'],
source_url=t['source'],
entities=json.dumps(t['entities']), # <- this is a dict
lang=t['lang'],
truncated=t['truncated'],
possibly_sensitive=t.get('possibly_sensitive', False),
coordinates=json.dumps(t['coordinates']), # <- dict?
in_reply_to_status_id=t['in_reply_to_status_id'],
in_reply_to_screen_name=t['in_reply_to_screen_name'],
in_reply_to_user_id=t['in_reply_to_user_id'],
favorite_count=t['favorite_count'],
retweet_count=t['retweet_count'],
is_headline=False,
quoted_status_id=t.get('quoted_status_id', None),
is_a_retweet=t.get,
retweeted_status_id=t.get('retweeted_status_id', None),
user_id=t['user']['id'])
if event_id:
event_tweet = EventTweet(tweet_id=t['id'], event_id=event_id)
else:
event_tweet = None
if u.get('entities'):
entities = json.dumps(u['entities'])
else:
entities = None
user = User(user_id=u['id'],
verified=u['verified'],
name=u['name'],
screen_name=u['screen_name'],
created_at=format_date(u['created_at']),
description=u['description'],
location=u['location'],
geo_enabled=u['geo_enabled'],
entities=entities,
lang=u['lang'],
url=u['url'],
followers_count=u['followers_count'],
favourites_count=u['favourites_count'],
listed_count=u['listed_count'],
friends_count=u['friends_count'],
statuses_count=u['statuses_count'],
utc_offset=u['utc_offset'],
time_zone=u['time_zone'])
return tweet, user, event_tweet
def post(self, request , *args , **Kwargs):
# print "in post method"
slack_request_msg = request.data
# checking if request coming from correct workspace
if slack_request_msg.get('token') != VeriFication_Token :
return Response(status = status.HTTP_403_FORBIDDEN)
#verification challenge
if slack_request_msg.get('type') == 'url_verification' :
return Response(data=slack_request_msg , status = status.HTTP_200_OK)
# event listening
if 'event' in slack_request_msg:
msg_event = slack_request_msg.get('event')
# check if msg coming from slack api
if msg_event.get('subtype') == 'bot_message':
return Response(status=status.HTTP_200_OK)
# process user's message
user = msg_event.get('user')
text = msg_event.get('text')
channel = msg_event.get('channel')
# respond with hello msg
bot_text = 'Hi <@{}> :wave:'.format(user)
if 'go' in text.lower():
slack_client.api_call(method='chat.postMessage',
channel=channel,
text=bot_text)
# tweet_obj = Tweet(tweet_id=1, tweet_text="Hi Hesham", tweet_time=datetime.datetime.now())
# tweet_obj.save()
# x = Tweet.objects()
# for t in x :
# print t.tweet_id , t.tweet_text
# print "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
# to get all tweets in the timeline
All_Tweets = tweepy.Cursor(api.home_timeline).items()
for iter in All_Tweets :
newTweet = Tweet.objects(tweet_id=iter.id)
# check if this tweet dosen't exist before
if not newTweet :
tweet_obj = Tweet(tweet_id = iter.id,tweet_text= iter.text,tweet_time = iter.created_at)
tweet_obj.save()
else :
break
return Response(status=status.HTTP_200_OK)
return Response(status=status.HTTP_200_OK)
def on_success(self, data):
if 'text' in data:
data['fetched_timestamp'] = datetime.datetime.now()
data['fresh_tweet'] = True
tweet = Tweet()
tweet.tweet_id = data['id_str']
tweet.tweets.append(data)
tweet.save()
print("saved", self.i)
self.i += 1
def handle_tweet():
message = request.form["tweet"]
user = User.query.filter(User.username == session["username"]).first()
tweet = Tweet(user=user, text=message)
db.session.add(tweet)
db.session.commit()
return redirect("/")
def on_data(self, data):
self.count += 1
# table.insert({"text": json.loads(data)["text"]})
print data
Tweet(data=json.loads(data)).save()
print "#############################"
if self.count == self.total:
return False
else:
return True
