def main():
t = TwitterAPI()
query_1 = sys.argv[1]
query_2 = sys.argv[2]
tweet_limit = sys.argv[3]
search_results_1 = t.search_tweets(query_1, tweet_limit)
search_results_2 = t.search_tweets(query_2, tweet_limit)
print improved_similarity(search_results_1, search_results_2)
def main():
t = TwitterAPI()
query_1 = sys.argv[1]
query_2 = sys.argv[2]
tweet_limit = sys.argv[3]
search_results_1 = t.search_tweets(query_1, tweet_limit)
search_results_2 = t.search_tweets(query_2, tweet_limit)
print improved_similarity(search_results_1, search_results_2)
def followers(self, celebrity):
match = self._get_match(celebrity)
if match is None:
raise ValueError(celebrity)
(celebrity_name, celebrity_handle) = match
tapi = TwitterAPI()
follower_count = tapi.followers(celebrity_handle)
return "%s has %d Twitter followers." % (celebrity_name.title(),
follower_count)
def main():
t = TwitterAPI()
query_1 = sys.argv[1]
query_2 = sys.argv[2]
tweet_limit = sys.argv[3]
search_results_1 = t.search_tweets(query_1, tweet_limit)
search_results_2 = t.search_tweets(query_2, tweet_limit)
query_token_set_1 = set(tokenize(' '.join(search_results_1)))
query_token_set_2 = set(tokenize(' '.join(search_results_2)))
print jaccard_similarity(query_token_set_1, query_token_set_2)
def main():
t = TwitterAPI()
query_1 = sys.argv[1]
query_2 = sys.argv[2]
tweet_limit = sys.argv[3]
search_results_1 = t.search_tweets(query_1, tweet_limit)
search_results_2 = t.search_tweets(query_2, tweet_limit)
query_token_set_1 = set(tokenize(' '.join(search_results_1)))
query_token_set_2 = set(tokenize(' '.join(search_results_2)))
print jaccard_similarity(query_token_set_1, query_token_set_2)
def process(request):
user_identifier = request.POST['user_identifier']
user_identifier_type = request.POST['user_identifier_type']
twitter_api = TwitterAPI()
try:
response = redirect('result')
if user_identifier_type == "user_id":
user = Users.objects.get(user_id=user_identifier)
response.set_cookie('app_user_id', user_identifier)
else:
user = Users.objects.get(user_name=user_identifier)
response.set_cookie('app_user_name', user_identifier)
try:
api = twitter_api.get_api_object(user.user_access_token,
user.user_access_secret)
response.set_cookie('app_access_token', user.user_access_token)
response.set_cookie('app_access_secret', user.user_access_secret)
except tweepy.TweepError:
user.delete()
print "Authentication failed. Redirecting user to relogin"
redirect_url = twitter_api.auth_handler.get_authorization_url()
request_token = twitter_api.auth_handler.request_token
response = redirect(redirect_url)
response.set_cookie('request_token', request_token)
if user_identifier_type == "user_id":
response.set_cookie('app_user_id', user_identifier)
else:
response.set_cookie('app_user_name', user_identifier)
return response
except Users.DoesNotExist:
print "User not found in DB. Will redirect user to fetch verifier token"
response = redirect(twitter_api.auth_handler.get_authorization_url())
if user_identifier_type == "user_id":
response.set_cookie('app_user_id', user_identifier)
else:
response.set_cookie('app_user_name', user_identifier)
request_token = twitter_api.auth_handler.request_token
response.set_cookie('request_token', request_token)
return response
def process(request):
user_identifier = request.POST['user_identifier']
user_identifier_type = request.POST['user_identifier_type']
twitter_api = TwitterAPI()
try:
response = redirect('result')
if user_identifier_type == "user_id":
user = Users.objects.get(user_id=user_identifier)
response.set_cookie('app_user_id', user_identifier)
else:
user = Users.objects.get(user_name=user_identifier)
response.set_cookie('app_user_name', user_identifier)
try:
api = twitter_api.get_api_object(user.user_access_token, user.user_access_secret)
response.set_cookie('app_access_token', user.user_access_token)
response.set_cookie('app_access_secret', user.user_access_secret)
except tweepy.TweepError:
user.delete()
print "Authentication failed. Redirecting user to relogin"
redirect_url = twitter_api.auth_handler.get_authorization_url()
request_token = twitter_api.auth_handler.request_token
response = redirect(redirect_url)
response.set_cookie('request_token', request_token)
if user_identifier_type == "user_id":
response.set_cookie('app_user_id', user_identifier)
else:
response.set_cookie('app_user_name', user_identifier)
return response
except Users.DoesNotExist:
print "User not found in DB. Will redirect user to fetch verifier token"
response = redirect(twitter_api.auth_handler.get_authorization_url())
if user_identifier_type == "user_id":
response.set_cookie('app_user_id', user_identifier)
else:
response.set_cookie('app_user_name', user_identifier)
request_token = twitter_api.auth_handler.request_token
response.set_cookie('request_token', request_token)
return response
def list(self):
if (time.time() - self.last_request) >= CACHE_TIME:
t = TwitterAPI()
active = t.get_active_tweets("#riverguard", "36.528580",
"-6.213026", RADIUS)["statuses"]
passive = t.get_passive_tweets("36.528580", "-6.213026",
RADIUS)["statuses"]
passive = t.extract_image_tweets(passive)[:15]
v = VisionAPI()
passive = v.tag_images(passive)
relevant_tags = ["waste", "algae", "fish", "water", "litter"]
passive = self.filter_non_relevant_tweets(passive, relevant_tags)
tweets = active + passive
self.log.info('getting tweets from twitter')
for tweet in tweets:
self.repository.create(tweet)
self.last_request = time.time()
return self.repository.list()
def result(request):
twitter_api = TwitterAPI()
if ('app_access_token'
in request.COOKIES.keys()) or ('app_access_secret'
in request.COOKIES.keys()):
api = twitter_api.get_api_object(
request.COOKIES.get('app_access_token'),
request.COOKIES.get('app_access_secret'))
user_id = api.me().id_str
else:
api = twitter_api.get_api_object_from_verifier(
request.GET['oauth_verifier'],
request.COOKIES.get('request_token'))
user_access_token = twitter_api.auth_handler.access_token
user_access_secret = twitter_api.auth_handler.access_token_secret
user_id = api.me().id_str
user_name = api.me().screen_name
new_user_entry = Users(user_name=user_name,
user_id=str(user_id),
user_access_secret=user_access_secret,
user_access_token=user_access_token)
new_user_entry.save()
calculated_value = PostTimeCalculator.calculate(user_id, api)
best_time_to_post = calculated_value['best_time_to_post']
best_day_to_post = calculated_value['best_day_to_post']
return render(
request, 'TimeToPost/result.html', {
'best_time_to_post': best_time_to_post,
'best_day_to_post': best_day_to_post
})
def result(request):
twitter_api = TwitterAPI()
if ('app_access_token' in request.COOKIES.keys()) or ('app_access_secret' in request.COOKIES.keys()):
api = twitter_api.get_api_object(request.COOKIES.get('app_access_token'), request.COOKIES.get('app_access_secret'))
user_id = api.me().id_str
else:
api = twitter_api.get_api_object_from_verifier(request.GET['oauth_verifier'], request.COOKIES.get('request_token'))
user_access_token = twitter_api.auth_handler.access_token
user_access_secret = twitter_api.auth_handler.access_token_secret
user_id = api.me().id_str
user_name = api.me().screen_name
new_user_entry = Users(user_name=user_name, user_id=str(user_id), user_access_secret=user_access_secret, user_access_token=user_access_token)
new_user_entry.save()
calculated_value = PostTimeCalculator.calculate(user_id, api)
best_time_to_post = calculated_value['best_time_to_post']
best_day_to_post = calculated_value['best_day_to_post']
return render(request, 'TimeToPost/result.html', {'best_time_to_post': best_time_to_post, 'best_day_to_post': best_day_to_post})
def init_twitter_api(
self
): # this will throw StopIteration if all proxies have been tried...
if self.proxies:
try:
self.client_args['proxies'] = next(
self.proxies)['proxy_dict'] # this will throw out
# logger.info("client_args: %s"%json.dumps(self.client_args))
except StopIteration as exc:
raise
except Exception as exc:
self.init_twitter_api()
if self.twitter_api:
del self.twitter_api
# crawler_id=self.crawler_id,
self.twitter_api = TwitterAPI(apikeys=self.apikeys,
client_args=self.client_args)
def eval_emotion(twitter_name):
print("Start evaluating")
waston_emotion = WastonEmotion(cfg.waston_name, cfg.waston_password)
twitter_api = TwitterAPI(cfg.twitter_key, cfg.twitter_secret)
twitter_api.fetch_bearer_key()
data = twitter_api.get_timelines_by_screen_name(twitter_name)
print("Data fetched")
ret = "<!DOCTYPE html><html><head><title>Show Items</title><script type='text/javascript' src='../static/jquery.js'></script> \
<script>"
username, profile_image_url = twitter_api.get_user_name_and_image(twitter_name)
ret += 'profile_image_url="{}";'.format(profile_image_url)
ret += 'username="******";'.format(username)
ret += 'background_file_name="../static/background02.png";'
ret += 'fear="#D0C9C1";'
ret += 'sadness="#C4CCC7";'
ret += 'joy="#F4DFC1";'
ret += 'anger="#D5BBAA";'
ret += 'disgust="#988A81";\n'
ret += 'color_list=['
color_list = []
emotion_list = []
for item in data:
res = waston_emotion.getEmotion(item['text'])
if 'emotion' in res:
emotion_list.append(get_max_score_key(res['emotion']['document']['emotion']))
print(item['text'])
emotion_count = len(emotion_list)
for idx, item in enumerate(emotion_list):
ret += "[{},{}],".format(idx/(emotion_count - 1), item)
ret += "];\n"
ret += "</script><script type='text/javascript' src='../static/main.js'></script>"
ret += "</head><body><canvas id='canvas1' height='640', width='932'></canvas><img src='../static/background02.png' style='display:none' id='bg-img'><img src='"+ profile_image_url+ "' style='display:none' id='profile-img'></body></html>"
return ret
class DialogueAgent:
def __init__(self, is_terminal, db):
self.__is_terminal = is_terminal
self.__utils = Utils()
self.__twiapi = TwitterAPI(db)
self.__db = db
def weather_text(self, day=0):
# jsonの取得
weather = self.__utils.get_wether(day)
# データの整理
date = weather['date'][5:]
rainfall = weather['rainfallchance']['period']
for temp in weather['temperature']['range']:
if temp['centigrade'] == 'max':
temp_max = temp['content'] # 11
if temp['centigrade'] == 'min':
temp_min = temp['content'] # 5
telop = weather['weather']
# テキストの生成
content = '{}は{}, 最低{}℃, 最高{}℃\n'.format(
date, telop, temp_min, temp_max)
content += '降水確率は, {} です！'.format(', '.join(
'{}:{}%'.format(x['hour'], x['content']) for x in rainfall))
return content
def lang8_reminder_text(self):
text = ''
if not self.__utils.is_today_lang8():
text = '今日のlang8がまだ投稿されてないです！'
return text
def lang8_check_text(self):
members = self.__utils.lastday_lang8()
if len(members) == 0:
text = '昨日は全員がlang8に投稿しました。'
elif len(members) == 5:
text = '昨日は全員がlang8に投稿し忘れました！'
else:
text = '昨日は{}がlang8に投稿し忘れました！'.format('と'.join(members))
return text
def generate_reply(self, text, method='markov'):
if text == 'お題' or text == 'おだい':
return self.__utils.get_odai()
if method == 'echo':
return text
if method == 'markov':
# 名詞をseedにする
seeds = list()
for tok in self.__utils.morph_parse(text):
if '名詞' in tok.part_of_speech:
seeds.append(tok.surface)
if len(seeds) == 0:
seeds.append(text)
# seedをqueryにしてツイッター検索をかける
sentences = list()
query = ' OR '.join(seeds)
count = len(seeds) * 20
for tweet in self.__twiapi.search(query, count=count)["statuses"]:
sentences.append(self.__utils.clean_tweet(tweet['text']))
# マルコフ連鎖で文を生成
gen_sentence = self.__utils.markov_generate(sentences)
return gen_sentence
def speech(self, text, tweet_id=None, screen_name=None, avoid_dupl='。'):
if self.__is_terminal:
print(text)
else:
if tweet_id is not None and screen_name is not None:
self.__twiapi.reply(text, tweet_id, screen_name)
else:
while self.__db.id_duplicate(text):
text += avoid_dupl
self.__twiapi.tweet(text)
self.__db.add_recent_tweet(text)
def get_message(self):
if self.__is_terminal:
for line in iter(sys.stdin.readline, '\n'):
yield (line.strip(), None, None)
else:
# リプを取得
for mention in self.__twiapi.get_mentions():
text = mention['text']
tweet_id = mention['id_str']
screen_name = mention['user']['screen_name']
# ユーザ情報(プロフィール)
name = mention['user']['name']
zone = mention['user']['time_zone']
location = mention['user']['location']
description = mention['user']['description']
yield (' '.join(text.split()[1:]), tweet_id, screen_name)
import time
import redis
import sentry_sdk
from dateutil import parser
from tracker_api import TrackerAPI
from twitter_api import TwitterAPI
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
db = redis.from_url(os.environ.get('REDIS_URL'))
sentry_sdk.init(os.environ.get('SENTRY_URL'))
tracker_api = TrackerAPI()
twitter_api = TwitterAPI()
def load_data():
data = tracker_api.get_data_by_country()
global_data = {
'confirmed': f"{data['latest']['confirmed']:,}",
'deaths': f"{data['latest']['deaths']:,}",
'last_updated': data['locations'][0]['last_updated']
}
db.hmset('global_latest', global_data)
def compose_tweet(confirmed: int, deaths: int, handle: str,
last_updated: datetime.datetime):
time_delta = (
def __init__(self, is_terminal, db):
self.__is_terminal = is_terminal
self.__utils = Utils()
self.__twiapi = TwitterAPI(db)
self.__db = db
queries = loop.run_until_complete(api.get_tweets(queries))
loop.close()
tweets = [tweet for query in queries for tweet in query]
return tweets
if __name__ == "__main__":
# Get credentials
consumer_key = os.getenv('TWITTER_KEY')
consumer_secret = os.getenv('TWITTER_SECRET_KEY')
access_token = os.getenv('TWITTER_ACCESS_TOKEN')
access_token_secret = os.getenv('TWITTER_ACCESS_TOKEN_SECRET')
# Get tweets
api = TwitterAPI(consumer_key, consumer_secret, access_token,
access_token_secret)
queries = ['pizza', 'virus', 'corona']
tweets = get_tweets_from_api(api, queries)
# Export data
importer_exporter = ImporterExporter()
# Local
if os.getenv('ENV') == 'dev':
filename = os.getenv('LOCAL_FILENAME')
header = api.get_tweet_contents()
importer_exporter.local_export(filename, header=header, data=tweets)
# S3
else:
filename = os.getenv('S3_FILENAME')
import tweepy
from database import Database
from flask import Flask, jsonify, request
from flask_cors import CORS
from models import Tweet
from tweepy import OAuthHandler
from twitter_api import TwitterAPI
BAD = 1
GOOD = 2
IGNORE = 3
app = Flask(__name__)
database = Database(app)
source_twitter = TwitterAPI(strings.SOURCE, database)
destination_twitter = TwitterAPI(strings.DESTINATION, database)
CORS(app)
@app.route('/tweets', methods=['GET'])
def get_tweets():
"""
Call the twitter api to get up to 200 tweets from
source timeline.
Return:
json: the tweet list in json format.
"""
def __init__(self, path, log):
self.last_request = time.time() - CACHE_TIME
self.log = log
self.twitterAPI = TwitterAPI()
self.repository = Persistence(path, logging.getLogger(__name__))
self.repository.init_db()
import strings
from flask import Flask
from database import Database
from models import Tweet
from twitter_api import TwitterAPI
from datetime import datetime
print("- INITIALIZING SCRIPT TO FETCH TWEETS ")
print("HORA CERTA ROGERINHO: %s" % (str(datetime.now())))
app = Flask(__name__)
print("- INSTANTIATING DATABASE ")
database = Database(app)
print("- INSTANTIATING TWITTER API")
source_twitter = TwitterAPI(strings.SOURCE, database)
print("- GETTING TWEET LIST")
tweet_list = source_twitter.get_timeline_tweets()
print("- SAVE TWEETS TO THE DATABASE")
if (tweet_list):
print("- %d TWEETS FETCHED" % (len(tweet_list)))
for tweet in tweet_list:
tweet_id = str(tweet['id'])
text = tweet['text']
origin = tweet['origin']
sentiment = 0
tweet = Tweet(tweet_id, text, sentiment, origin)
database.post_tweet(tweet)
print(" - TWEET %s SAVED TO DATABASE WITH SUCCESS" % (tweet_id))
DEBUG = False
if not valid_arguments(arguments,allowed_switches):
# Exit
print_DEBUG('Exit with errors due invalid arguments')
print "ERROR: Exit"
sys.exit(2)
# Is '--help' present? Then print Help and exit(0)
if '--help' in arguments:
print_DEBUG('Help called')
print_help()
exit(0)
twitter = TwitterAPI()
# Collections
# Mongo DB Name 'twitter_bot'
connection = mongo_connect('twitter_bot')
# Mongo Collection name = 'track' to store a couple of metrics
# Is going to be helpful to detect whether or not we already run today
collection_track = connection['track']
# Mongo Collection name = 'followers' for our current followers
collection_followers = connection['followers']
# Mongo Collection name = 'followers_archive' to keep track of former followers
from VisionAPI import VisionAPI
from twitter_api import TwitterAPI
from flask import jsonify
t = TwitterAPI()
active = t.get_active_tweets("#riverguard", "36.528580", "-6.213026",
"5")["statuses"]
print len(active)
passive = t.get_passive_tweets("36.528580", "-6.213026", "5")["statuses"]
print len(passive)
passive = t.extract_image_tweets(passive)[:15]
print len(passive)
v = VisionAPI()
passive = v.tag_images(passive)
print len(passive)
tweets = active + passive
result = jsonify(tweets)
print result
