def capture_relations(day=1, month=1, year=2018):
'''
Performs a capture of retweet relation between all Actors from the date specified
Returning a CSV
'''
header_json = json.load(open("helpers/relations_attributes.json"))
ids = []
actors = Actor.query.all()
csv = CsvBuilder(header_json)
for actor in actors:
ids.append(actor.id)
for actor in actors:
print("Capturing RT-relations of", actor.name)
user = TwitterUser(actor.id)
id_subset = [x for x in ids if x != actor.id]
if user.existence == True:
tweets = user.retrieve_tweets_from(day=day,
month=month,
year=year,
raw=True)
mentions_ids = extract_retweeted_author_id(tweets)
for mention in mentions_ids:
if mention[0] in id_subset:
retweeted = Actor.query.filter_by(
id=str(mention[0])).first().username
csv.add_row(actor.username + ";" + retweeted + ";" +
str(mention[1]) + "\n")
return csv
def add_actor():
if request.method == 'POST':
try:
username = request.form['username']
user = TwitterUser(username)
if user.existence == True:
name = user.name
name = unidecode(name)
f = Actor(id=int(user.id), username=username, name=name)
db.session.add(f)
db.session.commit()
if not scheduler.get_job(job_id=user.id):
scheduler.add_job(tweets_job,
'interval',
minutes=10080,
replace_existing=False,
id=user.id,
args=[user.id])
print("Ator", username, "adicionado")
else:
print("Usuario", username, "não existe!")
except:
pass
return redirect("/")
def test_api_get_actor_account_date_zero_reports(self):
user = TwitterUser('Renova_BR')
user = Actor(user.id, user.username, user.name)
with app.app_context():
TestAPIRoutes.setUp()
db.session.add(user)
db.session.commit()
response = api_get_actor_account_date('Renova_BR', '01-01-2018')
TestAPIRoutes.tearDown()
assert '500' == json.loads(response.get_data().decode())['code']
def check_actors_usernames():
print("Checking if actors usernames remain the same...")
actors = Actor.query.all()
for actor in actors:
user = TwitterUser(actor.id)
if user.existence == True:
if user.username != actor.username:
print(actor.username, "changed to:", user.username)
Actor.query.filter_by(id=actor.id).update(
dict(username=user.username))
db.session.commit()
def capture_tweets(id, day=1, month=1, year=2018):
'''
Performs a capture of all Tweets from the Actor specified by id from the date specified
Returning a CSV
'''
header_json = json.load(open("helpers/tweets_attributes.json"))
csv = CsvBuilder(header_json)
user = TwitterUser(id)
if user.existence == True:
print("Capturing " + user.name + " tweets")
tweets = user.retrieve_tweets_from(day, month, year)
for tweet in tweets:
row = list_to_row(tweet)
csv.add_row(row)
else:
csv = "none"
return csv
def test_api_get_actors(self):
user = TwitterUser('CNN')
user = Actor(user.id, user.username, user.name)
with app.app_context():
TestAPIRoutes.setUp()
db.session.add(user)
db.session.commit()
response = api_get_actors()
TestAPIRoutes.tearDown()
assert '{"actors":["CNN"],"code":"200","length":1,"message":"Success"}\n' == response.get_data(
).decode()
def test_api_get_actor_account_date(self):
csv_content = 'nome;seguidores;tweets;seguindo;curtidas;retweets;favorites;hashtags;mentions\n923257662569148417;1715;146;193;104;0;0;;;\n'
a = ActorReport('01-01-2018', '12:00', csv_content.encode())
user = TwitterUser('Renova_BR')
user = Actor(user.id, user.username, user.name)
with app.app_context():
TestAPIRoutes.setUp()
db.session.add(a)
db.session.add(user)
db.session.commit()
response = api_get_actor_account_date('Renova_BR', '01-01-2018')
TestAPIRoutes.tearDown()
assert '{"12:00":{"date":"01-01-2018","followers_count":"193","following_count":"104","likes_count":"0","name":"1715","tweets_count":"0","username":"******"},"code":"200","message":["Success"]}\n' == response.get_data(
).decode()
def test_api_get_actor_account_date_tweets(self):
csv_content = 'Data;Texto;Autor(RT);Retweets;Curtidas;Hashtags;Mentions\n2018-06-08 14:25:30;@MarcioQuessada Pressionar e discutir. Essa mudanca dificilmente vai vir deles!;;0;0; ;MarcioQuessada;'
user = TwitterUser('Renova_BR')
user = Actor(user.id, user.username, user.name)
t = TweetReport('12-06-2018', '19:00', user.id, csv_content.encode())
with app.app_context():
TestAPIRoutes.setUp()
db.session.add(t)
db.session.add(user)
db.session.commit()
response = api_get_actor_account_date_tweets(
'Renova_BR', '12-06-2018')
TestAPIRoutes.tearDown()
assert '{"code":"200","message":"Success","tweets":[{"date":"2018-06-08 14:25:30","hashtags":" ","likes":"0","mention":"MarcioQuessada","retweets":"0","rt_author":"","text":"@MarcioQuessada Pressionar e discutir. Essa mudanca dificilmente vai vir deles!"}]}\n' == response.get_data(
).decode()
def capture_actors():
'''
Performs a capture of quantitative information of all Actors returning a CSV
'''
actors = Actor.query.all()
header_json = json.load(open("helpers/actors_attributes.json"))
csv = CsvBuilder(header_json)
print("Capturing Actors information")
for actor in actors:
user = TwitterUser(actor.id)
if user.existence == True:
print("Retrieving information from ", user.username)
row = list_to_row([
actor.id, user.name, user.username, user.followers_count,
user.following_count, user.likes_count, user.tweets_count
])
csv.add_row(row)
return csv
def capture_relations_timeline(day=1, month=1, year=2018):
'''
Performs a capture of retweet relation between all Actors from the date specified and separeted by week
Returning a CSV
'''
header_json = json.load(open("helpers/relations_attributes.json"))
header_json = header_json[:2]
ids = []
actors = Actor.query.all()
date = datetime(day=day, month=month, year=year)
date_list = []
while date <= datetime.now():
date_list.append(date)
header_json.append({'attribute': str(date).split(" ")[0]})
date = date + timedelta(weeks=1)
date_list.append(datetime.now())
csv = CsvBuilder(header_json)
for actor in actors:
ids.append(actor.id)
relations = {}
for actor in actors:
print("Capturing RT-relations of", actor.name, "week by week")
user = TwitterUser(actor.id)
id_subset = [x for x in ids if x != actor.id]
if user.existence == True:
tweets = user.retrieve_tweets_from(day=day,
month=month,
year=year,
raw=True)
mentions_ids = extract_retweeted_author_id(tweets)
for mention in mentions_ids:
if mention[0] in id_subset:
retweeted = Actor.query.filter_by(
id=str(mention[0])).first().username
relations[actor.username + ";" + retweeted] = {}
by_week = split_tweets(tweets, date_list)
for week in by_week.keys():
week_mentions = extract_retweeted_author_id(by_week[week])
for mention in week_mentions:
if mention[0] in id_subset:
retweeted = Actor.query.filter_by(
id=str(mention[0])).first().username
relations[actor.username + ";" +
retweeted][week] = mention[1]
for relation in relations.keys():
row = relation
for i in range(0, len(date_list) - 1):
if str(date_list[i].date()) in relations[relation].keys():
row = row + ";" + str(relations[relation][str(
date_list[i].date())])
else:
row = row + ";0"
row = row + ";\n"
csv.add_row(row)
return csv
from modules.twitter_user import TwitterUser
import json
from app.models import Actor
from app import db
from unidecode import unidecode
# Adding actors from helpers/politicians.json
print("Adding Actors")
actors = json.load(open("helpers/politicians.json"))
for row in actors:
username = row["twitter_handle"]
user = TwitterUser(username)
if user.existence == True:
name = user.name
name = unidecode(name)
if not Actor.query.filter_by(id=user.id).first():
f = Actor(id = int(user.id), username=username, name= name)
db.session.add(f)
db.session.commit()
print(name, "added")
def test_name_retrieval(self):
user = TwitterUser('siqueiralex')
assert 'Alex Siqueira' == user.name
assert '52126452' == user.id
def test_account_existance(self):
user = TwitterUser('')
assert user.existence == False
def api_get_actor_account_date(username, date):
## If not specified date, API will return current values from Tweepy API.
all_actors_collection_dates = []
actorreports = ActorReport.query.all()
for actorreport in actorreports:
all_actors_collection_dates.append(actorreport.date)
all_actors_collection_dates = list(set(all_actors_collection_dates))
actor = Actor.query.filter_by(username=username).first()
if not actor:
data = {
'code': '400',
'message': 'Bad Request',
'details': 'Invalid username.'
}
return jsonify(data)
tweets_collection_dates = []
tweetreports = TweetReport.query.filter_by(actor_id=actor.id)
for tweetreport in tweetreports:
tweets_collection_dates.append(tweetreport.date)
tweets_collection_dates = list(set(tweets_collection_dates))
if date == None:
user = TwitterUser(actor.id)
if user.existence == False:
data = {
'code': '400',
'message': 'Bad Request',
'details': 'Invalid username.'
}
return jsonify(data)
else:
if user.username != actor.username:
Actor.query.filter_by(id=actor.id).update(
dict(username=user.username))
db.session.commit()
data = {
'code': '200',
'message': 'Success',
'username': user.username,
'name': user.name,
'followers_count': user.followers_count,
'tweets_count': user.tweets_count,
'following_count': user.following_count,
'likes_count': user.likes_count,
'tweets_collection_dates': tweets_collection_dates,
'actor_collection_dates': all_actors_collection_dates
}
return jsonify(data)
else:
# What to do if there's multiple records for the same date?
reports = ActorReport.query.filter_by(date=date)
if reports.count() == 0:
data = {
'code': '500',
'message': 'Internal Server Error',
'details': 'Sorry, no data for specific date.'
}
return jsonify(data)
data = {}
for report in reports:
lines = report.csv_content.decode().split('\n')
for line in lines:
aux = line.split(';')
try:
if len(aux) > 5 and actor.id == aux[0]:
hour_data = {
'date': date,
'username': aux[2],
'name': aux[1],
'followers_count': aux[3],
'tweets_count': aux[6],
'following_count': aux[4],
'likes_count': aux[5]
}
data[report.hour] = hour_data
except:
pass
if data:
data['code'] = '200'
data['message'] = 'Success',
return jsonify(data)
data = {
'code': '400',
'message': 'Bad Request',
'details': 'Sorry. No data for specific date.'
}
return jsonify(data)