class TwitterProducer:
logging.basicConfig(filename='tweets.py', level=logging.NOTSET)
def run(self):
self.stream_mentions()
# def _producer(self):
# pass
def __init__(self):
self.api = Api(CONSUMER, CONSUMER_SECRET, ACCESS_TOKEN,
ACCESS_TOKEN_SECRET)
self._producer = KafkaProducer(bootstrap_servers='localhost:9092',
max_in_flight_requests_per_connection=5,
acks='all',
api_version=(0, 10),
retries=100000000000)
def stream_mentions(self):
with open('output.txt', 'a') as f:
for msg in self.api.GetStreamFilter(track=['@tolumide_ng'],
languages=['en']):
f.write(json.dumps(msg))
logging.info(msg)
self._producer.send('twitter_mentions', None, msg)
f.write('\n')
def stream_timeline(self, user):
with open('timeline.txt', 'a') as f:
statuses = self.api.GetUserTimeline(user_id=user)
# print([s.text for s in statuses])
for s in statuses:
# f.write(json.dumps(s.text))
val = bytes(s.text, encoding='utf-8')
print(val)
self._producer.send('twitter_timelines', val)
f.write('\n')
f.close()
def get_followers(self, user):
users = self.api.GetFriends(user)
logging.info([u.name for u in users])
def get_friends(api: twitter.Api, user: str):
"""Get following users
Args:
api: twitter api
user: user to check the following of
"""
new_following = set(
map(
lambda user: twitter.User(id=user.id, screen_name=user.screen_name
), api.GetFriends(screen_name=user)))
if len(new_following) > 0:
logger.info("Inserting new following of %s", user)
new_following_values = tuple(
map(lambda e: (user, e.id, e.screen_name), new_following))
DbManager.insert_into(table_name="followed_users",
columns=("follower_name", "followed_id",
"followed_name"),
values=new_following_values,
multiple_rows=True)
# Log in
api = Api(consumer_key=consumer_key,
consumer_secret=consumer_secret,
access_token_key=access_token,
access_token_secret=access_secret)
# Load data
print("Loading users who follow" + username + "...")
followers = []
followers.extend(api.GetFollowers(screen_name=username))
print("Loading users that" + username + "follows...")
friends = []
friends.extend(api.GetFriends(screen_name=username))
# Create graph
print("Adding followers relationships...")
for user in followers:
graph.add_edge(user.name, username)
print("Adding following relationships...")
for user in friends:
graph.add_edge(username, user.name)
# Save graph
print("")
print("The personal profile was analyzed succesfully.")
print("")
print("Saving the file as " + username + "-personal-network.gexf...")
def connect_json(request, site):
creds = OAUTH_CREDENTIALS.get(site.upper())
if not creds:
raise Http404('Site %s not found' % site)
if not creds.get('FRIENDSHIPS', True):
raise Http404('Friendships not implemented')
if request.user.oauth_tokens.filter(site = site).count() == 0:
raise Http404('User not connected to %s' % site)
if request.method == 'GET':
page = 1
if request.GET.get('page'):
try:
page = int(request.GET.get('page'))
except:
pass
token = request.user.oauth_tokens.get(site = site)
connections = core.OAuthToken.objects.filter(
site = site
).values_list('username', flat = True).distinct()
connections = [u.lower() for u in connections]
friends = []
followers = core.OAuthToken.objects.filter(
site = site,
user__pk__in = request.user.following.values_list('followee__pk', flat = True)
).values_list('username', flat = True)
if site == 'twitter':
from twitter import Api
api = Api(
consumer_key = creds.get('CONSUMER_KEY'),
consumer_secret = creds.get('CONSUMER_SECRET'),
access_token_key = token.token,
access_token_secret = token.secret
)
for user in api.GetFriends():
if user.screen_name.lower() in connections:
friends.append(
{
'username': user.screen_name.lower(),
'full_name': user.name,
'image': user.profile_image_url,
'following': user.screen_name.lower() in followers,
'show_username': True
}
)
elif site == 'facebook':
def add_friends():
for user in result['data']:
username_or_id = user.get('username', user.get('id'))
if username_or_id in connections:
friends.append(
{
'username': username_or_id,
'full_name': user.get('name'),
'following': user.get('id') in followers,
'image': 'https://graph.facebook.com/%s/picture' % username_or_id,
'show_username': False
}
)
result = simplejson.load(
urlopen('https://graph.facebook.com/me/friends/?access_token=%s&fields=id,username,name' % token.token)
)
if not 'data' in result:
if 'error' in result and result['error'].get('type') == 'OAuthException':
return HttpResponseRedirect(
reverse('oauth_auth', args = [site])
)
raise Exception(result)
add_friends()
while result.get('paging', {}).get('next'):
print 'Moving to next page'
result = simplejson.load(
urlopen(result['paging']['next'])
)
add_friends()
return HttpResponse(
simplejson.dumps(friends),
mimetype = 'application/json'
)
elif request.method == 'POST':
username_or_id = request.POST.get('id', request.POST.get('username')).lower()
token = core.OAuthToken.objects.get(username = username_or_id, site = site)
follow, created = request.user.following.get_or_create(
followee = token.user
)
if created:
follow.notify(request.network)
return HttpResponse('OK', mimetype = 'text/plain')
