def complete_signin(request_key, verifier, postgres_handle):
auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
session = TwitterSession.get_by_request_key(request_key, postgres_handle)
if session.access_key:
#this happens if we get multiple same exact responses from twitter
#perhaps crazy clicking or back / forward browsing
credentials = TwitterCredentials.get_by_access_key(
session.access_key, postgres_handle)
else:
auth.set_request_token(request_key, session.request_secret)
auth.get_access_token(verifier)
# may have signed up already
credentials = TwitterCredentials.get_by_access_key(
auth.access_token.key, postgres_handle)
if not credentials:
credentials = TwitterCredentials.create(auth.access_token.key,
auth.access_token.secret,
postgres_handle)
session.access_key = credentials.access_key
if not credentials.twitter_user:
#probably don't have the user in our db yet
user = TwitterUser.upsert_from_api_user(credentials.api_handle.me(),
postgres_handle)
credentials.twitter_id = user.id
credentials.save()
#email
screen_name = credentials.twitter_user.screen_name
email_utils.send_email('*****@*****.**', ['*****@*****.**'],
'%s signed up' % screen_name, 'smarttypes signup!')
return session.save()
def complete_signin(request_key, verifier, postgres_handle):
auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
session = TwitterSession.get_by_request_key(request_key, postgres_handle)
if session.access_key:
#this happens if we get multiple same exact responses from twitter
#perhaps crazy clicking or back / forward browsing
credentials = TwitterCredentials.get_by_access_key(session.access_key, postgres_handle)
else:
auth.set_request_token(request_key, session.request_secret)
auth.get_access_token(verifier)
# may have signed up already
credentials = TwitterCredentials.get_by_access_key(auth.access_token.key, postgres_handle)
if not credentials:
credentials = TwitterCredentials.create(auth.access_token.key, auth.access_token.secret, postgres_handle)
session.access_key = credentials.access_key
if not credentials.twitter_user:
#probably don't have the user in our db yet
user = TwitterUser.upsert_from_api_user(credentials.api_handle.me(), postgres_handle)
credentials.twitter_id = user.id
credentials.save()
#email
screen_name = credentials.twitter_user.screen_name
email_utils.send_email('*****@*****.**', ['*****@*****.**'],
'%s signed up' % screen_name, 'smarttypes signup!')
return session.save()
def complete_signin(request_key, verifier, postgres_handle):
auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
session = TwitterSession.get_by_request_key(request_key, postgres_handle)
auth.set_request_token(request_key, session.request_secret)
auth.get_access_token(verifier)
# may have signed up already
credentials = TwitterCredentials.get_by_access_key(auth.access_token.key, postgres_handle)
if not credentials:
credentials = TwitterCredentials.create(auth.access_token.key, auth.access_token.secret, postgres_handle)
session.access_key = credentials.access_key
if not credentials.twitter_user:
user = TwitterUser.upsert_from_api_user(credentials.api_handle.me(), postgres_handle)
credentials.twitter_id = user.id
credentials.save()
screen_name = credentials.twitter_user.screen_name
email_utils.send_email('*****@*****.**', ['*****@*****.**'],
'%s signed up' % screen_name, 'smarttypes signup!')
return session.save()
def pull_some_users(access_key):
postgres_handle = PostgresHandle(smarttypes.connection_string)
creds = TwitterCredentials.get_by_access_key(access_key, postgres_handle)
root_user = load_user_and_the_people_they_follow(creds, creds.root_user_id, postgres_handle)
load_this_user_id = root_user.get_id_of_someone_in_my_network_to_load()
while load_this_user_id:
load_user_and_the_people_they_follow(creds, load_this_user_id, postgres_handle)
load_this_user_id = root_user.get_id_of_someone_in_my_network_to_load()
#load_this_user_id = None
print "Finished loading all related users for %s!" % root_user.screen_name
def list_cred_details():
for creds in TwitterCredentials.get_all(postgres_handle):
creds_user = creds.twitter_user
root_user = creds.root_user
creds_username = creds_user.screen_name if creds_user else 'None'
root_username = root_user.screen_name if root_user else 'None'
last_api_query = creds.last_root_user_api_query if creds.last_root_user_api_query else datetime(
2000, 1, 1)
print '%s %s %s' % (
string.ljust(creds_username, 20), string.ljust(root_username, 20),
string.ljust(creds.email if creds.email else '', 30))
def list_cred_details():
for creds in TwitterCredentials.get_all(postgres_handle):
creds_user = creds.twitter_user
root_user = creds.root_user
creds_username = creds_user.screen_name if creds_user else 'None'
root_username = root_user.screen_name if root_user else 'None'
last_api_query = creds.last_root_user_api_query if creds.last_root_user_api_query else datetime(2000,1,1)
print '%s %s %s' % (
string.ljust(creds_username, 20),
string.ljust(root_username, 20),
string.ljust(creds.email if creds.email else '', 30)
)
def pull_some_users(access_key):
postgres_handle = PostgresHandle(smarttypes.connection_string)
creds = TwitterCredentials.get_by_access_key(access_key, postgres_handle)
root_user = load_user_and_the_people_they_follow(creds, creds.root_user_id,
postgres_handle)
load_this_user_id = root_user.get_id_of_someone_in_my_network_to_load()
while load_this_user_id:
load_user_and_the_people_they_follow(creds, load_this_user_id,
postgres_handle)
load_this_user_id = root_user.get_id_of_someone_in_my_network_to_load()
#load_this_user_id = None
print "Finished loading all related users for %s!" % root_user.screen_name
def list_cred_details():
for creds in TwitterCredentials.get_all(postgres_handle):
creds_user = creds.twitter_user
root_user = creds.root_user
creds_username = creds_user.screen_name if creds_user else 'None'
root_username = root_user.screen_name if root_user else 'None'
last_api_query = creds.last_root_user_api_query if creds.last_root_user_api_query else datetime(2000,1,1)
print 'Creds for: %s \t Email: %s \t Root users: %s \t Created: %s \t Last_api_query: %s' % (
string.ljust(creds_username, 12),
string.ljust(creds.email if creds.email else '', 30),
string.ljust(root_username, 12),
creds.createddate.strftime('%y_%m_%d'),
last_api_query.strftime('%y_%m_%d'),
)
def list_cred_details():
for creds in TwitterCredentials.get_all(postgres_handle):
creds_user = creds.twitter_user
root_user = creds.root_user
creds_username = creds_user.screen_name if creds_user else 'None'
root_username = root_user.screen_name if root_user else 'None'
last_api_query = creds.last_root_user_api_query if creds.last_root_user_api_query else datetime(
2000, 1, 1)
print 'Creds for: %s \t Email: %s \t Root users: %s \t Created: %s \t Last_api_query: %s' % (
string.ljust(creds_username, 12),
string.ljust(creds.email if creds.email else '', 30),
string.ljust(root_username, 12),
creds.createddate.strftime('%y_%m_%d'),
last_api_query.strftime('%y_%m_%d'),
)
from datetime import datetime
import smarttypes, sys, string
from smarttypes.config import *
from smarttypes.model.twitter_credentials import TwitterCredentials
from smarttypes.model.twitter_user import TwitterUser
from smarttypes.model.twitter_reduction import TwitterReduction
from smarttypes.utils.postgres_handle import PostgresHandle
postgres_handle = PostgresHandle(smarttypes.connection_string)
if __name__ == "__main__":
user_count_list = TwitterReduction.get_user_reduction_counts(
postgres_handle)
for user, count in user_count_list:
if count == 1:
creds = TwitterCredentials.get_by_twitter_id(
user.id, postgres_handle)
creds.maybe_send_initial_map_email()
def credentials(self):
from smarttypes.model.twitter_credentials import TwitterCredentials
if not self.access_key:
return None
return TwitterCredentials.get_by_access_key(self.access_key, self.postgres_handle)
def credentials(self):
from smarttypes.model.twitter_credentials import TwitterCredentials
creds = TwitterCredentials.get_by_twitter_id(self.id, self.postgres_handle)
if not creds:
creds = TwitterCredentials.get_by_root_user_id(self.id, self.postgres_handle)
return creds
root_user = load_user_and_the_people_they_follow(api_handle,
root_user.id,
postgres_handle,
is_root_user=True)
load_this_user_id = root_user.get_id_of_someone_in_my_network_to_load()
while load_this_user_id:
load_user_and_the_people_they_follow(api_handle, load_this_user_id,
postgres_handle)
load_this_user_id = root_user.get_id_of_someone_in_my_network_to_load()
#load_this_user_id = None
print "Finshed loading all related users for %s!" % root_user.screen_name
if __name__ == "__main__":
postgres_handle = PostgresHandle(smarttypes.connection_string)
max_processes = 4
i = 0
for creds in TwitterCredentials.get_all(
postgres_handle, order_by='last_root_user_api_query'):
if i >= max_processes:
break
root_user = creds.root_user
if root_user:
print "Starting a process to load root user: %s" % root_user.screen_name
creds.last_root_user_api_query = datetime.now()
creds.save()
postgres_handle.connection.commit()
p = Process(target=pull_some_users, args=(root_user.id, ))
p.start()
i += 1
def credentials(self):
from smarttypes.model.twitter_credentials import TwitterCredentials
creds = TwitterCredentials.get_by_twitter_id(self.id, self.postgres_handle)
if not creds:
creds = TwitterCredentials.get_by_root_user_id(self.id, self.postgres_handle)
return creds
for user_id in user_ids:
scores.append(pagerank.get(user_id, 0))
groups.append(
TwitterGroup.create_group(twitter_reduction.id, i, user_ids,
scores, postgres_handle))
postgres_handle.connection.commit()
###########################################
##makes for quicker queries in some cases
###########################################
twitter_reduction.save_group_info(postgres_handle)
postgres_handle.connection.commit()
###########################################
##mk_tag_clouds
###########################################
TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle)
postgres_handle.connection.commit()
if __name__ == "__main__":
postgres_handle = PostgresHandle(smarttypes.connection_string)
for creds in TwitterCredentials.get_all(postgres_handle):
root_user = creds.root_user
if root_user and root_user.screen_name == 'SmartTypes':
#distance = int(400 / (root_user.following_count / 100.0))
distance = 1000
reduce_graph(root_user.screen_name,
distance=distance,
min_followers=60)
creds = TwitterCredentials.get_by_access_key(access_key, postgres_handle)
root_user = load_user_and_the_people_they_follow(creds, creds.root_user_id, postgres_handle)
load_this_user_id = root_user.get_id_of_someone_in_my_network_to_load()
while load_this_user_id:
load_user_and_the_people_they_follow(creds, load_this_user_id, postgres_handle)
load_this_user_id = root_user.get_id_of_someone_in_my_network_to_load()
#load_this_user_id = None
print "Finished loading all related users for %s!" % root_user.screen_name
if __name__ == "__main__":
postgres_handle = PostgresHandle(smarttypes.connection_string)
max_processes = 8
i = 0
for creds in TwitterCredentials.get_all(postgres_handle, order_by='last_root_user_api_query'):
if i >= max_processes:
break
#this can happen because this script is run on a different db than the webserver
#this is only a temporary problem because i'm poor
#see scripts/check_for_db_updates.py - root_user code for related research
if creds.twitter_id and not creds.twitter_user:
TwitterUser.upsert_from_api_user(creds.api_handle.me(), postgres_handle)
postgres_handle.connection.commit()
if creds.root_user_id and not creds.root_user:
root_user = creds.api_handle.get_user(user_id=creds.root_user_id)
TwitterUser.upsert_from_api_user(root_user, postgres_handle)
postgres_handle.connection.commit()
group_graph.add_edges_from(group_edges)
pagerank = networkx.pagerank(group_graph, max_iter=500)
scores = []
for user_id in user_ids:
scores.append(pagerank.get(user_id, 0))
groups.append(TwitterGroup.create_group(twitter_reduction.id, i, user_ids, scores,
postgres_handle))
postgres_handle.connection.commit()
###########################################
##makes for quicker queries in some cases
###########################################
twitter_reduction.save_group_info(postgres_handle)
postgres_handle.connection.commit()
###########################################
##mk_tag_clouds
###########################################
TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle)
postgres_handle.connection.commit()
if __name__ == "__main__":
postgres_handle = PostgresHandle(smarttypes.connection_string)
for creds in TwitterCredentials.get_all(postgres_handle):
root_user = creds.root_user
if root_user and root_user.screen_name == 'SmartTypes':
#distance = int(400 / (root_user.following_count / 100.0))
distance = 1000
reduce_graph(root_user.screen_name, distance=distance, min_followers=60)
string.ljust(root_username, 20),
string.ljust(creds.email if creds.email else '', 30)
)
if __name__ == "__main__":
"""
if no args, show all creds
if args, first arg is creds_username, second is root_username
"""
if len(sys.argv) == 1:
list_cred_details()
elif len(sys.argv) == 2:
creds_user = TwitterUser.by_screen_name(sys.argv[1], postgres_handle)
creds = TwitterCredentials.get_by_twitter_id(creds_user.id, postgres_handle)
creds.root_user_id = None
creds.save()
postgres_handle.connection.commit()
else:
creds_user = TwitterUser.by_screen_name(sys.argv[1], postgres_handle)
root_user = TwitterUser.by_screen_name(sys.argv[2], postgres_handle)
if not root_user:
api_user = creds_user.credentials.api_handle.get_user(screen_name=sys.argv[2])
root_user = TwitterUser.upsert_from_api_user(api_user, postgres_handle)
postgres_handle.connection.commit()
creds = TwitterCredentials.get_by_twitter_id(creds_user.id, postgres_handle)
creds.root_user_id = root_user.id
creds.save()
postgres_handle.connection.commit()
def credentials(self):
from smarttypes.model.twitter_credentials import TwitterCredentials
if not self.access_key:
return None
return TwitterCredentials.get_by_access_key(self.access_key,
self.postgres_handle)