def write_distance_matrix_to_csv(): postgres_handle = PostgresHandle(smarttypes.connection_string) network = load_network_from_the_db(postgres_handle, 5) landmarks = get_landmarks(network) similarity_matrix = mk_similarity_matrix(network, landmarks) distance_matrix = dist_euclidean(similarity_matrix) write_similarity_matrix_to_csv(distance_matrix)
def pull_some_users(access_key): postgres_handle = PostgresHandle(smarttypes.connection_string) creds = TwitterCredentials.get_by_access_key(access_key, postgres_handle) root_user = load_user_and_the_people_they_follow(creds, creds.root_user_id, postgres_handle) load_this_user_id = root_user.get_id_of_someone_in_my_network_to_load() while load_this_user_id: load_user_and_the_people_they_follow(creds, load_this_user_id, postgres_handle) load_this_user_id = root_user.get_id_of_someone_in_my_network_to_load() #load_this_user_id = None print "Finished loading all related users for %s!" % root_user.screen_name
def application(environ, start_response): path = environ.get('PATH_INFO', '').lstrip('/') for regex, controller in urls: match = re.search(regex, path) if match: request = Request(environ) try: postgres_handle = PostgresHandle(smarttypes.connection_string) try: session = None if request.cookies.get('session'): session = TwitterSession.get_by_request_key( request.cookies['session'], postgres_handle) response_dict = controller(request, session, postgres_handle) web_response = WebResponse(request, controller.__name__, response_dict, session) response_headers = web_response.get_response_headers() response_string = web_response.get_response_str() if getattr(postgres_handle, '_connection', False): postgres_handle.connection.commit() status_code = '200 OK' except RedirectException, (redirect_ex): if getattr(postgres_handle, '_connection', False): postgres_handle.connection.commit() status_code = '303 See Other' response_headers = [('Location', redirect_ex.redirect_url)] response_string = [""] except: if getattr(postgres_handle, '_connection', False): postgres_handle.connection.rollback() raise finally: if getattr(postgres_handle, '_connection', False): postgres_handle.connection.close() #start response start_response(status_code, response_headers) return response_string except Exception: #can't use print statements with mod_wsgi error_string = traceback.format_exc() start_response('500 Internal Server Error', [('Content-Type', 'text/plain')]) if smarttypes.config.IS_PROD: email_utils.send_email( '*****@*****.**', ['*****@*****.**', '*****@*****.**'], error_string, 'smarttypes site error') return [error_string]
def pull_some_users(user_id): postgres_handle = PostgresHandle(smarttypes.connection_string) root_user = TwitterUser.get_by_id(user_id, postgres_handle) if not root_user: raise Exception('User ID: %s not in our DB!' % user_id) if not root_user.credentials: raise Exception('%s does not have api credentials!' % root_user.screen_name) api_handle = root_user.credentials.api_handle root_user = load_user_and_the_people_they_follow(api_handle, root_user.id, postgres_handle, is_root_user=True) load_this_user_id = root_user.get_id_of_someone_in_my_network_to_load() while load_this_user_id: load_user_and_the_people_they_follow(api_handle, load_this_user_id, postgres_handle) load_this_user_id = root_user.get_id_of_someone_in_my_network_to_load() #load_this_user_id = None print "Finshed loading all related users for %s!" % root_user.screen_name
from datetime import datetime import smarttypes, sys, string from smarttypes.config import * from smarttypes.model.twitter_credentials import TwitterCredentials from smarttypes.model.twitter_user import TwitterUser from smarttypes.model.twitter_reduction import TwitterReduction from smarttypes.utils.postgres_handle import PostgresHandle postgres_handle = PostgresHandle(smarttypes.connection_string) if __name__ == "__main__": user_count_list = TwitterReduction.get_user_reduction_counts( postgres_handle) for user, count in user_count_list: if count == 1: creds = TwitterCredentials.get_by_twitter_id( user.id, postgres_handle) creds.maybe_send_initial_map_email()
def reduce_graph(screen_name, distance=20, min_followers=60): postgres_handle = PostgresHandle(smarttypes.connection_string) ########################################### ##reduce ########################################### root_user = TwitterUser.by_screen_name(screen_name, postgres_handle) follower_followies_map = root_user.get_graph_info( distance=distance, min_followers=min_followers) gr = GraphReduce(screen_name, follower_followies_map) gr.reduce_with_linloglayout() ########################################### ##save reduction in db ########################################### root_user_id = root_user.id user_ids = [] x_coordinates = [] y_coordinates = [] in_links = [] out_links = [] for i in range(len(gr.layout_ids)): user_id = gr.layout_ids[i] user_ids.append(user_id) x_coordinates.append(gr.reduction[i][0]) y_coordinates.append(gr.reduction[i][1]) itr_in_links = PostgresHandle.spliter.join(gr.G.predecessors(user_id)) itr_out_links = PostgresHandle.spliter.join(gr.G.successors(user_id)) in_links.append(itr_in_links) out_links.append(itr_out_links) twitter_reduction = TwitterReduction.create_reduction( root_user_id, user_ids, x_coordinates, y_coordinates, in_links, out_links, postgres_handle) postgres_handle.connection.commit() ########################################### ##save groups in db ########################################### groups = [] for i in range(gr.n_groups): user_ids = [] for j in range(len(gr.layout_ids)): if i == gr.groups[j]: user_ids.append(gr.layout_ids[j]) #run pagerank to get the scores group_graph = networkx.DiGraph() group_edges = [] for user_id in user_ids: for following_id in set(user_ids).intersection( follower_followies_map[user_id]): group_edges.append((user_id, following_id)) print len(user_ids), len(group_edges) if not group_edges: continue group_graph.add_edges_from(group_edges) pagerank = networkx.pagerank(group_graph, max_iter=500) scores = [] for user_id in user_ids: scores.append(pagerank.get(user_id, 0)) groups.append( TwitterGroup.create_group(twitter_reduction.id, i, user_ids, scores, postgres_handle)) postgres_handle.connection.commit() ########################################### ##makes for quicker queries in some cases ########################################### twitter_reduction.save_group_info(postgres_handle) postgres_handle.connection.commit() ########################################### ##mk_tag_clouds ########################################### TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle) postgres_handle.connection.commit()
def do_linlog_reduction(): postgres_handle = PostgresHandle(smarttypes.connection_string) network = load_network_from_the_db(postgres_handle, 5) reduce_with_linloglayout(network) graph_reduction, communities = load_linlog_reduction_from_file() write_reduction_to_csv(network, graph_reduction)
def reduce_graph(screen_name, distance=20, min_followers=60, pickle_it=True, just_load_from_file=False): postgres_handle = PostgresHandle(smarttypes.connection_string) # if just_load_from_file: # print "Loading data from a pickle." # gr = GraphReduce(screen_name, {}) # f = open(gr.pickle_file_path) # twitter_reduction, groups = pickle.load(f) # twitter_reduction.id = None # twitter_reduction.postgres_handle = postgres_handle # twitter_reduction.save() # postgres_handle.connection.commit() # for group in groups: # group.id = None # group.reduction_id = twitter_reduction.id # group.postgres_handle = postgres_handle # group.save() # postgres_handle.connection.commit() # TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle) # postgres_handle.connection.commit() # print "All done!" # return 0 ######################## ##reduce ######################## root_user = TwitterUser.by_screen_name(screen_name, postgres_handle) follower_followies_map = root_user.get_graph_info( distance=distance, min_followers=min_followers) gr = GraphReduce(screen_name, follower_followies_map) #gr.reduce_with_exafmm() gr.reduce_with_linloglayout() ######################## ##save reduction in db ######################## root_user_id = root_user.id user_ids = [] x_coordinates = [] y_coordinates = [] in_links = [] out_links = [] for i in range(len(gr.layout_ids)): user_id = gr.layout_ids[i] user_ids.append(user_id) x_coordinates.append(gr.reduction[i][0]) y_coordinates.append(gr.reduction[i][1]) itr_in_links = PostgresHandle.spliter.join(gr.G.predecessors(user_id)) itr_out_links = PostgresHandle.spliter.join(gr.G.successors(user_id)) in_links.append(itr_in_links) out_links.append(itr_out_links) twitter_reduction = TwitterReduction.create_reduction( root_user_id, user_ids, x_coordinates, y_coordinates, in_links, out_links, postgres_handle) postgres_handle.connection.commit() ######################## ##save groups in db ######################## groups = [] for i in range(gr.n_clusters): user_ids = [] for j in range(len(gr.layout_ids)): if gr.layout_clusters[j][i] > 0: user_ids.append(gr.layout_ids[j]) #run pagerank to get the scores group_graph = networkx.DiGraph() group_edges = [] for user_id in user_ids: if user_id in follower_followies_map: for following_id in set(user_ids).intersection( follower_followies_map[user_id]): group_edges.append((user_id, following_id)) print len(user_ids), len(group_edges) if not group_edges: continue group_graph.add_edges_from(group_edges) pagerank = networkx.pagerank(group_graph, max_iter=500) scores = [] for user_id in user_ids: scores.append(pagerank.get(user_id, 0)) groups.append( TwitterGroup.create_group(twitter_reduction.id, i, user_ids, scores, postgres_handle)) postgres_handle.connection.commit() twitter_reduction.save_group_info(postgres_handle) postgres_handle.connection.commit() ######################## ##mk_tag_clouds ######################## TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle) postgres_handle.connection.commit()
#setup a remote db tunnel import subprocess, copy subprocess.call('ssh [email protected] -N -f -L 5433:localhost:5432', shell=True) from datetime import datetime import smarttypes, sys, string from smarttypes.config import * from smarttypes.model.twitter_credentials import TwitterCredentials from smarttypes.model.twitter_user import TwitterUser from smarttypes.utils.postgres_handle import PostgresHandle remote_postgres_handle = PostgresHandle(smarttypes.connection_string + " port='5433'") local_postgres_handle = PostgresHandle(smarttypes.connection_string) for creds in TwitterCredentials.get_all(remote_postgres_handle): remote_root_user = copy.deepcopy(creds.root_user) creds.postgres_handle = local_postgres_handle local_root_user = creds.root_user if remote_root_user and not local_root_user: remote_root_user.postgres_handle = local_postgres_handle remote_root_user.save() creds.save() local_postgres_handle.connection.commit()