g.vs['size'] = 1 #plot to file layout = Layout(layout_list) filepath = 'io/%s.png' % root_user.screen_name thumb_filepath = 'io/%s_thumb.png' % root_user.screen_name plot_graph(g, layout, filepath, size_tup=(600, 600)) #need to adjust vs['size'] if i want to do this #plot_graph(g, layout, thumb_filepath, size_tup=(50, 50)) if not smarttypes.config.IS_PROD: os.system('cp io/%s*.png /home/timmyt/projects/smarttypes/smarttypes/static/images/maps/.' % root_user.screen_name) else: os.system('scp io/%s*.png cottie:/home/timmyt/projects/smarttypes/smarttypes/static/images/maps/.' % root_user.screen_name) print 'save to disk' twitter_reduction = TwitterReduction.create_reduction(root_user.id, postgres_handle) postgres_handle.connection.commit() for community_idx, values_dict in community_stats.items(): #params: #reduction_id, index, center_coordinate, member_ids, #global_pagerank, community_pagerank, hybrid_pagerank if community_idx > 0: TwitterCommunity.create_community(twitter_reduction.id, community_idx, values_dict['center_coordinate'], values_dict['member_ids'], values_dict['global_pagerank'], values_dict['community_pagerank'], values_dict['hybrid_pagerank'], postgres_handle) postgres_handle.connection.commit() TwitterCommunity.mk_tag_clouds(twitter_reduction.id, postgres_handle) postgres_handle.connection.commit() #how long print datetime.now() - start_time
#coordinates = reduce_with_linloglayout(g, root_user) coordinates = reduce_with_semi_intelligent_agents(g) #id_communities vertex_clustering = id_communities(g, coordinates) #do this after community detection because it causes distortion coordinates = reproject_to_spherical_mercator(coordinates) #network_stats network_stats = get_network_stats(network, g, vertex_clustering) global_pagerank, community_pagerank, community_score = network_stats hybrid_pagerank = calculate_hybrid_pagerank(global_pagerank, community_pagerank, community_score) print "save reduction" reduction = TwitterReduction.create_reduction(root_user.id, [0, 0, 0], False, postgres_handle) postgres_handle.connection.commit() print "save reduction users" reduction_users = [] for i in range(len(member_ids)): tru = TwitterReductionUser(postgres_handle=postgres_handle) tru.reduction_id = reduction.id tru.user_id = member_ids[i] tru.coordinates = Point(coordinates[i][0], coordinates[i][1]) tru.pagerank = global_pagerank[i] tru.hybrid_pagerank = hybrid_pagerank[i] reduction_users.append(tru.save()) postgres_handle.connection.commit() print "save communities"
def reduce_graph(screen_name, distance=20, min_followers=60): postgres_handle = PostgresHandle(smarttypes.connection_string) ########################################### ##reduce ########################################### root_user = TwitterUser.by_screen_name(screen_name, postgres_handle) follower_followies_map = root_user.get_graph_info(distance=distance, min_followers=min_followers) gr = GraphReduce(screen_name, follower_followies_map) gr.reduce_with_linloglayout() ########################################### ##save reduction in db ########################################### root_user_id = root_user.id user_ids = [] x_coordinates = [] y_coordinates = [] in_links = [] out_links = [] for i in range(len(gr.layout_ids)): user_id = gr.layout_ids[i] user_ids.append(user_id) x_coordinates.append(gr.reduction[i][0]) y_coordinates.append(gr.reduction[i][1]) itr_in_links = PostgresHandle.spliter.join(gr.G.predecessors(user_id)) itr_out_links = PostgresHandle.spliter.join(gr.G.successors(user_id)) in_links.append(itr_in_links) out_links.append(itr_out_links) twitter_reduction = TwitterReduction.create_reduction(root_user_id, user_ids, x_coordinates, y_coordinates, in_links, out_links, postgres_handle) postgres_handle.connection.commit() ########################################### ##save groups in db ########################################### groups = [] for i in range(gr.n_groups): user_ids = [] for j in range(len(gr.layout_ids)): if i == gr.groups[j]: user_ids.append(gr.layout_ids[j]) #run pagerank to get the scores group_graph = networkx.DiGraph() group_edges = [] for user_id in user_ids: for following_id in set(user_ids).intersection(follower_followies_map[user_id]): group_edges.append((user_id, following_id)) print len(user_ids), len(group_edges) if not group_edges: continue group_graph.add_edges_from(group_edges) pagerank = networkx.pagerank(group_graph, max_iter=500) scores = [] for user_id in user_ids: scores.append(pagerank.get(user_id, 0)) groups.append(TwitterGroup.create_group(twitter_reduction.id, i, user_ids, scores, postgres_handle)) postgres_handle.connection.commit() ########################################### ##makes for quicker queries in some cases ########################################### twitter_reduction.save_group_info(postgres_handle) postgres_handle.connection.commit() ########################################### ##mk_tag_clouds ########################################### TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle) postgres_handle.connection.commit()
def reduce_graph(screen_name, distance=20, min_followers=60): postgres_handle = PostgresHandle(smarttypes.connection_string) ########################################### ##reduce ########################################### root_user = TwitterUser.by_screen_name(screen_name, postgres_handle) follower_followies_map = root_user.get_graph_info( distance=distance, min_followers=min_followers) gr = GraphReduce(screen_name, follower_followies_map) gr.reduce_with_linloglayout() ########################################### ##save reduction in db ########################################### root_user_id = root_user.id user_ids = [] x_coordinates = [] y_coordinates = [] in_links = [] out_links = [] for i in range(len(gr.layout_ids)): user_id = gr.layout_ids[i] user_ids.append(user_id) x_coordinates.append(gr.reduction[i][0]) y_coordinates.append(gr.reduction[i][1]) itr_in_links = PostgresHandle.spliter.join(gr.G.predecessors(user_id)) itr_out_links = PostgresHandle.spliter.join(gr.G.successors(user_id)) in_links.append(itr_in_links) out_links.append(itr_out_links) twitter_reduction = TwitterReduction.create_reduction( root_user_id, user_ids, x_coordinates, y_coordinates, in_links, out_links, postgres_handle) postgres_handle.connection.commit() ########################################### ##save groups in db ########################################### groups = [] for i in range(gr.n_groups): user_ids = [] for j in range(len(gr.layout_ids)): if i == gr.groups[j]: user_ids.append(gr.layout_ids[j]) #run pagerank to get the scores group_graph = networkx.DiGraph() group_edges = [] for user_id in user_ids: for following_id in set(user_ids).intersection( follower_followies_map[user_id]): group_edges.append((user_id, following_id)) print len(user_ids), len(group_edges) if not group_edges: continue group_graph.add_edges_from(group_edges) pagerank = networkx.pagerank(group_graph, max_iter=500) scores = [] for user_id in user_ids: scores.append(pagerank.get(user_id, 0)) groups.append( TwitterGroup.create_group(twitter_reduction.id, i, user_ids, scores, postgres_handle)) postgres_handle.connection.commit() ########################################### ##makes for quicker queries in some cases ########################################### twitter_reduction.save_group_info(postgres_handle) postgres_handle.connection.commit() ########################################### ##mk_tag_clouds ########################################### TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle) postgres_handle.connection.commit()
def reduce_graph(screen_name, distance=20, min_followers=60, pickle_it=True, just_load_from_file=False): postgres_handle = PostgresHandle(smarttypes.connection_string) # if just_load_from_file: # print "Loading data from a pickle." # gr = GraphReduce(screen_name, {}) # f = open(gr.pickle_file_path) # twitter_reduction, groups = pickle.load(f) # twitter_reduction.id = None # twitter_reduction.postgres_handle = postgres_handle # twitter_reduction.save() # postgres_handle.connection.commit() # for group in groups: # group.id = None # group.reduction_id = twitter_reduction.id # group.postgres_handle = postgres_handle # group.save() # postgres_handle.connection.commit() # TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle) # postgres_handle.connection.commit() # print "All done!" # return 0 ######################## ##reduce ######################## root_user = TwitterUser.by_screen_name(screen_name, postgres_handle) follower_followies_map = root_user.get_graph_info( distance=distance, min_followers=min_followers) gr = GraphReduce(screen_name, follower_followies_map) #gr.reduce_with_exafmm() gr.reduce_with_linloglayout() ######################## ##save reduction in db ######################## root_user_id = root_user.id user_ids = [] x_coordinates = [] y_coordinates = [] in_links = [] out_links = [] for i in range(len(gr.layout_ids)): user_id = gr.layout_ids[i] user_ids.append(user_id) x_coordinates.append(gr.reduction[i][0]) y_coordinates.append(gr.reduction[i][1]) itr_in_links = PostgresHandle.spliter.join(gr.G.predecessors(user_id)) itr_out_links = PostgresHandle.spliter.join(gr.G.successors(user_id)) in_links.append(itr_in_links) out_links.append(itr_out_links) twitter_reduction = TwitterReduction.create_reduction( root_user_id, user_ids, x_coordinates, y_coordinates, in_links, out_links, postgres_handle) postgres_handle.connection.commit() ######################## ##save groups in db ######################## groups = [] for i in range(gr.n_clusters): user_ids = [] for j in range(len(gr.layout_ids)): if gr.layout_clusters[j][i] > 0: user_ids.append(gr.layout_ids[j]) #run pagerank to get the scores group_graph = networkx.DiGraph() group_edges = [] for user_id in user_ids: if user_id in follower_followies_map: for following_id in set(user_ids).intersection( follower_followies_map[user_id]): group_edges.append((user_id, following_id)) print len(user_ids), len(group_edges) if not group_edges: continue group_graph.add_edges_from(group_edges) pagerank = networkx.pagerank(group_graph, max_iter=500) scores = [] for user_id in user_ids: scores.append(pagerank.get(user_id, 0)) groups.append( TwitterGroup.create_group(twitter_reduction.id, i, user_ids, scores, postgres_handle)) postgres_handle.connection.commit() twitter_reduction.save_group_info(postgres_handle) postgres_handle.connection.commit() ######################## ##mk_tag_clouds ######################## TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle) postgres_handle.connection.commit()
#id_communities vertex_clustering = id_communities(g, coordinates) #do this after community detection because it causes distortion coordinates = reproject_to_spherical_mercator(coordinates) #network_stats network_stats = get_network_stats(network, g, vertex_clustering) global_pagerank, community_pagerank, community_score = network_stats hybrid_pagerank = calculate_hybrid_pagerank(global_pagerank, community_pagerank, community_score) print "save reduction" reduction = TwitterReduction.create_reduction(root_user.id, [0, 0, 0], False, postgres_handle) postgres_handle.connection.commit() print "save reduction users" reduction_users = [] for i in range(len(member_ids)): tru = TwitterReductionUser(postgres_handle=postgres_handle) tru.reduction_id = reduction.id tru.user_id = member_ids[i] tru.coordinates = Point(coordinates[i][0], coordinates[i][1]) tru.pagerank = global_pagerank[i] tru.hybrid_pagerank = hybrid_pagerank[i] reduction_users.append(tru.save()) postgres_handle.connection.commit() print "save communities"