def group(request): if 'group_index' in request.params: group_index = int(request.params['group_index']) twitter_group = TwitterGroup.get_by_index(group_index) else: twitter_group = TwitterGroup.get_random_group() return { 'twitter_group':twitter_group, }
def next_or_previous_reduction_id(req, session, postgres_handle): reduction = None if 'reduction_id' in req.params: try: reduction_id = int(req.params['reduction_id']) except ValueError: reduction_id = -1 reduction = TwitterReduction.get_by_id(reduction_id, postgres_handle) #prev / next reduction new_reduction_id = -1 if reduction and 'next_or_previous' in req.params: if req.params['next_or_previous'] in ['prev_reduction', 'next_reduction']: ordered_reduction_list = TwitterReduction.get_ordered_id_list( reduction.root_user_id, postgres_handle) for i in range(len(ordered_reduction_list)): if reduction.id == ordered_reduction_list[i]: current_idx = i break if req.params['next_or_previous'] == 'prev_reduction': idx = current_idx - 1 if req.params['next_or_previous'] == 'next_reduction': idx = current_idx + 1 new_reduction_id = ordered_reduction_list[idx] return { 'content_type': 'application/json', 'json': { 'reduction_id': new_reduction_id, 'num_groups': len(TwitterGroup.all_groups(new_reduction_id, postgres_handle)) } }
def index(req, session, postgres_handle): root_user = None if 'user_id' in req.params: root_user = TwitterUser.get_by_id(req.params['user_id'], postgres_handle) if not root_user: root_user = TwitterUser.by_screen_name('SmartTypes', postgres_handle) reduction = TwitterReduction.get_latest_reduction(root_user.id, postgres_handle) if not reduction: root_user = TwitterUser.by_screen_name('SmartTypes', postgres_handle) reduction = TwitterReduction.get_latest_reduction( root_user.id, postgres_handle) return { 'active_tab': 'social_map', 'template_path': 'social_map/index.html', 'root_user': root_user, 'reduction': reduction, 'num_groups': len(TwitterGroup.all_groups(reduction.id, postgres_handle)), 'users_with_a_reduction': TwitterReduction.get_users_with_a_reduction(postgres_handle), }
def top_groups(self, num_groups=10): from smarttypes.model.twitter_group import TwitterGroup return_list = [] i = 0 for score, group_id in sorted(self.scores_groups, reverse=True): if i <= num_groups and score > .001: return_list.append((score, TwitterGroup.get_by_index(group_id, self.postgres_handle))) else: break i += 1 return return_list
def group_details(req, session, postgres_handle): if 'group_index' in req.params and 'reduction_id' in req.params: reduction = TwitterReduction.get_by_id(req.params['reduction_id'], postgres_handle) group_index = int(req.params['group_index']) twitter_group = TwitterGroup.get_by_index(reduction.id, group_index, postgres_handle) else: twitter_group = None return { 'template_path': 'social_map/group_details.html', 'twitter_group': twitter_group, }
def index(req, session, postgres_handle): root_user = None if 'user_id' in req.params: root_user = TwitterUser.get_by_id(req.params['user_id'], postgres_handle) if not root_user: root_user = TwitterUser.by_screen_name('SmartTypes', postgres_handle) reduction = TwitterReduction.get_latest_reduction(root_user.id, postgres_handle) if not reduction: root_user = TwitterUser.by_screen_name('SmartTypes', postgres_handle) reduction = TwitterReduction.get_latest_reduction(root_user.id, postgres_handle) return { 'active_tab': 'social_map', 'template_path': 'social_map/index.html', 'root_user': root_user, 'reduction': reduction, 'num_groups': len(TwitterGroup.all_groups(reduction.id, postgres_handle)), 'users_with_a_reduction': TwitterReduction.get_users_with_a_reduction(postgres_handle), }
def next_or_previous_reduction_id(req, session, postgres_handle): reduction = None if 'reduction_id' in req.params: try: reduction_id = int(req.params['reduction_id']) except ValueError: reduction_id = -1 reduction = TwitterReduction.get_by_id(reduction_id, postgres_handle) #prev / next reduction new_reduction_id = -1 if reduction and 'next_or_previous' in req.params: if req.params['next_or_previous'] in [ 'prev_reduction', 'next_reduction' ]: ordered_reduction_list = TwitterReduction.get_ordered_id_list( reduction.root_user_id, postgres_handle) for i in range(len(ordered_reduction_list)): if reduction.id == ordered_reduction_list[i]: current_idx = i break if req.params['next_or_previous'] == 'prev_reduction': idx = current_idx - 1 if req.params['next_or_previous'] == 'next_reduction': idx = current_idx + 1 new_reduction_id = ordered_reduction_list[idx] return { 'content_type': 'application/json', 'json': { 'reduction_id': new_reduction_id, 'num_groups': len(TwitterGroup.all_groups(new_reduction_id, postgres_handle)) } }
for j in range(len(A[i])): write_this = str(int(round(A[i][j]))) test_file.write(write_this+',') test_file.write('\n') group_adjacency = [] A = numpy.dot(users_data, numpy.transpose(items_data)) for i in range(num_features): membership_scores = [] for j in range(num_features): membership_scores.append((A[i][j] * A[j][i], j)) group_adjacency.append(membership_scores) index_to_twitter_id_dict = pickle.load(open('index_to_twitter_id.pickle', 'r')) user_group_map = {} TwitterGroup.bulk_delete('all') for i in range(num_features): membership_scores = [] for j in range(num_items): user_id = index_to_twitter_id_dict[j] follower_score = users_data[i][j] following_score = items_data[i][j] membership_score = following_score * following_score if membership_score > .001: membership_scores.append((membership_score, user_id)) if user_id not in user_group_map: user_group_map[user_id] = [(membership_score, i)] else: user_group_map[user_id].append((membership_score, i)) TwitterGroup.upsert_group(i, membership_scores, group_adjacency[i]) print "Done creating groups."
for j in range(len(A[i])): write_this = str(int(round(A[i][j]))) test_file.write(write_this + ',') test_file.write('\n') group_adjacency = [] A = numpy.dot(users_data, numpy.transpose(items_data)) for i in range(num_features): membership_scores = [] for j in range(num_features): membership_scores.append((A[i][j] * A[j][i], j)) group_adjacency.append(membership_scores) index_to_twitter_id_dict = pickle.load(open('index_to_twitter_id.pickle', 'r')) user_group_map = {} TwitterGroup.bulk_delete('all') for i in range(num_features): membership_scores = [] for j in range(num_items): user_id = index_to_twitter_id_dict[j] follower_score = users_data[i][j] following_score = items_data[i][j] membership_score = following_score * following_score if membership_score > .001: membership_scores.append((membership_score, user_id)) if user_id not in user_group_map: user_group_map[user_id] = [(membership_score, i)] else: user_group_map[user_id].append((membership_score, i)) TwitterGroup.upsert_group(i, membership_scores, group_adjacency[i]) print "Done creating groups."
def reduce_graph(screen_name, distance=20, min_followers=60): postgres_handle = PostgresHandle(smarttypes.connection_string) ########################################### ##reduce ########################################### root_user = TwitterUser.by_screen_name(screen_name, postgres_handle) follower_followies_map = root_user.get_graph_info(distance=distance, min_followers=min_followers) gr = GraphReduce(screen_name, follower_followies_map) gr.reduce_with_linloglayout() ########################################### ##save reduction in db ########################################### root_user_id = root_user.id user_ids = [] x_coordinates = [] y_coordinates = [] in_links = [] out_links = [] for i in range(len(gr.layout_ids)): user_id = gr.layout_ids[i] user_ids.append(user_id) x_coordinates.append(gr.reduction[i][0]) y_coordinates.append(gr.reduction[i][1]) itr_in_links = PostgresHandle.spliter.join(gr.G.predecessors(user_id)) itr_out_links = PostgresHandle.spliter.join(gr.G.successors(user_id)) in_links.append(itr_in_links) out_links.append(itr_out_links) twitter_reduction = TwitterReduction.create_reduction(root_user_id, user_ids, x_coordinates, y_coordinates, in_links, out_links, postgres_handle) postgres_handle.connection.commit() ########################################### ##save groups in db ########################################### groups = [] for i in range(gr.n_groups): user_ids = [] for j in range(len(gr.layout_ids)): if i == gr.groups[j]: user_ids.append(gr.layout_ids[j]) #run pagerank to get the scores group_graph = networkx.DiGraph() group_edges = [] for user_id in user_ids: for following_id in set(user_ids).intersection(follower_followies_map[user_id]): group_edges.append((user_id, following_id)) print len(user_ids), len(group_edges) if not group_edges: continue group_graph.add_edges_from(group_edges) pagerank = networkx.pagerank(group_graph, max_iter=500) scores = [] for user_id in user_ids: scores.append(pagerank.get(user_id, 0)) groups.append(TwitterGroup.create_group(twitter_reduction.id, i, user_ids, scores, postgres_handle)) postgres_handle.connection.commit() ########################################### ##makes for quicker queries in some cases ########################################### twitter_reduction.save_group_info(postgres_handle) postgres_handle.connection.commit() ########################################### ##mk_tag_clouds ########################################### TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle) postgres_handle.connection.commit()
def reduce_graph(screen_name, distance=20, min_followers=60): postgres_handle = PostgresHandle(smarttypes.connection_string) ########################################### ##reduce ########################################### root_user = TwitterUser.by_screen_name(screen_name, postgres_handle) follower_followies_map = root_user.get_graph_info( distance=distance, min_followers=min_followers) gr = GraphReduce(screen_name, follower_followies_map) gr.reduce_with_linloglayout() ########################################### ##save reduction in db ########################################### root_user_id = root_user.id user_ids = [] x_coordinates = [] y_coordinates = [] in_links = [] out_links = [] for i in range(len(gr.layout_ids)): user_id = gr.layout_ids[i] user_ids.append(user_id) x_coordinates.append(gr.reduction[i][0]) y_coordinates.append(gr.reduction[i][1]) itr_in_links = PostgresHandle.spliter.join(gr.G.predecessors(user_id)) itr_out_links = PostgresHandle.spliter.join(gr.G.successors(user_id)) in_links.append(itr_in_links) out_links.append(itr_out_links) twitter_reduction = TwitterReduction.create_reduction( root_user_id, user_ids, x_coordinates, y_coordinates, in_links, out_links, postgres_handle) postgres_handle.connection.commit() ########################################### ##save groups in db ########################################### groups = [] for i in range(gr.n_groups): user_ids = [] for j in range(len(gr.layout_ids)): if i == gr.groups[j]: user_ids.append(gr.layout_ids[j]) #run pagerank to get the scores group_graph = networkx.DiGraph() group_edges = [] for user_id in user_ids: for following_id in set(user_ids).intersection( follower_followies_map[user_id]): group_edges.append((user_id, following_id)) print len(user_ids), len(group_edges) if not group_edges: continue group_graph.add_edges_from(group_edges) pagerank = networkx.pagerank(group_graph, max_iter=500) scores = [] for user_id in user_ids: scores.append(pagerank.get(user_id, 0)) groups.append( TwitterGroup.create_group(twitter_reduction.id, i, user_ids, scores, postgres_handle)) postgres_handle.connection.commit() ########################################### ##makes for quicker queries in some cases ########################################### twitter_reduction.save_group_info(postgres_handle) postgres_handle.connection.commit() ########################################### ##mk_tag_clouds ########################################### TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle) postgres_handle.connection.commit()
def get_groups(self): from smarttypes.model.twitter_group import TwitterGroup return TwitterGroup.get_by_name_value('reduction_id', self.id, self.postgres_handle)
def reduce_graph(screen_name, distance=20, min_followers=60, pickle_it=True, just_load_from_file=False): postgres_handle = PostgresHandle(smarttypes.connection_string) # if just_load_from_file: # print "Loading data from a pickle." # gr = GraphReduce(screen_name, {}) # f = open(gr.pickle_file_path) # twitter_reduction, groups = pickle.load(f) # twitter_reduction.id = None # twitter_reduction.postgres_handle = postgres_handle # twitter_reduction.save() # postgres_handle.connection.commit() # for group in groups: # group.id = None # group.reduction_id = twitter_reduction.id # group.postgres_handle = postgres_handle # group.save() # postgres_handle.connection.commit() # TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle) # postgres_handle.connection.commit() # print "All done!" # return 0 ######################## ##reduce ######################## root_user = TwitterUser.by_screen_name(screen_name, postgres_handle) follower_followies_map = root_user.get_graph_info( distance=distance, min_followers=min_followers) gr = GraphReduce(screen_name, follower_followies_map) #gr.reduce_with_exafmm() gr.reduce_with_linloglayout() ######################## ##save reduction in db ######################## root_user_id = root_user.id user_ids = [] x_coordinates = [] y_coordinates = [] in_links = [] out_links = [] for i in range(len(gr.layout_ids)): user_id = gr.layout_ids[i] user_ids.append(user_id) x_coordinates.append(gr.reduction[i][0]) y_coordinates.append(gr.reduction[i][1]) itr_in_links = PostgresHandle.spliter.join(gr.G.predecessors(user_id)) itr_out_links = PostgresHandle.spliter.join(gr.G.successors(user_id)) in_links.append(itr_in_links) out_links.append(itr_out_links) twitter_reduction = TwitterReduction.create_reduction( root_user_id, user_ids, x_coordinates, y_coordinates, in_links, out_links, postgres_handle) postgres_handle.connection.commit() ######################## ##save groups in db ######################## groups = [] for i in range(gr.n_clusters): user_ids = [] for j in range(len(gr.layout_ids)): if gr.layout_clusters[j][i] > 0: user_ids.append(gr.layout_ids[j]) #run pagerank to get the scores group_graph = networkx.DiGraph() group_edges = [] for user_id in user_ids: if user_id in follower_followies_map: for following_id in set(user_ids).intersection( follower_followies_map[user_id]): group_edges.append((user_id, following_id)) print len(user_ids), len(group_edges) if not group_edges: continue group_graph.add_edges_from(group_edges) pagerank = networkx.pagerank(group_graph, max_iter=500) scores = [] for user_id in user_ids: scores.append(pagerank.get(user_id, 0)) groups.append( TwitterGroup.create_group(twitter_reduction.id, i, user_ids, scores, postgres_handle)) postgres_handle.connection.commit() twitter_reduction.save_group_info(postgres_handle) postgres_handle.connection.commit() ######################## ##mk_tag_clouds ######################## TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle) postgres_handle.connection.commit()