def group(request):
if 'group_index' in request.params:
group_index = int(request.params['group_index'])
twitter_group = TwitterGroup.get_by_index(group_index)
else:
twitter_group = TwitterGroup.get_random_group()
return {
'twitter_group':twitter_group,
}
def next_or_previous_reduction_id(req, session, postgres_handle):
reduction = None
if 'reduction_id' in req.params:
try:
reduction_id = int(req.params['reduction_id'])
except ValueError:
reduction_id = -1
reduction = TwitterReduction.get_by_id(reduction_id, postgres_handle)
#prev / next reduction
new_reduction_id = -1
if reduction and 'next_or_previous' in req.params:
if req.params['next_or_previous'] in ['prev_reduction', 'next_reduction']:
ordered_reduction_list = TwitterReduction.get_ordered_id_list(
reduction.root_user_id, postgres_handle)
for i in range(len(ordered_reduction_list)):
if reduction.id == ordered_reduction_list[i]:
current_idx = i
break
if req.params['next_or_previous'] == 'prev_reduction':
idx = current_idx - 1
if req.params['next_or_previous'] == 'next_reduction':
idx = current_idx + 1
new_reduction_id = ordered_reduction_list[idx]
return {
'content_type': 'application/json',
'json': {
'reduction_id': new_reduction_id,
'num_groups': len(TwitterGroup.all_groups(new_reduction_id, postgres_handle))
}
}
def index(req, session, postgres_handle):
root_user = None
if 'user_id' in req.params:
root_user = TwitterUser.get_by_id(req.params['user_id'],
postgres_handle)
if not root_user:
root_user = TwitterUser.by_screen_name('SmartTypes', postgres_handle)
reduction = TwitterReduction.get_latest_reduction(root_user.id,
postgres_handle)
if not reduction:
root_user = TwitterUser.by_screen_name('SmartTypes', postgres_handle)
reduction = TwitterReduction.get_latest_reduction(
root_user.id, postgres_handle)
return {
'active_tab':
'social_map',
'template_path':
'social_map/index.html',
'root_user':
root_user,
'reduction':
reduction,
'num_groups':
len(TwitterGroup.all_groups(reduction.id, postgres_handle)),
'users_with_a_reduction':
TwitterReduction.get_users_with_a_reduction(postgres_handle),
}
def top_groups(self, num_groups=10):
from smarttypes.model.twitter_group import TwitterGroup
return_list = []
i = 0
for score, group_id in sorted(self.scores_groups, reverse=True):
if i <= num_groups and score > .001:
return_list.append((score, TwitterGroup.get_by_index(group_id, self.postgres_handle)))
else:
break
i += 1
return return_list
def top_groups(self, num_groups=10):
from smarttypes.model.twitter_group import TwitterGroup
return_list = []
i = 0
for score, group_id in sorted(self.scores_groups, reverse=True):
if i <= num_groups and score > .001:
return_list.append((score, TwitterGroup.get_by_index(group_id, self.postgres_handle)))
else:
break
i += 1
return return_list
def group_details(req, session, postgres_handle):
if 'group_index' in req.params and 'reduction_id' in req.params:
reduction = TwitterReduction.get_by_id(req.params['reduction_id'], postgres_handle)
group_index = int(req.params['group_index'])
twitter_group = TwitterGroup.get_by_index(reduction.id, group_index, postgres_handle)
else:
twitter_group = None
return {
'template_path': 'social_map/group_details.html',
'twitter_group': twitter_group,
}
def group_details(req, session, postgres_handle):
if 'group_index' in req.params and 'reduction_id' in req.params:
reduction = TwitterReduction.get_by_id(req.params['reduction_id'],
postgres_handle)
group_index = int(req.params['group_index'])
twitter_group = TwitterGroup.get_by_index(reduction.id, group_index,
postgres_handle)
else:
twitter_group = None
return {
'template_path': 'social_map/group_details.html',
'twitter_group': twitter_group,
}
def index(req, session, postgres_handle):
root_user = None
if 'user_id' in req.params:
root_user = TwitterUser.get_by_id(req.params['user_id'], postgres_handle)
if not root_user:
root_user = TwitterUser.by_screen_name('SmartTypes', postgres_handle)
reduction = TwitterReduction.get_latest_reduction(root_user.id, postgres_handle)
if not reduction:
root_user = TwitterUser.by_screen_name('SmartTypes', postgres_handle)
reduction = TwitterReduction.get_latest_reduction(root_user.id, postgres_handle)
return {
'active_tab': 'social_map',
'template_path': 'social_map/index.html',
'root_user': root_user,
'reduction': reduction,
'num_groups': len(TwitterGroup.all_groups(reduction.id, postgres_handle)),
'users_with_a_reduction': TwitterReduction.get_users_with_a_reduction(postgres_handle),
}
def next_or_previous_reduction_id(req, session, postgres_handle):
reduction = None
if 'reduction_id' in req.params:
try:
reduction_id = int(req.params['reduction_id'])
except ValueError:
reduction_id = -1
reduction = TwitterReduction.get_by_id(reduction_id, postgres_handle)
#prev / next reduction
new_reduction_id = -1
if reduction and 'next_or_previous' in req.params:
if req.params['next_or_previous'] in [
'prev_reduction', 'next_reduction'
]:
ordered_reduction_list = TwitterReduction.get_ordered_id_list(
reduction.root_user_id, postgres_handle)
for i in range(len(ordered_reduction_list)):
if reduction.id == ordered_reduction_list[i]:
current_idx = i
break
if req.params['next_or_previous'] == 'prev_reduction':
idx = current_idx - 1
if req.params['next_or_previous'] == 'next_reduction':
idx = current_idx + 1
new_reduction_id = ordered_reduction_list[idx]
return {
'content_type': 'application/json',
'json': {
'reduction_id':
new_reduction_id,
'num_groups':
len(TwitterGroup.all_groups(new_reduction_id, postgres_handle))
}
}
for j in range(len(A[i])):
write_this = str(int(round(A[i][j])))
test_file.write(write_this+',')
test_file.write('\n')
group_adjacency = []
A = numpy.dot(users_data, numpy.transpose(items_data))
for i in range(num_features):
membership_scores = []
for j in range(num_features):
membership_scores.append((A[i][j] * A[j][i], j))
group_adjacency.append(membership_scores)
index_to_twitter_id_dict = pickle.load(open('index_to_twitter_id.pickle', 'r'))
user_group_map = {}
TwitterGroup.bulk_delete('all')
for i in range(num_features):
membership_scores = []
for j in range(num_items):
user_id = index_to_twitter_id_dict[j]
follower_score = users_data[i][j]
following_score = items_data[i][j]
membership_score = following_score * following_score
if membership_score > .001:
membership_scores.append((membership_score, user_id))
if user_id not in user_group_map:
user_group_map[user_id] = [(membership_score, i)]
else:
user_group_map[user_id].append((membership_score, i))
TwitterGroup.upsert_group(i, membership_scores, group_adjacency[i])
print "Done creating groups."
for j in range(len(A[i])):
write_this = str(int(round(A[i][j])))
test_file.write(write_this + ',')
test_file.write('\n')
group_adjacency = []
A = numpy.dot(users_data, numpy.transpose(items_data))
for i in range(num_features):
membership_scores = []
for j in range(num_features):
membership_scores.append((A[i][j] * A[j][i], j))
group_adjacency.append(membership_scores)
index_to_twitter_id_dict = pickle.load(open('index_to_twitter_id.pickle', 'r'))
user_group_map = {}
TwitterGroup.bulk_delete('all')
for i in range(num_features):
membership_scores = []
for j in range(num_items):
user_id = index_to_twitter_id_dict[j]
follower_score = users_data[i][j]
following_score = items_data[i][j]
membership_score = following_score * following_score
if membership_score > .001:
membership_scores.append((membership_score, user_id))
if user_id not in user_group_map:
user_group_map[user_id] = [(membership_score, i)]
else:
user_group_map[user_id].append((membership_score, i))
TwitterGroup.upsert_group(i, membership_scores, group_adjacency[i])
print "Done creating groups."
def reduce_graph(screen_name, distance=20, min_followers=60):
postgres_handle = PostgresHandle(smarttypes.connection_string)
###########################################
##reduce
###########################################
root_user = TwitterUser.by_screen_name(screen_name, postgres_handle)
follower_followies_map = root_user.get_graph_info(distance=distance,
min_followers=min_followers)
gr = GraphReduce(screen_name, follower_followies_map)
gr.reduce_with_linloglayout()
###########################################
##save reduction in db
###########################################
root_user_id = root_user.id
user_ids = []
x_coordinates = []
y_coordinates = []
in_links = []
out_links = []
for i in range(len(gr.layout_ids)):
user_id = gr.layout_ids[i]
user_ids.append(user_id)
x_coordinates.append(gr.reduction[i][0])
y_coordinates.append(gr.reduction[i][1])
itr_in_links = PostgresHandle.spliter.join(gr.G.predecessors(user_id))
itr_out_links = PostgresHandle.spliter.join(gr.G.successors(user_id))
in_links.append(itr_in_links)
out_links.append(itr_out_links)
twitter_reduction = TwitterReduction.create_reduction(root_user_id, user_ids,
x_coordinates, y_coordinates, in_links, out_links, postgres_handle)
postgres_handle.connection.commit()
###########################################
##save groups in db
###########################################
groups = []
for i in range(gr.n_groups):
user_ids = []
for j in range(len(gr.layout_ids)):
if i == gr.groups[j]:
user_ids.append(gr.layout_ids[j])
#run pagerank to get the scores
group_graph = networkx.DiGraph()
group_edges = []
for user_id in user_ids:
for following_id in set(user_ids).intersection(follower_followies_map[user_id]):
group_edges.append((user_id, following_id))
print len(user_ids), len(group_edges)
if not group_edges:
continue
group_graph.add_edges_from(group_edges)
pagerank = networkx.pagerank(group_graph, max_iter=500)
scores = []
for user_id in user_ids:
scores.append(pagerank.get(user_id, 0))
groups.append(TwitterGroup.create_group(twitter_reduction.id, i, user_ids, scores,
postgres_handle))
postgres_handle.connection.commit()
###########################################
##makes for quicker queries in some cases
###########################################
twitter_reduction.save_group_info(postgres_handle)
postgres_handle.connection.commit()
###########################################
##mk_tag_clouds
###########################################
TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle)
postgres_handle.connection.commit()
def reduce_graph(screen_name, distance=20, min_followers=60):
postgres_handle = PostgresHandle(smarttypes.connection_string)
###########################################
##reduce
###########################################
root_user = TwitterUser.by_screen_name(screen_name, postgres_handle)
follower_followies_map = root_user.get_graph_info(
distance=distance, min_followers=min_followers)
gr = GraphReduce(screen_name, follower_followies_map)
gr.reduce_with_linloglayout()
###########################################
##save reduction in db
###########################################
root_user_id = root_user.id
user_ids = []
x_coordinates = []
y_coordinates = []
in_links = []
out_links = []
for i in range(len(gr.layout_ids)):
user_id = gr.layout_ids[i]
user_ids.append(user_id)
x_coordinates.append(gr.reduction[i][0])
y_coordinates.append(gr.reduction[i][1])
itr_in_links = PostgresHandle.spliter.join(gr.G.predecessors(user_id))
itr_out_links = PostgresHandle.spliter.join(gr.G.successors(user_id))
in_links.append(itr_in_links)
out_links.append(itr_out_links)
twitter_reduction = TwitterReduction.create_reduction(
root_user_id, user_ids, x_coordinates, y_coordinates, in_links,
out_links, postgres_handle)
postgres_handle.connection.commit()
###########################################
##save groups in db
###########################################
groups = []
for i in range(gr.n_groups):
user_ids = []
for j in range(len(gr.layout_ids)):
if i == gr.groups[j]:
user_ids.append(gr.layout_ids[j])
#run pagerank to get the scores
group_graph = networkx.DiGraph()
group_edges = []
for user_id in user_ids:
for following_id in set(user_ids).intersection(
follower_followies_map[user_id]):
group_edges.append((user_id, following_id))
print len(user_ids), len(group_edges)
if not group_edges:
continue
group_graph.add_edges_from(group_edges)
pagerank = networkx.pagerank(group_graph, max_iter=500)
scores = []
for user_id in user_ids:
scores.append(pagerank.get(user_id, 0))
groups.append(
TwitterGroup.create_group(twitter_reduction.id, i, user_ids,
scores, postgres_handle))
postgres_handle.connection.commit()
###########################################
##makes for quicker queries in some cases
###########################################
twitter_reduction.save_group_info(postgres_handle)
postgres_handle.connection.commit()
###########################################
##mk_tag_clouds
###########################################
TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle)
postgres_handle.connection.commit()
def get_groups(self):
from smarttypes.model.twitter_group import TwitterGroup
return TwitterGroup.get_by_name_value('reduction_id', self.id,
self.postgres_handle)
def reduce_graph(screen_name,
distance=20,
min_followers=60,
pickle_it=True,
just_load_from_file=False):
postgres_handle = PostgresHandle(smarttypes.connection_string)
# if just_load_from_file:
# print "Loading data from a pickle."
# gr = GraphReduce(screen_name, {})
# f = open(gr.pickle_file_path)
# twitter_reduction, groups = pickle.load(f)
# twitter_reduction.id = None
# twitter_reduction.postgres_handle = postgres_handle
# twitter_reduction.save()
# postgres_handle.connection.commit()
# for group in groups:
# group.id = None
# group.reduction_id = twitter_reduction.id
# group.postgres_handle = postgres_handle
# group.save()
# postgres_handle.connection.commit()
# TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle)
# postgres_handle.connection.commit()
# print "All done!"
# return 0
########################
##reduce
########################
root_user = TwitterUser.by_screen_name(screen_name, postgres_handle)
follower_followies_map = root_user.get_graph_info(
distance=distance, min_followers=min_followers)
gr = GraphReduce(screen_name, follower_followies_map)
#gr.reduce_with_exafmm()
gr.reduce_with_linloglayout()
########################
##save reduction in db
########################
root_user_id = root_user.id
user_ids = []
x_coordinates = []
y_coordinates = []
in_links = []
out_links = []
for i in range(len(gr.layout_ids)):
user_id = gr.layout_ids[i]
user_ids.append(user_id)
x_coordinates.append(gr.reduction[i][0])
y_coordinates.append(gr.reduction[i][1])
itr_in_links = PostgresHandle.spliter.join(gr.G.predecessors(user_id))
itr_out_links = PostgresHandle.spliter.join(gr.G.successors(user_id))
in_links.append(itr_in_links)
out_links.append(itr_out_links)
twitter_reduction = TwitterReduction.create_reduction(
root_user_id, user_ids, x_coordinates, y_coordinates, in_links,
out_links, postgres_handle)
postgres_handle.connection.commit()
########################
##save groups in db
########################
groups = []
for i in range(gr.n_clusters):
user_ids = []
for j in range(len(gr.layout_ids)):
if gr.layout_clusters[j][i] > 0:
user_ids.append(gr.layout_ids[j])
#run pagerank to get the scores
group_graph = networkx.DiGraph()
group_edges = []
for user_id in user_ids:
if user_id in follower_followies_map:
for following_id in set(user_ids).intersection(
follower_followies_map[user_id]):
group_edges.append((user_id, following_id))
print len(user_ids), len(group_edges)
if not group_edges:
continue
group_graph.add_edges_from(group_edges)
pagerank = networkx.pagerank(group_graph, max_iter=500)
scores = []
for user_id in user_ids:
scores.append(pagerank.get(user_id, 0))
groups.append(
TwitterGroup.create_group(twitter_reduction.id, i, user_ids,
scores, postgres_handle))
postgres_handle.connection.commit()
twitter_reduction.save_group_info(postgres_handle)
postgres_handle.connection.commit()
########################
##mk_tag_clouds
########################
TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle)
postgres_handle.connection.commit()
def get_groups(self):
from smarttypes.model.twitter_group import TwitterGroup
return TwitterGroup.get_by_name_value('reduction_id', self.id, self.postgres_handle)
