Beispiel #1
0
def index(req, session, postgres_handle):

    root_user = None
    if 'user_id' in req.params:
        root_user = TwitterUser.get_by_id(req.params['user_id'],
                                          postgres_handle)
    if not root_user:
        root_user = TwitterUser.by_screen_name('SmartTypes', postgres_handle)

    reduction = TwitterReduction.get_latest_reduction(root_user.id,
                                                      postgres_handle)
    if not reduction:
        root_user = TwitterUser.by_screen_name('SmartTypes', postgres_handle)
        reduction = TwitterReduction.get_latest_reduction(
            root_user.id, postgres_handle)

    return {
        'active_tab':
        'social_map',
        'template_path':
        'social_map/index.html',
        'root_user':
        root_user,
        'reduction':
        reduction,
        'num_groups':
        len(TwitterGroup.all_groups(reduction.id, postgres_handle)),
        'users_with_a_reduction':
        TwitterReduction.get_users_with_a_reduction(postgres_handle),
    }
Beispiel #2
0
def next_or_previous_reduction_id(req, session, postgres_handle):
    reduction = None
    if 'reduction_id' in req.params:
        try:
            reduction_id = int(req.params['reduction_id'])
        except ValueError:
            reduction_id = -1
        reduction = TwitterReduction.get_by_id(reduction_id, postgres_handle)
    #prev / next reduction
    new_reduction_id = -1
    if reduction and 'next_or_previous' in req.params:
        if req.params['next_or_previous'] in ['prev_reduction', 'next_reduction']:
            ordered_reduction_list = TwitterReduction.get_ordered_id_list(
                reduction.root_user_id, postgres_handle)
            for i in range(len(ordered_reduction_list)):
                if reduction.id == ordered_reduction_list[i]:
                    current_idx = i
                    break
            if req.params['next_or_previous'] == 'prev_reduction':
                idx = current_idx - 1
            if req.params['next_or_previous'] == 'next_reduction':
                idx = current_idx + 1
            new_reduction_id = ordered_reduction_list[idx]
    return {
        'content_type': 'application/json',
        'json': {
            'reduction_id': new_reduction_id,
            'num_groups': len(TwitterGroup.all_groups(new_reduction_id, postgres_handle))
        }
    }
Beispiel #3
0
def index(req, session, postgres_handle):
    #if it's not a valid request keep reduction_id None
    #don't do work for bots that don't know what they're
    #looking for
    reduction = None
    user_reduction_counts = TwitterReduction.get_user_reduction_counts(
        postgres_handle)
    random.shuffle(user_reduction_counts)

    if req.path.split('/') > 1 and req.path.split(
            '/')[1]:  # path looks like '/something'
        root_user = TwitterUser.by_screen_name(
            req.path.split('/')[1], postgres_handle)
        if root_user:
            reduction = TwitterReduction.get_latest_reduction(
                root_user.id, postgres_handle)
        if not reduction and is_int(req.path.split('/')[1]):
            reduction = TwitterReduction.get_by_id(
                req.path.split('/')[1], postgres_handle)
    else:
        reduction = TwitterReduction.get_latest_reduction(
            user_reduction_counts[0][0].id, postgres_handle)

    return {
        'reduction_id':
        reduction.id
        if reduction and reduction.tiles_are_written_to_disk else None,
        'reduction':
        reduction
        if reduction and reduction.tiles_are_written_to_disk else None,
        'user_reduction_counts':
        user_reduction_counts
    }
Beispiel #4
0
def community_features(req, session, postgres_handle):
    reduction = None
    if req.path.split('/') > 3 and req.path.split('/')[3]:  # path looks like '/social_map/community_features/something'
        root_user = TwitterUser.by_screen_name(req.path.split('/')[3], postgres_handle)
        if root_user:
            reduction = TwitterReduction.get_latest_reduction(root_user.id, postgres_handle)
        if not reduction and is_int(req.path.split('/')[3]):
            reduction = TwitterReduction.get_by_id(req.path.split('/')[3], postgres_handle)
    return {
        'content_type': 'application/json',
        'json':reduction.get_geojson_community_features() if reduction else [],
    }
Beispiel #5
0
def index(req, session, postgres_handle):
    root_user = None
    if 'user_id' in req.params:
        root_user = TwitterUser.get_by_id(req.params['user_id'], postgres_handle)
    if not root_user:
        root_user = TwitterUser.by_screen_name('SmartTypes', postgres_handle)
    reduction = TwitterReduction.get_latest_reduction(root_user.id, postgres_handle)
    if not reduction:
        root_user = TwitterUser.by_screen_name('SmartTypes', postgres_handle)
        reduction = TwitterReduction.get_latest_reduction(root_user.id, postgres_handle)
    return {
        'active_tab': 'social_map',
        'template_path': 'social_map/index.html',
        'root_user': root_user,
        'reduction': reduction,
        'num_groups': len(TwitterGroup.all_groups(reduction.id, postgres_handle)),
        'users_with_a_reduction': TwitterReduction.get_users_with_a_reduction(postgres_handle),
    }
Beispiel #6
0
def group_details(req, session, postgres_handle):
    if 'group_index' in req.params and 'reduction_id' in req.params:
        reduction = TwitterReduction.get_by_id(req.params['reduction_id'], postgres_handle)
        group_index = int(req.params['group_index'])
        twitter_group = TwitterGroup.get_by_index(reduction.id, group_index, postgres_handle)
    else:
        twitter_group = None
    return {
        'template_path': 'social_map/group_details.html',
        'twitter_group': twitter_group,
    }
Beispiel #7
0
def group_details(req, session, postgres_handle):
    if 'group_index' in req.params and 'reduction_id' in req.params:
        reduction = TwitterReduction.get_by_id(req.params['reduction_id'],
                                               postgres_handle)
        group_index = int(req.params['group_index'])
        twitter_group = TwitterGroup.get_by_index(reduction.id, group_index,
                                                  postgres_handle)
    else:
        twitter_group = None
    return {
        'template_path': 'social_map/group_details.html',
        'twitter_group': twitter_group,
    }
Beispiel #8
0
def node_details(req, session, postgres_handle):
    twitter_user, in_links, out_links = None, [], []
    if 'node_id' in req.params and 'reduction_id' in req.params:
        reduction = TwitterReduction.get_by_id(req.params['reduction_id'], postgres_handle)
        twitter_user = TwitterUser.get_by_id(req.params['node_id'], postgres_handle)
        if twitter_user:
            in_links, out_links = reduction.get_in_and_out_links_for_user(req.params['node_id'])
    return {
        'template_path': 'social_map/node_details.html',
        'twitter_user': twitter_user,
        'in_links':in_links,
        'out_links':out_links,
    }
Beispiel #9
0
def index(req, session, postgres_handle):
    #if it's not a valid request keep reduction_id None
    #don't do work for bots that don't know what they're 
    #looking for
    reduction = None
    user_reduction_counts = TwitterReduction.get_user_reduction_counts(postgres_handle)
    random.shuffle(user_reduction_counts)

    if req.path.split('/') > 1 and req.path.split('/')[1]:  # path looks like '/something'
        root_user = TwitterUser.by_screen_name(req.path.split('/')[1], postgres_handle)
        if root_user:
            reduction = TwitterReduction.get_latest_reduction(root_user.id, postgres_handle)
        if not reduction and is_int(req.path.split('/')[1]):
            reduction = TwitterReduction.get_by_id(req.path.split('/')[1], postgres_handle)
    else:
        reduction = TwitterReduction.get_latest_reduction(user_reduction_counts[0][0].id, postgres_handle)

    return {
        'reduction_id': reduction.id if reduction and reduction.tiles_are_written_to_disk else None,
        'reduction': reduction if reduction and reduction.tiles_are_written_to_disk else None,
        'user_reduction_counts': user_reduction_counts
    }
Beispiel #10
0
def next_or_previous_reduction_id(req, session, postgres_handle):

    reduction = None
    if 'reduction_id' in req.params:
        try:
            reduction_id = int(req.params['reduction_id'])
        except ValueError:
            reduction_id = -1
        reduction = TwitterReduction.get_by_id(reduction_id, postgres_handle)

    #prev / next reduction
    new_reduction_id = -1
    if reduction and 'next_or_previous' in req.params:
        if req.params['next_or_previous'] in [
                'prev_reduction', 'next_reduction'
        ]:
            ordered_reduction_list = TwitterReduction.get_ordered_id_list(
                reduction.root_user_id, postgres_handle)
            for i in range(len(ordered_reduction_list)):
                if reduction.id == ordered_reduction_list[i]:
                    current_idx = i
                    break
            if req.params['next_or_previous'] == 'prev_reduction':
                idx = current_idx - 1
            if req.params['next_or_previous'] == 'next_reduction':
                idx = current_idx + 1
            new_reduction_id = ordered_reduction_list[idx]

    return {
        'content_type': 'application/json',
        'json': {
            'reduction_id':
            new_reduction_id,
            'num_groups':
            len(TwitterGroup.all_groups(new_reduction_id, postgres_handle))
        }
    }
Beispiel #11
0
def map_data(req, session, postgres_handle):
    reduction = None
    if 'reduction_id' in req.params:
        try:
            reduction_id = int(req.params['reduction_id'])
        except ValueError:
            reduction_id = -1
        reduction = TwitterReduction.get_by_id(reduction_id, postgres_handle)

    #details
    reduction_details = []
    if reduction:
        return_all = not smarttypes.config.IS_PROD  # for debugging
        #return_all = False
        reduction_details = reduction.get_details(return_all=return_all)

    return {'content_type': 'application/json', 'json': reduction_details}
Beispiel #12
0
def node_details(req, session, postgres_handle):

    twitter_user, in_links, out_links = None, [], []
    if 'node_id' in req.params and 'reduction_id' in req.params:
        reduction = TwitterReduction.get_by_id(req.params['reduction_id'],
                                               postgres_handle)
        twitter_user = TwitterUser.get_by_id(req.params['node_id'],
                                             postgres_handle)
        if twitter_user:
            in_links, out_links = reduction.get_in_and_out_links_for_user(
                req.params['node_id'])
    return {
        'template_path': 'social_map/node_details.html',
        'twitter_user': twitter_user,
        'in_links': in_links,
        'out_links': out_links,
    }
Beispiel #13
0
def map_data(req, session, postgres_handle):
    reduction = None
    if 'reduction_id' in req.params:
        try:
            reduction_id = int(req.params['reduction_id'])
        except ValueError:
            reduction_id = -1
        reduction = TwitterReduction.get_by_id(reduction_id, postgres_handle)

    #details
    reduction_details = []
    if reduction:
        return_all = not smarttypes.config.IS_PROD  # for debugging
        #return_all = False
        reduction_details = reduction.get_details(return_all=return_all)

    return {
        'content_type': 'application/json',
        'json': reduction_details
    }
Beispiel #14
0
def map_data(req, session, postgres_handle):
    reduction = None
    if 'reduction_id' in req.params:
        try:
            reduction_id = int(req.params['reduction_id'])
        except ValueError:
            reduction_id = -1
        reduction = TwitterReduction.get_by_id(reduction_id, postgres_handle)
    reduction_info = {
        'coordinates':[],
        'min_coord':0,
        'max_coord':1,
    }
    if reduction:
        return_all = not smarttypes.config.IS_PROD  # for debugging
        #return_all = False
        reduction_info = reduction.get_reduction_info(return_all=return_all)
    return {
        'content_type': 'application/json',
        'json': reduction_info
    }
from datetime import datetime
import smarttypes, sys, string
from smarttypes.config import *
from smarttypes.model.twitter_credentials import TwitterCredentials
from smarttypes.model.twitter_user import TwitterUser
from smarttypes.model.twitter_reduction import TwitterReduction

from smarttypes.utils.postgres_handle import PostgresHandle
postgres_handle = PostgresHandle(smarttypes.connection_string)

if __name__ == "__main__":

    user_count_list = TwitterReduction.get_user_reduction_counts(
        postgres_handle)
    for user, count in user_count_list:
        if count == 1:
            creds = TwitterCredentials.get_by_twitter_id(
                user.id, postgres_handle)
            creds.maybe_send_initial_map_email()
Beispiel #16
0
        queue.put(None)
    # wait for pending rendering jobs to complete
    queue.join()
    for i in range(num_threads):
        renderers[i].join()


if __name__ == "__main__":

    postgres_handle = PostgresHandle(smarttypes.connection_string)
    #get reduction_id
    qry = """
    select tr.id 
    from twitter_reduction tr
    where tr.tiles_are_written_to_disk = False
    order by tr.id desc limit 1;
    """
    reduction_id = postgres_handle.execute_query(qry, {})[0]['id']
    tile_dir = '../static/tiles/%s/' % reduction_id
    if not os.path.isdir(tile_dir):
        os.mkdir(tile_dir)
    style_file = 'mapnik.xml'
    min_zoom = 0
    max_zoom = 5
    bbox = (-180, -85.0511, 180, 85.0511)
    render_tiles(bbox, style_file, tile_dir, min_zoom, max_zoom)
    reduction = TwitterReduction.get_by_id(reduction_id, postgres_handle)
    reduction.tiles_are_written_to_disk = True
    reduction.save()
    postgres_handle.connection.commit()
Beispiel #17
0
    g.vs['size'] = 1

    #plot to file
    layout = Layout(layout_list)
    filepath = 'io/%s.png' % root_user.screen_name
    thumb_filepath = 'io/%s_thumb.png' % root_user.screen_name
    plot_graph(g, layout, filepath, size_tup=(600, 600))
    #need to adjust vs['size'] if i want to do this
    #plot_graph(g, layout, thumb_filepath, size_tup=(50, 50))
    if not smarttypes.config.IS_PROD:
        os.system('cp io/%s*.png /home/timmyt/projects/smarttypes/smarttypes/static/images/maps/.' % root_user.screen_name)
    else:
        os.system('scp io/%s*.png cottie:/home/timmyt/projects/smarttypes/smarttypes/static/images/maps/.' % root_user.screen_name)
        
        print 'save to disk'
        twitter_reduction = TwitterReduction.create_reduction(root_user.id, postgres_handle)
        postgres_handle.connection.commit()
        for community_idx, values_dict in community_stats.items():
            #params:
            #reduction_id, index, center_coordinate, member_ids, 
            #global_pagerank, community_pagerank, hybrid_pagerank
            if community_idx > 0:
                TwitterCommunity.create_community(twitter_reduction.id, community_idx, 
                    values_dict['center_coordinate'], values_dict['member_ids'], values_dict['global_pagerank'], 
                    values_dict['community_pagerank'], values_dict['hybrid_pagerank'], postgres_handle)
            postgres_handle.connection.commit()
        TwitterCommunity.mk_tag_clouds(twitter_reduction.id, postgres_handle)
        postgres_handle.connection.commit()

    #how long
    print datetime.now() - start_time
def reduce_graph(screen_name, distance=20, min_followers=60):

    postgres_handle = PostgresHandle(smarttypes.connection_string)

    ###########################################
    ##reduce
    ###########################################
    root_user = TwitterUser.by_screen_name(screen_name, postgres_handle)
    follower_followies_map = root_user.get_graph_info(distance=distance, 
        min_followers=min_followers)
    gr = GraphReduce(screen_name, follower_followies_map)
    gr.reduce_with_linloglayout()

    ###########################################
    ##save reduction in db
    ###########################################
    root_user_id = root_user.id
    user_ids = []
    x_coordinates = []
    y_coordinates = []
    in_links = []
    out_links = []
    for i in range(len(gr.layout_ids)):
        user_id = gr.layout_ids[i]
        user_ids.append(user_id)
        x_coordinates.append(gr.reduction[i][0])
        y_coordinates.append(gr.reduction[i][1])
        itr_in_links = PostgresHandle.spliter.join(gr.G.predecessors(user_id))
        itr_out_links = PostgresHandle.spliter.join(gr.G.successors(user_id))
        in_links.append(itr_in_links)
        out_links.append(itr_out_links)
    twitter_reduction = TwitterReduction.create_reduction(root_user_id, user_ids,
        x_coordinates, y_coordinates, in_links, out_links, postgres_handle)
    postgres_handle.connection.commit()

    ###########################################
    ##save groups in db
    ###########################################
    groups = []
    for i in range(gr.n_groups):
        user_ids = []
        for j in range(len(gr.layout_ids)):
            if i == gr.groups[j]:
                user_ids.append(gr.layout_ids[j])
        #run pagerank to get the scores
        group_graph = networkx.DiGraph()
        group_edges = []
        for user_id in user_ids:
            for following_id in set(user_ids).intersection(follower_followies_map[user_id]):
                group_edges.append((user_id, following_id))
        print len(user_ids), len(group_edges)
        if not group_edges:
            continue
        group_graph.add_edges_from(group_edges)
        pagerank = networkx.pagerank(group_graph, max_iter=500)
        scores = []
        for user_id in user_ids:
            scores.append(pagerank.get(user_id, 0))
        groups.append(TwitterGroup.create_group(twitter_reduction.id, i, user_ids, scores,
            postgres_handle))
    postgres_handle.connection.commit()

    ###########################################
    ##makes for quicker queries in some cases
    ###########################################
    twitter_reduction.save_group_info(postgres_handle)
    postgres_handle.connection.commit()

    ###########################################
    ##mk_tag_clouds
    ###########################################
    TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle)
    postgres_handle.connection.commit()
Beispiel #19
0
    #coordinates = reduce_with_linloglayout(g, root_user)
    coordinates = reduce_with_semi_intelligent_agents(g)
    
    #id_communities
    vertex_clustering = id_communities(g, coordinates)

    #do this after community detection because it causes distortion
    coordinates = reproject_to_spherical_mercator(coordinates)

    #network_stats
    network_stats = get_network_stats(network, g, vertex_clustering)
    global_pagerank, community_pagerank, community_score = network_stats
    hybrid_pagerank = calculate_hybrid_pagerank(global_pagerank, community_pagerank, community_score)

    print "save reduction"
    reduction = TwitterReduction.create_reduction(root_user.id, [0, 0, 0], False, postgres_handle)
    postgres_handle.connection.commit()

    print "save reduction users"
    reduction_users = []
    for i in range(len(member_ids)):
        tru = TwitterReductionUser(postgres_handle=postgres_handle)
        tru.reduction_id = reduction.id
        tru.user_id = member_ids[i]
        tru.coordinates = Point(coordinates[i][0], coordinates[i][1])
        tru.pagerank = global_pagerank[i]
        tru.hybrid_pagerank = hybrid_pagerank[i]
        reduction_users.append(tru.save())
        postgres_handle.connection.commit()

    print "save communities"
from datetime import datetime
import smarttypes, sys, string
from smarttypes.config import *
from smarttypes.model.twitter_credentials import TwitterCredentials
from smarttypes.model.twitter_user import TwitterUser
from smarttypes.model.twitter_reduction import TwitterReduction

from smarttypes.utils.postgres_handle import PostgresHandle

postgres_handle = PostgresHandle(smarttypes.connection_string)


if __name__ == "__main__":

    user_count_list = TwitterReduction.get_user_reduction_counts(postgres_handle)
    for user, count in user_count_list:
        if count == 1:
            creds = TwitterCredentials.get_by_twitter_id(user.id, postgres_handle)
            creds.maybe_send_initial_map_email()
Beispiel #21
0
def reduce_graph(screen_name, distance=20, min_followers=60):

    postgres_handle = PostgresHandle(smarttypes.connection_string)

    ###########################################
    ##reduce
    ###########################################
    root_user = TwitterUser.by_screen_name(screen_name, postgres_handle)
    follower_followies_map = root_user.get_graph_info(
        distance=distance, min_followers=min_followers)
    gr = GraphReduce(screen_name, follower_followies_map)
    gr.reduce_with_linloglayout()

    ###########################################
    ##save reduction in db
    ###########################################
    root_user_id = root_user.id
    user_ids = []
    x_coordinates = []
    y_coordinates = []
    in_links = []
    out_links = []
    for i in range(len(gr.layout_ids)):
        user_id = gr.layout_ids[i]
        user_ids.append(user_id)
        x_coordinates.append(gr.reduction[i][0])
        y_coordinates.append(gr.reduction[i][1])
        itr_in_links = PostgresHandle.spliter.join(gr.G.predecessors(user_id))
        itr_out_links = PostgresHandle.spliter.join(gr.G.successors(user_id))
        in_links.append(itr_in_links)
        out_links.append(itr_out_links)
    twitter_reduction = TwitterReduction.create_reduction(
        root_user_id, user_ids, x_coordinates, y_coordinates, in_links,
        out_links, postgres_handle)
    postgres_handle.connection.commit()

    ###########################################
    ##save groups in db
    ###########################################
    groups = []
    for i in range(gr.n_groups):
        user_ids = []
        for j in range(len(gr.layout_ids)):
            if i == gr.groups[j]:
                user_ids.append(gr.layout_ids[j])
        #run pagerank to get the scores
        group_graph = networkx.DiGraph()
        group_edges = []
        for user_id in user_ids:
            for following_id in set(user_ids).intersection(
                    follower_followies_map[user_id]):
                group_edges.append((user_id, following_id))
        print len(user_ids), len(group_edges)
        if not group_edges:
            continue
        group_graph.add_edges_from(group_edges)
        pagerank = networkx.pagerank(group_graph, max_iter=500)
        scores = []
        for user_id in user_ids:
            scores.append(pagerank.get(user_id, 0))
        groups.append(
            TwitterGroup.create_group(twitter_reduction.id, i, user_ids,
                                      scores, postgres_handle))
    postgres_handle.connection.commit()

    ###########################################
    ##makes for quicker queries in some cases
    ###########################################
    twitter_reduction.save_group_info(postgres_handle)
    postgres_handle.connection.commit()

    ###########################################
    ##mk_tag_clouds
    ###########################################
    TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle)
    postgres_handle.connection.commit()
Beispiel #22
0
if __name__ == "__main__":
       
    postgres_handle = PostgresHandle(smarttypes.connection_string)
    #get reduction_id 
    qry = """
    select tr.id 
    from twitter_reduction tr
    where tr.tiles_are_written_to_disk = False
    order by tr.id desc limit 1;
    """
    reduction_id = postgres_handle.execute_query(qry, {})[0]['id']
    tile_dir = '../static/tiles/%s/' % reduction_id
    if not os.path.isdir(tile_dir):
        os.mkdir(tile_dir)
    style_file = 'mapnik.xml'
    min_zoom = 0
    max_zoom = 5
    bbox = (-180, -85.0511, 180, 85.0511)
    render_tiles(bbox, style_file, tile_dir, min_zoom, max_zoom)
    reduction = TwitterReduction.get_by_id(reduction_id, postgres_handle)
    reduction.tiles_are_written_to_disk = True
    reduction.save()
    postgres_handle.connection.commit()






Beispiel #23
0
 def get_latest_reduction(self):
     from smarttypes.model.twitter_reduction import TwitterReduction
     return TwitterReduction.get_latest_reduction(self.id,
                                                  self.postgres_handle)
Beispiel #24
0
def reduce_graph(screen_name,
                 distance=20,
                 min_followers=60,
                 pickle_it=True,
                 just_load_from_file=False):

    postgres_handle = PostgresHandle(smarttypes.connection_string)

    # if just_load_from_file:
    #     print "Loading data from a pickle."
    #     gr = GraphReduce(screen_name, {})
    #     f = open(gr.pickle_file_path)
    #     twitter_reduction, groups = pickle.load(f)
    #     twitter_reduction.id = None
    #     twitter_reduction.postgres_handle = postgres_handle
    #     twitter_reduction.save()
    #     postgres_handle.connection.commit()
    #     for group in groups:
    #         group.id = None
    #         group.reduction_id = twitter_reduction.id
    #         group.postgres_handle = postgres_handle
    #         group.save()
    #         postgres_handle.connection.commit()
    #     TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle)
    #     postgres_handle.connection.commit()
    #     print "All done!"
    #     return 0

    ########################
    ##reduce
    ########################
    root_user = TwitterUser.by_screen_name(screen_name, postgres_handle)
    follower_followies_map = root_user.get_graph_info(
        distance=distance, min_followers=min_followers)
    gr = GraphReduce(screen_name, follower_followies_map)
    #gr.reduce_with_exafmm()
    gr.reduce_with_linloglayout()

    ########################
    ##save reduction in db
    ########################
    root_user_id = root_user.id
    user_ids = []
    x_coordinates = []
    y_coordinates = []
    in_links = []
    out_links = []
    for i in range(len(gr.layout_ids)):
        user_id = gr.layout_ids[i]
        user_ids.append(user_id)
        x_coordinates.append(gr.reduction[i][0])
        y_coordinates.append(gr.reduction[i][1])
        itr_in_links = PostgresHandle.spliter.join(gr.G.predecessors(user_id))
        itr_out_links = PostgresHandle.spliter.join(gr.G.successors(user_id))
        in_links.append(itr_in_links)
        out_links.append(itr_out_links)
    twitter_reduction = TwitterReduction.create_reduction(
        root_user_id, user_ids, x_coordinates, y_coordinates, in_links,
        out_links, postgres_handle)
    postgres_handle.connection.commit()

    ########################
    ##save groups in db
    ########################
    groups = []
    for i in range(gr.n_clusters):
        user_ids = []
        for j in range(len(gr.layout_ids)):
            if gr.layout_clusters[j][i] > 0:
                user_ids.append(gr.layout_ids[j])
        #run pagerank to get the scores
        group_graph = networkx.DiGraph()
        group_edges = []
        for user_id in user_ids:
            if user_id in follower_followies_map:
                for following_id in set(user_ids).intersection(
                        follower_followies_map[user_id]):
                    group_edges.append((user_id, following_id))
        print len(user_ids), len(group_edges)
        if not group_edges:
            continue
        group_graph.add_edges_from(group_edges)
        pagerank = networkx.pagerank(group_graph, max_iter=500)
        scores = []
        for user_id in user_ids:
            scores.append(pagerank.get(user_id, 0))
        groups.append(
            TwitterGroup.create_group(twitter_reduction.id, i, user_ids,
                                      scores, postgres_handle))
    postgres_handle.connection.commit()

    twitter_reduction.save_group_info(postgres_handle)
    postgres_handle.connection.commit()

    ########################
    ##mk_tag_clouds
    ########################
    TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle)
    postgres_handle.connection.commit()
Beispiel #25
0
    #id_communities
    vertex_clustering = id_communities(g, coordinates)

    #do this after community detection because it causes distortion
    coordinates = reproject_to_spherical_mercator(coordinates)

    #network_stats
    network_stats = get_network_stats(network, g, vertex_clustering)
    global_pagerank, community_pagerank, community_score = network_stats
    hybrid_pagerank = calculate_hybrid_pagerank(global_pagerank,
                                                community_pagerank,
                                                community_score)

    print "save reduction"
    reduction = TwitterReduction.create_reduction(root_user.id, [0, 0, 0],
                                                  False, postgres_handle)
    postgres_handle.connection.commit()

    print "save reduction users"
    reduction_users = []
    for i in range(len(member_ids)):
        tru = TwitterReductionUser(postgres_handle=postgres_handle)
        tru.reduction_id = reduction.id
        tru.user_id = member_ids[i]
        tru.coordinates = Point(coordinates[i][0], coordinates[i][1])
        tru.pagerank = global_pagerank[i]
        tru.hybrid_pagerank = hybrid_pagerank[i]
        reduction_users.append(tru.save())
        postgres_handle.connection.commit()

    print "save communities"
Beispiel #26
0
 def get_latest_reduction(self):
     from smarttypes.model.twitter_reduction import TwitterReduction
     return TwitterReduction.get_latest_reduction(self.id, self.postgres_handle)