def write_distance_matrix_to_csv():
postgres_handle = PostgresHandle(smarttypes.connection_string)
network = load_network_from_the_db(postgres_handle, 5)
landmarks = get_landmarks(network)
similarity_matrix = mk_similarity_matrix(network, landmarks)
distance_matrix = dist_euclidean(similarity_matrix)
write_similarity_matrix_to_csv(distance_matrix)
def pull_some_users(access_key):
postgres_handle = PostgresHandle(smarttypes.connection_string)
creds = TwitterCredentials.get_by_access_key(access_key, postgres_handle)
root_user = load_user_and_the_people_they_follow(creds, creds.root_user_id,
postgres_handle)
load_this_user_id = root_user.get_id_of_someone_in_my_network_to_load()
while load_this_user_id:
load_user_and_the_people_they_follow(creds, load_this_user_id,
postgres_handle)
load_this_user_id = root_user.get_id_of_someone_in_my_network_to_load()
#load_this_user_id = None
print "Finished loading all related users for %s!" % root_user.screen_name
def application(environ, start_response):
path = environ.get('PATH_INFO', '').lstrip('/')
for regex, controller in urls:
match = re.search(regex, path)
if match:
request = Request(environ)
try:
postgres_handle = PostgresHandle(smarttypes.connection_string)
try:
session = None
if request.cookies.get('session'):
session = TwitterSession.get_by_request_key(
request.cookies['session'], postgres_handle)
response_dict = controller(request, session,
postgres_handle)
web_response = WebResponse(request, controller.__name__,
response_dict, session)
response_headers = web_response.get_response_headers()
response_string = web_response.get_response_str()
if getattr(postgres_handle, '_connection', False):
postgres_handle.connection.commit()
status_code = '200 OK'
except RedirectException, (redirect_ex):
if getattr(postgres_handle, '_connection', False):
postgres_handle.connection.commit()
status_code = '303 See Other'
response_headers = [('Location', redirect_ex.redirect_url)]
response_string = [""]
except:
if getattr(postgres_handle, '_connection', False):
postgres_handle.connection.rollback()
raise
finally:
if getattr(postgres_handle, '_connection', False):
postgres_handle.connection.close()
#start response
start_response(status_code, response_headers)
return response_string
except Exception:
#can't use print statements with mod_wsgi
error_string = traceback.format_exc()
start_response('500 Internal Server Error',
[('Content-Type', 'text/plain')])
if smarttypes.config.IS_PROD:
email_utils.send_email(
'*****@*****.**',
['*****@*****.**', '*****@*****.**'],
error_string, 'smarttypes site error')
return [error_string]
def pull_some_users(user_id):
postgres_handle = PostgresHandle(smarttypes.connection_string)
root_user = TwitterUser.get_by_id(user_id, postgres_handle)
if not root_user:
raise Exception('User ID: %s not in our DB!' % user_id)
if not root_user.credentials:
raise Exception('%s does not have api credentials!' %
root_user.screen_name)
api_handle = root_user.credentials.api_handle
root_user = load_user_and_the_people_they_follow(api_handle,
root_user.id,
postgres_handle,
is_root_user=True)
load_this_user_id = root_user.get_id_of_someone_in_my_network_to_load()
while load_this_user_id:
load_user_and_the_people_they_follow(api_handle, load_this_user_id,
postgres_handle)
load_this_user_id = root_user.get_id_of_someone_in_my_network_to_load()
#load_this_user_id = None
print "Finshed loading all related users for %s!" % root_user.screen_name
from datetime import datetime
import smarttypes, sys, string
from smarttypes.config import *
from smarttypes.model.twitter_credentials import TwitterCredentials
from smarttypes.model.twitter_user import TwitterUser
from smarttypes.model.twitter_reduction import TwitterReduction
from smarttypes.utils.postgres_handle import PostgresHandle
postgres_handle = PostgresHandle(smarttypes.connection_string)
if __name__ == "__main__":
user_count_list = TwitterReduction.get_user_reduction_counts(
postgres_handle)
for user, count in user_count_list:
if count == 1:
creds = TwitterCredentials.get_by_twitter_id(
user.id, postgres_handle)
creds.maybe_send_initial_map_email()
import smarttypes
from smarttypes.config import *
from smarttypes.utils import time_utils
from datetime import datetime
import psycopg2
from smarttypes.utils.postgres_handle import PostgresHandle
postgres_handle = PostgresHandle(smarttypes.connection_string)
try:
postgres_handle.execute_query("CREATE LANGUAGE plpgsql;", return_results=False)
postgres_handle.connection.commit()
except psycopg2.ProgrammingError:
postgres_handle.connection.rollback()
pass
ts_modifieddate = """
CREATE OR REPLACE FUNCTION ts_modifieddate() RETURNS trigger
AS $$
BEGIN
NEW.modifieddate = now();
RETURN NEW;
END;
$$
LANGUAGE plpgsql;
"""
postgres_handle.execute_query(ts_modifieddate, return_results=False)
postgres_handle.connection.commit()
import smarttypes
from smarttypes.config import *
from smarttypes.utils import time_utils
from datetime import datetime, timedelta
import psycopg2
from smarttypes.utils.postgres_handle import PostgresHandle
postgres_handle = PostgresHandle(smarttypes.connection_string)
################################################
##get rid of old connections
##we have db dumps, so we do have an archive
##if ever needed
################################################
retention_days = 30 * 4 #about 4 months
delete_before_this_date = datetime.now() - timedelta(days=retention_days)
#delete users
sql = """
delete from twitter_user
where last_loaded_following_ids < %(delete_before_this_date)s;"""
#print sql % {'delete_before_this_date':delete_before_this_date}
postgres_handle.execute_query(sql, {'delete_before_this_date':delete_before_this_date}, return_results=False)
postgres_handle.connection.commit()
#drop tables
sql = """drop table twitter_user_following_%(postfix)s;"""
for year_week_st in time_utils.year_weeknum_strs(delete_before_this_date - timedelta(days=7), 20, forward=False):
#print sql % {'postfix':year_week_st}
def reduce_graph(screen_name, distance=20, min_followers=60):
postgres_handle = PostgresHandle(smarttypes.connection_string)
###########################################
##reduce
###########################################
root_user = TwitterUser.by_screen_name(screen_name, postgres_handle)
follower_followies_map = root_user.get_graph_info(
distance=distance, min_followers=min_followers)
gr = GraphReduce(screen_name, follower_followies_map)
gr.reduce_with_linloglayout()
###########################################
##save reduction in db
###########################################
root_user_id = root_user.id
user_ids = []
x_coordinates = []
y_coordinates = []
in_links = []
out_links = []
for i in range(len(gr.layout_ids)):
user_id = gr.layout_ids[i]
user_ids.append(user_id)
x_coordinates.append(gr.reduction[i][0])
y_coordinates.append(gr.reduction[i][1])
itr_in_links = PostgresHandle.spliter.join(gr.G.predecessors(user_id))
itr_out_links = PostgresHandle.spliter.join(gr.G.successors(user_id))
in_links.append(itr_in_links)
out_links.append(itr_out_links)
twitter_reduction = TwitterReduction.create_reduction(
root_user_id, user_ids, x_coordinates, y_coordinates, in_links,
out_links, postgres_handle)
postgres_handle.connection.commit()
###########################################
##save groups in db
###########################################
groups = []
for i in range(gr.n_groups):
user_ids = []
for j in range(len(gr.layout_ids)):
if i == gr.groups[j]:
user_ids.append(gr.layout_ids[j])
#run pagerank to get the scores
group_graph = networkx.DiGraph()
group_edges = []
for user_id in user_ids:
for following_id in set(user_ids).intersection(
follower_followies_map[user_id]):
group_edges.append((user_id, following_id))
print len(user_ids), len(group_edges)
if not group_edges:
continue
group_graph.add_edges_from(group_edges)
pagerank = networkx.pagerank(group_graph, max_iter=500)
scores = []
for user_id in user_ids:
scores.append(pagerank.get(user_id, 0))
groups.append(
TwitterGroup.create_group(twitter_reduction.id, i, user_ids,
scores, postgres_handle))
postgres_handle.connection.commit()
###########################################
##makes for quicker queries in some cases
###########################################
twitter_reduction.save_group_info(postgres_handle)
postgres_handle.connection.commit()
###########################################
##mk_tag_clouds
###########################################
TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle)
postgres_handle.connection.commit()
def do_linlog_reduction():
postgres_handle = PostgresHandle(smarttypes.connection_string)
network = load_network_from_the_db(postgres_handle, 5)
reduce_with_linloglayout(network)
graph_reduction, communities = load_linlog_reduction_from_file()
write_reduction_to_csv(network, graph_reduction)
def reduce_graph(screen_name,
distance=20,
min_followers=60,
pickle_it=True,
just_load_from_file=False):
postgres_handle = PostgresHandle(smarttypes.connection_string)
# if just_load_from_file:
# print "Loading data from a pickle."
# gr = GraphReduce(screen_name, {})
# f = open(gr.pickle_file_path)
# twitter_reduction, groups = pickle.load(f)
# twitter_reduction.id = None
# twitter_reduction.postgres_handle = postgres_handle
# twitter_reduction.save()
# postgres_handle.connection.commit()
# for group in groups:
# group.id = None
# group.reduction_id = twitter_reduction.id
# group.postgres_handle = postgres_handle
# group.save()
# postgres_handle.connection.commit()
# TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle)
# postgres_handle.connection.commit()
# print "All done!"
# return 0
########################
##reduce
########################
root_user = TwitterUser.by_screen_name(screen_name, postgres_handle)
follower_followies_map = root_user.get_graph_info(
distance=distance, min_followers=min_followers)
gr = GraphReduce(screen_name, follower_followies_map)
#gr.reduce_with_exafmm()
gr.reduce_with_linloglayout()
########################
##save reduction in db
########################
root_user_id = root_user.id
user_ids = []
x_coordinates = []
y_coordinates = []
in_links = []
out_links = []
for i in range(len(gr.layout_ids)):
user_id = gr.layout_ids[i]
user_ids.append(user_id)
x_coordinates.append(gr.reduction[i][0])
y_coordinates.append(gr.reduction[i][1])
itr_in_links = PostgresHandle.spliter.join(gr.G.predecessors(user_id))
itr_out_links = PostgresHandle.spliter.join(gr.G.successors(user_id))
in_links.append(itr_in_links)
out_links.append(itr_out_links)
twitter_reduction = TwitterReduction.create_reduction(
root_user_id, user_ids, x_coordinates, y_coordinates, in_links,
out_links, postgres_handle)
postgres_handle.connection.commit()
########################
##save groups in db
########################
groups = []
for i in range(gr.n_clusters):
user_ids = []
for j in range(len(gr.layout_ids)):
if gr.layout_clusters[j][i] > 0:
user_ids.append(gr.layout_ids[j])
#run pagerank to get the scores
group_graph = networkx.DiGraph()
group_edges = []
for user_id in user_ids:
if user_id in follower_followies_map:
for following_id in set(user_ids).intersection(
follower_followies_map[user_id]):
group_edges.append((user_id, following_id))
print len(user_ids), len(group_edges)
if not group_edges:
continue
group_graph.add_edges_from(group_edges)
pagerank = networkx.pagerank(group_graph, max_iter=500)
scores = []
for user_id in user_ids:
scores.append(pagerank.get(user_id, 0))
groups.append(
TwitterGroup.create_group(twitter_reduction.id, i, user_ids,
scores, postgres_handle))
postgres_handle.connection.commit()
twitter_reduction.save_group_info(postgres_handle)
postgres_handle.connection.commit()
########################
##mk_tag_clouds
########################
TwitterGroup.mk_tag_clouds(twitter_reduction.id, postgres_handle)
postgres_handle.connection.commit()
# Submit tile to be rendered into the queue
t = (name, tile_uri, x, y, z)
queue.put(t)
# Signal render threads to exit by sending empty request to queue
for i in range(num_threads):
queue.put(None)
# wait for pending rendering jobs to complete
queue.join()
for i in range(num_threads):
renderers[i].join()
if __name__ == "__main__":
postgres_handle = PostgresHandle(smarttypes.connection_string)
#get reduction_id
qry = """
select tr.id
from twitter_reduction tr
where tr.tiles_are_written_to_disk = False
order by tr.id desc limit 1;
"""
reduction_id = postgres_handle.execute_query(qry, {})[0]['id']
tile_dir = '../static/tiles/%s/' % reduction_id
if not os.path.isdir(tile_dir):
os.mkdir(tile_dir)
style_file = 'mapnik.xml'
min_zoom = 0
max_zoom = 5
bbox = (-180, -85.0511, 180, 85.0511)
#setup a remote db tunnel
import subprocess, copy
subprocess.call('ssh [email protected] -N -f -L 5433:localhost:5432', shell=True)
from datetime import datetime
import smarttypes, sys, string
from smarttypes.config import *
from smarttypes.model.twitter_credentials import TwitterCredentials
from smarttypes.model.twitter_user import TwitterUser
from smarttypes.utils.postgres_handle import PostgresHandle
remote_postgres_handle = PostgresHandle(smarttypes.connection_string + " port='5433'")
local_postgres_handle = PostgresHandle(smarttypes.connection_string)
for creds in TwitterCredentials.get_all(remote_postgres_handle):
remote_root_user = copy.deepcopy(creds.root_user)
creds.postgres_handle = local_postgres_handle
local_root_user = creds.root_user
if remote_root_user and not local_root_user:
remote_root_user.postgres_handle = local_postgres_handle
remote_root_user.save()
creds.save()
local_postgres_handle.connection.commit()
"""
think about circular references
"""
import smarttypes, psycopg2
from smarttypes.model.ppygis import Geometry
from smarttypes.utils.postgres_handle import PostgresHandle
postgres_handle = PostgresHandle(smarttypes.connection_string)
sql = """
select pg_type.oid from pg_type where typname = 'geometry';
"""
geometry_oid = postgres_handle.execute_query(sql)[0]['oid']
GEOMETRY = psycopg2.extensions.new_type((geometry_oid, ), "GEOMETRY",
Geometry.read_ewkb)
psycopg2.extensions.register_type(GEOMETRY)
# Submit tile to be rendered into the queue
t = (name, tile_uri, x, y, z)
queue.put(t)
# Signal render threads to exit by sending empty request to queue
for i in range(num_threads):
queue.put(None)
# wait for pending rendering jobs to complete
queue.join()
for i in range(num_threads):
renderers[i].join()
if __name__ == "__main__":
postgres_handle = PostgresHandle(smarttypes.connection_string)
#get reduction_id
qry = """
select tr.id
from twitter_reduction tr
where tr.tiles_are_written_to_disk = False
order by tr.id desc limit 1;
"""
reduction_id = postgres_handle.execute_query(qry, {})[0]['id']
tile_dir = '../static/tiles/%s/' % reduction_id
if not os.path.isdir(tile_dir):
os.mkdir(tile_dir)
style_file = 'mapnik.xml'
min_zoom = 0
max_zoom = 5
bbox = (-180, -85.0511, 180, 85.0511)
import smarttypes
from smarttypes.config import *
from smarttypes.utils import time_utils
from datetime import datetime, timedelta
import psycopg2
from smarttypes.utils.postgres_handle import PostgresHandle
postgres_handle = PostgresHandle(smarttypes.connection_string)
################################################
##get rid of old connections
##we have db dumps, so we do have an archive
##if ever needed
################################################
retention_days = 30 * 4 #about 4 months
delete_before_this_date = datetime.now() - timedelta(days=retention_days)
#delete users
sql = """
delete from twitter_user
where last_loaded_following_ids < %(delete_before_this_date)s;"""
#print sql % {'delete_before_this_date':delete_before_this_date}
postgres_handle.execute_query(
sql, {'delete_before_this_date': delete_before_this_date},
return_results=False)
postgres_handle.connection.commit()
#drop tables
sql = """drop table twitter_user_following_%(postfix)s;"""
for year_week_st in time_utils.year_weeknum_strs(delete_before_this_date -
""" think about circular references """ import smarttypes, psycopg2 from smarttypes.model.ppygis import Geometry from smarttypes.utils.postgres_handle import PostgresHandle postgres_handle = PostgresHandle(smarttypes.connection_string) sql = """ select pg_type.oid from pg_type where typname = 'geometry'; """ geometry_oid = postgres_handle.execute_query(sql)[0]['oid'] GEOMETRY = psycopg2.extensions.new_type((geometry_oid, ), "GEOMETRY", Geometry.read_ewkb) psycopg2.extensions.register_type(GEOMETRY)