def export(name, j):
path = os.path.join(awfy.path, name)
if os.path.exists(path):
os.remove(path)
with open(path, 'w') as fp:
util.json_dump(j, fp)
print('Exported: ' + name)
def save_cache(prefix, cache):
j = {
'graph': cache,
'version': awfy.version
}
with open(os.path.join(awfy.path, prefix + '.json'), 'w') as fp:
util.json_dump(j, fp)
def run_with_context(interactions_path, candidate_tree_path, dirname=None, to_original_graph=False, undirected=False):
if not os.path.exists(dirname):
os.makedirs(dirname)
try:
interactions = json.load(open(interactions_path))
except ValueError as e:
print(e)
interactions = load_json_by_line(interactions_path)
interactions = IU.clean_interactions(interactions, undirected=undirected)
output_path = get_output_path(candidate_tree_path, dirname)
K = 5
events = detect_events_given_path(candidate_tree_path, K)
contexted_events = []
for e in events:
context_dag = extract_event_context(interactions, e, undirected=undirected)
if to_original_graph:
context_dag = convert_to_original_graph(context_dag)
e = convert_to_original_graph(e)
contexted_events.append(add_subgraph_specific_attributes_to_graph(context_dag, [(e, {"event": True})]))
d3_events = [to_d3_graph(ce) for ce in contexted_events]
print("writing to {}".format(output_path))
json_dump(d3_events, output_path)
def main():
import argparse
parser = argparse.ArgumentParser("Dump meta information to json")
parser.add_argument("--interactions_path", required=True)
parser.add_argument("--interactions_output_path", required=True)
parser.add_argument("--people_path", required=True)
parser.add_argument("--people_output_path", required=True)
args = parser.parse_args()
json_dump(load_id2obj_dict(args.interactions_path, "message_id"), args.interactions_output_path)
json_dump(load_id2obj_dict(args.people_path, "id"), args.people_output_path)
def _do_handle (self):
# Read the URL
url = self.env['REQUEST_URI']
# Get a copy of the server (it did fork!)
server = get_server()
# Refer SCGI object by thread
my_thread = threading.currentThread()
my_thread.scgi_conn = self
my_thread.request_url = url
for published in server._web_paths:
if re.match (published._regex, url):
# POST
if published._method == 'POST':
post = self._process_post()
my_thread.post = post
# Validate
validator = PostValidator (post, published._validation)
errors = validator.Validate()
if errors:
resp = HTTP_Response(200, body=json_dump(errors))
resp['Content-Type'] = "application/json"
return resp
# Execute handler
ret = published (**published._kwargs)
# Deal with the returned info
if type(ret) == str:
self.response += ret
return self.response
elif type(ret) == dict:
info = json_dump(ret)
self.response += info
self.response['Content-Type'] = "application/json"
return self.response
elif isinstance(ret, HTTP_Response):
return ret
else:
self.response += ret
return self.response
# Not found
return HTTP_Error (404)
def run(candidate_tree_path,
k,
id2people,
id2interaction,
dirname=None,
to_original_graph=False):
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
output_path = get_output_path(candidate_tree_path, dirname)
events = detect_events_given_path(candidate_tree_path, k)
# add people and content
for e in events:
root = get_roots(e)[0]
for n in e.nodes_iter():
e.node[n]['sender'] = id2people[e.node[n]['sender_id']]
e.node[n]['recipients'] = [id2people[id_]
for id_ in e.node[n]['recipient_ids']]
# print(id2interaction[n])
e.node[n]['subject'] = id2interaction[n]['subject']
e.node[n]['body'] = id2interaction[n]['body']
for f in ('retweet_count', 'favorite_count'):
e.node[n][f] = id2interaction[n].get(f)
e.node[n]['body'] = id2interaction[n]['body']
e.node[n]['root'] = (n == root)
e.node[n]['datetime'] = str(e.node[n]['datetime'])
# # some simple clustering
# assignment = greedy_clustering_on_graph(e)
# for n in e.nodes_iter():
# e.node[n]['cluster_label'] = assignment[n]
if to_original_graph:
events = map(convert_to_original_graph,
events)
# import pdb; pdb.set_trace()
d3_events = [to_d3_graph(e)
for e in events]
json_dump(d3_events, output_path)
def main():
import argparse
import cPickle as pkl
from util import json_dump
parser = argparse.ArgumentParser('dump vis timeline data')
parser.add_argument('--cand_trees_path', required=True)
parser.add_argument('--output_path', required=True)
parser.add_argument('--interactions_path', required=True)
parser.add_argument('--people_path', required=True)
parser.add_argument('--corpus_dict_path', required=True)
parser.add_argument('--lda_model_path', required=True)
parser.add_argument('--people_repr_template', type=str,
default="{id}")
parser.add_argument('-k', type=int, default=10)
parser.add_argument('--undirected', default=False, action="store_true")
args = parser.parse_args()
summary_kws = build_default_summary_kws_from_path(
args.interactions_path,
args.people_path,
args.corpus_dict_path,
args.lda_model_path,
args.people_repr_template,
undirected=args.undirected
)
trees = pkl.load(open(args.cand_trees_path))
# add hashtags if there
print(len(trees))
first_node = trees[0].nodes()[0]
if 'hashtags' in trees[0].node[first_node]:
print('add hashtags')
summary_kws['hashtags'] = {}
data = run(trees,
args.k,
summary_kws,
args.undirected)
json_dump(data, args.output_path)
def make(theory):
"""Bootstrap a language from Johann.
Inputs: theory in ['sk', 'skj', 'skja', 'skrj']
"""
spec = SPECS[theory]
nullary_weights = spec.get('nullary_weights', {})
injective_probs = spec.get('injective_probs', {})
binary_probs = spec.get('binary_probs', {})
symmetric_probs = spec.get('symmetric_probs', {})
compound_prob = (
sum(injective_probs.values()) +
sum(binary_probs.values()) +
sum(symmetric_probs.values()))
assert compound_prob < 1
nullary_prob = 1.0 - compound_prob
nullary_probs = {
key: exp(-val)
for key, val in nullary_weights.iteritems()
}
scale = nullary_prob / sum(nullary_probs.values())
for key in nullary_probs.keys():
nullary_probs[key] *= scale
probs = {
'NULLARY': nullary_probs,
'INJECTIVE': injective_probs,
'BINARY': binary_probs,
'SYMMETRIC': symmetric_probs,
}
for arity, group in probs.items():
if not group:
del probs[arity]
with pomagma.util.chdir(os.path.dirname(os.path.abspath(__file__))):
util.json_dump(probs, '{}.json'.format(theory))
def toJSON (self):
tmp = filter (lambda x: x, [x.toJSON() for x in self.helps])
if tmp:
help = []
for e in reduce(lambda x,y: x+y, tmp):
if not e[0] in [x[0] for x in help]:
help.append(e)
else:
help = []
return json_dump({'html': self.html,
'js': Postprocess(self.js),
'headers': self.headers,
'helps': help})
def py2js_dic (d):
js_pairs = []
for key in d:
val = d[key]
if type(val) in (bool, int, float, list):
js_pairs.append ("'%s': %s" %(key, json_dump(val)))
elif type(val) == str:
if '/* code */' in val:
js_pairs.append ("'%s': %s" %(key, val))
else:
js_pairs.append ("'%s': '%s'" %(key, val))
else:
assert False, "Unknown data type"
return "{%s}" % (", ".join(js_pairs))
def _do_handle (self):
# Read the URL
url = self.env['REQUEST_URI']
# Get a copy of the server (it did fork!)
server = get_server()
# Security Checks
sec_error = False
if server.use_sec_cookie:
if not self.env['CTK_COOKIE']:
sec_error = "Cookie"
elif not server.sec_cookie:
server.sec_cookie = self.env['CTK_COOKIE'][:]
else:
if server.sec_cookie != self.env['CTK_COOKIE']:
sec_error = "Cookie"
if server.use_sec_submit:
if not server.sec_submit:
server.sec_submit = self.env['CTK_SUBMITTER_SECRET'][:]
if self.env['REQUEST_METHOD'] == 'POST':
if not server.sec_submit in url:
sec_error = "Submit"
if sec_error:
response = HTTP_Response (error=403, body="%s check failed" %(sec_error))
self.send (str(response))
return
# Refer SCGI object by thread
my_thread = threading.currentThread()
my_thread.scgi_conn = self
my_thread.request_url = url
for published in server._web_paths:
if re.match (published._regex, url):
# POST
if published._method == 'POST':
post = self._process_post()
my_thread.post = post
# Validate
validator = PostValidator (post, published._validation)
errors = validator.Validate()
if errors:
resp = HTTP_Response(200, body=json_dump(errors))
resp['Content-Type'] = "application/json"
return resp
# Execute handler
ret = published (**published._kwargs)
# Deal with the returned info
if type(ret) == str:
self.response += ret
return self.response
elif type(ret) == dict:
info = json_dump(ret)
self.response += info
self.response['Content-Type'] = "application/json"
return self.response
elif isinstance(ret, HTTP_Response):
return ret
else:
self.response += ret
return self.response
# Not found
return HTTP_Error (404)
def dump2interactions(db, collection_name, output_path):
articles = articles_articles(db, collection_name)
print('# valid articles: ', len(articles))
json_dump(articles, output_path)
return articles
# filter out articles with single company tag
if has_multiple_companies(a):
valid_articles.append(
transform_article(a)
)
return valid_articles
def dump2interactions(db, collection_name, output_path):
articles = articles_articles(db, collection_name)
print('# valid articles: ', len(articles))
json_dump(articles, output_path)
return articles
def collect_people_info(articles):
participant_ids = set(
itertools.chain(*[a['participant_ids'] for a in articles])
)
print('# unique participants: ', len(participant_ids))
return [{'id': p} for p in participant_ids]
if __name__ == '__main__':
# articles = dump2interactions(MongoClient()['bloomberg'],
# 'articles',
# 'data/bloomberg/interactions.json')
articles = json_load('data/bloomberg/interactions.json')
json_dump(collect_people_info(articles),
'data/bloomberg/people.json')
def save_metadata(prefix, data):
with open(os.path.join(awfy.path, "metadata-" + prefix + ".json"), "w") as fp:
util.json_dump(data, fp)
def execute():
metrics = generate_metrics()
util.upload_file("v1/containers.json", util.json_dump(metrics))
print "Updated metrics file uploaded to S3"
def save_cache(prefix, cache):
j = {"graph": cache, "version": awfy.version}
with open(os.path.join(awfy.path, prefix + ".json"), "w") as fp:
util.json_dump(j, fp)
def _do_handle(self):
# Read the URL
url = self.env['REQUEST_URI']
# Get a copy of the server (it did fork!)
server = get_server()
# Security Checks
sec_error = False
if server.use_sec_cookie:
if not self.env['CTK_COOKIE']:
sec_error = "Cookie"
elif not server.sec_cookie:
server.sec_cookie = self.env['CTK_COOKIE'][:]
else:
if server.sec_cookie != self.env['CTK_COOKIE']:
sec_error = "Cookie"
if server.use_sec_submit:
if not server.sec_submit:
server.sec_submit = self.env['CTK_SUBMITTER_SECRET'][:]
if self.env['REQUEST_METHOD'] == 'POST':
if not server.sec_submit in url:
sec_error = "Submit"
if sec_error:
response = HTTP_Response(error=403,
body="%s check failed" % (sec_error))
self.send(str(response))
return
# Refer SCGI object by thread
my_thread = threading.currentThread()
my_thread.scgi_conn = self
my_thread.request_url = url
base_path = urlparse.urlsplit(url).path
if len(base_path) > 1 and base_path[-1] == '/':
base_path = base_path[:-1] # remove trailing '/' if it exists
for published in server._web_paths:
if re.match(published._regex, base_path):
# POST
if published._method == 'POST':
post = self._process_post()
my_thread.post = post
# Validate
validator = PostValidator(post, published._validation)
errors = validator.Validate()
if errors:
resp = HTTP_Response(200, body=json_dump(errors))
resp['Content-Type'] = "application/json"
return resp
# Execute handler
ret = published(**published._kwargs)
# Deal with the returned info
if type(ret) == str:
self.response += ret
return self.response
elif type(ret) == dict:
info = json_dump(ret)
self.response += info
self.response['Content-Type'] = "application/json"
return self.response
elif isinstance(ret, HTTP_Response):
return ret
else:
self.response += ret
return self.response
# Not found
return HTTP_Error(404)
def save_metadata(prefix, data):
with open(os.path.join(awfy.path, 'metadata-' + prefix + '.json'),
'w') as fp:
util.json_dump(data, fp)
def save_cache(prefix, cache):
j = {'graph': cache, 'version': awfy.version}
with open(os.path.join(awfy.path, prefix + '.json'), 'w') as fp:
util.json_dump(j, fp)
def export(name, j):
path = os.path.join(awfy.path, name)
if os.path.exists(path):
os.remove(path)
with open(path, 'w') as fp:
util.json_dump(j, fp)
def main():
dirname = sys.argv[1]
output_path = sys.argv[2]
names = map(os.path.basename,
glob(os.path.join(dirname, "result-*.json")))
json_dump(names, output_path)
def json(self): return json_dump(self.to_data())
def save_metadata(prefix, data):
with open(os.path.join(awfy.path, 'metadata-' + prefix + '.json'), 'w') as fp:
util.json_dump(data, fp)
def main():
dirname = sys.argv[1]
output_path = sys.argv[2]
names = map(os.path.basename, glob(os.path.join(dirname, "result-*.json")))
json_dump(names, output_path)