Example #1
0
def export(name, j):
    path = os.path.join(awfy.path, name)
    if os.path.exists(path):
        os.remove(path)
    with open(path, 'w') as fp:
        util.json_dump(j, fp)
    print('Exported: ' + name)
Example #2
0
def save_cache(prefix, cache):
    j = {
        'graph': cache,
        'version': awfy.version
    }
    with open(os.path.join(awfy.path, prefix + '.json'), 'w') as fp:
        util.json_dump(j, fp)
def run_with_context(interactions_path, candidate_tree_path, dirname=None, to_original_graph=False, undirected=False):
    if not os.path.exists(dirname):
        os.makedirs(dirname)

    try:
        interactions = json.load(open(interactions_path))
    except ValueError as e:
        print(e)
        interactions = load_json_by_line(interactions_path)

    interactions = IU.clean_interactions(interactions, undirected=undirected)

    output_path = get_output_path(candidate_tree_path, dirname)

    K = 5
    events = detect_events_given_path(candidate_tree_path, K)

    contexted_events = []
    for e in events:
        context_dag = extract_event_context(interactions, e, undirected=undirected)

        if to_original_graph:
            context_dag = convert_to_original_graph(context_dag)
            e = convert_to_original_graph(e)

        contexted_events.append(add_subgraph_specific_attributes_to_graph(context_dag, [(e, {"event": True})]))
    d3_events = [to_d3_graph(ce) for ce in contexted_events]

    print("writing to {}".format(output_path))
    json_dump(d3_events, output_path)
def main():
    import argparse

    parser = argparse.ArgumentParser("Dump meta information to json")

    parser.add_argument("--interactions_path", required=True)
    parser.add_argument("--interactions_output_path", required=True)
    parser.add_argument("--people_path", required=True)
    parser.add_argument("--people_output_path", required=True)
    args = parser.parse_args()

    json_dump(load_id2obj_dict(args.interactions_path, "message_id"), args.interactions_output_path)
    json_dump(load_id2obj_dict(args.people_path, "id"), args.people_output_path)
Example #5
0
    def _do_handle (self):
        # Read the URL
        url = self.env['REQUEST_URI']

        # Get a copy of the server (it did fork!)
        server = get_server()

        # Refer SCGI object by thread
        my_thread = threading.currentThread()
        my_thread.scgi_conn   = self
        my_thread.request_url = url

        for published in server._web_paths:
            if re.match (published._regex, url):
                # POST
                if published._method == 'POST':
                    post = self._process_post()
                    my_thread.post = post

                    # Validate
                    validator = PostValidator (post, published._validation)
                    errors = validator.Validate()
                    if errors:
                        resp = HTTP_Response(200, body=json_dump(errors))
                        resp['Content-Type'] = "application/json"
                        return resp

                # Execute handler
                ret = published (**published._kwargs)

                # Deal with the returned info
                if type(ret) == str:
                    self.response += ret
                    return self.response

                elif type(ret) == dict:
                    info = json_dump(ret)
                    self.response += info
                    self.response['Content-Type'] = "application/json"
                    return self.response

                elif isinstance(ret, HTTP_Response):
                    return ret

                else:
                    self.response += ret
                    return self.response

        # Not found
        return HTTP_Error (404)
Example #6
0
def run(candidate_tree_path,
        k,
        id2people,
        id2interaction,
        dirname=None,
        to_original_graph=False):

    if dirname and not os.path.exists(dirname):
        os.makedirs(dirname)

    output_path = get_output_path(candidate_tree_path, dirname)

    events = detect_events_given_path(candidate_tree_path, k)
    
    # add people and content
    for e in events:
        root = get_roots(e)[0]
        for n in e.nodes_iter():
            e.node[n]['sender'] = id2people[e.node[n]['sender_id']]
            e.node[n]['recipients'] = [id2people[id_]
                                       for id_ in e.node[n]['recipient_ids']]
            # print(id2interaction[n])
            e.node[n]['subject'] = id2interaction[n]['subject']
            e.node[n]['body'] = id2interaction[n]['body']

            for f in ('retweet_count', 'favorite_count'):
                e.node[n][f] = id2interaction[n].get(f)
            
            e.node[n]['body'] = id2interaction[n]['body']
            e.node[n]['root'] = (n == root)
            e.node[n]['datetime'] = str(e.node[n]['datetime'])

        # # some simple clustering
        # assignment = greedy_clustering_on_graph(e)
        # for n in e.nodes_iter():
        #     e.node[n]['cluster_label'] = assignment[n]
            
    if to_original_graph:
        events = map(convert_to_original_graph,
                     events)
        # import pdb; pdb.set_trace()

    d3_events = [to_d3_graph(e)
                 for e in events]

    json_dump(d3_events, output_path)
def main():
    import argparse
    import cPickle as pkl
    from util import json_dump

    parser = argparse.ArgumentParser('dump vis timeline data')
    parser.add_argument('--cand_trees_path', required=True)
    parser.add_argument('--output_path', required=True)
    parser.add_argument('--interactions_path', required=True)
    parser.add_argument('--people_path', required=True)
    parser.add_argument('--corpus_dict_path', required=True)
    parser.add_argument('--lda_model_path', required=True)
    parser.add_argument('--people_repr_template', type=str,
                        default="{id}")
    parser.add_argument('-k', type=int, default=10)
    parser.add_argument('--undirected', default=False, action="store_true")

    args = parser.parse_args()
    
    summary_kws = build_default_summary_kws_from_path(
        args.interactions_path,
        args.people_path,
        args.corpus_dict_path,
        args.lda_model_path,
        args.people_repr_template,
        undirected=args.undirected
    )
    trees = pkl.load(open(args.cand_trees_path))
    
    # add hashtags if there
    print(len(trees))
    first_node = trees[0].nodes()[0]
    if 'hashtags' in trees[0].node[first_node]:
        print('add hashtags')
        summary_kws['hashtags'] = {}

    data = run(trees,
               args.k,
               summary_kws,
               args.undirected)
    json_dump(data, args.output_path)
Example #8
0
def make(theory):
    """Bootstrap a language from Johann.

    Inputs: theory in ['sk', 'skj', 'skja', 'skrj']

    """
    spec = SPECS[theory]
    nullary_weights = spec.get('nullary_weights', {})
    injective_probs = spec.get('injective_probs', {})
    binary_probs = spec.get('binary_probs', {})
    symmetric_probs = spec.get('symmetric_probs', {})

    compound_prob = (
        sum(injective_probs.values()) +
        sum(binary_probs.values()) +
        sum(symmetric_probs.values()))
    assert compound_prob < 1
    nullary_prob = 1.0 - compound_prob
    nullary_probs = {
        key: exp(-val)
        for key, val in nullary_weights.iteritems()
    }
    scale = nullary_prob / sum(nullary_probs.values())
    for key in nullary_probs.keys():
        nullary_probs[key] *= scale

    probs = {
        'NULLARY': nullary_probs,
        'INJECTIVE': injective_probs,
        'BINARY': binary_probs,
        'SYMMETRIC': symmetric_probs,
    }
    for arity, group in probs.items():
        if not group:
            del probs[arity]

    with pomagma.util.chdir(os.path.dirname(os.path.abspath(__file__))):
        util.json_dump(probs, '{}.json'.format(theory))
Example #9
0
    def toJSON (self):
        tmp = filter (lambda x: x, [x.toJSON() for x in self.helps])
        if tmp:
            help = []
            for e in reduce(lambda x,y: x+y, tmp):
                if not e[0] in [x[0] for x in help]:
                    help.append(e)
        else:
            help = []

        return json_dump({'html':    self.html,
                          'js':      Postprocess(self.js),
                          'headers': self.headers,
                          'helps':   help})
Example #10
0
def py2js_dic (d):
    js_pairs = []

    for key in d:
        val = d[key]
        if type(val) in (bool, int, float, list):
             js_pairs.append ("'%s': %s" %(key, json_dump(val)))
        elif type(val) == str:
            if '/* code */' in val:
                js_pairs.append ("'%s': %s" %(key, val))
            else:
                js_pairs.append ("'%s': '%s'" %(key, val))
        else:
            assert False, "Unknown data type"

    return "{%s}" % (", ".join(js_pairs))
Example #11
0
    def _do_handle (self):
        # Read the URL
        url = self.env['REQUEST_URI']

        # Get a copy of the server (it did fork!)
        server = get_server()

        # Security Checks
        sec_error = False
        if server.use_sec_cookie:
            if not self.env['CTK_COOKIE']:
                sec_error = "Cookie"
            elif not server.sec_cookie:
                server.sec_cookie = self.env['CTK_COOKIE'][:]
            else:
                if server.sec_cookie != self.env['CTK_COOKIE']:
                    sec_error = "Cookie"

        if server.use_sec_submit:
            if not server.sec_submit:
                server.sec_submit = self.env['CTK_SUBMITTER_SECRET'][:]

            if self.env['REQUEST_METHOD'] == 'POST':
                if not server.sec_submit in url:
                    sec_error = "Submit"

        if sec_error:
            response = HTTP_Response (error=403, body="%s check failed" %(sec_error))
            self.send (str(response))
            return

        # Refer SCGI object by thread
        my_thread = threading.currentThread()
        my_thread.scgi_conn   = self
        my_thread.request_url = url

        for published in server._web_paths:
            if re.match (published._regex, url):
                # POST
                if published._method == 'POST':
                    post = self._process_post()
                    my_thread.post = post

                    # Validate
                    validator = PostValidator (post, published._validation)
                    errors = validator.Validate()
                    if errors:
                        resp = HTTP_Response(200, body=json_dump(errors))
                        resp['Content-Type'] = "application/json"
                        return resp

                # Execute handler
                ret = published (**published._kwargs)

                # Deal with the returned info
                if type(ret) == str:
                    self.response += ret
                    return self.response

                elif type(ret) == dict:
                    info = json_dump(ret)
                    self.response += info
                    self.response['Content-Type'] = "application/json"
                    return self.response

                elif isinstance(ret, HTTP_Response):
                    return ret

                else:
                    self.response += ret
                    return self.response

        # Not found
        return HTTP_Error (404)
Example #12
0
def dump2interactions(db, collection_name, output_path):
    articles = articles_articles(db, collection_name)
    print('# valid articles: ', len(articles))
    json_dump(articles, output_path)
    return articles
Example #13
0
        # filter out articles with single company tag
        if has_multiple_companies(a):
            valid_articles.append(
                transform_article(a)
            )
    return valid_articles


def dump2interactions(db, collection_name, output_path):
    articles = articles_articles(db, collection_name)
    print('# valid articles: ', len(articles))
    json_dump(articles, output_path)
    return articles


def collect_people_info(articles):
    participant_ids = set(
        itertools.chain(*[a['participant_ids'] for a in articles])
        )
    print('# unique participants: ', len(participant_ids))
    return [{'id': p} for p in participant_ids]


if __name__ == '__main__':
    # articles = dump2interactions(MongoClient()['bloomberg'],
    #                              'articles',
    #                              'data/bloomberg/interactions.json')
    articles = json_load('data/bloomberg/interactions.json')
    json_dump(collect_people_info(articles),
              'data/bloomberg/people.json')
Example #14
0
def save_metadata(prefix, data):
    with open(os.path.join(awfy.path, "metadata-" + prefix + ".json"), "w") as fp:
        util.json_dump(data, fp)
Example #15
0
def execute():
    metrics = generate_metrics()
    util.upload_file("v1/containers.json", util.json_dump(metrics))
    print "Updated metrics file uploaded to S3"
Example #16
0
def save_cache(prefix, cache):
    j = {"graph": cache, "version": awfy.version}
    with open(os.path.join(awfy.path, prefix + ".json"), "w") as fp:
        util.json_dump(j, fp)
Example #17
0
    def _do_handle(self):
        # Read the URL
        url = self.env['REQUEST_URI']

        # Get a copy of the server (it did fork!)
        server = get_server()

        # Security Checks
        sec_error = False
        if server.use_sec_cookie:
            if not self.env['CTK_COOKIE']:
                sec_error = "Cookie"
            elif not server.sec_cookie:
                server.sec_cookie = self.env['CTK_COOKIE'][:]
            else:
                if server.sec_cookie != self.env['CTK_COOKIE']:
                    sec_error = "Cookie"

        if server.use_sec_submit:
            if not server.sec_submit:
                server.sec_submit = self.env['CTK_SUBMITTER_SECRET'][:]

            if self.env['REQUEST_METHOD'] == 'POST':
                if not server.sec_submit in url:
                    sec_error = "Submit"

        if sec_error:
            response = HTTP_Response(error=403,
                                     body="%s check failed" % (sec_error))
            self.send(str(response))
            return

        # Refer SCGI object by thread
        my_thread = threading.currentThread()
        my_thread.scgi_conn = self
        my_thread.request_url = url

        base_path = urlparse.urlsplit(url).path
        if len(base_path) > 1 and base_path[-1] == '/':
            base_path = base_path[:-1]  # remove trailing '/' if it exists

        for published in server._web_paths:
            if re.match(published._regex, base_path):
                # POST
                if published._method == 'POST':
                    post = self._process_post()
                    my_thread.post = post

                    # Validate
                    validator = PostValidator(post, published._validation)
                    errors = validator.Validate()
                    if errors:
                        resp = HTTP_Response(200, body=json_dump(errors))
                        resp['Content-Type'] = "application/json"
                        return resp

                # Execute handler
                ret = published(**published._kwargs)

                # Deal with the returned info
                if type(ret) == str:
                    self.response += ret
                    return self.response

                elif type(ret) == dict:
                    info = json_dump(ret)
                    self.response += info
                    self.response['Content-Type'] = "application/json"
                    return self.response

                elif isinstance(ret, HTTP_Response):
                    return ret

                else:
                    self.response += ret
                    return self.response

        # Not found
        return HTTP_Error(404)
def save_metadata(prefix, data):
    with open(os.path.join(awfy.path, 'metadata-' + prefix + '.json'),
              'w') as fp:
        util.json_dump(data, fp)
def save_cache(prefix, cache):
    j = {'graph': cache, 'version': awfy.version}
    with open(os.path.join(awfy.path, prefix + '.json'), 'w') as fp:
        util.json_dump(j, fp)
Example #20
0
def export(name, j):
    path = os.path.join(awfy.path, name)
    if os.path.exists(path):
        os.remove(path)
    with open(path, 'w') as fp:
        util.json_dump(j, fp)
def main():
    dirname = sys.argv[1]
    output_path = sys.argv[2]
    names = map(os.path.basename,
                glob(os.path.join(dirname, "result-*.json")))
    json_dump(names, output_path)
Example #22
0
	def json(self):
		return json_dump(self.to_data())
Example #23
0
def save_metadata(prefix, data):
    with open(os.path.join(awfy.path, 'metadata-' + prefix + '.json'), 'w') as fp:
        util.json_dump(data, fp)
Example #24
0
def main():
    dirname = sys.argv[1]
    output_path = sys.argv[2]
    names = map(os.path.basename, glob(os.path.join(dirname, "result-*.json")))
    json_dump(names, output_path)