Exemplo n.º 1
0
    def process_result_set(self):
        """
        @return: a Topo_Diff object consisting of the commit node and parent link
        """
        ret = Topo_Diff()

        hash_parent = None
        hash_child = None
        ts_created = None
        for _, _, r_set in self.iter__r_set():
            for row in r_set:
                for ret_dict in row:

                    assert None == hash_parent  # assert hash values set once only
                    assert None == hash_child
                    assert None == ts_created

                    hash_parent = ret_dict['head_parent_commit']['hash']
                    hash_child = ret_dict['head_commit']['hash']
                    ts_created = ret_dict['ts_created']

        ret.node_set_add = [{'id': self.n_id, '__label_set': ['__Commit']}]
        l = Link.Link_Ptr(src_id=hash_parent, dst_id=hash_child)
        l['id'] = self.l_id
        l['__type'] = '__Parent'
        ret.link_set_add = [l]
        ret.meta['ts_created'] = ts_created
        return ret
Exemplo n.º 2
0
    def gen_full_db_op_set(self, test_label):
        n_0, n_0_id = generate_random_node_dict(test_label)
        n_1, n_1_id = generate_random_node_dict(test_label)
        l_0, l_0_id = generate_random_link_dict(test_label, n_0_id, n_1_id)

        n_set = [n_0, n_1]
        l_set = [l_0]
        topo_diff = Topo_Diff(node_set_add=n_set, link_set_add=l_set)

        attr_diff = Attr_Diff()
        attr_diff.add_node_attr_write(n_0_id, 'attr_0', 0)

        test_rzdoc = generate_random_RZDoc(test_label)

        op_set = [
                  DBO_rzdoc__clone(),
                  DBO_add_node_set(meta_attr_list_to_meta_attr_map(n_set)),
                  DBO_add_link_set(meta_attr_list_to_meta_attr_map(l_set, meta_attr='__type')),
                  DBO_diff_commit__attr(attr_diff),
                  DBO_diff_commit__topo(topo_diff),
                  DBO_rm_node_set(id_set=[n_0_id]),

                  # block chain
                  DBO_block_chain__init(test_rzdoc),
                  DBO_block_chain__commit(commit_obj=topo_diff.to_json_dict()),

                  # rzdoc
                  DBO_rzdoc__create(test_rzdoc),
                  DBO_rzdoc__delete(test_rzdoc),
                  DBO_rzdoc__list(),
                  DBO_rzdoc__lookup_by_name(test_rzdoc.name),
                  ]
        return op_set
Exemplo n.º 3
0
    def gen_full_db_op_set(self, test_label):
        n_0, n_0_id = generate_random_node_dict(test_label)
        n_1, n_1_id = generate_random_node_dict(test_label)
        l_0, l_0_id = generate_random_link_dict(test_label, n_0_id, n_1_id)

        n_set = [n_0, n_1]
        l_set = [l_0]
        topo_diff = Topo_Diff(node_set_add=n_set, link_set_add=l_set)

        attr_diff = Attr_Diff()
        attr_diff.add_node_attr_write(n_0_id, 'attr_0', 0)

        test_rzdoc = generate_random_RZDoc(test_label)

        op_set = [
            DBO_rzdoc__clone(),
            DBO_add_node_set(meta_attr_list_to_meta_attr_map(n_set)),
            DBO_add_link_set(
                meta_attr_list_to_meta_attr_map(l_set, meta_attr='__type')),
            DBO_diff_commit__attr(attr_diff),
            DBO_diff_commit__topo(topo_diff),
            DBO_rm_node_set(id_set=[n_0_id]),

            # block chain
            DBO_block_chain__init(test_rzdoc),
            DBO_block_chain__commit(commit_obj=topo_diff.to_json_dict()),

            # rzdoc
            DBO_rzdoc__create(test_rzdoc),
            DBO_rzdoc__delete(test_rzdoc),
            DBO_rzdoc__list(),
            DBO_rzdoc__lookup_by_name(test_rzdoc.name),
        ]
        return op_set
Exemplo n.º 4
0
    def process_result_set(self):
        """
        @return: a Topo_Diff object consisting of the commit node and parent link
        """
        ret = Topo_Diff()

        hash_parent = None
        hash_child = None
        ts_created = None
        for _, _, r_set in self.iter__r_set():
            for row in r_set:
                for ret_dict in row:

                    assert None == hash_parent  # assert hash values set once only
                    assert None == hash_child
                    assert None == ts_created

                    hash_parent = ret_dict['head_parent_commit']['hash']
                    hash_child = ret_dict['head_commit']['hash']
                    ts_created = ret_dict['ts_created']

        ret.node_set_add = [{'id': self.n_id,
                             '__label_set': ['__Commit']}
                           ]
        l = Link.Link_Ptr(src_id=hash_parent, dst_id=hash_child)
        l['id'] = self.l_id
        l['__type'] = '__Parent'
        ret.link_set_add = [l]
        ret.meta['ts_created'] = ts_created
        return ret
Exemplo n.º 5
0
    def sanitize_input(req):
        rzdoc_name = request.get_json().get('rzdoc_name')
        topo_diff_dict = request.get_json()['topo_diff']
        topo_diff = Topo_Diff.from_json_dict(topo_diff_dict)

        sanitize_input__topo_diff(topo_diff)
        return rzdoc_name, topo_diff
Exemplo n.º 6
0
    def test_REST_post_triggers_ws_multicast__topo_diff(self):
        class NS_test(BaseNamespace):
            def on_diff_commit__topo(self, *data):
                greenlet.getcurrent().data = data
                raise KeyboardInterrupt(
                )  # TODO: cleanup: properly close socket

        test_label = neo4j_test_util.rand_label()
        n, n_id = test_util.generate_random_node_dict(test_label)
        topo_diff = Topo_Diff(node_set_add=[n])

        def c_0():

            with RZ_websocket(namespace=NS_test) as (sock, _):
                c1_t.switch()  # allow peer to POST
                sock.wait(8)  # allow self to receive

        def c_1():
            data = json.dumps({'topo_diff': topo_diff.to_json_dict()})
            req = urllib2.Request(
                url='http://rhizi.local:8080/graph/diff-commit-topo',
                data=data,
                headers={'Content-Type': 'application/json'})

            f = urllib2.urlopen(req)
            f.close()
            c0_t.switch()

        c0_t = greenlet(c_0)
        c0_t.data = None
        c1_t = greenlet(c_1)
        c0_t.switch()

        self.assertTrue(None != c0_t.data)
        self.assertEqual(2, len(c1_t.data))
Exemplo n.º 7
0
    def sanitize_input(req):
        diff_set_dict = req.get_json()['diff_set']
        topo_diff_dict = diff_set_dict['__diff_set_topo'][0]
        topo_diff = Topo_Diff.from_json_dict(topo_diff_dict)

        sanitize_input__topo_diff(topo_diff)
        return topo_diff
Exemplo n.º 8
0
    def sanitize_input(req):
        rzdoc_name = request.get_json().get('rzdoc_name')
        topo_diff_dict = request.get_json()['topo_diff']
        topo_diff = Topo_Diff.from_json_dict(topo_diff_dict)

        sanitize_input__topo_diff(topo_diff)
        return rzdoc_name, topo_diff
Exemplo n.º 9
0
    def sanitize_input(req):
        diff_set_dict = req.get_json()['diff_set']
        topo_diff_dict = diff_set_dict['__diff_set_topo'][0]
        topo_diff = Topo_Diff.from_json_dict(topo_diff_dict)

        sanitize_input__topo_diff(topo_diff)
        return topo_diff;
Exemplo n.º 10
0
    def test_ws_event__topo_diff(self):
        class NS_test(BaseNamespace):
            def on_diff_commit__topo(self, *data):
                greenlet.getcurrent().data = data
                raise KeyboardInterrupt(
                )  # TODO: cleanup: properly close socket

        test_label = neo4j_test_util.rand_label()
        n_0, n_0_id = test_util.generate_random_node_dict(test_label)
        n_1, n_1_id = test_util.generate_random_node_dict(test_label)
        l, l_id = test_util.generate_random_link_dict(test_label, n_0_id,
                                                      n_1_id)
        topo_diff = Topo_Diff(node_set_add=[n_0, n_1], link_set_add=[l])

        def c_0():
            with RZ_websocket(namespace=NS_test) as (_, ns_sock):
                c1_t.switch()  # allow peer to connect
                data = json.dumps(topo_diff, cls=Topo_Diff.JSON_Encoder)
                ns_sock.emit('diff_commit__topo', data)
                c1_t.switch()

        def c_1():
            with RZ_websocket(namespace=NS_test) as (sock, _):
                c0_t.switch()  # allow peer to emit
                sock.wait(8)  # allow self to receive

        c0_t = greenlet(c_0)
        c1_t = greenlet(c_1)
        c1_t.data = None
        c0_t.switch()

        self.assertTrue(None != c1_t.data)
        self.assertEqual(2, len(c1_t.data))

        diff_in = Topo_Diff.from_json_dict(c1_t.data[0])
        commit_ret = Topo_Diff.Commit_Result_Type.from_json_dict(c1_t.data[1])

        self.assertEqual(Topo_Diff, type(diff_in))
        self.assertEqual(Topo_Diff.Commit_Result_Type, type(commit_ret))
Exemplo n.º 11
0
def generate_random_diff__topo__minimal(test_label):
    """
    @return: a ~minimal Topo_Diff containing three nodes and two links
    """
    n_0, n_0_id = generate_random_node_dict(test_label)
    n_1, n_1_id = generate_random_node_dict(test_label)
    n_2, n_2_id = generate_random_node_dict(test_label)

    l_0, l_0_id = generate_random_link_dict(test_label, n_0_id, n_1_id)
    l_1, l_1_id = generate_random_link_dict(test_label, n_0_id, n_2_id)

    n_set = [n_0, n_1, n_2]
    l_set = [l_0, l_1]
    topo_diff = Topo_Diff(node_set_add=n_set, link_set_add=l_set)
    return topo_diff
Exemplo n.º 12
0
    def on_diff_commit__topo(self, json_data):

        # FIXME: sanitize input
        json_dict = json.loads(json_data)
        topo_diff = Topo_Diff.from_json_dict(json_dict['topo_diff'])
        log.info('ws: rx: topo diff: ' + str(topo_diff))

        ctx = self.__context__common(json_dict)
        kernel = self.request.kernel
        topo_diff, commit_ret = kernel.diff_commit__topo(topo_diff, ctx)

        # handle serialization
        topo_diff_dict = topo_diff.to_json_dict()

        assert Topo_Diff.Commit_Result_Type == type(commit_ret)

        return self.multicast_msg('diff_commit__topo', topo_diff_dict,
                                  commit_ret)
Exemplo n.º 13
0
    def test_ws_event__topo_diff(self):

        class NS_test(BaseNamespace):

            def on_diff_commit__topo(self, *data):
                greenlet.getcurrent().data = data
                raise KeyboardInterrupt()  # TODO: cleanup: properly close socket

        test_label = neo4j_test_util.rand_label()
        n_0, n_0_id = test_util.generate_random_node_dict(test_label)
        n_1, n_1_id = test_util.generate_random_node_dict(test_label)
        l, l_id = test_util.generate_random_link_dict(test_label, n_0_id, n_1_id)
        topo_diff = Topo_Diff(node_set_add=[n_0, n_1], link_set_add=[l])

        def c_0():
            with RZ_websocket(namespace=NS_test) as (_, ns_sock):
                c1_t.switch()  # allow peer to connect
                data = json.dumps(topo_diff, cls=Topo_Diff.JSON_Encoder)
                ns_sock.emit('diff_commit__topo', data)
                c1_t.switch()

        def c_1():
            with RZ_websocket(namespace=NS_test) as (sock, _):
                c0_t.switch()  # allow peer to emit
                sock.wait(8)  # allow self to receive

        c0_t = greenlet(c_0)
        c1_t = greenlet(c_1)
        c1_t.data = None
        c0_t.switch()

        self.assertTrue(None != c1_t.data)
        self.assertEqual(2, len(c1_t.data))

        diff_in = Topo_Diff.from_json_dict(c1_t.data[0])
        commit_ret = Topo_Diff.Commit_Result_Type.from_json_dict(c1_t.data[1])

        self.assertEqual(Topo_Diff, type(diff_in))
        self.assertEqual(Topo_Diff.Commit_Result_Type, type(commit_ret))
Exemplo n.º 14
0
    def test_diff_commit__attr(self):
        # create test node
        test_label = neo4j_test_util.rand_label()
        n_0, n_0_id = generate_random_node_dict(test_label)
        n_0['attr_0'] = 0
        topo_diff = Topo_Diff(node_set_add=[n_0])
        op = DBO_diff_commit__topo(topo_diff)
        self.db_ctl.exec_op(op)

        # apply attr_diff
        attr_diff = Attr_Diff()
        attr_diff.add_node_attr_write(n_0_id, 'attr_0', 0)
        attr_diff.add_node_attr_write(n_0_id, 'attr_1', 'a')
        attr_diff.add_node_attr_rm(n_0_id, 'attr_2')

        op = DBO_diff_commit__attr(attr_diff)
        ret_diff = self.db_ctl.exec_op(op)

        self.assertEqual(len(ret_diff.type__node), 1)
        self.assertTrue(None != ret_diff.type__node[n_0_id])

        # attr-set only
        attr_diff = Attr_Diff()
        attr_diff.add_node_attr_write(n_0_id, 'attr_2', 0)

        op = DBO_diff_commit__attr(attr_diff)
        ret_diff = self.db_ctl.exec_op(op)
        self.assertTrue(
            None != ret_diff.type__node[n_0_id]['__attr_write'].get('attr_2'))

        # attr-remove only
        attr_diff = Attr_Diff()
        attr_diff.add_node_attr_rm(n_0_id, 'attr_2')

        op = DBO_diff_commit__attr(attr_diff)
        ret_diff = self.db_ctl.exec_op(op)
        self.assertTrue(
            'attr_2' in ret_diff.type__node[n_0_id]['__attr_remove'])
Exemplo n.º 15
0
    def process_result_set(self):
        ret_nid_set_add = []
        ret_lid_set_add = []
        ret_nid_set_rm = []
        ret_lid_set_rm = []
        it = self.iter__sub_op()

        if self.n_add_map:
            for _, _, r_set in it.next().iter__r_set(
            ):  # iterate over result sets
                for row in r_set:
                    for ret_dict in row:
                        n_id = ret_dict['id']  # see query return statement
                        ret_nid_set_add.append(n_id)

        if self.l_add_map:
            for _, _, r_set in it.next().iter__r_set(
            ):  # iterate over result sets
                for row in r_set:
                    for ret_dict in row:
                        l_id = ret_dict['id']  # see query return statement
                        ret_lid_set_add.append(l_id)

        if self.l_rm_set:
            for _, _, row_set in it.next().iter__r_set():
                for l_id in row_set:
                    ret_lid_set_rm.extend(l_id)

        if self.n_rm_set:
            for _, _, row_set in it.next().iter__r_set():
                for n_id in row_set:
                    ret_nid_set_rm.extend(n_id)

        ret = Topo_Diff.Commit_Result_Type(node_id_set_add=ret_nid_set_add,
                                           link_id_set_add=ret_lid_set_add,
                                           node_id_set_rm=ret_nid_set_rm,
                                           link_id_set_rm=ret_lid_set_rm)
        return ret
Exemplo n.º 16
0
    def process_result_set(self):
        ret_n_set = []
        ret_l_set = []
        for _, _, r_set in self.iter__r_set():
            for row in r_set:
                n, n_lbl_set, l_set = row.items()  # see query return statement

                # reconstruct nodes
                assert None != n.get('id'), "db contains nodes with no id"

                n['__label_set'] = self.process_q_ret__n_label_set(n_lbl_set)

                ret_n_set.append(n)

                # reconstruct links from link tuples
                for l_tuple in l_set:
                    assert 3 == len(l_tuple)  # see query return statement

                    if None == l_tuple[0]:  # check if link dst is None
                        # as link matching is optional, collect may yield empty sets
                        continue

                    ret_l, ret_l_type, ret_l_dst_id = l_tuple
                    l = Link.Link_Ptr(src_id=n['id'], dst_id=ret_l_dst_id)
                    l['id'] = ret_l['id']
                    l['__type'] = self.process_q_ret__l_type(ret_l_type)

                    ret_l_set.append(l)

        if len(ret_n_set
               ) >= self.limit:  # TODO: generalize logic, mv to DB_Driver
            log.warning(
                'DB op result set larger than query limit: size: %d, limit: %d'
                % (len(ret_n_set), self.limit))

        topo_diff = Topo_Diff(node_set_add=ret_n_set, link_set_add=ret_l_set)
        return topo_diff
Exemplo n.º 17
0
    def test_diff_commit__topo(self):
        test_label = neo4j_test_util.rand_label()
        n_0, n_0_id = generate_random_node_dict(test_label)
        n_1, n_1_id = generate_random_node_dict(test_label)
        n_2, n_2_id = generate_random_node_dict(test_label)

        l_0, l_0_id = generate_random_link_dict(test_label, n_0_id, n_1_id)
        l_1, l_1_id = generate_random_link_dict(test_label, n_0_id, n_2_id)

        n_set = [n_0, n_1, n_2]
        l_set = [l_0, l_1]
        topo_diff = Topo_Diff(node_set_add=n_set, link_set_add=l_set)

        # commit diff
        op = DBO_diff_commit__topo(topo_diff)
        ret_topo_diff = self.db_ctl.exec_op(op)

        # test return type
        self.assertTrue(hasattr(ret_topo_diff, 'node_id_set_add'))
        self.assertTrue(hasattr(ret_topo_diff, 'link_id_set_add'))
        self.assertTrue(hasattr(ret_topo_diff, 'node_id_set_rm'))
        self.assertTrue(hasattr(ret_topo_diff, 'link_id_set_rm'))

        # test return set lengths
        self.assertEqual(len(ret_topo_diff.node_id_set_add), len(n_set))
        self.assertEqual(len(ret_topo_diff.link_id_set_add), len(l_set))
        self.assertEqual(len(ret_topo_diff.node_id_set_rm), 0)
        self.assertEqual(len(ret_topo_diff.link_id_set_rm), 0)

        # assert nodes persisted
        id_set = self.db_ctl.exec_op(
            DBO_match_node_set_by_id_attribute([n_0_id, n_1_id]))
        self.assertEqual(len(id_set), 2)

        # assert links persisted
        l_ptr_0 = Link.link_ptr(src_id=n_0_id, dst_id=n_1_id)
        l_ptr_1 = Link.link_ptr(src_id=n_0_id, dst_id=n_2_id)
        op = DBO_load_link_set.init_from_link_ptr_set([l_ptr_0, l_ptr_1])
        id_set = self.db_ctl.exec_op(op)
        self.assertEqual(len(id_set), 2)

        # remova links
        topo_diff = Topo_Diff(link_id_set_rm=[l_0_id, l_1_id])
        op = DBO_diff_commit__topo(topo_diff)
        ret_topo_diff = self.db_ctl.exec_op(op)
        self.assertEqual(len(ret_topo_diff.link_id_set_rm), 2)

        # assert links removed
        op = DBO_load_link_set.init_from_link_ptr_set([l_ptr_0, l_ptr_1])
        id_set = self.db_ctl.exec_op(op)
        self.assertEqual(len(id_set), 0)

        # removal nodes
        topo_diff = Topo_Diff(node_id_set_rm=[n_2_id])
        op = DBO_diff_commit__topo(topo_diff)
        ret_topo_diff = self.db_ctl.exec_op(op)
        self.assertEqual(len(ret_topo_diff.node_id_set_rm), 1)

        # assert nodes removed
        op = DBO_match_node_set_by_id_attribute([n_2_id])
        id_set = self.db_ctl.exec_op(op)
        self.assertEqual(len(id_set), 0)
Exemplo n.º 18
0
def merge_topos(topos, names):
    """
    Merge a number of Topo_Diff's

    result node set is union of nodes
    conflicting nodes:
        same name nodes take the first (id wise)
        attributes: take the first
    result link set is merged (well defined: replace dropped id by chosen id in all links)
    """
    result = Topo_Diff()
    node_names = dict()
    node_ids = set()
    dropped_nodes = set()
    links_src_dst = set()

    def rename(links, from_id, to_id):
        for link in links:
            if link['__src_id'] == from_id:
                link['__src_id'] = to_id
            if link['__dst_id'] == from_id:
                link['__dst_id'] = to_id

    def merge(topo):
        links = topo.link_set_add
        nodes = topo.node_set_add
        new_nodes = []
        for node in nodes:
            name = node['name'].lower()
            if name in node_names:
                dropped_nodes.add(node['name'])
                rename(links, node['id'], node_names[name]['id'])
            else:
                node_names[name] = node
                new_nodes.append(node)
                node_ids.add(node['id'])
        new_links = []
        for link in links:
            k = (link['__src_id'], link['__dst_id'])
            if k not in links_src_dst:
                links_src_dst.add(k)
                new_links.append(link)
        if len(dropped_nodes) > 0:
            print("dropped duplicate nodes count: %s" % len(dropped_nodes))
        if len(new_links) != len(links):
            print("dropped duplicate links count: %s" % (len(links) - len(new_links)))
        print("adding %s nodes, %s links" % (len(new_nodes), len(new_links)))
        result.node_set_add.extend(new_nodes)
        result.link_set_add.extend(new_links)

    for topo, name in zip(topos, names):
        print("merging %s" % name)
        merge(topo)
    # now realloc all ids since otherwise they are duplicates of originals
    renames = [(node['id'], create_id()) for node in result.node_set_add]
    def show(word):
        print("=" * 80)
        print(word)
        print(result.node_set_add)
        print(result.link_set_add)
    for src, dst in renames:
        rename(result.link_set_add, src, dst)
    for node, (_, new_id) in zip(result.node_set_add, renames):
        node['id'] = new_id
    return result
Exemplo n.º 19
0
     print('\n'.join('%30s %30s' % (d['name'].encode('utf-8').ljust(30),
                                    d['id'].encode('utf-8').ljust(30)) for d in kernel.rzdoc__search('')))
     raise SystemExit
 if args.list_names:
     print('\n'.join(d['name'].encode('utf-8') for d in kernel.rzdoc__search('')))
     raise SystemExit
 if args.delete:
     remove(args.delete)
 if args.rename_from and args.rename_to:
     rename(args.rename_from, args.rename_to)
 if args.clone:
     print(json.dumps(clone(args.clone).to_json_dict()))
 if args.create and args.create_name:
     with open(args.create) as fd:
         data = json.load(fd)
         create(args.create_name, Topo_Diff.from_json_dict(json.load(fd)))
 if args.merge_target:
     merge_sources = None
     if args.merge:
         merge_sources = args.merge.split(',')
     if args.merge_file:
         with open(args.merge_file) as fd:
             merge_sources = [noeol(line) for line in fd.readlines()]
     if merge_sources:
         merge(args.merge_target, merge_sources)
 if args.dump_single:
     dump([args.dump_single])
 if args.dump_all:
     dump(None)
 if args.load:
     load(args.load)