Exemplo n.º 1
0
def test_zmq_ipc():
    io_threads = 1
    zmq_transport = 'ipc'
    tornado_sockets = False
    num_ventilators = 2
    num_workers = 4
    num_sinks = 2
    is_dispatcher = True
    db_num = 1
    connection_info = {}
    r = connect_redis(db_num, connection_info)
    r.flushdb()
    with MultiprocessEnvironment(zmq_transport, tornado_sockets, io_threads,
                   num_ventilators, num_workers, num_sinks,
                   is_dispatcher, db_num) as g:
        procedure, data, result = 'sleep', [-1], 1
        for i in range(10):
            assert g.run(procedure, data) == result
        for i in range(3):
            procedure, data, result = 'test', [-1, i], i
        assert g.run(procedure, data) == result
        list_iter = [10, 10]
        num_iter = reduce(lambda x, y: (x + 1) * y, reversed(list_iter)) + 1
        procedure, data, result = 'test_deep', [-1, list_iter], num_iter
        assert g.run(procedure, data) == result
        g.wait_finish()
        assert g.is_clean()
    assert g.is_closed(linger=0.1)
    assert not r.keys()
    r.flushdb()
    import shutil
    from settings.settings import DIR_IPC
    shutil.rmtree(DIR_IPC)
Exemplo n.º 2
0
 def _get_group_id(self):
     """Get the name of the group for this instance. Group must be unique
     between different instances of the class."""
     from pipeline.models import connect_redis, code_done
     self.redis = connect_redis(self.db_num, self.connection_info)
     with self.redis.pipeline() as pipe:
         while True:
             try:
                 pipe.watch('group')
                 pipe.multi()
                 pipe.exists('group')
                 new = pipe.execute()[0]
                 if self.is_dispatcher:
                     new_code = -1
                 else:
                     if not new:
                         new_code = 0
                     else:
                         pipe.watch('group')
                         pipe.multi()
                         pipe.sort('group', start=0, num=1, desc=1)
                         sorted_keys = pipe.execute()[0]
                         if sorted_keys:
                             new_code = int(sorted_keys[0]) + 1
                         else:
                             new_code = 0
                 saved = self.db_add_group_id(new_code)
                 if saved == str(new_code):
                     continue
                 return code_done(str(new_code))
             except WatchError:
                 continue
             finally:
                 pipe.reset()
Exemplo n.º 3
0
def test_interrupt_zmq_tcp():
    io_threads = 1
    zmq_transport = 'tcp'
    tornado_sockets = False
    num_ventilators = 2
    num_workers = 4
    num_sinks = 2
    is_dispatcher = True
    db_num = 1
    connection_info = {}
    r = connect_redis(db_num, connection_info)
    r.flushdb()
    with MultiprocessEnvironment(zmq_transport, tornado_sockets, io_threads,
                                 num_ventilators, num_workers, num_sinks,
                                 is_dispatcher, db_num,
                                 connection_info) as g_dispatcher:
        is_dispatcher = False
        with MultiprocessEnvironment(zmq_transport, tornado_sockets,
                                     io_threads, num_ventilators, num_workers,
                                     num_sinks, is_dispatcher, db_num,
                                     connection_info) as g_node:
            import time
            procedure, data = 'test', [0.01, 1000, []]
            g_dispatcher.server.to_socket.send(procedure, SNDMORE)
            for d in data[:-1]:
                g_dispatcher.server.to_socket.send_json(d, SNDMORE)
            g_dispatcher.server.to_socket.send_json(data[-1])
            time.sleep(4)
            g_node.clean_exit()
            g_dispatcher.server.to_socket.poll()
            res = [g_dispatcher.server.to_socket.recv_json(RCVMORE)]
            while g_dispatcher.server.to_socket.RCVMORE:
                res.append(g_dispatcher.server.to_socket.recv_json(RCVMORE))
            print res
        g_dispatcher.wait_finish()
        assert g_dispatcher.is_clean()
    assert g_dispatcher.is_closed(linger=0.1)
    assert g_node.is_closed(linger=0.1)
    assert not r.keys()
    r.flushdb()
Exemplo n.º 4
0
def test_interrupt_zmq_tcp():
    io_threads = 1
    zmq_transport = 'tcp'
    tornado_sockets = False
    num_ventilators = 2
    num_workers = 4
    num_sinks = 2
    is_dispatcher = True
    db_num = 1
    connection_info = {}
    r = connect_redis(db_num, connection_info)
    r.flushdb()
    with MultiprocessEnvironment(zmq_transport, tornado_sockets, io_threads,
                   num_ventilators, num_workers, num_sinks,
                   is_dispatcher, db_num, connection_info) as g_dispatcher:
        is_dispatcher = False
        with MultiprocessEnvironment(zmq_transport, tornado_sockets,
                    io_threads, num_ventilators, num_workers, num_sinks,
                    is_dispatcher, db_num, connection_info) as g_node:
            import time
            procedure, data = 'test', [0.01, 1000, []]
            g_dispatcher.server.to_socket.send(procedure, SNDMORE)
            for d in data[:-1]:
                g_dispatcher.server.to_socket.send_json(d, SNDMORE)
            g_dispatcher.server.to_socket.send_json(data[-1])
            time.sleep(4)
            g_node.clean_exit()
            g_dispatcher.server.to_socket.poll()
            res = [g_dispatcher.server.to_socket.recv_json(RCVMORE)]
            while g_dispatcher.server.to_socket.RCVMORE:
                res.append(g_dispatcher.server.to_socket.recv_json(RCVMORE))
            print res
        g_dispatcher.wait_finish()
        assert g_dispatcher.is_clean()
    assert g_dispatcher.is_closed(linger=0.1)
    assert g_node.is_closed(linger=0.1)
    assert not r.keys()
    r.flushdb()
Exemplo n.º 5
0
def test_connection_redis():
    from pipeline.models import connect_redis
    connect_redis()