Esempio n. 1
0
    def send_to_butler(self, command_line, widget_proxy, live):
        you = u"You > {0}".format(command_line)
        self.parent.value.set_values(self.parent.value.get() + [you])
        self.parent.wMain.values = self.parent.value.get()
        self.parent.wMain.display()

        # handle special commands
        if command_line.lower() == "quit":
            IOLoop.instance().stop()
            sys.exit(0)  # why is this needed ?

        elif command_line.lower() == "reload":
            try:
                cli = MQSyncReq(zmq.Context())
                msg = MQMessage()
                msg.set_action('butler.reload.do')
                result = cli.request('butler', msg.get(), timeout=10).get()
                if result:
                    msg = "*** action reload : {0}".format(result)
                    self.parent.value.set_values(self.parent.value.get() + [msg])
            except:
                msg = u"*** action reload : error (is butler component ok ?)"
                self.parent.value.set_values(self.parent.value.get() + [msg])

        # handle butler
        else:
            self.parent.butler_cb(command_line, identity = "cli user", media = "chat", location = None, mood = None)

        self.parent.wMain.values = self.parent.value.get()
        self.parent.wMain.display()
Esempio n. 2
0
    def ready(self, ioloopstart=1):
        """ to call at the end of the __init__ of classes that inherits of XplPlugin
        """
        if self.dont_run_ready == True:
            return

        ### activate xpl hbeat
        if self.enable_hbeat_called == True:
            self.log.error(u"in ready() : enable_hbeat() function already called : the plugin may not be fully converted to the 0.4+ Domogik format")
        else:
            self.enable_hbeat()

        # send the status for the xpl hbeat
        self.myxpl.update_status(2)

        ### send plugin status : STATUS_ALIVE
        # TODO : why the dbmgr has no self._name defined ???????
        # temporary set as unknown to avoir blocking bugs
        if not hasattr(self, '_name'):
            self._name = "unknown"
        self._set_status(STATUS_ALIVE)

        ### Instantiate the MQ
        # nothing can be launched after this line (blocking call!!!!)
        self.log.info(u"Start IOLoop for MQ : nothing else can be executed in the __init__ after this! Make sure that the self.ready() call is the last line of your init!!!!")
        if ioloopstart == 1:
            IOLoop.instance().start()
Esempio n. 3
0
 def _connect(self):
     context = zmq.Context()
     url = "{0}:{1}".format(BinderDSettings.BROKER_HOST,
                            BinderDSettings.BROKER_PORT)
     worker = BinderDModule.Worker(self, context, url, bytes(self.TAG))
     IOLoop.instance().start()
     worker.shutdown()
Esempio n. 4
0
    def _setup_pipe_in(self):
        """setup listening pipe for subprocesses"""
        ctx = self.pub_socket.context

        # use UUID to authenticate pipe messages
        self._pipe_uuid = uuid.uuid4().bytes

        self._pipe_in = ctx.socket(zmq.PULL)
        self._pipe_in.linger = 0
        try:
            self._pipe_port = self._pipe_in.bind_to_random_port(
                "tcp://127.0.0.1")
        except zmq.ZMQError as e:
            warn("Couldn't bind IOStream to 127.0.0.1: %s" % e +
                 "\nsubprocess output will be unavailable.")
            self._pipe_flag = False
            self._pipe_in.close()
            del self._pipe_in
            return
        self._pipe_poller = zmq.Poller()
        self._pipe_poller.register(self._pipe_in, zmq.POLLIN)
        if IOLoop.initialized():
            # subprocess flush should trigger flush
            # if kernel is idle
            IOLoop.instance().add_handler(
                self._pipe_in,
                lambda s, event: self.flush(),
                IOLoop.READ,
            )
Esempio n. 5
0
def main():
    """main method"""

    url_worker = "ipc://backend.ipc"
    url_client = "ipc://frontend.ipc"

    # Prepare our context and sockets
    context = zmq.Context()
    frontend = context.socket(zmq.ROUTER)
    frontend.bind(url_client)
    backend = context.socket(zmq.ROUTER)
    backend.bind(url_worker)

    # create workers and clients threads
    for i in range(NBR_WORKERS):
        thread = threading.Thread(target=worker_thread, args=(url_worker, i, ))
        thread.daemon = True
        thread.start()

    for i in range(NBR_CLIENTS):
        thread_c = threading.Thread(target=client_thread, args=(url_client, i, ))
        thread_c.daemon = True
        thread_c.start()

    # create queue with the sockets
    queue = LRUQueue(backend, frontend)

    # start reactor
    IOLoop.instance().start()
Esempio n. 6
0
def main():
    http_fetcher = fetcher.HTTPFetcher()
    rb = robot.Robot(http_fetcher, [
        ('/.*', AHandler),
    ])
    worker.Worker(rb).start()
    IOLoop.instance().start()
Esempio n. 7
0
def main(basedir, n_workers, daemon_flag,
         client_socket, worker_socket,
         worker_bin, deployment, notify_socket,
         journal_socket):
    """main method"""
    # create queue with the sockets
    signal.signal(signal.SIGCHLD, signal_handler)
    LRUQueue(worker_socket, client_socket,
             basedir, daemon_flag, worker_bin,
             deployment)
    worker_env = {
        'WORKER_SOCKET': worker_socket,
        'LAF_DEPLOYMENT': deployment
    }
    if notify_socket:
        os.environ['NOTIFICATION_SOCK'] = notify_socket
        worker_env['NOTIFICATION_SOCK'] = notify_socket
    if journal_socket:
        os.environ['JOURNAL_SOCK'] = journal_socket
        worker_env['JOURNAL_SOCK'] = journal_socket
    for _ in range(int(n_workers)):
        subprocess.Popen([worker_bin, basedir],
                         env=worker_env,
                         shell=False)

    # start reactor
    IOLoop.instance().start()
Esempio n. 8
0
def device_manager(context):

    global manager
    global publisher

    worker_address = Config.get('services', 'worker_address')
    worker_port = Config.get('services', 'worker_port')
    worker_address = "tcp://" + worker_address + ":" + worker_port

    publisher_address = Config.get('device_service', 'publisher_address')
    publisher_port = Config.get('device_service', 'publisher_port')
    publisher_address = "tcp://" + publisher_address + ":" + publisher_port

    service = Config.get('device_service', 'service_name')

    publisher = DeviceServicePublisher(context, publisher_address)
    manager = DeviceServiceManager(context, worker_address, service)

    try:
        IOLoop.instance().start()
    except KeyboardInterrupt:
        IOLoop.instance().stop()
        publisher.shutdown()
        manager.shutdown()

    return
Esempio n. 9
0
    def on_message(self, msgid, content):
        """ Handle MQ messages
            @param msgid : message id
            @content : message content
        """
        if msgid == "plugin.status":
            # we may miss starting and stop-request events but we only want to do some checks on alive and stopped...
            # and sometimes it happens that we still receive a last 'alive' status before the 'stop' one
            if self.count == 0:
                print(u"Message skipped (we skip the first one) : msgid={0}, content={1}".format(msgid, content))
                self.count = 1
                return 

            print(u"Message received : msgid={0}, content={1}".format(msgid, content))
            if content['name'] == self.name and \
               content['type'] == self.type and \
               content['host'] == self.host:
                self.plugin_status = content['event']
                # plugin started
                if content['event'] == STATUS_ALIVE:
                    print(u"Plugin is started")
                    print(u"Stop listening to MQ as we get our result")
                    IOLoop.instance().stop() 
    
                # plugin stopped
                elif content['event'] == STATUS_STOPPED:
                    print(u"Plugin is stopped")
                    print(u"Stop listening to MQ as we get our result")
                    IOLoop.instance().stop() 
Esempio n. 10
0
def main():
    """main method"""

    url_worker = "ipc://backend.ipc"
    url_client = "ipc://frontend.ipc"

    # Prepare our context and sockets
    context = zmq.Context()
    frontend = context.socket(zmq.ROUTER)
    frontend.bind(url_client)
    backend = context.socket(zmq.ROUTER)
    backend.bind(url_worker)

    # create workers and clients threads
    for i in range(NBR_WORKERS):
        thread = threading.Thread(target=worker_thread, args=(url_worker, i, ))
        thread.daemon = True
        thread.start()

    for i in range(NBR_CLIENTS):
        thread_c = threading.Thread(target=client_thread,
                                    args=(url_client, i, ))
        thread_c.daemon = True
        thread_c.start()

    # create queue with the sockets
    queue = LRUQueue(backend, frontend)

    # start reactor
    IOLoop.instance().start()
Esempio n. 11
0
    def _setup_pipe_in(self):
        """setup listening pipe for subprocesses"""
        ctx = self.pub_socket.context

        # use UUID to authenticate pipe messages
        self._pipe_uuid = uuid.uuid4().bytes

        self._pipe_in = ctx.socket(zmq.PULL)
        self._pipe_in.linger = 0
        try:
            self._pipe_port = self._pipe_in.bind_to_random_port("tcp://127.0.0.1")
        except zmq.ZMQError as e:
            warn("Couldn't bind IOStream to 127.0.0.1: %s" % e +
                "\nsubprocess output will be unavailable."
            )
            self._pipe_flag = False
            self._pipe_in.close()
            del self._pipe_in
            return
        self._pipe_poller = zmq.Poller()
        self._pipe_poller.register(self._pipe_in, zmq.POLLIN)
        if IOLoop.initialized():
            # subprocess flush should trigger flush
            # if kernel is idle
            IOLoop.instance().add_handler(self._pipe_in,
                lambda s, event: self.flush(),
                IOLoop.READ,
            )
Esempio n. 12
0
    def on_message(self, msgid, content):
        """ Handle MQ messages
            @param msgid : message id
            @content : message content
        """
        if msgid == "plugin.status":
            # we may miss starting and stop-request events but we only want to do some checks on alive and stopped...
            # and sometimes it happens that we still receive a last 'alive' status before the 'stop' one
            if self.count == 0:
                print(
                    u"Message skipped (we skip the first one) : msgid={0}, content={1}"
                    .format(msgid, content))
                self.count = 1
                return

            print(u"Message received : msgid={0}, content={1}".format(
                msgid, content))
            if content['name'] == self.name and \
               content['type'] == self.type and \
               content['host'] == self.host:
                self.plugin_status = content['event']
                # plugin started
                if content['event'] == STATUS_ALIVE:
                    print(u"Plugin is started")
                    print(u"Stop listening to MQ as we get our result")
                    IOLoop.instance().stop()

                # plugin stopped
                elif content['event'] == STATUS_STOPPED:
                    print(u"Plugin is stopped")
                    print(u"Stop listening to MQ as we get our result")
                    IOLoop.instance().stop()
Esempio n. 13
0
 def on_message(self, msg):
     if _do_print:
         print 'client received:',
         pprint(msg)
     self.last_msg = msg
     IOLoop.instance().stop()
     return
Esempio n. 14
0
 def _publish_msg(self,
                  msg_type,
                  data=None,
                  metadata=None,
                  buffers=None,
                  **keys):
     """Helper for sending a comm message on IOPub"""
     if threading.current_thread(
     ).name != 'MainThread' and IOLoop.initialized():
         # make sure we never send on a zmq socket outside the main IOLoop thread
         IOLoop.instance().add_callback(lambda: self._publish_msg(
             msg_type, data, metadata, buffers, **keys))
         return
     data = {} if data is None else data
     metadata = {} if metadata is None else metadata
     content = json_clean(dict(data=data, comm_id=self.comm_id, **keys))
     self.session.send(
         self.iopub_socket,
         msg_type,
         content,
         metadata=json_clean(metadata),
         parent=self.kernel._parent_header,
         ident=self.topic,
         buffers=buffers,
     )
Esempio n. 15
0
    def ready(self, ioloopstart=1):
        """ to call at the end of the __init__ of classes that inherits of XplPlugin
        """
        if self.dont_run_ready == True:
            return

        ### activate xpl hbeat
        if self.enable_hbeat_called == True:
            self.log.error(u"in ready() : enable_hbeat() function already called : the plugin may not be fully converted to the 0.4+ Domogik format")
        else:
            self.enable_hbeat()

        # send the status for the xpl hbeat
        self.myxpl.update_status(2)

        ### send plugin status : STATUS_ALIVE
        # TODO : why the dbmgr has no self._name defined ???????
        # temporary set as unknown to avoir blocking bugs
        if not hasattr(self, '_name'):
            self._name = "unknown"
        self._set_status(STATUS_ALIVE)

        ### Instantiate the MQ
        # nothing can be launched after this line (blocking call!!!!)
        self.log.info(u"Start IOLoop for MQ : nothing else can be executed in the __init__ after this! Make sure that the self.ready() call is the last line of your init!!!!")
        if ioloopstart == 1:
            IOLoop.instance().start()
Esempio n. 16
0
def main():
    http_fetcher = fetcher.HTTPFetcher()
    rb = robot.Robot(http_fetcher, [
        ('/.*', AHandler),
    ])
    worker.Worker(rb).start()
    IOLoop.instance().start()
Esempio n. 17
0
def main(pat):
    
    fname = find_connection_file(pat)
    with open(fname) as f:
        cfg = json.load(f)
    
    url = "%s://%s:%s" % (cfg.get('transport', 'tcp'), cfg['ip'], cfg['iopub_port'])
    
    session = Session(key=cfg['key'])
    
    ctx = zmq.Context.instance()
    sub = ctx.socket(zmq.SUB)
    sub.subscribe = b''
    sub.connect(url)
    # import IPython
    # IPython.embed()
    # return
    
    stream = ZMQStream(sub)
    
    stream.on_recv(lambda msg_list: log_msg(session, msg_list))
    
    pc = PeriodicCallback(print_time, 5 * 60 * 1000)
    pc.start()
    IOLoop.instance().start()
Esempio n. 18
0
 def on_message(self, msg):
     if _do_print:
         print 'client received:',
         pprint(msg)
     self.last_msg = msg
     IOLoop.instance().stop()
     return
Esempio n. 19
0
    def __init__(self, socket, pipe=False):
        # type: (object, object) -> object
        """Create IOPub thread

        Parameters
        ----------

        socket: zmq.PUB Socket
            the socket on which messages will be sent.
        pipe: bool
            Whether this process should listen for IOPub messages
            piped from subprocesses.
        """
        self.socket = socket
        self.background_socket = BackgroundSocket(self)
        self._master_pid = os.getpid()
        self._pipe_flag = pipe
        self.io_loop = IOLoop()
        if pipe:
            self._setup_pipe_in()
        self._local = threading.local()
        self._events = {}
        self._setup_event_pipe()
        self.thread = threading.Thread(target=self._thread_main)
        self.thread.daemon = True
Esempio n. 20
0
def subscriber_manager(context):

    global subscriber
    global publisher
    global subscriber_topics

    session = Session()

    # Set up forwarder device
    logger.debug("Configuring forwarder device")
    forwarder_subscriber_address = session.query(RoutingDefiniton.route) \
                                          .filter_by(name="internal_pub") \
                                          .scalar()
    forwarder_publisher_address = session.query(RoutingDefiniton.route) \
                                         .filter_by(name="internal_sub") \
                                         .scalar()

    forwarder = ProcessDevice(zmq.FORWARDER, zmq.SUB, zmq.PUB)
    forwarder.connect_in(forwarder_subscriber_address)
    forwarder.setsockopt_in(zmq.SUBSCRIBE, "")
    logger.debug("forwarder connect in: {0}".format(forwarder_subscriber_address))

    forwarder.bind_out(forwarder_publisher_address)
    logger.debug("forwarder bind out: {0}".format(forwarder_publisher_address))

    # Set up subscriber listening to Farm-Monitor server
    subscriber_address = session.query(RoutingDefiniton.route)\
                                .filter_by(name="subscriber_address").scalar()
    subscriber_port = session.query(RoutingDefiniton.route)\
                             .filter_by(name="subscriber_port").scalar()
    subscriber_address = "tcp://" + subscriber_address + ":" + subscriber_port
    device_id = session.query(HardwareDefinition.serial_number).scalar()
    subscriber_topics.append(device_id)

    subscriber = Subscriber(context, subscriber_address, subscriber_topics)
    subscriber.subscriber.on_recv(callback=server_subscriber_recv)
    logger.debug("Subscriber listening to: {0} topics: {1}".format(subscriber_address,
                                                                   subscriber_topics))

    # Set up publisher to forward messages from Farm-Monitor to internal
    publisher_address = session.query(RoutingDefiniton.route)\
                               .filter_by(name="internal_pub").scalar()
    publisher = Publisher(context, publisher_address)
    logger.debug("Publisher configured to: {0}".format(publisher_address))

    session.close()
    try:
        logger.info("starting forwarder, Subscriber, and Publisher")
        # Start the forwarder
        forwarder.start()
        IOLoop.instance().start()

    except KeyboardInterrupt:
        logger.info("stopping forwarder, Subscriber, and Publisher")
        IOLoop.instance().stop()
        subscriber.shutdown()
        publisher.shutdown()

    return
Esempio n. 21
0
 def test_03_timeout_01(self):
     """Test MDPclient request w/ timeout.
     """
     client = MyClient(self.context, self.endpoint, self.service)
     client.request(b'XXX', 20) # 20 millisecs timeout
     IOLoop.instance().start()
     client.shutdown()
     self.assertEquals(client.timed_out, True)
     return
Esempio n. 22
0
 def test_03_timeout_01(self):
     """Test MNClient request w/ timeout.
     """
     client = MyClient(self.context, self.endpoint, self.service)
     client.request(b'XXX', 20)  # 20 millisecs timeout
     IOLoop.instance().start()
     client.shutdown()
     self.assertEqual(client.timed_out, True)
     return
Esempio n. 23
0
class AsyncServerAdapter(object):
    producer_class = Producer

    def __init__(self, backend_rep_uri, frontend_rep_uri, frontend_pub_uri,
                 control_pipe=None):
        self.uris = OrderedDict([
            ('backend_rep', backend_rep_uri),
            ('consumer_push_be', unique_ipc_uri()),
            ('consumer_pull_be', unique_ipc_uri()),
            ('frontend_rep_uri', frontend_rep_uri),
            ('frontend_pub_uri', frontend_pub_uri)
        ])
        self.control_pipe = control_pipe
        self.done = False
        logging.getLogger(log_label(self)).info("uris: %s", self.uris)

    def watchdog(self):
        if self.control_pipe is None:
            return
        elif not self.done and self.control_pipe.poll():
            self.done = True
            self.finish()

    def run(self):
        consumer = Process(target=Consumer(self.uris['backend_rep'],
                                        self.uris['consumer_push_be'],
                                        self.uris['consumer_pull_be']).run
        )
        producer = Process(target=self.producer_class(
                self.uris['frontend_rep_uri'],
                self.uris['frontend_pub_uri'],
                self.uris['consumer_pull_be'],
                self.uris['consumer_push_be']).run
        )
        self.io_loop = IOLoop()
        periodic_callback = PeriodicCallback(self.watchdog, 500, self.io_loop)
        periodic_callback.start()
        try:
            consumer.start()
            producer.start()
            self.io_loop.start()
        except KeyboardInterrupt:
            pass
        producer.terminate()
        consumer.terminate()
        logging.getLogger(log_label(self)).info('PRODUCER and CONSUMER have '
                                                'been terminated')

    def __del__(self):
        uris = [self.uris[label] for label in ('consumer_push_be',
                                               'consumer_pull_be', )]
        cleanup_ipc_uris(uris)

    def finish(self):
        logging.getLogger(log_label(self)).debug('"finish" request received')
        self.io_loop.stop()
Esempio n. 24
0
    def _schedule_flush(self):
        """schedule a flush in the main thread

        only works with a tornado/pyzmq eventloop running
        """
        if IOLoop.initialized():
            IOLoop.instance().add_callback(self.flush)
        else:
            # no async loop, at least force the timer
            self._start = 0
Esempio n. 25
0
def main(start_script_file, running_class_def=None):
    '''主函数,传入开始执行的主函数,然后以指定的类作为起始工作实例,为单实例机器的程序入口'''
    current_environment_pack = get_current_environment_pack(
        start_script_absolute_filename=start_script_file)
    if running_class_def:
        current_working_instance = running_class_def(current_environment_pack)
        #print id(current_working_engine),'这里是主进程'

    IOLoop.instance().start()
    pass
Esempio n. 26
0
 def __init__(self, socket, pipe=False):
     self.socket = socket
     self.background_socket = BackgroundSocket(self)
     self._master_pid = os.getpid()
     self._pipe_flag = pipe
     self.io_loop = IOLoop()
     if pipe:
         self._setup_pipe_in()
     self.thread = threading.Thread(target=self._thread_main)
     self.thread.daemon = True
Esempio n. 27
0
    def _schedule_flush(self):
        """schedule a flush in the main thread

        only works with a tornado/pyzmq eventloop running
        """
        if IOLoop.initialized():
            IOLoop.instance().add_callback(self.flush)
        else:
            # no async loop, at least force the timer
            self._start = 0
Esempio n. 28
0
 def _on_msg(self, msg):
     self._msgs.append(msg)
     if _do_print:
         print('broker received:', msg)
     if self.broker.do_reply:
         new_msg = msg[:4]
         new_msg.append(b'REPLY')
         self.broker.send_multipart(new_msg)
     else:
         IOLoop.instance().stop()
     return
Esempio n. 29
0
def main():
    from frontier import Frontier
    from options import parse_command_line

    parse_command_line()

    ft = Frontier([
        ('http://localhost/', 1),
    ])
    Master(ft).start()
    IOLoop.instance().start()
Esempio n. 30
0
def run(address, optional_address=None):
    context = zmq.Context()
    broker = DedalusRunner(context, address, optional_address)
    try:
        IOLoop.instance().start()
    except KeyboardInterrupt:
        _LOG.info("Interrupt received, stopping.")
    finally:
        # clean up
        broker.shutdown()
        context.term()
Esempio n. 31
0
def main():
    from frontier import Frontier
    from options import parse_command_line

    parse_command_line()

    ft = Frontier([
        ('http://localhost/', 1),
    ])
    Master(ft).start()
    IOLoop.instance().start()
 def test_01_simple_01(self):
     """Test MDPWorker simple req/reply.
     """
     self._start_broker()
     time.sleep(0.2)
     worker = MyWorker(self.context, self.endpoint, self.service)
     sender = DelayedCallback(self.send_req, 1000)
     sender.start()
     IOLoop.instance().start()
     worker.shutdown()
     self._stop_broker()
     return
Esempio n. 33
0
 def _on_msg(self, msg):
     self._msgs.append(msg)
     if _do_print:
         print 'broker received:',
         pprint(msg)
     if self.broker.do_reply:
         new_msg = msg[:4]
         new_msg.append(b'REPLY')
         self.broker.send_multipart(new_msg)
     else:
         IOLoop.instance().stop()
     return
Esempio n. 34
0
def run(address):
    context = zmq.Context()
    worker = WorkerRunner(context, address, WK_SERVICE)
    try:
        IOLoop.instance().start()
        worker.shutdown()
    except KeyboardInterrupt:
        _LOG.info("Interrupt received, stopping!")
    finally:
        # clean up
        worker.shutdown()
        context.term()
Esempio n. 35
0
    def _send_fireball(self, mtype, data, f_size):
        if len(data) < 3:
            logger.error('no data to send')
            return []

        self.loop = IOLoop().instance()
        self.socket.close()

        self.socket = self.context.socket(zmq.DEALER)
        self.socket.connect(self.remote)

        self.stream = ZMQStream(self.socket)
        self.stream.on_recv(self._handle_message_fireball)

        self.stream.io_loop.call_later(SNDTIMEO, self._fireball_timeout)

        self.response = []

        if PYVERSION == 3:
            if isinstance(data, bytes):
                data = data.decode('utf-8')

        data = json.loads(data)

        if not isinstance(data, list):
            data = [data]

        if (len(data) % f_size) == 0:
            self.num_responses = int((len(data) / f_size))
        else:
            self.num_responses = int((len(data) / f_size)) + 1

        logger.debug('responses expected: %i' % self.num_responses)

        batch = []
        for d in data:
            batch.append(d)
            if len(batch) == f_size:
                Msg(mtype=Msg.INDICATORS_CREATE, token=self.token, data=batch).send(self.socket)
                batch = []

        if len(batch):
            Msg(mtype=Msg.INDICATORS_CREATE, token=self.token, data=batch).send(self.socket)

        logger.debug("starting loop to receive")
        self.loop.start()

        # clean up FDs
        self.loop.close()
        self.stream.close()
        self.socket.close()
        return self.response
Esempio n. 36
0
File: low.py Progetto: themoo/Jelly
def main():
    context = zmq.Context(1)
    
    #qq = q.TheQueue(context, CLIENT_URL, WORKER_URL)
    qq = q.TheQueue(context, CLIENT_URL, WORKER_URL)
    for i in range(2):
        t = threading.Thread(target=w.TheWorker,args=(context,))
        t.start()
        #t.join()
    try:
        IOLoop.instance().start()
    except KeyboardInterrupt:
        times_str('ctrlc')
Esempio n. 37
0
def main():
    cfg = Loader('mq')
    my_conf = cfg.load()
    config = dict(my_conf[1])

    ctx = DaemonContext()
    ctx.open()

    context = zmq.Context()
    print(("tcp://{0}:{1}".format(config['ip'], config['req_rep_port'])))
    broker = MDPBroker(context, "tcp://{0}:{1}".format(config['ip'], config['req_rep_port']))
    IOLoop.instance().start()
    broker.shutdown()
Esempio n. 38
0
def run(address):

    context = zmq.Context()
    client = ClientRunner(context, address, SERVICE_ECHO)
    try:
        IOLoop.instance().start()
        client.shutdown()
    except KeyboardInterrupt:
        _LOG.info("Interrupt received, stopping!")
    finally:
        # clean up
        client.shutdown()
        context.term()
Esempio n. 39
0
    def __init__(self):
        '''
        Initialize database and xPL connection
        '''
        XplPlugin.__init__(self, 'dbmgr')
        MQRep.__init__(self, zmq.Context(), 'dbmgr')
        self.log.debug("Init database_manager instance")

        # Check for database connexion
        self._db = DbHelper()
        nb_test = 0
        db_ok = False
        while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY:
            nb_test += 1
            try:
                self._db.list_user_accounts()
                db_ok = True
            except:
                msg = "The database is not responding. Check your configuration of if the database is up. Test %s/%s" % (
                    nb_test, DATABASE_CONNECTION_NUM_TRY)
                print(msg)
                self.log.error(msg)
                msg = "Waiting for %s seconds" % DATABASE_CONNECTION_WAIT
                print(msg)
                self.log.info(msg)
                time.sleep(DATABASE_CONNECTION_WAIT)

        if nb_test >= DATABASE_CONNECTION_NUM_TRY:
            msg = "Exiting dbmgr!"
            print(msg)
            self.log.error(msg)
            self.force_leave()
            return

        msg = "Connected to the database"
        print(msg)
        self.log.info(msg)
        try:
            self._engine = self._db.get_engine()
        except:
            self.log.error("Error while starting database engine : %s" %
                           traceback.format_exc())
            self.force_leave()
            return

        Listener(self._request_config_cb, self.myxpl, {
            'schema': 'domogik.config',
            'xpltype': 'xpl-cmnd'
        })
        self.enable_hbeat()
        IOLoop.instance().start()
Esempio n. 40
0
def main():
    ft = frontier.Frontier([
        ('http://m.sohu.com/', 1),
    ])
    http_fetcher = fetcher.HTTPFetcher()
    rb = robot.Robot(http_fetcher, [
        ('/.*', AHandler),
    ])
    id = fork_processes(0)
    if id == master_id():
        master.Master(ft).start()
    else:
        worker.Worker(rb).start()
    IOLoop.instance().start()
Esempio n. 41
0
 def test_01_simple_01(self):
     """Test MDPWorker simple req/reply.
     """
     self._start_broker()
     time.sleep(0.2)
     worker = MyWorker(self.context, self.endpoint, self.service)
     sender = DelayedCallback(self.send_req, 500)
     stopper = DelayedCallback(self.stop_test, 2500)
     sender.start()
     stopper.start()
     IOLoop.instance().start()
     worker.shutdown()
     self._stop_broker()
     return
Esempio n. 42
0
 def test_04_receive_01(self):
     """Test MNClient message receive.
     """
     self._start_broker(do_reply=True)
     client = MyClient(self.context, self.endpoint, self.service)
     client.request(b'XXX')
     IOLoop.instance().start()
     client.shutdown()
     self._stop_broker()
     self.assertEqual(True, hasattr(client, 'last_msg'))
     self.assertEqual(3, len(client.last_msg))
     self.assertEqual(b'REPLY', client.last_msg[-1])
     self.assertEqual(self.service, client.last_msg[-2])
     return
Esempio n. 43
0
 def test_04_receive_01(self):
     """Test MDPclient message receive.
     """
     self._start_broker(do_reply=True)
     client = MyClient(self.context, self.endpoint, self.service)
     client.request(b'XXX')
     IOLoop.instance().start()
     client.shutdown()
     self._stop_broker()
     self.assertEquals(True, hasattr(client, b'last_msg'))
     self.assertEquals(3, len(client.last_msg))
     self.assertEquals(b'REPLY', client.last_msg[-1])
     self.assertEquals(self.service, client.last_msg[-2])
     return
Esempio n. 44
0
def main():
    ft = frontier.Frontier([
        ('http://m.sohu.com/', 1),
    ])
    http_fetcher = fetcher.HTTPFetcher()
    rb = robot.Robot(http_fetcher, [
        ('/.*', AHandler),
    ])
    id = fork_processes(0)
    if id == master_id():
        master.Master(ft).start()
    else:
        worker.Worker(rb).start()
    IOLoop.instance().start()
Esempio n. 45
0
def main():
    cfg = Loader('mq')
    my_conf = cfg.load()
    config = dict(my_conf[1])

    ctx = DaemonContext()
    ctx.open()

    context = zmq.Context()
    print(("tcp://{0}:{1}".format(config['ip'], config['req_rep_port'])))
    broker = MDPBroker(
        context, "tcp://{0}:{1}".format(config['ip'], config['req_rep_port']))
    IOLoop.instance().start()
    broker.shutdown()
Esempio n. 46
0
 def test_simple_worker(self):
     """Test MNWorker simple req/reply.
     """
     self._start_broker()
     time.sleep(0.2)
     worker = WorkerRunner(self.context, self.endpoint, self.service)
     sender = DelayedCallback(self.send_req, 500)
     stopper = DelayedCallback(self.stop_loop, 2500)
     sender.start()
     stopper.start()
     IOLoop.instance().start()
     worker.shutdown()
     self._stop_broker()
     return
Esempio n. 47
0
    def _send_fireball(self, mtype, data):
        if len(data) < 3:
            logger.error('no data to send')
            return []

        self.loop = IOLoop()
        self.socket = self.context.socket(zmq.DEALER)
        self.socket.connect(self.remote)

        timeout = time.time() + SNDTIMEO
        self.loop.add_timeout(timeout, self._fireball_timeout)
        self.response = []

        self.loop.add_handler(self.socket, self._handle_message_fireball,
                              zmq.POLLIN)

        if PYVERSION == 3:
            if isinstance(data, bytes):
                data = data.decode('utf-8')

        data = json.loads(data)

        if not isinstance(data, list):
            data = [data]

        if (len(data) % FIREBALL_SIZE) == 0:
            self.num_responses = int((len(data) / FIREBALL_SIZE))
        else:
            self.num_responses = int((len(data) / FIREBALL_SIZE)) + 1

        logger.debug('responses expected: %i' % self.num_responses)

        batch = []
        for d in data:
            batch.append(d)
            if len(batch) == FIREBALL_SIZE:
                Msg(mtype=Msg.INDICATORS_CREATE, token=self.token,
                    data=batch).send(self.socket)
                batch = []

        if len(batch):
            Msg(mtype=Msg.INDICATORS_CREATE, token=self.token,
                data=batch).send(self.socket)

        logger.debug("starting loop to receive")
        self.loop.start()
        self.loop.close()
        self.socket.close()
        return self.response
Esempio n. 48
0
 def _on_msg(self, msg):
     if _do_print:
         print 'broker received:',
         pprint(msg)
     self.target = msg.pop(0)
     if msg[1] == chr(1): # ready
         print 'READY'
         self.target = msg[0]
         return
     if msg[1] == chr(4): # ready
         print 'HB'
         return
     if msg[1] == chr(3): # reply
         IOLoop.instance().stop()
         return
     return
Esempio n. 49
0
    def __init__(self, socket, io_loop=None):
        self.socket = socket
        self.io_loop = io_loop or IOLoop.instance()
        self.poller = zmq.Poller()

        self._send_queue = Queue()
        self._recv_callback = None
        self._send_callback = None
        self._close_callback = None
        self._recv_copy = False
        self._flushed = False

        self._state = self.io_loop.ERROR
        with stack_context.NullContext():
            self.io_loop.add_handler(self.socket, self._handle_events,
                                     self._state)

        # shortcircuit some socket methods
        self.bind = self.socket.bind
        self.bind_to_random_port = self.socket.bind_to_random_port
        self.connect = self.socket.connect
        self.setsockopt = self.socket.setsockopt
        self.getsockopt = self.socket.getsockopt
        self.setsockopt_unicode = self.socket.setsockopt_unicode
        self.getsockopt_unicode = self.socket.getsockopt_unicode
Esempio n. 50
0
    def stop(self):
        self.terminated = True
        loop = IOLoop.instance()
        with loop._callback_lock:
            for timeout in loop._timeouts:
                timeout.callback = None

        for dc_or_stream in event_mod.callbacks.values():
            if hasattr(dc_or_stream, 'close'):
                def close_stream_callback(stream):
                    stream.close()

                loop.add_callback(close_stream_callback, dc_or_stream)
            else:
                dc_or_stream.stop()

        event_mod.callbacks = {}

#        if getattr(self, 'stream', None) is not None:
#            self.stream.close()

        with loop._callback_lock:
            for timeout in loop._timeouts:
                timeout.callback = None

        loop.stop()
Esempio n. 51
0
 def __init__(self, socket, io_loop=None):
     self.socket = socket
     self.io_loop = io_loop or IOLoop.instance()
     self.poller = zmq.Poller()
     
     self._send_queue = Queue()
     self._recv_callback = None
     self._send_callback = None
     self._close_callback = None
     self._recv_copy = False
     self._flushed = False
     
     self._state = self.io_loop.ERROR
     with stack_context.NullContext():
         self.io_loop.add_handler(
             self.socket, self._handle_events, self._state)
     
     # shortcircuit some socket methods
     self.bind = self.socket.bind
     self.bind_to_random_port = self.socket.bind_to_random_port
     self.connect = self.socket.connect
     self.setsockopt = self.socket.setsockopt
     self.getsockopt = self.socket.getsockopt
     self.setsockopt_unicode = self.socket.setsockopt_unicode
     self.getsockopt_unicode = self.socket.getsockopt_unicode
Esempio n. 52
0
class UdpListener (CCDaemon):
    """ UDP server to handle UDP stream. """

    log = skytools.getLogger ('d:UdpListener')

    def reload (self):
        super(UdpListener, self).reload()

        self.listen_host = self.cf.get ('listen-host')
        self.listen_port = self.cf.getint ('listen-port')
        self.stats_period = self.cf.getint ('stats-period', 30)

    def startup (self):
        super(UdpListener, self).startup()

        # plugins should be ready before we start receiving udp stream
        self.init_plugins()

        self.listen_addr = (self.listen_host, self.listen_port)
        self.sock = socket.socket (socket.AF_INET, socket.SOCK_DGRAM)
        self.sock.setblocking (0)
        try:
            self.sock.bind (self.listen_addr)
        except Exception, e:
            self.log.exception ("failed to bind to %s - %s", self.listen_addr, e)
            raise

        self.ioloop = IOLoop.instance()
        callback = functools.partial (self.handle_udp, self.sock)
        self.ioloop.add_handler (self.sock.fileno(), callback, self.ioloop.READ)

        self.timer_stats = PeriodicCallback (self.send_stats, self.stats_period * 1000, self.ioloop)
        self.timer_stats.start()
Esempio n. 53
0
    def __init__(self):
        #procControllerAddr = '165.227.24.226'  # I am client to HostController
        #procControllerPort = '5557'
        hostControllerPort = '5556' # I server to device

        print("Host Controller Starting\n")

        self.context = zmq.Context()   # get context
        self.loop = IOLoop.instance()

#       self.clientSetup = ClientSetup(context)  # instantiate the ClientSetup object
        self.serverSetup = ServerSetup(self.context) # instantiate the ServerSetup object


        # set up separate server and client sockets
        self.serverSocket = self.serverSetup.createServerSocket() # get a server socket
        self.serverSetup.serverBind(hostControllerPort, self.serverSocket) # bind to an address

#       self.clientSocket = self.clientSetup.createClientSocket() # get a client socket

# NOTE: setIdentity() MUST BE CALLED BEFORE clientConnect or the identity will
# not take effect
#       self.clientSetup.setIdentity(MasterId().getDevId()) # get the device id
#       self.clientSetup.clientConnect(hostControllerAddr, hostControllerPort, self.clientSocket) # connect to server using clientSocket
        self.serverSocket = ZMQStream(self.serverSocket)
        self.serverSocket.on_recv(self.onServerRecv)
        self.messages = Messages() # instantiate a Messages object

        self.inDict = {}
        self.outDict = {}
Esempio n. 54
0
    def __init__(self, id, uuid, socket, addr, time_=None, beacon=None, **kwargs):
        self.id = id
        self.uuid = uuid
        self.socket = socket
        self.addr = addr
        self.time = time_ or time.time()

        self.transport, host = addr.split('://', 1)
        self.host, self.beacon_router_port = host.rsplit(':', 1)

        # Set callbacks
        for k, v in kwargs.iteritems():
            if k.startswith('on_') and k.endswith('_cb'):
                setattr(self, k, v)

        if not self.ctx:
            self.ctx = zmq.Context.instance()
        self.loop = IOLoop.instance()

        # Set up our own dkv client interface to peer
        self.subscriber = self.ctx.socket(zmq.SUB)
        self.subscriber.setsockopt(zmq.SUBSCRIBE, b'')
        self.subscriber.connect(self.publisher_endpoint)

        # Wrap sockets in ZMQStreams for IOLoop handlers
        self.subscriber = ZMQStream(self.subscriber)
        self.subscriber.on_recv(self._on_subscriber_recv)
Esempio n. 55
0
    def __init__(self, frontier,
            data_in_sock='ipc:///tmp/robot-data-w2m.sock',
            data_out_sock='ipc:///tmp/robot-data-m2w.sock',
            msg_in_sock='ipc:///tmp/robot-msg-w2m.sock',
            msg_out_sock='ipc:///tmp/robot-msg-m2w.sock',
            io_loop=None):
        self.identity = 'master:%s:%s' % (socket.gethostname(), os.getpid())

        context = zmq.Context()

        self._io_loop = io_loop or IOLoop.instance()

        self._in_socket = context.socket(zmq.SUB)
        self._in_socket.setsockopt(zmq.SUBSCRIBE, '')
        self._in_socket.bind(data_in_sock)
        self._in_stream = ZMQStream(self._in_socket, io_loop)

        self._out_socket = context.socket(zmq.PUSH)
        self._out_socket.bind(data_out_sock)
        self._out_stream = ZMQStream(self._out_socket, io_loop)

        self._online_workers = set()
        self._running = False

        self._updater = PeriodicCallback(self._send_next, 100, io_loop=io_loop)
        self._reloader = PeriodicCallback(self.reload, 1000, io_loop=io_loop)

        self.frontier = frontier
        self.messenger = ServerMessenger(msg_in_sock, msg_out_sock,
                context, io_loop)
Esempio n. 56
0
    def __init__(self, port=5556):
        self.port = port
        self.ctx = zmq.Context()
        self.kvmap = {}
        self.loop = IOLoop.instance()

        # Set up our clone server sockets
        self.snapshot  = self.ctx.socket(zmq.ROUTER)
        self.publisher = self.ctx.socket(zmq.PUB)
        self.collector = self.ctx.socket(zmq.PULL)
        self.snapshot.bind("tcp://*:%d" % self.port)
        self.publisher.bind("tcp://*:%d" % (self.port + 1))
        self.collector.bind("tcp://*:%d" % (self.port + 2))

        # Wrap sockets in ZMQStreams for IOLoop handlers
        self.snapshot = ZMQStream(self.snapshot)
        self.publisher = ZMQStream(self.publisher)
        self.collector = ZMQStream(self.collector)

        # Register our handlers with reactor
        self.snapshot.on_recv(self.handle_snapshot)
        self.collector.on_recv(self.handle_collect)
        self.flush_callback = PeriodicCallback(self.flush_ttl, 1000)

        # basic log formatting:
        logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S",
                level=logging.INFO)
Esempio n. 57
0
 def _publish_msg(self, msg_type, data=None, metadata=None, buffers=None, **keys):
     """Helper for sending a comm message on IOPub"""
     if threading.current_thread().name != 'MainThread' and IOLoop.initialized():
         # make sure we never send on a zmq socket outside the main IOLoop thread
         IOLoop.instance().add_callback(lambda : self._publish_msg(msg_type, data, metadata, buffers, **keys))
         return
     data = {} if data is None else data
     metadata = {} if metadata is None else metadata
     content = json_clean(dict(data=data, comm_id=self.comm_id, **keys))
     self.session.send(self.iopub_socket, msg_type,
         content,
         metadata=json_clean(metadata),
         parent=self.kernel._parent_header,
         ident=self.topic,
         buffers=buffers,
     )
Esempio n. 58
0
    def __init__(self, primary, local, remote):
        # initialize the Binary Star
        self.ctx = zmq.Context()  # Our private context
        self.loop = IOLoop.instance()  # Reactor loop
        self.state = STATE_PRIMARY if primary else STATE_BACKUP

        self.event = None  # Current event
        self.peer_expiry = 0  # When peer is considered 'dead'
        self.voter_callback = None  # Voting socket handler
        self.master_callback = None  # Call when become master
        self.slave_callback = None  # Call when become slave

        # Create publisher for state going to peer
        self.statepub = self.ctx.socket(zmq.PUB)
        self.statepub.bind(local)

        # Create subscriber for state coming from peer
        self.statesub = self.ctx.socket(zmq.SUB)
        self.statesub.setsockopt_string(zmq.SUBSCRIBE, u'')
        self.statesub.connect(remote)

        # wrap statesub in ZMQStream for event triggers
        self.statesub = ZMQStream(self.statesub, self.loop)

        # setup basic reactor events
        self.heartbeat = PeriodicCallback(self.send_state,
                                          HEARTBEAT, self.loop)
        self.statesub.on_recv(self.recv_state)
Esempio n. 59
0
    def __init__(self, context=None, ioloop=None, **kwargs):  #{
        """
        Parameters
        ==========
        ioloop : IOLoop
            An existing IOLoop instance, if not passed, zmq.IOLoop.instance()
            will be used.
        context : Context
            An existing Context instance, if not passed, zmq.Context.instance()
            will be used.
        serializer : Serializer
            An instance of a Serializer subclass that will be used to serialize
            and deserialize args, kwargs and the result.
        """
        Context, _ = get_zmq_classes()

        if context is None:
            self.context = Context.instance()
        else:
            assert isinstance(context, Context)
            self.context = context

        self.ioloop   = IOLoop.instance() if ioloop is None else ioloop
        self._futures = {}  # {<req_id> : <Future>}

        super(TornadoRPCClient, self).__init__(**kwargs)
Esempio n. 60
0
    def wait_for_event(self, event, timeout = 30):
        """ Wait for a plugin to be in a state
            @param event : the event (STATUS_ALIVE, STATUS_STOPPED, ...)
            This is done by subscribing on the MQ plugin.status publisher
            If no status has been catched before the timeout, raise an error
        """
        print(u"Start listening to MQ...")
        IOLoop.instance().start() 
        # TODO : handle timeout

        # the following line will be processed when a IOLoop.instance().stop() will be called
        if self.plugin_status == event:
            return True
        else:
            print(u"Plugin not in status '{0}' : status = {1}".format(event, self.plugin_status))
            return False