Exemple #1
0
def launch_scheduler(in_addr,
                     out_addr,
                     mon_addr,
                     not_addr,
                     config=None,
                     logname='ZMQ',
                     log_addr=None,
                     loglevel=logging.DEBUG,
                     scheme='lru',
                     identity=b'task'):
    from zmq.eventloop import ioloop
    from zmq.eventloop.zmqstream import ZMQStream

    if config:
        # unwrap dict back into Config
        config = Config(config)

    ctx = zmq.Context()
    loop = ioloop.IOLoop()
    ins = ZMQStream(ctx.socket(zmq.XREP), loop)
    ins.setsockopt(zmq.IDENTITY, identity)
    ins.bind(in_addr)

    outs = ZMQStream(ctx.socket(zmq.XREP), loop)
    outs.setsockopt(zmq.IDENTITY, identity)
    outs.bind(out_addr)
    mons = ZMQStream(ctx.socket(zmq.PUB), loop)
    mons.connect(mon_addr)
    nots = ZMQStream(ctx.socket(zmq.SUB), loop)
    nots.setsockopt(zmq.SUBSCRIBE, '')
    nots.connect(not_addr)

    scheme = globals().get(scheme, None)
    # setup logging
    if log_addr:
        connect_logger(logname,
                       ctx,
                       log_addr,
                       root="scheduler",
                       loglevel=loglevel)
    else:
        local_logger(logname, loglevel)

    scheduler = TaskScheduler(client_stream=ins,
                              engine_stream=outs,
                              mon_stream=mons,
                              notifier_stream=nots,
                              scheme=scheme,
                              loop=loop,
                              logname=logname,
                              config=config)
    scheduler.start()
    try:
        loop.start()
    except KeyboardInterrupt:
        print("interrupted, exiting...", file=sys.__stderr__)
Exemple #2
0
def launch_scheduler(
    in_addr,
    out_addr,
    mon_addr,
    not_addr,
    config=None,
    logname="root",
    log_url=None,
    loglevel=logging.DEBUG,
    identity=b"task",
    in_thread=False,
):

    ZMQStream = zmqstream.ZMQStream

    if config:
        # unwrap dict back into Config
        config = Config(config)

    if in_thread:
        # use instance() to get the same Context/Loop as our parent
        ctx = zmq.Context.instance()
        loop = ioloop.IOLoop.instance()
    else:
        # in a process, don't use instance()
        # for safety with multiprocessing
        ctx = zmq.Context()
        loop = ioloop.IOLoop()
    ins = ZMQStream(ctx.socket(zmq.ROUTER), loop)
    ins.setsockopt(zmq.IDENTITY, identity)
    ins.bind(in_addr)

    outs = ZMQStream(ctx.socket(zmq.ROUTER), loop)
    outs.setsockopt(zmq.IDENTITY, identity)
    outs.bind(out_addr)
    mons = zmqstream.ZMQStream(ctx.socket(zmq.PUB), loop)
    mons.connect(mon_addr)
    nots = zmqstream.ZMQStream(ctx.socket(zmq.SUB), loop)
    nots.setsockopt(zmq.SUBSCRIBE, b"")
    nots.connect(not_addr)

    # setup logging.
    if in_thread:
        log = Application.instance().log
    else:
        if log_url:
            log = connect_logger(logname, ctx, log_url, root="scheduler", loglevel=loglevel)
        else:
            log = local_logger(logname, loglevel)

    scheduler = TaskScheduler(
        client_stream=ins, engine_stream=outs, mon_stream=mons, notifier_stream=nots, loop=loop, log=log, config=config
    )
    scheduler.start()
    if not in_thread:
        try:
            loop.start()
        except KeyboardInterrupt:
            print("interrupted, exiting...", file=sys.__stderr__)
Exemple #3
0
def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, reg_addr, config=None,
                     logname='root', log_url=None, loglevel=logging.DEBUG,
                     identity=b'task', in_thread=False):

    ZMQStream = zmqstream.ZMQStream

    if config:
        # unwrap dict back into Config
        config = Config(config)

    if in_thread:
        # use instance() to get the same Context/Loop as our parent
        ctx = zmq.Context.instance()
        loop = ioloop.IOLoop.instance()
    else:
        # in a process, don't use instance()
        # for safety with multiprocessing
        ctx = zmq.Context()
        loop = ioloop.IOLoop()
    ins = ZMQStream(ctx.socket(zmq.ROUTER), loop)
    util.set_hwm(ins, 0)
    ins.setsockopt(zmq.IDENTITY, identity + b'_in')
    ins.bind(in_addr)

    outs = ZMQStream(ctx.socket(zmq.ROUTER), loop)
    util.set_hwm(outs, 0)
    outs.setsockopt(zmq.IDENTITY, identity + b'_out')
    outs.bind(out_addr)
    mons = zmqstream.ZMQStream(ctx.socket(zmq.PUB), loop)
    util.set_hwm(mons, 0)
    mons.connect(mon_addr)
    nots = zmqstream.ZMQStream(ctx.socket(zmq.SUB), loop)
    nots.setsockopt(zmq.SUBSCRIBE, b'')
    nots.connect(not_addr)

    querys = ZMQStream(ctx.socket(zmq.DEALER), loop)
    querys.connect(reg_addr)

    # setup logging.
    if in_thread:
        log = Application.instance().log
    else:
        if log_url:
            log = connect_logger(
                logname, ctx, log_url, root="scheduler", loglevel=loglevel)
        else:
            log = local_logger(logname, loglevel)

    scheduler = TaskScheduler(client_stream=ins, engine_stream=outs,
                              mon_stream=mons, notifier_stream=nots,
                              query_stream=querys,
                              loop=loop, log=log,
                              config=config)
    scheduler.start()
    if not in_thread:
        try:
            loop.start()
        except KeyboardInterrupt:
            scheduler.log.critical("Interrupted, exiting...")
Exemple #4
0
def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, reg_addr, config=None,
                        logname='root', log_url=None, loglevel=logging.DEBUG,
                        identity=b'task', in_thread=False):

    ZMQStream = zmqstream.ZMQStream

    if config:
        # unwrap dict back into Config
        config = Config(config)

    if in_thread:
        # use instance() to get the same Context/Loop as our parent
        ctx = zmq.Context.instance()
        loop = ioloop.IOLoop.instance()
    else:
        # in a process, don't use instance()
        # for safety with multiprocessing
        ctx = zmq.Context()
        loop = ioloop.IOLoop()
    ins = ZMQStream(ctx.socket(zmq.ROUTER),loop)
    util.set_hwm(ins, 0)
    ins.setsockopt(zmq.IDENTITY, identity + b'_in')
    ins.bind(in_addr)

    outs = ZMQStream(ctx.socket(zmq.ROUTER),loop)
    util.set_hwm(outs, 0)
    outs.setsockopt(zmq.IDENTITY, identity + b'_out')
    outs.bind(out_addr)
    mons = zmqstream.ZMQStream(ctx.socket(zmq.PUB),loop)
    util.set_hwm(mons, 0)
    mons.connect(mon_addr)
    nots = zmqstream.ZMQStream(ctx.socket(zmq.SUB),loop)
    nots.setsockopt(zmq.SUBSCRIBE, b'')
    nots.connect(not_addr)
    
    querys = ZMQStream(ctx.socket(zmq.DEALER),loop)
    querys.connect(reg_addr)
    
    # setup logging.
    if in_thread:
        log = Application.instance().log
    else:
        if log_url:
            log = connect_logger(logname, ctx, log_url, root="scheduler", loglevel=loglevel)
        else:
            log = local_logger(logname, loglevel)

    scheduler = TaskScheduler(client_stream=ins, engine_stream=outs,
                            mon_stream=mons, notifier_stream=nots,
                            query_stream=querys,
                            loop=loop, log=log,
                            config=config)
    scheduler.start()
    if not in_thread:
        try:
            loop.start()
        except KeyboardInterrupt:
            scheduler.log.critical("Interrupted, exiting...")
Exemple #5
0
def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, config=None,logname='ZMQ', 
                            log_addr=None, loglevel=logging.DEBUG, scheme='lru',
                            identity=b'task'):
    from zmq.eventloop import ioloop
    from zmq.eventloop.zmqstream import ZMQStream
    
    if config:
        # unwrap dict back into Config
        config = Config(config)

    ctx = zmq.Context()
    loop = ioloop.IOLoop()
    ins = ZMQStream(ctx.socket(zmq.XREP),loop)
    ins.setsockopt(zmq.IDENTITY, identity)
    ins.bind(in_addr)
    
    outs = ZMQStream(ctx.socket(zmq.XREP),loop)
    outs.setsockopt(zmq.IDENTITY, identity)
    outs.bind(out_addr)
    mons = ZMQStream(ctx.socket(zmq.PUB),loop)
    mons.connect(mon_addr)
    nots = ZMQStream(ctx.socket(zmq.SUB),loop)
    nots.setsockopt(zmq.SUBSCRIBE, '')
    nots.connect(not_addr)
    
    scheme = globals().get(scheme, None)
    # setup logging
    if log_addr:
        connect_logger(logname, ctx, log_addr, root="scheduler", loglevel=loglevel)
    else:
        local_logger(logname, loglevel)
    
    scheduler = TaskScheduler(client_stream=ins, engine_stream=outs,
                            mon_stream=mons, notifier_stream=nots,
                            scheme=scheme, loop=loop, logname=logname,
                            config=config)
    scheduler.start()
    try:
        loop.start()
    except KeyboardInterrupt:
        print ("interrupted, exiting...", file=sys.__stderr__)