def test_ipc_files(self): fd, path = mkstemp() os.close(fd) self.assertTrue(os.path.exists(path)) register_ipc_file('ipc://' + path) _cleanup_ipc_files() self.assertFalse(os.path.exists(path))
def __init__( self, frontend=DEFAULT_FRONTEND, backend=DEFAULT_BACKEND, heartbeat=None, register=DEFAULT_REG, io_threads=DEFAULT_IOTHREADS, agent_timeout=DEFAULT_AGENT_TIMEOUT, receiver=DEFAULT_BROKER_RECEIVER, publisher=DEFAULT_PUBLISHER, db="python", dboptions=None, web_root=None, ): # before doing anything, we verify if a broker is already up and # running logger.debug("Verifying if there is a running broker") pid = verify_broker(frontend) if pid is not None: # oops. can't do this ! logger.debug("Ooops, we have a running broker on that socket") raise DuplicateBrokerError(pid) self.endpoints = { "frontend": frontend, "backend": backend, "register": register, "receiver": receiver, "publisher": publisher, } if heartbeat is not None: self.endpoints["heartbeat"] = heartbeat logger.debug("Initializing the broker.") for endpoint in self.endpoints.values(): if endpoint.startswith("ipc"): register_ipc_file(endpoint) self.context = zmq.Context(io_threads=io_threads) # setting up the sockets self._frontend = self.context.socket(zmq.ROUTER) self._frontend.identity = "broker-" + frontend self._frontend.bind(frontend) self._backend = self.context.socket(zmq.ROUTER) self.pid = str(os.getpid()) self._backend.identity = self.pid self._backend.bind(backend) self._registration = self.context.socket(zmq.PULL) self._registration.bind(register) self._receiver = self.context.socket(zmq.PULL) self._receiver.bind(receiver) self._publisher = self.context.socket(zmq.PUB) self._publisher.bind(publisher) # setting up the streams self.loop = ioloop.IOLoop() self._frontstream = zmqstream.ZMQStream(self._frontend, self.loop) self._frontstream.on_recv(self._handle_recv_front) self._backstream = zmqstream.ZMQStream(self._backend, self.loop) self._backstream.on_recv(self._handle_recv_back) self._regstream = zmqstream.ZMQStream(self._registration, self.loop) self._regstream.on_recv(self._handle_reg) self._rcvstream = zmqstream.ZMQStream(self._receiver, self.loop) self._rcvstream.on_recv(self._handle_recv) # heartbeat if heartbeat is not None: self.pong = Heartbeat(heartbeat, io_loop=self.loop, ctx=self.context, onregister=self._deregister) else: self.pong = None # status self.started = False self.poll_timeout = None # controller self.ctrl = BrokerController(self, self.loop, db=db, dboptions=dboptions, agent_timeout=agent_timeout) self.web_root = web_root
def __init__(self, broker=DEFAULT_FRONTEND, receiver=DEFAULT_AGENT_RECEIVER, ping_delay=10., ping_retries=3, params=None, timeout=DEFAULT_TIMEOUT_MOVF, max_age=DEFAULT_MAX_AGE, max_age_delta=DEFAULT_MAX_AGE_DELTA): logger.debug('Initializing the agent.') self.debug = logger.isEnabledFor(logging.DEBUG) self.params = params self.pid = os.getpid() self.timeout = timeout self.max_age = max_age self.max_age_delta = max_age_delta self.delayed_exit = None self.env = os.environ.copy() self.running = False self._workers = {} self._run_args = {} self._run_started_at = {} self._max_id = defaultdict(int) self._started = self._stopped = self._launched = 0 self.loop = ioloop.IOLoop() self.ctx = zmq.Context() # Setup the zmq sockets # Let's ask the broker its options self.broker = broker client = Client(self.broker) result = client.ping() self.endpoints = result['endpoints'] # backend socket - used to receive work from the broker self._backend = self.ctx.socket(zmq.REP) self._backend.identity = str(self.pid) self._backend.connect(self.endpoints['backend']) # register socket - used to register into the broker self._reg = self.ctx.socket(zmq.PUSH) self._reg.connect(self.endpoints['register']) # receiver socket - used to receive results from workers self._receiver_socket = receiver.format(pid=self.pid) register_ipc_file(self._receiver_socket) self._receiver = self.ctx.socket(zmq.PULL) self._receiver.bind(self._receiver_socket) # push socket - used to send back results to the broker self._push = self.ctx.socket(zmq.PUSH) self._push.set_hwm(8096 * 10) self._push.setsockopt(zmq.LINGER, -1) self._push.connect(self.endpoints['receiver']) # hearbeat socket - used to check if the broker is alive heartbeat = self.endpoints.get('heartbeat') if heartbeat is not None: self.ping = Stethoscope(heartbeat, onbeatlost=self.lost, delay=ping_delay, retries=ping_retries, ctx=self.ctx, io_loop=self.loop, onregister=self.register) else: self.ping = None # Setup the zmq streams. self._backstream = zmqstream.ZMQStream(self._backend, self.loop) self._backstream.on_recv(self._handle_recv_back) self._rcvstream = zmqstream.ZMQStream(self._receiver, self.loop) self._rcvstream.on_recv(self._handle_events) self._check = ioloop.PeriodicCallback(self._check_proc, ping_delay * 1000, io_loop=self.loop)
def __init__(self, frontend=DEFAULT_FRONTEND, backend=DEFAULT_BACKEND, heartbeat=None, register=DEFAULT_REG, io_threads=DEFAULT_IOTHREADS, agent_timeout=DEFAULT_AGENT_TIMEOUT, receiver=DEFAULT_BROKER_RECEIVER, publisher=DEFAULT_PUBLISHER, db='python', dboptions=None, web_root=None): # before doing anything, we verify if a broker is already up and # running logger.debug('Verifying if there is a running broker') pid = verify_broker(frontend) if pid is not None: # oops. can't do this ! logger.debug('Ooops, we have a running broker on that socket') raise DuplicateBrokerError(pid) self.endpoints = { 'frontend': frontend, 'backend': backend, 'register': register, 'receiver': receiver, 'publisher': publisher } if heartbeat is not None: self.endpoints['heartbeat'] = heartbeat logger.debug('Initializing the broker.') for endpoint in self.endpoints.values(): if endpoint.startswith('ipc'): register_ipc_file(endpoint) self.context = zmq.Context(io_threads=io_threads) # setting up the sockets self._frontend = self.context.socket(zmq.ROUTER) self._frontend.identity = 'broker-' + frontend self._frontend.bind(frontend) self._backend = self.context.socket(zmq.ROUTER) self.pid = str(os.getpid()) self._backend.identity = self.pid self._backend.bind(backend) self._registration = self.context.socket(zmq.PULL) self._registration.bind(register) self._receiver = self.context.socket(zmq.PULL) self._receiver.bind(receiver) self._publisher = self.context.socket(zmq.PUB) self._publisher.bind(publisher) # setting up the streams self.loop = ioloop.IOLoop() self._frontstream = zmqstream.ZMQStream(self._frontend, self.loop) self._frontstream.on_recv(self._handle_recv_front) self._backstream = zmqstream.ZMQStream(self._backend, self.loop) self._backstream.on_recv(self._handle_recv_back) self._regstream = zmqstream.ZMQStream(self._registration, self.loop) self._regstream.on_recv(self._handle_reg) self._rcvstream = zmqstream.ZMQStream(self._receiver, self.loop) self._rcvstream.on_recv(self._handle_recv) # heartbeat if heartbeat is not None: self.pong = Heartbeat(heartbeat, io_loop=self.loop, ctx=self.context, onregister=self._deregister) else: self.pong = None # status self.started = False self.poll_timeout = None # controller self.ctrl = BrokerController(self, self.loop, db=db, dboptions=dboptions, agent_timeout=agent_timeout) self.web_root = web_root
def __init__(self, frontend=DEFAULT_FRONTEND, backend=DEFAULT_BACKEND, heartbeat=DEFAULT_HEARTBEAT, register=DEFAULT_REG, io_threads=DEFAULT_IOTHREADS, worker_timeout=DEFAULT_TIMEOUT_MOVF, receiver=DEFAULT_RECEIVER, publisher=DEFAULT_PUBLISHER, dbdir=DEFAULT_DBDIR): # before doing anything, we verify if a broker is already up and # running logger.debug('Verifying if there is a running broker') pid = verify_broker(frontend) if pid is not None: # oops. can't do this ! logger.debug('Ooops, we have a running broker on that socket') raise DuplicateBrokerError(pid) logger.debug('Initializing the broker.') for endpoint in (frontend, backend, heartbeat): if endpoint.startswith('ipc'): register_ipc_file(endpoint) self.context = zmq.Context(io_threads=io_threads) # setting up the sockets self._frontend = self.context.socket(zmq.ROUTER) self._frontend.identity = 'broker-' + frontend self._frontend.bind(frontend) self._backend = self.context.socket(zmq.ROUTER) self._backend.bind(backend) self._registration = self.context.socket(zmq.PULL) self._registration.bind(register) self._receiver = self.context.socket(zmq.PULL) self._receiver.bind(receiver) self._publisher = self.context.socket(zmq.PUB) self._publisher.bind(publisher) # setting up the streams self.loop = ioloop.IOLoop() self._frontstream = zmqstream.ZMQStream(self._frontend, self.loop) self._frontstream.on_recv(self._handle_recv_front) self._backstream = zmqstream.ZMQStream(self._backend, self.loop) self._backstream.on_recv(self._handle_recv_back) self._regstream = zmqstream.ZMQStream(self._registration, self.loop) self._regstream.on_recv(self._handle_reg) self._rcvstream = zmqstream.ZMQStream(self._receiver, self.loop) self._rcvstream.on_recv(self._handle_rcv) # heartbeat self.pong = Heartbeat(heartbeat, io_loop=self.loop, ctx=self.context) # status self.started = False self.poll_timeout = None # workers registration and timers self._workers = [] self._worker_times = {} self.worker_timeout = worker_timeout self._runs = {} # local DB self._db = BrokerDB(self.loop, dbdir)