def test_ipc_files(self):
fd, path = mkstemp()
os.close(fd)
self.assertTrue(os.path.exists(path))
register_ipc_file('ipc://' + path)
_cleanup_ipc_files()
self.assertFalse(os.path.exists(path))
def test_ipc_files(self):
fd, path = mkstemp()
os.close(fd)
self.assertTrue(os.path.exists(path))
register_ipc_file('ipc://' + path)
_cleanup_ipc_files()
self.assertFalse(os.path.exists(path))
def __init__(
self,
frontend=DEFAULT_FRONTEND,
backend=DEFAULT_BACKEND,
heartbeat=None,
register=DEFAULT_REG,
io_threads=DEFAULT_IOTHREADS,
agent_timeout=DEFAULT_AGENT_TIMEOUT,
receiver=DEFAULT_BROKER_RECEIVER,
publisher=DEFAULT_PUBLISHER,
db="python",
dboptions=None,
web_root=None,
):
# before doing anything, we verify if a broker is already up and
# running
logger.debug("Verifying if there is a running broker")
pid = verify_broker(frontend)
if pid is not None: # oops. can't do this !
logger.debug("Ooops, we have a running broker on that socket")
raise DuplicateBrokerError(pid)
self.endpoints = {
"frontend": frontend,
"backend": backend,
"register": register,
"receiver": receiver,
"publisher": publisher,
}
if heartbeat is not None:
self.endpoints["heartbeat"] = heartbeat
logger.debug("Initializing the broker.")
for endpoint in self.endpoints.values():
if endpoint.startswith("ipc"):
register_ipc_file(endpoint)
self.context = zmq.Context(io_threads=io_threads)
# setting up the sockets
self._frontend = self.context.socket(zmq.ROUTER)
self._frontend.identity = "broker-" + frontend
self._frontend.bind(frontend)
self._backend = self.context.socket(zmq.ROUTER)
self.pid = str(os.getpid())
self._backend.identity = self.pid
self._backend.bind(backend)
self._registration = self.context.socket(zmq.PULL)
self._registration.bind(register)
self._receiver = self.context.socket(zmq.PULL)
self._receiver.bind(receiver)
self._publisher = self.context.socket(zmq.PUB)
self._publisher.bind(publisher)
# setting up the streams
self.loop = ioloop.IOLoop()
self._frontstream = zmqstream.ZMQStream(self._frontend, self.loop)
self._frontstream.on_recv(self._handle_recv_front)
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
self._regstream = zmqstream.ZMQStream(self._registration, self.loop)
self._regstream.on_recv(self._handle_reg)
self._rcvstream = zmqstream.ZMQStream(self._receiver, self.loop)
self._rcvstream.on_recv(self._handle_recv)
# heartbeat
if heartbeat is not None:
self.pong = Heartbeat(heartbeat, io_loop=self.loop, ctx=self.context, onregister=self._deregister)
else:
self.pong = None
# status
self.started = False
self.poll_timeout = None
# controller
self.ctrl = BrokerController(self, self.loop, db=db, dboptions=dboptions, agent_timeout=agent_timeout)
self.web_root = web_root
def __init__(self, broker=DEFAULT_FRONTEND,
receiver=DEFAULT_AGENT_RECEIVER,
ping_delay=10., ping_retries=3,
params=None, timeout=DEFAULT_TIMEOUT_MOVF,
max_age=DEFAULT_MAX_AGE, max_age_delta=DEFAULT_MAX_AGE_DELTA):
logger.debug('Initializing the agent.')
self.debug = logger.isEnabledFor(logging.DEBUG)
self.params = params
self.pid = os.getpid()
self.timeout = timeout
self.max_age = max_age
self.max_age_delta = max_age_delta
self.delayed_exit = None
self.env = os.environ.copy()
self.running = False
self._workers = {}
self._run_args = {}
self._run_started_at = {}
self._max_id = defaultdict(int)
self._started = self._stopped = self._launched = 0
self.loop = ioloop.IOLoop()
self.ctx = zmq.Context()
# Setup the zmq sockets
# Let's ask the broker its options
self.broker = broker
client = Client(self.broker)
result = client.ping()
self.endpoints = result['endpoints']
# backend socket - used to receive work from the broker
self._backend = self.ctx.socket(zmq.REP)
self._backend.identity = str(self.pid)
self._backend.connect(self.endpoints['backend'])
# register socket - used to register into the broker
self._reg = self.ctx.socket(zmq.PUSH)
self._reg.connect(self.endpoints['register'])
# receiver socket - used to receive results from workers
self._receiver_socket = receiver.format(pid=self.pid)
register_ipc_file(self._receiver_socket)
self._receiver = self.ctx.socket(zmq.PULL)
self._receiver.bind(self._receiver_socket)
# push socket - used to send back results to the broker
self._push = self.ctx.socket(zmq.PUSH)
self._push.set_hwm(8096 * 10)
self._push.setsockopt(zmq.LINGER, -1)
self._push.connect(self.endpoints['receiver'])
# hearbeat socket - used to check if the broker is alive
heartbeat = self.endpoints.get('heartbeat')
if heartbeat is not None:
self.ping = Stethoscope(heartbeat, onbeatlost=self.lost,
delay=ping_delay, retries=ping_retries,
ctx=self.ctx, io_loop=self.loop,
onregister=self.register)
else:
self.ping = None
# Setup the zmq streams.
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
self._rcvstream = zmqstream.ZMQStream(self._receiver, self.loop)
self._rcvstream.on_recv(self._handle_events)
self._check = ioloop.PeriodicCallback(self._check_proc,
ping_delay * 1000,
io_loop=self.loop)
def __init__(self,
frontend=DEFAULT_FRONTEND,
backend=DEFAULT_BACKEND,
heartbeat=None,
register=DEFAULT_REG,
io_threads=DEFAULT_IOTHREADS,
agent_timeout=DEFAULT_AGENT_TIMEOUT,
receiver=DEFAULT_BROKER_RECEIVER,
publisher=DEFAULT_PUBLISHER,
db='python',
dboptions=None,
web_root=None):
# before doing anything, we verify if a broker is already up and
# running
logger.debug('Verifying if there is a running broker')
pid = verify_broker(frontend)
if pid is not None: # oops. can't do this !
logger.debug('Ooops, we have a running broker on that socket')
raise DuplicateBrokerError(pid)
self.endpoints = {
'frontend': frontend,
'backend': backend,
'register': register,
'receiver': receiver,
'publisher': publisher
}
if heartbeat is not None:
self.endpoints['heartbeat'] = heartbeat
logger.debug('Initializing the broker.')
for endpoint in self.endpoints.values():
if endpoint.startswith('ipc'):
register_ipc_file(endpoint)
self.context = zmq.Context(io_threads=io_threads)
# setting up the sockets
self._frontend = self.context.socket(zmq.ROUTER)
self._frontend.identity = 'broker-' + frontend
self._frontend.bind(frontend)
self._backend = self.context.socket(zmq.ROUTER)
self.pid = str(os.getpid())
self._backend.identity = self.pid
self._backend.bind(backend)
self._registration = self.context.socket(zmq.PULL)
self._registration.bind(register)
self._receiver = self.context.socket(zmq.PULL)
self._receiver.bind(receiver)
self._publisher = self.context.socket(zmq.PUB)
self._publisher.bind(publisher)
# setting up the streams
self.loop = ioloop.IOLoop()
self._frontstream = zmqstream.ZMQStream(self._frontend, self.loop)
self._frontstream.on_recv(self._handle_recv_front)
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
self._regstream = zmqstream.ZMQStream(self._registration, self.loop)
self._regstream.on_recv(self._handle_reg)
self._rcvstream = zmqstream.ZMQStream(self._receiver, self.loop)
self._rcvstream.on_recv(self._handle_recv)
# heartbeat
if heartbeat is not None:
self.pong = Heartbeat(heartbeat,
io_loop=self.loop,
ctx=self.context,
onregister=self._deregister)
else:
self.pong = None
# status
self.started = False
self.poll_timeout = None
# controller
self.ctrl = BrokerController(self,
self.loop,
db=db,
dboptions=dboptions,
agent_timeout=agent_timeout)
self.web_root = web_root
def __init__(self, frontend=DEFAULT_FRONTEND, backend=DEFAULT_BACKEND,
heartbeat=DEFAULT_HEARTBEAT, register=DEFAULT_REG,
io_threads=DEFAULT_IOTHREADS,
worker_timeout=DEFAULT_TIMEOUT_MOVF,
receiver=DEFAULT_RECEIVER, publisher=DEFAULT_PUBLISHER,
dbdir=DEFAULT_DBDIR):
# before doing anything, we verify if a broker is already up and
# running
logger.debug('Verifying if there is a running broker')
pid = verify_broker(frontend)
if pid is not None: # oops. can't do this !
logger.debug('Ooops, we have a running broker on that socket')
raise DuplicateBrokerError(pid)
logger.debug('Initializing the broker.')
for endpoint in (frontend, backend, heartbeat):
if endpoint.startswith('ipc'):
register_ipc_file(endpoint)
self.context = zmq.Context(io_threads=io_threads)
# setting up the sockets
self._frontend = self.context.socket(zmq.ROUTER)
self._frontend.identity = 'broker-' + frontend
self._frontend.bind(frontend)
self._backend = self.context.socket(zmq.ROUTER)
self._backend.bind(backend)
self._registration = self.context.socket(zmq.PULL)
self._registration.bind(register)
self._receiver = self.context.socket(zmq.PULL)
self._receiver.bind(receiver)
self._publisher = self.context.socket(zmq.PUB)
self._publisher.bind(publisher)
# setting up the streams
self.loop = ioloop.IOLoop()
self._frontstream = zmqstream.ZMQStream(self._frontend, self.loop)
self._frontstream.on_recv(self._handle_recv_front)
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
self._regstream = zmqstream.ZMQStream(self._registration, self.loop)
self._regstream.on_recv(self._handle_reg)
self._rcvstream = zmqstream.ZMQStream(self._receiver, self.loop)
self._rcvstream.on_recv(self._handle_rcv)
# heartbeat
self.pong = Heartbeat(heartbeat, io_loop=self.loop, ctx=self.context)
# status
self.started = False
self.poll_timeout = None
# workers registration and timers
self._workers = []
self._worker_times = {}
self.worker_timeout = worker_timeout
self._runs = {}
# local DB
self._db = BrokerDB(self.loop, dbdir)
