def __init__(self, context, agent_addr, client_addr):
self._content = context
self._loop = asyncio.get_event_loop()
self._agent_addr = agent_addr
self._client_addr = client_addr
self._agent_socket = context.socket(zmq.ROUTER)
self._client_socket = context.socket(zmq.ROUTER)
self._logger = logging.getLogger("inginious.backend")
# Enable support for ipv6
self._agent_socket.ipv6 = True
self._client_socket.ipv6 = True
self._poller = Poller()
self._poller.register(self._agent_socket, zmq.POLLIN)
self._poller.register(self._client_socket, zmq.POLLIN)
# dict of available environments. Keys are first the type of environement (docker, mcq, kata...) then the
# name of the environment.
self._environments: Dict[str, Dict[str, EnvironmentInfo]] = {}
self._registered_clients = set() # addr of registered clients
self._registered_agents: Dict[bytes, AgentInfo] = {} # all registered agents
self._ping_count = {} # ping count per addr of agents
# addr of available agents. May contain multiple times the same agent, because some agent can
# manage multiple jobs at once!
self._available_agents = []
# These two share the same objects! Tuples should never be recreated.
self._waiting_jobs_pq = TopicPriorityQueue() # priority queue for waiting jobs
self._waiting_jobs: Dict[str, WaitingJob] = {} # all jobs waiting in queue
self._job_running: Dict[str, RunningJob] = {} # all running jobs
class AsyncioAuthenticator(Authenticator):
"""ZAP authentication for use in the asyncio IO loop"""
def __init__(self, context=None, loop=None):
super().__init__(context)
self.loop = loop or asyncio.get_event_loop()
self.__poller = None
self.__task = None
@asyncio.coroutine
def __handle_zap(self):
while True:
events = yield from self.__poller.poll()
if self.zap_socket in dict(events):
msg = yield from self.zap_socket.recv_multipart()
self.handle_zap_message(msg)
def start(self):
"""Start ZAP authentication"""
super().start()
self.__poller = Poller()
self.__poller.register(self.zap_socket, zmq.POLLIN)
self.__task = asyncio.ensure_future(self.__handle_zap())
def stop(self):
"""Stop ZAP authentication"""
if self.__task:
self.__task.cancel()
if self.__poller:
self.__poller.unregister(self.zap_socket)
self.__poller = None
super().stop()
def __init__(self, context, agent_addr, client_addr):
self._content = context
self._loop = asyncio.get_event_loop()
self._agent_addr = agent_addr
self._client_addr = client_addr
self._agent_socket = context.socket(zmq.ROUTER)
self._client_socket = context.socket(zmq.ROUTER)
self._logger = logging.getLogger("inginious.backend")
# Enable support for ipv6
self._agent_socket.ipv6 = True
self._client_socket.ipv6 = True
self._poller = Poller()
self._poller.register(self._agent_socket, zmq.POLLIN)
self._poller.register(self._client_socket, zmq.POLLIN)
# List of containers available
# {
# "name": ("last_id", "created_last", ["agent_addr1", "agent_addr2"])
# }
self._containers = {}
# Containers available per agent {"agent_addr": ["container_id_1", ...]}
self._containers_on_agent = {}
self._registered_clients = set() # addr of registered clients
self._registered_agents = {} # addr of registered agents
self._available_agents = [] # addr of available agents
self._ping_count = {} # ping count per addr of agents
self._waiting_jobs = OrderedDict() # rb queue for waiting jobs format:[(client_addr_as_bytes, ClientNewJob])]
self._job_running = {} # indicates on which agent which job is running. format: {BackendJobId:(addr_as_bytes,ClientNewJob,start_time)}
async def run_queue(context):
# Prepare our context and sockets
frontend = context.socket(zmq.ROUTER)
backend = context.socket(zmq.ROUTER)
frontend.bind("tcp://*:5559")
backend.bind("tcp://*:5560")
# Initialize poll set
poller = Poller()
poller.register(frontend, zmq.POLLIN)
poller.register(backend, zmq.POLLIN)
# Switch messages between sockets
while True:
socks = await poller.poll()
socks = dict(socks)
if socks.get(frontend) == zmq.POLLIN:
frames = await frontend.recv_multipart()
print('received from frontend: {}'.format(frames))
# Add the worker ident to the envelope - simplified for example.
frames.insert(0, b'Worker1')
await backend.send_multipart(frames)
if socks.get(backend) == zmq.POLLIN:
frames = await backend.recv_multipart()
print('received from backend: {}'.format(frames))
msg = frames[1:] # Slice off worker ident
await frontend.send_multipart(msg)
class AsyncioAuthenticator(Authenticator):
"""ZAP authentication for use in the asyncio IO loop"""
def __init__(self, context=None, loop=None):
super().__init__(context)
self.loop = loop or asyncio.get_event_loop()
self.__poller = None
self.__task = None
@asyncio.coroutine
def __handle_zap(self):
while True:
events = yield from self.__poller.poll()
if self.zap_socket in dict(events):
msg = yield from self.zap_socket.recv_multipart()
self.handle_zap_message(msg)
def start(self):
"""Start ZAP authentication"""
super().start()
self.__poller = Poller()
self.__poller.register(self.zap_socket, zmq.POLLIN)
self.__task = asyncio.ensure_future(self.__handle_zap())
def stop(self):
"""Stop ZAP authentication"""
if self.__task:
self.__task.cancel()
if self.__poller:
self.__poller.unregister(self.zap_socket)
self.__poller = None
super().stop()
def run_worker(context):
poller = Poller()
liveness = HEARTBEAT_LIVENESS
interval = INTERVAL_INIT
heartbeat_at = time.time() + HEARTBEAT_INTERVAL
worker = yield from worker_socket(context, poller)
cycles = 0
while True:
socks = yield from poller.poll(HEARTBEAT_INTERVAL * 1000)
socks = dict(socks)
# Handle worker activity on backend
if socks.get(worker) == zmq.POLLIN:
# Get message
# - 3-part envelope + content -> request
# - 1-part HEARTBEAT -> heartbeat
frames = yield from worker.recv_multipart()
if not frames:
break # Interrupted
if len(frames) == 3:
# Simulate various problems, after a few cycles
cycles += 1
if cycles > 3 and randint(0, 5) == 0:
print("I: Simulating a crash")
break
if cycles > 3 and randint(0, 5) == 0:
print("I: Simulating CPU overload")
yield from asyncio.sleep(3)
print("I: Normal reply")
yield from worker.send_multipart(frames)
liveness = HEARTBEAT_LIVENESS
yield from asyncio.sleep(1) # Do some heavy work
elif len(frames) == 1 and frames[0] == PPP_HEARTBEAT:
print("I: Queue heartbeat")
liveness = HEARTBEAT_LIVENESS
else:
print("E: Invalid message: %s" % frames)
interval = INTERVAL_INIT
else:
liveness -= 1
if liveness == 0:
print("W: Heartbeat failure, can't reach queue")
print("W: Reconnecting in %0.2fs..." % interval)
yield from asyncio.sleep(interval)
if interval < INTERVAL_MAX:
interval *= 2
poller.unregister(worker)
worker.setsockopt(zmq.LINGER, 0)
worker.close()
worker = yield from worker_socket(context, poller)
liveness = HEARTBEAT_LIVENESS
if time.time() > heartbeat_at:
heartbeat_at = time.time() + HEARTBEAT_INTERVAL
print("I: Worker heartbeat")
yield from worker.send(PPP_HEARTBEAT)
def register(self):
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
if settings.SCHEDULER_MONITOR:
self.monitor_poller = Poller()
self.monitor_poller.register(self.monitor_socket, POLLIN)
logger.info('%s - all sockets are successfully registered '
'in poller objects ...', self)
def __init__(self, context, agent_addr, client_addr):
self._content = context
self._loop = asyncio.get_event_loop()
self._agent_addr = agent_addr
self._client_addr = client_addr
self._agent_socket = context.socket(zmq.ROUTER)
self._client_socket = context.socket(zmq.ROUTER)
self._logger = logging.getLogger("inginious.backend")
# Enable support for ipv6
self._agent_socket.ipv6 = True
self._client_socket.ipv6 = True
self._poller = Poller()
self._poller.register(self._agent_socket, zmq.POLLIN)
self._poller.register(self._client_socket, zmq.POLLIN)
# List of containers available
# {
# "name": ("last_id", "created_last", ["agent_addr1", "agent_addr2"])
# }
self._containers = {}
# List of batch containers available
# {
# "name": {
# "description": "a description written in RST",
# "id": "container img id",
# "created": 123456789,
# "agents": ["agent_addr1", "agent_addr2"]
# "parameters": {
# "key": {
# "type:" "file", # or "text",
# "path": "path/to/file/inside/input/dir", # not mandatory in file, by default "key"
# "name": "name of the field", # not mandatory in file, default "key"
# "description": "a short description of what this field is used for" # not mandatory, default ""
# }
# }
# }
self._batch_containers = {}
# Batch containers available per agent {"agent_addr": ["batch_id_1", ...]}
self._batch_containers_on_agent = {}
# Containers available per agent {"agent_addr": ["container_id_1", ...]}
self._containers_on_agent = {}
self._registered_clients = set() # addr of registered clients
self._registered_agents = {} # addr of registered agents
self._available_agents = [] # addr of available agents
self._ping_count = {} # ping count per addr of agents
self._waiting_jobs = OrderedDict(
) # rb queue for waiting jobs format:[(client_addr_as_bytes, Union[ClientNewJob,ClientNewBatchJob])]
self._job_running = {
} # indicates on which agent which job is running. format: {BackendJobId:(addr_as_bytes,ClientNewJob,start_time)}
self._batch_job_running = {
} # indicates on which agent which job is running. format: {BackendJobId:(addr_as_bytes,ClientNewBatchJob,
def __init__(self, address):
super(Server, self).__init__()
self.address = address
self.queue = asyncio.Queue()
self.context = Context.instance()
self.socket = self.context.socket(zmq.PULL)
self.poller = Poller()
self._listen_future = None
def reinitreq(req=None):
if req:
req.close()
del req
req = ctx.socket(zmq.CLIENT)
req.connect(url)
poller = Poller()
poller.register(req, zmq.POLLIN)
return req, poller
async def handleZMQMessage():
subscriber = Ctx.socket(zmq.SUB)
subscriber.connect("tcp://raspberry_ip:5555")
subscriber.setsockopt(zmq.SUBSCRIBE, b"1")
poller = Poller()
poller.register(subscriber, zmq.POLLOUT)
while True:
topic, data = await subscriber.recv_multipart()
await sendPicture(data)
def establish_socket(self):
self.context = Context()
self.socket = self.context.socket(REP)
self.socket.bind(self.address)
self.poller = Poller()
self.poller.register(self.socket, POLLIN)
self.ping = 0
while self.auth() is False:
time.sleep(1)
def run():
subscriber = Ctx.socket(zmq.SUB)
subscriber.connect(Url)
subscription = b"%03d" % 5
subscriber.setsockopt(zmq.SUBSCRIBE, subscription)
poller = Poller()
poller.register(subscriber, zmq.POLLOUT)
while True:
topic, data = yield from subscriber.recv_multipart()
# assert topic == subscription
print(data)
def run():
subscriber = Ctx.socket(zmq.SUB)
subscriber.connect(Url)
subscription = b"%03d" % 5
subscriber.setsockopt(zmq.SUBSCRIBE, subscription)
poller = Poller()
poller.register(subscriber, zmq.POLLOUT)
while True:
topic, data = yield from subscriber.recv_multipart()
#assert topic == subscription
print(data)
async def receiver():
"""receive messages with polling"""
pull = ctx.socket(zmq.PULL)
pull.connect(url)
poller = Poller()
poller.register(pull, zmq.POLLIN)
while True:
events = await poller.poll()
if pull in dict(events):
print("recving", events)
msg = await pull.recv_multipart()
print('recvd', msg)
async def receiver(response):
print('listening on {}'.format(url))
"""receive messages with polling"""
pull = ctx.socket(zmq.PULL)
pull.connect(url)
poller = Poller()
poller.register(pull, zmq.POLLIN)
while True:
body = await req.stream.get()
events = await poller.poll()
msg = await pull.recv_multipart() if pull in dict(
events) else 'waiting ...'
response.write(msg)
def __init__(self, url, logger, request_timeout=None, database=None):
self._logger = logger
self._database = database
self._context = Context.instance()
self._poller = Poller()
self._request = self._context.socket(zmq.DEALER)
self._request_timeout = request_timeout or 60
self._rds_bus_url = url
self._request.connect(self._rds_bus_url)
self._request_dict = dict()
self._io_loop = asyncio.get_event_loop()
self._running = False
asyncio.ensure_future(self.start())
def run_queue():
context = Context(1)
frontend = context.socket(zmq.ROUTER) # ROUTER
backend = context.socket(zmq.ROUTER) # ROUTER
frontend.bind("tcp://*:5555") # For clients
backend.bind("tcp://*:5556") # For workers
poll_workers = Poller()
poll_workers.register(backend, zmq.POLLIN)
poll_both = Poller()
poll_both.register(frontend, zmq.POLLIN)
poll_both.register(backend, zmq.POLLIN)
workers = []
while True:
if workers:
socks = yield from poll_both.poll()
else:
socks = yield from poll_workers.poll()
socks = dict(socks)
# Handle worker activity on backend
if socks.get(backend) == zmq.POLLIN:
# Use worker address for LRU routing
msg = yield from backend.recv_multipart()
if not msg:
break
print('I: received msg: {}'.format(msg))
address = msg[0]
workers.append(address)
# Everything after the second (delimiter) frame is reply
reply = msg[2:]
# Forward message to client if it's not a READY
if reply[0] != LRU_READY:
print('I: sending -- reply: {}'.format(reply))
yield from frontend.send_multipart(reply)
else:
print('I: received ready -- address: {}'.format(address))
if socks.get(frontend) == zmq.POLLIN:
# Get client request, route to first available worker
msg = yield from frontend.recv_multipart()
worker = workers.pop(0)
request = [worker, b''] + msg
print('I: sending -- worker: {} msg: {}'.format(worker, msg))
yield from backend.send_multipart(request)
class Server(Killable):
_event_class = asyncio.Event
def __init__(self, address):
super(Server, self).__init__()
self.address = address
self.queue = asyncio.Queue()
self.context = Context.instance()
self.socket = self.context.socket(zmq.PULL)
self.poller = Poller()
self._listen_future = None
async def _receive_into_queue(self):
while self.alive:
try:
events = await self.poller.poll(timeout=1e-4)
if self.socket in dict(events):
data = await self.socket.recv()
await self.queue.put(data)
except zmq.error.ZMQError:
await asyncio.sleep(1e-4)
async def start(self):
self.socket.bind('tcp://%s:%s' % self.address)
self.poller.register(self.socket, zmq.POLLIN)
self._listen_future = asyncio.ensure_future(self._receive_into_queue())
await asyncio.sleep(0)
async def shutdown(self):
self.kill()
self.poller.unregister(self.socket)
if self._listen_future is not None and not self._listen_future.done():
self._listen_future.cancel()
self.socket.close(linger=0)
await asyncio.sleep(0)
def iter_messages(self):
async def wrapped():
while self.alive:
try:
result = self.queue.get_nowait()
except asyncio.QueueEmpty:
await asyncio.sleep(1e-6)
else:
await asyncio.sleep(0)
return result
return AsyncGenerator(wrapped)
async def app(store: StorageServerStore, context: Context, name: str,
command: str, arg: str):
print("Starting...")
dirserv_commands = context.socket(zmq.REQ)
dirserv_commands.connect("tcp://127.0.0.1:5350")
print("App is started")
if command == "declare":
self_addr = await asyncio.wait_for(ping(dirserv_commands, name), 5)
print("Directory server report this client is run on {}".format(
self_addr))
command_port: Socket = context.socket(zmq.ROUTER)
port = command_port.bind_to_random_port("tcp://127.0.0.1")
self_entrypoint_addr = "tcp://{}:{}".format(self_addr, port)
await asyncio.wait_for(
cast_address(dirserv_commands, name, self_entrypoint_addr), 5)
print("Address {} casted on directory server".format(
self_entrypoint_addr))
with open(arg, mode='rb') as f:
store.files[arg] = VirtualFile(arg, f.read(), [])
await declare_file(dirserv_commands, arg, name)
print("File {} is declared, serving file...".format(arg))
poller = Poller()
poller.register(command_port, zmq.POLLIN)
while True:
events: List[Tuple[Socket, int]] = await poller.poll()
for socket, mark in events:
frames: List[Frame] = await socket.recv_multipart(copy=False)
id_frame = frames.pop(0)
frames.pop(0)
command_frame = frames.pop(0)
command = str(command_frame.bytes, 'utf8')
if socket == command_port:
if command == 'fs.read_file':
await read_file_handler(store, frames, socket,
id_frame)
elif command == "disown":
await disown_file(dirserv_commands, arg, name)
context.destroy()
return
elif command == "show":
content = await download_file(context, dirserv_commands, arg)
print("==== Content of '{}' ====".format(arg))
print(str(content, 'utf8'))
context.destroy()
return
else:
print("Unknown command {}".format(command))
context.destroy()
return
def reconnect(self):
'''connects or reconnects to the broker'''
if self.poller:
self.poller.unregister(self.socket)
if self.socket:
self.socket.close()
self.context = Context()
self.socket = self.context.socket(zmq.DEALER)
self.socket.connect(self.QWeatherStationIP + ':' + self.QWeatherStationSocket)
self.subsocket = self.context.socket(zmq.SUB)
self.subsocket.connect(self.QWeatherStationIP + ':' + str(int(self.QWeatherStationSocket) + SUBSOCKET))
self.poller = Poller()
self.poller.register(self.socket,zmq.POLLIN)
self.poller.register(self.subsocket,zmq.POLLIN)
async def receiver(name=''):
"""receive messages with polling"""
rep = ctx.socket(zmq.STREAM)
for url in urls:
rep.connect(url)
poller = Poller()
poller.register(rep, zmq.POLLIN)
cnt = 0
while True:
cnt += 1
events = await poller.poll()
if rep in dict(events):
msg = rep.recv().result()
print(name, 'recvd', recv_zipped_pickle(msg))
print(name, 'send:', cnt)
rep.send(send_zipped_pickle(cnt), routing_id=msg.routing_id)
def run_queue(context):
frontend = context.socket(zmq.ROUTER) # ROUTER
backend = context.socket(zmq.ROUTER) # ROUTER
frontend.bind(FRONT_END_ADDRESS) # For clients
backend.bind(BACK_END_ADDRESS) # For workers
poll_workers = Poller()
poll_workers.register(backend, zmq.POLLIN)
poll_both = Poller()
poll_both.register(frontend, zmq.POLLIN)
poll_both.register(backend, zmq.POLLIN)
workers = WorkerQueue()
heartbeat_at = time.time() + HEARTBEAT_INTERVAL
while True:
if len(workers.queue) > 0:
poller = poll_both
else:
poller = poll_workers
socks = yield from poller.poll(HEARTBEAT_INTERVAL * 1000)
socks = dict(socks)
# Handle worker activity on backend
if socks.get(backend) == zmq.POLLIN:
# Use worker address for LRU routing
frames = yield from backend.recv_multipart()
if not frames:
break
address = frames[0]
workers.ready(Worker(address))
# Validate control message, or return reply to client
msg = frames[1:]
if len(msg) == 1:
if msg[0] not in (PPP_READY, PPP_HEARTBEAT):
print("E: Invalid message from worker: %s" % msg)
else:
yield from frontend.send_multipart(msg)
# Send heartbeats to idle workers if it's time
if time.time() >= heartbeat_at:
for worker in workers.queue:
msg = [worker, PPP_HEARTBEAT]
yield from backend.send_multipart(msg)
heartbeat_at = time.time() + HEARTBEAT_INTERVAL
if socks.get(frontend) == zmq.POLLIN:
frames = yield from frontend.recv_multipart()
if not frames:
break
frames.insert(0, next(workers))
backend.send_multipart(frames)
workers.purge()
async def receiver(name=''):
"""receive messages with polling"""
rep = ctx.socket(zmq.REP)
for url in urls:
rep.connect(url)
poller = Poller()
poller.register(rep, zmq.POLLIN)
cnt = 0
while True:
cnt += 1
events = await poller.poll()
if rep in dict(events):
msg = await rep.recv()
print(name, 'recvd', recv_zipped_pickle(msg))
print(name, 'send:', cnt)
A = numpy.ones((1024, 1024))
rep.send(send_zipped_pickle([cnt, A]))
def __init__(self, context, agent_addr, client_addr):
self._content = context
self._loop = asyncio.get_event_loop()
self._agent_addr = agent_addr
self._client_addr = client_addr
self._agent_socket = context.socket(zmq.ROUTER)
self._client_socket = context.socket(zmq.ROUTER)
self._logger = logging.getLogger("inginious.backend")
# Enable support for ipv6
self._agent_socket.ipv6 = True
self._client_socket.ipv6 = True
self._poller = Poller()
self._poller.register(self._agent_socket, zmq.POLLIN)
self._poller.register(self._client_socket, zmq.POLLIN)
# Dict of available environments
# {
# "name": ("last_id", "created_last", ["agent_addr1", "agent_addr2"], "type")
# }
self._environments = {}
self._registered_clients = set() # addr of registered clients
# Dict of registered agents
# {
# agent_address: {"name": "friendly_name", "environments": environment_dict}
# } environment_dict is a described in AgentHello.
self._registered_agents = {}
# addr of available agents. May contain multiple times the same agent, because some agent can
# manage multiple jobs at once!
self._available_agents = []
# ping count per addr of agents
self._ping_count = {}
# These two share the same objects! Tuples should never be recreated.
self._waiting_jobs_pq = TopicPriorityQueue(
) # priority queue for waiting jobs
self._waiting_jobs = {
} # mapping job to job message, with key: [(client_addr_as_bytes, ClientNewJob])]
self._job_running = {
} # indicates on which agent which job is running. format: {BackendJobId:(addr_as_bytes,ClientNewJob,start_time)}
def run_worker(context):
# Socket to receive messages on
receiver = context.socket(zmq.PULL)
receiver.connect("tcp://localhost:5557")
# Socket to send messages to
sender = context.socket(zmq.PUSH)
sender.connect("tcp://localhost:5558")
# Socket for control input
controller = context.socket(zmq.SUB)
controller.connect("tcp://localhost:5559")
controller.setsockopt(zmq.SUBSCRIBE, b"")
# Process messages from receiver and controller
poller = Poller()
poller.register(receiver, zmq.POLLIN)
poller.register(controller, zmq.POLLIN)
# Process messages from both sockets
while True:
socks = yield from poller.poll()
socks = dict(socks)
if socks.get(receiver) == zmq.POLLIN:
message = yield from receiver.recv()
# Process task
workload = int(message) # Workload in msecs
# Do the work
yield from asyncio.sleep(workload / 1000.0)
# Send results to sink
yield from sender.send(message)
# Simple progress indicator for the viewer
sys.stdout.write(".")
sys.stdout.flush()
# Any waiting controller command acts as 'KILL'
if socks.get(controller) == zmq.POLLIN:
break
def __init__(self, IP, loop=None, verbose=False, debug=False):
if loop is None:
#from zmq import Context,Poller
# import asyncio
# from zmq.asyncio import Context,Poller
asyncio.set_event_loop_policy(
asyncio.WindowsSelectorEventLoopPolicy())
self.loop = asyncio.get_event_loop()
else:
self.loop = loop
IpAndPort = re.search(IPREPATTERN, IP)
assert IpAndPort != None, 'Ip not understood (tcp://xxx.xxx.xxx.xxx:XXXX or txp://*:XXXX)'
self.StationIP = IpAndPort.group(1)
self.StationSocket = IpAndPort.group(2)
assert self.StationIP[:
6] == 'tcp://', 'Ip not understood (tcp://xxx.xxx.xxx.xxx:XXXX or txp://*:XXXX)'
assert len(
self.StationSocket
) == 4, 'Port not understood (tcp://xxx.xxx.xxx.xxx:XXXX or txp://*:XXXX)'
formatting = '{:}: %(levelname)s: %(message)s'.format(
'QWeatherStation')
if debug:
logging.basicConfig(format=formatting, level=logging.DEBUG)
if verbose:
logging.basicConfig(format=formatting, level=logging.INFO)
self.servers = {} # key:value = clientaddress:value, bytes:string
self.clients = {} # key:value = clientaddress:value, bytes:string
self.servermethods = {}
self.serverjobs = {}
self.pinged = []
self.requesttimeoutdict = {}
self.cnx = Context()
self.socket = self.cnx.socket(zmq.ROUTER)
self.socket.bind(self.StationIP + ':' + self.StationSocket)
self.proxy = ThreadProxy(zmq.XSUB, zmq.XPUB)
self.proxy.bind_in(self.StationIP + ':' +
str(int(self.StationSocket) + PUBLISHSOCKET))
self.proxy.bind_out(self.StationIP + ':' +
str(int(self.StationSocket) + SUBSOCKET))
self.proxy.start()
self.poller = Poller()
self.poller.register(self.socket, zmq.POLLIN)
logging.info('Ready to run on IP: {:}'.format(self.get_own_ip()))
def run(ident):
# Socket to talk to server
print("Connecting to hello world server. Ctrl-C to exit early.\n")
socket = Ctx.socket(zmq.REQ)
socket.connect(Url)
poller = Poller()
poller.register(socket, zmq.POLLOUT)
# Do multiple requests, waiting each time for a response
for request in range(10):
message = '{} Hello {}'.format(ident, request)
message = message.encode('utf-8')
print("Sending message: {}".format(message))
yield from socket.send(message)
# Get the reply.
message = yield from socket.recv()
print("Received reply: {}".format(message))
print('exiting')
return 'nothing'
def run(ident):
# Socket to talk to server
print("Connecting to hello world server. Ctrl-C to exit early.\n")
socket = Ctx.socket(zmq.REQ)
socket.connect(Url)
poller = Poller()
poller.register(socket, zmq.POLLOUT)
# Do multiple requests, waiting each time for a response
for request in range(10):
message = '{} Hello {}'.format(ident, request)
message = message.encode('utf-8')
print("Sending message: {}".format(message))
yield from socket.send(message)
# Get the reply.
message = yield from socket.recv()
print("Received reply: {}".format(message))
print('exiting')
return 'nothing'
async def receiver():
global request
pull = context.socket(zmq.PULL)
pull.connect(addr)
poller = Poller()
poller.register(pull, zmq.POLLIN)
while True:
print("Sending request %s >>>\n" % request)
events = await poller.poll()
if pull in dict(events):
messages_received = pull.recv_multipart()
for message_received in messages_received:
print("<<< RECEIVED BYTES\n", message_received)
unpacked_maessage_split = message_received.decode().split(",")
print(f'MESSAGE DECODED SPLIT: {unpacked_maessage_split}\n')
request += 1
def run():
print("Getting ready for hello world client. Ctrl-C to exit.\n")
socket = Ctx.socket(zmq.REP)
socket.bind(Url)
poller = Poller()
poller.register(socket, zmq.POLLIN)
while True:
# Wait for next request from client
message = yield from socket.recv()
print("Received request: {}".format(message))
# Do some 'work'
yield from asyncio.sleep(1)
# Send reply back to client
message = message.decode('utf-8')
message = '{}, world'.format(message)
message = message.encode('utf-8')
print("Sending reply: {}".format(message))
yield from socket.send(message)
def run():
print("Getting ready for hello world client. Ctrl-C to exit.\n")
socket = Ctx.socket(zmq.REP)
socket.bind(Url)
poller = Poller()
poller.register(socket, zmq.POLLIN)
while True:
# Wait for next request from client
message = yield from socket.recv()
print("Received request: {}".format(message))
# Do some 'work'
yield from asyncio.sleep(1)
# Send reply back to client
message = message.decode('utf-8')
message = '{}, world'.format(message)
message = message.encode('utf-8')
print("Sending reply: {}".format(message))
yield from socket.send(message)
async def execute_parser_request(endpoint, parameters, timeout=5):
socket = TickerParser.zmqContext.socket(REQ)
socket.connect("tcp://parser:6900")
socket.setsockopt(LINGER, 0)
poller = Poller()
poller.register(socket, POLLIN)
await socket.send_multipart([endpoint] + parameters)
responses = await poller.poll(timeout * 1000)
if len(responses) != 0:
response = await socket.recv_multipart()
socket.close()
return response
else:
socket.close()
raise Exception("time out")
return None
async def worker_task(
worker_url, ident
): #receive the request from the client or send the message to backend
#context = zmq.Context(1)
context = Context.instance()
worker = context.socket(zmq.DEALER)
worker.identity = u"Worker-{}".format(ident).encode("ascii")
worker.connect(worker_url)
# Tell broker we're ready for work
worker.send(PPP_READY)
poll_worker = Poller()
poll_worker.register(worker, zmq.POLLIN)
while True:
#socks = dict(poll_worker.poll(HEARTBEAT_INTERVAL * 1000))
socks = dict(await poll_worker.poll(HEARTBEAT_INTERVAL * 1000))
#frames = worker.recv_multipart() # format:[b'client',b'',message_body]
print("{}".format(worker.identity.decode("ascii")))
if socks.get(worker) == zmq.POLLIN:
frames = await worker.recv_multipart()
#print("worker gets:",frames)
if not frames:
break # Interrupted
if len(frames) == 3:
print("I: Normal reply")
t1 = time.time()
image = Image.open(BytesIO(frames[2]))
result = crnn_recognition(image, model)
t2 = time.time() - t1
print("recogition time:", t2)
image.save("/home/cad488/test_images/" +
time.asctime(time.localtime(time.time())) + ".jpg")
await worker.send_multipart(
[frames[0], b"", result.encode("ascii")])
else:
print("E: Invalid message: %s" % frames)
def run_client(context):
print("I: Connecting to server...")
client = context.socket(zmq.REQ)
client.connect(SERVER_ENDPOINT)
poll = Poller()
poll.register(client, zmq.POLLIN)
sequence = 0
retries_left = REQUEST_RETRIES
while retries_left:
sequence += 1
request = str(sequence)
print("I: Sending (%s)" % request)
yield from client.send_string(request)
expect_reply = True
while expect_reply:
socks = yield from poll.poll(REQUEST_TIMEOUT)
socks = dict(socks)
if socks.get(client) == zmq.POLLIN:
reply = yield from client.recv()
if not reply:
break
if int(reply) == sequence:
print("I: Server replied OK (%s)" % reply)
retries_left = REQUEST_RETRIES
expect_reply = False
else:
print("E: Malformed reply from server: %s" % reply)
else:
print("W: No response from server, retrying...")
# Socket is confused. Close and remove it.
print('W: confused')
client.setsockopt(zmq.LINGER, 0)
client.unbind(SERVER_ENDPOINT)
#client.close()
poll.unregister(client)
retries_left -= 1
if retries_left == 0:
print("E: Server seems to be offline, abandoning")
return
print("I: Reconnecting and resending (%s)" % request)
# Create new connection
client = context.socket(zmq.REQ)
client.connect(SERVER_ENDPOINT)
poll.register(client, zmq.POLLIN)
yield from client.send_string(request)
async def process_request(service,
authorId,
request,
timeout=60,
retries=3):
socket = Processor.zmqContext.socket(REQ)
payload, responseText = None, None
socket.connect(Processor.services[service])
socket.setsockopt(LINGER, 0)
poller = Poller()
poller.register(socket, POLLIN)
request["timestamp"] = time()
request["authorId"] = authorId
await socket.send_multipart(
[Processor.clientId,
service.encode(),
dumps(request)])
responses = await poller.poll(timeout * 1000)
if len(responses) != 0:
payload, responseText = await socket.recv_multipart()
socket.close()
payload = loads(payload)
payload = payload if bool(payload) else None
responseText = None if responseText == b"" else responseText.decode(
)
if service in ["chart", "heatmap", "depth"
] and payload is not None:
payload["data"] = BytesIO(decodebytes(
payload["data"].encode()))
else:
socket.close()
if retries == 1: raise Exception("time out")
else:
payload, responseText = await Processor.process_request(
service, authorId, request, retries=retries - 1)
return payload, responseText
async def execute_database_request(endpoint, parameters, timeout=1):
socket = DatabaseConnector.zmqContext.socket(REQ)
payload, responseText = None, None
socket.connect("tcp://database:6900")
socket.setsockopt(LINGER, 0)
poller = Poller()
poller.register(socket, POLLIN)
await socket.send_multipart([
endpoint,
bytes(str(int((time() + timeout) * 1000)), encoding='utf8'),
parameters
])
responses = await poller.poll(timeout * 1000)
if len(responses) != 0:
[response] = await socket.recv_multipart()
socket.close()
return loads(response)
else:
socket.close()
return None
def run_server(loop):
"""Server routine"""
# Prepare our context and sockets
# Socket to talk to clients
clients = Ctx.socket(zmq.ROUTER)
clients.bind(Url_client)
workers = Ctx.socket(zmq.DEALER)
workers.bind(Url_worker)
# Start the workers
tasks = []
for idx in range(5):
ident = 'worker {}'.format(idx)
task = asyncio.ensure_future(run_worker(ident))
tasks.append(task)
poller = Poller()
poller.register(clients, zmq.POLLIN)
poller.register(workers, zmq.POLLIN)
print('mtserver ready for requests')
while True:
events = yield from poller.poll()
events = dict(events)
if clients in events:
message = yield from clients.recv_multipart()
printdbg('(run) received from client message_parts: {}'.format(
message))
client, empty, message = message[:3]
printdbg('(run) received from client message: {}'.format(
message))
printdbg('(run) sending message to workers: {}'.format(message))
yield from workers.send_multipart([client, b'', message])
elif workers in events:
message = yield from workers.recv_multipart()
printdbg('(run) received from worker message_parts: {}'.format(
message))
client, empty, message = message[:3]
printdbg('(run) received from worker message: {}'.format(
message))
yield from clients.send_multipart([client, b'', message])
printdbg('(run) sent message to clients: {}'.format(message))
def run_queue(context):
frontend = context.socket(zmq.ROUTER) # ROUTER
backend = context.socket(zmq.ROUTER) # ROUTER
frontend.bind(FRONT_END_ADDRESS) # For clients
backend.bind(BACK_END_ADDRESS) # For workers
poll_workers = Poller()
poll_workers.register(backend, zmq.POLLIN)
poll_both = Poller()
poll_both.register(frontend, zmq.POLLIN)
poll_both.register(backend, zmq.POLLIN)
workers = WorkerQueue()
heartbeat_at = time.time() + HEARTBEAT_INTERVAL
while True:
if len(workers.queue) > 0:
poller = poll_both
else:
poller = poll_workers
socks = yield from poller.poll(HEARTBEAT_INTERVAL * 1000)
socks = dict(socks)
# Handle worker activity on backend
if socks.get(backend) == zmq.POLLIN:
# Use worker address for LRU routing
frames = yield from backend.recv_multipart()
if not frames:
break
address = frames[0]
workers.ready(Worker(address))
# Validate control message, or return reply to client
msg = frames[1:]
if len(msg) == 1:
if msg[0] not in (PPP_READY, PPP_HEARTBEAT):
print("E: Invalid message from worker: %s" % msg)
else:
yield from frontend.send_multipart(msg)
# Send heartbeats to idle workers if it's time
if time.time() >= heartbeat_at:
for worker in workers.queue:
msg = [worker, PPP_HEARTBEAT]
yield from backend.send_multipart(msg)
heartbeat_at = time.time() + HEARTBEAT_INTERVAL
if socks.get(frontend) == zmq.POLLIN:
frames = yield from frontend.recv_multipart()
if not frames:
break
frames.insert(0, next(workers))
backend.send_multipart(frames)
workers.purge()
def run_client(context):
print("I: Connecting to server...")
client = context.socket(zmq.REQ)
client.connect(SERVER_ENDPOINT)
poll = Poller()
poll.register(client, zmq.POLLIN)
sequence = 0
retries_left = REQUEST_RETRIES
while retries_left:
sequence += 1
request = str(sequence)
print("I: Sending (%s)" % request)
yield from client.send_string(request)
expect_reply = True
while expect_reply:
socks = yield from poll.poll(REQUEST_TIMEOUT)
socks = dict(socks)
if socks.get(client) == zmq.POLLIN:
reply = yield from client.recv()
if not reply:
break
if int(reply) == sequence:
print("I: Server replied OK (%s)" % reply)
retries_left = REQUEST_RETRIES
expect_reply = False
else:
print("E: Malformed reply from server: %s" % reply)
else:
print("W: No response from server, retrying...")
# Socket is confused. Close and remove it.
print('W: confused')
client.setsockopt(zmq.LINGER, 0)
client.unbind(SERVER_ENDPOINT)
#client.close()
poll.unregister(client)
retries_left -= 1
if retries_left == 0:
print("E: Server seems to be offline, abandoning")
return
print("I: Reconnecting and resending (%s)" % request)
# Create new connection
client = context.socket(zmq.REQ)
client.connect(SERVER_ENDPOINT)
poll.register(client, zmq.POLLIN)
yield from client.send_string(request)
def run_proxy(socket_from, socket_to):
poller = Poller()
poller.register(socket_from, zmq.POLLIN)
poller.register(socket_to, zmq.POLLIN)
printdbg('(run_proxy) started')
while True:
events = yield from poller.poll()
events = dict(events)
if socket_from in events:
msg = yield from socket_from.recv_multipart()
printdbg('(run_proxy) received from frontend -- msg: {}'.format(
msg))
yield from socket_to.send_multipart(msg)
printdbg('(run_proxy) sent to backend -- msg: {}'.format(msg))
elif socket_to in events:
msg = yield from socket_to.recv_multipart()
printdbg('(run_proxy) received from backend -- msg: {}'.format(
msg))
yield from socket_from.send_multipart(msg)
printdbg('(run_proxy) sent to frontend -- msg: {}'.format(msg))
def run_broker(context):
# Prepare our context and sockets
frontend = context.socket(zmq.ROUTER)
backend = context.socket(zmq.DEALER)
frontend.bind("tcp://*:5559")
backend.bind("tcp://*:5560")
# Initialize poll set
poller = Poller()
poller.register(frontend, zmq.POLLIN)
poller.register(backend, zmq.POLLIN)
# Switch messages between sockets
while True:
socks = yield from poller.poll()
socks = dict(socks)
if socks.get(frontend) == zmq.POLLIN:
message = yield from frontend.recv_multipart()
print('received from frontend: {}'.format(message))
yield from backend.send_multipart(message)
if socks.get(backend) == zmq.POLLIN:
message = yield from backend.recv_multipart()
print('received from backend: {}'.format(message))
yield from frontend.send_multipart(message)
def start(self):
"""Start ZAP authentication"""
super().start()
self.__poller = Poller()
self.__poller.register(self.zap_socket, zmq.POLLIN)
self.__task = asyncio.ensure_future(self.__handle_zap())
class Backend(object):
"""
Backend. Central point of the architecture, manages communication between clients (frontends) and agents.
Schedule jobs on agents.
"""
def __init__(self, context, agent_addr, client_addr):
self._content = context
self._loop = asyncio.get_event_loop()
self._agent_addr = agent_addr
self._client_addr = client_addr
self._agent_socket = context.socket(zmq.ROUTER)
self._client_socket = context.socket(zmq.ROUTER)
self._logger = logging.getLogger("inginious.backend")
# Enable support for ipv6
self._agent_socket.ipv6 = True
self._client_socket.ipv6 = True
self._poller = Poller()
self._poller.register(self._agent_socket, zmq.POLLIN)
self._poller.register(self._client_socket, zmq.POLLIN)
# List of containers available
# {
# "name": ("last_id", "created_last", ["agent_addr1", "agent_addr2"])
# }
self._containers = {}
# Containers available per agent {"agent_addr": ["container_id_1", ...]}
self._containers_on_agent = {}
self._registered_clients = set() # addr of registered clients
self._registered_agents = {} # addr of registered agents
self._available_agents = [] # addr of available agents
self._ping_count = {} # ping count per addr of agents
self._waiting_jobs = OrderedDict() # rb queue for waiting jobs format:[(client_addr_as_bytes, ClientNewJob])]
self._job_running = {} # indicates on which agent which job is running. format: {BackendJobId:(addr_as_bytes,ClientNewJob,start_time)}
async def handle_agent_message(self, agent_addr, message):
"""Dispatch messages received from agents to the right handlers"""
message_handlers = {
AgentHello: self.handle_agent_hello,
AgentJobStarted: self.handle_agent_job_started,
AgentJobDone: self.handle_agent_job_done,
AgentJobSSHDebug: self.handle_agent_job_ssh_debug,
Pong: self._handle_pong
}
try:
func = message_handlers[message.__class__]
except:
raise TypeError("Unknown message type %s" % message.__class__)
self._create_safe_task(func(agent_addr, message))
async def handle_client_message(self, client_addr, message):
"""Dispatch messages received from clients to the right handlers"""
# Verify that the client is registered
if message.__class__ != ClientHello and client_addr not in self._registered_clients:
await ZMQUtils.send_with_addr(self._client_socket, client_addr, Unknown())
return
message_handlers = {
ClientHello: self.handle_client_hello,
ClientNewJob: self.handle_client_new_job,
ClientKillJob: self.handle_client_kill_job,
ClientGetQueue: self.handle_client_get_queue,
Ping: self.handle_client_ping
}
try:
func = message_handlers[message.__class__]
except:
raise TypeError("Unknown message type %s" % message.__class__)
self._create_safe_task(func(client_addr, message))
async def send_container_update_to_client(self, client_addrs):
""" :param client_addrs: list of clients to which we should send the update """
self._logger.debug("Sending containers updates...")
available_containers = tuple(self._containers.keys())
msg = BackendUpdateContainers(available_containers)
for client in client_addrs:
await ZMQUtils.send_with_addr(self._client_socket, client, msg)
async def handle_client_hello(self, client_addr, _: ClientHello):
""" Handle an ClientHello message. Send available containers to the client """
self._logger.info("New client connected %s", client_addr)
self._registered_clients.add(client_addr)
await self.send_container_update_to_client([client_addr])
async def handle_client_ping(self, client_addr, _: Ping):
""" Handle an Ping message. Pong the client """
await ZMQUtils.send_with_addr(self._client_socket, client_addr, Pong())
async def handle_client_new_job(self, client_addr, message: ClientNewJob):
""" Handle an ClientNewJob message. Add a job to the queue and triggers an update """
self._logger.info("Adding a new job %s %s to the queue", client_addr, message.job_id)
self._waiting_jobs[(client_addr, message.job_id)] = message
await self.update_queue()
async def handle_client_kill_job(self, client_addr, message: ClientKillJob):
""" Handle an ClientKillJob message. Remove a job from the waiting list or send the kill message to the right agent. """
# Check if the job is not in the queue
if (client_addr, message.job_id) in self._waiting_jobs:
del self._waiting_jobs[(client_addr, message.job_id)]
# Do not forget to send a JobDone
await ZMQUtils.send_with_addr(self._client_socket, client_addr, BackendJobDone(message.job_id, ("killed", "You killed the job"),
0.0, {}, {}, {}, "", None, "", ""))
# If the job is running, transmit the info to the agent
elif (client_addr, message.job_id) in self._job_running:
agent_addr = self._job_running[(client_addr, message.job_id)][0]
await ZMQUtils.send_with_addr(self._agent_socket, agent_addr, BackendKillJob((client_addr, message.job_id)))
else:
self._logger.warning("Client %s attempted to kill unknown job %s", str(client_addr), str(message.job_id))
async def handle_client_get_queue(self, client_addr, _: ClientGetQueue):
""" Handles a ClientGetQueue message. Send back info about the job queue"""
#jobs_running: a list of tuples in the form
#(job_id, is_current_client_job, agent_name, info, launcher, started_at, max_end)
jobs_running = list()
for backend_job_id, content in self._job_running.items():
jobs_running.append((content[1].job_id, backend_job_id[0] == client_addr, self._registered_agents[content[0]],
content[1].course_id+"/"+content[1].task_id,
content[1].launcher, int(content[2]), int(content[2])+content[1].time_limit))
#jobs_waiting: a list of tuples in the form
#(job_id, is_current_client_job, info, launcher, max_time)
jobs_waiting = list()
for job_client_addr, msg in self._waiting_jobs.items():
if isinstance(msg, ClientNewJob):
jobs_waiting.append((msg.job_id, job_client_addr[0] == client_addr, msg.course_id+"/"+msg.task_id, msg.launcher,
msg.time_limit))
await ZMQUtils.send_with_addr(self._client_socket, client_addr, BackendGetQueue(jobs_running, jobs_waiting))
async def update_queue(self):
"""
Send waiting jobs to available agents
"""
# For now, round-robin
not_found_for_agent = []
while len(self._available_agents) > 0 and len(self._waiting_jobs) > 0:
agent_addr = self._available_agents.pop(0)
# Find first job that can be run on this agent
found = False
client_addr, job_id, job_msg = None, None, None
for (client_addr, job_id), job_msg in self._waiting_jobs.items():
if job_msg.environment in self._containers_on_agent[agent_addr]:
found = True
break
if not found:
self._logger.debug("Nothing to do for agent %s", agent_addr)
not_found_for_agent.append(agent_addr)
continue
# Remove the job from the queue
del self._waiting_jobs[(client_addr, job_id)]
job_id = (client_addr, job_msg.job_id)
self._job_running[job_id] = (agent_addr, job_msg, time.time())
self._logger.info("Sending job %s %s to agent %s", client_addr, job_msg.job_id, agent_addr)
await ZMQUtils.send_with_addr(self._agent_socket, agent_addr, BackendNewJob(job_id, job_msg.course_id, job_msg.task_id,
job_msg.inputdata, job_msg.environment,
job_msg.enable_network, job_msg.time_limit,
job_msg.hard_time_limit, job_msg.mem_limit,
job_msg.debug))
# Do not forget to add again for which we did not find jobs to do
self._available_agents += not_found_for_agent
async def handle_agent_hello(self, agent_addr, message: AgentHello):
"""
Handle an AgentAvailable message. Add agent_addr to the list of available agents
"""
self._logger.info("Agent %s (%s) said hello", agent_addr, message.friendly_name)
if agent_addr in self._registered_agents:
# Delete previous instance of this agent, if any
await self._delete_agent(agent_addr)
self._registered_agents[agent_addr] = message.friendly_name
self._available_agents.extend([agent_addr for _ in range(0, message.available_job_slots)])
self._containers_on_agent[agent_addr] = message.available_containers.keys()
self._ping_count[agent_addr] = 0
# update information about available containers
for container_name, container_info in message.available_containers.items():
if container_name in self._containers:
# check if the id is the same
if self._containers[container_name][0] == container_info["id"]:
# ok, just add the agent to the list of agents that have the container
self._logger.debug("Registering container %s for agent %s", container_name, str(agent_addr))
self._containers[container_name][2].append(agent_addr)
elif self._containers[container_name][1] > container_info["created"]:
# containers stored have been created after the new one
# add the agent, but emit a warning
self._logger.warning("Container %s has multiple version: \n"
"\t Currently registered agents have version %s (%i)\n"
"\t New agent %s has version %s (%i)",
container_name,
self._containers[container_name][0], self._containers[container_name][1],
str(agent_addr), container_info["id"], container_info["created"])
self._containers[container_name][2].append(agent_addr)
else: # self._containers[container_name][1] < container_info["created"]:
# containers stored have been created before the new one
# add the agent, update the infos, and emit a warning
self._logger.warning("Container %s has multiple version: \n"
"\t Currently registered agents have version %s (%i)\n"
"\t New agent %s has version %s (%i)",
container_name,
self._containers[container_name][0], self._containers[container_name][1],
str(agent_addr), container_info["id"], container_info["created"])
self._containers[container_name] = (container_info["id"], container_info["created"],
self._containers[container_name][2] + [agent_addr])
else:
# just add it
self._logger.debug("Registering container %s for agent %s", container_name, str(agent_addr))
self._containers[container_name] = (container_info["id"], container_info["created"], [agent_addr])
# update the queue
await self.update_queue()
# update clients
await self.send_container_update_to_client(self._registered_clients)
async def handle_agent_job_started(self, agent_addr, message: AgentJobStarted):
"""Handle an AgentJobStarted message. Send the data back to the client"""
self._logger.debug("Job %s %s started on agent %s", message.job_id[0], message.job_id[1], agent_addr)
await ZMQUtils.send_with_addr(self._client_socket, message.job_id[0], BackendJobStarted(message.job_id[1]))
async def handle_agent_job_done(self, agent_addr, message: AgentJobDone):
"""Handle an AgentJobDone message. Send the data back to the client, and start new job if needed"""
if agent_addr in self._registered_agents:
self._logger.info("Job %s %s finished on agent %s", message.job_id[0], message.job_id[1], agent_addr)
# Remove the job from the list of running jobs
del self._job_running[message.job_id]
# Sent the data back to the client
await ZMQUtils.send_with_addr(self._client_socket, message.job_id[0], BackendJobDone(message.job_id[1], message.result,
message.grade, message.problems,
message.tests, message.custom,
message.state, message.archive,
message.stdout, message.stderr))
# The agent is available now
self._available_agents.append(agent_addr)
else:
self._logger.warning("Job result %s %s from non-registered agent %s", message.job_id[0], message.job_id[1], agent_addr)
# update the queue
await self.update_queue()
async def handle_agent_job_ssh_debug(self, _, message: AgentJobSSHDebug):
"""Handle an AgentJobSSHDebug message. Send the data back to the client"""
await ZMQUtils.send_with_addr(self._client_socket, message.job_id[0], BackendJobSSHDebug(message.job_id[1], message.host, message.port,
message.password))
async def run(self):
self._logger.info("Backend started")
self._agent_socket.bind(self._agent_addr)
self._client_socket.bind(self._client_addr)
self._loop.call_later(1, self._create_safe_task, self._do_ping())
try:
while True:
socks = await self._poller.poll()
socks = dict(socks)
# New message from agent
if self._agent_socket in socks:
agent_addr, message = await ZMQUtils.recv_with_addr(self._agent_socket)
await self.handle_agent_message(agent_addr, message)
# New message from client
if self._client_socket in socks:
client_addr, message = await ZMQUtils.recv_with_addr(self._client_socket)
await self.handle_client_message(client_addr, message)
except asyncio.CancelledError:
return
except KeyboardInterrupt:
return
async def _handle_pong(self, agent_addr, _ : Pong):
""" Handle a pong """
self._ping_count[agent_addr] = 0
async def _do_ping(self):
""" Ping the agents """
# the list() call here is needed, as we remove entries from _registered_agents!
for agent_addr, friendly_name in list(self._registered_agents.items()):
try:
ping_count = self._ping_count.get(agent_addr, 0)
if ping_count > 5:
self._logger.warning("Agent %s (%s) does not respond: removing from list.", agent_addr, friendly_name)
delete_agent = True
else:
self._ping_count[agent_addr] = ping_count + 1
await ZMQUtils.send_with_addr(self._agent_socket, agent_addr, Ping())
delete_agent = False
except:
# This should not happen, but it's better to check anyway.
self._logger.exception("Failed to send ping to agent %s (%s). Removing it from list.", agent_addr, friendly_name)
delete_agent = True
if delete_agent:
try:
await self._delete_agent(agent_addr)
except:
self._logger.exception("Failed to delete agent %s (%s)!", agent_addr, friendly_name)
self._loop.call_later(1, self._create_safe_task, self._do_ping())
async def _delete_agent(self, agent_addr):
""" Deletes an agent """
self._available_agents = [agent for agent in self._available_agents if agent != agent_addr]
del self._registered_agents[agent_addr]
await self._recover_jobs(agent_addr)
async def _recover_jobs(self, agent_addr):
""" Recover the jobs sent to a crashed agent """
for (client_addr, job_id), (agent, job_msg, _) in reversed(list(self._job_running.items())):
if agent == agent_addr:
await ZMQUtils.send_with_addr(self._client_socket, client_addr,
BackendJobDone(job_id, ("crash", "Agent restarted"),
0.0, {}, {}, {}, "", None, None, None))
del self._job_running[(client_addr, job_id)]
await self.update_queue()
def _create_safe_task(self, coroutine):
""" Calls self._loop.create_task with a safe (== with logged exception) coroutine """
task = self._loop.create_task(coroutine)
task.add_done_callback(self.__log_safe_task)
return task
def __log_safe_task(self, task):
exception = task.exception()
if exception is not None:
self._logger.exception("An exception occurred while running a Task", exc_info=exception)
class SchedulerConnection(object):
__slots__ = (
'address',
# context object to open socket connections
'context',
# pull socket to receive check definitions from scheduler
'pull',
# poller object for `pull` socket
'poller',
# monitor socket for `pull` socket
'monitor_socket',
# poller object for monitor socket
'monitor_poller',
'first_missing',
)
def __init__(self, address):
self.pull = self.poller = None
self.monitor_poller = self.monitor_socket = None
self.address = address
self.context = Context.instance()
self.open()
self.first_missing = None
def __str__(self):
return self.address
def __repr__(self):
return 'Scheduler({})'.format(self.address)
def open(self):
self.pull = self.context.socket(PULL)
logger.info('%s - opening pull socket ...', self)
self.pull.connect(self.address)
if settings.SCHEDULER_MONITOR:
logger.info('%s - opening monitor socket ...', self)
self.monitor_socket = self.pull.get_monitor_socket(
events=EVENT_DISCONNECTED
)
self.register()
def register(self):
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
if settings.SCHEDULER_MONITOR:
self.monitor_poller = Poller()
self.monitor_poller.register(self.monitor_socket, POLLIN)
logger.info('%s - all sockets are successfully registered '
'in poller objects ...', self)
def close(self):
"""Unregister open sockets from poller objects and close them."""
self.unregister()
logger.info('%s - closing open sockets ...', self)
self.pull.close()
if settings.SCHEDULER_MONITOR:
self.monitor_socket.close()
logger.info('%s - connection closed successfully ...', self)
def unregister(self):
"""Unregister open sockets from poller object."""
logger.info('%s - unregistering sockets from poller objects ...', self)
self.poller.unregister(self.pull)
if settings.SCHEDULER_MONITOR:
self.monitor_poller.unregister(self.monitor_socket)
def reconnect(self):
self.close()
self.open()
self.first_missing = None
@asyncio.coroutine
def receive(self):
check = None
events = yield from self.poller.poll(timeout=2000)
if self.pull in dict(events):
check = yield from self.pull.recv_multipart()
check = jsonapi.loads(check[0])
if check:
self.first_missing = None
elif self.first_missing is None:
self.first_missing = datetime.now(tz=pytz.utc)
if self.first_missing:
diff = datetime.now(tz=pytz.utc) - self.first_missing
delta = timedelta(minutes=settings.SCHEDULER_LIVENESS_IN_MINUTES)
if diff > delta:
logger.warning(
'Alamo worker `%s` pid `%s` try to reconnect to '
'`%s` scheduler.',
settings.WORKER_FQDN, settings.WORKER_PID, self
)
self.reconnect()
return check
@asyncio.coroutine
def receive_event(self):
event = None
events = yield from self.monitor_poller.poll(timeout=2000)
if self.monitor_socket in dict(events):
msg = yield from self.monitor_socket.recv_multipart(
flags=NOBLOCK)
event = parse_monitor_message(msg)
return event
def run_broker(loop):
""" main broker method """
print('(run_broker) starting')
url_worker = "inproc://workers"
url_client = "inproc://clients"
client_nbr = NBR_CLIENTS * 3
# Prepare our context and sockets
context = Context()
frontend = context.socket(zmq.ROUTER)
frontend.bind(url_client)
backend = context.socket(zmq.ROUTER)
backend.bind(url_worker)
print('(run_broker) creating workers and clients')
# create workers and clients threads
worker_tasks = []
for idx in range(NBR_WORKERS):
task = asyncio.ensure_future(run_worker(url_worker, context, idx))
worker_tasks.append(task)
client_tasks = []
for idx in range(NBR_CLIENTS):
task = asyncio.ensure_future(run_client(url_client, context, idx))
client_tasks.append(task)
print('(run_broker) after creating workers and clients')
# Logic of LRU loop
# - Poll backend always, frontend only if 1+ worker ready
# - If worker replies, queue worker as ready and forward reply
# to client if necessary
# - If client requests, pop next worker and send request to it
# Queue of available workers
available_workers = 0
workers_list = []
# init poller
poller = Poller()
# Always poll for worker activity on backend
poller.register(backend, zmq.POLLIN)
# Poll front-end only if we have available workers
poller.register(frontend, zmq.POLLIN)
while True:
socks = yield from poller.poll()
socks = dict(socks)
# Handle worker activity on backend
if (backend in socks and socks[backend] == zmq.POLLIN):
# Queue worker address for LRU routing
message = yield from backend.recv_multipart()
assert available_workers < NBR_WORKERS
worker_addr = message[0]
# add worker back to the list of workers
available_workers += 1
workers_list.append(worker_addr)
# Second frame is empty
empty = message[1]
assert empty == b""
# Third frame is READY or else a client reply address
client_addr = message[2]
# If client reply, send rest back to frontend
if client_addr != b'READY':
# Following frame is empty
empty = message[3]
assert empty == b""
reply = message[4]
yield from frontend.send_multipart([client_addr, b"", reply])
printdbg('(run_broker) to frontend -- reply: "{}"'.format(
reply))
client_nbr -= 1
if client_nbr == 0:
printdbg('(run_broker) exiting')
break # Exit after N messages
# poll on frontend only if workers are available
if available_workers > 0:
if (frontend in socks and socks[frontend] == zmq.POLLIN):
# Now get next client request, route to LRU worker
# Client request is [address][empty][request]
response = yield from frontend.recv_multipart()
[client_addr, empty, request] = response
assert empty == b""
# Dequeue and drop the next worker address
available_workers += -1
worker_id = workers_list.pop()
yield from backend.send_multipart(
[worker_id, b"", client_addr, b"", request])
printdbg('(run_broker) to backend -- request: "{}"'.format(
request))
#out of infinite loop: do some housekeeping
printdbg('(run_broker) finished')
for worker_task in worker_tasks:
worker_task.cancel()
printdbg('(run_broker) workers cancelled')
yield from asyncio.sleep(1)
frontend.close()
backend.close()
#context.term() # Caution: calling term() blocks.
loop.stop()
printdbg('(run_broker) returning')
return 'finished ok'
