def subscribe(actor_id, worker_ch):
"""
Main loop for the Actor executor worker. Subscribes to the actor's inbox and executes actor
containers when message arrive. Also subscribes to the worker channel for future communications.
:return:
"""
actor_ch = ActorMsgChannel(actor_id)
t = threading.Thread(target=process_worker_ch, args=(worker_ch, actor_id, actor_ch))
t.start()
print("Worker subscribing to actor channel...")
while keep_running:
update_worker_status(actor_id, worker_ch.name, READY)
try:
msg = actor_ch.get(timeout=2)
except channelpy.ChannelTimeoutException:
continue
print("Received message {}. Starting actor container...".format(str(msg)))
message = msg.pop("msg", "")
try:
stats, logs = execute_actor(actor_id, worker_ch, image, message, msg)
except DockerStartContainerError as e:
print("Got DockerStartContainerError: {}".format(str(e)))
Actor.set_status(actor_id, ERROR)
continue
# add the execution to the actor store
print("Actor container finished successfully. Got stats object:{}".format(str(stats)))
exc_id = Execution.add_execution(actor_id, stats)
Execution.set_logs(exc_id, logs)
def execute_actor(actor_id, worker_ch, image, msg, d={}):
result = {'cpu': 0,
'io': 0,
'runtime': 0 }
cli = docker.AutoVersionClient(base_url=dd)
d['MSG'] = msg
container = cli.create_container(image=image, environment=d)
try:
cli.start(container=container.get('Id'))
except Exception as e:
# if there was an error starting the container, user will need to debig
raise DockerStartContainerError("Could not start container {}. Exception {}".format(container.get('Id'), str(e)))
start = timeit.default_timer()
update_worker_status(actor_id, worker_ch, BUSY)
running = True
try:
stats_obj = cli.stats(container=container.get('Id'), decode=True)
except ReadTimeout:
# if the container execution is so fast that the inital stats object cannot be created,
# we skip the running loop and return a minimal stats object
result['cpu'] = 1
result['runtime'] = 1
return result
while running:
try:
stats = next(stats_obj)
result['cpu'] += stats['cpu_stats']['cpu_usage']['total_usage']
result['io'] += stats['network']['rx_bytes']
except ReadTimeoutError:
# container stopped before another stats record could be read, just ignore and move on
running = False
if running:
try:
cli.wait(container=container.get('Id'), timeout=1)
running = False
except ReadTimeout:
# the wait timed out so check if we are beyond the max_run_time
runtime = timeit.default_timer() - start
if max_run_time > 0 and max_run_time < runtime:
cli.stop(container.get('Id'))
running = False
stop = timeit.default_timer()
logs = cli.logs(container.get('Id'))
result['runtime'] = int(stop - start)
return result, logs
