def subscribe(actor_id, worker_ch): """ Main loop for the Actor executor worker. Subscribes to the actor's inbox and executes actor containers when message arrive. Also subscribes to the worker channel for future communications. :return: """ actor_ch = ActorMsgChannel(actor_id) t = threading.Thread(target=process_worker_ch, args=(worker_ch, actor_id, actor_ch)) t.start() print("Worker subscribing to actor channel...") while keep_running: update_worker_status(actor_id, worker_ch.name, READY) try: msg = actor_ch.get(timeout=2) except channelpy.ChannelTimeoutException: continue print("Received message {}. Starting actor container...".format(str(msg))) message = msg.pop("msg", "") try: stats, logs = execute_actor(actor_id, worker_ch, image, message, msg) except DockerStartContainerError as e: print("Got DockerStartContainerError: {}".format(str(e))) Actor.set_status(actor_id, ERROR) continue # add the execution to the actor store print("Actor container finished successfully. Got stats object:{}".format(str(stats))) exc_id = Execution.add_execution(actor_id, stats) Execution.set_logs(exc_id, logs)
def execute_actor(actor_id, worker_ch, image, msg, d={}): result = {'cpu': 0, 'io': 0, 'runtime': 0 } cli = docker.AutoVersionClient(base_url=dd) d['MSG'] = msg container = cli.create_container(image=image, environment=d) try: cli.start(container=container.get('Id')) except Exception as e: # if there was an error starting the container, user will need to debig raise DockerStartContainerError("Could not start container {}. Exception {}".format(container.get('Id'), str(e))) start = timeit.default_timer() update_worker_status(actor_id, worker_ch, BUSY) running = True try: stats_obj = cli.stats(container=container.get('Id'), decode=True) except ReadTimeout: # if the container execution is so fast that the inital stats object cannot be created, # we skip the running loop and return a minimal stats object result['cpu'] = 1 result['runtime'] = 1 return result while running: try: stats = next(stats_obj) result['cpu'] += stats['cpu_stats']['cpu_usage']['total_usage'] result['io'] += stats['network']['rx_bytes'] except ReadTimeoutError: # container stopped before another stats record could be read, just ignore and move on running = False if running: try: cli.wait(container=container.get('Id'), timeout=1) running = False except ReadTimeout: # the wait timed out so check if we are beyond the max_run_time runtime = timeit.default_timer() - start if max_run_time > 0 and max_run_time < runtime: cli.stop(container.get('Id')) running = False stop = timeit.default_timer() logs = cli.logs(container.get('Id')) result['runtime'] = int(stop - start) return result, logs