class Interchange(object): """ Interchange is a task orchestrator for distributed systems. 1. Asynchronously queue large volume of tasks (>100K) 2. Allow for workers to join and leave the union 3. Detect workers that have failed using heartbeats 4. Service single and batch requests from workers 5. Be aware of requests worker resource capacity, eg. schedule only jobs that fit into walltime. TODO: We most likely need a PUB channel to send out global commands, like shutdown """ def __init__(self, config, client_address="127.0.0.1", interchange_address="127.0.0.1", client_ports=(50055, 50056, 50057), worker_ports=None, worker_port_range=(54000, 55000), cores_per_worker=1.0, worker_debug=False, launch_cmd=None, heartbeat_threshold=60, logdir=".", logging_level=logging.INFO, poll_period=10, endpoint_id=None, suppress_failure=False, max_heartbeats_missed=2 ): """ Parameters ---------- config : funcx.Config object Funcx config object that describes how compute should be provisioned client_address : str The ip address at which the parsl client can be reached. Default: "127.0.0.1" interchange_address : str The ip address at which the workers will be able to reach the Interchange. Default: "127.0.0.1" client_ports : triple(int, int, int) The ports at which the client can be reached launch_cmd : str TODO : update worker_ports : tuple(int, int) The specific two ports at which workers will connect to the Interchange. Default: None worker_port_range : tuple(int, int) The interchange picks ports at random from the range which will be used by workers. This is overridden when the worker_ports option is set. Defauls: (54000, 55000) cores_per_worker : float cores to be assigned to each worker. Oversubscription is possible by setting cores_per_worker < 1.0. Default=1 worker_debug : Bool Enables worker debug logging. heartbeat_threshold : int Number of seconds since the last heartbeat after which worker is considered lost. logdir : str Parsl log directory paths. Logs and temp files go here. Default: '.' logging_level : int Logging level as defined in the logging module. Default: logging.INFO (20) endpoint_id : str Identity string that identifies the endpoint to the broker poll_period : int The main thread polling period, in milliseconds. Default: 10ms suppress_failure : Bool When set to True, the interchange will attempt to suppress failures. Default: False max_heartbeats_missed : int Number of heartbeats missed before setting kill_event """ self.logdir = logdir try: os.makedirs(self.logdir) except FileExistsError: pass start_file_logger("{}/interchange.log".format(self.logdir), level=logging_level) logger.info("logger location {}".format(logger.handlers)) logger.info("Initializing Interchange process with Endpoint ID: {}".format(endpoint_id)) self.config = config logger.info("Got config : {}".format(config)) self.strategy = self.config.strategy self.client_address = client_address self.interchange_address = interchange_address self.suppress_failure = suppress_failure self.poll_period = poll_period self.serializer = FuncXSerializer() logger.info("Attempting connection to client at {} on ports: {},{},{}".format( client_address, client_ports[0], client_ports[1], client_ports[2])) self.context = zmq.Context() self.task_incoming = self.context.socket(zmq.DEALER) self.task_incoming.set_hwm(0) self.task_incoming.RCVTIMEO = 10 # in milliseconds logger.info("Task incoming on tcp://{}:{}".format(client_address, client_ports[0])) self.task_incoming.connect("tcp://{}:{}".format(client_address, client_ports[0])) self.results_outgoing = self.context.socket(zmq.DEALER) self.results_outgoing.set_hwm(0) logger.info("Results outgoing on tcp://{}:{}".format(client_address, client_ports[1])) self.results_outgoing.connect("tcp://{}:{}".format(client_address, client_ports[1])) self.command_channel = self.context.socket(zmq.DEALER) self.command_channel.RCVTIMEO = 1000 # in milliseconds # self.command_channel.set_hwm(0) logger.info("Command channel on tcp://{}:{}".format(client_address, client_ports[2])) self.command_channel.connect("tcp://{}:{}".format(client_address, client_ports[2])) logger.info("Connected to client") self.pending_task_queue = {} self.containers = {} self.total_pending_task_count = 0 self.fxs = FuncXClient() logger.info("Interchange address is {}".format(self.interchange_address)) self.worker_ports = worker_ports self.worker_port_range = worker_port_range self.task_outgoing = self.context.socket(zmq.ROUTER) self.task_outgoing.set_hwm(0) self.results_incoming = self.context.socket(zmq.ROUTER) self.results_incoming.set_hwm(0) # initalize the last heartbeat time to start the loop self.last_heartbeat = time.time() self.max_heartbeats_missed = max_heartbeats_missed self.endpoint_id = endpoint_id if self.worker_ports: self.worker_task_port = self.worker_ports[0] self.worker_result_port = self.worker_ports[1] self.task_outgoing.bind("tcp://*:{}".format(self.worker_task_port)) self.results_incoming.bind("tcp://*:{}".format(self.worker_result_port)) else: self.worker_task_port = self.task_outgoing.bind_to_random_port('tcp://*', min_port=worker_port_range[0], max_port=worker_port_range[1], max_tries=100) self.worker_result_port = self.results_incoming.bind_to_random_port('tcp://*', min_port=worker_port_range[0], max_port=worker_port_range[1], max_tries=100) logger.info("Bound to ports {},{} for incoming worker connections".format( self.worker_task_port, self.worker_result_port)) self._ready_manager_queue = {} self.heartbeat_threshold = heartbeat_threshold self.blocks = {} # type: Dict[str, str] self.block_id_map = {} self.launch_cmd = launch_cmd self.last_core_hr_counter = 0 if not launch_cmd: self.launch_cmd = ("funcx-manager {debug} {max_workers} " "-c {cores_per_worker} " "--poll {poll_period} " "--task_url={task_url} " "--result_url={result_url} " "--logdir={logdir} " "--block_id={{block_id}} " "--hb_period={heartbeat_period} " "--hb_threshold={heartbeat_threshold} " "--worker_mode={worker_mode} " "--scheduler_mode={scheduler_mode} " "--worker_type={{worker_type}} ") self.current_platform = {'parsl_v': PARSL_VERSION, 'python_v': "{}.{}.{}".format(sys.version_info.major, sys.version_info.minor, sys.version_info.micro), 'os': platform.system(), 'hname': platform.node(), 'dir': os.getcwd()} logger.info("Platform info: {}".format(self.current_platform)) self._block_counter = 0 try: self.load_config() except Exception as e: logger.exception("Caught exception") raise def load_config(self): """ Load the config """ logger.info("Loading endpoint local config") working_dir = self.config.working_dir if self.config.working_dir is None: working_dir = "{}/{}".format(self.logdir, "worker_logs") logger.info("Setting working_dir: {}".format(working_dir)) self.config.provider.script_dir = working_dir if hasattr(self.config.provider, 'channel'): self.config.provider.channel.script_dir = os.path.join(working_dir, 'submit_scripts') self.config.provider.channel.makedirs(self.config.provider.channel.script_dir, exist_ok=True) os.makedirs(self.config.provider.script_dir, exist_ok=True) debug_opts = "--debug" if self.config.worker_debug else "" max_workers = "" if self.config.max_workers_per_node == float('inf') \ else "--max_workers={}".format(self.config.max_workers_per_node) worker_task_url = f"tcp://{self.interchange_address}:{self.worker_task_port}" worker_result_url = f"tcp://{self.interchange_address}:{self.worker_result_port}" l_cmd = self.launch_cmd.format(debug=debug_opts, max_workers=max_workers, cores_per_worker=self.config.cores_per_worker, #mem_per_worker=self.config.mem_per_worker, prefetch_capacity=self.config.prefetch_capacity, task_url=worker_task_url, result_url=worker_result_url, nodes_per_block=self.config.provider.nodes_per_block, heartbeat_period=self.config.heartbeat_period, heartbeat_threshold=self.config.heartbeat_threshold, poll_period=self.config.poll_period, worker_mode=self.config.worker_mode, scheduler_mode=self.config.scheduler_mode, logdir=working_dir) self.launch_cmd = l_cmd logger.info("Launch command: {}".format(self.launch_cmd)) if self.config.scaling_enabled: logger.info("Scaling ...") self.scale_out(self.config.provider.init_blocks) def get_tasks(self, count): """ Obtains a batch of tasks from the internal pending_task_queue Parameters ---------- count: int Count of tasks to get from the queue Returns ------- List of upto count tasks. May return fewer than count down to an empty list eg. [{'task_id':<x>, 'buffer':<buf>} ... ] """ tasks = [] for i in range(0, count): try: x = self.pending_task_queue.get(block=False) except queue.Empty: break else: tasks.append(x) return tasks def migrate_tasks_to_internal(self, kill_event, status_request): """Pull tasks from the incoming tasks 0mq pipe onto the internal pending task queue Parameters: ----------- kill_event : threading.Event Event to let the thread know when it is time to die. """ logger.info("[TASK_PULL_THREAD] Starting") task_counter = 0 poller = zmq.Poller() poller.register(self.task_incoming, zmq.POLLIN) while not kill_event.is_set(): # Check when the last heartbeat was. # logger.debug(f"[TASK_PULL_THREAD] Last heartbeat: {self.last_heartbeat}") if int(time.time() - self.last_heartbeat) > (self.heartbeat_threshold * self.max_heartbeats_missed): logger.critical("[TASK_PULL_THREAD] Missed too many heartbeats. Setting kill event.") kill_event.set() break try: msg = self.task_incoming.recv_pyobj() self.last_heartbeat = time.time() except zmq.Again: # We just timed out while attempting to receive logger.debug("[TASK_PULL_THREAD] {} tasks in internal queue".format(self.total_pending_task_count)) continue if msg == 'STOP': kill_event.set() break elif msg == 'STATUS_REQUEST': logger.info("Got STATUS_REQUEST") status_request.set() else: logger.info("[TASK_PULL_THREAD] Received task:{}".format(msg)) task_type = self.get_container(msg['task_id'].split(";")[1]) msg['container'] = task_type if task_type not in self.pending_task_queue: self.pending_task_queue[task_type] = queue.Queue(maxsize=10 ** 6) self.pending_task_queue[task_type].put(msg) self.total_pending_task_count += 1 logger.debug("[TASK_PULL_THREAD] pending task count: {}".format(self.total_pending_task_count)) task_counter += 1 logger.debug("[TASK_PULL_THREAD] Fetched task:{}".format(task_counter)) def get_container(self, container_uuid): """ Get the container image location if it is not known to the interchange""" if container_uuid not in self.containers: if container_uuid == 'RAW' or not container_uuid: self.containers[container_uuid] = 'RAW' else: try: container = self.fxs.get_container(container_uuid, self.config.container_type) except Exception: logger.exception("[FETCH_CONTAINER] Unable to resolve container location") self.containers[container_uuid] = 'RAW' else: logger.info("[FETCH_CONTAINER] Got container info: {}".format(container)) self.containers[container_uuid] = container.get('location', 'RAW') return self.containers[container_uuid] def get_total_tasks_outstanding(self): """ Get the outstanding tasks in total """ outstanding = {} for task_type in self.pending_task_queue: outstanding[task_type] = outstanding.get(task_type, 0) + self.pending_task_queue[task_type].qsize() for manager in self._ready_manager_queue: for task_type in self._ready_manager_queue[manager]['tasks']: outstanding[task_type] = outstanding.get(task_type, 0) + len(self._ready_manager_queue[manager]['tasks'][task_type]) return outstanding def get_total_live_workers(self): """ Get the total active workers """ active = 0 for manager in self._ready_manager_queue: if self._ready_manager_queue[manager]['active']: active += self._ready_manager_queue[manager]['max_worker_count'] return active def get_outstanding_breakdown(self): """ Get outstanding breakdown per manager and in the interchange queues Returns ------- List of status for online elements [ (element, tasks_pending, status) ... ] """ pending_on_interchange = self.total_pending_task_count # Reporting pending on interchange is a deviation from Parsl reply = [('interchange', pending_on_interchange, True)] for manager in self._ready_manager_queue: resp = (manager.decode('utf-8'), sum([len(tids) for tids in self._ready_manager_queue[manager]['tasks'].values()]), self._ready_manager_queue[manager]['active']) reply.append(resp) return reply def _hold_block(self, block_id): """ Sends hold command to all managers which are in a specific block Parameters ---------- block_id : str Block identifier of the block to be put on hold """ for manager in self._ready_manager_queue: if self._ready_manager_queue[manager]['active'] and \ self._ready_manager_queue[manager]['block_id'] == block_id: logger.debug("[HOLD_BLOCK]: Sending hold to manager: {}".format(manager)) self.hold_manager(manager) def hold_manager(self, manager): """ Put manager on hold Parameters ---------- manager : str Manager id to be put on hold while being killed """ if manager in self._ready_manager_queue: self._ready_manager_queue[manager]['active'] = False reply = True else: reply = False def _command_server(self, kill_event): """ Command server to run async command to the interchange """ logger.debug("[COMMAND] Command Server Starting") while not kill_event.is_set(): try: command_req = self.command_channel.recv_pyobj() logger.debug("[COMMAND] Received command request: {}".format(command_req)) if command_req == "OUTSTANDING_C": reply = self.get_total_outstanding() elif command_req == "MANAGERS": reply = self.get_outstanding_breakdown() elif command_req.startswith("HOLD_WORKER"): cmd, s_manager = command_req.split(';') manager = s_manager.encode('utf-8') logger.info("[CMD] Received HOLD_WORKER for {}".format(manager)) if manager in self._ready_manager_queue: self._ready_manager_queue[manager]['active'] = False reply = True else: reply = False elif command_req == "HEARTBEAT": logger.info("[CMD] Received heartbeat message from hub") reply = "HBT,{}".format(self.endpoint_id) elif command_req == "SHUTDOWN": logger.info("[CMD] Received SHUTDOWN command") kill_event.set() reply = True else: reply = None logger.debug("[COMMAND] Reply: {}".format(reply)) self.command_channel.send_pyobj(reply) except zmq.Again: logger.debug("[COMMAND] is alive") continue def stop(self): """Prepare the interchange for shutdown""" self._kill_event.set() self._task_puller_thread.join() self._command_thread.join() def start(self, poll_period=None): """ Start the Interchange Parameters: ---------- poll_period : int poll_period in milliseconds """ logger.info("Incoming ports bound") if poll_period is None: poll_period = self.poll_period start = time.time() count = 0 self._kill_event = threading.Event() self._status_request = threading.Event() self._task_puller_thread = threading.Thread(target=self.migrate_tasks_to_internal, args=(self._kill_event, self._status_request, )) self._task_puller_thread.start() self._command_thread = threading.Thread(target=self._command_server, args=(self._kill_event, )) self._command_thread.start() try: logger.debug("Starting strategy.") self.strategy.start(self) except RuntimeError as e: # This is raised when re-registering an endpoint as strategy already exists logger.debug("Failed to start strategy.") logger.info(e) poller = zmq.Poller() # poller.register(self.task_incoming, zmq.POLLIN) poller.register(self.task_outgoing, zmq.POLLIN) poller.register(self.results_incoming, zmq.POLLIN) # These are managers which we should examine in an iteration # for scheduling a job (or maybe any other attention?). # Anything altering the state of the manager should add it # onto this list. interesting_managers = set() while not self._kill_event.is_set(): self.socks = dict(poller.poll(timeout=poll_period)) # Listen for requests for work if self.task_outgoing in self.socks and self.socks[self.task_outgoing] == zmq.POLLIN: logger.debug("[MAIN] starting task_outgoing section") message = self.task_outgoing.recv_multipart() manager = message[0] if manager not in self._ready_manager_queue: reg_flag = False try: msg = json.loads(message[1].decode('utf-8')) reg_flag = True except Exception: logger.warning("[MAIN] Got a non-json registration message from manager:{}".format( manager)) logger.debug("[MAIN] Message :\n{}\n".format(message)) # By default we set up to ignore bad nodes/registration messages. self._ready_manager_queue[manager] = {'last': time.time(), 'reg_time': time.time(), 'free_capacity': {'total_workers': 0}, 'max_worker_count': 0, 'active': True, 'tasks': collections.defaultdict(set), 'total_tasks': 0} if reg_flag is True: interesting_managers.add(manager) logger.info("[MAIN] Adding manager: {} to ready queue".format(manager)) self._ready_manager_queue[manager].update(msg) logger.info("[MAIN] Registration info for manager {}: {}".format(manager, msg)) if (msg['python_v'].rsplit(".", 1)[0] != self.current_platform['python_v'].rsplit(".", 1)[0] or msg['parsl_v'] != self.current_platform['parsl_v']): logger.warn("[MAIN] Manager {} has incompatible version info with the interchange".format(manager)) if self.suppress_failure is False: logger.debug("Setting kill event") self._kill_event.set() e = ManagerLost(manager) result_package = {'task_id': -1, 'exception': self.serializer.serialize(e)} pkl_package = pickle.dumps(result_package) self.results_outgoing.send(pkl_package) logger.warning("[MAIN] Sent failure reports, unregistering manager") else: logger.debug("[MAIN] Suppressing shutdown due to version incompatibility") else: # Registration has failed. if self.suppress_failure is False: logger.debug("Setting kill event for bad manager") self._kill_event.set() e = BadRegistration(manager, critical=True) result_package = {'task_id': -1, 'exception': self.serializer.serialize(e)} pkl_package = pickle.dumps(result_package) self.results_outgoing.send(pkl_package) else: logger.debug("[MAIN] Suppressing bad registration from manager:{}".format( manager)) else: self._ready_manager_queue[manager]['last'] = time.time() if message[1] == b'HEARTBEAT': logger.debug("[MAIN] Manager {} sends heartbeat".format(manager)) self.task_outgoing.send_multipart([manager, b'', PKL_HEARTBEAT_CODE]) else: manager_adv = pickle.loads(message[1]) logger.debug("[MAIN] Manager {} requested {}".format(manager, manager_adv)) self._ready_manager_queue[manager]['free_capacity'].update(manager_adv) self._ready_manager_queue[manager]['free_capacity']['total_workers'] = sum(manager_adv.values()) interesting_managers.add(manager) # If we had received any requests, check if there are tasks that could be passed logger.debug("[MAIN] Managers count (total/interesting): {}/{}".format( len(self._ready_manager_queue), len(interesting_managers))) task_dispatch, dispatched_task = naive_interchange_task_dispatch(interesting_managers, self.pending_task_queue, self._ready_manager_queue, scheduler_mode=self.config.scheduler_mode) self.total_pending_task_count -= dispatched_task for manager in task_dispatch: tasks = task_dispatch[manager] if tasks: logger.info("[MAIN] Sending task message {} to manager {}".format(tasks, manager)) self.task_outgoing.send_multipart([manager, b'', pickle.dumps(tasks)]) # Receive any results and forward to client if self.results_incoming in self.socks and self.socks[self.results_incoming] == zmq.POLLIN: logger.debug("[MAIN] entering results_incoming section") manager, *b_messages = self.results_incoming.recv_multipart() if manager not in self._ready_manager_queue: logger.warning("[MAIN] Received a result from a un-registered manager: {}".format(manager)) else: logger.info("[MAIN] Got {} result items in batch".format(len(b_messages))) for b_message in b_messages: r = pickle.loads(b_message) # logger.debug("[MAIN] Received result for task {} from {}".format(r['task_id'], manager)) task_type = self.containers[r['task_id'].split(';')[1]] self._ready_manager_queue[manager]['tasks'][task_type].remove(r['task_id']) self._ready_manager_queue[manager]['total_tasks'] -= len(b_messages) self.results_outgoing.send_multipart(b_messages) logger.debug("[MAIN] Current tasks: {}".format(self._ready_manager_queue[manager]['tasks'])) logger.debug("[MAIN] leaving results_incoming section") # logger.debug("[MAIN] entering bad_managers section") bad_managers = [manager for manager in self._ready_manager_queue if time.time() - self._ready_manager_queue[manager]['last'] > self.heartbeat_threshold] for manager in bad_managers: logger.debug("[MAIN] Last: {} Current: {}".format(self._ready_manager_queue[manager]['last'], time.time())) logger.warning("[MAIN] Too many heartbeats missed for manager {}".format(manager)) e = ManagerLost(manager) for task_type in self._ready_manager_queue[manager]['tasks']: for tid in self._ready_manager_queue[manager]['tasks'][task_type]: result_package = {'task_id': tid, 'exception': self.serializer.serialize(e)} pkl_package = pickle.dumps(result_package) self.results_outgoing.send(pkl_package) logger.warning("[MAIN] Sent failure reports, unregistering manager") self._ready_manager_queue.pop(manager, 'None') if manager in interesting_managers: interesting_managers.remove(manager) logger.debug("[MAIN] ending one main loop iteration") if self._status_request.is_set(): logger.info("status request response") result_package = self.get_status_report() pkl_package = pickle.dumps(result_package) self.results_outgoing.send(pkl_package) logger.info("[MAIN] Sent info response") self._status_request.clear() delta = time.time() - start logger.info("Processed {} tasks in {} seconds".format(count, delta)) logger.warning("Exiting") def get_status_report(self): """ Get utilization numbers """ total_cores = 0 total_mem = 0 core_hrs = 0 active_managers = 0 free_capacity = 0 outstanding_tasks = self.get_total_tasks_outstanding() pending_tasks = self.total_pending_task_count num_managers = len(self._ready_manager_queue) live_workers = self.get_total_live_workers() for manager in self._ready_manager_queue: total_cores += self._ready_manager_queue[manager]['cores'] total_mem += self._ready_manager_queue[manager]['mem'] active_dur = abs(time.time() - self._ready_manager_queue[manager]['reg_time']) core_hrs += (active_dur * total_cores) / 3600 if self._ready_manager_queue[manager]['active']: active_managers += 1 free_capacity += self._ready_manager_queue[manager]['free_capacity']['total_workers'] result_package = {'task_id': -2, 'info': {'total_cores': total_cores, 'total_mem' : total_mem, 'new_core_hrs': core_hrs - self.last_core_hr_counter, 'total_core_hrs': round(core_hrs, 2), 'managers': num_managers, 'active_managers': active_managers, 'total_workers': live_workers, 'idle_workers': free_capacity, 'pending_tasks': pending_tasks, 'outstanding_tasks': outstanding_tasks, 'worker_mode': self.config.worker_mode, 'scheduler_mode': self.config.scheduler_mode, 'scaling_enabled': self.config.scaling_enabled, 'mem_per_worker': self.config.mem_per_worker, 'cores_per_worker': self.config.cores_per_worker, 'prefetch_capacity': self.config.prefetch_capacity, 'max_blocks': self.config.provider.max_blocks, 'min_blocks': self.config.provider.min_blocks, 'max_workers_per_node': self.config.max_workers_per_node, 'nodes_per_block': self.config.provider.nodes_per_block }} self.last_core_hr_counter = core_hrs return result_package def scale_out(self, blocks=1, task_type=None): """Scales out the number of blocks by "blocks" Raises: NotImplementedError """ r = [] for i in range(blocks): if self.config.provider: self._block_counter += 1 external_block_id = str(self._block_counter) if not task_type and self.config.scheduler_mode == 'hard': launch_cmd = self.launch_cmd.format(block_id=external_block_id, worker_type='RAW') else: launch_cmd = self.launch_cmd.format(block_id=external_block_id, worker_type=task_type) if not task_type: internal_block = self.config.provider.submit(launch_cmd, 1) else: internal_block = self.config.provider.submit(launch_cmd, 1, task_type) logger.debug("Launched block {}->{}".format(external_block_id, internal_block)) if not internal_block: raise(ScalingFailed(self.provider.label, "Attempts to provision nodes via provider has failed")) self.blocks[external_block_id] = internal_block self.block_id_map[internal_block] = external_block_id else: logger.error("No execution provider available") r = None return r def scale_in(self, blocks=None, block_ids=[], task_type=None): """Scale in the number of active blocks by specified amount. Parameters ---------- blocks : int # of blocks to terminate block_ids : [str.. ] List of external block ids to terminate """ if task_type: logger.info("Scaling in blocks of specific task type {}. Let the provider decide which to kill".format(task_type)) if self.config.scaling_enabled and self.config.provider: to_kill, r = self.config.provider.cancel(blocks, task_type) logger.info("Get the killed blocks: {}, and status: {}".format(to_kill, r)) for job in to_kill: logger.info("[scale_in] Getting the block_id map {} for job {}".format(self.block_id_map, job)) block_id = self.block_id_map[job] logger.info("[scale_in] Holding block {}".format(block_id)) self._hold_block(block_id) self.blocks.pop(block_id) return r if block_ids: block_ids_to_kill = block_ids else: block_ids_to_kill = list(self.blocks.keys())[:blocks] # Try a polite terminate # TODO : Missing logic to hold blocks for block_id in block_ids_to_kill: self._hold_block(block_id) # Now kill via provider to_kill = [self.blocks.pop(bid) for bid in block_ids_to_kill] if self.config.scaling_enabled and self.config.provider: r = self.config.provider.cancel(to_kill) return r def provider_status(self): """ Get status of all blocks from the provider """ status = [] if self.config.provider: logger.debug("[MAIN] Getting the status of {} blocks.".format(list(self.blocks.values()))) status = self.config.provider.status(list(self.blocks.values())) logger.debug("[MAIN] The status is {}".format(status)) return status
class EndpointInterchange: """Interchange is a task orchestrator for distributed systems. 1. Asynchronously queue large volume of tasks (>100K) 2. Allow for workers to join and leave the union 3. Detect workers that have failed using heartbeats 4. Service single and batch requests from workers 5. Be aware of requests worker resource capacity, eg. schedule only jobs that fit into walltime. TODO: We most likely need a PUB channel to send out global commandzs, like shutdown """ def __init__( self, config, client_address="127.0.0.1", interchange_address="127.0.0.1", client_ports: Tuple[int, int, int] = (50055, 50056, 50057), launch_cmd=None, logdir=".", endpoint_id=None, keys_dir=".curve", suppress_failure=True, endpoint_dir=".", endpoint_name="default", reg_info=None, funcx_client_options=None, results_ack_handler=None, ): """ Parameters ---------- config : funcx.Config object Funcx config object that describes how compute should be provisioned client_address : str The ip address at which the parsl client can be reached. Default: "127.0.0.1" interchange_address : str The ip address at which the workers will be able to reach the Interchange. Default: "127.0.0.1" client_ports : Tuple[int, int, int] The ports at which the client can be reached launch_cmd : str TODO : update logdir : str Parsl log directory paths. Logs and temp files go here. Default: '.' keys_dir : str Directory from where keys used for communicating with the funcX service (forwarders) are stored endpoint_id : str Identity string that identifies the endpoint to the broker suppress_failure : Bool When set to True, the interchange will attempt to suppress failures. Default: False endpoint_dir : str Endpoint directory path to store registration info in endpoint_name : str Name of endpoint reg_info : Dict Registration info from initial registration on endpoint start, if it succeeded funcx_client_options : Dict FuncXClient initialization options """ self.logdir = logdir log.info( "Initializing EndpointInterchange process with Endpoint ID: {}". format(endpoint_id)) self.config = config log.info(f"Got config: {config}") self.client_address = client_address self.interchange_address = interchange_address self.client_ports = client_ports self.suppress_failure = suppress_failure self.endpoint_dir = endpoint_dir self.endpoint_name = endpoint_name if funcx_client_options is None: funcx_client_options = {} self.funcx_client = FuncXClient(**funcx_client_options) self.initial_registration_complete = False if reg_info: self.initial_registration_complete = True self.apply_reg_info(reg_info) self.heartbeat_period = self.config.heartbeat_period self.heartbeat_threshold = self.config.heartbeat_threshold # initalize the last heartbeat time to start the loop self.last_heartbeat = time.time() self.keys_dir = keys_dir self.serializer = FuncXSerializer() self.pending_task_queue = Queue() self.containers = {} self.total_pending_task_count = 0 self._quiesce_event = threading.Event() self._kill_event = threading.Event() self.results_ack_handler = results_ack_handler log.info(f"Interchange address is {self.interchange_address}") self.endpoint_id = endpoint_id self.current_platform = { "parsl_v": PARSL_VERSION, "python_v": "{}.{}.{}".format(sys.version_info.major, sys.version_info.minor, sys.version_info.micro), "libzmq_v": zmq.zmq_version(), "pyzmq_v": zmq.pyzmq_version(), "os": platform.system(), "hname": platform.node(), "funcx_sdk_version": funcx_sdk_version, "funcx_endpoint_version": funcx_endpoint_version, "registration": self.endpoint_id, "dir": os.getcwd(), } log.info(f"Platform info: {self.current_platform}") try: self.load_config() except Exception: log.exception("Caught exception") raise self.tasks = set() self.task_status_deltas = {} self._test_start = False def load_config(self): """Load the config""" log.info("Loading endpoint local config") self.results_passthrough = mpQueue() self.executors = {} for executor in self.config.executors: log.info(f"Initializing executor: {executor.label}") executor.funcx_service_address = self.config.funcx_service_address if not executor.endpoint_id: executor.endpoint_id = self.endpoint_id else: if not executor.endpoint_id == self.endpoint_id: raise Exception("InconsistentEndpointId") self.executors[executor.label] = executor if executor.run_dir is None: executor.run_dir = self.logdir def start_executors(self): log.info("Starting Executors") for executor in self.config.executors: if hasattr(executor, "passthrough") and executor.passthrough is True: executor.start(results_passthrough=self.results_passthrough) def apply_reg_info(self, reg_info): self.client_address = reg_info["public_ip"] self.client_ports = ( reg_info["tasks_port"], reg_info["results_port"], reg_info["commands_port"], ) def register_endpoint(self): reg_info = register_endpoint(self.funcx_client, self.endpoint_id, self.endpoint_dir, self.endpoint_name) self.apply_reg_info(reg_info) return reg_info def migrate_tasks_to_internal(self, quiesce_event): """Pull tasks from the incoming tasks 0mq pipe onto the internal pending task queue Parameters: ----------- quiesce_event : threading.Event Event to let the thread know when it is time to die. """ log.info("[TASK_PULL_THREAD] Starting") try: self._task_puller_loop(quiesce_event) except Exception: log.exception("[TASK_PULL_THREAD] Unhandled exception") finally: quiesce_event.set() self.task_incoming.close() log.info("[TASK_PULL_THREAD] Thread loop exiting") def _task_puller_loop(self, quiesce_event): task_counter = 0 # Create the incoming queue in the thread to keep # zmq.context in the same thread. zmq.context is not thread-safe self.task_incoming = TaskQueue( self.client_address, port=self.client_ports[0], identity=self.endpoint_id, mode="client", set_hwm=True, keys_dir=self.keys_dir, RCVTIMEO=1000, linger=0, ) self.task_incoming.put("forwarder", pickle.dumps(self.current_platform)) log.info( f"Task incoming on tcp://{self.client_address}:{self.client_ports[0]}" ) self.last_heartbeat = time.time() while not quiesce_event.is_set(): try: if int(time.time() - self.last_heartbeat) > self.heartbeat_threshold: log.critical( "[TASK_PULL_THREAD] Missed too many heartbeats. " "Setting quiesce event.") quiesce_event.set() break try: # TODO : Check the kwarg options for get raw_msg = self.task_incoming.get()[0] self.last_heartbeat = time.time() except zmq.Again: # We just timed out while attempting to receive log.debug( "[TASK_PULL_THREAD] {} tasks in internal queue".format( self.total_pending_task_count)) continue except Exception: log.exception( "[TASK_PULL_THREAD] Unknown exception while waiting for tasks" ) # YADU: TODO We need to do the routing here try: msg = Message.unpack(raw_msg) except Exception: log.exception( "[TASK_PULL_THREAD] Failed to unpack message from forwarder" ) pass if msg == "STOP": self._kill_event.set() quiesce_event.set() break elif isinstance(msg, Heartbeat): log.info( "[TASK_PULL_THREAD] Got heartbeat from funcx-forwarder" ) elif isinstance(msg, Task): log.info(f"[TASK_PULL_THREAD] Received task:{msg.task_id}") self.pending_task_queue.put(msg) self.total_pending_task_count += 1 self.task_status_deltas[ msg.task_id] = TaskStatusCode.WAITING_FOR_NODES task_counter += 1 log.debug( "[TASK_PULL_THREAD] Task counter:%s Pending Tasks: %s", task_counter, self.total_pending_task_count, ) elif isinstance(msg, ResultsAck): self.results_ack_handler.ack(msg.task_id) else: log.warning( f"[TASK_PULL_THREAD] Unknown message type received: {msg}" ) except Exception: log.exception( "[TASK_PULL_THREAD] Something really bad happened") continue def get_container(self, container_uuid): """Get the container image location if it is not known to the interchange""" if container_uuid not in self.containers: if container_uuid == "RAW" or not container_uuid: self.containers[container_uuid] = "RAW" else: try: container = self.funcx_client.get_container( container_uuid, self.config.container_type) except Exception: log.exception( "[FETCH_CONTAINER] Unable to resolve container location" ) self.containers[container_uuid] = "RAW" else: log.info( f"[FETCH_CONTAINER] Got container info: {container}") self.containers[container_uuid] = container.get( "location", "RAW") return self.containers[container_uuid] def _command_server(self, quiesce_event): """Command server to run async command to the interchange We want to be able to receive the following not yet implemented/updated commands: - OutstandingCount - ListManagers (get outstanding broken down by manager) - HoldWorker - Shutdown """ log.debug("[COMMAND] Command Server Starting") try: self._command_server_loop(quiesce_event) except Exception: log.exception("[COMMAND] Unhandled exception") finally: quiesce_event.set() self.command_channel.close() log.info("[COMMAND] Thread loop exiting") def _command_server_loop(self, quiesce_event): self.command_channel = TaskQueue( self.client_address, port=self.client_ports[2], identity=self.endpoint_id, mode="client", RCVTIMEO=1000, # in milliseconds keys_dir=self.keys_dir, set_hwm=True, linger=0, ) # TODO :Register all channels with the authentication string. self.command_channel.put( "forwarder", pickle.dumps({"registration": self.endpoint_id})) while not quiesce_event.is_set(): try: # Wait for 1000 ms buffer = self.command_channel.get(timeout=1000) log.debug(f"[COMMAND] Received command request {buffer}") command = Message.unpack(buffer) if command.type not in COMMAND_TYPES: log.error( "Received incorrect message type on command channel") self.command_channel.put(bytes()) continue if command.type is MessageType.HEARTBEAT_REQ: log.info( "[COMMAND] Received synchonous HEARTBEAT_REQ from hub") log.info( f"[COMMAND] Replying with Heartbeat({self.endpoint_id})" ) reply = Heartbeat(self.endpoint_id) log.debug(f"[COMMAND] Reply: {reply}") self.command_channel.put(reply.pack()) except zmq.Again: # log.debug("[COMMAND] is alive") continue def quiesce(self): """Temporarily stop everything on the interchange in order to reach a consistent state before attempting to start again. This must be called on the main thread """ log.info( "Interchange Quiesce in progress (stopping and joining all threads)" ) self._quiesce_event.set() self._task_puller_thread.join() self._command_thread.join() log.info("Saving unacked results to disk") try: self.results_ack_handler.persist() except Exception: log.exception("Caught exception while saving unacked results") log.warning( "Interchange will continue without saving unacked results") # this must be called last to ensure the next interchange run will occur self._quiesce_event.clear() def stop(self): """Prepare the interchange for shutdown""" log.info("Shutting down EndpointInterchange") # TODO: shut down executors gracefully # kill_event must be set before quiesce_event because we need to guarantee that # once the quiesce is complete, the interchange will not try to start again self._kill_event.set() self._quiesce_event.set() def handle_sigterm(self, sig_num, curr_stack_frame): log.warning( "Received SIGTERM, attempting to save unacked results to disk") try: self.results_ack_handler.persist() except Exception: log.exception("Caught exception while saving unacked results") else: log.info("Unacked results successfully saved to disk") sys.exit(1) def start(self): """Start the Interchange""" log.info("Starting EndpointInterchange") signal.signal(signal.SIGTERM, self.handle_sigterm) self._quiesce_event.clear() self._kill_event.clear() # NOTE: currently we only start the executors once because # the current behavior is to keep them running decoupled while # the endpoint is waiting for reconnection self.start_executors() while not self._kill_event.is_set(): self._start_threads_and_main() self.quiesce() # this check is solely for testing to force this loop to only run once if self._test_start: break log.info("EndpointInterchange shutdown complete.") def _start_threads_and_main(self): # re-register on every loop start if not self.initial_registration_complete: # Register the endpoint log.info("Running endpoint registration retry loop") reg_info = retry_call(self.register_endpoint, delay=10, max_delay=300, backoff=1.2) log.info("Endpoint registered with UUID: {}".format( reg_info["endpoint_id"])) self.initial_registration_complete = False log.info( "Attempting connection to client at {} on ports: {},{},{}".format( self.client_address, self.client_ports[0], self.client_ports[1], self.client_ports[2], )) self._task_puller_thread = threading.Thread( target=self.migrate_tasks_to_internal, args=(self._quiesce_event, )) self._task_puller_thread.start() self._command_thread = threading.Thread(target=self._command_server, args=(self._quiesce_event, )) self._command_thread.start() try: self._main_loop() except Exception: log.exception("[MAIN] Unhandled exception") finally: self.results_outgoing.close() log.info("[MAIN] Thread loop exiting") def _main_loop(self): self.results_outgoing = TaskQueue( self.client_address, port=self.client_ports[1], identity=self.endpoint_id, mode="client", keys_dir=self.keys_dir, # Fail immediately if results cannot be sent back SNDTIMEO=0, set_hwm=True, linger=0, ) self.results_outgoing.put( "forwarder", pickle.dumps({"registration": self.endpoint_id})) # TODO: this resend must happen after any endpoint re-registration to # ensure there are not unacked results left resend_results_messages = self.results_ack_handler.get_unacked_results_list( ) if len(resend_results_messages) > 0: log.info( "[MAIN] Resending %s previously unacked results", len(resend_results_messages), ) # TODO: this should be a multipart send rather than a loop for results in resend_results_messages: self.results_outgoing.put("forwarder", results) executor = list(self.executors.values())[0] last = time.time() while not self._quiesce_event.is_set(): if last + self.heartbeat_threshold < time.time(): log.debug("[MAIN] alive") last = time.time() try: # Adding results heartbeat to essentially force a TCP keepalive # without meddling with OS TCP keepalive defaults self.results_outgoing.put("forwarder", b"HEARTBEAT") except Exception: log.exception( "[MAIN] Sending heartbeat to the forwarder over the results " "channel has failed") raise self.results_ack_handler.check_ack_counts() try: task = self.pending_task_queue.get(block=True, timeout=0.01) executor.submit_raw(task.pack()) except queue.Empty: pass except Exception: log.exception( "[MAIN] Unhandled issue while waiting for pending tasks") try: results = self.results_passthrough.get(False, 0.01) task_id = results["task_id"] if task_id: self.results_ack_handler.put(task_id, results["message"]) log.info(f"Passing result to forwarder for task {task_id}") # results will be a pickled dict with task_id, container_id, # and results/exception self.results_outgoing.put("forwarder", results["message"]) except queue.Empty: pass except Exception: log.exception( "[MAIN] Something broke while forwarding results from executor " "to forwarder queues") continue def get_status_report(self): """Get utilization numbers""" total_cores = 0 total_mem = 0 core_hrs = 0 active_managers = 0 free_capacity = 0 outstanding_tasks = self.get_total_tasks_outstanding() pending_tasks = self.total_pending_task_count num_managers = len(self._ready_manager_queue) live_workers = self.get_total_live_workers() for manager in self._ready_manager_queue: total_cores += self._ready_manager_queue[manager]["cores"] total_mem += self._ready_manager_queue[manager]["mem"] active_dur = abs(time.time() - self._ready_manager_queue[manager]["reg_time"]) core_hrs += (active_dur * total_cores) / 3600 if self._ready_manager_queue[manager]["active"]: active_managers += 1 free_capacity += self._ready_manager_queue[manager][ "free_capacity"]["total_workers"] result_package = { "task_id": -2, "info": { "total_cores": total_cores, "total_mem": total_mem, "new_core_hrs": core_hrs - self.last_core_hr_counter, "total_core_hrs": round(core_hrs, 2), "managers": num_managers, "active_managers": active_managers, "total_workers": live_workers, "idle_workers": free_capacity, "pending_tasks": pending_tasks, "outstanding_tasks": outstanding_tasks, "worker_mode": self.config.worker_mode, "scheduler_mode": self.config.scheduler_mode, "scaling_enabled": self.config.scaling_enabled, "mem_per_worker": self.config.mem_per_worker, "cores_per_worker": self.config.cores_per_worker, "prefetch_capacity": self.config.prefetch_capacity, "max_blocks": self.config.provider.max_blocks, "min_blocks": self.config.provider.min_blocks, "max_workers_per_node": self.config.max_workers_per_node, "nodes_per_block": self.config.provider.nodes_per_block, }, } self.last_core_hr_counter = core_hrs return result_package def scale_out(self, blocks=1, task_type=None): """Scales out the number of blocks by "blocks" Raises: NotImplementedError """ r = [] for _i in range(blocks): if self.config.provider: self._block_counter += 1 external_block_id = str(self._block_counter) if not task_type and self.config.scheduler_mode == "hard": launch_cmd = self.launch_cmd.format( block_id=external_block_id, worker_type="RAW") else: launch_cmd = self.launch_cmd.format( block_id=external_block_id, worker_type=task_type) if not task_type: internal_block = self.config.provider.submit(launch_cmd, 1) else: internal_block = self.config.provider.submit( launch_cmd, 1, task_type) log.debug( f"Launched block {external_block_id}->{internal_block}") if not internal_block: raise (ScalingFailed( self.provider.label, "Attempts to provision nodes via provider has failed", )) self.blocks[external_block_id] = internal_block self.block_id_map[internal_block] = external_block_id else: log.error("No execution provider available") r = None return r def scale_in(self, blocks=None, block_ids=None, task_type=None): """Scale in the number of active blocks by specified amount. Parameters ---------- blocks : int # of blocks to terminate block_ids : [str.. ] List of external block ids to terminate """ if block_ids is None: block_ids = [] if task_type: log.info( "Scaling in blocks of specific task type %s. " "Let the provider decide which to kill", task_type, ) if self.config.scaling_enabled and self.config.provider: to_kill, r = self.config.provider.cancel(blocks, task_type) log.info(f"Get the killed blocks: {to_kill}, and status: {r}") for job in to_kill: log.info( "[scale_in] Getting the block_id map {} for job {}". format(self.block_id_map, job)) block_id = self.block_id_map[job] log.info(f"[scale_in] Holding block {block_id}") self._hold_block(block_id) self.blocks.pop(block_id) return r if block_ids: block_ids_to_kill = block_ids else: block_ids_to_kill = list(self.blocks.keys())[:blocks] # Try a polite terminate # TODO : Missing logic to hold blocks for block_id in block_ids_to_kill: self._hold_block(block_id) # Now kill via provider to_kill = [self.blocks.pop(bid) for bid in block_ids_to_kill] if self.config.scaling_enabled and self.config.provider: r = self.config.provider.cancel(to_kill) return r def provider_status(self): """Get status of all blocks from the provider""" status = [] if self.config.provider: log.debug("[MAIN] Getting the status of {} blocks.".format( list(self.blocks.values()))) status = self.config.provider.status(list(self.blocks.values())) log.debug(f"[MAIN] The status is {status}") return status