def __init__(self, container): self.container = container # Define the callables that can be added to Container public API self.container_api = [self.spawn_process, self.terminate_process] # Add the public callables to Container for call in self.container_api: setattr(self.container, call.__name__, call) self.proc_id_pool = IDPool() # Temporary registry of running processes self.procs_by_name = {} self.procs = {} # mapping of greenlets we spawn to service_instances for error handling self._spawned_proc_to_process = {} # The pyon worker process supervisor self.proc_sup = IonProcessSupervisor( heartbeat_secs=CFG.cc.timeout.heartbeat, failure_notify_callback=self._spawned_proc_failed)
def __init__(self, container): self.container = container # Define the callables that can be added to Container public API self.container_api = [self.spawn_process] # Add the public callables to Container for call in self.container_api: setattr(self.container, call.__name__, call) self.proc_id_pool = IDPool() # Temporary registry of running processes self.procs_by_name = {} self.procs = {} # The pyon worker process supervisor self.proc_sup = IonProcessSupervisor(heartbeat_secs=CFG.cc.timeout.heartbeat)
def __init__(self, container): self.container = container # Define the callables that can be added to Container public API self.container_api = [self.spawn_process, self.terminate_process] # Add the public callables to Container for call in self.container_api: setattr(self.container, call.__name__, call) self.proc_id_pool = IDPool() # Temporary registry of running processes self.procs_by_name = {} self.procs = {} # mapping of greenlets we spawn to service_instances for error handling self._spawned_proc_to_process = {} # The pyon worker process supervisor self.proc_sup = IonProcessSupervisor(heartbeat_secs=CFG.cc.timeout.heartbeat, failure_notify_callback=self._spawned_proc_failed)
class ProcManager(object): def __init__(self, container): self.container = container # Define the callables that can be added to Container public API self.container_api = [self.spawn_process] # Add the public callables to Container for call in self.container_api: setattr(self.container, call.__name__, call) self.proc_id_pool = IDPool() # Temporary registry of running processes self.procs_by_name = {} self.procs = {} # The pyon worker process supervisor self.proc_sup = IonProcessSupervisor(heartbeat_secs=CFG.cc.timeout.heartbeat) def start(self): log.debug("ProcManager starting ...") self.proc_sup.start() log.debug("ProcManager started, OK.") def stop(self): log.debug("ProcManager stopping ...") # Call quit on procs to give them ability to clean up # TODO: This can be done concurrently for procid, proc in self.procs.iteritems(): try: # These are service processes with full life cycle proc.quit() except Exception: log.exception("Process %s quit failed" % procid) # TODO: Have a choice of shutdown behaviors for waiting on children, timeouts, etc self.proc_sup.shutdown(CFG.cc.timeout.shutdown) log.debug("ProcManager stopped, OK.") def spawn_process(self, name=None, module=None, cls=None, config=None, process_type=None): """ Spawn a process within the container. Processes can be of different type. """ # Generate a new process id # TODO: Ensure it is system-wide unique process_id = str(self.proc_id_pool.get_id()) log.debug("ProcManager.spawn_process(name=%s, module.cls=%s.%s, proc_type=%s) as pid=%s", name, module, cls, process_type, process_id) if config is None: config = DictModifier(CFG) log.debug("spawn_process() pid=%s config=%s", process_id, config) # PROCESS TYPE. # One of: service, stream_process, agent, simple, immediate service_cls = named_any("%s.%s" % (module, cls)) process_type = process_type or getattr(service_cls, "process_type", "service") service_instance = None try: # spawn service by type if process_type == "service": service_instance = self._spawn_service_process(process_id, name, module, cls, config) elif process_type == "stream_process": service_instance = self._spawn_stream_process(process_id, name, module, cls, config) elif process_type == "agent": service_instance = self._spawn_agent_process(process_id, name, module, cls, config) elif process_type == "immediate": service_instance = self._spawn_immediate_process(process_id, name, module, cls, config) elif process_type == "simple": service_instance = self._spawn_simple_process(process_id, name, module, cls, config) else: raise Exception("Unknown process type: %s" % process_type) service_instance._proc_type = process_type self._register_process(service_instance, name) service_instance.errcause = "OK" log.info("AppManager.spawn_process: %s.%s -> pid=%s OK" % (module, cls, process_id)) return process_id except Exception: errcause = service_instance.errcause if service_instance else "instantiating service" log.exception("Error spawning %s %s process (process_id: %s): %s" % (name, process_type, process_id, errcause)) raise def _spawn_service_process(self, process_id, name, module, cls, config): """ Spawn a process acting as a service worker. Attach to service queue with service definition, attach to service pid """ service_instance = self._create_service_instance(process_id, name, module, cls, config) self._service_init(service_instance) listen_name = config.get("process", {}).get("listen_name", service_instance.name) log.debug("Service Process (%s) listen_name: %s", name, listen_name) self._set_service_endpoint(service_instance, listen_name) self._set_service_endpoint(service_instance, service_instance.id) self._service_start(service_instance) return service_instance def _spawn_stream_process(self, process_id, name, module, cls, config): """ Spawn a process acting as a data stream process. Attach to subscription queue with process function. """ service_instance = self._create_service_instance(process_id, name, module, cls, config) self._service_init(service_instance) listen_name = config.get("process", {}).get("listen_name", name) # Throws an exception if no listen name is given! self._set_subscription_endpoint(service_instance, listen_name) # Spawn the greenlet self._service_start(service_instance) publish_streams = config.get("process", {}).get("publish_streams", None) # Add the publishers if any... self._set_publisher_endpoints(service_instance, publish_streams) return service_instance def _spawn_agent_process(self, process_id, name, module, cls, config): """ Spawn a process acting as agent process. Attach to service pid. """ service_instance = self._create_service_instance(process_id, name, module, cls, config) self._service_init(service_instance) self._set_service_endpoint(service_instance, service_instance.id) self._service_start(service_instance) return service_instance def _spawn_simple_process(self, process_id, name, module, cls, config): """ Spawn a process acting as simple process. No attachments. """ service_instance = self._create_service_instance(process_id, name, module, cls, config) self._service_init(service_instance) self._service_start(service_instance) return service_instance def _spawn_immediate_process(self, process_id, name, module, cls, config): """ Spawn a process acting as immediate one off process. No attachments. """ service_instance = self._create_service_instance(process_id, name, module, cls, config) self._service_init(service_instance) self._service_start(service_instance) return service_instance def _create_service_instance(self, process_id, name, module, cls, config): # SERVICE INSTANCE. service_instance = for_name(module, cls) assert isinstance(service_instance, BaseService), "Instantiated service not a BaseService %r" % service_instance # Prepare service instance service_instance.errcause = "" service_instance.id = "%s.%s" % (self.container.id, process_id) service_instance.container = self.container service_instance.CFG = config service_instance._proc_name = name # start service dependencies (RPC clients) self._start_service_dependencies(service_instance) return service_instance def _start_service_dependencies(self, service_instance): service_instance.errcause = "setting service dependencies" log.debug("spawn_process dependencies: %s" % service_instance.dependencies) # TODO: Service dependency != process dependency for dependency in service_instance.dependencies: client = getattr(service_instance.clients, dependency) assert client, "Client for dependency not found: %s" % dependency # @TODO: should be in a start_client in RPCClient chain client.process = service_instance client.node = self.container.node def _service_init(self, service_instance): # Init process service_instance.errcause = "initializing service" service_instance.init() def _service_start(self, service_instance): # Start process # TODO: Check for timeout service_instance.errcause = "starting service" service_instance.start() def _set_service_endpoint(self, service_instance, listen_name): service_instance.errcause = "setting process service endpoint" # Service RPC endpoint rsvc = ProcessRPCServer(node=self.container.node, name=listen_name, service=service_instance, process=service_instance) # Start an ION process with the right kind of endpoint factory self.proc_sup.spawn((CFG.cc.proctype or 'green', None), listener=rsvc, name=listen_name) if not rsvc.get_ready_event().wait(timeout=10): raise exception.ContainerError('_set_service_endpoint for listen_name: %s did not report ok' % listen_name) log.debug("Process %s service listener ready: %s", service_instance.id, listen_name) def _set_subscription_endpoint(self, service_instance, listen_name): service_instance.errcause = "setting process subscription endpoint" service_instance.stream_subscriber_registrar = StreamSubscriberRegistrar(process=service_instance, node=self.container.node) sub = service_instance.stream_subscriber_registrar.create_subscriber(exchange_name=listen_name,callback=lambda m,h: service_instance.process(m)) self.proc_sup.spawn((CFG.cc.proctype or 'green', None), listener=sub, name=listen_name) if not sub.get_ready_event().wait(timeout=10): raise exception.ContainerError('_set_subscription_endpoint for listen_name: %s did not report ok' % listen_name) log.debug("Process %s stream listener ready: %s", service_instance.id, listen_name) def _set_publisher_endpoints(self, service_instance, publisher_streams=None): service_instance.stream_publisher_registrar = StreamPublisherRegistrar(process=service_instance, node=self.container.node) publisher_streams = publisher_streams or {} for name, stream_id in publisher_streams.iteritems(): # problem is here pub = service_instance.stream_publisher_registrar.create_publisher(stream_id) setattr(service_instance, name, pub) def _register_process(self, service_instance, name): # Add to local process dict self.procs_by_name[name] = service_instance self.procs[service_instance.id] = service_instance # Add to directory service_instance.errcause = "registering" self.container.directory.register("/Containers/%s/Processes" % self.container.id, service_instance.id, name=name) def terminate_process(self, process_id): service_instance = self.procs.get(process_id, None) if not service_instance: return service_instance.quit() # find the proc lp = list(self.proc_sup.children) lps = [p for p in lp if p.listener._process == service_instance] assert len(lps) > 0 for p in lps: p.notify_stop() p.stop() del self.procs[process_id]
class ProcManager(object): def __init__(self, container): self.container = container # Define the callables that can be added to Container public API self.container_api = [self.spawn_process, self.terminate_process] # Add the public callables to Container for call in self.container_api: setattr(self.container, call.__name__, call) self.proc_id_pool = IDPool() # Temporary registry of running processes self.procs_by_name = {} self.procs = {} # mapping of greenlets we spawn to service_instances for error handling self._spawned_proc_to_process = {} # The pyon worker process supervisor self.proc_sup = IonProcessSupervisor(heartbeat_secs=CFG.cc.timeout.heartbeat, failure_notify_callback=self._spawned_proc_failed) def start(self): log.debug("ProcManager starting ...") self.proc_sup.start() log.debug("ProcManager started, OK.") def stop(self): log.debug("ProcManager stopping ...") # Call quit on procs to give them ability to clean up # TODO: This can be done concurrently while self.procs: try: procid = self.procs.keys()[0] # These are service processes with full life cycle #proc.quit() self.terminate_process(procid) except Exception: log.exception("Process %s quit failed" % procid) # TODO: Have a choice of shutdown behaviors for waiting on children, timeouts, etc self.proc_sup.shutdown(CFG.cc.timeout.shutdown) log.debug("ProcManager stopped, OK.") def spawn_process(self, name=None, module=None, cls=None, config=None): """ Spawn a process within the container. Processes can be of different type. """ # Generate a new process id # TODO: Ensure it is system-wide unique process_id = "%s.%s" % (self.container.id, self.proc_id_pool.get_id()) log.debug("ProcManager.spawn_process(name=%s, module.cls=%s.%s) as pid=%s", name, module, cls, process_id) if not config: # Use system CFG. It has the command line args in it config = DictModifier(CFG) else: # Use provided config. Must be dict or DotDict if not isinstance(config, DotDict): config = DotDict(config) config = DictModifier(CFG, config) if self.container.spawn_args: # Override config with spawn args dict_merge(config, self.container.spawn_args, inplace=True) log.debug("spawn_process() pid=%s config=%s", process_id, config) # PROCESS TYPE. Determines basic process context (messaging, service interface) # One of: service, stream_process, agent, simple, immediate service_cls = named_any("%s.%s" % (module, cls)) process_type = get_safe(config, "process.type") or getattr(service_cls, "process_type", "service") service_instance = None try: # spawn service by type if process_type == "service": service_instance = self._spawn_service_process(process_id, name, module, cls, config) elif process_type == "stream_process": service_instance = self._spawn_stream_process(process_id, name, module, cls, config) elif process_type == "agent": service_instance = self._spawn_agent_process(process_id, name, module, cls, config) elif process_type == "standalone": service_instance = self._spawn_standalone_process(process_id, name, module, cls, config) elif process_type == "immediate": service_instance = self._spawn_immediate_process(process_id, name, module, cls, config) elif process_type == "simple": service_instance = self._spawn_simple_process(process_id, name, module, cls, config) else: raise BadRequest("Unknown process type: %s" % process_type) service_instance._proc_type = process_type self._register_process(service_instance, name) service_instance.errcause = "OK" log.info("AppManager.spawn_process: %s.%s -> pid=%s OK" % (module, cls, process_id)) return service_instance.id except Exception: errcause = service_instance.errcause if service_instance else "instantiating service" log.exception("Error spawning %s %s process (process_id: %s): %s" % (name, process_type, process_id, errcause)) raise def _spawned_proc_failed(self, proc_sup, gproc): log.error("ProcManager._spawned_proc_failed: %s", gproc) # for now - don't worry about the mapping, if we get a failure, just kill the container. # leave the mapping in place for potential expansion later. # # look it up in mapping # if not gproc in self._spawned_proc_to_process: # log.warn("No record of gproc %s in our map (%s)", gproc, self._spawned_proc_to_process) # return # svc = self._spawned_proc_to_process.get(gproc, "Unknown") # # # make sure svc is in our list # if not svc in self.procs.values(): # log.warn("svc %s not found in procs list", svc) # return self.container.fail_fast("Container process (%s) failed: %s" % (svc, gproc.exception)) # ----------------------------------------------------------------- # PROCESS TYPE: service def _spawn_service_process(self, process_id, name, module, cls, config): """ Spawn a process acting as a service worker. Attach to service queue with service definition, attach to service pid """ service_instance = self._create_service_instance(process_id, name, module, cls, config) self._service_init(service_instance) listen_name = get_safe(config, "process.listen_name") or service_instance.name log.debug("Service Process (%s) listen_name: %s", name, listen_name) self._set_service_endpoint(service_instance, listen_name) self._set_service_endpoint(service_instance, service_instance.id) self._service_start(service_instance) # Directory registration self.container.directory.register_safe("/Services", listen_name, interface=service_instance.name) self.container.directory.register_safe("/Services/%s" % listen_name, service_instance.id) return service_instance # ----------------------------------------------------------------- # PROCESS TYPE: stream process def _spawn_stream_process(self, process_id, name, module, cls, config): """ Spawn a process acting as a data stream process. Attach to subscription queue with process function. """ service_instance = self._create_service_instance(process_id, name, module, cls, config) self._service_init(service_instance) listen_name = get_safe(config, "process.listen_name") or name # Throws an exception if no listen name is given! self._set_subscription_endpoint(service_instance, listen_name) # Add publishers if any... publish_streams = get_safe(config, "process.publish_streams") self._set_publisher_endpoints(service_instance, publish_streams) self._set_service_endpoint(service_instance, service_instance.id) # Start the service self._service_start(service_instance) return service_instance # ----------------------------------------------------------------- # PROCESS TYPE: agent def _spawn_agent_process(self, process_id, name, module, cls, config): """ Spawn a process acting as agent process. Attach to service pid. """ service_instance = self._create_service_instance(process_id, name, module, cls, config) if not isinstance(service_instance, ResourceAgent): raise ContainerConfigError("Agent process must extend ResourceAgent") self._service_init(service_instance) self._set_service_endpoint(service_instance, service_instance.id) self._service_start(service_instance) # Directory registration caps = service_instance.get_capabilities() self.container.directory.register("/Agents", service_instance.id, **dict(name=service_instance._proc_name, container=service_instance.container.id, resource_id=service_instance.resource_id, agent_id=service_instance.agent_id, def_id=service_instance.agent_def_id, capabilities=caps)) if not service_instance.resource_id: log.warn("Agent process id=%s does not define resource_id!!" % service_instance.id) return service_instance # ----------------------------------------------------------------- # PROCESS TYPE: standalone def _spawn_standalone_process(self, process_id, name, module, cls, config): """ Spawn a process acting as standalone process. Attach to service pid. """ service_instance = self._create_service_instance(process_id, name, module, cls, config) self._service_init(service_instance) self._set_service_endpoint(service_instance, service_instance.id) # Add publishers if any... publish_streams = get_safe(config, "process.publish_streams") self._set_publisher_endpoints(service_instance, publish_streams) self._service_start(service_instance) return service_instance # ----------------------------------------------------------------- # PROCESS TYPE: simple def _spawn_simple_process(self, process_id, name, module, cls, config): """ Spawn a process acting as simple process. No attachments. """ service_instance = self._create_service_instance(process_id, name, module, cls, config) self._service_init(service_instance) # Add publishers if any... publish_streams = get_safe(config, "process.publish_streams") self._set_publisher_endpoints(service_instance, publish_streams) self._service_start(service_instance) return service_instance # ----------------------------------------------------------------- # PROCESS TYPE: immediate def _spawn_immediate_process(self, process_id, name, module, cls, config): """ Spawn a process acting as immediate one off process. No attachments. """ service_instance = self._create_service_instance(process_id, name, module, cls, config) self._service_init(service_instance) self._service_start(service_instance) return service_instance def _create_service_instance(self, process_id, name, module, cls, config): # SERVICE INSTANCE. service_instance = for_name(module, cls) if not isinstance(service_instance, BaseService): raise ContainerConfigError("Instantiated service not a BaseService %r" % service_instance) # Prepare service instance service_instance.errcause = "" service_instance.id = process_id service_instance.container = self.container service_instance.CFG = config service_instance._proc_name = name # start service dependencies (RPC clients) self._start_service_dependencies(service_instance) return service_instance def _start_service_dependencies(self, service_instance): service_instance.errcause = "setting service dependencies" log.debug("spawn_process dependencies: %s" % service_instance.dependencies) # TODO: Service dependency != process dependency for dependency in service_instance.dependencies: client = getattr(service_instance.clients, dependency) assert client, "Client for dependency not found: %s" % dependency # @TODO: should be in a start_client in RPCClient chain client.process = service_instance client.node = self.container.node def _service_init(self, service_instance): # Init process service_instance.errcause = "initializing service" service_instance.init() def _service_start(self, service_instance): # Start process # TODO: Check for timeout service_instance.errcause = "starting service" service_instance.start() def _set_service_endpoint(self, service_instance, listen_name): service_instance.errcause = "setting process service endpoint" # Service RPC endpoint rsvc = ProcessRPCServer(node=self.container.node, from_name=listen_name, service=service_instance, process=service_instance) # Start an ION process with the right kind of endpoint factory proc = self.proc_sup.spawn((CFG.cc.proctype or 'green', None), listener=rsvc, name=listen_name, proc_name=service_instance._proc_name) self.proc_sup.ensure_ready(proc, "_set_service_endpoint for listen_name: %s" % listen_name) # map gproc to service_instance self._spawned_proc_to_process[proc.proc] = service_instance log.debug("Process %s service listener ready: %s", service_instance.id, listen_name) def _set_subscription_endpoint(self, service_instance, listen_name): service_instance.errcause = "setting process subscription endpoint" service_instance.stream_subscriber_registrar = StreamSubscriberRegistrar(process=service_instance, node=self.container.node) sub = service_instance.stream_subscriber_registrar.create_subscriber(exchange_name=listen_name,callback=lambda m,h: service_instance.process(m)) proc = self.proc_sup.spawn((CFG.cc.proctype or 'green', None), listener=sub, name=listen_name, proc_name=service_instance._proc_name) self.proc_sup.ensure_ready(proc, '_set_subscription_endpoint for listen_name: %s' % listen_name) # map gproc to service_instance self._spawned_proc_to_process[proc.proc] = service_instance log.debug("Process %s stream listener ready: %s", service_instance.id, listen_name) def _set_publisher_endpoints(self, service_instance, publisher_streams=None): service_instance.stream_publisher_registrar = StreamPublisherRegistrar(process=service_instance, node=self.container.node) publisher_streams = publisher_streams or {} for name, stream_id in publisher_streams.iteritems(): # problem is here pub = service_instance.stream_publisher_registrar.create_publisher(stream_id) setattr(service_instance, name, pub) def _register_process(self, service_instance, name): # Add to local process dict self.procs_by_name[name] = service_instance self.procs[service_instance.id] = service_instance # Add to directory service_instance.errcause = "registering" self.container.directory.register_safe("/Containers/%s/Processes" % self.container.id, service_instance.id, name=name) def terminate_process(self, process_id): service_instance = self.procs.get(process_id, None) if not service_instance: raise BadRequest("Cannot terminate. Process id='%s' unknown on container id='%s'" % ( process_id, self.container.id)) service_instance.quit() # find the proc lp = list(self.proc_sup.children) lps = [p for p in lp if p.listener._process == service_instance] for p in lps: p.notify_stop() p.stop() del self.procs[process_id] self.container.directory.unregister_safe("/Containers/%s/Processes" % self.container.id, service_instance.id) # Cleanup for specific process types if service_instance._proc_type == "service": listen_name = get_safe(service_instance.CFG, "process.listen_name", service_instance.name) self.container.directory.unregister_safe("/Services/%s" % listen_name, service_instance.id) remaining_workers = self.container.directory.find_entries("/Services/%s" % listen_name) if remaining_workers and len(remaining_workers) == 2: self.container.directory.unregister_safe("/Services", listen_name) elif service_instance._proc_type == "agent": self.container.directory.unregister_safe("/Agents", service_instance.id)
class ProcManager(object): def __init__(self, container): self.container = container # Define the callables that can be added to Container public API self.container_api = [self.spawn_process, self.terminate_process] # Add the public callables to Container for call in self.container_api: setattr(self.container, call.__name__, call) self.proc_id_pool = IDPool() # Temporary registry of running processes self.procs_by_name = {} self.procs = {} # mapping of greenlets we spawn to service_instances for error handling self._spawned_proc_to_process = {} # The pyon worker process supervisor self.proc_sup = IonProcessSupervisor( heartbeat_secs=CFG.cc.timeout.heartbeat, failure_notify_callback=self._spawned_proc_failed) def start(self): log.debug("ProcManager starting ...") self.proc_sup.start() log.debug("ProcManager started, OK.") def stop(self): log.debug("ProcManager stopping ...") # Call quit on procs to give them ability to clean up # @TODO terminate_process is not gl-safe # gls = map(lambda k: spawn(self.terminate_process, k), self.procs.keys()) # join(gls) map(self.terminate_process, self.procs.keys()) # TODO: Have a choice of shutdown behaviors for waiting on children, timeouts, etc self.proc_sup.shutdown(CFG.cc.timeout.shutdown) log.debug("ProcManager stopped, OK.") def spawn_process(self, name=None, module=None, cls=None, config=None, process_id=None): """ Spawn a process within the container. Processes can be of different type. """ if process_id and not is_valid_identifier(process_id, ws_sub='_'): raise BadRequest("Given process_id %s is not a valid identifier" % process_id) # Generate a new process id if not provided # TODO: Ensure it is system-wide unique process_id = process_id or "%s.%s" % (self.container.id, self.proc_id_pool.get_id()) log.debug( "ProcManager.spawn_process(name=%s, module.cls=%s.%s) as pid=%s", name, module, cls, process_id) if not config: # Use system CFG. It has the command line args in it config = DictModifier(CFG) else: # Use provided config. Must be dict or DotDict if not isinstance(config, DotDict): config = DotDict(config) config = DictModifier(CFG, config) if self.container.spawn_args: # Override config with spawn args dict_merge(config, self.container.spawn_args, inplace=True) #log.debug("spawn_process() pid=%s config=%s", process_id, config) # PROCESS TYPE. Determines basic process context (messaging, service interface) # One of: service, stream_process, agent, simple, immediate service_cls = named_any("%s.%s" % (module, cls)) process_type = get_safe(config, "process.type") or getattr( service_cls, "process_type", "service") service_instance = None try: # spawn service by type if process_type == "service": service_instance = self._spawn_service_process( process_id, name, module, cls, config) elif process_type == "stream_process": service_instance = self._spawn_stream_process( process_id, name, module, cls, config) elif process_type == "agent": service_instance = self._spawn_agent_process( process_id, name, module, cls, config) elif process_type == "standalone": service_instance = self._spawn_standalone_process( process_id, name, module, cls, config) elif process_type == "immediate": service_instance = self._spawn_immediate_process( process_id, name, module, cls, config) elif process_type == "simple": service_instance = self._spawn_simple_process( process_id, name, module, cls, config) else: raise BadRequest("Unknown process type: %s" % process_type) service_instance._proc_type = process_type self._register_process(service_instance, name) service_instance.errcause = "OK" log.info("AppManager.spawn_process: %s.%s -> pid=%s OK" % (module, cls, process_id)) return service_instance.id except Exception: errcause = service_instance.errcause if service_instance else "instantiating service" log.exception("Error spawning %s %s process (process_id: %s): %s" % (name, process_type, process_id, errcause)) raise def _spawned_proc_failed(self, proc_sup, gproc): log.error("ProcManager._spawned_proc_failed: %s", gproc) # for now - don't worry about the mapping, if we get a failure, just kill the container. # leave the mapping in place for potential expansion later. # # look it up in mapping # if not gproc in self._spawned_proc_to_process: # log.warn("No record of gproc %s in our map (%s)", gproc, self._spawned_proc_to_process) # return # svc = self._spawned_proc_to_process.get(gproc, "Unknown") # # # make sure svc is in our list # if not svc in self.procs.values(): # log.warn("svc %s not found in procs list", svc) # return self.container.fail_fast("Container process (%s) failed: %s" % (svc, gproc.exception)) # ----------------------------------------------------------------- # PROCESS TYPE: service def _spawn_service_process(self, process_id, name, module, cls, config): """ Spawn a process acting as a service worker. Attach to service queue with service definition, attach to service pid """ service_instance = self._create_service_instance( process_id, name, module, cls, config) self._service_init(service_instance) self._service_start(service_instance) listen_name = get_safe(config, "process.listen_name") or service_instance.name log.debug("Service Process (%s) listen_name: %s", name, listen_name) self._set_service_endpoint(service_instance, listen_name) self._set_service_endpoint(service_instance, service_instance.id) # Directory registration self.container.directory.register_safe("/Services", listen_name, interface=service_instance.name) self.container.directory.register_safe("/Services/%s" % listen_name, service_instance.id) return service_instance # ----------------------------------------------------------------- # PROCESS TYPE: stream process def _spawn_stream_process(self, process_id, name, module, cls, config): """ Spawn a process acting as a data stream process. Attach to subscription queue with process function. """ service_instance = self._create_service_instance( process_id, name, module, cls, config) self._service_init(service_instance) # Start the service self._service_start(service_instance) listen_name = get_safe(config, "process.listen_name") or name # Throws an exception if no listen name is given! self._set_subscription_endpoint(service_instance, listen_name) # Add publishers if any... publish_streams = get_safe(config, "process.publish_streams") self._set_publisher_endpoints(service_instance, publish_streams) self._set_service_endpoint(service_instance, service_instance.id) return service_instance # ----------------------------------------------------------------- # PROCESS TYPE: agent def _spawn_agent_process(self, process_id, name, module, cls, config): """ Spawn a process acting as agent process. Attach to service pid. """ service_instance = self._create_service_instance( process_id, name, module, cls, config) if not isinstance(service_instance, ResourceAgent): raise ContainerConfigError( "Agent process must extend ResourceAgent") # Set the resource ID if we get it through the config resource_id = get_safe(service_instance.CFG, "agent.resource_id") if resource_id: service_instance.resource_id = resource_id # Now call the on_init of the agent. self._service_init(service_instance) if not service_instance.resource_id: log.warn("New agent pid=%s has no resource_id set" % process_id) self._service_start(service_instance) self._set_service_endpoint(service_instance, service_instance.id) # Directory registration caps = service_instance.get_capabilities() self.container.directory.register( "/Agents", service_instance.id, **dict(name=service_instance._proc_name, container=service_instance.container.id, resource_id=service_instance.resource_id, agent_id=service_instance.agent_id, def_id=service_instance.agent_def_id, capabilities=caps)) if not service_instance.resource_id: log.warn("Agent process id=%s does not define resource_id!!" % service_instance.id) return service_instance # ----------------------------------------------------------------- # PROCESS TYPE: standalone def _spawn_standalone_process(self, process_id, name, module, cls, config): """ Spawn a process acting as standalone process. Attach to service pid. """ service_instance = self._create_service_instance( process_id, name, module, cls, config) self._service_init(service_instance) self._service_start(service_instance) self._set_service_endpoint(service_instance, service_instance.id) # Add publishers if any... publish_streams = get_safe(config, "process.publish_streams") self._set_publisher_endpoints(service_instance, publish_streams) return service_instance # ----------------------------------------------------------------- # PROCESS TYPE: simple def _spawn_simple_process(self, process_id, name, module, cls, config): """ Spawn a process acting as simple process. No attachments. """ service_instance = self._create_service_instance( process_id, name, module, cls, config) self._service_init(service_instance) self._service_start(service_instance) # Add publishers if any... publish_streams = get_safe(config, "process.publish_streams") self._set_publisher_endpoints(service_instance, publish_streams) return service_instance # ----------------------------------------------------------------- # PROCESS TYPE: immediate def _spawn_immediate_process(self, process_id, name, module, cls, config): """ Spawn a process acting as immediate one off process. No attachments. """ service_instance = self._create_service_instance( process_id, name, module, cls, config) self._service_init(service_instance) self._service_start(service_instance) return service_instance def _create_service_instance(self, process_id, name, module, cls, config): # SERVICE INSTANCE. service_instance = for_name(module, cls) if not isinstance(service_instance, BaseService): raise ContainerConfigError( "Instantiated service not a BaseService %r" % service_instance) # Prepare service instance service_instance.errcause = "" service_instance.id = process_id service_instance.container = self.container service_instance.CFG = config service_instance._proc_name = name # start service dependencies (RPC clients) self._start_service_dependencies(service_instance) return service_instance def _start_service_dependencies(self, service_instance): service_instance.errcause = "setting service dependencies" log.debug("spawn_process dependencies: %s" % service_instance.dependencies) # TODO: Service dependency != process dependency for dependency in service_instance.dependencies: client = getattr(service_instance.clients, dependency) assert client, "Client for dependency not found: %s" % dependency # @TODO: should be in a start_client in RPCClient chain client.process = service_instance client.node = self.container.node def _service_init(self, service_instance): # Init process service_instance.errcause = "initializing service" service_instance.init() def _service_start(self, service_instance): # Start process # TODO: Check for timeout service_instance.errcause = "starting service" service_instance.start() def _set_service_endpoint(self, service_instance, listen_name): service_instance.errcause = "setting process service endpoint" # Service RPC endpoint rsvc = ProcessRPCServer(node=self.container.node, from_name=listen_name, service=service_instance, process=service_instance) # Start an ION process with the right kind of endpoint factory proc = self.proc_sup.spawn((CFG.cc.proctype or 'green', None), listener=rsvc, name=listen_name, proc_name=service_instance._proc_name) self.proc_sup.ensure_ready( proc, "_set_service_endpoint for listen_name: %s" % listen_name) # map gproc to service_instance self._spawned_proc_to_process[proc.proc] = service_instance log.debug("Process %s service listener ready: %s", service_instance.id, listen_name) def _set_subscription_endpoint(self, service_instance, listen_name): service_instance.errcause = "setting process subscription endpoint" service_instance.stream_subscriber_registrar = StreamSubscriberRegistrar( process=service_instance, node=self.container.node) sub = service_instance.stream_subscriber_registrar.create_subscriber( exchange_name=listen_name, callback=lambda m, h: service_instance.call_process(m)) proc = self.proc_sup.spawn((CFG.cc.proctype or 'green', None), listener=sub, name=listen_name, proc_name=service_instance._proc_name) self.proc_sup.ensure_ready( proc, '_set_subscription_endpoint for listen_name: %s' % listen_name) # map gproc to service_instance self._spawned_proc_to_process[proc.proc] = service_instance log.debug("Process %s stream listener ready: %s", service_instance.id, listen_name) def _set_publisher_endpoints(self, service_instance, publisher_streams=None): service_instance.stream_publisher_registrar = StreamPublisherRegistrar( process=service_instance, node=self.container.node) publisher_streams = publisher_streams or {} for name, stream_id in publisher_streams.iteritems(): # problem is here pub = service_instance.stream_publisher_registrar.create_publisher( stream_id) setattr(service_instance, name, pub) def _register_process(self, service_instance, name): # Add to local process dict self.procs_by_name[name] = service_instance self.procs[service_instance.id] = service_instance # Add to directory service_instance.errcause = "registering" self.container.directory.register_safe("/Containers/%s/Processes" % self.container.id, service_instance.id, name=name) self.container.event_pub.publish_event( event_type="ProcessLifecycleEvent", origin=service_instance.id, origin_type="ContainerProcess", sub_type="SPAWN", container_id=self.container.id, process_type=service_instance._proc_type, process_name=service_instance._proc_name, state=ProcessStateEnum.SPAWN) def terminate_process(self, process_id): service_instance = self.procs.get(process_id, None) if not service_instance: raise BadRequest( "Cannot terminate. Process id='%s' unknown on container id='%s'" % (process_id, self.container.id)) service_instance.quit() # find the proc lp = list(self.proc_sup.children) lps = [p for p in lp if p.listener._process == service_instance] for p in lps: p.notify_stop() p.stop() del self.procs[process_id] self.container.directory.unregister_safe( "/Containers/%s/Processes" % self.container.id, service_instance.id) # Cleanup for specific process types if service_instance._proc_type == "service": listen_name = get_safe(service_instance.CFG, "process.listen_name", service_instance.name) self.container.directory.unregister_safe( "/Services/%s" % listen_name, service_instance.id) remaining_workers = self.container.directory.find_entries( "/Services/%s" % listen_name) if remaining_workers and len(remaining_workers) == 2: self.container.directory.unregister_safe( "/Services", listen_name) elif service_instance._proc_type == "agent": self.container.directory.unregister_safe("/Agents", service_instance.id) self.container.event_pub.publish_event( event_type="ProcessLifecycleEvent", origin=service_instance.id, origin_type="ContainerProcess", sub_type="TERMINATE", container_id=self.container.id, process_type=service_instance._proc_type, process_name=service_instance._proc_name, state=ProcessStateEnum.TERMINATE)