def spawn_process(self, name=None, module=None, cls=None, config=None, process_id=None): """ Spawn a process within the container. Processes can be of different type. """ if process_id and not is_valid_identifier(process_id, ws_sub='_'): raise BadRequest("Given process_id %s is not a valid identifier" % process_id) # PROCESS ID. Generate a new process id if not provided # TODO: Ensure it is system-wide unique process_id = process_id or "%s.%s" % (self.container.id, self.proc_id_pool.get_id()) log.debug("ProcManager.spawn_process(name=%s, module.cls=%s.%s, config=%s) as pid=%s", name, module, cls, config, process_id) # CONFIG process_cfg = self._create_process_config(config) try: service_cls = named_any("%s.%s" % (module, cls)) except AttributeError as ae: # Try to nail down the error import importlib importlib.import_module(module) raise # PROCESS TYPE. Determines basic process context (messaging, service interface) process_type = get_safe(process_cfg, "process.type") or getattr(service_cls, "process_type", PROCTYPE_SERVICE) process_start_mode = get_safe(config, "process.start_mode") process_instance = None # alert we have a spawning process, but we don't have the instance yet, so give the class instead (more accurate than name) # Note: this uses a str as first argument instead of a process instance self._call_proc_state_changed("%s.%s" % (module, cls), ProcessStateEnum.PENDING) try: # Additional attributes to set with the process instance proc_attr = {"_proc_type": process_type, "_proc_spawn_cfg": config } # SPAWN. Determined by type if process_type == PROCTYPE_SERVICE: process_instance = self._spawn_service_process(process_id, name, module, cls, process_cfg, proc_attr) elif process_type == PROCTYPE_STREAMPROC: process_instance = self._spawn_stream_process(process_id, name, module, cls, process_cfg, proc_attr) elif process_type == PROCTYPE_AGENT: process_instance = self._spawn_agent_process(process_id, name, module, cls, process_cfg, proc_attr) elif process_type == PROCTYPE_STANDALONE: process_instance = self._spawn_standalone_process(process_id, name, module, cls, process_cfg, proc_attr) elif process_type == PROCTYPE_IMMEDIATE: process_instance = self._spawn_immediate_process(process_id, name, module, cls, process_cfg, proc_attr) elif process_type == PROCTYPE_SIMPLE: process_instance = self._spawn_simple_process(process_id, name, module, cls, process_cfg, proc_attr) else: raise BadRequest("Unknown process type: %s" % process_type) # REGISTER. self._register_process(process_instance, name) process_instance.errcause = "OK" log.info("ProcManager.spawn_process: %s.%s -> pid=%s OK", module, cls, process_id) if process_type == PROCTYPE_IMMEDIATE: log.debug('Terminating immediate process: %s', process_instance.id) self.terminate_process(process_instance.id) # Terminate process also triggers TERMINATING/TERMINATED self._call_proc_state_changed(process_instance, ProcessStateEnum.EXITED) else: # Update local policies for the new process if self.container.has_capability(self.container.CCAP.GOVERNANCE_CONTROLLER): self.container.governance_controller.update_process_policies( process_instance, safe_mode=True, force_update=False) return process_instance.id except IonProcessError: errcause = process_instance.errcause if process_instance else "instantiating process" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) return None except Exception: errcause = process_instance.errcause if process_instance else "instantiating process" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) # trigger failed notification - catches problems in init/start self._call_proc_state_changed(process_instance, ProcessStateEnum.FAILED) raise
def test_is_valid_identifier(self): self.assertTrue(is_valid_identifier("jhwfjff.ef. hfieo()-ffeh", NORMAL_VALID)) self.assertFalse(is_valid_identifier("jhwfjff.ef. hfieo()-ffeh", NORMAL_VALID, ";"))
def spawn_process(self, name=None, module=None, cls=None, config=None, process_id=None): """ Spawn a process within the container. Processes can be of different type. """ if process_id and not is_valid_identifier(process_id, ws_sub='_'): raise BadRequest("Given process_id %s is not a valid identifier" % process_id) # Generate a new process id if not provided # TODO: Ensure it is system-wide unique process_id = process_id or "%s.%s" % (self.container.id, self.proc_id_pool.get_id()) log.debug( "ProcManager.spawn_process(name=%s, module.cls=%s.%s, config=%s) as pid=%s", name, module, cls, config, process_id) process_cfg = deepcopy(CFG) if config: # Use provided config. Must be dict or DotDict if not isinstance(config, DotDict): config = DotDict(config) dict_merge(process_cfg, config, inplace=True) if self.container.spawn_args: # Override config with spawn args dict_merge(process_cfg, self.container.spawn_args, inplace=True) #log.debug("spawn_process() pid=%s process_cfg=%s", process_id, process_cfg) # PROCESS TYPE. Determines basic process context (messaging, service interface) # One of: service, stream_process, agent, simple, immediate service_cls = named_any("%s.%s" % (module, cls)) process_type = get_safe(process_cfg, "process.type") or getattr( service_cls, "process_type", "service") process_start_mode = get_safe(config, "process.start_mode") process_instance = None # alert we have a spawning process, but we don't have the instance yet, so give the class instead (more accurate than name) self._call_proc_state_changed("%s.%s" % (module, cls), ProcessStateEnum.PENDING) try: # spawn service by type if process_type == "service": process_instance = self._spawn_service_process( process_id, name, module, cls, process_cfg) elif process_type == "stream_process": process_instance = self._spawn_stream_process( process_id, name, module, cls, process_cfg) elif process_type == "agent": process_instance = self._spawn_agent_process( process_id, name, module, cls, process_cfg) elif process_type == "standalone": process_instance = self._spawn_standalone_process( process_id, name, module, cls, process_cfg) elif process_type == "immediate": process_instance = self._spawn_immediate_process( process_id, name, module, cls, process_cfg) elif process_type == "simple": process_instance = self._spawn_simple_process( process_id, name, module, cls, process_cfg) else: raise BadRequest("Unknown process type: %s" % process_type) process_instance._proc_type = process_type self._register_process(process_instance, name) process_instance.errcause = "OK" log.info("ProcManager.spawn_process: %s.%s -> pid=%s OK", module, cls, process_id) if process_type == 'immediate': log.info('Terminating immediate process: %s', process_instance.id) self.terminate_process(process_instance.id) # terminate process also triggers TERMINATING/TERMINATED self._call_proc_state_changed(process_instance, ProcessStateEnum.EXITED) else: #Shouldn't be any policies for immediate processes self.update_container_policies(process_instance) return process_instance.id except IonProcessError: errcause = process_instance.errcause if process_instance else "instantiating process" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) return None except Exception: errcause = process_instance.errcause if process_instance else "instantiating process" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) # trigger failed notification - catches problems in init/start self._call_proc_state_changed(process_instance, ProcessStateEnum.FAILED) raise
def test_is_valid_identifier(self): self.assertTrue( is_valid_identifier('jhwfjff.ef. hfieo()-ffeh', NORMAL_VALID)) self.assertFalse( is_valid_identifier('jhwfjff.ef. hfieo()-ffeh', NORMAL_VALID, ';'))
def spawn_process(self, name=None, module=None, cls=None, config=None, process_id=None): """ Spawn a process within the container. Processes can be of different type. """ if process_id and not is_valid_identifier(process_id, ws_sub='_'): raise BadRequest("Given process_id %s is not a valid identifier" % process_id) # Generate a new process id if not provided # TODO: Ensure it is system-wide unique process_id = process_id or "%s.%s" % (self.container.id, self.proc_id_pool.get_id()) log.debug("ProcManager.spawn_process(name=%s, module.cls=%s.%s, config=%s) as pid=%s", name, module, cls, config, process_id) process_cfg = deepcopy(CFG) if config: # Use provided config. Must be dict or DotDict if not isinstance(config, DotDict): config = DotDict(config) if config.get_safe("process.config_ref"): # Use a reference config_ref = config.get_safe("process.config_ref") log.info("Enhancing new process spawn config from ref=%s" % config_ref) matches = re.match(r'^([A-Za-z]+):([A-Za-z0-9]+)/(.+)$', config_ref) if matches: ref_type, ref_id, ref_ext = matches.groups() if ref_type == "resources": if self.container.has_capability(self.container.CCAP.RESOURCE_REGISTRY): try: obj = self.container.resource_registry.read(ref_id) if obj and hasattr(obj, ref_ext): ref_config = getattr(obj, ref_ext) if isinstance(ref_config, dict): dict_merge(process_cfg, ref_config, inplace=True) else: raise BadRequest("config_ref %s exists but not dict" % config_ref) else: raise BadRequest("config_ref %s - attribute not found" % config_ref) except NotFound as nf: log.warn("config_ref %s - object not found" % config_ref) raise else: log.error("Container missing RESOURCE_REGISTRY capability to resolve process config ref %s" % config_ref) else: raise BadRequest("Unknown reference type in: %s" % config_ref) dict_merge(process_cfg, config, inplace=True) if self.container.spawn_args: # Override config with spawn args dict_merge(process_cfg, self.container.spawn_args, inplace=True) #log.debug("spawn_process() pid=%s process_cfg=%s", process_id, process_cfg) # PROCESS TYPE. Determines basic process context (messaging, service interface) # One of the constants defined at the top of this file service_cls = named_any("%s.%s" % (module, cls)) process_type = get_safe(process_cfg, "process.type") or getattr(service_cls, "process_type", "service") process_start_mode = get_safe(config, "process.start_mode") process_instance = None # alert we have a spawning process, but we don't have the instance yet, so give the class instead (more accurate than name) self._call_proc_state_changed("%s.%s" % (module, cls), ProcessStateEnum.PENDING) try: # spawn service by type if process_type == SERVICE_PROCESS_TYPE: process_instance = self._spawn_service_process(process_id, name, module, cls, process_cfg) elif process_type == STREAM_PROCESS_TYPE: process_instance = self._spawn_stream_process(process_id, name, module, cls, process_cfg) elif process_type == AGENT_PROCESS_TYPE: process_instance = self._spawn_agent_process(process_id, name, module, cls, process_cfg) elif process_type == STANDALONE_PROCESS_TYPE: process_instance = self._spawn_standalone_process(process_id, name, module, cls, process_cfg) elif process_type == IMMEDIATE_PROCESS_TYPE: process_instance = self._spawn_immediate_process(process_id, name, module, cls, process_cfg) elif process_type == SIMPLE_PROCESS_TYPE: process_instance = self._spawn_simple_process(process_id, name, module, cls, process_cfg) else: raise BadRequest("Unknown process type: %s" % process_type) process_instance._proc_type = process_type self._register_process(process_instance, name) process_instance.errcause = "OK" log.info("ProcManager.spawn_process: %s.%s -> pid=%s OK", module, cls, process_id) if process_type == IMMEDIATE_PROCESS_TYPE: log.info('Terminating immediate process: %s', process_instance.id) self.terminate_process(process_instance.id) # terminate process also triggers TERMINATING/TERMINATED self._call_proc_state_changed(process_instance, ProcessStateEnum.EXITED) else: #Update local policies for the new process if self.container.has_capability(self.container.CCAP.GOVERNANCE_CONTROLLER): self.container.governance_controller.update_container_policies(process_instance, safe_mode=True) return process_instance.id except IonProcessError: errcause = process_instance.errcause if process_instance else "instantiating process" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) return None except Exception: errcause = process_instance.errcause if process_instance else "instantiating process" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) # trigger failed notification - catches problems in init/start self._call_proc_state_changed(process_instance, ProcessStateEnum.FAILED) raise
def spawn_process(self, name=None, module=None, cls=None, config=None, process_id=None): """ Spawn a process within the container. Processes can be of different type. """ if process_id and not is_valid_identifier(process_id, ws_sub='_'): raise BadRequest("Given process_id %s is not a valid identifier" % process_id) # Generate a new process id if not provided # TODO: Ensure it is system-wide unique process_id = process_id or "%s.%s" % (self.container.id, self.proc_id_pool.get_id()) log.debug("ProcManager.spawn_process(name=%s, module.cls=%s.%s) as pid=%s", name, module, cls, process_id) if not config: # Use system CFG. It has the command line args in it config = DictModifier(CFG) else: # Use provided config. Must be dict or DotDict if not isinstance(config, DotDict): config = DotDict(config) config = DictModifier(CFG, config) if self.container.spawn_args: # Override config with spawn args dict_merge(config, self.container.spawn_args, inplace=True) #log.debug("spawn_process() pid=%s config=%s", process_id, config) # PROCESS TYPE. Determines basic process context (messaging, service interface) # One of: service, stream_process, agent, simple, immediate service_cls = named_any("%s.%s" % (module, cls)) process_type = get_safe(config, "process.type") or getattr(service_cls, "process_type", "service") service_instance = None try: # spawn service by type if process_type == "service": service_instance = self._spawn_service_process(process_id, name, module, cls, config) elif process_type == "stream_process": service_instance = self._spawn_stream_process(process_id, name, module, cls, config) elif process_type == "agent": service_instance = self._spawn_agent_process(process_id, name, module, cls, config) elif process_type == "standalone": service_instance = self._spawn_standalone_process(process_id, name, module, cls, config) elif process_type == "immediate": service_instance = self._spawn_immediate_process(process_id, name, module, cls, config) elif process_type == "simple": service_instance = self._spawn_simple_process(process_id, name, module, cls, config) else: raise BadRequest("Unknown process type: %s" % process_type) service_instance._proc_type = process_type self._register_process(service_instance, name) service_instance.errcause = "OK" log.info("AppManager.spawn_process: %s.%s -> pid=%s OK", module, cls, process_id) if process_type == 'immediate': log.info('Terminating immediate process: %s', service_instance.id) self.terminate_process(service_instance.id) return service_instance.id except Exception: errcause = service_instance.errcause if service_instance else "instantiating service" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) raise
def define_user(self, user_id='', first_name='', last_name='', username='', password='', email='', attributes=None): if user_id: raise NotImplementedError("Update not supported: user_id=%s" % user_id) if not email: raise BadRequest('Email is required') username = username or email user = self._get_user_by_email(email) if user: raise BadRequest("Email already taken") if not username or not is_valid_identifier(username, valid_chars=EMAIL_VALID): raise BadRequest("Argument username invalid: %s" % username) if attributes and type(attributes) is not dict: raise BadRequest("Argument attributes invalid type") if not first_name: first_name = username attributes = attributes or {} full_name = ("%s %s" % (first_name, last_name)) if last_name else first_name IdentityUtils.check_password_policy(password) contact = ContactInformation(individual_names_given=first_name, individual_name_family=last_name, email=email) user_profile = UserIdentityDetails(contact=contact, profile=attributes) actor_obj = ActorIdentity(name=full_name, details=user_profile) # Support fast setting of credentials without expensive compute of bcrypt hash, for quick preload pwd_salt, pwd_hash = None, None if attributes and "scion_init_pwdsalt" in attributes and "scion_init_pwdhash" in attributes: pwd_salt, pwd_hash = attributes.pop( "scion_init_pwdsalt"), attributes.pop("scion_init_pwdhash") user_exists = self.idm_client.is_user_existing(username) if user_exists: raise BadRequest("Username already taken") actor_id = self.idm_client.create_actor_identity(actor_obj) if pwd_salt and pwd_hash: # Add to credentials actor_obj1 = self.rr.read(actor_id) cred_obj = None for cred in actor_obj1.credentials: if cred.username == username: cred_obj = cred break if not cred_obj: cred_obj = Credentials() cred_obj.username = username actor_obj1.credentials.append(cred_obj) actor_obj1.alt_ids.append("UNAME:" + username) cred_obj.identity_provider = "SciON" cred_obj.authentication_service = "SciON IdM" cred_obj.password_salt = pwd_salt cred_obj.password_hash = pwd_hash self.rr.update(actor_obj1) else: self.idm_client.set_actor_credentials(actor_id, username, password) return actor_id
def spawn_process(self, name=None, module=None, cls=None, config=None, process_id=None): """ Spawn a process within the container. Processes can be of different type. """ if process_id and not is_valid_identifier(process_id, ws_sub='_'): raise BadRequest("Given process_id %s is not a valid identifier" % process_id) # Generate a new process id if not provided # TODO: Ensure it is system-wide unique process_id = process_id or "%s.%s" % (self.container.id, self.proc_id_pool.get_id()) log.debug("ProcManager.spawn_process(name=%s, module.cls=%s.%s, config=%s) as pid=%s", name, module, cls, config, process_id) process_cfg = deepcopy(CFG) if config: # Use provided config. Must be dict or DotDict if not isinstance(config, DotDict): config = DotDict(config) dict_merge(process_cfg, config, inplace=True) if self.container.spawn_args: # Override config with spawn args dict_merge(process_cfg, self.container.spawn_args, inplace=True) #log.debug("spawn_process() pid=%s process_cfg=%s", process_id, process_cfg) # PROCESS TYPE. Determines basic process context (messaging, service interface) # One of: service, stream_process, agent, simple, immediate service_cls = named_any("%s.%s" % (module, cls)) process_type = get_safe(process_cfg, "process.type") or getattr(service_cls, "process_type", "service") process_start_mode = get_safe(config, "process.start_mode") process_instance = None # alert we have a spawning process, but we don't have the instance yet, so give the class instead (more accurate than name) self._call_proc_state_changed("%s.%s" % (module, cls), ProcessStateEnum.PENDING) try: # spawn service by type if process_type == "service": process_instance = self._spawn_service_process(process_id, name, module, cls, process_cfg) elif process_type == "stream_process": process_instance = self._spawn_stream_process(process_id, name, module, cls, process_cfg) elif process_type == "agent": process_instance = self._spawn_agent_process(process_id, name, module, cls, process_cfg) elif process_type == "standalone": process_instance = self._spawn_standalone_process(process_id, name, module, cls, process_cfg) elif process_type == "immediate": process_instance = self._spawn_immediate_process(process_id, name, module, cls, process_cfg) elif process_type == "simple": process_instance = self._spawn_simple_process(process_id, name, module, cls, process_cfg) else: raise BadRequest("Unknown process type: %s" % process_type) process_instance._proc_type = process_type self._register_process(process_instance, name) process_instance.errcause = "OK" log.info("ProcManager.spawn_process: %s.%s -> pid=%s OK", module, cls, process_id) if process_type == 'immediate': log.info('Terminating immediate process: %s', process_instance.id) self.terminate_process(process_instance.id) # terminate process also triggers TERMINATING/TERMINATED self._call_proc_state_changed(process_instance, ProcessStateEnum.EXITED) else: #Shouldn't be any policies for immediate processes self.update_container_policies(process_instance) return process_instance.id except IonProcessError: errcause = process_instance.errcause if process_instance else "instantiating process" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) return None except Exception: errcause = process_instance.errcause if process_instance else "instantiating process" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) # trigger failed notification - catches problems in init/start self._call_proc_state_changed(process_instance, ProcessStateEnum.FAILED) raise
def spawn_process(self, name=None, module=None, cls=None, config=None, process_id=None): """ Spawn a process within the container. Processes can be of different type. """ if process_id and not is_valid_identifier(process_id, ws_sub='_'): raise BadRequest("Given process_id %s is not a valid identifier" % process_id) # Generate a new process id if not provided # TODO: Ensure it is system-wide unique process_id = process_id or "%s.%s" % (self.container.id, self.proc_id_pool.get_id()) log.debug( "ProcManager.spawn_process(name=%s, module.cls=%s.%s) as pid=%s", name, module, cls, process_id) if not config: # Use system CFG. It has the command line args in it config = DictModifier(CFG) else: # Use provided config. Must be dict or DotDict if not isinstance(config, DotDict): config = DotDict(config) config = DictModifier(CFG, config) if self.container.spawn_args: # Override config with spawn args dict_merge(config, self.container.spawn_args, inplace=True) #log.debug("spawn_process() pid=%s config=%s", process_id, config) # PROCESS TYPE. Determines basic process context (messaging, service interface) # One of: service, stream_process, agent, simple, immediate service_cls = named_any("%s.%s" % (module, cls)) process_type = get_safe(config, "process.type") or getattr( service_cls, "process_type", "service") service_instance = None try: # spawn service by type if process_type == "service": service_instance = self._spawn_service_process( process_id, name, module, cls, config) elif process_type == "stream_process": service_instance = self._spawn_stream_process( process_id, name, module, cls, config) elif process_type == "agent": service_instance = self._spawn_agent_process( process_id, name, module, cls, config) elif process_type == "standalone": service_instance = self._spawn_standalone_process( process_id, name, module, cls, config) elif process_type == "immediate": service_instance = self._spawn_immediate_process( process_id, name, module, cls, config) elif process_type == "simple": service_instance = self._spawn_simple_process( process_id, name, module, cls, config) else: raise BadRequest("Unknown process type: %s" % process_type) service_instance._proc_type = process_type self._register_process(service_instance, name) service_instance.errcause = "OK" log.info("AppManager.spawn_process: %s.%s -> pid=%s OK", module, cls, process_id) if process_type == 'immediate': log.info('Terminating immediate process: %s', service_instance.id) self.terminate_process(service_instance.id) return service_instance.id except Exception: errcause = service_instance.errcause if service_instance else "instantiating service" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) raise
def define_user(self, user_id='', first_name='', last_name='', username='', password='', email='', attributes=None): if user_id: raise NotImplementedError("Update not supported: user_id=%s" % user_id) if not email: raise BadRequest('Email is required') username = username or email user = self._get_user_by_email(email) if user: raise BadRequest("Email already taken") if not username or not is_valid_identifier(username, valid_chars=EMAIL_VALID): raise BadRequest("Argument username invalid: %s" % username) if attributes and type(attributes) is not dict: raise BadRequest("Argument attributes invalid type") if not first_name: first_name = username attributes = attributes or {} full_name = ("%s %s" % (first_name, last_name)) if last_name else first_name IdentityUtils.check_password_policy(password) contact = ContactInformation(individual_names_given=first_name, individual_name_family=last_name, email=email) user_profile = UserIdentityDetails(contact=contact, profile=attributes) actor_obj = ActorIdentity(name=full_name, details=user_profile) # Support fast setting of credentials without expensive compute of bcrypt hash, for quick preload pwd_salt, pwd_hash = None, None if attributes and "scion_init_pwdsalt" in attributes and "scion_init_pwdhash" in attributes: pwd_salt, pwd_hash = attributes.pop("scion_init_pwdsalt"), attributes.pop("scion_init_pwdhash") user_exists = self.idm_client.is_user_existing(username) if user_exists: raise BadRequest("Username already taken") actor_id = self.idm_client.create_actor_identity(actor_obj) if pwd_salt and pwd_hash: # Add to credentials actor_obj1 = self.rr.read(actor_id) cred_obj = None for cred in actor_obj1.credentials: if cred.username == username: cred_obj = cred break if not cred_obj: cred_obj = Credentials() cred_obj.username = username actor_obj1.credentials.append(cred_obj) actor_obj1.alt_ids.append("UNAME:" + username) cred_obj.identity_provider = "SciON" cred_obj.authentication_service = "SciON IdM" cred_obj.password_salt = pwd_salt cred_obj.password_hash = pwd_hash self.rr.update(actor_obj1) else: self.idm_client.set_actor_credentials(actor_id, username, password) return actor_id