def _load_capabilities(self): self._cap_initialized = [] # List of capability constants initialized in container self._capabilities = [] # List of capability constants active in container self._cap_instances = {} # Dict mapping capability->manager instance self._cap_definitions = Config(["res/config/container_capabilities.yml"]).data['capabilities'] profile_filename = CFG.get_safe("container.profile", "development") if not profile_filename.endswith(".yml"): profile_filename = "res/profile/%s.yml" % profile_filename log.info("Loading CC capability profile from file: %s", profile_filename) profile_cfg = Config([profile_filename]).data if not isinstance(profile_cfg, dict) or profile_cfg['type'] != "profile" or not "profile" in profile_cfg: raise ContainerError("Container capability profile invalid: %s" % profile_filename) self.cap_profile = profile_cfg['profile'] if "capabilities" in self.cap_profile and self.cap_profile['capabilities']: dict_merge(self._cap_definitions, self.cap_profile['capabilities'], True) CCAP.clear() cap_list = self._cap_definitions.keys() CCAP.update(zip(cap_list, cap_list)) if "config" in self.cap_profile and self.cap_profile['config']: log.info("Container CFG was changed based on profile: %s", profile_filename)
def start_app(self, appdef=None, config=None): """ @brief Start an app from an app definition. Note: apps can come in one of 2 variants: 1 processapp: In-line defined process to be started 2 regular app: Full app definition """ log.debug("AppManager.start_app(appdef=%s) ..." % appdef) appdef = DotDict(appdef) if 'config' in appdef: app_cfg = appdef.config.copy() if config: dict_merge(app_cfg, config, inplace=True) config = app_cfg if 'processapp' in appdef: # Case 1: Appdef contains definition of process to start name, module, cls = appdef.processapp try: pid = self.container.spawn_process(name, module, cls, config) appdef._pid = pid self.apps.append(appdef) except Exception, ex: log.exception("Appl %s start from processapp failed" % appdef.name)
def _get_execution_engine_config(self): ee_base_cfg = CFG.get_safe("container.execution_engine") or {} if ee_base_cfg.get("type", None) != "scioncc": raise ContainerConfigError("Execution engine config invalid: %s", ee_base_cfg) ee_cfg = deepcopy(ee_base_cfg) # If we are a child process, merge in child config override proc_name = multiprocessing.current_process().name ee_cfg["container"] = dict(child_proc_name=proc_name, is_child=False) child_cfgs = ee_base_cfg.get("child_configs", None) or {} if proc_name.startswith("Container-child-"): ee_cfg["container"]["is_child"] = True if proc_name in child_cfgs: log.info("Applying execution engine config override for child: %s", proc_name) dict_merge(ee_cfg, child_cfgs[proc_name], inplace=True) else: for cfg_name, ch_cfg in child_cfgs.iteritems(): pattern = ch_cfg.get("name_pattern", None) if pattern and re.match(pattern, proc_name): log.info("Applying execution engine config override %s for child: %s", cfg_name, proc_name) dict_merge(ee_cfg, ch_cfg, inplace=True) break ee_cfg.pop("child_configs", None) return ee_cfg
def __init__(self, *args, **kwargs): BaseContainerAgent.__init__(self, *args, **kwargs) # set id and name (as they are set in base class call) self.id = string.replace('%s_%d' % (os.uname()[1], os.getpid()), ".", "_") self.name = "cc_agent_%s" % self.id Container.instance = self # TODO: Bug: Replacing CFG instance not work because references are already public. Update directly dict_merge(CFG, kwargs) from pyon.core import bootstrap bootstrap.sys_name = CFG.system.name or bootstrap.sys_name log.debug("Container (sysname=%s) initializing ..." % bootstrap.sys_name) # Keep track of the overrides from the command-line, so they can trump app/rel file data self.spawn_args = DictModifier(CFG, kwargs) # Load object and service registry etc. bootstrap_pyon() # Create this Container's specific ExchangeManager instance self.ex_manager = ExchangeManager(self) # Create this Container's specific ProcManager instance self.proc_manager = ProcManager(self) # Create this Container's specific AppManager instance self.app_manager = AppManager(self) # DatastoreManager - controls access to Datastores (both mock and couch backed) self.datastore_manager = DatastoreManager() log.debug("Container initialized, OK.")
def start_rel(self, rel=None, config=None): """ @brief Recurse over the rel and start apps defined there. Note: apps in a rel file can come in one of 2 forms: 1 processapp: In-line defined process to be started as app 2 app file: Reference to an app definition in an app file If the rel file provides an app config block, it is provided to spawn the process. Any given function config dict is merged on top of this. """ log.debug("AppManager.start_rel(rel=%s) ...", rel) if rel is None: return if self.use_pd: log.info("Sending rel file to PD") import json rel_def = json.loads(json.dumps(rel)) # HACK to get rid of OrderedDict (not serializable) cmd_res = self.pd_client.start_rel_blocking(rel_def, timeout=None) return cmd_res for rel_app_cfg in rel.apps: name = rel_app_cfg.name log.debug("app definition in rel: %s" % str(rel_app_cfg)) if 'processapp' in rel_app_cfg: # Case 1: Rel contains definition of process to start as app name, module, cls = rel_app_cfg.processapp rel_cfg = None if 'config' in rel_app_cfg: rel_cfg = deepcopy(rel_app_cfg.config) if config: dict_merge(rel_cfg, config, inplace=True) if 'replicas' in rel_app_cfg: proc_replicas = int(rel_app_cfg["replicas"]) if self.max_proc_replicas > 0: if proc_replicas > self.max_proc_replicas: log.info("Limiting number of proc replicas to %s from %s", self.max_proc_replicas, proc_replicas) proc_replicas = min(proc_replicas, self.max_proc_replicas) if proc_replicas < 1 or proc_replicas > 100: log.warn("Invalid number of process replicas: %s", proc_replicas) proc_replicas = 1 for i in xrange(proc_replicas): proc_name = "%s.%s" % (name, i) if i else name self.container.spawn_process(proc_name, module, cls, rel_cfg) else: self.container.spawn_process(name, module, cls, rel_cfg) self.apps.append(DotDict(type="application", name=name, processapp=rel_app_cfg.processapp)) else: # Case 2: Rel contains reference to app file to start app_file_path = 'res/apps/%s.yml' % (name) rel_cfg = rel_app_cfg.get('config', None) if config: dict_merge(rel_cfg, config, inplace=True) self.start_app_from_url(app_file_path, config=rel_cfg)
def __init__(self, *args, **kwargs): BaseContainerAgent.__init__(self, *args, **kwargs) self._is_started = False # set id and name (as they are set in base class call) self.id = string.replace('%s_%d' % (os.uname()[1], os.getpid()), ".", "_") self.name = "cc_agent_%s" % self.id Container.instance = self # TODO: Bug: Replacing CFG instance not work because references are already public. Update directly dict_merge(CFG, kwargs, inplace=True) from pyon.core import bootstrap bootstrap.container_instance = self bootstrap.assert_configuration(CFG) log.debug("Container (sysname=%s) initializing ..." % bootstrap.get_sys_name()) # Keep track of the overrides from the command-line, so they can trump app/rel file data self.spawn_args = kwargs # Load object and service registry etc. bootstrap_pyon() # Create this Container's specific ExchangeManager instance self.ex_manager = ExchangeManager(self) # Create this Container's specific ProcManager instance self.proc_manager = ProcManager(self) # Create this Container's specific AppManager instance self.app_manager = AppManager(self) # DatastoreManager - controls access to Datastores (both mock and couch backed) self.datastore_manager = DatastoreManager() # File System - Interface to the OS File System, using correct path names and setups self.file_system = FileSystem(CFG) # Governance Controller - manages the governance related interceptors self.governance_controller = GovernanceController(self) # sFlow manager - controls sFlow stat emission self.sflow_manager = SFlowManager(self) # Coordinates the container start self._is_started = False self._capabilities = [] self._status = "INIT" # protection for when the container itself is used as a Process for clients self.container = self log.debug("Container initialized, OK.")
def do_work(container): """ Performs initial startup actions with the container as requested in arguments. Then remains in container shell or infinite wait until container stops. Returns when container should stop. Raises an exception if anything failed. """ if opts.proc: # Run a one-off process (with the -x argument) mod, proc = opts.proc.rsplit('.', 1) print "pycc: Starting process %s" % opts.proc container.spawn_process(proc, mod, proc, config={'process': { 'type': 'immediate' }}) # And end return if opts.rel: # Start a rel file start_ok = container.start_rel_from_url(opts.rel) if not start_ok: raise Exception("Cannot start deploy file '%s'" % opts.rel) if opts.mx: container.spawn_process("ContainerUI", "ion.core.containerui", "ContainerUI") print "pycc: Container UI started ... listening on http://localhost:8080" if opts.signalparent: import os import signal print 'pycc: Signal parent pid %d that pycc pid %d service start process is complete...' % ( os.getppid(), os.getpid()) os.kill(os.getppid(), signal.SIGUSR1) def is_parent_gone(): while os.getppid() != 1: gevent.sleep(1) print 'pycc: Now I am an orphan ... notifying serve_forever to stop' os.kill(os.getpid(), signal.SIGINT) import gevent ipg = gevent.spawn(is_parent_gone) from pyon.util.containers import dict_merge from pyon.public import CFG dict_merge(CFG, {'system': {'watch_parent': ipg}}, True) if not opts.noshell and not opts.daemon: # Keep container running while there is an interactive shell from pyon.container.shell_api import get_shell_api setup_ipython(get_shell_api(container)) else: # Keep container running until process terminated container.serve_forever()
def test_dict_merge(self): # dict_merge(base, upd, inplace=False): org_dict = { "a": "str_a", "d": { "d-x": 1, "d-y": None, "d-d": { "d-d-1": 1, "d-d-2": 2 } } } base_dict = copy.deepcopy(org_dict) dd = DictDiffer(base_dict, org_dict) self.assertTrue(len(base_dict), 2) self.assertTrue(len(dd.unchanged()), len(base_dict)) # Case 1: Add new value delta_dict = {"c": "NEW_C"} mod_dict = dict_merge(base_dict, delta_dict) dd = DictDiffer(base_dict, org_dict) self.assertTrue(len(base_dict), 2) self.assertTrue(len(dd.unchanged()), len(base_dict)) dd = DictDiffer(mod_dict, org_dict) self.assertTrue(len(mod_dict), 3) self.assertTrue(len(dd.unchanged()), len(org_dict)) self.assertTrue(dd.added(), 1) # Case 2: Change simple type value delta_dict = {"a": 5} base_dict = copy.deepcopy(org_dict) mod_dict = dict_merge(base_dict, delta_dict) dd = DictDiffer(mod_dict, org_dict) self.assertTrue(len(mod_dict), len(org_dict)) self.assertTrue(len(dd.unchanged()), len(org_dict) - 1) self.assertTrue(len(dd.changed()), 1) self.assertTrue(mod_dict['a'], 5) # Case 3: Add new value on lower level delta_dict = {"d": {"new": "NEW_ENTRY"}} base_dict = copy.deepcopy(org_dict) mod_dict = dict_merge(base_dict, delta_dict) dd = DictDiffer(mod_dict, org_dict) self.assertTrue(len(mod_dict), len(org_dict)) self.assertTrue(len(mod_dict['d']), len(org_dict['d']) + 1) self.assertTrue(mod_dict['d']['new'], "NEW_ENTRY") dd = DictDiffer(mod_dict['d'], org_dict['d']) self.assertTrue(len(dd.unchanged()), len(org_dict['d'])) self.assertTrue(dd.added(), 1)
def apply_profile_configuration(system_cfg, bootstrap_config): profile_filename = bootstrap_config.get_safe("container.profile", None) if not profile_filename: return if not profile_filename.endswith(".yml"): profile_filename = "res/profile/%s.yml" % profile_filename from pyon.util.config import Config profile_cfg = Config([profile_filename]).data config_override = profile_cfg.get_safe("profile.config") if config_override and isinstance(config_override, dict): from pyon.util.containers import dict_merge dict_merge(system_cfg, config_override, inplace=True)
def _call_plugins(self, method, process, config, **kwargs): bootstrap_plugins = config.get_safe("bootstrap_plugins", None) if bootstrap_plugins is None: log.warn("Bootstrapper called without bootstrap_plugins config") # Finding the system actor ID. If found, construct call context headers. # This may be called very early in bootstrap with no system actor yet existing system_actor = get_system_actor() if system_actor: actor_headers = get_system_actor_header(system_actor) else: # Use default actor headers, not roles. actor_headers = build_actor_header() # Set the call context of the current process with process.push_context(actor_headers): for plugin_info in bootstrap_plugins: plugin_mod, plugin_cls = plugin_info.get("plugin", [None,None]) plugin_cfg = plugin_info.get("config", None) plugin_cfg = dict_merge(config, plugin_cfg) if plugin_cfg is not None else config try: log.info("Bootstrapping plugin %s.%s ...", plugin_mod, plugin_cls) plugin = for_name(plugin_mod, plugin_cls) plugin_func = getattr(plugin, method) plugin_func(process, plugin_cfg, **kwargs) except AbortBootstrap as abort: raise except Exception as ex: log.exception("Error bootstrapping plugin %s.%s", plugin_mod, plugin_cls)
def _call_plugins(self, method, process, config, **kwargs): bootstrap_plugins = config.get_safe("bootstrap_plugins", None) if bootstrap_plugins is None: log.warn("Bootstrapper called without bootstrap_plugins config") # Finding the system actor ID. If found, construct call context headers. # This may be called very early in bootstrap with no system actor yet existing system_actor, _ = process.container.resource_registry.find_resources( RT.ActorIdentity, name=self.CFG.system.system_actor, id_only=False) actor_headers = get_system_actor_header( system_actor[0] if system_actor else None) # Set the call context of the current process with process.push_context(actor_headers): for plugin_info in bootstrap_plugins: plugin_mod, plugin_cls = plugin_info.get( "plugin", [None, None]) plugin_cfg = plugin_info.get("config", None) plugin_cfg = dict_merge( config, plugin_cfg) if plugin_cfg is not None else config try: log.info("Bootstrapping plugin %s.%s ...", plugin_mod, plugin_cls) plugin = for_name(plugin_mod, plugin_cls) plugin_func = getattr(plugin, method) plugin_func(process, plugin_cfg, **kwargs) except AbortBootstrap as abort: raise except Exception as ex: log.exception("Error bootstrapping plugin %s.%s", plugin_mod, plugin_cls)
def _call_plugins(self, method, process, config, **kwargs): bootstrap_plugins = config.get_safe("bootstrap_plugins", None) if bootstrap_plugins is None: log.warn("Bootstrapper called without bootstrap_plugins config") # Finding the system actor ID. If found, construct call context headers. # This may be called very early in bootstrap with no system actor yet existing system_actor = get_system_actor() if system_actor: actor_headers = get_system_actor_header(system_actor) else: # Use default actor headers, not roles. actor_headers = build_actor_header() # Set the call context of the current process with process.push_context(actor_headers): for plugin_info in bootstrap_plugins: plugin_mod, plugin_cls = plugin_info.get( "plugin", [None, None]) plugin_cfg = plugin_info.get("config", None) plugin_cfg = dict_merge( config, plugin_cfg) if plugin_cfg is not None else config try: log.info("Bootstrapping plugin %s.%s ...", plugin_mod, plugin_cls) plugin = for_name(plugin_mod, plugin_cls) plugin_func = getattr(plugin, method) plugin_func(process, plugin_cfg, **kwargs) except AbortBootstrap as abort: raise except Exception as ex: log.exception("Error bootstrapping plugin %s.%s", plugin_mod, plugin_cls)
def _generate_skeleton_config_block(self): log.info("Generating skeleton config block for %s", self.agent_instance_obj.name) # merge the agent config into the default config agent_config = dict_merge(self._get_agent().agent_default_config, self.agent_instance_obj.agent_config, True) org_obj = self._generate_org() # Create agent_config. agent_config['instance_id'] = self.agent_instance_obj._id agent_config['instance_name'] = self.agent_instance_obj.name agent_config['org_governance_name'] = org_obj.org_governance_name if org_obj else '' agent_config['provider_id'] = org_obj._id if org_obj else '' agent_config['actor_id'] = self.actor_id agent_config['device_type'] = self._generate_device_type() agent_config['driver_config'] = self._generate_driver_config() agent_config['stream_config'] = self._generate_stream_config() agent_config['agent'] = self._generate_agent_config() agent_config['aparam_alerts_config'] = self._generate_alerts_config() agent_config['startup_config'] = self._generate_startup_config() agent_config['children'] = self._generate_children() log.info("DONE generating skeleton config block for %s", self.agent_instance_obj.name) return agent_config
def _call_plugins(self, method, process, config, **kwargs): bootstrap_plugins = config.get_safe("bootstrap_plugins", None) if bootstrap_plugins is None: log.warn("Bootstrapper called without bootstrap_plugins config") # Finding the system actor ID. If found, construct call context headers. # This may be called very early in bootstrap with no system actor yet existing system_actor, _ = process.container.resource_registry.find_resources( RT.ActorIdentity, name=self.CFG.system.system_actor, id_only=True ) system_actor_id = system_actor[0] if system_actor else "anonymous" actor_headers = { "ion-actor-id": system_actor_id, "ion-actor-roles": {"ION": ["ION_MANAGER", "ORG_MANAGER"]} if system_actor else {}, } # Set the call context of the current process with process.push_context(actor_headers): for plugin_info in bootstrap_plugins: plugin_mod, plugin_cls = plugin_info.get("plugin", [None, None]) plugin_cfg = plugin_info.get("config", None) plugin_cfg = dict_merge(config, plugin_cfg) if plugin_cfg is not None else config try: log.info("Bootstrapping plugin %s.%s ...", plugin_mod, plugin_cls) plugin = for_name(plugin_mod, plugin_cls) plugin_func = getattr(plugin, method) plugin_func(process, plugin_cfg, **kwargs) except AbortBootstrap as abort: raise except Exception as ex: log.exception("Error bootstrapping plugin %s.%s", plugin_mod, plugin_cls)
def _generate_driver_config(self): log.debug("_generate_driver_config for %s", self.agent_instance_obj.name) # get default config driver_config = super(ExternalDatasetAgentConfigurationBuilder, self)._generate_driver_config() agent_instance_obj = self.agent_instance_obj agent_obj = self._get_agent() parser_cfg = copy.deepcopy(agent_obj.parser_default_config) poller_cfg = copy.deepcopy(agent_obj.poller_default_config) # Create driver config. base_driver_config = { 'parser': { 'uri': agent_obj.parser_uri, 'module': agent_obj.parser_module, 'class': agent_obj.parser_class, 'config': parser_cfg, }, 'poller': { 'uri': agent_obj.poller_uri, 'module': agent_obj.poller_module, 'class': agent_obj.poller_class, 'config': poller_cfg, }, } res_driver_config = dict_merge(base_driver_config, driver_config) return res_driver_config
def _generate_skeleton_config_block(self): log.info("Generating skeleton config block for %s", self.agent_instance_obj.name) # merge the agent config into the default config agent_config = dict_merge(self._get_agent().agent_default_config, self.agent_instance_obj.agent_config, True) org_obj = self._generate_org() # Create agent_config. agent_config['instance_id'] = self.agent_instance_obj._id agent_config['instance_name'] = self.agent_instance_obj.name agent_config[ 'org_governance_name'] = org_obj.org_governance_name if org_obj else '' agent_config['provider_id'] = org_obj._id if org_obj else '' agent_config['actor_id'] = self.actor_id agent_config['device_type'] = self._generate_device_type() agent_config['driver_config'] = self._generate_driver_config() agent_config['stream_config'] = self._generate_stream_config() agent_config['agent'] = self._generate_agent_config() agent_config['aparam_alerts_config'] = self._generate_alerts_config() agent_config['startup_config'] = self._generate_startup_config() agent_config['children'] = self._generate_children() log.info("DONE generating skeleton config block for %s", self.agent_instance_obj.name) return agent_config
def do_work(container): """ Performs initial startup actions with the container as requested in arguments. Then remains in container shell or infinite wait until container stops. Returns when container should stop. Raises an exception if anything failed. """ if opts.proc: # Run a one-off process (with the -x argument) mod, proc = opts.proc.rsplit('.', 1) print "pycc: Starting process %s" % opts.proc container.spawn_process(proc, mod, proc, config={'process':{'type':'immediate'}}) # And end return if opts.rel: # Start a rel file start_ok = container.start_rel_from_url(opts.rel) if not start_ok: raise Exception("Cannot start deploy file '%s'" % opts.rel) if opts.mx: container.spawn_process("ContainerUI", "ion.core.containerui", "ContainerUI") print "pycc: Container UI started ... listening on http://localhost:8080" if opts.signalparent: import os import signal print 'pycc: Signal parent pid %d that pycc pid %d service start process is complete...' % (os.getppid(), os.getpid()) os.kill(os.getppid(), signal.SIGUSR1) def is_parent_gone(): while os.getppid() != 1: gevent.sleep(1) print 'pycc: Now I am an orphan ... notifying serve_forever to stop' os.kill(os.getpid(), signal.SIGINT) import gevent ipg = gevent.spawn(is_parent_gone) from pyon.util.containers import dict_merge from pyon.public import CFG dict_merge(CFG, {'system':{'watch_parent': ipg}}, True) if not opts.noshell and not opts.daemon: # Keep container running while there is an interactive shell from pyon.container.shell_api import get_shell_api setup_ipython(get_shell_api(container)) else: # Keep container running until process terminated container.serve_forever()
def start_app(self, appdef=None, config=None): """ @brief Start an app from an app definition. Note: apps can come in one of 2 variants: 1 processapp: In-line defined process to be started 2 regular app: Full app definition """ log.debug("AppManager.start_app(appdef=%s) ..." % appdef) appdef = DotDict(appdef) if 'config' in appdef: app_cfg = deepcopy(appdef.config) if config: dict_merge(app_cfg, config, inplace=True) config = app_cfg if 'processapp' in appdef: # Case 1: Appdef contains definition of process to start name, module, cls = appdef.processapp try: pid = self.container.spawn_process(name, module, cls, config) appdef._pid = pid self.apps.append(appdef) except Exception: log.exception("Appl %s start from processapp failed" % appdef.name) else: # Case 2: Appdef contains full app start params modpath = appdef.mod try: mod = named_any(modpath) appdef._mod_loaded = mod # Start the app supid, state = mod.start(self.container, START_PERMANENT, appdef, config) appdef._supid = supid appdef._state = state log.debug("App '%s' started. Root sup-id=%s" % (appdef.name, supid)) self.apps.append(appdef) except Exception: log.exception("Appl %s start from appdef failed" % appdef.name)
def acquire_data(self, streaming_args=None): if self.current_state != self.AGENTSTATE_CONNECTED: raise BadRequest("Illegal agent state: %s" % self.current_state) try: args = dict_merge(self.agent_config, streaming_args) if streaming_args else self.agent_config res = self.on_acquire_data(args) except Exception: self.current_state = self.AGENTSTATE_ERROR raise
def test_dict_merge(self): # dict_merge(base, upd, inplace=False): org_dict = {"a":"str_a", "d": {"d-x": 1, "d-y": None, "d-d": {"d-d-1": 1, "d-d-2": 2}}} base_dict = copy.deepcopy(org_dict) dd = DictDiffer(base_dict, org_dict) self.assertTrue(len(base_dict), 2) self.assertTrue(len(dd.unchanged()), len(base_dict)) # Case 1: Add new value delta_dict = {"c" : "NEW_C"} mod_dict = dict_merge(base_dict, delta_dict) dd = DictDiffer(base_dict, org_dict) self.assertTrue(len(base_dict), 2) self.assertTrue(len(dd.unchanged()), len(base_dict)) dd = DictDiffer(mod_dict, org_dict) self.assertTrue(len(mod_dict), 3) self.assertTrue(len(dd.unchanged()), len(org_dict)) self.assertTrue(dd.added(), 1) # Case 2: Change simple type value delta_dict = {"a" : 5} base_dict = copy.deepcopy(org_dict) mod_dict = dict_merge(base_dict, delta_dict) dd = DictDiffer(mod_dict, org_dict) self.assertTrue(len(mod_dict), len(org_dict)) self.assertTrue(len(dd.unchanged()), len(org_dict)-1) self.assertTrue(len(dd.changed()), 1) self.assertTrue(mod_dict['a'], 5) # Case 3: Add new value on lower level delta_dict = {"d": {"new":"NEW_ENTRY"}} base_dict = copy.deepcopy(org_dict) mod_dict = dict_merge(base_dict, delta_dict) dd = DictDiffer(mod_dict, org_dict) self.assertTrue(len(mod_dict), len(org_dict)) self.assertTrue(len(mod_dict['d']), len(org_dict['d']) + 1) self.assertTrue(mod_dict['d']['new'], "NEW_ENTRY") dd = DictDiffer(mod_dict['d'], org_dict['d']) self.assertTrue(len(dd.unchanged()), len(org_dict['d'])) self.assertTrue(dd.added(), 1)
def acquire_data(self, streaming_args=None): if self.current_state != self.AGENTSTATE_CONNECTED: raise BadRequest("Illegal agent state: %s" % self.current_state) try: args = dict_merge( self.agent_config, streaming_args) if streaming_args else self.agent_config res = self.on_acquire_data(args) except Exception: self.current_state = self.AGENTSTATE_ERROR raise
def start_rel(self, rel=None, config=None): """ @brief Recurse over the rel and start apps defined there. Note: apps in a rel file can come in one of 2 forms: 1 processapp: In-line defined process to be started as app 2 app file: Reference to an app definition in an app file If the rel file provides an app config block, it is provided to spawn the process. Any given function config dict is merged on top of this. """ log.debug("AppManager.start_rel(rel=%s) ...", rel) if rel is None: rel = {} for rel_app_cfg in rel.apps: name = rel_app_cfg.name log.debug("app definition in rel: %s" % str(rel_app_cfg)) if 'processapp' in rel_app_cfg: # Case 1: Rel contains definition of process to start as app name, module, cls = rel_app_cfg.processapp rel_cfg = None if 'config' in rel_app_cfg: rel_cfg = rel_app_cfg.config.copy() if config: dict_merge(rel_cfg, config, inplace=True) self.container.spawn_process(name, module, cls, rel_cfg) self.apps.append( DotDict(type="application", name=name, processapp=rel_app_cfg.processapp)) else: # Case 2: Rel contains reference to app file to start app_file_path = 'res/apps/%s.yml' % (name) rel_cfg = rel_app_cfg.get('config', None) if config: dict_merge(rel_cfg, config, inplace=True) self.start_app_from_url(app_file_path, config=rel_cfg)
def prepare_container(): import threading threading.current_thread().name = "CC-Main" # SIDE EFFECT: The import of pyon.public triggers many module initializers: # pyon.core.bootstrap (Config load, logging setup), etc. from pyon.public import Container, CFG from pyon.util.containers import dict_merge from pyon.util.config import Config # Check if user opted to override logging config if opts.logcfg: from pyon.util.config import logging_conf_paths, initialize_logging # Re-initialize logging logging_conf_paths.append(opts.logcfg) initialize_logging() # Set that system is not testing. We are running as standalone container dict_merge(CFG, {'system':{'testing':False}}, True) # Also set the immediate flag, but only if specified - it is an override if opts.immediate: dict_merge(CFG, {'system':{'immediate':True}}, True) # Load any additional config paths and merge them into main config if len(opts.config): cfg = Config(opts.config) dict_merge(CFG, cfg.data, True) # Create the container instance container = Container(*args, **kwargs) return container
def connect(self, connect_args=None): if self.current_state == self.AGENTSTATE_CONNECTED: return elif self.current_state != self.AGENTSTATE_INITIALIZED: raise BadRequest("Illegal agent state: %s" % self.current_state) try: self.stream_pub = StreamPublisher(process=self, stream=self.stream_name) args = dict_merge(self.agent_config, connect_args) if connect_args else self.agent_config res = self.on_connect(args) except Exception: self.current_state = self.AGENTSTATE_ERROR raise self.current_state = self.AGENTSTATE_CONNECTED
def start_rel(self, rel=None, config=None): """ @brief Recurse over the rel and start apps defined there. Note: apps in a rel file can come in one of 2 forms: 1 processapp: In-line defined process to be started as app 2 app file: Reference to an app definition in an app file If the rel file provides an app config block, it is provided to spawn the process. Any given function config dict is merged on top of this. """ log.debug("AppManager.start_rel(rel=%s) ...", rel) if rel is None: rel = {} for rel_app_cfg in rel.apps: name = rel_app_cfg.name log.debug("app definition in rel: %s" % str(rel_app_cfg)) if 'processapp' in rel_app_cfg: # Case 1: Rel contains definition of process to start as app name, module, cls = rel_app_cfg.processapp rel_cfg = None if 'config' in rel_app_cfg: rel_cfg = deepcopy(rel_app_cfg.config) if config: dict_merge(rel_cfg, config, inplace=True) self.container.spawn_process(name, module, cls, rel_cfg) self.apps.append(DotDict(type="application", name=name, processapp=rel_app_cfg.processapp)) else: # Case 2: Rel contains reference to app file to start app_file_path = 'res/apps/%s.yml' % (name) rel_cfg = rel_app_cfg.get('config', None) if config: dict_merge(rel_cfg, config, inplace=True) self.start_app_from_url(app_file_path, config=rel_cfg)
def _load_capabilities(self): self._cap_initialized = [ ] # List of capability constants initialized in container self._capabilities = [ ] # List of capability constants active in container self._cap_instances = {} # Dict mapping capability->manager instance self._cap_definitions = Config( ["res/config/container_capabilities.yml"]).data['capabilities'] profile_filename = CFG.get_safe("container.profile", "development") if not profile_filename.endswith(".yml"): profile_filename = "res/profile/%s.yml" % profile_filename log.debug("Loading CC capability profile from file: %s", profile_filename) profile_cfg = Config([profile_filename]).data if not isinstance( profile_cfg, dict ) or profile_cfg['type'] != "profile" or not "profile" in profile_cfg: raise ContainerError("Container capability profile invalid: %s" % profile_filename) self.cap_profile = profile_cfg['profile'] if "capabilities" in self.cap_profile and self.cap_profile[ 'capabilities']: dict_merge(self._cap_definitions, self.cap_profile['capabilities'], True) CCAP.clear() cap_list = self._cap_definitions.keys() CCAP.update(zip(cap_list, cap_list)) if "config" in self.cap_profile and self.cap_profile['config']: log.info("Container CFG was changed based on profile: %s", profile_filename)
def start_streaming(self, streaming_args=None): if self.current_state == self.AGENTSTATE_STREAMING: return if self.current_state == self.AGENTSTATE_INITIALIZED: self.connect(self.agent_config) if self.current_state != self.AGENTSTATE_CONNECTED: raise BadRequest("Illegal agent state: %s" % self.current_state) log.info("Start streaming") try: args = dict_merge(self.agent_config, streaming_args) if streaming_args else self.agent_config res = self.on_start_streaming(args) except Exception: self.current_state = self.AGENTSTATE_ERROR raise self.current_state = self.AGENTSTATE_STREAMING
def prepare_container(): import threading threading.current_thread().name = "CC-Main" # SIDE EFFECT: The import of pyon.public triggers many module initializers: # pyon.core.bootstrap (Config load, logging setup), etc. from pyon.public import Container, CFG from pyon.util.containers import dict_merge from pyon.util.config import Config # Check if user opted to override logging config # Requires re-initializing logging if opts.logcfg: from pyon.util.config import LOGGING_CFG, logging_conf_paths, read_logging_config, initialize_logging import ast # Dict of config values if '{' in opts.logcfg: try: eval_value = ast.literal_eval(opts.logcfg) except ValueError: raise Exception("Value error in logcfg arg '%s'" % opts.logcfg) dict_merge(LOGGING_CFG, eval_value) initialize_logging() # YAML file containing config values else: logging_conf_paths.append(opts.logcfg) read_logging_config() initialize_logging() # Set that system is not testing. We are running as standalone container dict_merge(CFG, {'system': {'testing': False}}, True) # Also set the immediate flag, but only if specified - it is an override if opts.immediate: dict_merge(CFG, {'system': {'immediate': True}}, True) # Load any additional config paths and merge them into main config if len(opts.config): ipython_cfg = Config(opts.config) dict_merge(CFG, ipython_cfg.data, True) # Create the container instance container = Container(*args, **kwargs) return container
def connect(self, connect_args=None): if self.current_state == self.AGENTSTATE_CONNECTED: return elif self.current_state != self.AGENTSTATE_INITIALIZED: raise BadRequest("Illegal agent state: %s" % self.current_state) try: self.stream_pub = StreamPublisher(process=self, stream=self.stream_name) args = dict_merge( self.agent_config, connect_args) if connect_args else self.agent_config res = self.on_connect(args) except Exception: self.current_state = self.AGENTSTATE_ERROR raise self.current_state = self.AGENTSTATE_CONNECTED
def prepare_container(): import threading threading.current_thread().name = "CC-Main" # SIDE EFFECT: The import of pyon.public triggers many module initializers: # pyon.core.bootstrap (Config load, logging setup), etc. from pyon.public import Container, CFG from pyon.util.containers import dict_merge from pyon.util.config import Config # Check if user opted to override logging config # Requires re-initializing logging if opts.logcfg: from pyon.util.config import LOGGING_CFG, logging_conf_paths, read_logging_config, initialize_logging import ast # Dict of config values if '{' in opts.logcfg: try: eval_value = ast.literal_eval(opts.logcfg) except ValueError: raise Exception("Value error in logcfg arg '%s'" % opts.logcfg) dict_merge(LOGGING_CFG, eval_value) initialize_logging() # YAML file containing config values else: logging_conf_paths.append(opts.logcfg) read_logging_config() initialize_logging() # Set that system is not testing. We are running as standalone container dict_merge(CFG, {'system':{'testing':False}}, True) # Also set the immediate flag, but only if specified - it is an override if opts.immediate: dict_merge(CFG, {'system':{'immediate':True}}, True) # Load any additional config paths and merge them into main config if len(opts.config): ipython_cfg = Config(opts.config) dict_merge(CFG, ipython_cfg.data, True) # Create the container instance container = Container(*args, **kwargs) return container
def start_streaming(self, streaming_args=None): if self.current_state == self.AGENTSTATE_STREAMING: return if self.current_state == self.AGENTSTATE_INITIALIZED: self.connect(self.agent_config) if self.current_state != self.AGENTSTATE_CONNECTED: raise BadRequest("Illegal agent state: %s" % self.current_state) log.info("Start streaming") try: args = dict_merge( self.agent_config, streaming_args) if streaming_args else self.agent_config res = self.on_start_streaming(args) except Exception: self.current_state = self.AGENTSTATE_ERROR raise self.current_state = self.AGENTSTATE_STREAMING
def load(self, ignore_not_found=False): """ Load each path in order. Remember paths already loaded and only load new ones. """ data = self.dict_class() for path in self.paths: if path in self.paths_loaded: continue try: with open(path, 'r') as file: path_data = yaml.load(file.read()) if path_data is not None: data = dict_merge(data, path_data) self.paths_loaded.add(path) except IOError: if not ignore_not_found: raise ConfigNotFound("Config URL '%s' not found" % path) self.data = data
def _override_config(config_override): if type(config_override) is not dict: print "pyon: WARNING: config_override is not dict but", config_override from pyon.util.containers import dict_merge dict_merge(LOGGING_CFG, config_override)
def _get_alt_cfg(cfg_merge): cfg_clone = deepcopy(CFG) dict_merge(cfg_clone, cfg_merge, inplace=True) return DotDict(**cfg_clone)
def start_rel(self, rel=None, config=None): """ @brief Recurse over the rel and start apps defined there. Note: apps in a rel file can come in one of 2 forms: 1 processapp: In-line defined process to be started as app 2 app file: Reference to an app definition in an app file If the rel file provides an app config block, it is provided to spawn the process. Any given function config dict is merged on top of this. """ log.debug("AppManager.start_rel(rel=%s) ...", rel) if rel is None: return if self.use_pd: log.info("Sending rel file to PD") import json rel_def = json.loads(json.dumps( rel)) # HACK to get rid of OrderedDict (not serializable) cmd_res = self.pd_client.start_rel_blocking(rel_def, timeout=None) return cmd_res for rel_app_cfg in rel.apps: name = rel_app_cfg.name log.debug("app definition in rel: %s" % str(rel_app_cfg)) if 'processapp' in rel_app_cfg: # Case 1: Rel contains definition of process to start as app name, module, cls = rel_app_cfg.processapp rel_cfg = None if 'config' in rel_app_cfg: rel_cfg = deepcopy(rel_app_cfg.config) if config: dict_merge(rel_cfg, config, inplace=True) if 'replicas' in rel_app_cfg: proc_replicas = int(rel_app_cfg["replicas"]) if self.max_proc_replicas > 0: if proc_replicas > self.max_proc_replicas: log.info( "Limiting number of proc replicas to %s from %s", self.max_proc_replicas, proc_replicas) proc_replicas = min(proc_replicas, self.max_proc_replicas) if proc_replicas < 1 or proc_replicas > 100: log.warn("Invalid number of process replicas: %s", proc_replicas) proc_replicas = 1 for i in xrange(proc_replicas): proc_name = "%s.%s" % (name, i) if i else name self.container.spawn_process(proc_name, module, cls, rel_cfg) else: self.container.spawn_process(name, module, cls, rel_cfg) self.apps.append( DotDict(type="application", name=name, processapp=rel_app_cfg.processapp)) else: # Case 2: Rel contains reference to app file to start app_file_path = 'res/apps/%s.yml' % (name) rel_cfg = rel_app_cfg.get('config', None) if config: dict_merge(rel_cfg, config, inplace=True) self.start_app_from_url(app_file_path, config=rel_cfg)
def prepare_container(): """ Walks through pyon initialization in a deterministic way and initializes Container. In particular make sure configuration is loaded in correct order and pycc startup arguments are considered. """ # SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults import pyon import threading threading.current_thread().name = "CC-Main" import logging global log log = logging.getLogger('pycc') from pyon.core import bootstrap, config from pyon.util.containers import get_safe, dict_merge # Set global testing flag to False. We are running as capability container, because # we started through the pycc program. bootstrap.testing = False # Set sysname if provided in startup argument if opts.sysname: bootstrap.set_sys_name(opts.sysname) # Trigger any initializing default logic in get_sys_name bootstrap.get_sys_name() command_line_config = kwargs # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores. bootstrap_config = None # This holds the new CFG object for pyon. Build it up in proper sequence and conditions. pyon_config = config.read_standard_configuration( ) # Initial pyon.yml + pyon.local.yml # Load config override if provided. Supports variants literal and list of paths config_override = None if opts.config: if '{' in opts.config: # Variant 1: Dict of config values try: eval_value = ast.literal_eval(opts.config) config_override = eval_value except ValueError: raise Exception("Value error in config arg '%s'" % opts.config) else: # Variant 2: List of paths from pyon.util.config import Config config_override = Config([opts.config]).data # Determine bootstrap_config if opts.config_from_directory: # Load minimal bootstrap config if option "config_from_directory" bootstrap_config = config.read_local_configuration( ['res/config/pyon_min_boot.yml']) config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) log.info( "config_from_directory=True. Minimal bootstrap configuration: %s", bootstrap_config) else: # Otherwise: Set to standard set of local config files plus command line overrides bootstrap_config = deepcopy(pyon_config) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) # Override sysname from config file or command line if not opts.sysname and bootstrap_config.get_safe("system.name", None): new_sysname = bootstrap_config.get_safe("system.name") bootstrap.set_sys_name(new_sysname) # Force_clean - deletes sysname datastores if opts.force_clean: from pyon.datastore import clear_db_util log.info("force_clean=True. DROP DATASTORES for sysname=%s", bootstrap.get_sys_name()) clear_db_util.clear_db(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name()) from pyon.core.interfaces.interfaces import InterfaceAdmin iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config) # If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config # WARNING: If multiple containers start concurrently, this may fail if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True: log.debug("auto_store_interfaces=True.") stored_config = deepcopy(pyon_config) config.apply_configuration(stored_config, config_override) config.apply_configuration(stored_config, command_line_config) iadm.create_core_datastores() iadm.store_config(stored_config) # Determine the final pyon_config: # - Start from standard config already set (pyon.yml + local YML files) # - Optionally load config from directory if opts.config_from_directory: config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config) # - Apply container profile specific config config.apply_profile_configuration(pyon_config, bootstrap_config) # - Reapply pyon.local.yml here again for good measure config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) # - Last apply any separate command line config overrides config.apply_configuration(pyon_config, config_override) config.apply_configuration(pyon_config, command_line_config) iadm.set_config(pyon_config) # Set the immediate flag when command line override specified if opts.immediate: dict_merge(pyon_config, {"system": { "immediate": True }}, inplace=True) # Determine system bootmode for bootstrapping actions (unless explicitly specified) if not pyon_config.get_safe("bootmode"): set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode") if set_bootmode == "auto": if iadm.system_data_exists(): dict_merge(pyon_config, {"bootmode": "restart"}, inplace=True) log.info( "System bootmode auto-detection is ON. Determined bootmode=%s", pyon_config.get_safe("bootmode", "initial")) elif set_bootmode == "secondary": dict_merge(pyon_config, {"bootmode": "secondary"}, inplace=True) log.info("System bootmode override. Set to bootmode=%s", pyon_config.get_safe("bootmode", "")) log.info("System in bootmode=%s", pyon_config.get_safe("bootmode", "initial")) # Bootstrap the pyon framework's core. Load configuration etc. bootstrap.bootstrap_pyon(pyon_cfg=pyon_config) # Delete any queues/exchanges owned by sysname if option "broker_clean" is set if opts.broker_clean: log.info("broker_clean=True, sysname: %s", bootstrap.get_sys_name()) from putil.rabbitmq.rabbit_util import RabbitManagementUtil rabbit_util = RabbitManagementUtil( pyon_config, sysname=bootstrap.get_sys_name()) deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname() log.info("Exchanges deleted (%s): %s" % (len(deleted_exchanges), ", ".join(deleted_exchanges))) log.info("Queues deleted (%s): %s" % (len(deleted_queues), ", ".join(deleted_queues))) if opts.force_clean: from pyon.util.file_sys import FileSystem FileSystem._clean(pyon_config) # If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True: iadm.store_interfaces(idempotent=True) iadm.declare_core_exchange_resources() iadm.close() if opts.no_container: log.info("no_container=True. Stopping here.") return None # Create the container instance from pyon.container.cc import Container container = Container(*args, **command_line_config) container.version = version return container
def prepare_container(): """ Walks through pyon initialization in a deterministic way and initializes Container. In particular make sure configuration is loaded in correct order and pycc startup arguments are considered. """ import threading threading.current_thread().name = "CC-Main" # SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults import pyon from pyon.core import bootstrap, config # Set global testing flag to False. We are running as capability container, because # we started through the pycc program. bootstrap.testing = False # Set sysname if provided in startup argument if opts.sysname: bootstrap.set_sys_name(opts.sysname) # Trigger any initializing default logic in get_sys_name bootstrap.get_sys_name() command_line_config = kwargs # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores. bootstrap_config = None # This holds the new CFG object for pyon. Build it up in proper sequence and conditions. pyon_config = config.read_standard_configuration() # Initial pyon.yml + pyon.local.yml # Load config override if provided. Supports variants literal and list of paths config_override = None if opts.config: if "{" in opts.config: # Variant 1: Dict of config values try: eval_value = ast.literal_eval(opts.config) config_override = eval_value except ValueError: raise Exception("Value error in config arg '%s'" % opts.config) else: # Variant 2: List of paths from pyon.util.config import Config config_override = Config([opts.config]).data # Determine bootstrap_config if opts.config_from_directory: # Load minimal bootstrap config if option "config_from_directory" bootstrap_config = config.read_local_configuration(["res/config/pyon_min_boot.yml"]) config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config else: # Otherwise: Set to standard set of local config files plus command line overrides bootstrap_config = deepcopy(pyon_config) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) # Override sysname from config file or command line if not opts.sysname and bootstrap_config.get_safe("system.name", None): new_sysname = bootstrap_config.get_safe("system.name") bootstrap.set_sys_name(new_sysname) # Delete sysname datastores if option "force_clean" is set if opts.force_clean: from pyon.datastore import clear_couch_util print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name() clear_couch_util.clear_couch( bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name() ) pyon_config.container.filesystem.force_clean = True from pyon.core.interfaces.interfaces import InterfaceAdmin iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config) # If auto_bootstrap, load config and interfaces into directory # Note: this is idempotent and will not alter anything if this is not the first container to run if bootstrap_config.system.auto_bootstrap: print "pycc: auto_bootstrap=True." stored_config = deepcopy(pyon_config) config.apply_configuration(stored_config, config_override) config.apply_configuration(stored_config, command_line_config) iadm.create_core_datastores() iadm.store_config(stored_config) # Determine the final pyon_config # - Start from standard config already set (pyon.yml + local YML files) # - Optionally load config from directory if opts.config_from_directory: config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config) # - Apply container profile specific config config.apply_profile_configuration(pyon_config, bootstrap_config) # - Reapply pyon.local.yml here again for good measure config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) # - Last apply any separate command line config overrides config.apply_configuration(pyon_config, config_override) config.apply_configuration(pyon_config, command_line_config) # Also set the immediate flag, but only if specified - it is an override if opts.immediate: from pyon.util.containers import dict_merge dict_merge(pyon_config, {"system": {"immediate": True}}, True) # Bootstrap pyon's core. Load configuration etc. bootstrap.bootstrap_pyon(pyon_cfg=pyon_config) # Delete any queues/exchanges owned by sysname if option "broker_clean" is set if opts.broker_clean: print "pycc: broker_clean=True, sysname:", bootstrap.get_sys_name() # build connect str connect_str = "-q -H %s -P %s -u %s -p %s -V %s" % ( pyon_config.get_safe("server.amqp_priv.host", pyon_config.get_safe("server.amqp.host", "localhost")), pyon_config.get_safe("container.exchange.management.port", "55672"), pyon_config.get_safe("container.exchange.management.username", "guest"), pyon_config.get_safe("container.exchange.management.password", "guest"), "/", ) from putil.rabbithelper import clean_by_sysname deleted_exchanges, deleted_queues = clean_by_sysname(connect_str, bootstrap.get_sys_name()) print " exchanges deleted (%s): %s" % (len(deleted_exchanges), ",".join(deleted_exchanges)) print " queues deleted (%s): %s" % (len(deleted_queues), ",".join(deleted_queues)) if opts.force_clean: path = os.path.join(pyon_config.get_safe("container.filesystem.root", "/tmp/ion"), bootstrap.get_sys_name()) print "force_clean: Removing", path FileSystem._clean(pyon_config) # Auto-bootstrap interfaces if bootstrap_config.system.auto_bootstrap: iadm.store_interfaces(idempotent=True) iadm.close() if opts.no_container: print "pycc: no_container=True. Stopping here." return None # Create the container instance from pyon.container.cc import Container container = Container(*args, **command_line_config) return container
def spawn_process(self, name=None, module=None, cls=None, config=None): """ Spawn a process within the container. Processes can be of different type. """ # Generate a new process id # TODO: Ensure it is system-wide unique process_id = "%s.%s" % (self.container.id, self.proc_id_pool.get_id()) log.debug("ProcManager.spawn_process(name=%s, module.cls=%s.%s) as pid=%s", name, module, cls, process_id) if not config: # Use system CFG. It has the command line args in it config = DictModifier(CFG) else: # Use provided config. Must be dict or DotDict if not isinstance(config, DotDict): config = DotDict(config) config = DictModifier(CFG, config) if self.container.spawn_args: # Override config with spawn args dict_merge(config, self.container.spawn_args, inplace=True) log.debug("spawn_process() pid=%s config=%s", process_id, config) # PROCESS TYPE. Determines basic process context (messaging, service interface) # One of: service, stream_process, agent, simple, immediate service_cls = named_any("%s.%s" % (module, cls)) process_type = get_safe(config, "process.type") or getattr(service_cls, "process_type", "service") service_instance = None try: # spawn service by type if process_type == "service": service_instance = self._spawn_service_process(process_id, name, module, cls, config) elif process_type == "stream_process": service_instance = self._spawn_stream_process(process_id, name, module, cls, config) elif process_type == "agent": service_instance = self._spawn_agent_process(process_id, name, module, cls, config) elif process_type == "standalone": service_instance = self._spawn_standalone_process(process_id, name, module, cls, config) elif process_type == "immediate": service_instance = self._spawn_immediate_process(process_id, name, module, cls, config) elif process_type == "simple": service_instance = self._spawn_simple_process(process_id, name, module, cls, config) else: raise BadRequest("Unknown process type: %s" % process_type) service_instance._proc_type = process_type self._register_process(service_instance, name) service_instance.errcause = "OK" log.info("AppManager.spawn_process: %s.%s -> pid=%s OK" % (module, cls, process_id)) return service_instance.id except Exception: errcause = service_instance.errcause if service_instance else "instantiating service" log.exception("Error spawning %s %s process (process_id: %s): %s" % (name, process_type, process_id, errcause)) raise
def prepare_container(): """ Walks through pyon initialization in a deterministic way and initializes Container. In particular make sure configuration is loaded in correct order and pycc startup arguments are considered. """ import threading threading.current_thread().name = "CC-Main" # SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults import pyon from pyon.core import bootstrap, config # Set global testing flag to False. We are running as capability container. This is NO TEST. bootstrap.testing = False # Set sysname if provided in startup argument if opts.sysname: bootstrap.set_sys_name(opts.sysname) # Trigger any initializing default logic in get_sys_name bootstrap.get_sys_name() command_line_config = kwargs # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores. bootstrap_config = None # This holds the new CFG object for pyon. Build it up in proper sequence and conditions. pyon_config = config.read_standard_configuration( ) # Initial pyon.yml + pyon.local.yml # Load config override if provided. Supports variants literal and list of paths config_override = None if opts.config: if '{' in opts.config: # Variant 1: Dict of config values try: eval_value = ast.literal_eval(opts.config) config_override = eval_value except ValueError: raise Exception("Value error in config arg '%s'" % opts.config) else: # Variant 2: List of paths from pyon.util.config import Config config_override = Config([opts.config]).data # Determine bootstrap_config if opts.config_from_directory: # Load minimal bootstrap config if option "config_from_directory" bootstrap_config = config.read_local_configuration( ['res/config/pyon_min_boot.yml']) config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config else: # Otherwise: Set to standard set of local config files plus command line overrides bootstrap_config = deepcopy(pyon_config) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) # Override sysname from config file or command line if not opts.sysname and bootstrap_config.get_safe("system.name", None): new_sysname = bootstrap_config.get_safe("system.name") bootstrap.set_sys_name(new_sysname) # Delete sysname datastores if option "force_clean" is set if opts.force_clean: from pyon.datastore import clear_couch_util print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name( ) clear_couch_util.clear_couch(bootstrap_config, prefix=bootstrap.get_sys_name()) pyon_config.container.filesystem.force_clean = True from pyon.core.interfaces.interfaces import InterfaceAdmin iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config) # If auto_bootstrap, load config and interfaces into directory # Note: this is idempotent and will not alter anything if this is not the first container to run if bootstrap_config.system.auto_bootstrap: print "pycc: auto_bootstrap=True." stored_config = deepcopy(pyon_config) config.apply_configuration(stored_config, config_override) config.apply_configuration(stored_config, command_line_config) iadm.create_core_datastores() iadm.store_config(stored_config) # Determine the final pyon_config # - Start from standard config already set (pyon.yml + local YML files) # - Optionally load config from directory if opts.config_from_directory: config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config) config.apply_local_configuration( pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS ) # apply pyon.local.yml again over top # - Last apply any separate command line config overrides config.apply_configuration(pyon_config, config_override) config.apply_configuration(pyon_config, command_line_config) # Also set the immediate flag, but only if specified - it is an override if opts.immediate: from pyon.util.containers import dict_merge dict_merge(pyon_config, {'system': {'immediate': True}}, True) # Bootstrap pyon's core. Load configuration etc. bootstrap.bootstrap_pyon(pyon_cfg=pyon_config) # Delete any queues/exchanges owned by sysname if option "broker_clean" is set if opts.broker_clean: print "pycc: broker_clean=True, sysname:", bootstrap.get_sys_name() # build connect str connect_str = "-q -H %s -P 55672 -u %s -p %s -V %s" % ( pyon_config.get_safe( 'server.amqp_priv.host', pyon_config.get_safe('server.amqp.host', 'localhost')), pyon_config.get_safe('container.exchange.management.username', 'guest'), pyon_config.get_safe('container.exchange.management.password', 'guest'), '/') from putil.rabbithelper import clean_by_sysname deleted_exchanges, deleted_queues = clean_by_sysname( connect_str, bootstrap.get_sys_name()) print " exchanges deleted (%s): %s" % ( len(deleted_exchanges), ",".join(deleted_exchanges)) print " queues deleted (%s): %s" % ( len(deleted_queues), ",".join(deleted_queues)) # Auto-bootstrap interfaces if bootstrap_config.system.auto_bootstrap: iadm.store_interfaces(idempotent=True) iadm.close() if opts.no_container: print "pycc: no_container=True. Stopping here." return None # Create the container instance from pyon.container.cc import Container container = Container(*args, **command_line_config) return container
def prepare_container(): """ Walks through pyon initialization in a deterministic way and initializes Container. In particular make sure configuration is loaded in correct order and pycc startup arguments are considered. """ # SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults import pyon import threading threading.current_thread().name = "CC-Main" import logging global log log = logging.getLogger('pycc') from pyon.core import bootstrap, config from pyon.util.containers import get_safe, dict_merge # Set global testing flag to False. We are running as capability container, because # we started through the pycc program. bootstrap.testing = False # Set sysname if provided in startup argument if opts.sysname: bootstrap.set_sys_name(opts.sysname) # Trigger any initializing default logic in get_sys_name bootstrap.get_sys_name() command_line_config = kwargs # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores. bootstrap_config = None # This holds the new CFG object for pyon. Build it up in proper sequence and conditions. pyon_config = config.read_standard_configuration() # Initial pyon.yml + pyon.local.yml # Load config override if provided. Supports variants literal and list of paths config_override = None if opts.config: if '{' in opts.config: # Variant 1: Dict of config values try: eval_value = ast.literal_eval(opts.config) config_override = eval_value except ValueError: raise Exception("Value error in config arg '%s'" % opts.config) else: # Variant 2: List of paths from pyon.util.config import Config config_override = Config([opts.config]).data # Determine bootstrap_config if opts.config_from_directory: # Load minimal bootstrap config if option "config_from_directory" bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml']) config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) log.info("config_from_directory=True. Minimal bootstrap configuration: %s", bootstrap_config) else: # Otherwise: Set to standard set of local config files plus command line overrides bootstrap_config = deepcopy(pyon_config) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) # Override sysname from config file or command line if not opts.sysname and bootstrap_config.get_safe("system.name", None): new_sysname = bootstrap_config.get_safe("system.name") bootstrap.set_sys_name(new_sysname) # Force_clean - deletes sysname datastores if opts.force_clean: from pyon.datastore import clear_db_util log.info("force_clean=True. DROP DATASTORES for sysname=%s", bootstrap.get_sys_name()) clear_db_util.clear_db(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name()) from pyon.core.interfaces.interfaces import InterfaceAdmin iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config) # If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config # WARNING: If multiple containers start concurrently, this may fail if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True: log.debug("auto_store_interfaces=True.") stored_config = deepcopy(pyon_config) config.apply_configuration(stored_config, config_override) config.apply_configuration(stored_config, command_line_config) iadm.create_core_datastores() iadm.store_config(stored_config) # Determine the final pyon_config: # - Start from standard config already set (pyon.yml + local YML files) # - Optionally load config from directory if opts.config_from_directory: config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config) # - Apply container profile specific config config.apply_profile_configuration(pyon_config, bootstrap_config) # - Reapply pyon.local.yml here again for good measure config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) # - Last apply any separate command line config overrides config.apply_configuration(pyon_config, config_override) config.apply_configuration(pyon_config, command_line_config) iadm.set_config(pyon_config) # Set the immediate flag when command line override specified if opts.immediate: dict_merge(pyon_config, {"system": {"immediate": True}}, inplace=True) # Determine system bootmode for bootstrapping actions (unless explicitly specified) if not pyon_config.get_safe("bootmode"): set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode") if set_bootmode == "auto": if iadm.system_data_exists(): dict_merge(pyon_config, {"bootmode": "restart"}, inplace=True) log.info("System bootmode auto-detection is ON. Determined bootmode=%s", pyon_config.get_safe("bootmode", "initial")) elif set_bootmode == "secondary": dict_merge(pyon_config, {"bootmode": "secondary"}, inplace=True) log.info("System bootmode override. Set to bootmode=%s", pyon_config.get_safe("bootmode", "")) log.info("System in bootmode=%s", pyon_config.get_safe("bootmode", "initial")) # Bootstrap the pyon framework's core. Load configuration etc. bootstrap.bootstrap_pyon(pyon_cfg=pyon_config) # Delete any queues/exchanges owned by sysname if option "broker_clean" is set if opts.broker_clean: log.info("broker_clean=True, sysname: %s", bootstrap.get_sys_name()) from putil.rabbitmq.rabbit_util import RabbitManagementUtil rabbit_util = RabbitManagementUtil(pyon_config, sysname=bootstrap.get_sys_name()) deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname() log.info("Exchanges deleted (%s): %s" % (len(deleted_exchanges), ", ".join(deleted_exchanges))) log.info("Queues deleted (%s): %s" % (len(deleted_queues), ", ".join(deleted_queues))) if opts.force_clean: from pyon.util.file_sys import FileSystem FileSystem._clean(pyon_config) # If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True: iadm.store_interfaces(idempotent=True) iadm.declare_core_exchange_resources() iadm.close() if opts.no_container: log.info("no_container=True. Stopping here.") return None # Create the container instance from pyon.container.cc import Container container = Container(*args, **command_line_config) container.version = version return container
def spawn_process(self, name=None, module=None, cls=None, config=None, process_id=None): """ Spawn a process within the container. Processes can be of different type. """ if process_id and not is_valid_identifier(process_id, ws_sub='_'): raise BadRequest("Given process_id %s is not a valid identifier" % process_id) # Generate a new process id if not provided # TODO: Ensure it is system-wide unique process_id = process_id or "%s.%s" % (self.container.id, self.proc_id_pool.get_id()) log.debug( "ProcManager.spawn_process(name=%s, module.cls=%s.%s, config=%s) as pid=%s", name, module, cls, config, process_id) process_cfg = deepcopy(CFG) if config: # Use provided config. Must be dict or DotDict if not isinstance(config, DotDict): config = DotDict(config) dict_merge(process_cfg, config, inplace=True) if self.container.spawn_args: # Override config with spawn args dict_merge(process_cfg, self.container.spawn_args, inplace=True) #log.debug("spawn_process() pid=%s process_cfg=%s", process_id, process_cfg) # PROCESS TYPE. Determines basic process context (messaging, service interface) # One of: service, stream_process, agent, simple, immediate service_cls = named_any("%s.%s" % (module, cls)) process_type = get_safe(process_cfg, "process.type") or getattr( service_cls, "process_type", "service") process_start_mode = get_safe(config, "process.start_mode") process_instance = None # alert we have a spawning process, but we don't have the instance yet, so give the class instead (more accurate than name) self._call_proc_state_changed("%s.%s" % (module, cls), ProcessStateEnum.PENDING) try: # spawn service by type if process_type == "service": process_instance = self._spawn_service_process( process_id, name, module, cls, process_cfg) elif process_type == "stream_process": process_instance = self._spawn_stream_process( process_id, name, module, cls, process_cfg) elif process_type == "agent": process_instance = self._spawn_agent_process( process_id, name, module, cls, process_cfg) elif process_type == "standalone": process_instance = self._spawn_standalone_process( process_id, name, module, cls, process_cfg) elif process_type == "immediate": process_instance = self._spawn_immediate_process( process_id, name, module, cls, process_cfg) elif process_type == "simple": process_instance = self._spawn_simple_process( process_id, name, module, cls, process_cfg) else: raise BadRequest("Unknown process type: %s" % process_type) process_instance._proc_type = process_type self._register_process(process_instance, name) process_instance.errcause = "OK" log.info("ProcManager.spawn_process: %s.%s -> pid=%s OK", module, cls, process_id) if process_type == 'immediate': log.info('Terminating immediate process: %s', process_instance.id) self.terminate_process(process_instance.id) # terminate process also triggers TERMINATING/TERMINATED self._call_proc_state_changed(process_instance, ProcessStateEnum.EXITED) else: #Shouldn't be any policies for immediate processes self.update_container_policies(process_instance) return process_instance.id except IonProcessError: errcause = process_instance.errcause if process_instance else "instantiating process" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) return None except Exception: errcause = process_instance.errcause if process_instance else "instantiating process" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) # trigger failed notification - catches problems in init/start self._call_proc_state_changed(process_instance, ProcessStateEnum.FAILED) raise
def spawn_process(self, name=None, module=None, cls=None, config=None, process_id=None): """ Spawn a process within the container. Processes can be of different type. """ if process_id and not is_valid_identifier(process_id, ws_sub='_'): raise BadRequest("Given process_id %s is not a valid identifier" % process_id) # Generate a new process id if not provided # TODO: Ensure it is system-wide unique process_id = process_id or "%s.%s" % (self.container.id, self.proc_id_pool.get_id()) log.debug("ProcManager.spawn_process(name=%s, module.cls=%s.%s, config=%s) as pid=%s", name, module, cls, config, process_id) process_cfg = deepcopy(CFG) if config: # Use provided config. Must be dict or DotDict if not isinstance(config, DotDict): config = DotDict(config) if config.get_safe("process.config_ref"): # Use a reference config_ref = config.get_safe("process.config_ref") log.info("Enhancing new process spawn config from ref=%s" % config_ref) matches = re.match(r'^([A-Za-z]+):([A-Za-z0-9]+)/(.+)$', config_ref) if matches: ref_type, ref_id, ref_ext = matches.groups() if ref_type == "resources": if self.container.has_capability(self.container.CCAP.RESOURCE_REGISTRY): try: obj = self.container.resource_registry.read(ref_id) if obj and hasattr(obj, ref_ext): ref_config = getattr(obj, ref_ext) if isinstance(ref_config, dict): dict_merge(process_cfg, ref_config, inplace=True) else: raise BadRequest("config_ref %s exists but not dict" % config_ref) else: raise BadRequest("config_ref %s - attribute not found" % config_ref) except NotFound as nf: log.warn("config_ref %s - object not found" % config_ref) raise else: log.error("Container missing RESOURCE_REGISTRY capability to resolve process config ref %s" % config_ref) else: raise BadRequest("Unknown reference type in: %s" % config_ref) dict_merge(process_cfg, config, inplace=True) if self.container.spawn_args: # Override config with spawn args dict_merge(process_cfg, self.container.spawn_args, inplace=True) #log.debug("spawn_process() pid=%s process_cfg=%s", process_id, process_cfg) # PROCESS TYPE. Determines basic process context (messaging, service interface) # One of the constants defined at the top of this file service_cls = named_any("%s.%s" % (module, cls)) process_type = get_safe(process_cfg, "process.type") or getattr(service_cls, "process_type", "service") process_start_mode = get_safe(config, "process.start_mode") process_instance = None # alert we have a spawning process, but we don't have the instance yet, so give the class instead (more accurate than name) self._call_proc_state_changed("%s.%s" % (module, cls), ProcessStateEnum.PENDING) try: # spawn service by type if process_type == SERVICE_PROCESS_TYPE: process_instance = self._spawn_service_process(process_id, name, module, cls, process_cfg) elif process_type == STREAM_PROCESS_TYPE: process_instance = self._spawn_stream_process(process_id, name, module, cls, process_cfg) elif process_type == AGENT_PROCESS_TYPE: process_instance = self._spawn_agent_process(process_id, name, module, cls, process_cfg) elif process_type == STANDALONE_PROCESS_TYPE: process_instance = self._spawn_standalone_process(process_id, name, module, cls, process_cfg) elif process_type == IMMEDIATE_PROCESS_TYPE: process_instance = self._spawn_immediate_process(process_id, name, module, cls, process_cfg) elif process_type == SIMPLE_PROCESS_TYPE: process_instance = self._spawn_simple_process(process_id, name, module, cls, process_cfg) else: raise BadRequest("Unknown process type: %s" % process_type) process_instance._proc_type = process_type self._register_process(process_instance, name) process_instance.errcause = "OK" log.info("ProcManager.spawn_process: %s.%s -> pid=%s OK", module, cls, process_id) if process_type == IMMEDIATE_PROCESS_TYPE: log.info('Terminating immediate process: %s', process_instance.id) self.terminate_process(process_instance.id) # terminate process also triggers TERMINATING/TERMINATED self._call_proc_state_changed(process_instance, ProcessStateEnum.EXITED) else: #Update local policies for the new process if self.container.has_capability(self.container.CCAP.GOVERNANCE_CONTROLLER): self.container.governance_controller.update_container_policies(process_instance, safe_mode=True) return process_instance.id except IonProcessError: errcause = process_instance.errcause if process_instance else "instantiating process" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) return None except Exception: errcause = process_instance.errcause if process_instance else "instantiating process" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) # trigger failed notification - catches problems in init/start self._call_proc_state_changed(process_instance, ProcessStateEnum.FAILED) raise
def apply_configuration(system_cfg, config_override): if not config_override: return from pyon.util.containers import dict_merge dict_merge(system_cfg, config_override, inplace=True)
def spawn_process(self, name=None, module=None, cls=None, config=None, process_id=None): """ Spawn a process within the container. Processes can be of different type. """ if process_id and not is_valid_identifier(process_id, ws_sub='_'): raise BadRequest("Given process_id %s is not a valid identifier" % process_id) # Generate a new process id if not provided # TODO: Ensure it is system-wide unique process_id = process_id or "%s.%s" % (self.container.id, self.proc_id_pool.get_id()) log.debug( "ProcManager.spawn_process(name=%s, module.cls=%s.%s) as pid=%s", name, module, cls, process_id) if not config: # Use system CFG. It has the command line args in it config = DictModifier(CFG) else: # Use provided config. Must be dict or DotDict if not isinstance(config, DotDict): config = DotDict(config) config = DictModifier(CFG, config) if self.container.spawn_args: # Override config with spawn args dict_merge(config, self.container.spawn_args, inplace=True) #log.debug("spawn_process() pid=%s config=%s", process_id, config) # PROCESS TYPE. Determines basic process context (messaging, service interface) # One of: service, stream_process, agent, simple, immediate service_cls = named_any("%s.%s" % (module, cls)) process_type = get_safe(config, "process.type") or getattr( service_cls, "process_type", "service") service_instance = None try: # spawn service by type if process_type == "service": service_instance = self._spawn_service_process( process_id, name, module, cls, config) elif process_type == "stream_process": service_instance = self._spawn_stream_process( process_id, name, module, cls, config) elif process_type == "agent": service_instance = self._spawn_agent_process( process_id, name, module, cls, config) elif process_type == "standalone": service_instance = self._spawn_standalone_process( process_id, name, module, cls, config) elif process_type == "immediate": service_instance = self._spawn_immediate_process( process_id, name, module, cls, config) elif process_type == "simple": service_instance = self._spawn_simple_process( process_id, name, module, cls, config) else: raise BadRequest("Unknown process type: %s" % process_type) service_instance._proc_type = process_type self._register_process(service_instance, name) service_instance.errcause = "OK" log.info("AppManager.spawn_process: %s.%s -> pid=%s OK", module, cls, process_id) if process_type == 'immediate': log.info('Terminating immediate process: %s', service_instance.id) self.terminate_process(service_instance.id) return service_instance.id except Exception: errcause = service_instance.errcause if service_instance else "instantiating service" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) raise
def _create_process_config(self, config): """ Prepare the config for the new process. Clone system config and apply process overrides. Support including config by reference of a resource attribute or object from object store. """ process_cfg = deepcopy(CFG) if config: # Use provided config. Must be dict or DotDict if not isinstance(config, DotDict): config = DotDict(config) if config.get_safe("process.config_ref"): # Use a reference config_ref = config.get_safe("process.config_ref") log.info("Enhancing new process spawn config from ref=%s" % config_ref) matches = re.match(r'^([A-Za-z]+):([A-Za-z0-9_\.]+)/(.*)$', config_ref) if matches: ref_type, ref_id, ref_ext = matches.groups() if ref_type == "resources": if self.container.has_capability(self.container.CCAP.RESOURCE_REGISTRY): try: obj = self.container.resource_registry.read(ref_id) if obj and hasattr(obj, ref_ext): ref_config = getattr(obj, ref_ext) if isinstance(ref_config, dict): dict_merge(process_cfg, ref_config, inplace=True) else: raise BadRequest("config_ref %s exists but not dict" % config_ref) else: raise BadRequest("config_ref %s - attribute not found" % config_ref) except NotFound as nf: log.warn("config_ref %s - object not found" % config_ref) raise else: log.error("Container missing RESOURCE_REGISTRY capability to resolve process config ref %s" % config_ref) elif ref_type == "objects": if self.container.has_capability(self.container.CCAP.OBJECT_STORE): try: obj = self.container.object_store.read_doc(ref_id) ref_config = obj if ref_ext: ref_config = get_safe(obj, ref_ext, None) if ref_config is None: raise BadRequest("config_ref %s - attribute not found" % config_ref) if isinstance(ref_config, dict): dict_merge(process_cfg, ref_config, inplace=True) else: raise BadRequest("config_ref %s exists but not dict" % config_ref) except NotFound as nf: log.warn("config_ref %s - object not found" % config_ref) raise else: log.error("Container missing OBJECT_STORE capability to resolve process config ref %s" % config_ref) else: raise BadRequest("Unknown reference type in: %s" % config_ref) dict_merge(process_cfg, config, inplace=True) if self.container.spawn_args: # Override config with spawn args dict_merge(process_cfg, self.container.spawn_args, inplace=True) #log.debug("spawn_process() pid=%s process_cfg=%s", process_id, process_cfg) return process_cfg
def prepare_container(): """ Walks through pyon initialization in a deterministic way and initializes Container. In particular make sure configuration is loaded in correct order and pycc startup arguments are considered. """ import threading threading.current_thread().name = "CC-Main" # SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults import pyon from pyon.core import bootstrap, config # Set global testing flag to False. We are running as capability container. This is NO TEST. bootstrap.testing = False # Set sysname if provided in startup argument if opts.sysname: bootstrap.set_sys_name(opts.sysname) # Trigger any initializing default logic in get_sys_name bootstrap.get_sys_name() command_line_config = kwargs # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores. bootstrap_config = None # This holds the new CFG object for pyon. Build it up in proper sequence and conditions. pyon_config = config.read_standard_configuration() # Load config override if provided. Supports variants literal and list of paths config_override = None if opts.config: if '{' in opts.config: # Variant 1: Dict of config values try: eval_value = ast.literal_eval(opts.config) config_override = eval_value except ValueError: raise Exception("Value error in config arg '%s'" % opts.config) else: # Variant 2: List of paths from pyon.util.config import Config config_override = Config([opts.config]).data # Determine bootstrap_config if opts.config_from_directory: # Load minimal bootstrap config if option "config_from_directory" bootstrap_config = config.read_local_configuration( ['res/config/pyon_min_boot.yml']) config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config else: # Otherwise: Set to standard set of local config files plus command line overrides bootstrap_config = pyon_config.copy() config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) # Override sysname from config file or command line if not opts.sysname and bootstrap_config.get_safe("system.name", None): new_sysname = bootstrap_config.get_safe("system.name") bootstrap.set_sys_name(new_sysname) # Delete sysname datastores if option "force_clean" is set if opts.force_clean: from pyon.datastore import clear_couch_util print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name( ) clear_couch_util.clear_couch(bootstrap_config, prefix=bootstrap.get_sys_name()) # If auto_bootstrap, load config and interfaces into directory # Note: this is idempotent and will not alter anything if this is not the first container to run if bootstrap_config.system.auto_bootstrap: print "pycc: auto_bootstrap=True." stored_config = pyon_config.copy() config.apply_configuration(stored_config, config_override) config.apply_configuration(stored_config, command_line_config) config.auto_bootstrap_config(bootstrap_config, system_cfg=stored_config) # Determine the final pyon_config # - Start from standard config already set (pyon.yml + local YML files) # - Optionally load config from directory if opts.config_from_directory: config.apply_remote_config(pyon_config) # - Last apply any separate command line config overrides config.apply_configuration(pyon_config, config_override) config.apply_configuration(pyon_config, command_line_config) # Load logging override config if provided. Supports variants literal and path. logging_config_override = None if opts.logcfg: if '{' in opts.logcfg: # Variant 1: Value is dict of config values try: eval_value = ast.literal_eval(opts.logcfg) logging_config_override = eval_value except ValueError: raise Exception("Value error in logcfg arg '%s'" % opts.logcfg) else: # Variant 2: Value is path to YAML file containing config values pyon.DEFAULT_LOGGING_PATHS.append(opts.logcfg) # Also set the immediate flag, but only if specified - it is an override if opts.immediate: dict_merge(pyon_config, {'system': {'immediate': True}}, True) # Bootstrap pyon's core. Load configuration etc. bootstrap.bootstrap_pyon( logging_config_override=logging_config_override, pyon_cfg=pyon_config) # Auto-bootstrap interfaces # @WARN: This currently imports ALL modules, executing ALL static initializers as side effect!!!!!!! if bootstrap_config.system.auto_bootstrap: config.auto_bootstrap_interfaces(bootstrap_config) if opts.no_container: print "pycc: no_container=True. Stopping here." return None # Create the container instance from pyon.container.cc import Container container = Container(*args, **command_line_config) return container
def prepare_container(): """ Walks through pyon initialization in a deterministic way and initializes Container. In particular make sure configuration is loaded in correct order and pycc startup arguments are considered. """ import threading threading.current_thread().name = "CC-Main" # SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults import pyon from pyon.core import bootstrap, config # Set global testing flag to False. We are running as capability container. This is NO TEST. bootstrap.testing = False # Set sysname if provided in startup argument if opts.sysname: bootstrap.set_sys_name(opts.sysname) # Trigger any initializing default logic in get_sys_name bootstrap.get_sys_name() command_line_config = kwargs # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores. bootstrap_config = None # This holds the new CFG object for pyon. Build it up in proper sequence and conditions. pyon_config = config.read_standard_configuration() # Load config override if provided. Supports variants literal and list of paths config_override = None if opts.config: if '{' in opts.config: # Variant 1: Dict of config values try: eval_value = ast.literal_eval(opts.config) config_override = eval_value except ValueError: raise Exception("Value error in config arg '%s'" % opts.config) else: # Variant 2: List of paths from pyon.util.config import Config config_override = Config([opts.config]).data # Determine bootstrap_config if opts.config_from_directory: # Load minimal bootstrap config if option "config_from_directory" bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml']) config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config else: # Otherwise: Set to standard set of local config files plus command line overrides bootstrap_config = pyon_config.copy() config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) # Override sysname from config file or command line if not opts.sysname and bootstrap_config.get_safe("system.name", None): new_sysname = bootstrap_config.get_safe("system.name") bootstrap.set_sys_name(new_sysname) # Delete sysname datastores if option "force_clean" is set if opts.force_clean: from pyon.datastore import clear_couch_util print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name() clear_couch_util.clear_couch(bootstrap_config, prefix=bootstrap.get_sys_name()) # If auto_bootstrap, load config and interfaces into directory # Note: this is idempotent and will not alter anything if this is not the first container to run if bootstrap_config.system.auto_bootstrap: print "pycc: auto_bootstrap=True." stored_config = pyon_config.copy() config.apply_configuration(stored_config, config_override) config.apply_configuration(stored_config, command_line_config) config.auto_bootstrap_config(bootstrap_config, system_cfg=stored_config) # Determine the final pyon_config # - Start from standard config already set (pyon.yml + local YML files) # - Optionally load config from directory if opts.config_from_directory: config.apply_remote_config(pyon_config) # - Last apply any separate command line config overrides config.apply_configuration(pyon_config, config_override) config.apply_configuration(pyon_config, command_line_config) # Load logging override config if provided. Supports variants literal and path. logging_config_override = None if opts.logcfg: if '{' in opts.logcfg: # Variant 1: Value is dict of config values try: eval_value = ast.literal_eval(opts.logcfg) logging_config_override = eval_value except ValueError: raise Exception("Value error in logcfg arg '%s'" % opts.logcfg) else: # Variant 2: Value is path to YAML file containing config values pyon.DEFAULT_LOGGING_PATHS.append(opts.logcfg) # Also set the immediate flag, but only if specified - it is an override if opts.immediate: dict_merge(pyon_config, {'system':{'immediate':True}}, True) # Bootstrap pyon's core. Load configuration etc. bootstrap.bootstrap_pyon( logging_config_override=logging_config_override, pyon_cfg=pyon_config) # Auto-bootstrap interfaces # @WARN: This currently imports ALL modules, executing ALL static initializers as side effect!!!!!!! if bootstrap_config.system.auto_bootstrap: config.auto_bootstrap_interfaces(bootstrap_config) if opts.no_container: print "pycc: no_container=True. Stopping here." return None # Create the container instance from pyon.container.cc import Container container = Container(*args, **command_line_config) return container