Esempio n. 1
0
    def _load_capabilities(self):
        self._cap_initialized = []  # List of capability constants initialized in container
        self._capabilities = []     # List of capability constants active in container
        self._cap_instances = {}    # Dict mapping capability->manager instance

        self._cap_definitions = Config(["res/config/container_capabilities.yml"]).data['capabilities']

        profile_filename = CFG.get_safe("container.profile", "development")
        if not profile_filename.endswith(".yml"):
            profile_filename = "res/profile/%s.yml" % profile_filename
        log.info("Loading CC capability profile from file: %s", profile_filename)
        profile_cfg = Config([profile_filename]).data
        if not isinstance(profile_cfg, dict) or profile_cfg['type'] != "profile" or not "profile" in profile_cfg:
            raise ContainerError("Container capability profile invalid: %s" % profile_filename)

        self.cap_profile = profile_cfg['profile']

        if "capabilities" in self.cap_profile and self.cap_profile['capabilities']:
            dict_merge(self._cap_definitions, self.cap_profile['capabilities'], True)

        CCAP.clear()
        cap_list = self._cap_definitions.keys()
        CCAP.update(zip(cap_list, cap_list))

        if "config" in self.cap_profile and self.cap_profile['config']:
            log.info("Container CFG was changed based on profile: %s", profile_filename)
Esempio n. 2
0
    def start_app(self, appdef=None, config=None):
        """
        @brief Start an app from an app definition.
        Note: apps can come in one of 2 variants:
        1 processapp: In-line defined process to be started
        2 regular app: Full app definition
        """
        log.debug("AppManager.start_app(appdef=%s) ..." % appdef)

        appdef = DotDict(appdef)

        if 'config' in appdef:
            app_cfg = appdef.config.copy()
            if config:
                dict_merge(app_cfg, config, inplace=True)
            config = app_cfg

        if 'processapp' in appdef:
            # Case 1: Appdef contains definition of process to start
            name, module, cls = appdef.processapp
            try:
                pid = self.container.spawn_process(name, module, cls, config)
                appdef._pid = pid
                self.apps.append(appdef)
            except Exception, ex:
                log.exception("Appl %s start from processapp failed" %
                              appdef.name)
Esempio n. 3
0
    def start_app(self, appdef=None, config=None):
        """
        @brief Start an app from an app definition.
        Note: apps can come in one of 2 variants:
        1 processapp: In-line defined process to be started
        2 regular app: Full app definition
        """
        log.debug("AppManager.start_app(appdef=%s) ..." % appdef)

        appdef = DotDict(appdef)

        if 'config' in appdef:
            app_cfg = appdef.config.copy()
            if config:
                dict_merge(app_cfg, config, inplace=True)
            config = app_cfg

        if 'processapp' in appdef:
            # Case 1: Appdef contains definition of process to start
            name, module, cls = appdef.processapp
            try:
                pid = self.container.spawn_process(name, module, cls, config)
                appdef._pid = pid
                self.apps.append(appdef)
            except Exception, ex:
                log.exception("Appl %s start from processapp failed" % appdef.name)
Esempio n. 4
0
    def _get_execution_engine_config(self):
        ee_base_cfg = CFG.get_safe("container.execution_engine") or {}
        if ee_base_cfg.get("type", None) != "scioncc":
            raise ContainerConfigError("Execution engine config invalid: %s", ee_base_cfg)

        ee_cfg = deepcopy(ee_base_cfg)

        # If we are a child process, merge in child config override
        proc_name = multiprocessing.current_process().name
        ee_cfg["container"] = dict(child_proc_name=proc_name, is_child=False)
        child_cfgs = ee_base_cfg.get("child_configs", None) or {}
        if proc_name.startswith("Container-child-"):
            ee_cfg["container"]["is_child"] = True
            if proc_name in child_cfgs:
                log.info("Applying execution engine config override for child: %s", proc_name)
                dict_merge(ee_cfg, child_cfgs[proc_name], inplace=True)
            else:
                for cfg_name, ch_cfg in child_cfgs.iteritems():
                    pattern = ch_cfg.get("name_pattern", None)
                    if pattern and re.match(pattern, proc_name):
                        log.info("Applying execution engine config override %s for child: %s", cfg_name, proc_name)
                        dict_merge(ee_cfg, ch_cfg, inplace=True)
                        break

        ee_cfg.pop("child_configs", None)
        return ee_cfg
Esempio n. 5
0
File: cc.py Progetto: wfrench/pyon
    def __init__(self, *args, **kwargs):
        BaseContainerAgent.__init__(self, *args, **kwargs)

        # set id and name (as they are set in base class call)
        self.id = string.replace('%s_%d' % (os.uname()[1], os.getpid()), ".", "_")
        self.name = "cc_agent_%s" % self.id

        Container.instance = self

        # TODO: Bug: Replacing CFG instance not work because references are already public. Update directly
        dict_merge(CFG, kwargs)
        from pyon.core import bootstrap
        bootstrap.sys_name = CFG.system.name or bootstrap.sys_name
        log.debug("Container (sysname=%s) initializing ..." % bootstrap.sys_name)

        # Keep track of the overrides from the command-line, so they can trump app/rel file data
        self.spawn_args = DictModifier(CFG, kwargs)

        # Load object and service registry etc.
        bootstrap_pyon()

        # Create this Container's specific ExchangeManager instance
        self.ex_manager = ExchangeManager(self)

        # Create this Container's specific ProcManager instance
        self.proc_manager = ProcManager(self)

        # Create this Container's specific AppManager instance
        self.app_manager = AppManager(self)

        # DatastoreManager - controls access to Datastores (both mock and couch backed)
        self.datastore_manager = DatastoreManager()
        
        log.debug("Container initialized, OK.")
Esempio n. 6
0
    def start_rel(self, rel=None, config=None):
        """
        @brief Recurse over the rel and start apps defined there.
        Note: apps in a rel file can come in one of 2 forms:
        1 processapp: In-line defined process to be started as app
        2 app file: Reference to an app definition in an app file
        If the rel file provides an app config block, it is provided to spawn the process.
        Any given function config dict is merged on top of this.
        """
        log.debug("AppManager.start_rel(rel=%s) ...", rel)

        if rel is None:
            return

        if self.use_pd:
            log.info("Sending rel file to PD")
            import json
            rel_def = json.loads(json.dumps(rel))     # HACK to get rid of OrderedDict (not serializable)
            cmd_res = self.pd_client.start_rel_blocking(rel_def, timeout=None)
            return cmd_res

        for rel_app_cfg in rel.apps:
            name = rel_app_cfg.name
            log.debug("app definition in rel: %s" % str(rel_app_cfg))

            if 'processapp' in rel_app_cfg:
                # Case 1: Rel contains definition of process to start as app
                name, module, cls = rel_app_cfg.processapp

                rel_cfg = None
                if 'config' in rel_app_cfg:
                    rel_cfg = deepcopy(rel_app_cfg.config)
                    if config:
                        dict_merge(rel_cfg, config, inplace=True)

                if 'replicas' in rel_app_cfg:
                    proc_replicas = int(rel_app_cfg["replicas"])
                    if self.max_proc_replicas > 0:
                        if proc_replicas > self.max_proc_replicas:
                            log.info("Limiting number of proc replicas to %s from %s", self.max_proc_replicas, proc_replicas)
                        proc_replicas = min(proc_replicas, self.max_proc_replicas)
                    if proc_replicas < 1 or proc_replicas > 100:
                        log.warn("Invalid number of process replicas: %s", proc_replicas)
                        proc_replicas = 1
                    for i in xrange(proc_replicas):
                        proc_name = "%s.%s" % (name, i) if i else name
                        self.container.spawn_process(proc_name, module, cls, rel_cfg)
                else:
                    self.container.spawn_process(name, module, cls, rel_cfg)
                self.apps.append(DotDict(type="application", name=name, processapp=rel_app_cfg.processapp))

            else:
                # Case 2: Rel contains reference to app file to start
                app_file_path = 'res/apps/%s.yml' % (name)
                rel_cfg = rel_app_cfg.get('config', None)
                if config:
                    dict_merge(rel_cfg, config, inplace=True)
                self.start_app_from_url(app_file_path, config=rel_cfg)
Esempio n. 7
0
    def __init__(self, *args, **kwargs):
        BaseContainerAgent.__init__(self, *args, **kwargs)

        self._is_started = False

        # set id and name (as they are set in base class call)
        self.id = string.replace('%s_%d' % (os.uname()[1], os.getpid()), ".",
                                 "_")
        self.name = "cc_agent_%s" % self.id

        Container.instance = self

        # TODO: Bug: Replacing CFG instance not work because references are already public. Update directly
        dict_merge(CFG, kwargs, inplace=True)
        from pyon.core import bootstrap
        bootstrap.container_instance = self
        bootstrap.assert_configuration(CFG)
        log.debug("Container (sysname=%s) initializing ..." %
                  bootstrap.get_sys_name())

        # Keep track of the overrides from the command-line, so they can trump app/rel file data
        self.spawn_args = kwargs

        # Load object and service registry etc.
        bootstrap_pyon()

        # Create this Container's specific ExchangeManager instance
        self.ex_manager = ExchangeManager(self)

        # Create this Container's specific ProcManager instance
        self.proc_manager = ProcManager(self)

        # Create this Container's specific AppManager instance
        self.app_manager = AppManager(self)

        # DatastoreManager - controls access to Datastores (both mock and couch backed)
        self.datastore_manager = DatastoreManager()

        # File System - Interface to the OS File System, using correct path names and setups
        self.file_system = FileSystem(CFG)

        # Governance Controller - manages the governance related interceptors
        self.governance_controller = GovernanceController(self)

        # sFlow manager - controls sFlow stat emission
        self.sflow_manager = SFlowManager(self)

        # Coordinates the container start
        self._is_started = False
        self._capabilities = []
        self._status = "INIT"

        # protection for when the container itself is used as a Process for clients
        self.container = self

        log.debug("Container initialized, OK.")
Esempio n. 8
0
    def do_work(container):
        """
        Performs initial startup actions with the container as requested in arguments.
        Then remains in container shell or infinite wait until container stops.
        Returns when container should stop. Raises an exception if anything failed.
        """
        if opts.proc:
            # Run a one-off process (with the -x argument)
            mod, proc = opts.proc.rsplit('.', 1)
            print "pycc: Starting process %s" % opts.proc
            container.spawn_process(proc,
                                    mod,
                                    proc,
                                    config={'process': {
                                        'type': 'immediate'
                                    }})
            # And end
            return

        if opts.rel:
            # Start a rel file
            start_ok = container.start_rel_from_url(opts.rel)
            if not start_ok:
                raise Exception("Cannot start deploy file '%s'" % opts.rel)

        if opts.mx:
            container.spawn_process("ContainerUI", "ion.core.containerui",
                                    "ContainerUI")
            print "pycc: Container UI started ... listening on http://localhost:8080"

        if opts.signalparent:
            import os
            import signal
            print 'pycc: Signal parent pid %d that pycc pid %d service start process is complete...' % (
                os.getppid(), os.getpid())
            os.kill(os.getppid(), signal.SIGUSR1)

            def is_parent_gone():
                while os.getppid() != 1:
                    gevent.sleep(1)
                print 'pycc: Now I am an orphan ... notifying serve_forever to stop'
                os.kill(os.getpid(), signal.SIGINT)

            import gevent
            ipg = gevent.spawn(is_parent_gone)
            from pyon.util.containers import dict_merge
            from pyon.public import CFG
            dict_merge(CFG, {'system': {'watch_parent': ipg}}, True)
        if not opts.noshell and not opts.daemon:
            # Keep container running while there is an interactive shell
            from pyon.container.shell_api import get_shell_api
            setup_ipython(get_shell_api(container))
        else:
            # Keep container running until process terminated
            container.serve_forever()
Esempio n. 9
0
File: cc.py Progetto: ooici-dm/pyon
    def __init__(self, *args, **kwargs):
        BaseContainerAgent.__init__(self, *args, **kwargs)

        self._is_started = False

        # set id and name (as they are set in base class call)
        self.id = string.replace('%s_%d' % (os.uname()[1], os.getpid()), ".", "_")
        self.name = "cc_agent_%s" % self.id

        Container.instance = self

        # TODO: Bug: Replacing CFG instance not work because references are already public. Update directly
        dict_merge(CFG, kwargs, inplace=True)
        from pyon.core import bootstrap
        bootstrap.container_instance = self
        bootstrap.assert_configuration(CFG)
        log.debug("Container (sysname=%s) initializing ..." % bootstrap.get_sys_name())

        # Keep track of the overrides from the command-line, so they can trump app/rel file data
        self.spawn_args = kwargs

        # Load object and service registry etc.
        bootstrap_pyon()

        # Create this Container's specific ExchangeManager instance
        self.ex_manager = ExchangeManager(self)

        # Create this Container's specific ProcManager instance
        self.proc_manager = ProcManager(self)

        # Create this Container's specific AppManager instance
        self.app_manager = AppManager(self)

        # DatastoreManager - controls access to Datastores (both mock and couch backed)
        self.datastore_manager = DatastoreManager()

        # File System - Interface to the OS File System, using correct path names and setups
        self.file_system = FileSystem(CFG)

        # Governance Controller - manages the governance related interceptors
        self.governance_controller = GovernanceController(self)

        # sFlow manager - controls sFlow stat emission
        self.sflow_manager = SFlowManager(self)

        # Coordinates the container start
        self._is_started = False
        self._capabilities = []
        self._status = "INIT"

        # protection for when the container itself is used as a Process for clients
        self.container = self

        log.debug("Container initialized, OK.")
Esempio n. 10
0
    def test_dict_merge(self):
        # dict_merge(base, upd, inplace=False):
        org_dict = {
            "a": "str_a",
            "d": {
                "d-x": 1,
                "d-y": None,
                "d-d": {
                    "d-d-1": 1,
                    "d-d-2": 2
                }
            }
        }

        base_dict = copy.deepcopy(org_dict)
        dd = DictDiffer(base_dict, org_dict)
        self.assertTrue(len(base_dict), 2)
        self.assertTrue(len(dd.unchanged()), len(base_dict))

        # Case 1: Add new value
        delta_dict = {"c": "NEW_C"}
        mod_dict = dict_merge(base_dict, delta_dict)
        dd = DictDiffer(base_dict, org_dict)
        self.assertTrue(len(base_dict), 2)
        self.assertTrue(len(dd.unchanged()), len(base_dict))

        dd = DictDiffer(mod_dict, org_dict)
        self.assertTrue(len(mod_dict), 3)
        self.assertTrue(len(dd.unchanged()), len(org_dict))
        self.assertTrue(dd.added(), 1)

        # Case 2: Change simple type value
        delta_dict = {"a": 5}
        base_dict = copy.deepcopy(org_dict)
        mod_dict = dict_merge(base_dict, delta_dict)

        dd = DictDiffer(mod_dict, org_dict)
        self.assertTrue(len(mod_dict), len(org_dict))
        self.assertTrue(len(dd.unchanged()), len(org_dict) - 1)
        self.assertTrue(len(dd.changed()), 1)
        self.assertTrue(mod_dict['a'], 5)

        # Case 3: Add new value on lower level
        delta_dict = {"d": {"new": "NEW_ENTRY"}}
        base_dict = copy.deepcopy(org_dict)
        mod_dict = dict_merge(base_dict, delta_dict)

        dd = DictDiffer(mod_dict, org_dict)
        self.assertTrue(len(mod_dict), len(org_dict))
        self.assertTrue(len(mod_dict['d']), len(org_dict['d']) + 1)
        self.assertTrue(mod_dict['d']['new'], "NEW_ENTRY")
        dd = DictDiffer(mod_dict['d'], org_dict['d'])
        self.assertTrue(len(dd.unchanged()), len(org_dict['d']))
        self.assertTrue(dd.added(), 1)
Esempio n. 11
0
def apply_profile_configuration(system_cfg, bootstrap_config):
    profile_filename = bootstrap_config.get_safe("container.profile", None)
    if not profile_filename:
        return
    if not profile_filename.endswith(".yml"):
        profile_filename = "res/profile/%s.yml" % profile_filename
    from pyon.util.config import Config
    profile_cfg = Config([profile_filename]).data
    config_override = profile_cfg.get_safe("profile.config")
    if config_override and isinstance(config_override, dict):
        from pyon.util.containers import dict_merge
        dict_merge(system_cfg, config_override, inplace=True)
Esempio n. 12
0
def apply_profile_configuration(system_cfg, bootstrap_config):
    profile_filename = bootstrap_config.get_safe("container.profile", None)
    if not profile_filename:
        return
    if not profile_filename.endswith(".yml"):
        profile_filename = "res/profile/%s.yml" % profile_filename
    from pyon.util.config import Config
    profile_cfg = Config([profile_filename]).data
    config_override = profile_cfg.get_safe("profile.config")
    if config_override and isinstance(config_override, dict):
        from pyon.util.containers import dict_merge
        dict_merge(system_cfg, config_override, inplace=True)
Esempio n. 13
0
    def _call_plugins(self, method, process, config, **kwargs):
        bootstrap_plugins = config.get_safe("bootstrap_plugins", None)
        if bootstrap_plugins is None:
            log.warn("Bootstrapper called without bootstrap_plugins config")

        # Finding the system actor ID. If found, construct call context headers.
        # This may be called very early in bootstrap with no system actor yet existing
        system_actor = get_system_actor()
        if system_actor:
            actor_headers = get_system_actor_header(system_actor)
        else:
            # Use default actor headers, not roles.
            actor_headers = build_actor_header()

        # Set the call context of the current process
        with process.push_context(actor_headers):

            for plugin_info in bootstrap_plugins:
                plugin_mod, plugin_cls = plugin_info.get("plugin", [None,None])
                plugin_cfg = plugin_info.get("config", None)
                plugin_cfg = dict_merge(config, plugin_cfg) if plugin_cfg is not None else config

                try:
                    log.info("Bootstrapping plugin %s.%s ...", plugin_mod, plugin_cls)
                    plugin = for_name(plugin_mod, plugin_cls)
                    plugin_func = getattr(plugin, method)
                    plugin_func(process, plugin_cfg, **kwargs)
                except AbortBootstrap as abort:
                    raise
                except Exception as ex:
                    log.exception("Error bootstrapping plugin %s.%s", plugin_mod, plugin_cls)
Esempio n. 14
0
    def _call_plugins(self, method, process, config, **kwargs):
        bootstrap_plugins = config.get_safe("bootstrap_plugins", None)
        if bootstrap_plugins is None:
            log.warn("Bootstrapper called without bootstrap_plugins config")

        # Finding the system actor ID. If found, construct call context headers.
        # This may be called very early in bootstrap with no system actor yet existing
        system_actor, _ = process.container.resource_registry.find_resources(
            RT.ActorIdentity, name=self.CFG.system.system_actor, id_only=False)

        actor_headers = get_system_actor_header(
            system_actor[0] if system_actor else None)

        # Set the call context of the current process
        with process.push_context(actor_headers):

            for plugin_info in bootstrap_plugins:
                plugin_mod, plugin_cls = plugin_info.get(
                    "plugin", [None, None])
                plugin_cfg = plugin_info.get("config", None)
                plugin_cfg = dict_merge(
                    config, plugin_cfg) if plugin_cfg is not None else config

                try:
                    log.info("Bootstrapping plugin %s.%s ...", plugin_mod,
                             plugin_cls)
                    plugin = for_name(plugin_mod, plugin_cls)
                    plugin_func = getattr(plugin, method)
                    plugin_func(process, plugin_cfg, **kwargs)
                except AbortBootstrap as abort:
                    raise
                except Exception as ex:
                    log.exception("Error bootstrapping plugin %s.%s",
                                  plugin_mod, plugin_cls)
Esempio n. 15
0
    def _call_plugins(self, method, process, config, **kwargs):
        bootstrap_plugins = config.get_safe("bootstrap_plugins", None)
        if bootstrap_plugins is None:
            log.warn("Bootstrapper called without bootstrap_plugins config")

        # Finding the system actor ID. If found, construct call context headers.
        # This may be called very early in bootstrap with no system actor yet existing
        system_actor = get_system_actor()
        if system_actor:
            actor_headers = get_system_actor_header(system_actor)
        else:
            # Use default actor headers, not roles.
            actor_headers = build_actor_header()

        # Set the call context of the current process
        with process.push_context(actor_headers):

            for plugin_info in bootstrap_plugins:
                plugin_mod, plugin_cls = plugin_info.get(
                    "plugin", [None, None])
                plugin_cfg = plugin_info.get("config", None)
                plugin_cfg = dict_merge(
                    config, plugin_cfg) if plugin_cfg is not None else config

                try:
                    log.info("Bootstrapping plugin %s.%s ...", plugin_mod,
                             plugin_cls)
                    plugin = for_name(plugin_mod, plugin_cls)
                    plugin_func = getattr(plugin, method)
                    plugin_func(process, plugin_cfg, **kwargs)
                except AbortBootstrap as abort:
                    raise
                except Exception as ex:
                    log.exception("Error bootstrapping plugin %s.%s",
                                  plugin_mod, plugin_cls)
    def _generate_skeleton_config_block(self):
        log.info("Generating skeleton config block for %s", self.agent_instance_obj.name)

        # merge the agent config into the default config
        agent_config = dict_merge(self._get_agent().agent_default_config, self.agent_instance_obj.agent_config, True)

        org_obj = self._generate_org()

        # Create agent_config.
        agent_config['instance_id']        = self.agent_instance_obj._id
        agent_config['instance_name']        = self.agent_instance_obj.name
        agent_config['org_governance_name']  = org_obj.org_governance_name if org_obj else ''
        agent_config['provider_id']          = org_obj._id if org_obj else ''
        agent_config['actor_id']             = self.actor_id
        agent_config['device_type']          = self._generate_device_type()
        agent_config['driver_config']        = self._generate_driver_config()
        agent_config['stream_config']        = self._generate_stream_config()
        agent_config['agent']                = self._generate_agent_config()
        agent_config['aparam_alerts_config'] = self._generate_alerts_config()
        agent_config['startup_config']       = self._generate_startup_config()
        agent_config['children']             = self._generate_children()

        log.info("DONE generating skeleton config block for %s", self.agent_instance_obj.name)

        return agent_config
Esempio n. 17
0
    def _call_plugins(self, method, process, config, **kwargs):
        bootstrap_plugins = config.get_safe("bootstrap_plugins", None)
        if bootstrap_plugins is None:
            log.warn("Bootstrapper called without bootstrap_plugins config")

        # Finding the system actor ID. If found, construct call context headers.
        # This may be called very early in bootstrap with no system actor yet existing
        system_actor, _ = process.container.resource_registry.find_resources(
            RT.ActorIdentity, name=self.CFG.system.system_actor, id_only=True
        )
        system_actor_id = system_actor[0] if system_actor else "anonymous"

        actor_headers = {
            "ion-actor-id": system_actor_id,
            "ion-actor-roles": {"ION": ["ION_MANAGER", "ORG_MANAGER"]} if system_actor else {},
        }

        # Set the call context of the current process
        with process.push_context(actor_headers):

            for plugin_info in bootstrap_plugins:
                plugin_mod, plugin_cls = plugin_info.get("plugin", [None, None])
                plugin_cfg = plugin_info.get("config", None)
                plugin_cfg = dict_merge(config, plugin_cfg) if plugin_cfg is not None else config

                try:
                    log.info("Bootstrapping plugin %s.%s ...", plugin_mod, plugin_cls)
                    plugin = for_name(plugin_mod, plugin_cls)
                    plugin_func = getattr(plugin, method)
                    plugin_func(process, plugin_cfg, **kwargs)
                except AbortBootstrap as abort:
                    raise
                except Exception as ex:
                    log.exception("Error bootstrapping plugin %s.%s", plugin_mod, plugin_cls)
    def _generate_driver_config(self):
        log.debug("_generate_driver_config for %s",
                  self.agent_instance_obj.name)
        # get default config
        driver_config = super(ExternalDatasetAgentConfigurationBuilder,
                              self)._generate_driver_config()

        agent_instance_obj = self.agent_instance_obj
        agent_obj = self._get_agent()

        parser_cfg = copy.deepcopy(agent_obj.parser_default_config)
        poller_cfg = copy.deepcopy(agent_obj.poller_default_config)

        # Create driver config.
        base_driver_config = {
            'parser': {
                'uri': agent_obj.parser_uri,
                'module': agent_obj.parser_module,
                'class': agent_obj.parser_class,
                'config': parser_cfg,
            },
            'poller': {
                'uri': agent_obj.poller_uri,
                'module': agent_obj.poller_module,
                'class': agent_obj.poller_class,
                'config': poller_cfg,
            },
        }

        res_driver_config = dict_merge(base_driver_config, driver_config)

        return res_driver_config
    def _generate_driver_config(self):
        log.debug("_generate_driver_config for %s", self.agent_instance_obj.name)
        # get default config
        driver_config = super(ExternalDatasetAgentConfigurationBuilder, self)._generate_driver_config()

        agent_instance_obj = self.agent_instance_obj
        agent_obj = self._get_agent()

        parser_cfg = copy.deepcopy(agent_obj.parser_default_config)
        poller_cfg = copy.deepcopy(agent_obj.poller_default_config)

        # Create driver config.
        base_driver_config = {
            'parser': {
                'uri': agent_obj.parser_uri,
                'module': agent_obj.parser_module,
                'class': agent_obj.parser_class,
                'config': parser_cfg,
            },
            'poller': {
                'uri': agent_obj.poller_uri,
                'module': agent_obj.poller_module,
                'class': agent_obj.poller_class,
                'config': poller_cfg,
            },
        }

        res_driver_config = dict_merge(base_driver_config, driver_config)

        return res_driver_config
Esempio n. 20
0
    def _generate_skeleton_config_block(self):
        log.info("Generating skeleton config block for %s",
                 self.agent_instance_obj.name)

        # merge the agent config into the default config
        agent_config = dict_merge(self._get_agent().agent_default_config,
                                  self.agent_instance_obj.agent_config, True)

        org_obj = self._generate_org()

        # Create agent_config.
        agent_config['instance_id'] = self.agent_instance_obj._id
        agent_config['instance_name'] = self.agent_instance_obj.name
        agent_config[
            'org_governance_name'] = org_obj.org_governance_name if org_obj else ''
        agent_config['provider_id'] = org_obj._id if org_obj else ''
        agent_config['actor_id'] = self.actor_id
        agent_config['device_type'] = self._generate_device_type()
        agent_config['driver_config'] = self._generate_driver_config()
        agent_config['stream_config'] = self._generate_stream_config()
        agent_config['agent'] = self._generate_agent_config()
        agent_config['aparam_alerts_config'] = self._generate_alerts_config()
        agent_config['startup_config'] = self._generate_startup_config()
        agent_config['children'] = self._generate_children()

        log.info("DONE generating skeleton config block for %s",
                 self.agent_instance_obj.name)

        return agent_config
Esempio n. 21
0
    def do_work(container):
        """
        Performs initial startup actions with the container as requested in arguments.
        Then remains in container shell or infinite wait until container stops.
        Returns when container should stop. Raises an exception if anything failed.
        """
        if opts.proc:
            # Run a one-off process (with the -x argument)
            mod, proc = opts.proc.rsplit('.', 1)
            print "pycc: Starting process %s" % opts.proc
            container.spawn_process(proc, mod, proc, config={'process':{'type':'immediate'}})
            # And end
            return

        if opts.rel:
            # Start a rel file
            start_ok = container.start_rel_from_url(opts.rel)
            if not start_ok:
                raise Exception("Cannot start deploy file '%s'" % opts.rel)

        if opts.mx:
            container.spawn_process("ContainerUI", "ion.core.containerui", "ContainerUI")
            print "pycc: Container UI started ... listening on http://localhost:8080"

        if opts.signalparent:
            import os
            import signal
            print 'pycc: Signal parent pid %d that pycc pid %d service start process is complete...' % (os.getppid(), os.getpid())
            os.kill(os.getppid(), signal.SIGUSR1)

            def is_parent_gone():
                while os.getppid() != 1:
                    gevent.sleep(1)
                print 'pycc: Now I am an orphan ... notifying serve_forever to stop'
                os.kill(os.getpid(), signal.SIGINT)
            import gevent
            ipg = gevent.spawn(is_parent_gone)
            from pyon.util.containers import dict_merge
            from pyon.public import CFG
            dict_merge(CFG, {'system':{'watch_parent': ipg}}, True)
        if not opts.noshell and not opts.daemon:
            # Keep container running while there is an interactive shell
            from pyon.container.shell_api import get_shell_api
            setup_ipython(get_shell_api(container))
        else:
            # Keep container running until process terminated
            container.serve_forever()
Esempio n. 22
0
    def start_app(self, appdef=None, config=None):
        """
        @brief Start an app from an app definition.
        Note: apps can come in one of 2 variants:
        1 processapp: In-line defined process to be started
        2 regular app: Full app definition
        """
        log.debug("AppManager.start_app(appdef=%s) ..." % appdef)

        appdef = DotDict(appdef)

        if 'config' in appdef:
            app_cfg = deepcopy(appdef.config)
            if config:
                dict_merge(app_cfg, config, inplace=True)
            config = app_cfg

        if 'processapp' in appdef:
            # Case 1: Appdef contains definition of process to start
            name, module, cls = appdef.processapp
            try:
                pid = self.container.spawn_process(name, module, cls, config)
                appdef._pid = pid
                self.apps.append(appdef)
            except Exception:
                log.exception("Appl %s start from processapp failed" %
                              appdef.name)
        else:
            # Case 2: Appdef contains full app start params
            modpath = appdef.mod
            try:
                mod = named_any(modpath)
                appdef._mod_loaded = mod

                # Start the app
                supid, state = mod.start(self.container, START_PERMANENT,
                                         appdef, config)
                appdef._supid = supid
                appdef._state = state

                log.debug("App '%s' started. Root sup-id=%s" %
                          (appdef.name, supid))

                self.apps.append(appdef)
            except Exception:
                log.exception("Appl %s start from appdef failed" % appdef.name)
Esempio n. 23
0
 def acquire_data(self, streaming_args=None):
     if self.current_state != self.AGENTSTATE_CONNECTED:
         raise BadRequest("Illegal agent state: %s" % self.current_state)
     try:
         args = dict_merge(self.agent_config, streaming_args) if streaming_args else self.agent_config
         res = self.on_acquire_data(args)
     except Exception:
         self.current_state = self.AGENTSTATE_ERROR
         raise
Esempio n. 24
0
    def test_dict_merge(self):
        # dict_merge(base, upd, inplace=False):
        org_dict = {"a":"str_a", "d": {"d-x": 1, "d-y": None, "d-d": {"d-d-1": 1, "d-d-2": 2}}}

        base_dict = copy.deepcopy(org_dict)
        dd = DictDiffer(base_dict, org_dict)
        self.assertTrue(len(base_dict), 2)
        self.assertTrue(len(dd.unchanged()), len(base_dict))

        # Case 1: Add new value
        delta_dict = {"c" : "NEW_C"}
        mod_dict = dict_merge(base_dict, delta_dict)
        dd = DictDiffer(base_dict, org_dict)
        self.assertTrue(len(base_dict), 2)
        self.assertTrue(len(dd.unchanged()), len(base_dict))

        dd = DictDiffer(mod_dict, org_dict)
        self.assertTrue(len(mod_dict), 3)
        self.assertTrue(len(dd.unchanged()), len(org_dict))
        self.assertTrue(dd.added(), 1)

        # Case 2: Change simple type value
        delta_dict = {"a" : 5}
        base_dict = copy.deepcopy(org_dict)
        mod_dict = dict_merge(base_dict, delta_dict)

        dd = DictDiffer(mod_dict, org_dict)
        self.assertTrue(len(mod_dict), len(org_dict))
        self.assertTrue(len(dd.unchanged()), len(org_dict)-1)
        self.assertTrue(len(dd.changed()), 1)
        self.assertTrue(mod_dict['a'], 5)

        # Case 3: Add new value on lower level
        delta_dict = {"d": {"new":"NEW_ENTRY"}}
        base_dict = copy.deepcopy(org_dict)
        mod_dict = dict_merge(base_dict, delta_dict)

        dd = DictDiffer(mod_dict, org_dict)
        self.assertTrue(len(mod_dict), len(org_dict))
        self.assertTrue(len(mod_dict['d']), len(org_dict['d']) + 1)
        self.assertTrue(mod_dict['d']['new'], "NEW_ENTRY")
        dd = DictDiffer(mod_dict['d'], org_dict['d'])
        self.assertTrue(len(dd.unchanged()), len(org_dict['d']))
        self.assertTrue(dd.added(), 1)
Esempio n. 25
0
    def start_app(self, appdef=None, config=None):
        """
        @brief Start an app from an app definition.
        Note: apps can come in one of 2 variants:
        1 processapp: In-line defined process to be started
        2 regular app: Full app definition
        """
        log.debug("AppManager.start_app(appdef=%s) ..." % appdef)

        appdef = DotDict(appdef)

        if 'config' in appdef:
            app_cfg = deepcopy(appdef.config)
            if config:
                dict_merge(app_cfg, config, inplace=True)
            config = app_cfg

        if 'processapp' in appdef:
            # Case 1: Appdef contains definition of process to start
            name, module, cls = appdef.processapp
            try:
                pid = self.container.spawn_process(name, module, cls, config)
                appdef._pid = pid
                self.apps.append(appdef)
            except Exception:
                log.exception("Appl %s start from processapp failed" % appdef.name)
        else:
            # Case 2: Appdef contains full app start params
            modpath = appdef.mod
            try:
                mod = named_any(modpath)
                appdef._mod_loaded = mod

                # Start the app
                supid, state = mod.start(self.container, START_PERMANENT, appdef, config)
                appdef._supid = supid
                appdef._state = state

                log.debug("App '%s' started. Root sup-id=%s" % (appdef.name, supid))

                self.apps.append(appdef)
            except Exception:
                log.exception("Appl %s start from appdef failed" % appdef.name)
Esempio n. 26
0
 def acquire_data(self, streaming_args=None):
     if self.current_state != self.AGENTSTATE_CONNECTED:
         raise BadRequest("Illegal agent state: %s" % self.current_state)
     try:
         args = dict_merge(
             self.agent_config,
             streaming_args) if streaming_args else self.agent_config
         res = self.on_acquire_data(args)
     except Exception:
         self.current_state = self.AGENTSTATE_ERROR
         raise
Esempio n. 27
0
    def start_rel(self, rel=None, config=None):
        """
        @brief Recurse over the rel and start apps defined there.
        Note: apps in a rel file can come in one of 2 forms:
        1 processapp: In-line defined process to be started as app
        2 app file: Reference to an app definition in an app file
        If the rel file provides an app config block, it is provided to spawn the process.
        Any given function config dict is merged on top of this.
        """
        log.debug("AppManager.start_rel(rel=%s) ...", rel)

        if rel is None: rel = {}

        for rel_app_cfg in rel.apps:
            name = rel_app_cfg.name
            log.debug("app definition in rel: %s" % str(rel_app_cfg))

            if 'processapp' in rel_app_cfg:
                # Case 1: Rel contains definition of process to start as app
                name, module, cls = rel_app_cfg.processapp

                rel_cfg = None
                if 'config' in rel_app_cfg:
                    rel_cfg = rel_app_cfg.config.copy()
                    if config:
                        dict_merge(rel_cfg, config, inplace=True)

                self.container.spawn_process(name, module, cls, rel_cfg)
                self.apps.append(
                    DotDict(type="application",
                            name=name,
                            processapp=rel_app_cfg.processapp))

            else:
                # Case 2: Rel contains reference to app file to start
                app_file_path = 'res/apps/%s.yml' % (name)
                rel_cfg = rel_app_cfg.get('config', None)
                if config:
                    dict_merge(rel_cfg, config, inplace=True)
                self.start_app_from_url(app_file_path, config=rel_cfg)
Esempio n. 28
0
    def prepare_container():
        import threading
        threading.current_thread().name = "CC-Main"

        # SIDE EFFECT: The import of pyon.public triggers many module initializers:
        # pyon.core.bootstrap (Config load, logging setup), etc.
        from pyon.public import Container, CFG
        from pyon.util.containers import dict_merge
        from pyon.util.config import Config

        # Check if user opted to override logging config
        if opts.logcfg:
            from pyon.util.config import logging_conf_paths, initialize_logging
            # Re-initialize logging
            logging_conf_paths.append(opts.logcfg)
            initialize_logging()

        # Set that system is not testing. We are running as standalone container
        dict_merge(CFG, {'system':{'testing':False}}, True)

        # Also set the immediate flag, but only if specified - it is an override
        if opts.immediate:
            dict_merge(CFG, {'system':{'immediate':True}}, True)

        # Load any additional config paths and merge them into main config
        if len(opts.config):
            cfg = Config(opts.config)
            dict_merge(CFG, cfg.data, True)

        # Create the container instance
        container = Container(*args, **kwargs)

        return container
Esempio n. 29
0
 def connect(self, connect_args=None):
     if self.current_state == self.AGENTSTATE_CONNECTED:
         return
     elif self.current_state != self.AGENTSTATE_INITIALIZED:
         raise BadRequest("Illegal agent state: %s" % self.current_state)
     try:
         self.stream_pub = StreamPublisher(process=self, stream=self.stream_name)
         args = dict_merge(self.agent_config, connect_args) if connect_args else self.agent_config
         res = self.on_connect(args)
     except Exception:
         self.current_state = self.AGENTSTATE_ERROR
         raise
     self.current_state = self.AGENTSTATE_CONNECTED
Esempio n. 30
0
    def start_rel(self, rel=None, config=None):
        """
        @brief Recurse over the rel and start apps defined there.
        Note: apps in a rel file can come in one of 2 forms:
        1 processapp: In-line defined process to be started as app
        2 app file: Reference to an app definition in an app file
        If the rel file provides an app config block, it is provided to spawn the process.
        Any given function config dict is merged on top of this.
        """
        log.debug("AppManager.start_rel(rel=%s) ...", rel)

        if rel is None:
            rel = {}

        for rel_app_cfg in rel.apps:
            name = rel_app_cfg.name
            log.debug("app definition in rel: %s" % str(rel_app_cfg))

            if 'processapp' in rel_app_cfg:
                # Case 1: Rel contains definition of process to start as app
                name, module, cls = rel_app_cfg.processapp

                rel_cfg = None
                if 'config' in rel_app_cfg:
                    rel_cfg = deepcopy(rel_app_cfg.config)
                    if config:
                        dict_merge(rel_cfg, config, inplace=True)

                self.container.spawn_process(name, module, cls, rel_cfg)
                self.apps.append(DotDict(type="application", name=name, processapp=rel_app_cfg.processapp))

            else:
                # Case 2: Rel contains reference to app file to start
                app_file_path = 'res/apps/%s.yml' % (name)
                rel_cfg = rel_app_cfg.get('config', None)
                if config:
                    dict_merge(rel_cfg, config, inplace=True)
                self.start_app_from_url(app_file_path, config=rel_cfg)
Esempio n. 31
0
    def _load_capabilities(self):
        self._cap_initialized = [
        ]  # List of capability constants initialized in container
        self._capabilities = [
        ]  # List of capability constants active in container
        self._cap_instances = {}  # Dict mapping capability->manager instance

        self._cap_definitions = Config(
            ["res/config/container_capabilities.yml"]).data['capabilities']

        profile_filename = CFG.get_safe("container.profile", "development")
        if not profile_filename.endswith(".yml"):
            profile_filename = "res/profile/%s.yml" % profile_filename
        log.debug("Loading CC capability profile from file: %s",
                  profile_filename)
        profile_cfg = Config([profile_filename]).data
        if not isinstance(
                profile_cfg, dict
        ) or profile_cfg['type'] != "profile" or not "profile" in profile_cfg:
            raise ContainerError("Container capability profile invalid: %s" %
                                 profile_filename)

        self.cap_profile = profile_cfg['profile']

        if "capabilities" in self.cap_profile and self.cap_profile[
                'capabilities']:
            dict_merge(self._cap_definitions, self.cap_profile['capabilities'],
                       True)

        CCAP.clear()
        cap_list = self._cap_definitions.keys()
        CCAP.update(zip(cap_list, cap_list))

        if "config" in self.cap_profile and self.cap_profile['config']:
            log.info("Container CFG was changed based on profile: %s",
                     profile_filename)
Esempio n. 32
0
 def start_streaming(self, streaming_args=None):
     if self.current_state == self.AGENTSTATE_STREAMING:
         return
     if self.current_state == self.AGENTSTATE_INITIALIZED:
         self.connect(self.agent_config)
     if self.current_state != self.AGENTSTATE_CONNECTED:
         raise BadRequest("Illegal agent state: %s" % self.current_state)
     log.info("Start streaming")
     try:
         args = dict_merge(self.agent_config, streaming_args) if streaming_args else self.agent_config
         res = self.on_start_streaming(args)
     except Exception:
         self.current_state = self.AGENTSTATE_ERROR
         raise
     self.current_state = self.AGENTSTATE_STREAMING
Esempio n. 33
0
    def prepare_container():
        import threading
        threading.current_thread().name = "CC-Main"

        # SIDE EFFECT: The import of pyon.public triggers many module initializers:
        # pyon.core.bootstrap (Config load, logging setup), etc.
        from pyon.public import Container, CFG
        from pyon.util.containers import dict_merge
        from pyon.util.config import Config

        # Check if user opted to override logging config
        # Requires re-initializing logging
        if opts.logcfg:
            from pyon.util.config import LOGGING_CFG, logging_conf_paths, read_logging_config, initialize_logging
            import ast
            # Dict of config values
            if '{' in opts.logcfg:
                try:
                    eval_value = ast.literal_eval(opts.logcfg)
                except ValueError:
                    raise Exception("Value error in logcfg arg '%s'" %
                                    opts.logcfg)
                dict_merge(LOGGING_CFG, eval_value)
                initialize_logging()
            # YAML file containing config values
            else:
                logging_conf_paths.append(opts.logcfg)
                read_logging_config()
                initialize_logging()

        # Set that system is not testing. We are running as standalone container
        dict_merge(CFG, {'system': {'testing': False}}, True)

        # Also set the immediate flag, but only if specified - it is an override
        if opts.immediate:
            dict_merge(CFG, {'system': {'immediate': True}}, True)

        # Load any additional config paths and merge them into main config
        if len(opts.config):
            ipython_cfg = Config(opts.config)
            dict_merge(CFG, ipython_cfg.data, True)

        # Create the container instance
        container = Container(*args, **kwargs)

        return container
Esempio n. 34
0
 def connect(self, connect_args=None):
     if self.current_state == self.AGENTSTATE_CONNECTED:
         return
     elif self.current_state != self.AGENTSTATE_INITIALIZED:
         raise BadRequest("Illegal agent state: %s" % self.current_state)
     try:
         self.stream_pub = StreamPublisher(process=self,
                                           stream=self.stream_name)
         args = dict_merge(
             self.agent_config,
             connect_args) if connect_args else self.agent_config
         res = self.on_connect(args)
     except Exception:
         self.current_state = self.AGENTSTATE_ERROR
         raise
     self.current_state = self.AGENTSTATE_CONNECTED
Esempio n. 35
0
    def prepare_container():
        import threading
        threading.current_thread().name = "CC-Main"

        # SIDE EFFECT: The import of pyon.public triggers many module initializers:
        # pyon.core.bootstrap (Config load, logging setup), etc.
        from pyon.public import Container, CFG
        from pyon.util.containers import dict_merge
        from pyon.util.config import Config

        # Check if user opted to override logging config
        # Requires re-initializing logging
        if opts.logcfg:
            from pyon.util.config import LOGGING_CFG, logging_conf_paths, read_logging_config, initialize_logging
            import ast
            # Dict of config values
            if '{' in opts.logcfg:
                try:
                    eval_value = ast.literal_eval(opts.logcfg)
                except ValueError:
                    raise Exception("Value error in logcfg arg '%s'" % opts.logcfg)
                dict_merge(LOGGING_CFG, eval_value)
                initialize_logging()
            # YAML file containing config values
            else:
                logging_conf_paths.append(opts.logcfg)
                read_logging_config()
                initialize_logging()

        # Set that system is not testing. We are running as standalone container
        dict_merge(CFG, {'system':{'testing':False}}, True)

        # Also set the immediate flag, but only if specified - it is an override
        if opts.immediate:
            dict_merge(CFG, {'system':{'immediate':True}}, True)

        # Load any additional config paths and merge them into main config
        if len(opts.config):
            ipython_cfg = Config(opts.config)
            dict_merge(CFG, ipython_cfg.data, True)

        # Create the container instance
        container = Container(*args, **kwargs)

        return container
Esempio n. 36
0
 def start_streaming(self, streaming_args=None):
     if self.current_state == self.AGENTSTATE_STREAMING:
         return
     if self.current_state == self.AGENTSTATE_INITIALIZED:
         self.connect(self.agent_config)
     if self.current_state != self.AGENTSTATE_CONNECTED:
         raise BadRequest("Illegal agent state: %s" % self.current_state)
     log.info("Start streaming")
     try:
         args = dict_merge(
             self.agent_config,
             streaming_args) if streaming_args else self.agent_config
         res = self.on_start_streaming(args)
     except Exception:
         self.current_state = self.AGENTSTATE_ERROR
         raise
     self.current_state = self.AGENTSTATE_STREAMING
Esempio n. 37
0
    def load(self, ignore_not_found=False):
        """ Load each path in order. Remember paths already loaded and only load new ones. """
        data = self.dict_class()

        for path in self.paths:
            if path in self.paths_loaded: continue

            try:
                with open(path, 'r') as file:
                    path_data = yaml.load(file.read())
                    if path_data is not None:
                        data = dict_merge(data, path_data)
                self.paths_loaded.add(path)
            except IOError:
                if not ignore_not_found:
                    raise ConfigNotFound("Config URL '%s' not found" % path)

        self.data = data
Esempio n. 38
0
    def load(self, ignore_not_found=False):
        """ Load each path in order. Remember paths already loaded and only load new ones. """
        data = self.dict_class()
        
        for path in self.paths:
            if path in self.paths_loaded: continue
            
            try:
                with open(path, 'r') as file:
                    path_data = yaml.load(file.read())
                    if path_data is not None:
                        data = dict_merge(data, path_data)
                self.paths_loaded.add(path)
            except IOError:
                if not ignore_not_found:
                    raise ConfigNotFound("Config URL '%s' not found" % path)

        self.data = data
Esempio n. 39
0
def _override_config(config_override):
    if type(config_override) is not dict:
        print "pyon: WARNING: config_override is not dict but", config_override
    from pyon.util.containers import dict_merge
    dict_merge(LOGGING_CFG, config_override)
Esempio n. 40
0
 def _get_alt_cfg(cfg_merge):
     cfg_clone = deepcopy(CFG)
     dict_merge(cfg_clone, cfg_merge, inplace=True)
     return DotDict(**cfg_clone)
Esempio n. 41
0
    def start_rel(self, rel=None, config=None):
        """
        @brief Recurse over the rel and start apps defined there.
        Note: apps in a rel file can come in one of 2 forms:
        1 processapp: In-line defined process to be started as app
        2 app file: Reference to an app definition in an app file
        If the rel file provides an app config block, it is provided to spawn the process.
        Any given function config dict is merged on top of this.
        """
        log.debug("AppManager.start_rel(rel=%s) ...", rel)

        if rel is None:
            return

        if self.use_pd:
            log.info("Sending rel file to PD")
            import json
            rel_def = json.loads(json.dumps(
                rel))  # HACK to get rid of OrderedDict (not serializable)
            cmd_res = self.pd_client.start_rel_blocking(rel_def, timeout=None)
            return cmd_res

        for rel_app_cfg in rel.apps:
            name = rel_app_cfg.name
            log.debug("app definition in rel: %s" % str(rel_app_cfg))

            if 'processapp' in rel_app_cfg:
                # Case 1: Rel contains definition of process to start as app
                name, module, cls = rel_app_cfg.processapp

                rel_cfg = None
                if 'config' in rel_app_cfg:
                    rel_cfg = deepcopy(rel_app_cfg.config)
                    if config:
                        dict_merge(rel_cfg, config, inplace=True)

                if 'replicas' in rel_app_cfg:
                    proc_replicas = int(rel_app_cfg["replicas"])
                    if self.max_proc_replicas > 0:
                        if proc_replicas > self.max_proc_replicas:
                            log.info(
                                "Limiting number of proc replicas to %s from %s",
                                self.max_proc_replicas, proc_replicas)
                        proc_replicas = min(proc_replicas,
                                            self.max_proc_replicas)
                    if proc_replicas < 1 or proc_replicas > 100:
                        log.warn("Invalid number of process replicas: %s",
                                 proc_replicas)
                        proc_replicas = 1
                    for i in xrange(proc_replicas):
                        proc_name = "%s.%s" % (name, i) if i else name
                        self.container.spawn_process(proc_name, module, cls,
                                                     rel_cfg)
                else:
                    self.container.spawn_process(name, module, cls, rel_cfg)
                self.apps.append(
                    DotDict(type="application",
                            name=name,
                            processapp=rel_app_cfg.processapp))

            else:
                # Case 2: Rel contains reference to app file to start
                app_file_path = 'res/apps/%s.yml' % (name)
                rel_cfg = rel_app_cfg.get('config', None)
                if config:
                    dict_merge(rel_cfg, config, inplace=True)
                self.start_app_from_url(app_file_path, config=rel_cfg)
Esempio n. 42
0
    def prepare_container():
        """
        Walks through pyon initialization in a deterministic way and initializes Container.
        In particular make sure configuration is loaded in correct order and
        pycc startup arguments are considered.
        """
        # SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults
        import pyon

        import threading
        threading.current_thread().name = "CC-Main"

        import logging
        global log
        log = logging.getLogger('pycc')

        from pyon.core import bootstrap, config
        from pyon.util.containers import get_safe, dict_merge

        # Set global testing flag to False. We are running as capability container, because
        # we started through the pycc program.
        bootstrap.testing = False

        # Set sysname if provided in startup argument
        if opts.sysname:
            bootstrap.set_sys_name(opts.sysname)
        # Trigger any initializing default logic in get_sys_name
        bootstrap.get_sys_name()

        command_line_config = kwargs

        # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
        bootstrap_config = None

        # This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
        pyon_config = config.read_standard_configuration(
        )  # Initial pyon.yml + pyon.local.yml

        # Load config override if provided. Supports variants literal and list of paths
        config_override = None
        if opts.config:
            if '{' in opts.config:
                # Variant 1: Dict of config values
                try:
                    eval_value = ast.literal_eval(opts.config)
                    config_override = eval_value
                except ValueError:
                    raise Exception("Value error in config arg '%s'" %
                                    opts.config)
            else:
                # Variant 2: List of paths
                from pyon.util.config import Config
                config_override = Config([opts.config]).data

        # Determine bootstrap_config
        if opts.config_from_directory:
            # Load minimal bootstrap config if option "config_from_directory"
            bootstrap_config = config.read_local_configuration(
                ['res/config/pyon_min_boot.yml'])
            config.apply_local_configuration(bootstrap_config,
                                             pyon.DEFAULT_LOCAL_CONFIG_PATHS)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)
            log.info(
                "config_from_directory=True. Minimal bootstrap configuration: %s",
                bootstrap_config)
        else:
            # Otherwise: Set to standard set of local config files plus command line overrides
            bootstrap_config = deepcopy(pyon_config)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)

        # Override sysname from config file or command line
        if not opts.sysname and bootstrap_config.get_safe("system.name", None):
            new_sysname = bootstrap_config.get_safe("system.name")
            bootstrap.set_sys_name(new_sysname)

        # Force_clean - deletes sysname datastores
        if opts.force_clean:
            from pyon.datastore import clear_db_util
            log.info("force_clean=True. DROP DATASTORES for sysname=%s",
                     bootstrap.get_sys_name())
            clear_db_util.clear_db(bootstrap_config,
                                   prefix=bootstrap.get_sys_name(),
                                   sysname=bootstrap.get_sys_name())

        from pyon.core.interfaces.interfaces import InterfaceAdmin
        iadm = InterfaceAdmin(bootstrap.get_sys_name(),
                              config=bootstrap_config)

        # If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config
        # WARNING: If multiple containers start concurrently, this may fail
        if get_safe(bootstrap_config,
                    "bootstrap.auto_store_interfaces") is True:
            log.debug("auto_store_interfaces=True.")
            stored_config = deepcopy(pyon_config)
            config.apply_configuration(stored_config, config_override)
            config.apply_configuration(stored_config, command_line_config)
            iadm.create_core_datastores()
            iadm.store_config(stored_config)

        # Determine the final pyon_config:
        # - Start from standard config already set (pyon.yml + local YML files)
        # - Optionally load config from directory
        if opts.config_from_directory:
            config.apply_remote_config(bootstrap_cfg=bootstrap_config,
                                       system_cfg=pyon_config)
        # - Apply container profile specific config
        config.apply_profile_configuration(pyon_config, bootstrap_config)
        # - Reapply pyon.local.yml here again for good measure
        config.apply_local_configuration(pyon_config,
                                         pyon.DEFAULT_LOCAL_CONFIG_PATHS)
        # - Last apply any separate command line config overrides
        config.apply_configuration(pyon_config, config_override)
        config.apply_configuration(pyon_config, command_line_config)
        iadm.set_config(pyon_config)

        # Set the immediate flag when command line override specified
        if opts.immediate:
            dict_merge(pyon_config, {"system": {
                "immediate": True
            }},
                       inplace=True)

        # Determine system bootmode for bootstrapping actions (unless explicitly specified)
        if not pyon_config.get_safe("bootmode"):
            set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode")
            if set_bootmode == "auto":
                if iadm.system_data_exists():
                    dict_merge(pyon_config, {"bootmode": "restart"},
                               inplace=True)
                log.info(
                    "System bootmode auto-detection is ON. Determined bootmode=%s",
                    pyon_config.get_safe("bootmode", "initial"))
            elif set_bootmode == "secondary":
                dict_merge(pyon_config, {"bootmode": "secondary"},
                           inplace=True)
                log.info("System bootmode override. Set to bootmode=%s",
                         pyon_config.get_safe("bootmode", ""))
        log.info("System in bootmode=%s",
                 pyon_config.get_safe("bootmode", "initial"))

        # Bootstrap the pyon framework's core. Load configuration etc.
        bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)

        # Delete any queues/exchanges owned by sysname if option "broker_clean" is set
        if opts.broker_clean:
            log.info("broker_clean=True, sysname: %s",
                     bootstrap.get_sys_name())

            from putil.rabbitmq.rabbit_util import RabbitManagementUtil
            rabbit_util = RabbitManagementUtil(
                pyon_config, sysname=bootstrap.get_sys_name())
            deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname()
            log.info("Exchanges deleted (%s): %s" %
                     (len(deleted_exchanges), ", ".join(deleted_exchanges)))
            log.info("Queues deleted (%s): %s" %
                     (len(deleted_queues), ", ".join(deleted_queues)))

        if opts.force_clean:
            from pyon.util.file_sys import FileSystem
            FileSystem._clean(pyon_config)

        # If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging
        if get_safe(bootstrap_config,
                    "bootstrap.auto_store_interfaces") is True:
            iadm.store_interfaces(idempotent=True)
            iadm.declare_core_exchange_resources()

        iadm.close()

        if opts.no_container:
            log.info("no_container=True. Stopping here.")
            return None

        # Create the container instance
        from pyon.container.cc import Container
        container = Container(*args, **command_line_config)
        container.version = version

        return container
Esempio n. 43
0
File: pycc.py Progetto: ooici/pyon
    def prepare_container():
        """
        Walks through pyon initialization in a deterministic way and initializes Container.
        In particular make sure configuration is loaded in correct order and
        pycc startup arguments are considered.
        """
        import threading

        threading.current_thread().name = "CC-Main"

        # SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
        import pyon

        from pyon.core import bootstrap, config

        # Set global testing flag to False. We are running as capability container, because
        # we started through the pycc program.
        bootstrap.testing = False

        # Set sysname if provided in startup argument
        if opts.sysname:
            bootstrap.set_sys_name(opts.sysname)
        # Trigger any initializing default logic in get_sys_name
        bootstrap.get_sys_name()

        command_line_config = kwargs

        # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
        bootstrap_config = None

        # This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
        pyon_config = config.read_standard_configuration()  # Initial pyon.yml + pyon.local.yml

        # Load config override if provided. Supports variants literal and list of paths
        config_override = None
        if opts.config:
            if "{" in opts.config:
                # Variant 1: Dict of config values
                try:
                    eval_value = ast.literal_eval(opts.config)
                    config_override = eval_value
                except ValueError:
                    raise Exception("Value error in config arg '%s'" % opts.config)
            else:
                # Variant 2: List of paths
                from pyon.util.config import Config

                config_override = Config([opts.config]).data

        # Determine bootstrap_config
        if opts.config_from_directory:
            # Load minimal bootstrap config if option "config_from_directory"
            bootstrap_config = config.read_local_configuration(["res/config/pyon_min_boot.yml"])
            config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)
            print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
        else:
            # Otherwise: Set to standard set of local config files plus command line overrides
            bootstrap_config = deepcopy(pyon_config)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)

        # Override sysname from config file or command line
        if not opts.sysname and bootstrap_config.get_safe("system.name", None):
            new_sysname = bootstrap_config.get_safe("system.name")
            bootstrap.set_sys_name(new_sysname)

        # Delete sysname datastores if option "force_clean" is set
        if opts.force_clean:
            from pyon.datastore import clear_couch_util

            print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name()
            clear_couch_util.clear_couch(
                bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name()
            )
            pyon_config.container.filesystem.force_clean = True

        from pyon.core.interfaces.interfaces import InterfaceAdmin

        iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)

        # If auto_bootstrap, load config and interfaces into directory
        # Note: this is idempotent and will not alter anything if this is not the first container to run
        if bootstrap_config.system.auto_bootstrap:
            print "pycc: auto_bootstrap=True."
            stored_config = deepcopy(pyon_config)
            config.apply_configuration(stored_config, config_override)
            config.apply_configuration(stored_config, command_line_config)
            iadm.create_core_datastores()
            iadm.store_config(stored_config)

        # Determine the final pyon_config
        # - Start from standard config already set (pyon.yml + local YML files)
        # - Optionally load config from directory
        if opts.config_from_directory:
            config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config)
        # - Apply container profile specific config
        config.apply_profile_configuration(pyon_config, bootstrap_config)
        # - Reapply pyon.local.yml here again for good measure
        config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
        # - Last apply any separate command line config overrides
        config.apply_configuration(pyon_config, config_override)
        config.apply_configuration(pyon_config, command_line_config)

        # Also set the immediate flag, but only if specified - it is an override
        if opts.immediate:
            from pyon.util.containers import dict_merge

            dict_merge(pyon_config, {"system": {"immediate": True}}, True)

        # Bootstrap pyon's core. Load configuration etc.
        bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)

        # Delete any queues/exchanges owned by sysname if option "broker_clean" is set
        if opts.broker_clean:
            print "pycc: broker_clean=True, sysname:", bootstrap.get_sys_name()

            # build connect str
            connect_str = "-q -H %s -P %s -u %s -p %s -V %s" % (
                pyon_config.get_safe("server.amqp_priv.host", pyon_config.get_safe("server.amqp.host", "localhost")),
                pyon_config.get_safe("container.exchange.management.port", "55672"),
                pyon_config.get_safe("container.exchange.management.username", "guest"),
                pyon_config.get_safe("container.exchange.management.password", "guest"),
                "/",
            )

            from putil.rabbithelper import clean_by_sysname

            deleted_exchanges, deleted_queues = clean_by_sysname(connect_str, bootstrap.get_sys_name())
            print "      exchanges deleted (%s): %s" % (len(deleted_exchanges), ",".join(deleted_exchanges))
            print "         queues deleted (%s): %s" % (len(deleted_queues), ",".join(deleted_queues))

        if opts.force_clean:
            path = os.path.join(pyon_config.get_safe("container.filesystem.root", "/tmp/ion"), bootstrap.get_sys_name())
            print "force_clean: Removing", path
            FileSystem._clean(pyon_config)

        # Auto-bootstrap interfaces
        if bootstrap_config.system.auto_bootstrap:
            iadm.store_interfaces(idempotent=True)

        iadm.close()

        if opts.no_container:
            print "pycc: no_container=True. Stopping here."
            return None

        # Create the container instance
        from pyon.container.cc import Container

        container = Container(*args, **command_line_config)

        return container
Esempio n. 44
0
    def spawn_process(self, name=None, module=None, cls=None, config=None):
        """
        Spawn a process within the container. Processes can be of different type.
        """
        # Generate a new process id
        # TODO: Ensure it is system-wide unique
        process_id =  "%s.%s" % (self.container.id, self.proc_id_pool.get_id())
        log.debug("ProcManager.spawn_process(name=%s, module.cls=%s.%s) as pid=%s", name, module, cls, process_id)

        if not config:
            # Use system CFG. It has the command line args in it
            config = DictModifier(CFG)
        else:
            # Use provided config. Must be dict or DotDict
            if not isinstance(config, DotDict):
                config = DotDict(config)
            config = DictModifier(CFG, config)
            if self.container.spawn_args:
                # Override config with spawn args
                dict_merge(config, self.container.spawn_args, inplace=True)

        log.debug("spawn_process() pid=%s config=%s", process_id, config)

        # PROCESS TYPE. Determines basic process context (messaging, service interface)
        # One of: service, stream_process, agent, simple, immediate

        service_cls = named_any("%s.%s" % (module, cls))
        process_type = get_safe(config, "process.type") or getattr(service_cls, "process_type", "service")

        service_instance = None
        try:
            # spawn service by type
            if process_type == "service":
                service_instance = self._spawn_service_process(process_id, name, module, cls, config)

            elif process_type == "stream_process":
                service_instance = self._spawn_stream_process(process_id, name, module, cls, config)

            elif process_type == "agent":
                service_instance = self._spawn_agent_process(process_id, name, module, cls, config)

            elif process_type == "standalone":
                service_instance = self._spawn_standalone_process(process_id, name, module, cls, config)

            elif process_type == "immediate":
                service_instance = self._spawn_immediate_process(process_id, name, module, cls, config)

            elif process_type == "simple":
                service_instance = self._spawn_simple_process(process_id, name, module, cls, config)

            else:
                raise BadRequest("Unknown process type: %s" % process_type)

            service_instance._proc_type = process_type
            self._register_process(service_instance, name)

            service_instance.errcause = "OK"
            log.info("AppManager.spawn_process: %s.%s -> pid=%s OK" % (module, cls, process_id))
            return service_instance.id

        except Exception:
            errcause = service_instance.errcause if service_instance else "instantiating service"
            log.exception("Error spawning %s %s process (process_id: %s): %s" % (name, process_type, process_id, errcause))
            raise
Esempio n. 45
0
    def prepare_container():
        """
        Walks through pyon initialization in a deterministic way and initializes Container.
        In particular make sure configuration is loaded in correct order and
        pycc startup arguments are considered.
        """
        import threading
        threading.current_thread().name = "CC-Main"

        # SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
        import pyon

        from pyon.core import bootstrap, config

        # Set global testing flag to False. We are running as capability container. This is NO TEST.
        bootstrap.testing = False

        # Set sysname if provided in startup argument
        if opts.sysname:
            bootstrap.set_sys_name(opts.sysname)
        # Trigger any initializing default logic in get_sys_name
        bootstrap.get_sys_name()

        command_line_config = kwargs

        # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
        bootstrap_config = None

        # This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
        pyon_config = config.read_standard_configuration(
        )  # Initial pyon.yml + pyon.local.yml

        # Load config override if provided. Supports variants literal and list of paths
        config_override = None
        if opts.config:
            if '{' in opts.config:
                # Variant 1: Dict of config values
                try:
                    eval_value = ast.literal_eval(opts.config)
                    config_override = eval_value
                except ValueError:
                    raise Exception("Value error in config arg '%s'" %
                                    opts.config)
            else:
                # Variant 2: List of paths
                from pyon.util.config import Config
                config_override = Config([opts.config]).data

        # Determine bootstrap_config
        if opts.config_from_directory:
            # Load minimal bootstrap config if option "config_from_directory"
            bootstrap_config = config.read_local_configuration(
                ['res/config/pyon_min_boot.yml'])
            config.apply_local_configuration(bootstrap_config,
                                             pyon.DEFAULT_LOCAL_CONFIG_PATHS)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)
            print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
        else:
            # Otherwise: Set to standard set of local config files plus command line overrides
            bootstrap_config = deepcopy(pyon_config)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)

        # Override sysname from config file or command line
        if not opts.sysname and bootstrap_config.get_safe("system.name", None):
            new_sysname = bootstrap_config.get_safe("system.name")
            bootstrap.set_sys_name(new_sysname)

        # Delete sysname datastores if option "force_clean" is set
        if opts.force_clean:
            from pyon.datastore import clear_couch_util
            print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name(
            )
            clear_couch_util.clear_couch(bootstrap_config,
                                         prefix=bootstrap.get_sys_name())
            pyon_config.container.filesystem.force_clean = True

        from pyon.core.interfaces.interfaces import InterfaceAdmin
        iadm = InterfaceAdmin(bootstrap.get_sys_name(),
                              config=bootstrap_config)

        # If auto_bootstrap, load config and interfaces into directory
        # Note: this is idempotent and will not alter anything if this is not the first container to run
        if bootstrap_config.system.auto_bootstrap:
            print "pycc: auto_bootstrap=True."
            stored_config = deepcopy(pyon_config)
            config.apply_configuration(stored_config, config_override)
            config.apply_configuration(stored_config, command_line_config)
            iadm.create_core_datastores()
            iadm.store_config(stored_config)

        # Determine the final pyon_config
        # - Start from standard config already set (pyon.yml + local YML files)
        # - Optionally load config from directory
        if opts.config_from_directory:
            config.apply_remote_config(bootstrap_cfg=bootstrap_config,
                                       system_cfg=pyon_config)
            config.apply_local_configuration(
                pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS
            )  # apply pyon.local.yml again over top
        # - Last apply any separate command line config overrides
        config.apply_configuration(pyon_config, config_override)
        config.apply_configuration(pyon_config, command_line_config)

        # Also set the immediate flag, but only if specified - it is an override
        if opts.immediate:
            from pyon.util.containers import dict_merge
            dict_merge(pyon_config, {'system': {'immediate': True}}, True)

        # Bootstrap pyon's core. Load configuration etc.
        bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)

        # Delete any queues/exchanges owned by sysname if option "broker_clean" is set
        if opts.broker_clean:
            print "pycc: broker_clean=True, sysname:", bootstrap.get_sys_name()

            # build connect str
            connect_str = "-q -H %s -P 55672 -u %s -p %s -V %s" % (
                pyon_config.get_safe(
                    'server.amqp_priv.host',
                    pyon_config.get_safe('server.amqp.host', 'localhost')),
                pyon_config.get_safe('container.exchange.management.username',
                                     'guest'),
                pyon_config.get_safe('container.exchange.management.password',
                                     'guest'), '/')

            from putil.rabbithelper import clean_by_sysname
            deleted_exchanges, deleted_queues = clean_by_sysname(
                connect_str, bootstrap.get_sys_name())
            print "      exchanges deleted (%s): %s" % (
                len(deleted_exchanges), ",".join(deleted_exchanges))
            print "         queues deleted (%s): %s" % (
                len(deleted_queues), ",".join(deleted_queues))

        # Auto-bootstrap interfaces
        if bootstrap_config.system.auto_bootstrap:
            iadm.store_interfaces(idempotent=True)

        iadm.close()

        if opts.no_container:
            print "pycc: no_container=True. Stopping here."
            return None

        # Create the container instance
        from pyon.container.cc import Container
        container = Container(*args, **command_line_config)

        return container
Esempio n. 46
0
    def prepare_container():
        """
        Walks through pyon initialization in a deterministic way and initializes Container.
        In particular make sure configuration is loaded in correct order and
        pycc startup arguments are considered.
        """
        # SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults
        import pyon

        import threading
        threading.current_thread().name = "CC-Main"

        import logging
        global log
        log = logging.getLogger('pycc')

        from pyon.core import bootstrap, config
        from pyon.util.containers import get_safe, dict_merge

        # Set global testing flag to False. We are running as capability container, because
        # we started through the pycc program.
        bootstrap.testing = False

        # Set sysname if provided in startup argument
        if opts.sysname:
            bootstrap.set_sys_name(opts.sysname)
        # Trigger any initializing default logic in get_sys_name
        bootstrap.get_sys_name()

        command_line_config = kwargs

        # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
        bootstrap_config = None

        # This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
        pyon_config = config.read_standard_configuration()      # Initial pyon.yml + pyon.local.yml

        # Load config override if provided. Supports variants literal and list of paths
        config_override = None
        if opts.config:
            if '{' in opts.config:
                # Variant 1: Dict of config values
                try:
                    eval_value = ast.literal_eval(opts.config)
                    config_override = eval_value
                except ValueError:
                    raise Exception("Value error in config arg '%s'" % opts.config)
            else:
                # Variant 2: List of paths
                from pyon.util.config import Config
                config_override = Config([opts.config]).data

        # Determine bootstrap_config
        if opts.config_from_directory:
            # Load minimal bootstrap config if option "config_from_directory"
            bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
            config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)
            log.info("config_from_directory=True. Minimal bootstrap configuration: %s", bootstrap_config)
        else:
            # Otherwise: Set to standard set of local config files plus command line overrides
            bootstrap_config = deepcopy(pyon_config)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)

        # Override sysname from config file or command line
        if not opts.sysname and bootstrap_config.get_safe("system.name", None):
            new_sysname = bootstrap_config.get_safe("system.name")
            bootstrap.set_sys_name(new_sysname)

        # Force_clean - deletes sysname datastores
        if opts.force_clean:
            from pyon.datastore import clear_db_util
            log.info("force_clean=True. DROP DATASTORES for sysname=%s", bootstrap.get_sys_name())
            clear_db_util.clear_db(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name())

        from pyon.core.interfaces.interfaces import InterfaceAdmin
        iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)

        # If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config
        # WARNING: If multiple containers start concurrently, this may fail
        if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True:
            log.debug("auto_store_interfaces=True.")
            stored_config = deepcopy(pyon_config)
            config.apply_configuration(stored_config, config_override)
            config.apply_configuration(stored_config, command_line_config)
            iadm.create_core_datastores()
            iadm.store_config(stored_config)

        # Determine the final pyon_config:
        # - Start from standard config already set (pyon.yml + local YML files)
        # - Optionally load config from directory
        if opts.config_from_directory:
            config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config)
        # - Apply container profile specific config
        config.apply_profile_configuration(pyon_config, bootstrap_config)
        # - Reapply pyon.local.yml here again for good measure
        config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
        # - Last apply any separate command line config overrides
        config.apply_configuration(pyon_config, config_override)
        config.apply_configuration(pyon_config, command_line_config)
        iadm.set_config(pyon_config)

        # Set the immediate flag when command line override specified
        if opts.immediate:
            dict_merge(pyon_config, {"system": {"immediate": True}}, inplace=True)

        # Determine system bootmode for bootstrapping actions (unless explicitly specified)
        if not pyon_config.get_safe("bootmode"):
            set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode")
            if set_bootmode == "auto":
                if iadm.system_data_exists():
                    dict_merge(pyon_config, {"bootmode": "restart"}, inplace=True)
                log.info("System bootmode auto-detection is ON. Determined bootmode=%s", pyon_config.get_safe("bootmode", "initial"))
            elif set_bootmode == "secondary":
                dict_merge(pyon_config, {"bootmode": "secondary"}, inplace=True)
                log.info("System bootmode override. Set to bootmode=%s", pyon_config.get_safe("bootmode", ""))
        log.info("System in bootmode=%s", pyon_config.get_safe("bootmode", "initial"))

        # Bootstrap the pyon framework's core. Load configuration etc.
        bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)

        # Delete any queues/exchanges owned by sysname if option "broker_clean" is set
        if opts.broker_clean:
            log.info("broker_clean=True, sysname: %s", bootstrap.get_sys_name())

            from putil.rabbitmq.rabbit_util import RabbitManagementUtil
            rabbit_util = RabbitManagementUtil(pyon_config, sysname=bootstrap.get_sys_name())
            deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname()
            log.info("Exchanges deleted (%s): %s" % (len(deleted_exchanges), ", ".join(deleted_exchanges)))
            log.info("Queues deleted (%s): %s" % (len(deleted_queues), ", ".join(deleted_queues)))

        if opts.force_clean:
            from pyon.util.file_sys import FileSystem
            FileSystem._clean(pyon_config)

        # If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging
        if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True:
            iadm.store_interfaces(idempotent=True)
            iadm.declare_core_exchange_resources()

        iadm.close()

        if opts.no_container:
            log.info("no_container=True. Stopping here.")
            return None


        # Create the container instance
        from pyon.container.cc import Container
        container = Container(*args, **command_line_config)
        container.version = version

        return container
Esempio n. 47
0
    def spawn_process(self,
                      name=None,
                      module=None,
                      cls=None,
                      config=None,
                      process_id=None):
        """
        Spawn a process within the container. Processes can be of different type.
        """

        if process_id and not is_valid_identifier(process_id, ws_sub='_'):
            raise BadRequest("Given process_id %s is not a valid identifier" %
                             process_id)

        # Generate a new process id if not provided
        # TODO: Ensure it is system-wide unique
        process_id = process_id or "%s.%s" % (self.container.id,
                                              self.proc_id_pool.get_id())
        log.debug(
            "ProcManager.spawn_process(name=%s, module.cls=%s.%s, config=%s) as pid=%s",
            name, module, cls, config, process_id)

        process_cfg = deepcopy(CFG)
        if config:
            # Use provided config. Must be dict or DotDict
            if not isinstance(config, DotDict):
                config = DotDict(config)
            dict_merge(process_cfg, config, inplace=True)
            if self.container.spawn_args:
                # Override config with spawn args
                dict_merge(process_cfg,
                           self.container.spawn_args,
                           inplace=True)

        #log.debug("spawn_process() pid=%s process_cfg=%s", process_id, process_cfg)

        # PROCESS TYPE. Determines basic process context (messaging, service interface)
        # One of: service, stream_process, agent, simple, immediate

        service_cls = named_any("%s.%s" % (module, cls))
        process_type = get_safe(process_cfg, "process.type") or getattr(
            service_cls, "process_type", "service")

        process_start_mode = get_safe(config, "process.start_mode")

        process_instance = None

        # alert we have a spawning process, but we don't have the instance yet, so give the class instead (more accurate than name)
        self._call_proc_state_changed("%s.%s" % (module, cls),
                                      ProcessStateEnum.PENDING)

        try:
            # spawn service by type
            if process_type == "service":
                process_instance = self._spawn_service_process(
                    process_id, name, module, cls, process_cfg)

            elif process_type == "stream_process":
                process_instance = self._spawn_stream_process(
                    process_id, name, module, cls, process_cfg)

            elif process_type == "agent":
                process_instance = self._spawn_agent_process(
                    process_id, name, module, cls, process_cfg)

            elif process_type == "standalone":
                process_instance = self._spawn_standalone_process(
                    process_id, name, module, cls, process_cfg)

            elif process_type == "immediate":
                process_instance = self._spawn_immediate_process(
                    process_id, name, module, cls, process_cfg)

            elif process_type == "simple":
                process_instance = self._spawn_simple_process(
                    process_id, name, module, cls, process_cfg)

            else:
                raise BadRequest("Unknown process type: %s" % process_type)

            process_instance._proc_type = process_type
            self._register_process(process_instance, name)

            process_instance.errcause = "OK"
            log.info("ProcManager.spawn_process: %s.%s -> pid=%s OK", module,
                     cls, process_id)

            if process_type == 'immediate':
                log.info('Terminating immediate process: %s',
                         process_instance.id)
                self.terminate_process(process_instance.id)

                # terminate process also triggers TERMINATING/TERMINATED
                self._call_proc_state_changed(process_instance,
                                              ProcessStateEnum.EXITED)
            else:
                #Shouldn't be any policies for immediate processes
                self.update_container_policies(process_instance)

            return process_instance.id

        except IonProcessError:
            errcause = process_instance.errcause if process_instance else "instantiating process"
            log.exception("Error spawning %s %s process (process_id: %s): %s",
                          name, process_type, process_id, errcause)
            return None

        except Exception:
            errcause = process_instance.errcause if process_instance else "instantiating process"
            log.exception("Error spawning %s %s process (process_id: %s): %s",
                          name, process_type, process_id, errcause)

            # trigger failed notification - catches problems in init/start
            self._call_proc_state_changed(process_instance,
                                          ProcessStateEnum.FAILED)

            raise
Esempio n. 48
0
def _override_config(config_override):
    if type(config_override) is not dict:
        print "pyon: WARNING: config_override is not dict but", config_override
    from pyon.util.containers import dict_merge
    dict_merge(LOGGING_CFG, config_override)
Esempio n. 49
0
    def spawn_process(self, name=None, module=None, cls=None, config=None, process_id=None):
        """
        Spawn a process within the container. Processes can be of different type.
        """
        if process_id and not is_valid_identifier(process_id, ws_sub='_'):
            raise BadRequest("Given process_id %s is not a valid identifier" % process_id)

        # Generate a new process id if not provided
        # TODO: Ensure it is system-wide unique
        process_id = process_id or "%s.%s" % (self.container.id, self.proc_id_pool.get_id())
        log.debug("ProcManager.spawn_process(name=%s, module.cls=%s.%s, config=%s) as pid=%s", name, module, cls, config, process_id)

        process_cfg = deepcopy(CFG)
        if config:
            # Use provided config. Must be dict or DotDict
            if not isinstance(config, DotDict):
                config = DotDict(config)
            if config.get_safe("process.config_ref"):
                # Use a reference
                config_ref = config.get_safe("process.config_ref")
                log.info("Enhancing new process spawn config from ref=%s" % config_ref)
                matches = re.match(r'^([A-Za-z]+):([A-Za-z0-9]+)/(.+)$', config_ref)
                if matches:
                    ref_type, ref_id, ref_ext = matches.groups()
                    if ref_type == "resources":
                        if self.container.has_capability(self.container.CCAP.RESOURCE_REGISTRY):
                            try:
                                obj = self.container.resource_registry.read(ref_id)
                                if obj and hasattr(obj, ref_ext):
                                    ref_config = getattr(obj, ref_ext)
                                    if isinstance(ref_config, dict):
                                        dict_merge(process_cfg, ref_config, inplace=True)
                                    else:
                                        raise BadRequest("config_ref %s exists but not dict" % config_ref)
                                else:
                                    raise BadRequest("config_ref %s - attribute not found" % config_ref)
                            except NotFound as nf:
                                log.warn("config_ref %s - object not found" % config_ref)
                                raise
                        else:
                            log.error("Container missing RESOURCE_REGISTRY capability to resolve process config ref %s" % config_ref)
                    else:
                        raise BadRequest("Unknown reference type in: %s" % config_ref)

            dict_merge(process_cfg, config, inplace=True)
            if self.container.spawn_args:
                # Override config with spawn args
                dict_merge(process_cfg, self.container.spawn_args, inplace=True)

        #log.debug("spawn_process() pid=%s process_cfg=%s", process_id, process_cfg)

        # PROCESS TYPE. Determines basic process context (messaging, service interface)
        # One of the constants defined at the top of this file

        service_cls = named_any("%s.%s" % (module, cls))
        process_type = get_safe(process_cfg, "process.type") or getattr(service_cls, "process_type", "service")

        process_start_mode = get_safe(config, "process.start_mode")

        process_instance = None

        # alert we have a spawning process, but we don't have the instance yet, so give the class instead (more accurate than name)
        self._call_proc_state_changed("%s.%s" % (module, cls), ProcessStateEnum.PENDING)

        try:
            # spawn service by type
            if process_type == SERVICE_PROCESS_TYPE:
                process_instance = self._spawn_service_process(process_id, name, module, cls, process_cfg)

            elif process_type == STREAM_PROCESS_TYPE:
                process_instance = self._spawn_stream_process(process_id, name, module, cls, process_cfg)

            elif process_type == AGENT_PROCESS_TYPE:
                process_instance = self._spawn_agent_process(process_id, name, module, cls, process_cfg)

            elif process_type == STANDALONE_PROCESS_TYPE:
                process_instance = self._spawn_standalone_process(process_id, name, module, cls, process_cfg)

            elif process_type == IMMEDIATE_PROCESS_TYPE:
                process_instance = self._spawn_immediate_process(process_id, name, module, cls, process_cfg)

            elif process_type == SIMPLE_PROCESS_TYPE:
                process_instance = self._spawn_simple_process(process_id, name, module, cls, process_cfg)

            else:
                raise BadRequest("Unknown process type: %s" % process_type)

            process_instance._proc_type = process_type
            self._register_process(process_instance, name)

            process_instance.errcause = "OK"
            log.info("ProcManager.spawn_process: %s.%s -> pid=%s OK", module, cls, process_id)

            if process_type == IMMEDIATE_PROCESS_TYPE:
                log.info('Terminating immediate process: %s', process_instance.id)
                self.terminate_process(process_instance.id)

                # terminate process also triggers TERMINATING/TERMINATED
                self._call_proc_state_changed(process_instance, ProcessStateEnum.EXITED)

            else:
                #Update local policies for the new process
                if self.container.has_capability(self.container.CCAP.GOVERNANCE_CONTROLLER):
                    self.container.governance_controller.update_container_policies(process_instance, safe_mode=True)

            return process_instance.id

        except IonProcessError:
            errcause = process_instance.errcause if process_instance else "instantiating process"
            log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause)
            return None

        except Exception:
            errcause = process_instance.errcause if process_instance else "instantiating process"
            log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause)

            # trigger failed notification - catches problems in init/start
            self._call_proc_state_changed(process_instance, ProcessStateEnum.FAILED)

            raise
Esempio n. 50
0
def apply_configuration(system_cfg, config_override):
    if not config_override:
        return
    from pyon.util.containers import dict_merge
    dict_merge(system_cfg, config_override, inplace=True)
Esempio n. 51
0
    def spawn_process(self,
                      name=None,
                      module=None,
                      cls=None,
                      config=None,
                      process_id=None):
        """
        Spawn a process within the container. Processes can be of different type.
        """

        if process_id and not is_valid_identifier(process_id, ws_sub='_'):
            raise BadRequest("Given process_id %s is not a valid identifier" %
                             process_id)

        # Generate a new process id if not provided
        # TODO: Ensure it is system-wide unique
        process_id = process_id or "%s.%s" % (self.container.id,
                                              self.proc_id_pool.get_id())
        log.debug(
            "ProcManager.spawn_process(name=%s, module.cls=%s.%s) as pid=%s",
            name, module, cls, process_id)

        if not config:
            # Use system CFG. It has the command line args in it
            config = DictModifier(CFG)
        else:
            # Use provided config. Must be dict or DotDict
            if not isinstance(config, DotDict):
                config = DotDict(config)
            config = DictModifier(CFG, config)
            if self.container.spawn_args:
                # Override config with spawn args
                dict_merge(config, self.container.spawn_args, inplace=True)

        #log.debug("spawn_process() pid=%s config=%s", process_id, config)

        # PROCESS TYPE. Determines basic process context (messaging, service interface)
        # One of: service, stream_process, agent, simple, immediate

        service_cls = named_any("%s.%s" % (module, cls))
        process_type = get_safe(config, "process.type") or getattr(
            service_cls, "process_type", "service")

        service_instance = None
        try:
            # spawn service by type
            if process_type == "service":
                service_instance = self._spawn_service_process(
                    process_id, name, module, cls, config)

            elif process_type == "stream_process":
                service_instance = self._spawn_stream_process(
                    process_id, name, module, cls, config)

            elif process_type == "agent":
                service_instance = self._spawn_agent_process(
                    process_id, name, module, cls, config)

            elif process_type == "standalone":
                service_instance = self._spawn_standalone_process(
                    process_id, name, module, cls, config)

            elif process_type == "immediate":
                service_instance = self._spawn_immediate_process(
                    process_id, name, module, cls, config)

            elif process_type == "simple":
                service_instance = self._spawn_simple_process(
                    process_id, name, module, cls, config)

            else:
                raise BadRequest("Unknown process type: %s" % process_type)

            service_instance._proc_type = process_type
            self._register_process(service_instance, name)

            service_instance.errcause = "OK"
            log.info("AppManager.spawn_process: %s.%s -> pid=%s OK", module,
                     cls, process_id)

            if process_type == 'immediate':
                log.info('Terminating immediate process: %s',
                         service_instance.id)
                self.terminate_process(service_instance.id)

            return service_instance.id

        except Exception:
            errcause = service_instance.errcause if service_instance else "instantiating service"
            log.exception("Error spawning %s %s process (process_id: %s): %s",
                          name, process_type, process_id, errcause)
            raise
Esempio n. 52
0
    def _create_process_config(self, config):
        """ Prepare the config for the new process. Clone system config and apply process overrides.
        Support including config by reference of a resource attribute or object from object store.
        """
        process_cfg = deepcopy(CFG)
        if config:
            # Use provided config. Must be dict or DotDict
            if not isinstance(config, DotDict):
                config = DotDict(config)
            if config.get_safe("process.config_ref"):
                # Use a reference
                config_ref = config.get_safe("process.config_ref")
                log.info("Enhancing new process spawn config from ref=%s" % config_ref)
                matches = re.match(r'^([A-Za-z]+):([A-Za-z0-9_\.]+)/(.*)$', config_ref)
                if matches:
                    ref_type, ref_id, ref_ext = matches.groups()
                    if ref_type == "resources":
                        if self.container.has_capability(self.container.CCAP.RESOURCE_REGISTRY):
                            try:
                                obj = self.container.resource_registry.read(ref_id)
                                if obj and hasattr(obj, ref_ext):
                                    ref_config = getattr(obj, ref_ext)
                                    if isinstance(ref_config, dict):
                                        dict_merge(process_cfg, ref_config, inplace=True)
                                    else:
                                        raise BadRequest("config_ref %s exists but not dict" % config_ref)
                                else:
                                    raise BadRequest("config_ref %s - attribute not found" % config_ref)
                            except NotFound as nf:
                                log.warn("config_ref %s - object not found" % config_ref)
                                raise
                        else:
                            log.error("Container missing RESOURCE_REGISTRY capability to resolve process config ref %s" % config_ref)
                    elif ref_type == "objects":
                        if self.container.has_capability(self.container.CCAP.OBJECT_STORE):
                            try:
                                obj = self.container.object_store.read_doc(ref_id)
                                ref_config = obj
                                if ref_ext:
                                    ref_config = get_safe(obj, ref_ext, None)
                                    if ref_config is None:
                                        raise BadRequest("config_ref %s - attribute not found" % config_ref)

                                if isinstance(ref_config, dict):
                                    dict_merge(process_cfg, ref_config, inplace=True)
                                else:
                                    raise BadRequest("config_ref %s exists but not dict" % config_ref)
                            except NotFound as nf:
                                log.warn("config_ref %s - object not found" % config_ref)
                                raise
                        else:
                            log.error("Container missing OBJECT_STORE capability to resolve process config ref %s" % config_ref)
                    else:
                        raise BadRequest("Unknown reference type in: %s" % config_ref)

            dict_merge(process_cfg, config, inplace=True)
            if self.container.spawn_args:
                # Override config with spawn args
                dict_merge(process_cfg, self.container.spawn_args, inplace=True)

        #log.debug("spawn_process() pid=%s process_cfg=%s", process_id, process_cfg)
        return process_cfg
Esempio n. 53
0
    def prepare_container():
        """
        Walks through pyon initialization in a deterministic way and initializes Container.
        In particular make sure configuration is loaded in correct order and
        pycc startup arguments are considered.
        """
        import threading
        threading.current_thread().name = "CC-Main"

        # SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
        import pyon

        from pyon.core import bootstrap, config

        # Set global testing flag to False. We are running as capability container. This is NO TEST.
        bootstrap.testing = False

        # Set sysname if provided in startup argument
        if opts.sysname:
            bootstrap.set_sys_name(opts.sysname)
        # Trigger any initializing default logic in get_sys_name
        bootstrap.get_sys_name()

        command_line_config = kwargs

        # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
        bootstrap_config = None

        # This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
        pyon_config = config.read_standard_configuration()

        # Load config override if provided. Supports variants literal and list of paths
        config_override = None
        if opts.config:
            if '{' in opts.config:
                # Variant 1: Dict of config values
                try:
                    eval_value = ast.literal_eval(opts.config)
                    config_override = eval_value
                except ValueError:
                    raise Exception("Value error in config arg '%s'" %
                                    opts.config)
            else:
                # Variant 2: List of paths
                from pyon.util.config import Config
                config_override = Config([opts.config]).data

        # Determine bootstrap_config
        if opts.config_from_directory:
            # Load minimal bootstrap config if option "config_from_directory"
            bootstrap_config = config.read_local_configuration(
                ['res/config/pyon_min_boot.yml'])
            config.apply_local_configuration(bootstrap_config,
                                             pyon.DEFAULT_LOCAL_CONFIG_PATHS)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)
            print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
        else:
            # Otherwise: Set to standard set of local config files plus command line overrides
            bootstrap_config = pyon_config.copy()
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)

        # Override sysname from config file or command line
        if not opts.sysname and bootstrap_config.get_safe("system.name", None):
            new_sysname = bootstrap_config.get_safe("system.name")
            bootstrap.set_sys_name(new_sysname)

        # Delete sysname datastores if option "force_clean" is set
        if opts.force_clean:
            from pyon.datastore import clear_couch_util
            print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name(
            )
            clear_couch_util.clear_couch(bootstrap_config,
                                         prefix=bootstrap.get_sys_name())

        # If auto_bootstrap, load config and interfaces into directory
        # Note: this is idempotent and will not alter anything if this is not the first container to run
        if bootstrap_config.system.auto_bootstrap:
            print "pycc: auto_bootstrap=True."
            stored_config = pyon_config.copy()
            config.apply_configuration(stored_config, config_override)
            config.apply_configuration(stored_config, command_line_config)
            config.auto_bootstrap_config(bootstrap_config,
                                         system_cfg=stored_config)

        # Determine the final pyon_config
        # - Start from standard config already set (pyon.yml + local YML files)
        # - Optionally load config from directory
        if opts.config_from_directory:
            config.apply_remote_config(pyon_config)
        # - Last apply any separate command line config overrides
        config.apply_configuration(pyon_config, config_override)
        config.apply_configuration(pyon_config, command_line_config)

        # Load logging override config if provided. Supports variants literal and path.
        logging_config_override = None
        if opts.logcfg:
            if '{' in opts.logcfg:
                # Variant 1: Value is dict of config values
                try:
                    eval_value = ast.literal_eval(opts.logcfg)
                    logging_config_override = eval_value
                except ValueError:
                    raise Exception("Value error in logcfg arg '%s'" %
                                    opts.logcfg)
            else:
                # Variant 2: Value is path to YAML file containing config values
                pyon.DEFAULT_LOGGING_PATHS.append(opts.logcfg)

        # Also set the immediate flag, but only if specified - it is an override
        if opts.immediate:
            dict_merge(pyon_config, {'system': {'immediate': True}}, True)

        # Bootstrap pyon's core. Load configuration etc.
        bootstrap.bootstrap_pyon(
            logging_config_override=logging_config_override,
            pyon_cfg=pyon_config)

        # Auto-bootstrap interfaces
        # @WARN: This currently imports ALL modules, executing ALL static initializers as side effect!!!!!!!
        if bootstrap_config.system.auto_bootstrap:
            config.auto_bootstrap_interfaces(bootstrap_config)

        if opts.no_container:
            print "pycc: no_container=True. Stopping here."
            return None

        # Create the container instance
        from pyon.container.cc import Container
        container = Container(*args, **command_line_config)

        return container
Esempio n. 54
0
    def prepare_container():
        """
        Walks through pyon initialization in a deterministic way and initializes Container.
        In particular make sure configuration is loaded in correct order and
        pycc startup arguments are considered.
        """
        import threading
        threading.current_thread().name = "CC-Main"

        # SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
        import pyon

        from pyon.core import bootstrap, config

        # Set global testing flag to False. We are running as capability container. This is NO TEST.
        bootstrap.testing = False

        # Set sysname if provided in startup argument
        if opts.sysname:
            bootstrap.set_sys_name(opts.sysname)
        # Trigger any initializing default logic in get_sys_name
        bootstrap.get_sys_name()

        command_line_config = kwargs

        # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
        bootstrap_config = None

        # This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
        pyon_config = config.read_standard_configuration()

        # Load config override if provided. Supports variants literal and list of paths
        config_override = None
        if opts.config:
            if '{' in opts.config:
                # Variant 1: Dict of config values
                try:
                    eval_value = ast.literal_eval(opts.config)
                    config_override = eval_value
                except ValueError:
                    raise Exception("Value error in config arg '%s'" % opts.config)
            else:
                # Variant 2: List of paths
                from pyon.util.config import Config
                config_override = Config([opts.config]).data

        # Determine bootstrap_config
        if opts.config_from_directory:
            # Load minimal bootstrap config if option "config_from_directory"
            bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
            config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)
            print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
        else:
            # Otherwise: Set to standard set of local config files plus command line overrides
            bootstrap_config = pyon_config.copy()
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)

        # Override sysname from config file or command line
        if not opts.sysname and bootstrap_config.get_safe("system.name", None):
            new_sysname = bootstrap_config.get_safe("system.name")
            bootstrap.set_sys_name(new_sysname)

        # Delete sysname datastores if option "force_clean" is set
        if opts.force_clean:
            from pyon.datastore import clear_couch_util
            print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name()
            clear_couch_util.clear_couch(bootstrap_config, prefix=bootstrap.get_sys_name())

        # If auto_bootstrap, load config and interfaces into directory
        # Note: this is idempotent and will not alter anything if this is not the first container to run
        if bootstrap_config.system.auto_bootstrap:
            print "pycc: auto_bootstrap=True."
            stored_config = pyon_config.copy()
            config.apply_configuration(stored_config, config_override)
            config.apply_configuration(stored_config, command_line_config)
            config.auto_bootstrap_config(bootstrap_config, system_cfg=stored_config)

        # Determine the final pyon_config
        # - Start from standard config already set (pyon.yml + local YML files)
        # - Optionally load config from directory
        if opts.config_from_directory:
            config.apply_remote_config(pyon_config)
        # - Last apply any separate command line config overrides
        config.apply_configuration(pyon_config, config_override)
        config.apply_configuration(pyon_config, command_line_config)

        # Load logging override config if provided. Supports variants literal and path.
        logging_config_override = None
        if opts.logcfg:
            if '{' in opts.logcfg:
                # Variant 1: Value is dict of config values
                try:
                    eval_value = ast.literal_eval(opts.logcfg)
                    logging_config_override = eval_value
                except ValueError:
                    raise Exception("Value error in logcfg arg '%s'" % opts.logcfg)
            else:
                # Variant 2: Value is path to YAML file containing config values
                pyon.DEFAULT_LOGGING_PATHS.append(opts.logcfg)

        # Also set the immediate flag, but only if specified - it is an override
        if opts.immediate:
            dict_merge(pyon_config, {'system':{'immediate':True}}, True)

        # Bootstrap pyon's core. Load configuration etc.
        bootstrap.bootstrap_pyon(
            logging_config_override=logging_config_override,
            pyon_cfg=pyon_config)

        # Auto-bootstrap interfaces
        # @WARN: This currently imports ALL modules, executing ALL static initializers as side effect!!!!!!!
        if bootstrap_config.system.auto_bootstrap:
            config.auto_bootstrap_interfaces(bootstrap_config)

        if opts.no_container:
            print "pycc: no_container=True. Stopping here."
            return None

        # Create the container instance
        from pyon.container.cc import Container
        container = Container(*args, **command_line_config)

        return container
Esempio n. 55
0
def apply_configuration(system_cfg, config_override):
    if not config_override:
        return
    from pyon.util.containers import dict_merge
    dict_merge(system_cfg, config_override, inplace=True)