Esempio n. 1
0
    def prepare_container():
        """
        Walks through pyon initialization in a deterministic way and initializes Container.
        In particular make sure configuration is loaded in correct order and
        pycc startup arguments are considered.
        """
        import threading
        threading.current_thread().name = "CC-Main"

        # SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
        import pyon

        from pyon.core import bootstrap, config

        # Set global testing flag to False. We are running as capability container. This is NO TEST.
        bootstrap.testing = False

        # Set sysname if provided in startup argument
        if opts.sysname:
            bootstrap.set_sys_name(opts.sysname)
        # Trigger any initializing default logic in get_sys_name
        bootstrap.get_sys_name()

        command_line_config = kwargs

        # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
        bootstrap_config = None

        # This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
        pyon_config = config.read_standard_configuration()

        # Load config override if provided. Supports variants literal and list of paths
        config_override = None
        if opts.config:
            if '{' in opts.config:
                # Variant 1: Dict of config values
                try:
                    eval_value = ast.literal_eval(opts.config)
                    config_override = eval_value
                except ValueError:
                    raise Exception("Value error in config arg '%s'" % opts.config)
            else:
                # Variant 2: List of paths
                from pyon.util.config import Config
                config_override = Config([opts.config]).data

        # Determine bootstrap_config
        if opts.config_from_directory:
            # Load minimal bootstrap config if option "config_from_directory"
            bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
            config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)
            print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
        else:
            # Otherwise: Set to standard set of local config files plus command line overrides
            bootstrap_config = pyon_config.copy()
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)

        # Override sysname from config file or command line
        if not opts.sysname and bootstrap_config.get_safe("system.name", None):
            new_sysname = bootstrap_config.get_safe("system.name")
            bootstrap.set_sys_name(new_sysname)

        # Delete sysname datastores if option "force_clean" is set
        if opts.force_clean:
            from pyon.datastore import clear_couch_util
            print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name()
            clear_couch_util.clear_couch(bootstrap_config, prefix=bootstrap.get_sys_name())

        # If auto_bootstrap, load config and interfaces into directory
        # Note: this is idempotent and will not alter anything if this is not the first container to run
        if bootstrap_config.system.auto_bootstrap:
            print "pycc: auto_bootstrap=True."
            stored_config = pyon_config.copy()
            config.apply_configuration(stored_config, config_override)
            config.apply_configuration(stored_config, command_line_config)
            config.auto_bootstrap_config(bootstrap_config, system_cfg=stored_config)

        # Determine the final pyon_config
        # - Start from standard config already set (pyon.yml + local YML files)
        # - Optionally load config from directory
        if opts.config_from_directory:
            config.apply_remote_config(pyon_config)
        # - Last apply any separate command line config overrides
        config.apply_configuration(pyon_config, config_override)
        config.apply_configuration(pyon_config, command_line_config)

        # Load logging override config if provided. Supports variants literal and path.
        logging_config_override = None
        if opts.logcfg:
            if '{' in opts.logcfg:
                # Variant 1: Value is dict of config values
                try:
                    eval_value = ast.literal_eval(opts.logcfg)
                    logging_config_override = eval_value
                except ValueError:
                    raise Exception("Value error in logcfg arg '%s'" % opts.logcfg)
            else:
                # Variant 2: Value is path to YAML file containing config values
                pyon.DEFAULT_LOGGING_PATHS.append(opts.logcfg)

        # Also set the immediate flag, but only if specified - it is an override
        if opts.immediate:
            dict_merge(pyon_config, {'system':{'immediate':True}}, True)

        # Bootstrap pyon's core. Load configuration etc.
        bootstrap.bootstrap_pyon(
            logging_config_override=logging_config_override,
            pyon_cfg=pyon_config)

        # Auto-bootstrap interfaces
        # @WARN: This currently imports ALL modules, executing ALL static initializers as side effect!!!!!!!
        if bootstrap_config.system.auto_bootstrap:
            config.auto_bootstrap_interfaces(bootstrap_config)

        if opts.no_container:
            print "pycc: no_container=True. Stopping here."
            return None

        # Create the container instance
        from pyon.container.cc import Container
        container = Container(*args, **command_line_config)

        return container
Esempio n. 2
0
File: pycc.py Progetto: ooici/pyon
    def prepare_container():
        """
        Walks through pyon initialization in a deterministic way and initializes Container.
        In particular make sure configuration is loaded in correct order and
        pycc startup arguments are considered.
        """
        import threading

        threading.current_thread().name = "CC-Main"

        # SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
        import pyon

        from pyon.core import bootstrap, config

        # Set global testing flag to False. We are running as capability container, because
        # we started through the pycc program.
        bootstrap.testing = False

        # Set sysname if provided in startup argument
        if opts.sysname:
            bootstrap.set_sys_name(opts.sysname)
        # Trigger any initializing default logic in get_sys_name
        bootstrap.get_sys_name()

        command_line_config = kwargs

        # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
        bootstrap_config = None

        # This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
        pyon_config = config.read_standard_configuration()  # Initial pyon.yml + pyon.local.yml

        # Load config override if provided. Supports variants literal and list of paths
        config_override = None
        if opts.config:
            if "{" in opts.config:
                # Variant 1: Dict of config values
                try:
                    eval_value = ast.literal_eval(opts.config)
                    config_override = eval_value
                except ValueError:
                    raise Exception("Value error in config arg '%s'" % opts.config)
            else:
                # Variant 2: List of paths
                from pyon.util.config import Config

                config_override = Config([opts.config]).data

        # Determine bootstrap_config
        if opts.config_from_directory:
            # Load minimal bootstrap config if option "config_from_directory"
            bootstrap_config = config.read_local_configuration(["res/config/pyon_min_boot.yml"])
            config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)
            print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
        else:
            # Otherwise: Set to standard set of local config files plus command line overrides
            bootstrap_config = deepcopy(pyon_config)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)

        # Override sysname from config file or command line
        if not opts.sysname and bootstrap_config.get_safe("system.name", None):
            new_sysname = bootstrap_config.get_safe("system.name")
            bootstrap.set_sys_name(new_sysname)

        # Delete sysname datastores if option "force_clean" is set
        if opts.force_clean:
            from pyon.datastore import clear_couch_util

            print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name()
            clear_couch_util.clear_couch(
                bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name()
            )
            pyon_config.container.filesystem.force_clean = True

        from pyon.core.interfaces.interfaces import InterfaceAdmin

        iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)

        # If auto_bootstrap, load config and interfaces into directory
        # Note: this is idempotent and will not alter anything if this is not the first container to run
        if bootstrap_config.system.auto_bootstrap:
            print "pycc: auto_bootstrap=True."
            stored_config = deepcopy(pyon_config)
            config.apply_configuration(stored_config, config_override)
            config.apply_configuration(stored_config, command_line_config)
            iadm.create_core_datastores()
            iadm.store_config(stored_config)

        # Determine the final pyon_config
        # - Start from standard config already set (pyon.yml + local YML files)
        # - Optionally load config from directory
        if opts.config_from_directory:
            config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config)
        # - Apply container profile specific config
        config.apply_profile_configuration(pyon_config, bootstrap_config)
        # - Reapply pyon.local.yml here again for good measure
        config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
        # - Last apply any separate command line config overrides
        config.apply_configuration(pyon_config, config_override)
        config.apply_configuration(pyon_config, command_line_config)

        # Also set the immediate flag, but only if specified - it is an override
        if opts.immediate:
            from pyon.util.containers import dict_merge

            dict_merge(pyon_config, {"system": {"immediate": True}}, True)

        # Bootstrap pyon's core. Load configuration etc.
        bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)

        # Delete any queues/exchanges owned by sysname if option "broker_clean" is set
        if opts.broker_clean:
            print "pycc: broker_clean=True, sysname:", bootstrap.get_sys_name()

            # build connect str
            connect_str = "-q -H %s -P %s -u %s -p %s -V %s" % (
                pyon_config.get_safe("server.amqp_priv.host", pyon_config.get_safe("server.amqp.host", "localhost")),
                pyon_config.get_safe("container.exchange.management.port", "55672"),
                pyon_config.get_safe("container.exchange.management.username", "guest"),
                pyon_config.get_safe("container.exchange.management.password", "guest"),
                "/",
            )

            from putil.rabbithelper import clean_by_sysname

            deleted_exchanges, deleted_queues = clean_by_sysname(connect_str, bootstrap.get_sys_name())
            print "      exchanges deleted (%s): %s" % (len(deleted_exchanges), ",".join(deleted_exchanges))
            print "         queues deleted (%s): %s" % (len(deleted_queues), ",".join(deleted_queues))

        if opts.force_clean:
            path = os.path.join(pyon_config.get_safe("container.filesystem.root", "/tmp/ion"), bootstrap.get_sys_name())
            print "force_clean: Removing", path
            FileSystem._clean(pyon_config)

        # Auto-bootstrap interfaces
        if bootstrap_config.system.auto_bootstrap:
            iadm.store_interfaces(idempotent=True)

        iadm.close()

        if opts.no_container:
            print "pycc: no_container=True. Stopping here."
            return None

        # Create the container instance
        from pyon.container.cc import Container

        container = Container(*args, **command_line_config)

        return container
Esempio n. 3
0
    def prepare_container():
        """
        Walks through pyon initialization in a deterministic way and initializes Container.
        In particular make sure configuration is loaded in correct order and
        pycc startup arguments are considered.
        """
        # SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults
        import pyon

        import threading
        threading.current_thread().name = "CC-Main"

        import logging
        global log
        log = logging.getLogger('pycc')

        from pyon.core import bootstrap, config
        from pyon.util.containers import get_safe, dict_merge

        # Set global testing flag to False. We are running as capability container, because
        # we started through the pycc program.
        bootstrap.testing = False

        # Set sysname if provided in startup argument
        if opts.sysname:
            bootstrap.set_sys_name(opts.sysname)
        # Trigger any initializing default logic in get_sys_name
        bootstrap.get_sys_name()

        command_line_config = kwargs

        # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
        bootstrap_config = None

        # This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
        pyon_config = config.read_standard_configuration()      # Initial pyon.yml + pyon.local.yml

        # Load config override if provided. Supports variants literal and list of paths
        config_override = None
        if opts.config:
            if '{' in opts.config:
                # Variant 1: Dict of config values
                try:
                    eval_value = ast.literal_eval(opts.config)
                    config_override = eval_value
                except ValueError:
                    raise Exception("Value error in config arg '%s'" % opts.config)
            else:
                # Variant 2: List of paths
                from pyon.util.config import Config
                config_override = Config([opts.config]).data

        # Determine bootstrap_config
        if opts.config_from_directory:
            # Load minimal bootstrap config if option "config_from_directory"
            bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
            config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)
            log.info("config_from_directory=True. Minimal bootstrap configuration: %s", bootstrap_config)
        else:
            # Otherwise: Set to standard set of local config files plus command line overrides
            bootstrap_config = deepcopy(pyon_config)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)

        # Override sysname from config file or command line
        if not opts.sysname and bootstrap_config.get_safe("system.name", None):
            new_sysname = bootstrap_config.get_safe("system.name")
            bootstrap.set_sys_name(new_sysname)

        # Force_clean - deletes sysname datastores
        if opts.force_clean:
            from pyon.datastore import clear_db_util
            log.info("force_clean=True. DROP DATASTORES for sysname=%s", bootstrap.get_sys_name())
            clear_db_util.clear_db(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name())

        from pyon.core.interfaces.interfaces import InterfaceAdmin
        iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)

        # If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config
        # WARNING: If multiple containers start concurrently, this may fail
        if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True:
            log.debug("auto_store_interfaces=True.")
            stored_config = deepcopy(pyon_config)
            config.apply_configuration(stored_config, config_override)
            config.apply_configuration(stored_config, command_line_config)
            iadm.create_core_datastores()
            iadm.store_config(stored_config)

        # Determine the final pyon_config:
        # - Start from standard config already set (pyon.yml + local YML files)
        # - Optionally load config from directory
        if opts.config_from_directory:
            config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config)
        # - Apply container profile specific config
        config.apply_profile_configuration(pyon_config, bootstrap_config)
        # - Reapply pyon.local.yml here again for good measure
        config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
        # - Last apply any separate command line config overrides
        config.apply_configuration(pyon_config, config_override)
        config.apply_configuration(pyon_config, command_line_config)
        iadm.set_config(pyon_config)

        # Set the immediate flag when command line override specified
        if opts.immediate:
            dict_merge(pyon_config, {"system": {"immediate": True}}, inplace=True)

        # Determine system bootmode for bootstrapping actions (unless explicitly specified)
        if not pyon_config.get_safe("bootmode"):
            set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode")
            if set_bootmode == "auto":
                if iadm.system_data_exists():
                    dict_merge(pyon_config, {"bootmode": "restart"}, inplace=True)
                log.info("System bootmode auto-detection is ON. Determined bootmode=%s", pyon_config.get_safe("bootmode", "initial"))
            elif set_bootmode == "secondary":
                dict_merge(pyon_config, {"bootmode": "secondary"}, inplace=True)
                log.info("System bootmode override. Set to bootmode=%s", pyon_config.get_safe("bootmode", ""))
        log.info("System in bootmode=%s", pyon_config.get_safe("bootmode", "initial"))

        # Bootstrap the pyon framework's core. Load configuration etc.
        bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)

        # Delete any queues/exchanges owned by sysname if option "broker_clean" is set
        if opts.broker_clean:
            log.info("broker_clean=True, sysname: %s", bootstrap.get_sys_name())

            from putil.rabbitmq.rabbit_util import RabbitManagementUtil
            rabbit_util = RabbitManagementUtil(pyon_config, sysname=bootstrap.get_sys_name())
            deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname()
            log.info("Exchanges deleted (%s): %s" % (len(deleted_exchanges), ", ".join(deleted_exchanges)))
            log.info("Queues deleted (%s): %s" % (len(deleted_queues), ", ".join(deleted_queues)))

        if opts.force_clean:
            from pyon.util.file_sys import FileSystem
            FileSystem._clean(pyon_config)

        # If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging
        if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True:
            iadm.store_interfaces(idempotent=True)
            iadm.declare_core_exchange_resources()

        iadm.close()

        if opts.no_container:
            log.info("no_container=True. Stopping here.")
            return None


        # Create the container instance
        from pyon.container.cc import Container
        container = Container(*args, **command_line_config)
        container.version = version

        return container
Esempio n. 4
0
    def prepare_container():
        """
        Walks through pyon initialization in a deterministic way and initializes Container.
        In particular make sure configuration is loaded in correct order and
        pycc startup arguments are considered.
        """
        import threading
        threading.current_thread().name = "CC-Main"

        # SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
        import pyon

        from pyon.core import bootstrap, config

        # Set global testing flag to False. We are running as capability container. This is NO TEST.
        bootstrap.testing = False

        # Set sysname if provided in startup argument
        if opts.sysname:
            bootstrap.set_sys_name(opts.sysname)
        # Trigger any initializing default logic in get_sys_name
        bootstrap.get_sys_name()

        command_line_config = kwargs

        # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
        bootstrap_config = None

        # This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
        pyon_config = config.read_standard_configuration(
        )  # Initial pyon.yml + pyon.local.yml

        # Load config override if provided. Supports variants literal and list of paths
        config_override = None
        if opts.config:
            if '{' in opts.config:
                # Variant 1: Dict of config values
                try:
                    eval_value = ast.literal_eval(opts.config)
                    config_override = eval_value
                except ValueError:
                    raise Exception("Value error in config arg '%s'" %
                                    opts.config)
            else:
                # Variant 2: List of paths
                from pyon.util.config import Config
                config_override = Config([opts.config]).data

        # Determine bootstrap_config
        if opts.config_from_directory:
            # Load minimal bootstrap config if option "config_from_directory"
            bootstrap_config = config.read_local_configuration(
                ['res/config/pyon_min_boot.yml'])
            config.apply_local_configuration(bootstrap_config,
                                             pyon.DEFAULT_LOCAL_CONFIG_PATHS)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)
            print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
        else:
            # Otherwise: Set to standard set of local config files plus command line overrides
            bootstrap_config = deepcopy(pyon_config)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)

        # Override sysname from config file or command line
        if not opts.sysname and bootstrap_config.get_safe("system.name", None):
            new_sysname = bootstrap_config.get_safe("system.name")
            bootstrap.set_sys_name(new_sysname)

        # Delete sysname datastores if option "force_clean" is set
        if opts.force_clean:
            from pyon.datastore import clear_couch_util
            print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name(
            )
            clear_couch_util.clear_couch(bootstrap_config,
                                         prefix=bootstrap.get_sys_name())
            pyon_config.container.filesystem.force_clean = True

        from pyon.core.interfaces.interfaces import InterfaceAdmin
        iadm = InterfaceAdmin(bootstrap.get_sys_name(),
                              config=bootstrap_config)

        # If auto_bootstrap, load config and interfaces into directory
        # Note: this is idempotent and will not alter anything if this is not the first container to run
        if bootstrap_config.system.auto_bootstrap:
            print "pycc: auto_bootstrap=True."
            stored_config = deepcopy(pyon_config)
            config.apply_configuration(stored_config, config_override)
            config.apply_configuration(stored_config, command_line_config)
            iadm.create_core_datastores()
            iadm.store_config(stored_config)

        # Determine the final pyon_config
        # - Start from standard config already set (pyon.yml + local YML files)
        # - Optionally load config from directory
        if opts.config_from_directory:
            config.apply_remote_config(bootstrap_cfg=bootstrap_config,
                                       system_cfg=pyon_config)
            config.apply_local_configuration(
                pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS
            )  # apply pyon.local.yml again over top
        # - Last apply any separate command line config overrides
        config.apply_configuration(pyon_config, config_override)
        config.apply_configuration(pyon_config, command_line_config)

        # Also set the immediate flag, but only if specified - it is an override
        if opts.immediate:
            from pyon.util.containers import dict_merge
            dict_merge(pyon_config, {'system': {'immediate': True}}, True)

        # Bootstrap pyon's core. Load configuration etc.
        bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)

        # Delete any queues/exchanges owned by sysname if option "broker_clean" is set
        if opts.broker_clean:
            print "pycc: broker_clean=True, sysname:", bootstrap.get_sys_name()

            # build connect str
            connect_str = "-q -H %s -P 55672 -u %s -p %s -V %s" % (
                pyon_config.get_safe(
                    'server.amqp_priv.host',
                    pyon_config.get_safe('server.amqp.host', 'localhost')),
                pyon_config.get_safe('container.exchange.management.username',
                                     'guest'),
                pyon_config.get_safe('container.exchange.management.password',
                                     'guest'), '/')

            from putil.rabbithelper import clean_by_sysname
            deleted_exchanges, deleted_queues = clean_by_sysname(
                connect_str, bootstrap.get_sys_name())
            print "      exchanges deleted (%s): %s" % (
                len(deleted_exchanges), ",".join(deleted_exchanges))
            print "         queues deleted (%s): %s" % (
                len(deleted_queues), ",".join(deleted_queues))

        # Auto-bootstrap interfaces
        if bootstrap_config.system.auto_bootstrap:
            iadm.store_interfaces(idempotent=True)

        iadm.close()

        if opts.no_container:
            print "pycc: no_container=True. Stopping here."
            return None

        # Create the container instance
        from pyon.container.cc import Container
        container = Container(*args, **command_line_config)

        return container
Esempio n. 5
0
    def prepare_container():
        """
        Walks through pyon initialization in a deterministic way and initializes Container.
        In particular make sure configuration is loaded in correct order and
        pycc startup arguments are considered.
        """
        # SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults
        import pyon

        import threading
        threading.current_thread().name = "CC-Main"

        import logging
        global log
        log = logging.getLogger('pycc')

        from pyon.core import bootstrap, config
        from pyon.util.containers import get_safe, dict_merge

        # Set global testing flag to False. We are running as capability container, because
        # we started through the pycc program.
        bootstrap.testing = False

        # Set sysname if provided in startup argument
        if opts.sysname:
            bootstrap.set_sys_name(opts.sysname)
        # Trigger any initializing default logic in get_sys_name
        bootstrap.get_sys_name()

        command_line_config = kwargs

        # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
        bootstrap_config = None

        # This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
        pyon_config = config.read_standard_configuration(
        )  # Initial pyon.yml + pyon.local.yml

        # Load config override if provided. Supports variants literal and list of paths
        config_override = None
        if opts.config:
            if '{' in opts.config:
                # Variant 1: Dict of config values
                try:
                    eval_value = ast.literal_eval(opts.config)
                    config_override = eval_value
                except ValueError:
                    raise Exception("Value error in config arg '%s'" %
                                    opts.config)
            else:
                # Variant 2: List of paths
                from pyon.util.config import Config
                config_override = Config([opts.config]).data

        # Determine bootstrap_config
        if opts.config_from_directory:
            # Load minimal bootstrap config if option "config_from_directory"
            bootstrap_config = config.read_local_configuration(
                ['res/config/pyon_min_boot.yml'])
            config.apply_local_configuration(bootstrap_config,
                                             pyon.DEFAULT_LOCAL_CONFIG_PATHS)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)
            log.info(
                "config_from_directory=True. Minimal bootstrap configuration: %s",
                bootstrap_config)
        else:
            # Otherwise: Set to standard set of local config files plus command line overrides
            bootstrap_config = deepcopy(pyon_config)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)

        # Override sysname from config file or command line
        if not opts.sysname and bootstrap_config.get_safe("system.name", None):
            new_sysname = bootstrap_config.get_safe("system.name")
            bootstrap.set_sys_name(new_sysname)

        # Force_clean - deletes sysname datastores
        if opts.force_clean:
            from pyon.datastore import clear_db_util
            log.info("force_clean=True. DROP DATASTORES for sysname=%s",
                     bootstrap.get_sys_name())
            clear_db_util.clear_db(bootstrap_config,
                                   prefix=bootstrap.get_sys_name(),
                                   sysname=bootstrap.get_sys_name())

        from pyon.core.interfaces.interfaces import InterfaceAdmin
        iadm = InterfaceAdmin(bootstrap.get_sys_name(),
                              config=bootstrap_config)

        # If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config
        # WARNING: If multiple containers start concurrently, this may fail
        if get_safe(bootstrap_config,
                    "bootstrap.auto_store_interfaces") is True:
            log.debug("auto_store_interfaces=True.")
            stored_config = deepcopy(pyon_config)
            config.apply_configuration(stored_config, config_override)
            config.apply_configuration(stored_config, command_line_config)
            iadm.create_core_datastores()
            iadm.store_config(stored_config)

        # Determine the final pyon_config:
        # - Start from standard config already set (pyon.yml + local YML files)
        # - Optionally load config from directory
        if opts.config_from_directory:
            config.apply_remote_config(bootstrap_cfg=bootstrap_config,
                                       system_cfg=pyon_config)
        # - Apply container profile specific config
        config.apply_profile_configuration(pyon_config, bootstrap_config)
        # - Reapply pyon.local.yml here again for good measure
        config.apply_local_configuration(pyon_config,
                                         pyon.DEFAULT_LOCAL_CONFIG_PATHS)
        # - Last apply any separate command line config overrides
        config.apply_configuration(pyon_config, config_override)
        config.apply_configuration(pyon_config, command_line_config)
        iadm.set_config(pyon_config)

        # Set the immediate flag when command line override specified
        if opts.immediate:
            dict_merge(pyon_config, {"system": {
                "immediate": True
            }},
                       inplace=True)

        # Determine system bootmode for bootstrapping actions (unless explicitly specified)
        if not pyon_config.get_safe("bootmode"):
            set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode")
            if set_bootmode == "auto":
                if iadm.system_data_exists():
                    dict_merge(pyon_config, {"bootmode": "restart"},
                               inplace=True)
                log.info(
                    "System bootmode auto-detection is ON. Determined bootmode=%s",
                    pyon_config.get_safe("bootmode", "initial"))
            elif set_bootmode == "secondary":
                dict_merge(pyon_config, {"bootmode": "secondary"},
                           inplace=True)
                log.info("System bootmode override. Set to bootmode=%s",
                         pyon_config.get_safe("bootmode", ""))
        log.info("System in bootmode=%s",
                 pyon_config.get_safe("bootmode", "initial"))

        # Bootstrap the pyon framework's core. Load configuration etc.
        bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)

        # Delete any queues/exchanges owned by sysname if option "broker_clean" is set
        if opts.broker_clean:
            log.info("broker_clean=True, sysname: %s",
                     bootstrap.get_sys_name())

            from putil.rabbitmq.rabbit_util import RabbitManagementUtil
            rabbit_util = RabbitManagementUtil(
                pyon_config, sysname=bootstrap.get_sys_name())
            deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname()
            log.info("Exchanges deleted (%s): %s" %
                     (len(deleted_exchanges), ", ".join(deleted_exchanges)))
            log.info("Queues deleted (%s): %s" %
                     (len(deleted_queues), ", ".join(deleted_queues)))

        if opts.force_clean:
            from pyon.util.file_sys import FileSystem
            FileSystem._clean(pyon_config)

        # If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging
        if get_safe(bootstrap_config,
                    "bootstrap.auto_store_interfaces") is True:
            iadm.store_interfaces(idempotent=True)
            iadm.declare_core_exchange_resources()

        iadm.close()

        if opts.no_container:
            log.info("no_container=True. Stopping here.")
            return None

        # Create the container instance
        from pyon.container.cc import Container
        container = Container(*args, **command_line_config)
        container.version = version

        return container
Esempio n. 6
0
    def prepare_container():
        """
        Walks through pyon initialization in a deterministic way and initializes Container.
        In particular make sure configuration is loaded in correct order and
        pycc startup arguments are considered.
        """
        import threading
        threading.current_thread().name = "CC-Main"

        # SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
        import pyon

        from pyon.core import bootstrap, config

        # Set global testing flag to False. We are running as capability container. This is NO TEST.
        bootstrap.testing = False

        # Set sysname if provided in startup argument
        if opts.sysname:
            bootstrap.set_sys_name(opts.sysname)
        # Trigger any initializing default logic in get_sys_name
        bootstrap.get_sys_name()

        command_line_config = kwargs

        # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
        bootstrap_config = None

        # This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
        pyon_config = config.read_standard_configuration()

        # Load config override if provided. Supports variants literal and list of paths
        config_override = None
        if opts.config:
            if '{' in opts.config:
                # Variant 1: Dict of config values
                try:
                    eval_value = ast.literal_eval(opts.config)
                    config_override = eval_value
                except ValueError:
                    raise Exception("Value error in config arg '%s'" %
                                    opts.config)
            else:
                # Variant 2: List of paths
                from pyon.util.config import Config
                config_override = Config([opts.config]).data

        # Determine bootstrap_config
        if opts.config_from_directory:
            # Load minimal bootstrap config if option "config_from_directory"
            bootstrap_config = config.read_local_configuration(
                ['res/config/pyon_min_boot.yml'])
            config.apply_local_configuration(bootstrap_config,
                                             pyon.DEFAULT_LOCAL_CONFIG_PATHS)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)
            print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
        else:
            # Otherwise: Set to standard set of local config files plus command line overrides
            bootstrap_config = pyon_config.copy()
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)

        # Override sysname from config file or command line
        if not opts.sysname and bootstrap_config.get_safe("system.name", None):
            new_sysname = bootstrap_config.get_safe("system.name")
            bootstrap.set_sys_name(new_sysname)

        # Delete sysname datastores if option "force_clean" is set
        if opts.force_clean:
            from pyon.datastore import clear_couch_util
            print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name(
            )
            clear_couch_util.clear_couch(bootstrap_config,
                                         prefix=bootstrap.get_sys_name())

        # If auto_bootstrap, load config and interfaces into directory
        # Note: this is idempotent and will not alter anything if this is not the first container to run
        if bootstrap_config.system.auto_bootstrap:
            print "pycc: auto_bootstrap=True."
            stored_config = pyon_config.copy()
            config.apply_configuration(stored_config, config_override)
            config.apply_configuration(stored_config, command_line_config)
            config.auto_bootstrap_config(bootstrap_config,
                                         system_cfg=stored_config)

        # Determine the final pyon_config
        # - Start from standard config already set (pyon.yml + local YML files)
        # - Optionally load config from directory
        if opts.config_from_directory:
            config.apply_remote_config(pyon_config)
        # - Last apply any separate command line config overrides
        config.apply_configuration(pyon_config, config_override)
        config.apply_configuration(pyon_config, command_line_config)

        # Load logging override config if provided. Supports variants literal and path.
        logging_config_override = None
        if opts.logcfg:
            if '{' in opts.logcfg:
                # Variant 1: Value is dict of config values
                try:
                    eval_value = ast.literal_eval(opts.logcfg)
                    logging_config_override = eval_value
                except ValueError:
                    raise Exception("Value error in logcfg arg '%s'" %
                                    opts.logcfg)
            else:
                # Variant 2: Value is path to YAML file containing config values
                pyon.DEFAULT_LOGGING_PATHS.append(opts.logcfg)

        # Also set the immediate flag, but only if specified - it is an override
        if opts.immediate:
            dict_merge(pyon_config, {'system': {'immediate': True}}, True)

        # Bootstrap pyon's core. Load configuration etc.
        bootstrap.bootstrap_pyon(
            logging_config_override=logging_config_override,
            pyon_cfg=pyon_config)

        # Auto-bootstrap interfaces
        # @WARN: This currently imports ALL modules, executing ALL static initializers as side effect!!!!!!!
        if bootstrap_config.system.auto_bootstrap:
            config.auto_bootstrap_interfaces(bootstrap_config)

        if opts.no_container:
            print "pycc: no_container=True. Stopping here."
            return None

        # Create the container instance
        from pyon.container.cc import Container
        container = Container(*args, **command_line_config)

        return container