Exemplo n.º 1
0
    def _force_clean(cls, recreate=False, initial=False):
        # Database resources
        from pyon.core.bootstrap import get_sys_name, CFG
        from pyon.datastore.datastore_common import DatastoreFactory
        datastore = DatastoreFactory.get_datastore(config=CFG, variant=DatastoreFactory.DS_BASE, scope=get_sys_name())
        if initial:
            datastore._init_database(datastore.database)

        dbs = datastore.list_datastores()
        clean_prefix = '%s_' % get_sys_name().lower()
        things_to_clean = [x for x in dbs if x.startswith(clean_prefix)]
        try:
            for thing in things_to_clean:
                datastore.delete_datastore(datastore_name=thing)
                if recreate:
                    datastore.create_datastore(datastore_name=thing)

        finally:
            datastore.close()

        # Broker resources
        from putil.rabbitmq.rabbit_util import RabbitManagementUtil
        rabbit_util = RabbitManagementUtil(CFG, sysname=bootstrap.get_sys_name())
        deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname()
        log.info("Deleted %s exchanges, %s queues" % (len(deleted_exchanges), len(deleted_queues)))

        # File system
        from pyon.util.file_sys import FileSystem
        FileSystem._clean(CFG)
Exemplo n.º 2
0
    def _make_management_call(self, url, method="get", data=None):
        """
        Makes a call to the Rabbit HTTP management API using the passed in HTTP method.
        """
        log.debug("Calling rabbit API management (%s): %s", method, url)

        meth = getattr(requests, method)

        try:
            from putil.rabbitmq.rabbit_util import RabbitManagementUtil
            mgmt_cfg = RabbitManagementUtil.get_mgmt_config(CFG)

            with gevent.timeout.Timeout(10):
                r = meth(url,
                         auth=(mgmt_cfg["username"], mgmt_cfg["password"]),
                         data=data)
            r.raise_for_status()

            if not r.content == "":
                content = json.loads(r.content)
            else:
                content = None

        except gevent.timeout.Timeout as ex:
            raise Timeout(str(ex))
        except requests.exceptions.Timeout as ex:
            raise Timeout(str(ex))
        except (requests.exceptions.ConnectionError, socket.error) as ex:
            raise ServiceUnavailable(str(ex))
        except requests.exceptions.RequestException as ex:
            # the generic base exception all requests' exceptions inherit from, raise our
            # general server error too.
            raise ServerError(str(ex))

        return content
Exemplo n.º 3
0
    def _make_management_call(self, url, method="get", data=None):
        """
        Makes a call to the Rabbit HTTP management API using the passed in HTTP method.
        """
        log.debug("Calling rabbit API management (%s): %s", method, url)

        meth = getattr(requests, method)

        try:
            from putil.rabbitmq.rabbit_util import RabbitManagementUtil
            mgmt_cfg = RabbitManagementUtil.get_mgmt_config(CFG)

            with gevent.timeout.Timeout(10):
                r = meth(url, auth=(mgmt_cfg["username"], mgmt_cfg["password"]), data=data)
            r.raise_for_status()

            if not r.content == "":
                content = json.loads(r.content)
            else:
                content = None

        except gevent.timeout.Timeout as ex:
            raise Timeout(str(ex))
        except requests.exceptions.Timeout as ex:
            raise Timeout(str(ex))
        except (requests.exceptions.ConnectionError, socket.error) as ex:
            raise ServiceUnavailable(str(ex))
        except requests.exceptions.RequestException as ex:
            # the generic base exception all requests' exceptions inherit from, raise our
            # general server error too.
            raise ServerError(str(ex))

        return content
Exemplo n.º 4
0
    def declare_core_exchange_resources(self):
        """
        Create important messaging resources (queues, bindings, etc), so that they are ready before
        the first containers start and processes get spawned concurrently.
        """
        from putil.rabbitmq.rabbit_util import RabbitManagementUtil
        rabbit_util = RabbitManagementUtil(self.config, sysname=self.sysname)

        # Event persister queue
        rabbit_util.declare_exchange("events")
        rabbit_util.declare_queue("events", "event_persister")
        rabbit_util.bind_queue("events", "event_persister", "#")
Exemplo n.º 5
0
    def _get_management_url(self, *feats):
        """
        Builds a URL to be used with the Rabbit HTTP management API.
        """
        node = self._priv_nodes.get(ION_DEFAULT_BROKER, self.default_node)
        host = node.client.parameters.host

        from putil.rabbitmq.rabbit_util import RabbitManagementUtil
        url = RabbitManagementUtil.get_mgmt_url(CFG, feats)

        return url
Exemplo n.º 6
0
    def _get_management_url(self, *feats):
        """
        Builds a URL to be used with the Rabbit HTTP management API.
        """
        node = self._priv_nodes.get(ION_DEFAULT_BROKER, self.default_node)
        host = node.client.parameters.host

        from putil.rabbitmq.rabbit_util import RabbitManagementUtil
        url = RabbitManagementUtil.get_mgmt_url(CFG, feats)

        return url
Exemplo n.º 7
0
    def declare_core_exchange_resources(self):
        """
        Create important messaging resources (queues, bindings, etc), so that they are ready before
        the first containers start and processes get spawned concurrently.
        """
        from putil.rabbitmq.rabbit_util import RabbitManagementUtil
        rabbit_util = RabbitManagementUtil(self.config, sysname=self.sysname)

        # Event persister queue
        rabbit_util.declare_exchange("events")
        rabbit_util.declare_queue("events", "event_persister")
        rabbit_util.bind_queue("events", "event_persister", "#")
Exemplo n.º 8
0
    def prepare_container():
        """
        Walks through pyon initialization in a deterministic way and initializes Container.
        In particular make sure configuration is loaded in correct order and
        pycc startup arguments are considered.
        """
        # SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults
        import pyon

        import threading
        threading.current_thread().name = "CC-Main"

        import logging
        global log
        log = logging.getLogger('pycc')

        from pyon.core import bootstrap, config
        from pyon.util.containers import get_safe, dict_merge

        # Set global testing flag to False. We are running as capability container, because
        # we started through the pycc program.
        bootstrap.testing = False

        # Set sysname if provided in startup argument
        if opts.sysname:
            bootstrap.set_sys_name(opts.sysname)
        # Trigger any initializing default logic in get_sys_name
        bootstrap.get_sys_name()

        command_line_config = kwargs

        # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
        bootstrap_config = None

        # This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
        pyon_config = config.read_standard_configuration()      # Initial pyon.yml + pyon.local.yml

        # Load config override if provided. Supports variants literal and list of paths
        config_override = None
        if opts.config:
            if '{' in opts.config:
                # Variant 1: Dict of config values
                try:
                    eval_value = ast.literal_eval(opts.config)
                    config_override = eval_value
                except ValueError:
                    raise Exception("Value error in config arg '%s'" % opts.config)
            else:
                # Variant 2: List of paths
                from pyon.util.config import Config
                config_override = Config([opts.config]).data

        # Determine bootstrap_config
        if opts.config_from_directory:
            # Load minimal bootstrap config if option "config_from_directory"
            bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
            config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)
            log.info("config_from_directory=True. Minimal bootstrap configuration: %s", bootstrap_config)
        else:
            # Otherwise: Set to standard set of local config files plus command line overrides
            bootstrap_config = deepcopy(pyon_config)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)

        # Override sysname from config file or command line
        if not opts.sysname and bootstrap_config.get_safe("system.name", None):
            new_sysname = bootstrap_config.get_safe("system.name")
            bootstrap.set_sys_name(new_sysname)

        # Force_clean - deletes sysname datastores
        if opts.force_clean:
            from pyon.datastore import clear_db_util
            log.info("force_clean=True. DROP DATASTORES for sysname=%s", bootstrap.get_sys_name())
            clear_db_util.clear_db(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name())

        from pyon.core.interfaces.interfaces import InterfaceAdmin
        iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)

        # If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config
        # WARNING: If multiple containers start concurrently, this may fail
        if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True:
            log.debug("auto_store_interfaces=True.")
            stored_config = deepcopy(pyon_config)
            config.apply_configuration(stored_config, config_override)
            config.apply_configuration(stored_config, command_line_config)
            iadm.create_core_datastores()
            iadm.store_config(stored_config)

        # Determine the final pyon_config:
        # - Start from standard config already set (pyon.yml + local YML files)
        # - Optionally load config from directory
        if opts.config_from_directory:
            config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config)
        # - Apply container profile specific config
        config.apply_profile_configuration(pyon_config, bootstrap_config)
        # - Reapply pyon.local.yml here again for good measure
        config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
        # - Last apply any separate command line config overrides
        config.apply_configuration(pyon_config, config_override)
        config.apply_configuration(pyon_config, command_line_config)
        iadm.set_config(pyon_config)

        # Set the immediate flag when command line override specified
        if opts.immediate:
            dict_merge(pyon_config, {"system": {"immediate": True}}, inplace=True)

        # Determine system bootmode for bootstrapping actions (unless explicitly specified)
        if not pyon_config.get_safe("bootmode"):
            set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode")
            if set_bootmode == "auto":
                if iadm.system_data_exists():
                    dict_merge(pyon_config, {"bootmode": "restart"}, inplace=True)
                log.info("System bootmode auto-detection is ON. Determined bootmode=%s", pyon_config.get_safe("bootmode", "initial"))
            elif set_bootmode == "secondary":
                dict_merge(pyon_config, {"bootmode": "secondary"}, inplace=True)
                log.info("System bootmode override. Set to bootmode=%s", pyon_config.get_safe("bootmode", ""))
        log.info("System in bootmode=%s", pyon_config.get_safe("bootmode", "initial"))

        # Bootstrap the pyon framework's core. Load configuration etc.
        bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)

        # Delete any queues/exchanges owned by sysname if option "broker_clean" is set
        if opts.broker_clean:
            log.info("broker_clean=True, sysname: %s", bootstrap.get_sys_name())

            from putil.rabbitmq.rabbit_util import RabbitManagementUtil
            rabbit_util = RabbitManagementUtil(pyon_config, sysname=bootstrap.get_sys_name())
            deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname()
            log.info("Exchanges deleted (%s): %s" % (len(deleted_exchanges), ", ".join(deleted_exchanges)))
            log.info("Queues deleted (%s): %s" % (len(deleted_queues), ", ".join(deleted_queues)))

        if opts.force_clean:
            from pyon.util.file_sys import FileSystem
            FileSystem._clean(pyon_config)

        # If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging
        if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True:
            iadm.store_interfaces(idempotent=True)
            iadm.declare_core_exchange_resources()

        iadm.close()

        if opts.no_container:
            log.info("no_container=True. Stopping here.")
            return None


        # Create the container instance
        from pyon.container.cc import Container
        container = Container(*args, **command_line_config)
        container.version = version

        return container
Exemplo n.º 9
0
    def prepare_container():
        """
        Walks through pyon initialization in a deterministic way and initializes Container.
        In particular make sure configuration is loaded in correct order and
        pycc startup arguments are considered.
        """
        # SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults
        import pyon

        import threading
        threading.current_thread().name = "CC-Main"

        import logging
        global log
        log = logging.getLogger('pycc')

        from pyon.core import bootstrap, config
        from pyon.util.containers import get_safe, dict_merge

        # Set global testing flag to False. We are running as capability container, because
        # we started through the pycc program.
        bootstrap.testing = False

        # Set sysname if provided in startup argument
        if opts.sysname:
            bootstrap.set_sys_name(opts.sysname)
        # Trigger any initializing default logic in get_sys_name
        bootstrap.get_sys_name()

        command_line_config = kwargs

        # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
        bootstrap_config = None

        # This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
        pyon_config = config.read_standard_configuration(
        )  # Initial pyon.yml + pyon.local.yml

        # Load config override if provided. Supports variants literal and list of paths
        config_override = None
        if opts.config:
            if '{' in opts.config:
                # Variant 1: Dict of config values
                try:
                    eval_value = ast.literal_eval(opts.config)
                    config_override = eval_value
                except ValueError:
                    raise Exception("Value error in config arg '%s'" %
                                    opts.config)
            else:
                # Variant 2: List of paths
                from pyon.util.config import Config
                config_override = Config([opts.config]).data

        # Determine bootstrap_config
        if opts.config_from_directory:
            # Load minimal bootstrap config if option "config_from_directory"
            bootstrap_config = config.read_local_configuration(
                ['res/config/pyon_min_boot.yml'])
            config.apply_local_configuration(bootstrap_config,
                                             pyon.DEFAULT_LOCAL_CONFIG_PATHS)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)
            log.info(
                "config_from_directory=True. Minimal bootstrap configuration: %s",
                bootstrap_config)
        else:
            # Otherwise: Set to standard set of local config files plus command line overrides
            bootstrap_config = deepcopy(pyon_config)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)

        # Override sysname from config file or command line
        if not opts.sysname and bootstrap_config.get_safe("system.name", None):
            new_sysname = bootstrap_config.get_safe("system.name")
            bootstrap.set_sys_name(new_sysname)

        # Force_clean - deletes sysname datastores
        if opts.force_clean:
            from pyon.datastore import clear_db_util
            log.info("force_clean=True. DROP DATASTORES for sysname=%s",
                     bootstrap.get_sys_name())
            clear_db_util.clear_db(bootstrap_config,
                                   prefix=bootstrap.get_sys_name(),
                                   sysname=bootstrap.get_sys_name())

        from pyon.core.interfaces.interfaces import InterfaceAdmin
        iadm = InterfaceAdmin(bootstrap.get_sys_name(),
                              config=bootstrap_config)

        # If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config
        # WARNING: If multiple containers start concurrently, this may fail
        if get_safe(bootstrap_config,
                    "bootstrap.auto_store_interfaces") is True:
            log.debug("auto_store_interfaces=True.")
            stored_config = deepcopy(pyon_config)
            config.apply_configuration(stored_config, config_override)
            config.apply_configuration(stored_config, command_line_config)
            iadm.create_core_datastores()
            iadm.store_config(stored_config)

        # Determine the final pyon_config:
        # - Start from standard config already set (pyon.yml + local YML files)
        # - Optionally load config from directory
        if opts.config_from_directory:
            config.apply_remote_config(bootstrap_cfg=bootstrap_config,
                                       system_cfg=pyon_config)
        # - Apply container profile specific config
        config.apply_profile_configuration(pyon_config, bootstrap_config)
        # - Reapply pyon.local.yml here again for good measure
        config.apply_local_configuration(pyon_config,
                                         pyon.DEFAULT_LOCAL_CONFIG_PATHS)
        # - Last apply any separate command line config overrides
        config.apply_configuration(pyon_config, config_override)
        config.apply_configuration(pyon_config, command_line_config)
        iadm.set_config(pyon_config)

        # Set the immediate flag when command line override specified
        if opts.immediate:
            dict_merge(pyon_config, {"system": {
                "immediate": True
            }},
                       inplace=True)

        # Determine system bootmode for bootstrapping actions (unless explicitly specified)
        if not pyon_config.get_safe("bootmode"):
            set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode")
            if set_bootmode == "auto":
                if iadm.system_data_exists():
                    dict_merge(pyon_config, {"bootmode": "restart"},
                               inplace=True)
                log.info(
                    "System bootmode auto-detection is ON. Determined bootmode=%s",
                    pyon_config.get_safe("bootmode", "initial"))
            elif set_bootmode == "secondary":
                dict_merge(pyon_config, {"bootmode": "secondary"},
                           inplace=True)
                log.info("System bootmode override. Set to bootmode=%s",
                         pyon_config.get_safe("bootmode", ""))
        log.info("System in bootmode=%s",
                 pyon_config.get_safe("bootmode", "initial"))

        # Bootstrap the pyon framework's core. Load configuration etc.
        bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)

        # Delete any queues/exchanges owned by sysname if option "broker_clean" is set
        if opts.broker_clean:
            log.info("broker_clean=True, sysname: %s",
                     bootstrap.get_sys_name())

            from putil.rabbitmq.rabbit_util import RabbitManagementUtil
            rabbit_util = RabbitManagementUtil(
                pyon_config, sysname=bootstrap.get_sys_name())
            deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname()
            log.info("Exchanges deleted (%s): %s" %
                     (len(deleted_exchanges), ", ".join(deleted_exchanges)))
            log.info("Queues deleted (%s): %s" %
                     (len(deleted_queues), ", ".join(deleted_queues)))

        if opts.force_clean:
            from pyon.util.file_sys import FileSystem
            FileSystem._clean(pyon_config)

        # If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging
        if get_safe(bootstrap_config,
                    "bootstrap.auto_store_interfaces") is True:
            iadm.store_interfaces(idempotent=True)
            iadm.declare_core_exchange_resources()

        iadm.close()

        if opts.no_container:
            log.info("no_container=True. Stopping here.")
            return None

        # Create the container instance
        from pyon.container.cc import Container
        container = Container(*args, **command_line_config)
        container.version = version

        return container