def _force_clean(cls, recreate=False, initial=False): # Database resources from pyon.core.bootstrap import get_sys_name, CFG from pyon.datastore.datastore_common import DatastoreFactory datastore = DatastoreFactory.get_datastore(config=CFG, variant=DatastoreFactory.DS_BASE, scope=get_sys_name()) if initial: datastore._init_database(datastore.database) dbs = datastore.list_datastores() clean_prefix = '%s_' % get_sys_name().lower() things_to_clean = [x for x in dbs if x.startswith(clean_prefix)] try: for thing in things_to_clean: datastore.delete_datastore(datastore_name=thing) if recreate: datastore.create_datastore(datastore_name=thing) finally: datastore.close() # Broker resources from putil.rabbitmq.rabbit_util import RabbitManagementUtil rabbit_util = RabbitManagementUtil(CFG, sysname=bootstrap.get_sys_name()) deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname() log.info("Deleted %s exchanges, %s queues" % (len(deleted_exchanges), len(deleted_queues))) # File system from pyon.util.file_sys import FileSystem FileSystem._clean(CFG)
def _make_management_call(self, url, method="get", data=None): """ Makes a call to the Rabbit HTTP management API using the passed in HTTP method. """ log.debug("Calling rabbit API management (%s): %s", method, url) meth = getattr(requests, method) try: from putil.rabbitmq.rabbit_util import RabbitManagementUtil mgmt_cfg = RabbitManagementUtil.get_mgmt_config(CFG) with gevent.timeout.Timeout(10): r = meth(url, auth=(mgmt_cfg["username"], mgmt_cfg["password"]), data=data) r.raise_for_status() if not r.content == "": content = json.loads(r.content) else: content = None except gevent.timeout.Timeout as ex: raise Timeout(str(ex)) except requests.exceptions.Timeout as ex: raise Timeout(str(ex)) except (requests.exceptions.ConnectionError, socket.error) as ex: raise ServiceUnavailable(str(ex)) except requests.exceptions.RequestException as ex: # the generic base exception all requests' exceptions inherit from, raise our # general server error too. raise ServerError(str(ex)) return content
def _make_management_call(self, url, method="get", data=None): """ Makes a call to the Rabbit HTTP management API using the passed in HTTP method. """ log.debug("Calling rabbit API management (%s): %s", method, url) meth = getattr(requests, method) try: from putil.rabbitmq.rabbit_util import RabbitManagementUtil mgmt_cfg = RabbitManagementUtil.get_mgmt_config(CFG) with gevent.timeout.Timeout(10): r = meth(url, auth=(mgmt_cfg["username"], mgmt_cfg["password"]), data=data) r.raise_for_status() if not r.content == "": content = json.loads(r.content) else: content = None except gevent.timeout.Timeout as ex: raise Timeout(str(ex)) except requests.exceptions.Timeout as ex: raise Timeout(str(ex)) except (requests.exceptions.ConnectionError, socket.error) as ex: raise ServiceUnavailable(str(ex)) except requests.exceptions.RequestException as ex: # the generic base exception all requests' exceptions inherit from, raise our # general server error too. raise ServerError(str(ex)) return content
def declare_core_exchange_resources(self): """ Create important messaging resources (queues, bindings, etc), so that they are ready before the first containers start and processes get spawned concurrently. """ from putil.rabbitmq.rabbit_util import RabbitManagementUtil rabbit_util = RabbitManagementUtil(self.config, sysname=self.sysname) # Event persister queue rabbit_util.declare_exchange("events") rabbit_util.declare_queue("events", "event_persister") rabbit_util.bind_queue("events", "event_persister", "#")
def _get_management_url(self, *feats): """ Builds a URL to be used with the Rabbit HTTP management API. """ node = self._priv_nodes.get(ION_DEFAULT_BROKER, self.default_node) host = node.client.parameters.host from putil.rabbitmq.rabbit_util import RabbitManagementUtil url = RabbitManagementUtil.get_mgmt_url(CFG, feats) return url
def _get_management_url(self, *feats): """ Builds a URL to be used with the Rabbit HTTP management API. """ node = self._priv_nodes.get(ION_DEFAULT_BROKER, self.default_node) host = node.client.parameters.host from putil.rabbitmq.rabbit_util import RabbitManagementUtil url = RabbitManagementUtil.get_mgmt_url(CFG, feats) return url
def declare_core_exchange_resources(self): """ Create important messaging resources (queues, bindings, etc), so that they are ready before the first containers start and processes get spawned concurrently. """ from putil.rabbitmq.rabbit_util import RabbitManagementUtil rabbit_util = RabbitManagementUtil(self.config, sysname=self.sysname) # Event persister queue rabbit_util.declare_exchange("events") rabbit_util.declare_queue("events", "event_persister") rabbit_util.bind_queue("events", "event_persister", "#")
def prepare_container(): """ Walks through pyon initialization in a deterministic way and initializes Container. In particular make sure configuration is loaded in correct order and pycc startup arguments are considered. """ # SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults import pyon import threading threading.current_thread().name = "CC-Main" import logging global log log = logging.getLogger('pycc') from pyon.core import bootstrap, config from pyon.util.containers import get_safe, dict_merge # Set global testing flag to False. We are running as capability container, because # we started through the pycc program. bootstrap.testing = False # Set sysname if provided in startup argument if opts.sysname: bootstrap.set_sys_name(opts.sysname) # Trigger any initializing default logic in get_sys_name bootstrap.get_sys_name() command_line_config = kwargs # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores. bootstrap_config = None # This holds the new CFG object for pyon. Build it up in proper sequence and conditions. pyon_config = config.read_standard_configuration() # Initial pyon.yml + pyon.local.yml # Load config override if provided. Supports variants literal and list of paths config_override = None if opts.config: if '{' in opts.config: # Variant 1: Dict of config values try: eval_value = ast.literal_eval(opts.config) config_override = eval_value except ValueError: raise Exception("Value error in config arg '%s'" % opts.config) else: # Variant 2: List of paths from pyon.util.config import Config config_override = Config([opts.config]).data # Determine bootstrap_config if opts.config_from_directory: # Load minimal bootstrap config if option "config_from_directory" bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml']) config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) log.info("config_from_directory=True. Minimal bootstrap configuration: %s", bootstrap_config) else: # Otherwise: Set to standard set of local config files plus command line overrides bootstrap_config = deepcopy(pyon_config) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) # Override sysname from config file or command line if not opts.sysname and bootstrap_config.get_safe("system.name", None): new_sysname = bootstrap_config.get_safe("system.name") bootstrap.set_sys_name(new_sysname) # Force_clean - deletes sysname datastores if opts.force_clean: from pyon.datastore import clear_db_util log.info("force_clean=True. DROP DATASTORES for sysname=%s", bootstrap.get_sys_name()) clear_db_util.clear_db(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name()) from pyon.core.interfaces.interfaces import InterfaceAdmin iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config) # If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config # WARNING: If multiple containers start concurrently, this may fail if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True: log.debug("auto_store_interfaces=True.") stored_config = deepcopy(pyon_config) config.apply_configuration(stored_config, config_override) config.apply_configuration(stored_config, command_line_config) iadm.create_core_datastores() iadm.store_config(stored_config) # Determine the final pyon_config: # - Start from standard config already set (pyon.yml + local YML files) # - Optionally load config from directory if opts.config_from_directory: config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config) # - Apply container profile specific config config.apply_profile_configuration(pyon_config, bootstrap_config) # - Reapply pyon.local.yml here again for good measure config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) # - Last apply any separate command line config overrides config.apply_configuration(pyon_config, config_override) config.apply_configuration(pyon_config, command_line_config) iadm.set_config(pyon_config) # Set the immediate flag when command line override specified if opts.immediate: dict_merge(pyon_config, {"system": {"immediate": True}}, inplace=True) # Determine system bootmode for bootstrapping actions (unless explicitly specified) if not pyon_config.get_safe("bootmode"): set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode") if set_bootmode == "auto": if iadm.system_data_exists(): dict_merge(pyon_config, {"bootmode": "restart"}, inplace=True) log.info("System bootmode auto-detection is ON. Determined bootmode=%s", pyon_config.get_safe("bootmode", "initial")) elif set_bootmode == "secondary": dict_merge(pyon_config, {"bootmode": "secondary"}, inplace=True) log.info("System bootmode override. Set to bootmode=%s", pyon_config.get_safe("bootmode", "")) log.info("System in bootmode=%s", pyon_config.get_safe("bootmode", "initial")) # Bootstrap the pyon framework's core. Load configuration etc. bootstrap.bootstrap_pyon(pyon_cfg=pyon_config) # Delete any queues/exchanges owned by sysname if option "broker_clean" is set if opts.broker_clean: log.info("broker_clean=True, sysname: %s", bootstrap.get_sys_name()) from putil.rabbitmq.rabbit_util import RabbitManagementUtil rabbit_util = RabbitManagementUtil(pyon_config, sysname=bootstrap.get_sys_name()) deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname() log.info("Exchanges deleted (%s): %s" % (len(deleted_exchanges), ", ".join(deleted_exchanges))) log.info("Queues deleted (%s): %s" % (len(deleted_queues), ", ".join(deleted_queues))) if opts.force_clean: from pyon.util.file_sys import FileSystem FileSystem._clean(pyon_config) # If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True: iadm.store_interfaces(idempotent=True) iadm.declare_core_exchange_resources() iadm.close() if opts.no_container: log.info("no_container=True. Stopping here.") return None # Create the container instance from pyon.container.cc import Container container = Container(*args, **command_line_config) container.version = version return container
def prepare_container(): """ Walks through pyon initialization in a deterministic way and initializes Container. In particular make sure configuration is loaded in correct order and pycc startup arguments are considered. """ # SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults import pyon import threading threading.current_thread().name = "CC-Main" import logging global log log = logging.getLogger('pycc') from pyon.core import bootstrap, config from pyon.util.containers import get_safe, dict_merge # Set global testing flag to False. We are running as capability container, because # we started through the pycc program. bootstrap.testing = False # Set sysname if provided in startup argument if opts.sysname: bootstrap.set_sys_name(opts.sysname) # Trigger any initializing default logic in get_sys_name bootstrap.get_sys_name() command_line_config = kwargs # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores. bootstrap_config = None # This holds the new CFG object for pyon. Build it up in proper sequence and conditions. pyon_config = config.read_standard_configuration( ) # Initial pyon.yml + pyon.local.yml # Load config override if provided. Supports variants literal and list of paths config_override = None if opts.config: if '{' in opts.config: # Variant 1: Dict of config values try: eval_value = ast.literal_eval(opts.config) config_override = eval_value except ValueError: raise Exception("Value error in config arg '%s'" % opts.config) else: # Variant 2: List of paths from pyon.util.config import Config config_override = Config([opts.config]).data # Determine bootstrap_config if opts.config_from_directory: # Load minimal bootstrap config if option "config_from_directory" bootstrap_config = config.read_local_configuration( ['res/config/pyon_min_boot.yml']) config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) log.info( "config_from_directory=True. Minimal bootstrap configuration: %s", bootstrap_config) else: # Otherwise: Set to standard set of local config files plus command line overrides bootstrap_config = deepcopy(pyon_config) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) # Override sysname from config file or command line if not opts.sysname and bootstrap_config.get_safe("system.name", None): new_sysname = bootstrap_config.get_safe("system.name") bootstrap.set_sys_name(new_sysname) # Force_clean - deletes sysname datastores if opts.force_clean: from pyon.datastore import clear_db_util log.info("force_clean=True. DROP DATASTORES for sysname=%s", bootstrap.get_sys_name()) clear_db_util.clear_db(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name()) from pyon.core.interfaces.interfaces import InterfaceAdmin iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config) # If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config # WARNING: If multiple containers start concurrently, this may fail if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True: log.debug("auto_store_interfaces=True.") stored_config = deepcopy(pyon_config) config.apply_configuration(stored_config, config_override) config.apply_configuration(stored_config, command_line_config) iadm.create_core_datastores() iadm.store_config(stored_config) # Determine the final pyon_config: # - Start from standard config already set (pyon.yml + local YML files) # - Optionally load config from directory if opts.config_from_directory: config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config) # - Apply container profile specific config config.apply_profile_configuration(pyon_config, bootstrap_config) # - Reapply pyon.local.yml here again for good measure config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) # - Last apply any separate command line config overrides config.apply_configuration(pyon_config, config_override) config.apply_configuration(pyon_config, command_line_config) iadm.set_config(pyon_config) # Set the immediate flag when command line override specified if opts.immediate: dict_merge(pyon_config, {"system": { "immediate": True }}, inplace=True) # Determine system bootmode for bootstrapping actions (unless explicitly specified) if not pyon_config.get_safe("bootmode"): set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode") if set_bootmode == "auto": if iadm.system_data_exists(): dict_merge(pyon_config, {"bootmode": "restart"}, inplace=True) log.info( "System bootmode auto-detection is ON. Determined bootmode=%s", pyon_config.get_safe("bootmode", "initial")) elif set_bootmode == "secondary": dict_merge(pyon_config, {"bootmode": "secondary"}, inplace=True) log.info("System bootmode override. Set to bootmode=%s", pyon_config.get_safe("bootmode", "")) log.info("System in bootmode=%s", pyon_config.get_safe("bootmode", "initial")) # Bootstrap the pyon framework's core. Load configuration etc. bootstrap.bootstrap_pyon(pyon_cfg=pyon_config) # Delete any queues/exchanges owned by sysname if option "broker_clean" is set if opts.broker_clean: log.info("broker_clean=True, sysname: %s", bootstrap.get_sys_name()) from putil.rabbitmq.rabbit_util import RabbitManagementUtil rabbit_util = RabbitManagementUtil( pyon_config, sysname=bootstrap.get_sys_name()) deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname() log.info("Exchanges deleted (%s): %s" % (len(deleted_exchanges), ", ".join(deleted_exchanges))) log.info("Queues deleted (%s): %s" % (len(deleted_queues), ", ".join(deleted_queues))) if opts.force_clean: from pyon.util.file_sys import FileSystem FileSystem._clean(pyon_config) # If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True: iadm.store_interfaces(idempotent=True) iadm.declare_core_exchange_resources() iadm.close() if opts.no_container: log.info("no_container=True. Stopping here.") return None # Create the container instance from pyon.container.cc import Container container = Container(*args, **command_line_config) container.version = version return container