def _force_clean(cls, recreate=False, initial=False):
# Database resources
from pyon.core.bootstrap import get_sys_name, CFG
from pyon.datastore.datastore_common import DatastoreFactory
datastore = DatastoreFactory.get_datastore(config=CFG, variant=DatastoreFactory.DS_BASE, scope=get_sys_name())
if initial:
datastore._init_database(datastore.database)
dbs = datastore.list_datastores()
clean_prefix = '%s_' % get_sys_name().lower()
things_to_clean = [x for x in dbs if x.startswith(clean_prefix)]
try:
for thing in things_to_clean:
datastore.delete_datastore(datastore_name=thing)
if recreate:
datastore.create_datastore(datastore_name=thing)
finally:
datastore.close()
# Broker resources
from putil.rabbitmq.rabbit_util import RabbitManagementUtil
rabbit_util = RabbitManagementUtil(CFG, sysname=bootstrap.get_sys_name())
deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname()
log.info("Deleted %s exchanges, %s queues" % (len(deleted_exchanges), len(deleted_queues)))
# File system
from pyon.util.file_sys import FileSystem
FileSystem._clean(CFG)
def _make_management_call(self, url, method="get", data=None):
"""
Makes a call to the Rabbit HTTP management API using the passed in HTTP method.
"""
log.debug("Calling rabbit API management (%s): %s", method, url)
meth = getattr(requests, method)
try:
from putil.rabbitmq.rabbit_util import RabbitManagementUtil
mgmt_cfg = RabbitManagementUtil.get_mgmt_config(CFG)
with gevent.timeout.Timeout(10):
r = meth(url,
auth=(mgmt_cfg["username"], mgmt_cfg["password"]),
data=data)
r.raise_for_status()
if not r.content == "":
content = json.loads(r.content)
else:
content = None
except gevent.timeout.Timeout as ex:
raise Timeout(str(ex))
except requests.exceptions.Timeout as ex:
raise Timeout(str(ex))
except (requests.exceptions.ConnectionError, socket.error) as ex:
raise ServiceUnavailable(str(ex))
except requests.exceptions.RequestException as ex:
# the generic base exception all requests' exceptions inherit from, raise our
# general server error too.
raise ServerError(str(ex))
return content
def _make_management_call(self, url, method="get", data=None):
"""
Makes a call to the Rabbit HTTP management API using the passed in HTTP method.
"""
log.debug("Calling rabbit API management (%s): %s", method, url)
meth = getattr(requests, method)
try:
from putil.rabbitmq.rabbit_util import RabbitManagementUtil
mgmt_cfg = RabbitManagementUtil.get_mgmt_config(CFG)
with gevent.timeout.Timeout(10):
r = meth(url, auth=(mgmt_cfg["username"], mgmt_cfg["password"]), data=data)
r.raise_for_status()
if not r.content == "":
content = json.loads(r.content)
else:
content = None
except gevent.timeout.Timeout as ex:
raise Timeout(str(ex))
except requests.exceptions.Timeout as ex:
raise Timeout(str(ex))
except (requests.exceptions.ConnectionError, socket.error) as ex:
raise ServiceUnavailable(str(ex))
except requests.exceptions.RequestException as ex:
# the generic base exception all requests' exceptions inherit from, raise our
# general server error too.
raise ServerError(str(ex))
return content
def declare_core_exchange_resources(self):
"""
Create important messaging resources (queues, bindings, etc), so that they are ready before
the first containers start and processes get spawned concurrently.
"""
from putil.rabbitmq.rabbit_util import RabbitManagementUtil
rabbit_util = RabbitManagementUtil(self.config, sysname=self.sysname)
# Event persister queue
rabbit_util.declare_exchange("events")
rabbit_util.declare_queue("events", "event_persister")
rabbit_util.bind_queue("events", "event_persister", "#")
def _get_management_url(self, *feats):
"""
Builds a URL to be used with the Rabbit HTTP management API.
"""
node = self._priv_nodes.get(ION_DEFAULT_BROKER, self.default_node)
host = node.client.parameters.host
from putil.rabbitmq.rabbit_util import RabbitManagementUtil
url = RabbitManagementUtil.get_mgmt_url(CFG, feats)
return url
def _get_management_url(self, *feats):
"""
Builds a URL to be used with the Rabbit HTTP management API.
"""
node = self._priv_nodes.get(ION_DEFAULT_BROKER, self.default_node)
host = node.client.parameters.host
from putil.rabbitmq.rabbit_util import RabbitManagementUtil
url = RabbitManagementUtil.get_mgmt_url(CFG, feats)
return url
def declare_core_exchange_resources(self):
"""
Create important messaging resources (queues, bindings, etc), so that they are ready before
the first containers start and processes get spawned concurrently.
"""
from putil.rabbitmq.rabbit_util import RabbitManagementUtil
rabbit_util = RabbitManagementUtil(self.config, sysname=self.sysname)
# Event persister queue
rabbit_util.declare_exchange("events")
rabbit_util.declare_queue("events", "event_persister")
rabbit_util.bind_queue("events", "event_persister", "#")
def prepare_container():
"""
Walks through pyon initialization in a deterministic way and initializes Container.
In particular make sure configuration is loaded in correct order and
pycc startup arguments are considered.
"""
# SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults
import pyon
import threading
threading.current_thread().name = "CC-Main"
import logging
global log
log = logging.getLogger('pycc')
from pyon.core import bootstrap, config
from pyon.util.containers import get_safe, dict_merge
# Set global testing flag to False. We are running as capability container, because
# we started through the pycc program.
bootstrap.testing = False
# Set sysname if provided in startup argument
if opts.sysname:
bootstrap.set_sys_name(opts.sysname)
# Trigger any initializing default logic in get_sys_name
bootstrap.get_sys_name()
command_line_config = kwargs
# This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
bootstrap_config = None
# This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
pyon_config = config.read_standard_configuration() # Initial pyon.yml + pyon.local.yml
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if opts.config:
if '{' in opts.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(opts.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" % opts.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([opts.config]).data
# Determine bootstrap_config
if opts.config_from_directory:
# Load minimal bootstrap config if option "config_from_directory"
bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
log.info("config_from_directory=True. Minimal bootstrap configuration: %s", bootstrap_config)
else:
# Otherwise: Set to standard set of local config files plus command line overrides
bootstrap_config = deepcopy(pyon_config)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not opts.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Force_clean - deletes sysname datastores
if opts.force_clean:
from pyon.datastore import clear_db_util
log.info("force_clean=True. DROP DATASTORES for sysname=%s", bootstrap.get_sys_name())
clear_db_util.clear_db(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name())
from pyon.core.interfaces.interfaces import InterfaceAdmin
iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)
# If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config
# WARNING: If multiple containers start concurrently, this may fail
if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True:
log.debug("auto_store_interfaces=True.")
stored_config = deepcopy(pyon_config)
config.apply_configuration(stored_config, config_override)
config.apply_configuration(stored_config, command_line_config)
iadm.create_core_datastores()
iadm.store_config(stored_config)
# Determine the final pyon_config:
# - Start from standard config already set (pyon.yml + local YML files)
# - Optionally load config from directory
if opts.config_from_directory:
config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config)
# - Apply container profile specific config
config.apply_profile_configuration(pyon_config, bootstrap_config)
# - Reapply pyon.local.yml here again for good measure
config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
# - Last apply any separate command line config overrides
config.apply_configuration(pyon_config, config_override)
config.apply_configuration(pyon_config, command_line_config)
iadm.set_config(pyon_config)
# Set the immediate flag when command line override specified
if opts.immediate:
dict_merge(pyon_config, {"system": {"immediate": True}}, inplace=True)
# Determine system bootmode for bootstrapping actions (unless explicitly specified)
if not pyon_config.get_safe("bootmode"):
set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode")
if set_bootmode == "auto":
if iadm.system_data_exists():
dict_merge(pyon_config, {"bootmode": "restart"}, inplace=True)
log.info("System bootmode auto-detection is ON. Determined bootmode=%s", pyon_config.get_safe("bootmode", "initial"))
elif set_bootmode == "secondary":
dict_merge(pyon_config, {"bootmode": "secondary"}, inplace=True)
log.info("System bootmode override. Set to bootmode=%s", pyon_config.get_safe("bootmode", ""))
log.info("System in bootmode=%s", pyon_config.get_safe("bootmode", "initial"))
# Bootstrap the pyon framework's core. Load configuration etc.
bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)
# Delete any queues/exchanges owned by sysname if option "broker_clean" is set
if opts.broker_clean:
log.info("broker_clean=True, sysname: %s", bootstrap.get_sys_name())
from putil.rabbitmq.rabbit_util import RabbitManagementUtil
rabbit_util = RabbitManagementUtil(pyon_config, sysname=bootstrap.get_sys_name())
deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname()
log.info("Exchanges deleted (%s): %s" % (len(deleted_exchanges), ", ".join(deleted_exchanges)))
log.info("Queues deleted (%s): %s" % (len(deleted_queues), ", ".join(deleted_queues)))
if opts.force_clean:
from pyon.util.file_sys import FileSystem
FileSystem._clean(pyon_config)
# If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging
if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True:
iadm.store_interfaces(idempotent=True)
iadm.declare_core_exchange_resources()
iadm.close()
if opts.no_container:
log.info("no_container=True. Stopping here.")
return None
# Create the container instance
from pyon.container.cc import Container
container = Container(*args, **command_line_config)
container.version = version
return container
def prepare_container():
"""
Walks through pyon initialization in a deterministic way and initializes Container.
In particular make sure configuration is loaded in correct order and
pycc startup arguments are considered.
"""
# SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults
import pyon
import threading
threading.current_thread().name = "CC-Main"
import logging
global log
log = logging.getLogger('pycc')
from pyon.core import bootstrap, config
from pyon.util.containers import get_safe, dict_merge
# Set global testing flag to False. We are running as capability container, because
# we started through the pycc program.
bootstrap.testing = False
# Set sysname if provided in startup argument
if opts.sysname:
bootstrap.set_sys_name(opts.sysname)
# Trigger any initializing default logic in get_sys_name
bootstrap.get_sys_name()
command_line_config = kwargs
# This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
bootstrap_config = None
# This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
pyon_config = config.read_standard_configuration(
) # Initial pyon.yml + pyon.local.yml
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if opts.config:
if '{' in opts.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(opts.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" %
opts.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([opts.config]).data
# Determine bootstrap_config
if opts.config_from_directory:
# Load minimal bootstrap config if option "config_from_directory"
bootstrap_config = config.read_local_configuration(
['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config,
pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
log.info(
"config_from_directory=True. Minimal bootstrap configuration: %s",
bootstrap_config)
else:
# Otherwise: Set to standard set of local config files plus command line overrides
bootstrap_config = deepcopy(pyon_config)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not opts.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Force_clean - deletes sysname datastores
if opts.force_clean:
from pyon.datastore import clear_db_util
log.info("force_clean=True. DROP DATASTORES for sysname=%s",
bootstrap.get_sys_name())
clear_db_util.clear_db(bootstrap_config,
prefix=bootstrap.get_sys_name(),
sysname=bootstrap.get_sys_name())
from pyon.core.interfaces.interfaces import InterfaceAdmin
iadm = InterfaceAdmin(bootstrap.get_sys_name(),
config=bootstrap_config)
# If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config
# WARNING: If multiple containers start concurrently, this may fail
if get_safe(bootstrap_config,
"bootstrap.auto_store_interfaces") is True:
log.debug("auto_store_interfaces=True.")
stored_config = deepcopy(pyon_config)
config.apply_configuration(stored_config, config_override)
config.apply_configuration(stored_config, command_line_config)
iadm.create_core_datastores()
iadm.store_config(stored_config)
# Determine the final pyon_config:
# - Start from standard config already set (pyon.yml + local YML files)
# - Optionally load config from directory
if opts.config_from_directory:
config.apply_remote_config(bootstrap_cfg=bootstrap_config,
system_cfg=pyon_config)
# - Apply container profile specific config
config.apply_profile_configuration(pyon_config, bootstrap_config)
# - Reapply pyon.local.yml here again for good measure
config.apply_local_configuration(pyon_config,
pyon.DEFAULT_LOCAL_CONFIG_PATHS)
# - Last apply any separate command line config overrides
config.apply_configuration(pyon_config, config_override)
config.apply_configuration(pyon_config, command_line_config)
iadm.set_config(pyon_config)
# Set the immediate flag when command line override specified
if opts.immediate:
dict_merge(pyon_config, {"system": {
"immediate": True
}},
inplace=True)
# Determine system bootmode for bootstrapping actions (unless explicitly specified)
if not pyon_config.get_safe("bootmode"):
set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode")
if set_bootmode == "auto":
if iadm.system_data_exists():
dict_merge(pyon_config, {"bootmode": "restart"},
inplace=True)
log.info(
"System bootmode auto-detection is ON. Determined bootmode=%s",
pyon_config.get_safe("bootmode", "initial"))
elif set_bootmode == "secondary":
dict_merge(pyon_config, {"bootmode": "secondary"},
inplace=True)
log.info("System bootmode override. Set to bootmode=%s",
pyon_config.get_safe("bootmode", ""))
log.info("System in bootmode=%s",
pyon_config.get_safe("bootmode", "initial"))
# Bootstrap the pyon framework's core. Load configuration etc.
bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)
# Delete any queues/exchanges owned by sysname if option "broker_clean" is set
if opts.broker_clean:
log.info("broker_clean=True, sysname: %s",
bootstrap.get_sys_name())
from putil.rabbitmq.rabbit_util import RabbitManagementUtil
rabbit_util = RabbitManagementUtil(
pyon_config, sysname=bootstrap.get_sys_name())
deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname()
log.info("Exchanges deleted (%s): %s" %
(len(deleted_exchanges), ", ".join(deleted_exchanges)))
log.info("Queues deleted (%s): %s" %
(len(deleted_queues), ", ".join(deleted_queues)))
if opts.force_clean:
from pyon.util.file_sys import FileSystem
FileSystem._clean(pyon_config)
# If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging
if get_safe(bootstrap_config,
"bootstrap.auto_store_interfaces") is True:
iadm.store_interfaces(idempotent=True)
iadm.declare_core_exchange_resources()
iadm.close()
if opts.no_container:
log.info("no_container=True. Stopping here.")
return None
# Create the container instance
from pyon.container.cc import Container
container = Container(*args, **command_line_config)
container.version = version
return container