def prepare_container():
"""
Walks through pyon initialization in a deterministic way and initializes Container.
In particular make sure configuration is loaded in correct order and
pycc startup arguments are considered.
"""
# SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults
import pyon
import threading
threading.current_thread().name = "CC-Main"
import logging
global log
log = logging.getLogger('pycc')
from pyon.core import bootstrap, config
from pyon.util.containers import get_safe, dict_merge
# Set global testing flag to False. We are running as capability container, because
# we started through the pycc program.
bootstrap.testing = False
# Set sysname if provided in startup argument
if opts.sysname:
bootstrap.set_sys_name(opts.sysname)
# Trigger any initializing default logic in get_sys_name
bootstrap.get_sys_name()
command_line_config = kwargs
# This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
bootstrap_config = None
# This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
pyon_config = config.read_standard_configuration() # Initial pyon.yml + pyon.local.yml
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if opts.config:
if '{' in opts.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(opts.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" % opts.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([opts.config]).data
# Determine bootstrap_config
if opts.config_from_directory:
# Load minimal bootstrap config if option "config_from_directory"
bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
log.info("config_from_directory=True. Minimal bootstrap configuration: %s", bootstrap_config)
else:
# Otherwise: Set to standard set of local config files plus command line overrides
bootstrap_config = deepcopy(pyon_config)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not opts.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Force_clean - deletes sysname datastores
if opts.force_clean:
from pyon.datastore import clear_db_util
log.info("force_clean=True. DROP DATASTORES for sysname=%s", bootstrap.get_sys_name())
clear_db_util.clear_db(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name())
from pyon.core.interfaces.interfaces import InterfaceAdmin
iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)
# If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config
# WARNING: If multiple containers start concurrently, this may fail
if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True:
log.debug("auto_store_interfaces=True.")
stored_config = deepcopy(pyon_config)
config.apply_configuration(stored_config, config_override)
config.apply_configuration(stored_config, command_line_config)
iadm.create_core_datastores()
iadm.store_config(stored_config)
# Determine the final pyon_config:
# - Start from standard config already set (pyon.yml + local YML files)
# - Optionally load config from directory
if opts.config_from_directory:
config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config)
# - Apply container profile specific config
config.apply_profile_configuration(pyon_config, bootstrap_config)
# - Reapply pyon.local.yml here again for good measure
config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
# - Last apply any separate command line config overrides
config.apply_configuration(pyon_config, config_override)
config.apply_configuration(pyon_config, command_line_config)
iadm.set_config(pyon_config)
# Set the immediate flag when command line override specified
if opts.immediate:
dict_merge(pyon_config, {"system": {"immediate": True}}, inplace=True)
# Determine system bootmode for bootstrapping actions (unless explicitly specified)
if not pyon_config.get_safe("bootmode"):
set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode")
if set_bootmode == "auto":
if iadm.system_data_exists():
dict_merge(pyon_config, {"bootmode": "restart"}, inplace=True)
log.info("System bootmode auto-detection is ON. Determined bootmode=%s", pyon_config.get_safe("bootmode", "initial"))
elif set_bootmode == "secondary":
dict_merge(pyon_config, {"bootmode": "secondary"}, inplace=True)
log.info("System bootmode override. Set to bootmode=%s", pyon_config.get_safe("bootmode", ""))
log.info("System in bootmode=%s", pyon_config.get_safe("bootmode", "initial"))
# Bootstrap the pyon framework's core. Load configuration etc.
bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)
# Delete any queues/exchanges owned by sysname if option "broker_clean" is set
if opts.broker_clean:
log.info("broker_clean=True, sysname: %s", bootstrap.get_sys_name())
from putil.rabbitmq.rabbit_util import RabbitManagementUtil
rabbit_util = RabbitManagementUtil(pyon_config, sysname=bootstrap.get_sys_name())
deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname()
log.info("Exchanges deleted (%s): %s" % (len(deleted_exchanges), ", ".join(deleted_exchanges)))
log.info("Queues deleted (%s): %s" % (len(deleted_queues), ", ".join(deleted_queues)))
if opts.force_clean:
from pyon.util.file_sys import FileSystem
FileSystem._clean(pyon_config)
# If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging
if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True:
iadm.store_interfaces(idempotent=True)
iadm.declare_core_exchange_resources()
iadm.close()
if opts.no_container:
log.info("no_container=True. Stopping here.")
return None
# Create the container instance
from pyon.container.cc import Container
container = Container(*args, **command_line_config)
container.version = version
return container
def prepare_container():
"""
Walks through pyon initialization in a deterministic way and initializes Container.
In particular make sure configuration is loaded in correct order and
pycc startup arguments are considered.
"""
import threading
threading.current_thread().name = "CC-Main"
# SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
import pyon
from pyon.core import bootstrap, config
# Set global testing flag to False. We are running as capability container, because
# we started through the pycc program.
bootstrap.testing = False
# Set sysname if provided in startup argument
if opts.sysname:
bootstrap.set_sys_name(opts.sysname)
# Trigger any initializing default logic in get_sys_name
bootstrap.get_sys_name()
command_line_config = kwargs
# This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
bootstrap_config = None
# This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
pyon_config = config.read_standard_configuration() # Initial pyon.yml + pyon.local.yml
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if opts.config:
if "{" in opts.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(opts.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" % opts.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([opts.config]).data
# Determine bootstrap_config
if opts.config_from_directory:
# Load minimal bootstrap config if option "config_from_directory"
bootstrap_config = config.read_local_configuration(["res/config/pyon_min_boot.yml"])
config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
else:
# Otherwise: Set to standard set of local config files plus command line overrides
bootstrap_config = deepcopy(pyon_config)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not opts.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Delete sysname datastores if option "force_clean" is set
if opts.force_clean:
from pyon.datastore import clear_couch_util
print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name()
clear_couch_util.clear_couch(
bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name()
)
pyon_config.container.filesystem.force_clean = True
from pyon.core.interfaces.interfaces import InterfaceAdmin
iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)
# If auto_bootstrap, load config and interfaces into directory
# Note: this is idempotent and will not alter anything if this is not the first container to run
if bootstrap_config.system.auto_bootstrap:
print "pycc: auto_bootstrap=True."
stored_config = deepcopy(pyon_config)
config.apply_configuration(stored_config, config_override)
config.apply_configuration(stored_config, command_line_config)
iadm.create_core_datastores()
iadm.store_config(stored_config)
# Determine the final pyon_config
# - Start from standard config already set (pyon.yml + local YML files)
# - Optionally load config from directory
if opts.config_from_directory:
config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config)
# - Apply container profile specific config
config.apply_profile_configuration(pyon_config, bootstrap_config)
# - Reapply pyon.local.yml here again for good measure
config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
# - Last apply any separate command line config overrides
config.apply_configuration(pyon_config, config_override)
config.apply_configuration(pyon_config, command_line_config)
# Also set the immediate flag, but only if specified - it is an override
if opts.immediate:
from pyon.util.containers import dict_merge
dict_merge(pyon_config, {"system": {"immediate": True}}, True)
# Bootstrap pyon's core. Load configuration etc.
bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)
# Delete any queues/exchanges owned by sysname if option "broker_clean" is set
if opts.broker_clean:
print "pycc: broker_clean=True, sysname:", bootstrap.get_sys_name()
# build connect str
connect_str = "-q -H %s -P %s -u %s -p %s -V %s" % (
pyon_config.get_safe("server.amqp_priv.host", pyon_config.get_safe("server.amqp.host", "localhost")),
pyon_config.get_safe("container.exchange.management.port", "55672"),
pyon_config.get_safe("container.exchange.management.username", "guest"),
pyon_config.get_safe("container.exchange.management.password", "guest"),
"/",
)
from putil.rabbithelper import clean_by_sysname
deleted_exchanges, deleted_queues = clean_by_sysname(connect_str, bootstrap.get_sys_name())
print " exchanges deleted (%s): %s" % (len(deleted_exchanges), ",".join(deleted_exchanges))
print " queues deleted (%s): %s" % (len(deleted_queues), ",".join(deleted_queues))
if opts.force_clean:
path = os.path.join(pyon_config.get_safe("container.filesystem.root", "/tmp/ion"), bootstrap.get_sys_name())
print "force_clean: Removing", path
FileSystem._clean(pyon_config)
# Auto-bootstrap interfaces
if bootstrap_config.system.auto_bootstrap:
iadm.store_interfaces(idempotent=True)
iadm.close()
if opts.no_container:
print "pycc: no_container=True. Stopping here."
return None
# Create the container instance
from pyon.container.cc import Container
container = Container(*args, **command_line_config)
return container
def prepare_container():
"""
Walks through pyon initialization in a deterministic way and initializes Container.
In particular make sure configuration is loaded in correct order and
pycc startup arguments are considered.
"""
# SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults
import pyon
import threading
threading.current_thread().name = "CC-Main"
import logging
global log
log = logging.getLogger('pycc')
from pyon.core import bootstrap, config
from pyon.util.containers import get_safe, dict_merge
# Set global testing flag to False. We are running as capability container, because
# we started through the pycc program.
bootstrap.testing = False
# Set sysname if provided in startup argument
if opts.sysname:
bootstrap.set_sys_name(opts.sysname)
# Trigger any initializing default logic in get_sys_name
bootstrap.get_sys_name()
command_line_config = kwargs
# This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
bootstrap_config = None
# This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
pyon_config = config.read_standard_configuration(
) # Initial pyon.yml + pyon.local.yml
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if opts.config:
if '{' in opts.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(opts.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" %
opts.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([opts.config]).data
# Determine bootstrap_config
if opts.config_from_directory:
# Load minimal bootstrap config if option "config_from_directory"
bootstrap_config = config.read_local_configuration(
['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config,
pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
log.info(
"config_from_directory=True. Minimal bootstrap configuration: %s",
bootstrap_config)
else:
# Otherwise: Set to standard set of local config files plus command line overrides
bootstrap_config = deepcopy(pyon_config)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not opts.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Force_clean - deletes sysname datastores
if opts.force_clean:
from pyon.datastore import clear_db_util
log.info("force_clean=True. DROP DATASTORES for sysname=%s",
bootstrap.get_sys_name())
clear_db_util.clear_db(bootstrap_config,
prefix=bootstrap.get_sys_name(),
sysname=bootstrap.get_sys_name())
from pyon.core.interfaces.interfaces import InterfaceAdmin
iadm = InterfaceAdmin(bootstrap.get_sys_name(),
config=bootstrap_config)
# If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config
# WARNING: If multiple containers start concurrently, this may fail
if get_safe(bootstrap_config,
"bootstrap.auto_store_interfaces") is True:
log.debug("auto_store_interfaces=True.")
stored_config = deepcopy(pyon_config)
config.apply_configuration(stored_config, config_override)
config.apply_configuration(stored_config, command_line_config)
iadm.create_core_datastores()
iadm.store_config(stored_config)
# Determine the final pyon_config:
# - Start from standard config already set (pyon.yml + local YML files)
# - Optionally load config from directory
if opts.config_from_directory:
config.apply_remote_config(bootstrap_cfg=bootstrap_config,
system_cfg=pyon_config)
# - Apply container profile specific config
config.apply_profile_configuration(pyon_config, bootstrap_config)
# - Reapply pyon.local.yml here again for good measure
config.apply_local_configuration(pyon_config,
pyon.DEFAULT_LOCAL_CONFIG_PATHS)
# - Last apply any separate command line config overrides
config.apply_configuration(pyon_config, config_override)
config.apply_configuration(pyon_config, command_line_config)
iadm.set_config(pyon_config)
# Set the immediate flag when command line override specified
if opts.immediate:
dict_merge(pyon_config, {"system": {
"immediate": True
}},
inplace=True)
# Determine system bootmode for bootstrapping actions (unless explicitly specified)
if not pyon_config.get_safe("bootmode"):
set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode")
if set_bootmode == "auto":
if iadm.system_data_exists():
dict_merge(pyon_config, {"bootmode": "restart"},
inplace=True)
log.info(
"System bootmode auto-detection is ON. Determined bootmode=%s",
pyon_config.get_safe("bootmode", "initial"))
elif set_bootmode == "secondary":
dict_merge(pyon_config, {"bootmode": "secondary"},
inplace=True)
log.info("System bootmode override. Set to bootmode=%s",
pyon_config.get_safe("bootmode", ""))
log.info("System in bootmode=%s",
pyon_config.get_safe("bootmode", "initial"))
# Bootstrap the pyon framework's core. Load configuration etc.
bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)
# Delete any queues/exchanges owned by sysname if option "broker_clean" is set
if opts.broker_clean:
log.info("broker_clean=True, sysname: %s",
bootstrap.get_sys_name())
from putil.rabbitmq.rabbit_util import RabbitManagementUtil
rabbit_util = RabbitManagementUtil(
pyon_config, sysname=bootstrap.get_sys_name())
deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname()
log.info("Exchanges deleted (%s): %s" %
(len(deleted_exchanges), ", ".join(deleted_exchanges)))
log.info("Queues deleted (%s): %s" %
(len(deleted_queues), ", ".join(deleted_queues)))
if opts.force_clean:
from pyon.util.file_sys import FileSystem
FileSystem._clean(pyon_config)
# If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging
if get_safe(bootstrap_config,
"bootstrap.auto_store_interfaces") is True:
iadm.store_interfaces(idempotent=True)
iadm.declare_core_exchange_resources()
iadm.close()
if opts.no_container:
log.info("no_container=True. Stopping here.")
return None
# Create the container instance
from pyon.container.cc import Container
container = Container(*args, **command_line_config)
container.version = version
return container