def test_store_core(self):
# Store system CFG properties
ion_config = config.read_standard_configuration()
self.iadm.store_config(ion_config)
# Validate the CFG entries are stored in DB
entries = self.dir.lookup('/System')
self.assertTrue(entries)
def test_store_core(self):
# Store system CFG properties
ion_config = config.read_standard_configuration()
self.iadm.store_config(ion_config)
# Validate the CFG entries are stored in DB
entries = self.dir.lookup('/Config/CFG')
self.assertTrue(entries)
def set_config(pyon_cfg=None):
"""
Initialize pyon global configuration
"""
from pyon.core import config
if pyon_cfg:
# Variant 1: if provided, set pyon_cfg as pyon global CFG
config.apply_configuration(CFG, pyon_cfg)
else:
# Variant 2: default, load standard configuration sequence
std_cfg = config.read_standard_configuration()
config.apply_configuration(CFG, std_cfg)
assert_configuration(CFG)
def set_config(pyon_cfg=None):
"""
Initialize pyon global configuration
"""
from pyon.core import config
if pyon_cfg:
# Variant 1: if provided, set pyon_cfg as pyon global CFG
config.apply_configuration(CFG, pyon_cfg)
else:
# Variant 2: default, load standard configuration sequence
std_cfg = config.read_standard_configuration()
config.apply_configuration(CFG, std_cfg)
assert_configuration(CFG)
def arg_initialize(self):
if bootstrap.pyon_initialized:
raise BadRequest("Pyon already initalized")
parser = argparse.ArgumentParser()
parser.add_argument("-s",
"--sysname",
dest="sysname",
help="System name")
options, extra = parser.parse_known_args()
args, command_line_config = parse_args(extra)
# -------------------------------------------------------------------------
# Store config and interfaces
# Set global testing flag to False. We are running as standalone script. This is NO TEST.
bootstrap.testing = False
# Set sysname if provided in startup argument
if options.sysname:
bootstrap.set_sys_name(options.sysname)
# bootstrap_config - Used for running this store_interfaces script
bootstrap_config = config.read_local_configuration(
['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config,
pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not options.sysname and bootstrap_config.get_safe(
"system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# ion_config - Holds the new CFG object for the system (independent of this tool's config)
ion_config = config.read_standard_configuration()
config.apply_configuration(ion_config, command_line_config)
# Bootstrap pyon's core. Load configuration etc.
bootstrap.bootstrap_pyon(pyon_cfg=ion_config)
def startup(self):
ion_config = config.read_standard_configuration()
self.GEO_WS = ion_config['eoi']['geoserver']['geoserver_ooi_workspace']
self.SERVER = ion_config['eoi']['geoserver']['server']+"/geoserver/rest"
self.U_NAME = ion_config['eoi']['geoserver']['user_name']
self.P_WD = ion_config['eoi']['geoserver']['password']
self.PORT = ion_config['eoi']['importer_service']['port']
self.GEO_STORE = ion_config['eoi']['geoserver']['geoserver_ooi_store']
self.SESSION_START_UP_ln1 = ion_config['eoi']['postgres']['session_startup_ln1']
self.SESSION_START_UP_ln2 = ion_config['eoi']['postgres']['session_startup_ln2']
self.POSTGRES_USER = ion_config['eoi']['postgres']['user_name']
self.POSTGRES_PASSWORD = ion_config['eoi']['postgres']['password']
self.POSTGRES_DB = ion_config['eoi']['postgres']['database']
self.POSTGRES_PORT = ion_config['eoi']['postgres']['port']
self.POSTGRES_HOST = ion_config['eoi']['postgres']['host']
self.LAYER_PREFIX = ion_config['eoi']['geoserver']['layer_prefix']
self.LAYER_SUFFIX = ion_config['eoi']['geoserver']['layer_suffix']
log.info('Serving on '+str(self.PORT)+'...')
server = WSGIServer(('', self.PORT), self.application).serve_forever()
def arg_initialize(self):
if bootstrap.pyon_initialized:
raise BadRequest("Pyon already initalized")
parser = argparse.ArgumentParser()
parser.add_argument("-s", "--sysname", dest="sysname", help="System name")
options, extra = parser.parse_known_args()
args, command_line_config = parse_args(extra)
# -------------------------------------------------------------------------
# Store config and interfaces
# Set global testing flag to False. We are running as standalone script. This is NO TEST.
bootstrap.testing = False
# Set sysname if provided in startup argument
if options.sysname:
bootstrap.set_sys_name(options.sysname)
# bootstrap_config - Used for running this store_interfaces script
bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not options.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# ion_config - Holds the new CFG object for the system (independent of this tool's config)
ion_config = config.read_standard_configuration()
config.apply_configuration(ion_config, command_line_config)
# Bootstrap pyon's core. Load configuration etc.
bootstrap.bootstrap_pyon(pyon_cfg=ion_config)
def prepare_container():
"""
Walks through pyon initialization in a deterministic way and initializes Container.
In particular make sure configuration is loaded in correct order and
pycc startup arguments are considered.
"""
import threading
threading.current_thread().name = "CC-Main"
# SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
import pyon
from pyon.core import bootstrap, config
# Set global testing flag to False. We are running as capability container, because
# we started through the pycc program.
bootstrap.testing = False
# Set sysname if provided in startup argument
if opts.sysname:
bootstrap.set_sys_name(opts.sysname)
# Trigger any initializing default logic in get_sys_name
bootstrap.get_sys_name()
command_line_config = kwargs
# This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
bootstrap_config = None
# This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
pyon_config = config.read_standard_configuration() # Initial pyon.yml + pyon.local.yml
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if opts.config:
if "{" in opts.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(opts.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" % opts.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([opts.config]).data
# Determine bootstrap_config
if opts.config_from_directory:
# Load minimal bootstrap config if option "config_from_directory"
bootstrap_config = config.read_local_configuration(["res/config/pyon_min_boot.yml"])
config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
else:
# Otherwise: Set to standard set of local config files plus command line overrides
bootstrap_config = deepcopy(pyon_config)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not opts.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Delete sysname datastores if option "force_clean" is set
if opts.force_clean:
from pyon.datastore import clear_couch_util
print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name()
clear_couch_util.clear_couch(
bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name()
)
pyon_config.container.filesystem.force_clean = True
from pyon.core.interfaces.interfaces import InterfaceAdmin
iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)
# If auto_bootstrap, load config and interfaces into directory
# Note: this is idempotent and will not alter anything if this is not the first container to run
if bootstrap_config.system.auto_bootstrap:
print "pycc: auto_bootstrap=True."
stored_config = deepcopy(pyon_config)
config.apply_configuration(stored_config, config_override)
config.apply_configuration(stored_config, command_line_config)
iadm.create_core_datastores()
iadm.store_config(stored_config)
# Determine the final pyon_config
# - Start from standard config already set (pyon.yml + local YML files)
# - Optionally load config from directory
if opts.config_from_directory:
config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config)
# - Apply container profile specific config
config.apply_profile_configuration(pyon_config, bootstrap_config)
# - Reapply pyon.local.yml here again for good measure
config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
# - Last apply any separate command line config overrides
config.apply_configuration(pyon_config, config_override)
config.apply_configuration(pyon_config, command_line_config)
# Also set the immediate flag, but only if specified - it is an override
if opts.immediate:
from pyon.util.containers import dict_merge
dict_merge(pyon_config, {"system": {"immediate": True}}, True)
# Bootstrap pyon's core. Load configuration etc.
bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)
# Delete any queues/exchanges owned by sysname if option "broker_clean" is set
if opts.broker_clean:
print "pycc: broker_clean=True, sysname:", bootstrap.get_sys_name()
# build connect str
connect_str = "-q -H %s -P %s -u %s -p %s -V %s" % (
pyon_config.get_safe("server.amqp_priv.host", pyon_config.get_safe("server.amqp.host", "localhost")),
pyon_config.get_safe("container.exchange.management.port", "55672"),
pyon_config.get_safe("container.exchange.management.username", "guest"),
pyon_config.get_safe("container.exchange.management.password", "guest"),
"/",
)
from putil.rabbithelper import clean_by_sysname
deleted_exchanges, deleted_queues = clean_by_sysname(connect_str, bootstrap.get_sys_name())
print " exchanges deleted (%s): %s" % (len(deleted_exchanges), ",".join(deleted_exchanges))
print " queues deleted (%s): %s" % (len(deleted_queues), ",".join(deleted_queues))
if opts.force_clean:
path = os.path.join(pyon_config.get_safe("container.filesystem.root", "/tmp/ion"), bootstrap.get_sys_name())
print "force_clean: Removing", path
FileSystem._clean(pyon_config)
# Auto-bootstrap interfaces
if bootstrap_config.system.auto_bootstrap:
iadm.store_interfaces(idempotent=True)
iadm.close()
if opts.no_container:
print "pycc: no_container=True. Stopping here."
return None
# Create the container instance
from pyon.container.cc import Container
container = Container(*args, **command_line_config)
return container
def __init__(self):
pyon_config = config.read_standard_configuration() # Initial pyon.yml + pyon.local.yml
log.info("Setting up geoserver importer service...")
self.startup()
def prepare_container():
"""
Walks through pyon initialization in a deterministic way and initializes Container.
In particular make sure configuration is loaded in correct order and
pycc startup arguments are considered.
"""
# SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults
import pyon
import threading
threading.current_thread().name = "CC-Main"
import logging
global log
log = logging.getLogger('pycc')
from pyon.core import bootstrap, config
from pyon.util.containers import get_safe, dict_merge
# Set global testing flag to False. We are running as capability container, because
# we started through the pycc program.
bootstrap.testing = False
# Set sysname if provided in startup argument
if opts.sysname:
bootstrap.set_sys_name(opts.sysname)
# Trigger any initializing default logic in get_sys_name
bootstrap.get_sys_name()
command_line_config = kwargs
# This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
bootstrap_config = None
# This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
pyon_config = config.read_standard_configuration() # Initial pyon.yml + pyon.local.yml
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if opts.config:
if '{' in opts.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(opts.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" % opts.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([opts.config]).data
# Determine bootstrap_config
if opts.config_from_directory:
# Load minimal bootstrap config if option "config_from_directory"
bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
log.info("config_from_directory=True. Minimal bootstrap configuration: %s", bootstrap_config)
else:
# Otherwise: Set to standard set of local config files plus command line overrides
bootstrap_config = deepcopy(pyon_config)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not opts.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Force_clean - deletes sysname datastores
if opts.force_clean:
from pyon.datastore import clear_db_util
log.info("force_clean=True. DROP DATASTORES for sysname=%s", bootstrap.get_sys_name())
clear_db_util.clear_db(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name())
from pyon.core.interfaces.interfaces import InterfaceAdmin
iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)
# If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config
# WARNING: If multiple containers start concurrently, this may fail
if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True:
log.debug("auto_store_interfaces=True.")
stored_config = deepcopy(pyon_config)
config.apply_configuration(stored_config, config_override)
config.apply_configuration(stored_config, command_line_config)
iadm.create_core_datastores()
iadm.store_config(stored_config)
# Determine the final pyon_config:
# - Start from standard config already set (pyon.yml + local YML files)
# - Optionally load config from directory
if opts.config_from_directory:
config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config)
# - Apply container profile specific config
config.apply_profile_configuration(pyon_config, bootstrap_config)
# - Reapply pyon.local.yml here again for good measure
config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
# - Last apply any separate command line config overrides
config.apply_configuration(pyon_config, config_override)
config.apply_configuration(pyon_config, command_line_config)
iadm.set_config(pyon_config)
# Set the immediate flag when command line override specified
if opts.immediate:
dict_merge(pyon_config, {"system": {"immediate": True}}, inplace=True)
# Determine system bootmode for bootstrapping actions (unless explicitly specified)
if not pyon_config.get_safe("bootmode"):
set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode")
if set_bootmode == "auto":
if iadm.system_data_exists():
dict_merge(pyon_config, {"bootmode": "restart"}, inplace=True)
log.info("System bootmode auto-detection is ON. Determined bootmode=%s", pyon_config.get_safe("bootmode", "initial"))
elif set_bootmode == "secondary":
dict_merge(pyon_config, {"bootmode": "secondary"}, inplace=True)
log.info("System bootmode override. Set to bootmode=%s", pyon_config.get_safe("bootmode", ""))
log.info("System in bootmode=%s", pyon_config.get_safe("bootmode", "initial"))
# Bootstrap the pyon framework's core. Load configuration etc.
bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)
# Delete any queues/exchanges owned by sysname if option "broker_clean" is set
if opts.broker_clean:
log.info("broker_clean=True, sysname: %s", bootstrap.get_sys_name())
from putil.rabbitmq.rabbit_util import RabbitManagementUtil
rabbit_util = RabbitManagementUtil(pyon_config, sysname=bootstrap.get_sys_name())
deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname()
log.info("Exchanges deleted (%s): %s" % (len(deleted_exchanges), ", ".join(deleted_exchanges)))
log.info("Queues deleted (%s): %s" % (len(deleted_queues), ", ".join(deleted_queues)))
if opts.force_clean:
from pyon.util.file_sys import FileSystem
FileSystem._clean(pyon_config)
# If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging
if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True:
iadm.store_interfaces(idempotent=True)
iadm.declare_core_exchange_resources()
iadm.close()
if opts.no_container:
log.info("no_container=True. Stopping here.")
return None
# Create the container instance
from pyon.container.cc import Container
container = Container(*args, **command_line_config)
container.version = version
return container
def prepare_container():
"""
Walks through pyon initialization in a deterministic way and initializes Container.
In particular make sure configuration is loaded in correct order and
pycc startup arguments are considered.
"""
import threading
threading.current_thread().name = "CC-Main"
# SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
import pyon
from pyon.core import bootstrap, config
# Set global testing flag to False. We are running as capability container. This is NO TEST.
bootstrap.testing = False
# Set sysname if provided in startup argument
if opts.sysname:
bootstrap.set_sys_name(opts.sysname)
# Trigger any initializing default logic in get_sys_name
bootstrap.get_sys_name()
command_line_config = kwargs
# This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
bootstrap_config = None
# This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
pyon_config = config.read_standard_configuration(
) # Initial pyon.yml + pyon.local.yml
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if opts.config:
if '{' in opts.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(opts.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" %
opts.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([opts.config]).data
# Determine bootstrap_config
if opts.config_from_directory:
# Load minimal bootstrap config if option "config_from_directory"
bootstrap_config = config.read_local_configuration(
['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config,
pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
else:
# Otherwise: Set to standard set of local config files plus command line overrides
bootstrap_config = deepcopy(pyon_config)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not opts.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Delete sysname datastores if option "force_clean" is set
if opts.force_clean:
from pyon.datastore import clear_couch_util
print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name(
)
clear_couch_util.clear_couch(bootstrap_config,
prefix=bootstrap.get_sys_name())
pyon_config.container.filesystem.force_clean = True
from pyon.core.interfaces.interfaces import InterfaceAdmin
iadm = InterfaceAdmin(bootstrap.get_sys_name(),
config=bootstrap_config)
# If auto_bootstrap, load config and interfaces into directory
# Note: this is idempotent and will not alter anything if this is not the first container to run
if bootstrap_config.system.auto_bootstrap:
print "pycc: auto_bootstrap=True."
stored_config = deepcopy(pyon_config)
config.apply_configuration(stored_config, config_override)
config.apply_configuration(stored_config, command_line_config)
iadm.create_core_datastores()
iadm.store_config(stored_config)
# Determine the final pyon_config
# - Start from standard config already set (pyon.yml + local YML files)
# - Optionally load config from directory
if opts.config_from_directory:
config.apply_remote_config(bootstrap_cfg=bootstrap_config,
system_cfg=pyon_config)
config.apply_local_configuration(
pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS
) # apply pyon.local.yml again over top
# - Last apply any separate command line config overrides
config.apply_configuration(pyon_config, config_override)
config.apply_configuration(pyon_config, command_line_config)
# Also set the immediate flag, but only if specified - it is an override
if opts.immediate:
from pyon.util.containers import dict_merge
dict_merge(pyon_config, {'system': {'immediate': True}}, True)
# Bootstrap pyon's core. Load configuration etc.
bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)
# Delete any queues/exchanges owned by sysname if option "broker_clean" is set
if opts.broker_clean:
print "pycc: broker_clean=True, sysname:", bootstrap.get_sys_name()
# build connect str
connect_str = "-q -H %s -P 55672 -u %s -p %s -V %s" % (
pyon_config.get_safe(
'server.amqp_priv.host',
pyon_config.get_safe('server.amqp.host', 'localhost')),
pyon_config.get_safe('container.exchange.management.username',
'guest'),
pyon_config.get_safe('container.exchange.management.password',
'guest'), '/')
from putil.rabbithelper import clean_by_sysname
deleted_exchanges, deleted_queues = clean_by_sysname(
connect_str, bootstrap.get_sys_name())
print " exchanges deleted (%s): %s" % (
len(deleted_exchanges), ",".join(deleted_exchanges))
print " queues deleted (%s): %s" % (
len(deleted_queues), ",".join(deleted_queues))
# Auto-bootstrap interfaces
if bootstrap_config.system.auto_bootstrap:
iadm.store_interfaces(idempotent=True)
iadm.close()
if opts.no_container:
print "pycc: no_container=True. Stopping here."
return None
# Create the container instance
from pyon.container.cc import Container
container = Container(*args, **command_line_config)
return container
def main():
'''
Store configuration and interfaces into the datastore
How to run this from command line:
bin/store_interfaces -s system name [ -of filename | -sf filename | -fc true|false]
-of Load object definition file
-sf Load service definition file
-fc Force clean the database
Example:
Load all object and service definitions
bin/python bin/store_interfaces -s mysysname
Load all object and service definitions with force clean the database
bin/python bin/store_interfaces -s mysysname -fc
Load object definition from a file
bin/python bin/store_interfaces -s mysysname -of obj/data/core/org.yml
Load service definition from a file
bin/python bin/store_interfaces -s mysysname -sf obj/services/core/datastore_service.yml
'''
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', type=str, help='Additional config files to load or dict config content.', default=[])
parser.add_argument('-fc', '--force_clean', action='store_true', help='Force clean.')
parser.add_argument("-of", "--object", dest="fobject", help="Load object definition from a file")
parser.add_argument("-s", "--sysname", dest="sysname", help="System name")
parser.add_argument("-sf", "--service", dest="fservice", help="Load service definition from a file")
options, extra = parser.parse_known_args()
args, command_line_config = parse_args(extra)
print "store_interfaces: Storing SciON config and interfaces, with options:" , str(options)
# -------------------------------------------------------------------------
# Store config and interfaces
# Set global testing flag to False. We are running as standalone script. This is NO TEST.
bootstrap.testing = False
# Set sysname if provided in startup argument
if options.sysname:
bootstrap.set_sys_name(options.sysname)
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if options.config:
if '{' in options.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(options.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" % options.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([options.config]).data
# bootstrap_config - Used for running this store_interfaces script
bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
if config_override:
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not options.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Delete sysname datastores if option "force_clean" is set
if options.force_clean:
from pyon.datastore import clear_db_util
from pyon.util.file_sys import FileSystem
print "store_interfaces: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name()
pyon_config = config.read_standard_configuration() # Initial pyon.yml + pyon.local.yml
clear_db_util.clear_db(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name())
FileSystem._clean(pyon_config)
# ion_config - Holds the new CFG object for the system (independent of this tool's config)
ion_config = config.read_standard_configuration()
if config_override:
config.apply_configuration(ion_config, config_override)
config.apply_configuration(ion_config, command_line_config)
# -------------------------------------------------------------------------
# Store config and interfaces
iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)
# Make sure core datastores exist
iadm.create_core_datastores()
# Store system CFG properties
iadm.store_config(ion_config)
# Store system interfaces
iadm.store_interfaces(options.fobject, options.fservice)
iadm.close()
def prepare_container():
"""
Walks through pyon initialization in a deterministic way and initializes Container.
In particular make sure configuration is loaded in correct order and
pycc startup arguments are considered.
"""
# SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults
import pyon
import threading
threading.current_thread().name = "CC-Main"
import logging
global log
log = logging.getLogger('pycc')
from pyon.core import bootstrap, config
from pyon.util.containers import get_safe, dict_merge
# Set global testing flag to False. We are running as capability container, because
# we started through the pycc program.
bootstrap.testing = False
# Set sysname if provided in startup argument
if opts.sysname:
bootstrap.set_sys_name(opts.sysname)
# Trigger any initializing default logic in get_sys_name
bootstrap.get_sys_name()
command_line_config = kwargs
# This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
bootstrap_config = None
# This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
pyon_config = config.read_standard_configuration(
) # Initial pyon.yml + pyon.local.yml
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if opts.config:
if '{' in opts.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(opts.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" %
opts.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([opts.config]).data
# Determine bootstrap_config
if opts.config_from_directory:
# Load minimal bootstrap config if option "config_from_directory"
bootstrap_config = config.read_local_configuration(
['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config,
pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
log.info(
"config_from_directory=True. Minimal bootstrap configuration: %s",
bootstrap_config)
else:
# Otherwise: Set to standard set of local config files plus command line overrides
bootstrap_config = deepcopy(pyon_config)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not opts.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Force_clean - deletes sysname datastores
if opts.force_clean:
from pyon.datastore import clear_db_util
log.info("force_clean=True. DROP DATASTORES for sysname=%s",
bootstrap.get_sys_name())
clear_db_util.clear_db(bootstrap_config,
prefix=bootstrap.get_sys_name(),
sysname=bootstrap.get_sys_name())
from pyon.core.interfaces.interfaces import InterfaceAdmin
iadm = InterfaceAdmin(bootstrap.get_sys_name(),
config=bootstrap_config)
# If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config
# WARNING: If multiple containers start concurrently, this may fail
if get_safe(bootstrap_config,
"bootstrap.auto_store_interfaces") is True:
log.debug("auto_store_interfaces=True.")
stored_config = deepcopy(pyon_config)
config.apply_configuration(stored_config, config_override)
config.apply_configuration(stored_config, command_line_config)
iadm.create_core_datastores()
iadm.store_config(stored_config)
# Determine the final pyon_config:
# - Start from standard config already set (pyon.yml + local YML files)
# - Optionally load config from directory
if opts.config_from_directory:
config.apply_remote_config(bootstrap_cfg=bootstrap_config,
system_cfg=pyon_config)
# - Apply container profile specific config
config.apply_profile_configuration(pyon_config, bootstrap_config)
# - Reapply pyon.local.yml here again for good measure
config.apply_local_configuration(pyon_config,
pyon.DEFAULT_LOCAL_CONFIG_PATHS)
# - Last apply any separate command line config overrides
config.apply_configuration(pyon_config, config_override)
config.apply_configuration(pyon_config, command_line_config)
iadm.set_config(pyon_config)
# Set the immediate flag when command line override specified
if opts.immediate:
dict_merge(pyon_config, {"system": {
"immediate": True
}},
inplace=True)
# Determine system bootmode for bootstrapping actions (unless explicitly specified)
if not pyon_config.get_safe("bootmode"):
set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode")
if set_bootmode == "auto":
if iadm.system_data_exists():
dict_merge(pyon_config, {"bootmode": "restart"},
inplace=True)
log.info(
"System bootmode auto-detection is ON. Determined bootmode=%s",
pyon_config.get_safe("bootmode", "initial"))
elif set_bootmode == "secondary":
dict_merge(pyon_config, {"bootmode": "secondary"},
inplace=True)
log.info("System bootmode override. Set to bootmode=%s",
pyon_config.get_safe("bootmode", ""))
log.info("System in bootmode=%s",
pyon_config.get_safe("bootmode", "initial"))
# Bootstrap the pyon framework's core. Load configuration etc.
bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)
# Delete any queues/exchanges owned by sysname if option "broker_clean" is set
if opts.broker_clean:
log.info("broker_clean=True, sysname: %s",
bootstrap.get_sys_name())
from putil.rabbitmq.rabbit_util import RabbitManagementUtil
rabbit_util = RabbitManagementUtil(
pyon_config, sysname=bootstrap.get_sys_name())
deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname()
log.info("Exchanges deleted (%s): %s" %
(len(deleted_exchanges), ", ".join(deleted_exchanges)))
log.info("Queues deleted (%s): %s" %
(len(deleted_queues), ", ".join(deleted_queues)))
if opts.force_clean:
from pyon.util.file_sys import FileSystem
FileSystem._clean(pyon_config)
# If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging
if get_safe(bootstrap_config,
"bootstrap.auto_store_interfaces") is True:
iadm.store_interfaces(idempotent=True)
iadm.declare_core_exchange_resources()
iadm.close()
if opts.no_container:
log.info("no_container=True. Stopping here.")
return None
# Create the container instance
from pyon.container.cc import Container
container = Container(*args, **command_line_config)
container.version = version
return container
def main():
"""
Store configuration and interfaces into the datastore
How to run this from command line:
bin/store_interfaces -s system name [ -of filename | -sf filename | -fc true|false]
-of Load object definition file
-sf Load service definition file
-fc Force clean the database
Example:
Load all object and service definitions
bin/python bin/store_interfaces -s mysysname
Load all object and service definitions with force clean the database
bin/python bin/store_interfaces -s mysysname -fc
Load object definition from a file
bin/python bin/store_interfaces -s mysysname -of obj/data/core/org.yml
Load service definition from a file
bin/python bin/store_interfaces -s mysysname -sf obj/services/core/datastore_service.yml
"""
parser = argparse.ArgumentParser()
parser.add_argument(
'-c',
'--config',
type=str,
help='Additional config files to load or dict config content.',
default=[])
parser.add_argument('-fc',
'--force_clean',
action='store_true',
help='Force clean.')
parser.add_argument("-of",
"--object",
dest="fobject",
help="Load object definition from a file")
parser.add_argument("-s", "--sysname", dest="sysname", help="System name")
parser.add_argument("-sf",
"--service",
dest="fservice",
help="Load service definition from a file")
options, extra = parser.parse_known_args()
args, command_line_config = parse_args(extra)
log.info("Storing SciON config and interfaces, with options: %s",
str(options))
from pyon.core import log as logutil
logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS)
# -------------------------------------------------------------------------
# Store config and interfaces
# Set global testing flag to False. We are running as standalone script. This is NO TEST.
bootstrap.testing = False
# Set sysname if provided in startup argument
if options.sysname:
bootstrap.set_sys_name(options.sysname)
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if options.config:
if '{' in options.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(options.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" %
options.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([options.config]).data
# bootstrap_config - Used for running this store_interfaces script
bootstrap_config = config.read_local_configuration(
['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config,
pyon.DEFAULT_LOCAL_CONFIG_PATHS)
if config_override:
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not options.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Delete sysname datastores if option "force_clean" is set
if options.force_clean:
from pyon.datastore import clear_db_util
from pyon.util.file_sys import FileSystem
log.info("force_clean=True. DROP DATASTORES for sysname=%s" %
bootstrap.get_sys_name())
pyon_config = config.read_standard_configuration(
) # Initial pyon.yml + pyon.local.yml
clear_db_util.clear_db(bootstrap_config,
prefix=bootstrap.get_sys_name(),
sysname=bootstrap.get_sys_name())
FileSystem._clean(pyon_config)
# ion_config - Holds the new CFG object for the system (independent of this tool's config)
ion_config = config.read_standard_configuration()
if config_override:
config.apply_configuration(ion_config, config_override)
config.apply_configuration(ion_config, command_line_config)
# -------------------------------------------------------------------------
# Store config and interfaces
iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)
# Make sure core datastores exist
iadm.create_core_datastores()
# Store system CFG properties
iadm.store_config(ion_config)
# Store system interfaces
iadm.store_interfaces(options.fobject, options.fservice)
iadm.close()
def prepare_container():
"""
Walks through pyon initialization in a deterministic way and initializes Container.
In particular make sure configuration is loaded in correct order and
pycc startup arguments are considered.
"""
import threading
threading.current_thread().name = "CC-Main"
# SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
import pyon
from pyon.core import bootstrap, config
# Set global testing flag to False. We are running as capability container. This is NO TEST.
bootstrap.testing = False
# Set sysname if provided in startup argument
if opts.sysname:
bootstrap.set_sys_name(opts.sysname)
# Trigger any initializing default logic in get_sys_name
bootstrap.get_sys_name()
command_line_config = kwargs
# This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
bootstrap_config = None
# This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
pyon_config = config.read_standard_configuration()
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if opts.config:
if '{' in opts.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(opts.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" %
opts.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([opts.config]).data
# Determine bootstrap_config
if opts.config_from_directory:
# Load minimal bootstrap config if option "config_from_directory"
bootstrap_config = config.read_local_configuration(
['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config,
pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
else:
# Otherwise: Set to standard set of local config files plus command line overrides
bootstrap_config = pyon_config.copy()
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not opts.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Delete sysname datastores if option "force_clean" is set
if opts.force_clean:
from pyon.datastore import clear_couch_util
print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name(
)
clear_couch_util.clear_couch(bootstrap_config,
prefix=bootstrap.get_sys_name())
# If auto_bootstrap, load config and interfaces into directory
# Note: this is idempotent and will not alter anything if this is not the first container to run
if bootstrap_config.system.auto_bootstrap:
print "pycc: auto_bootstrap=True."
stored_config = pyon_config.copy()
config.apply_configuration(stored_config, config_override)
config.apply_configuration(stored_config, command_line_config)
config.auto_bootstrap_config(bootstrap_config,
system_cfg=stored_config)
# Determine the final pyon_config
# - Start from standard config already set (pyon.yml + local YML files)
# - Optionally load config from directory
if opts.config_from_directory:
config.apply_remote_config(pyon_config)
# - Last apply any separate command line config overrides
config.apply_configuration(pyon_config, config_override)
config.apply_configuration(pyon_config, command_line_config)
# Load logging override config if provided. Supports variants literal and path.
logging_config_override = None
if opts.logcfg:
if '{' in opts.logcfg:
# Variant 1: Value is dict of config values
try:
eval_value = ast.literal_eval(opts.logcfg)
logging_config_override = eval_value
except ValueError:
raise Exception("Value error in logcfg arg '%s'" %
opts.logcfg)
else:
# Variant 2: Value is path to YAML file containing config values
pyon.DEFAULT_LOGGING_PATHS.append(opts.logcfg)
# Also set the immediate flag, but only if specified - it is an override
if opts.immediate:
dict_merge(pyon_config, {'system': {'immediate': True}}, True)
# Bootstrap pyon's core. Load configuration etc.
bootstrap.bootstrap_pyon(
logging_config_override=logging_config_override,
pyon_cfg=pyon_config)
# Auto-bootstrap interfaces
# @WARN: This currently imports ALL modules, executing ALL static initializers as side effect!!!!!!!
if bootstrap_config.system.auto_bootstrap:
config.auto_bootstrap_interfaces(bootstrap_config)
if opts.no_container:
print "pycc: no_container=True. Stopping here."
return None
# Create the container instance
from pyon.container.cc import Container
container = Container(*args, **command_line_config)
return container
def prepare_container():
"""
Walks through pyon initialization in a deterministic way and initializes Container.
In particular make sure configuration is loaded in correct order and
pycc startup arguments are considered.
"""
import threading
threading.current_thread().name = "CC-Main"
# SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
import pyon
from pyon.core import bootstrap, config
# Set global testing flag to False. We are running as capability container. This is NO TEST.
bootstrap.testing = False
# Set sysname if provided in startup argument
if opts.sysname:
bootstrap.set_sys_name(opts.sysname)
# Trigger any initializing default logic in get_sys_name
bootstrap.get_sys_name()
command_line_config = kwargs
# This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
bootstrap_config = None
# This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
pyon_config = config.read_standard_configuration()
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if opts.config:
if '{' in opts.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(opts.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" % opts.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([opts.config]).data
# Determine bootstrap_config
if opts.config_from_directory:
# Load minimal bootstrap config if option "config_from_directory"
bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
else:
# Otherwise: Set to standard set of local config files plus command line overrides
bootstrap_config = pyon_config.copy()
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not opts.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Delete sysname datastores if option "force_clean" is set
if opts.force_clean:
from pyon.datastore import clear_couch_util
print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name()
clear_couch_util.clear_couch(bootstrap_config, prefix=bootstrap.get_sys_name())
# If auto_bootstrap, load config and interfaces into directory
# Note: this is idempotent and will not alter anything if this is not the first container to run
if bootstrap_config.system.auto_bootstrap:
print "pycc: auto_bootstrap=True."
stored_config = pyon_config.copy()
config.apply_configuration(stored_config, config_override)
config.apply_configuration(stored_config, command_line_config)
config.auto_bootstrap_config(bootstrap_config, system_cfg=stored_config)
# Determine the final pyon_config
# - Start from standard config already set (pyon.yml + local YML files)
# - Optionally load config from directory
if opts.config_from_directory:
config.apply_remote_config(pyon_config)
# - Last apply any separate command line config overrides
config.apply_configuration(pyon_config, config_override)
config.apply_configuration(pyon_config, command_line_config)
# Load logging override config if provided. Supports variants literal and path.
logging_config_override = None
if opts.logcfg:
if '{' in opts.logcfg:
# Variant 1: Value is dict of config values
try:
eval_value = ast.literal_eval(opts.logcfg)
logging_config_override = eval_value
except ValueError:
raise Exception("Value error in logcfg arg '%s'" % opts.logcfg)
else:
# Variant 2: Value is path to YAML file containing config values
pyon.DEFAULT_LOGGING_PATHS.append(opts.logcfg)
# Also set the immediate flag, but only if specified - it is an override
if opts.immediate:
dict_merge(pyon_config, {'system':{'immediate':True}}, True)
# Bootstrap pyon's core. Load configuration etc.
bootstrap.bootstrap_pyon(
logging_config_override=logging_config_override,
pyon_cfg=pyon_config)
# Auto-bootstrap interfaces
# @WARN: This currently imports ALL modules, executing ALL static initializers as side effect!!!!!!!
if bootstrap_config.system.auto_bootstrap:
config.auto_bootstrap_interfaces(bootstrap_config)
if opts.no_container:
print "pycc: no_container=True. Stopping here."
return None
# Create the container instance
from pyon.container.cc import Container
container = Container(*args, **command_line_config)
return container
def main():
'''
Store configuration and interfaces into the datastore
How to run this from command line:
bin/store_interfaces -s system name [ -of filename | -sf filename | -fc true|false]
-of Load object definition file
-sf Load service definition file
-fc Force clean the database
Example:
Load all object and service definitions
bin/python bin/store_interfaces -s mysysname
Load all object and service definitions with force clean the database
bin/python bin/store_interfaces -s mysysname -fc
Load object definition from a file
bin/python bin/store_interfaces -s mysysname -of obj/data/coi/org.yml
Load service definition from a file
bin/python bin/store_interfaces -s mysysname -sf obj/services/coi/datastore_service.yml
'''
parser = argparse.ArgumentParser()
parser.add_argument('-fc', '--force_clean', action='store_true',
help='Force clean.')
parser.add_argument("-of", "--object", dest="fobject",
help="Load object definition from a file")
parser.add_argument("-s", "--sysname", dest="sysname", help="System name")
parser.add_argument("-sf", "--service", dest="fservice",
help="Load service definition from a file")
options, extra = parser.parse_known_args()
args, command_line_config = parse_args(extra)
print "store_interfaces: Storing ION config and interfaces in datastore, with options:" , str(options)
# -------------------------------------------------------------------------
# Store config and interfaces
# Set global testing flag to False. We are running as standalone script. This is NO TEST.
bootstrap.testing = False
# Set sysname if provided in startup argument
if options.sysname:
bootstrap.set_sys_name(options.sysname)
# Load minimal bootstrap config
bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, command_line_config)
# Delete sysname datastores if option "force_clean" is set
if options.force_clean:
from pyon.datastore import clear_couch_util
print "store_interfaces: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name()
clear_couch_util.clear_couch(bootstrap_config, prefix=bootstrap.get_sys_name())
# This holds the new CFG object for the system
# @TODO: Could add command line --config
ion_config = config.read_standard_configuration()
config.apply_configuration(ion_config, command_line_config)
# -------------------------------------------------------------------------
# Store config and interfaces
iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)
# Make sure core datastores exist
iadm.create_core_datastores()
# Store system CFG properties
iadm.store_config(ion_config)
# Store system interfaces
iadm.store_interfaces(options.fobject, options.fservice)
iadm.close()