def __init__(self, *args, **kwargs):
BaseContainerAgent.__init__(self, *args, **kwargs)
# set id and name (as they are set in base class call)
self.id = string.replace('%s_%d' % (os.uname()[1], os.getpid()), ".", "_")
self.name = "cc_agent_%s" % self.id
Container.instance = self
# TODO: Bug: Replacing CFG instance not work because references are already public. Update directly
dict_merge(CFG, kwargs)
from pyon.core import bootstrap
bootstrap.sys_name = CFG.system.name or bootstrap.sys_name
log.debug("Container (sysname=%s) initializing ..." % bootstrap.sys_name)
# Keep track of the overrides from the command-line, so they can trump app/rel file data
self.spawn_args = DictModifier(CFG, kwargs)
# Load object and service registry etc.
bootstrap_pyon()
# Create this Container's specific ExchangeManager instance
self.ex_manager = ExchangeManager(self)
# Create this Container's specific ProcManager instance
self.proc_manager = ProcManager(self)
# Create this Container's specific AppManager instance
self.app_manager = AppManager(self)
# DatastoreManager - controls access to Datastores (both mock and couch backed)
self.datastore_manager = DatastoreManager()
log.debug("Container initialized, OK.")
def start_paster():
from gevent import monkey
monkey.patch_all()
from pyon.core.bootstrap import bootstrap_pyon
bootstrap_pyon()
paste.script.command.run()
def main():
parser = argparse.ArgumentParser(description="ScionCC Control")
parser.add_argument(
"pidfile",
help="pidfile to use. If not specified, uses the first one found.")
parser.add_argument("command",
help="command to send to the container agent",
choices=IContainerAgent.names())
parser.add_argument("commandargs",
metavar="arg",
nargs="*",
help="arguments to the command being sent")
opts = parser.parse_args()
pidfile = opts.pidfile
if not pidfile:
raise Exception("No pidfile specified")
parms = {}
with open(pidfile, 'r') as pf:
parms = msgpack.loads(pf.read())
assert parms, "No content in pidfile"
bootstrap_pyon()
node, ioloop = make_node(parms['messaging'])
node.setup_interceptors(CFG.interceptor)
cc = ContainerAgentClient(node=node,
to_name=(parms['container-xp'],
parms['container-agent']))
# make a manual call - this is to avoid having to have the IonObject for the call
methdefs = [
x[1] for x in IContainerAgent.namesAndDescriptions()
if x[0] == opts.command
]
assert len(methdefs) == 1
arg_names = methdefs[0].positional # ('name', 'module', 'cls', 'config')
msg_args = dict(
zip(arg_names, opts.commandargs)
) # ('name', <usrinp1>, 'cls', <usrinp2>) -> { 'name' : <usrinp1>, 'cls': <usrinp2> }
retval = cc.request(msg_args, op=opts.command)
# special case: status
if opts.command == "status":
statstr = retval
print "Status:", statstr
if statstr != "RUNNING":
node.client.close()
sys.exit(2)
else:
print "Returned", retval
node.client.close()
def __init__(self, *args, **kwargs):
BaseContainerAgent.__init__(self, *args, **kwargs)
self._is_started = False
# set id and name (as they are set in base class call)
self.id = string.replace('%s_%d' % (os.uname()[1], os.getpid()), ".",
"_")
self.name = "cc_agent_%s" % self.id
Container.instance = self
# TODO: Bug: Replacing CFG instance not work because references are already public. Update directly
dict_merge(CFG, kwargs, inplace=True)
from pyon.core import bootstrap
bootstrap.container_instance = self
bootstrap.assert_configuration(CFG)
log.debug("Container (sysname=%s) initializing ..." %
bootstrap.get_sys_name())
# Keep track of the overrides from the command-line, so they can trump app/rel file data
self.spawn_args = kwargs
# Load object and service registry etc.
bootstrap_pyon()
# Create this Container's specific ExchangeManager instance
self.ex_manager = ExchangeManager(self)
# Create this Container's specific ProcManager instance
self.proc_manager = ProcManager(self)
# Create this Container's specific AppManager instance
self.app_manager = AppManager(self)
# DatastoreManager - controls access to Datastores (both mock and couch backed)
self.datastore_manager = DatastoreManager()
# File System - Interface to the OS File System, using correct path names and setups
self.file_system = FileSystem(CFG)
# Governance Controller - manages the governance related interceptors
self.governance_controller = GovernanceController(self)
# sFlow manager - controls sFlow stat emission
self.sflow_manager = SFlowManager(self)
# Coordinates the container start
self._is_started = False
self._capabilities = []
self._status = "INIT"
# protection for when the container itself is used as a Process for clients
self.container = self
log.debug("Container initialized, OK.")
def __init__(self, *args, **kwargs):
BaseContainerAgent.__init__(self, *args, **kwargs)
self._is_started = False
# set id and name (as they are set in base class call)
self.id = string.replace('%s_%d' % (os.uname()[1], os.getpid()), ".", "_")
self.name = "cc_agent_%s" % self.id
Container.instance = self
# TODO: Bug: Replacing CFG instance not work because references are already public. Update directly
dict_merge(CFG, kwargs, inplace=True)
from pyon.core import bootstrap
bootstrap.container_instance = self
bootstrap.assert_configuration(CFG)
log.debug("Container (sysname=%s) initializing ..." % bootstrap.get_sys_name())
# Keep track of the overrides from the command-line, so they can trump app/rel file data
self.spawn_args = kwargs
# Load object and service registry etc.
bootstrap_pyon()
# Create this Container's specific ExchangeManager instance
self.ex_manager = ExchangeManager(self)
# Create this Container's specific ProcManager instance
self.proc_manager = ProcManager(self)
# Create this Container's specific AppManager instance
self.app_manager = AppManager(self)
# DatastoreManager - controls access to Datastores (both mock and couch backed)
self.datastore_manager = DatastoreManager()
# File System - Interface to the OS File System, using correct path names and setups
self.file_system = FileSystem(CFG)
# Governance Controller - manages the governance related interceptors
self.governance_controller = GovernanceController(self)
# sFlow manager - controls sFlow stat emission
self.sflow_manager = SFlowManager(self)
# Coordinates the container start
self._is_started = False
self._capabilities = []
self._status = "INIT"
# protection for when the container itself is used as a Process for clients
self.container = self
log.debug("Container initialized, OK.")
def begin(self):
from interface.services.cei.iprocess_dispatcher_service import ProcessDispatcherServiceClient
from pyon.net.messaging import make_node
from pyon.core import bootstrap
from pyon.public import CFG
self.base_pids = []
self.rpc_timeout = 2
self._procs_by_test = {}
if not bootstrap.pyon_initialized:
bootstrap.bootstrap_pyon()
self.node, self.ioloop = make_node()
self.node.setup_interceptors(CFG.interceptor)
self.pd_cli = ProcessDispatcherServiceClient(node=self.node)
def begin(self):
from interface.services.cei.iprocess_dispatcher_service import ProcessDispatcherServiceClient
from pyon.net.messaging import make_node
from pyon.core import bootstrap
from pyon.public import CFG
self.base_pids = []
self.rpc_timeout = 2
self._procs_by_test = {}
if not bootstrap.pyon_initialized:
bootstrap.bootstrap_pyon()
self.node, self.ioloop = make_node()
self.node.setup_interceptors(CFG.interceptor)
self.pd_cli = ProcessDispatcherServiceClient(node=self.node)
def __init__(self):
bootstrap_pyon()
dsm = DatastoreManager()
self.datastore = dsm.get_datastore(ds_name='coverage')
if self.datastore is None:
raise RuntimeError("Unable to load datastore for coverage")
else:
self.entity_table_name = self.datastore._get_datastore_name()
log.trace("Got datastore: %s type %s" % (self.datastore._get_datastore_name(), str(type(self.datastore))))
self.span_store = dsm.get_datastore(ds_name='coverage_spans')
if self.span_store is None:
raise RuntimeError("Unable to load datastore for coverage_spans")
else:
self.span_table_name = self.span_store._get_datastore_name()
log.trace("Got datastore: %s type %s", self.span_store._get_datastore_name(), type(self.span_store))
def main():
parser = argparse.ArgumentParser(description="ScionCC Control")
parser.add_argument("pidfile", help="pidfile to use. If not specified, uses the first one found.")
parser.add_argument("command", help="command to send to the container agent", choices=IContainerAgent.names())
parser.add_argument("commandargs", metavar="arg", nargs="*", help="arguments to the command being sent")
opts = parser.parse_args()
pidfile = opts.pidfile
if not pidfile:
raise Exception("No pidfile specified")
parms = {}
with open(pidfile, 'r') as pf:
parms = msgpack.loads(pf.read())
assert parms, "No content in pidfile"
bootstrap_pyon()
node, ioloop = make_node(parms['messaging'])
node.setup_interceptors(CFG.interceptor)
cc = ContainerAgentClient(node=node, to_name=(parms['container-xp'], parms['container-agent']))
# make a manual call - this is to avoid having to have the IonObject for the call
methdefs = [x[1] for x in IContainerAgent.namesAndDescriptions() if x[0] == opts.command]
assert len(methdefs) == 1
arg_names = methdefs[0].positional # ('name', 'module', 'cls', 'config')
msg_args = dict(zip(arg_names, opts.commandargs)) # ('name', <usrinp1>, 'cls', <usrinp2>) -> { 'name' : <usrinp1>, 'cls': <usrinp2> }
retval = cc.request(msg_args, op=opts.command)
# special case: status
if opts.command == "status":
statstr = retval
print "Status:", statstr
if statstr != "RUNNING":
node.client.close()
sys.exit(2)
else:
print "Returned", retval
node.client.close()
def arg_initialize(self):
if bootstrap.pyon_initialized:
raise BadRequest("Pyon already initalized")
parser = argparse.ArgumentParser()
parser.add_argument("-s",
"--sysname",
dest="sysname",
help="System name")
options, extra = parser.parse_known_args()
args, command_line_config = parse_args(extra)
# -------------------------------------------------------------------------
# Store config and interfaces
# Set global testing flag to False. We are running as standalone script. This is NO TEST.
bootstrap.testing = False
# Set sysname if provided in startup argument
if options.sysname:
bootstrap.set_sys_name(options.sysname)
# bootstrap_config - Used for running this store_interfaces script
bootstrap_config = config.read_local_configuration(
['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config,
pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not options.sysname and bootstrap_config.get_safe(
"system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# ion_config - Holds the new CFG object for the system (independent of this tool's config)
ion_config = config.read_standard_configuration()
config.apply_configuration(ion_config, command_line_config)
# Bootstrap pyon's core. Load configuration etc.
bootstrap.bootstrap_pyon(pyon_cfg=ion_config)
def begin(self):
from interface.services.cei.iprocess_dispatcher_service import ProcessDispatcherServiceClient
from pyon.net.messaging import make_node
from pyon.core import bootstrap
from pyon.public import CFG
self.rpc_timeout = 2
self.base_pids = []
self._procs_by_test = {}
if not bootstrap.pyon_initialized:
bootstrap.bootstrap_pyon()
self.node, self.ioloop = make_node()
self.node.setup_interceptors(CFG.interceptor)
self.pd_cli = ProcessDispatcherServiceClient(node=self.node)
# Set base_pids once
from pyon.core.exception import Timeout
try:
self.base_pids = [ proc.process_id for proc in self.pd_cli.list_processes(timeout=20) ]
except Timeout:
pass
def begin(self):
self._active_queues = set()
self._test_changes = {}
self._queues_declared = [] # ordered list of queues declared
self._queues = defaultdict(list) # queue name -> list of accesses
# Make sure we initialize pyon before anything in this plugin executes
from pyon.core import bootstrap
if not bootstrap.pyon_initialized:
bootstrap.bootstrap_pyon()
from pyon.ion.exchange import ExchangeManager
from pyon.util.containers import DotDict
from pyon.core.bootstrap import CFG
from mock import Mock
containermock = Mock()
containermock.resource_registry.find_resources.return_value = ([], None)
self.ex_manager = ExchangeManager(containermock) # needs to be able to setattr
self.ex_manager._nodes['priviledged'] = DotDict(client=DotDict(parameters=DotDict(host=CFG.get_safe('server.amqp.host', 'localhost'))))
def report(self, stream):
""" all tests have completed but --with-pycc has not yet stopped external container.
request that containers log statistics now
"""
# initialize pyon so we can get system name
from pyon.core import bootstrap
if not bootstrap.pyon_initialized:
bootstrap.bootstrap_pyon()
from pyon.public import get_sys_name, CFG
# make request: bin/pycc --sysname mgmt -x ion.processes.test.manage_system.ReportStats
null = open('/dev/null', 'w')
cmd = ['bin/pycc', '--sysname', get_sys_name(), '-x', 'ion.processes.test.manage_system.ReportStats' ]
status = subprocess.call(cmd, stdout=null, stderr=null)
if status==0:
stream.write('container statistics: a report request has been sent\n')
time.sleep(5) # give time to handle before container shutdown begins
else:
stream.write('container statistics: failed to send report request (logging anyway -- who needs a container?)\n')
from ooi.timer import get_accumulators
for a in get_accumulators().values():
a.log()
def arg_initialize(self):
if bootstrap.pyon_initialized:
raise BadRequest("Pyon already initalized")
parser = argparse.ArgumentParser()
parser.add_argument("-s", "--sysname", dest="sysname", help="System name")
options, extra = parser.parse_known_args()
args, command_line_config = parse_args(extra)
# -------------------------------------------------------------------------
# Store config and interfaces
# Set global testing flag to False. We are running as standalone script. This is NO TEST.
bootstrap.testing = False
# Set sysname if provided in startup argument
if options.sysname:
bootstrap.set_sys_name(options.sysname)
# bootstrap_config - Used for running this store_interfaces script
bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not options.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# ion_config - Holds the new CFG object for the system (independent of this tool's config)
ion_config = config.read_standard_configuration()
config.apply_configuration(ion_config, command_line_config)
# Bootstrap pyon's core. Load configuration etc.
bootstrap.bootstrap_pyon(pyon_cfg=ion_config)
def prepare_container():
"""
Walks through pyon initialization in a deterministic way and initializes Container.
In particular make sure configuration is loaded in correct order and
pycc startup arguments are considered.
"""
import threading
threading.current_thread().name = "CC-Main"
# SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
import pyon
from pyon.core import bootstrap, config
# Set global testing flag to False. We are running as capability container. This is NO TEST.
bootstrap.testing = False
# Set sysname if provided in startup argument
if opts.sysname:
bootstrap.set_sys_name(opts.sysname)
# Trigger any initializing default logic in get_sys_name
bootstrap.get_sys_name()
command_line_config = kwargs
# This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
bootstrap_config = None
# This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
pyon_config = config.read_standard_configuration()
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if opts.config:
if '{' in opts.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(opts.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" % opts.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([opts.config]).data
# Determine bootstrap_config
if opts.config_from_directory:
# Load minimal bootstrap config if option "config_from_directory"
bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
else:
# Otherwise: Set to standard set of local config files plus command line overrides
bootstrap_config = pyon_config.copy()
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not opts.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Delete sysname datastores if option "force_clean" is set
if opts.force_clean:
from pyon.datastore import clear_couch_util
print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name()
clear_couch_util.clear_couch(bootstrap_config, prefix=bootstrap.get_sys_name())
# If auto_bootstrap, load config and interfaces into directory
# Note: this is idempotent and will not alter anything if this is not the first container to run
if bootstrap_config.system.auto_bootstrap:
print "pycc: auto_bootstrap=True."
stored_config = pyon_config.copy()
config.apply_configuration(stored_config, config_override)
config.apply_configuration(stored_config, command_line_config)
config.auto_bootstrap_config(bootstrap_config, system_cfg=stored_config)
# Determine the final pyon_config
# - Start from standard config already set (pyon.yml + local YML files)
# - Optionally load config from directory
if opts.config_from_directory:
config.apply_remote_config(pyon_config)
# - Last apply any separate command line config overrides
config.apply_configuration(pyon_config, config_override)
config.apply_configuration(pyon_config, command_line_config)
# Load logging override config if provided. Supports variants literal and path.
logging_config_override = None
if opts.logcfg:
if '{' in opts.logcfg:
# Variant 1: Value is dict of config values
try:
eval_value = ast.literal_eval(opts.logcfg)
logging_config_override = eval_value
except ValueError:
raise Exception("Value error in logcfg arg '%s'" % opts.logcfg)
else:
# Variant 2: Value is path to YAML file containing config values
pyon.DEFAULT_LOGGING_PATHS.append(opts.logcfg)
# Also set the immediate flag, but only if specified - it is an override
if opts.immediate:
dict_merge(pyon_config, {'system':{'immediate':True}}, True)
# Bootstrap pyon's core. Load configuration etc.
bootstrap.bootstrap_pyon(
logging_config_override=logging_config_override,
pyon_cfg=pyon_config)
# Auto-bootstrap interfaces
# @WARN: This currently imports ALL modules, executing ALL static initializers as side effect!!!!!!!
if bootstrap_config.system.auto_bootstrap:
config.auto_bootstrap_interfaces(bootstrap_config)
if opts.no_container:
print "pycc: no_container=True. Stopping here."
return None
# Create the container instance
from pyon.container.cc import Container
container = Container(*args, **command_line_config)
return container
def generate_validation_report(self):
# WARNING!!!!
# At this point, all the code (py files) should be generated. Now we can bootstrap pyon
# which reads these py files.
# THEN we can load all the modules, which depend on pyon
from pyon.core import bootstrap
bootstrap.bootstrap_pyon()
validation_results = "Report generated on " + self.currtime + "\n"
self.load_mods("interface/services", True)
base_subtypes = self.find_subtypes(BaseService)
self.load_mods("ion", False)
self.load_mods("examples", False)
for base_subtype in base_subtypes:
base_subtype_name = base_subtype.__module__ + "." + base_subtype.__name__
compare_methods = {}
for method_tuple in inspect.getmembers(base_subtype,
inspect.ismethod):
method_name = method_tuple[0]
method = method_tuple[1]
# Ignore private methods
if method_name.startswith("_"):
continue
# Ignore methods not implemented in the class
if method_name not in base_subtype.__dict__:
continue
compare_methods[method_name] = method
# Find implementing subtypes of each base interface
impl_subtypes = self.find_subtypes(base_subtype)
if len(impl_subtypes) == 0:
validation_results += "\nBase service: %s \n" % base_subtype_name
validation_results += " No impl subtypes found\n"
for impl_subtype in self.find_subtypes(base_subtype):
impl_subtype_name = impl_subtype.__module__ + "." + impl_subtype.__name__
# Compare parameters
added_class_names = False
found_error = False
for key in compare_methods:
if key not in impl_subtype.__dict__:
found_error = True
if not added_class_names:
added_class_names = True
validation_results += "\nBase service: %s\n" % base_subtype_name
validation_results += "Impl subtype: %s\n" % impl_subtype_name
validation_results += " Method '%s' not implemented\n" % key
else:
base_params = inspect.getargspec(compare_methods[key])
impl_params = inspect.getargspec(
impl_subtype.__dict__[key])
if base_params != impl_params:
found_error = True
if not added_class_names:
added_class_names = True
validation_results += "\nBase service: %s\n" % base_subtype_name
validation_results += "Impl subtype: %s\n" % impl_subtype_name
validation_results += " Method '%s' implementation is out of sync\n" % key
validation_results += " Base: %s\n" % str(
base_params)
validation_results += " Impl: %s\n" % str(
impl_params)
if found_error is False:
validation_results += "\nBase service: %s\n" % base_subtype_name
validation_results += "Impl subtype: %s\n" % impl_subtype_name
validation_results += " OK\n"
reportfile = os.path.join('interface', 'validation_report.txt')
try:
os.unlink(reportfile)
except:
pass
print " Writing service implementation validation report to '" + reportfile + "'"
with open(reportfile, 'w') as f:
f.write(validation_results)
def initialize_ion_int_tests():
# Bootstrap pyon CFG, logging and object/resource interfaces
bootstrap_pyon()
if bootstrap.is_testing():
IonIntegrationTestCase._force_clean(False)
pre_initialize_ion()
def begin(self):
"""Called before any tests are collected or run. Use this to
perform any setup needed before testing begins.
"""
# Make sure we initialize pyon before anything in this plugin executes
from pyon.core import bootstrap
if not bootstrap.pyon_initialized:
bootstrap.bootstrap_pyon()
try:
from pyon.public import get_sys_name, CFG
self.sysname = get_sys_name()
# Clean exchanges and system queues out there
try:
rmh = RabbitManagementHelper(make_parser(), '-H %s -P 55672 -u %s -p %s -V %s'
% (CFG.server.amqp.host, CFG.server.amqp.username,
CFG.server.amqp.password, CFG.server.amqp.vhost))
exchanges = rmh.list_names('exchanges')
deleted = rmh.delete_names_with_prefix('exchange', exchanges, self.sysname)
debug.write('Deleted exchanges:\n%s \n' % '\n'.join(deleted))
queues = rmh.list_names('queues')
deleted = rmh.delete_names_with_prefix('queue', queues, self.sysname)
debug.write('Deleted queues:\n%s \n' % '\n'.join(deleted))
except Exception as e:
pass
# Force datastore loader to use the same sysname
from pyon.datastore.datastore_admin import DatastoreAdmin
self.datastore_admin = DatastoreAdmin(config=CFG)
self.datastore_admin.clear_datastore(prefix=self.sysname)
def die(signum, frame):
# For whatever reason, the parent doesn't die some times
# when getting KeyboardInterrupt. Hence this signal
# handler.
# Signal is pass through. The child pycc gets
# its own KeyboardInterrupt and will shut down accordingly.
debug.write('Received Keyboard Interrupt. Exiting now.\n')
os._exit(9)
signal.signal(signal.SIGINT, die)
def no_zombie(signum, frame):
# Debug to figure out who's dying
debug.write('SIGCHLD received\n')
stack = []
while frame:
stack.append(frame)
frame =frame.f_back
stack.reverse()
for frame in stack:
debug.write('Frame %s in %s at line %s\n' %
(frame.f_code.co_name,
frame.f_code.co_filename, frame.f_lineno))
debug.write('Child is dead...Clean up now so there is no zombie\n')
(pid, status) = os.wait()
exitstatus, signum = status & 0xff, (status & 0xff00) >> 8
debug.write('Child pid %d with exit status %d and signum %d\n' % (pid, exitstatus, signum))
# Could be dangerous. Comment this out.
# signal.signal(signal.SIGCHLD, no_zombie)
def container_started_cb(signum, frame):
"""Callback when child pycc service is ready"""
self.container_started = True
signal.signal(signal.SIGUSR1, container_started_cb)
# Make sure the pycc process has the same sysname as the nose
ccargs = ['bin/pycc', '--noshell', '-sp', '--sysname=%s' % self.sysname,
'--logcfg=res/config/logging.pycc.yml',
'--rel=%s' % self.rel,
"--config={'system': {'auto_bootstrap': True}}"]
debug.write('Starting cc process: %s\n' % ' '.join(ccargs))
newenv = os.environ.copy()
po = subprocess.Popen(ccargs, env=newenv, close_fds=True)
self.ccs.append(po)
# Wait for container to be ready
while not self.container_started:
time.sleep(0.2)
debug.write('Child container is ready...\n')
# Dump datastore
self.datastore_admin.dump_datastore(path='res/dd', compact=True)
debug.write('Dump child container state to file...\n')
# Clean again to make sure the first nosetest starts on a clean
# slate
self.datastore_admin.clear_datastore(prefix=self.sysname)
# Enable CEI mode for the tests
os.environ['CEI_LAUNCH_TEST'] = '1'
debug.write('Start nose tests now...\n')
except Exception as e:
self.container_shutdown()
raise e
def initialize_ion_int_tests():
# Bootstrap pyon CFG, logging and object/resource interfaces
bootstrap_pyon()
if bootstrap.is_testing():
IonIntegrationTestCase._force_clean(False, initial=True)
pre_initialize_ion()
def prepare_container():
"""
Walks through pyon initialization in a deterministic way and initializes Container.
In particular make sure configuration is loaded in correct order and
pycc startup arguments are considered.
"""
import threading
threading.current_thread().name = "CC-Main"
# SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
import pyon
from pyon.core import bootstrap, config
# Set global testing flag to False. We are running as capability container, because
# we started through the pycc program.
bootstrap.testing = False
# Set sysname if provided in startup argument
if opts.sysname:
bootstrap.set_sys_name(opts.sysname)
# Trigger any initializing default logic in get_sys_name
bootstrap.get_sys_name()
command_line_config = kwargs
# This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
bootstrap_config = None
# This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
pyon_config = config.read_standard_configuration() # Initial pyon.yml + pyon.local.yml
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if opts.config:
if "{" in opts.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(opts.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" % opts.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([opts.config]).data
# Determine bootstrap_config
if opts.config_from_directory:
# Load minimal bootstrap config if option "config_from_directory"
bootstrap_config = config.read_local_configuration(["res/config/pyon_min_boot.yml"])
config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
else:
# Otherwise: Set to standard set of local config files plus command line overrides
bootstrap_config = deepcopy(pyon_config)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not opts.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Delete sysname datastores if option "force_clean" is set
if opts.force_clean:
from pyon.datastore import clear_couch_util
print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name()
clear_couch_util.clear_couch(
bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name()
)
pyon_config.container.filesystem.force_clean = True
from pyon.core.interfaces.interfaces import InterfaceAdmin
iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)
# If auto_bootstrap, load config and interfaces into directory
# Note: this is idempotent and will not alter anything if this is not the first container to run
if bootstrap_config.system.auto_bootstrap:
print "pycc: auto_bootstrap=True."
stored_config = deepcopy(pyon_config)
config.apply_configuration(stored_config, config_override)
config.apply_configuration(stored_config, command_line_config)
iadm.create_core_datastores()
iadm.store_config(stored_config)
# Determine the final pyon_config
# - Start from standard config already set (pyon.yml + local YML files)
# - Optionally load config from directory
if opts.config_from_directory:
config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config)
# - Apply container profile specific config
config.apply_profile_configuration(pyon_config, bootstrap_config)
# - Reapply pyon.local.yml here again for good measure
config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
# - Last apply any separate command line config overrides
config.apply_configuration(pyon_config, config_override)
config.apply_configuration(pyon_config, command_line_config)
# Also set the immediate flag, but only if specified - it is an override
if opts.immediate:
from pyon.util.containers import dict_merge
dict_merge(pyon_config, {"system": {"immediate": True}}, True)
# Bootstrap pyon's core. Load configuration etc.
bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)
# Delete any queues/exchanges owned by sysname if option "broker_clean" is set
if opts.broker_clean:
print "pycc: broker_clean=True, sysname:", bootstrap.get_sys_name()
# build connect str
connect_str = "-q -H %s -P %s -u %s -p %s -V %s" % (
pyon_config.get_safe("server.amqp_priv.host", pyon_config.get_safe("server.amqp.host", "localhost")),
pyon_config.get_safe("container.exchange.management.port", "55672"),
pyon_config.get_safe("container.exchange.management.username", "guest"),
pyon_config.get_safe("container.exchange.management.password", "guest"),
"/",
)
from putil.rabbithelper import clean_by_sysname
deleted_exchanges, deleted_queues = clean_by_sysname(connect_str, bootstrap.get_sys_name())
print " exchanges deleted (%s): %s" % (len(deleted_exchanges), ",".join(deleted_exchanges))
print " queues deleted (%s): %s" % (len(deleted_queues), ",".join(deleted_queues))
if opts.force_clean:
path = os.path.join(pyon_config.get_safe("container.filesystem.root", "/tmp/ion"), bootstrap.get_sys_name())
print "force_clean: Removing", path
FileSystem._clean(pyon_config)
# Auto-bootstrap interfaces
if bootstrap_config.system.auto_bootstrap:
iadm.store_interfaces(idempotent=True)
iadm.close()
if opts.no_container:
print "pycc: no_container=True. Stopping here."
return None
# Create the container instance
from pyon.container.cc import Container
container = Container(*args, **command_line_config)
return container
def generate_validation_report (self):
# WARNING!!!!
# At this point, all the code (py files) should be generated. Now we can bootstrap pyon
# which reads these py files.
# THEN we can load all the modules, which depend on pyon
from pyon.core import bootstrap
bootstrap.bootstrap_pyon()
validation_results = "Report generated on " + self.currtime + "\n"
self.load_mods("interface/services", True)
base_subtypes = self.find_subtypes(BaseService)
self.load_mods("ion", False)
self.load_mods("examples", False)
for base_subtype in base_subtypes:
base_subtype_name = base_subtype.__module__ + "." + base_subtype.__name__
compare_methods = {}
for method_tuple in inspect.getmembers(base_subtype, inspect.ismethod):
method_name = method_tuple[0]
method = method_tuple[1]
# Ignore private methods
if method_name.startswith("_"):
continue
# Ignore methods not implemented in the class
if method_name not in base_subtype.__dict__:
continue
compare_methods[method_name] = method
# Find implementing subtypes of each base interface
impl_subtypes = self.find_subtypes(base_subtype)
if len(impl_subtypes) == 0:
validation_results += "\nBase service: %s \n" % base_subtype_name
validation_results += " No impl subtypes found\n"
for impl_subtype in self.find_subtypes(base_subtype):
impl_subtype_name = impl_subtype.__module__ + "." + impl_subtype.__name__
# Compare parameters
added_class_names = False
found_error = False
for key in compare_methods:
if key not in impl_subtype.__dict__:
found_error = True
if not added_class_names:
added_class_names = True
validation_results += "\nBase service: %s\n" % base_subtype_name
validation_results += "Impl subtype: %s\n" % impl_subtype_name
validation_results += " Method '%s' not implemented\n" % key
else:
base_params = inspect.getargspec(compare_methods[key])
impl_params = inspect.getargspec(impl_subtype.__dict__[key])
if base_params != impl_params:
found_error = True
if not added_class_names:
added_class_names = True
validation_results += "\nBase service: %s\n" % base_subtype_name
validation_results += "Impl subtype: %s\n" % impl_subtype_name
validation_results += " Method '%s' implementation is out of sync\n" % key
validation_results += " Base: %s\n" % str(base_params)
validation_results += " Impl: %s\n" % str(impl_params)
if found_error == False:
validation_results += "\nBase service: %s\n" % base_subtype_name
validation_results += "Impl subtype: %s\n" % impl_subtype_name
validation_results += " OK\n"
reportfile = os.path.join('interface', 'validation_report.txt')
try:
os.unlink(reportfile)
except:
pass
print "Writing validation report to '" + reportfile + "'"
with open(reportfile, 'w') as f:
f.write(validation_results)
def prepare_container():
"""
Walks through pyon initialization in a deterministic way and initializes Container.
In particular make sure configuration is loaded in correct order and
pycc startup arguments are considered.
"""
import threading
threading.current_thread().name = "CC-Main"
# SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
import pyon
from pyon.core import bootstrap, config
# Set global testing flag to False. We are running as capability container. This is NO TEST.
bootstrap.testing = False
# Set sysname if provided in startup argument
if opts.sysname:
bootstrap.set_sys_name(opts.sysname)
# Trigger any initializing default logic in get_sys_name
bootstrap.get_sys_name()
command_line_config = kwargs
# This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
bootstrap_config = None
# This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
pyon_config = config.read_standard_configuration()
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if opts.config:
if '{' in opts.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(opts.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" %
opts.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([opts.config]).data
# Determine bootstrap_config
if opts.config_from_directory:
# Load minimal bootstrap config if option "config_from_directory"
bootstrap_config = config.read_local_configuration(
['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config,
pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
else:
# Otherwise: Set to standard set of local config files plus command line overrides
bootstrap_config = pyon_config.copy()
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not opts.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Delete sysname datastores if option "force_clean" is set
if opts.force_clean:
from pyon.datastore import clear_couch_util
print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name(
)
clear_couch_util.clear_couch(bootstrap_config,
prefix=bootstrap.get_sys_name())
# If auto_bootstrap, load config and interfaces into directory
# Note: this is idempotent and will not alter anything if this is not the first container to run
if bootstrap_config.system.auto_bootstrap:
print "pycc: auto_bootstrap=True."
stored_config = pyon_config.copy()
config.apply_configuration(stored_config, config_override)
config.apply_configuration(stored_config, command_line_config)
config.auto_bootstrap_config(bootstrap_config,
system_cfg=stored_config)
# Determine the final pyon_config
# - Start from standard config already set (pyon.yml + local YML files)
# - Optionally load config from directory
if opts.config_from_directory:
config.apply_remote_config(pyon_config)
# - Last apply any separate command line config overrides
config.apply_configuration(pyon_config, config_override)
config.apply_configuration(pyon_config, command_line_config)
# Load logging override config if provided. Supports variants literal and path.
logging_config_override = None
if opts.logcfg:
if '{' in opts.logcfg:
# Variant 1: Value is dict of config values
try:
eval_value = ast.literal_eval(opts.logcfg)
logging_config_override = eval_value
except ValueError:
raise Exception("Value error in logcfg arg '%s'" %
opts.logcfg)
else:
# Variant 2: Value is path to YAML file containing config values
pyon.DEFAULT_LOGGING_PATHS.append(opts.logcfg)
# Also set the immediate flag, but only if specified - it is an override
if opts.immediate:
dict_merge(pyon_config, {'system': {'immediate': True}}, True)
# Bootstrap pyon's core. Load configuration etc.
bootstrap.bootstrap_pyon(
logging_config_override=logging_config_override,
pyon_cfg=pyon_config)
# Auto-bootstrap interfaces
# @WARN: This currently imports ALL modules, executing ALL static initializers as side effect!!!!!!!
if bootstrap_config.system.auto_bootstrap:
config.auto_bootstrap_interfaces(bootstrap_config)
if opts.no_container:
print "pycc: no_container=True. Stopping here."
return None
# Create the container instance
from pyon.container.cc import Container
container = Container(*args, **command_line_config)
return container
def begin(self):
"""Called before any tests are collected or run. Use this to
perform any setup needed before testing begins.
"""
# Make sure we initialize pyon before anything in this plugin executes
from pyon.core import bootstrap
if not bootstrap.pyon_initialized:
bootstrap.bootstrap_pyon()
try:
from pyon.public import get_sys_name, CFG
self.sysname = get_sys_name()
# Clean exchanges and system queues out there
try:
connect_str = '-H %s -P 55672 -u %s -p %s -V %s' % (CFG.server.amqp.host,
CFG.server.amqp.username,
CFG.server.amqp.password,
CFG.server.amqp.vhost)
deleted_exchanges, deleted_queues = clean_by_sysname(connect_str, self.sysname)
debug.write('Deleted exchanges:\n%s \n' % '\n'.join(deleted_exchanges))
debug.write('Deleted queues:\n%s \n' % '\n'.join(deleted_queues))
except Exception as e:
pass
# Force datastore loader to use the same sysname
from pyon.datastore.datastore_admin import DatastoreAdmin
self.datastore_admin = DatastoreAdmin(config=CFG)
self.datastore_admin.clear_datastore(prefix=self.sysname)
def die(signum, frame):
# For whatever reason, the parent doesn't die some times
# when getting KeyboardInterrupt. Hence this signal
# handler.
# Signal is pass through. The child pycc gets
# its own KeyboardInterrupt and will shut down accordingly.
debug.write('Received Keyboard Interrupt. Exiting now.\n')
os._exit(9)
signal.signal(signal.SIGINT, die)
def no_zombie(signum, frame):
# Debug to figure out who's dying
debug.write('SIGCHLD received\n')
stack = []
while frame:
stack.append(frame)
frame =frame.f_back
stack.reverse()
for frame in stack:
debug.write('Frame %s in %s at line %s\n' %
(frame.f_code.co_name,
frame.f_code.co_filename, frame.f_lineno))
debug.write('Child is dead...Clean up now so there is no zombie\n')
(pid, status) = os.wait()
exitstatus, signum = status & 0xff, (status & 0xff00) >> 8
debug.write('Child pid %d with exit status %d and signum %d\n' % (pid, exitstatus, signum))
# Could be dangerous. Comment this out.
# signal.signal(signal.SIGCHLD, no_zombie)
def container_started_cb(signum, frame):
"""Callback when child pycc service is ready"""
self.container_started = True
signal.signal(signal.SIGUSR1, container_started_cb)
# Make sure the pycc process has the same sysname as the nose
ccargs = ['bin/pycc', '-o', '--noshell', '-sp', '--sysname=%s' % self.sysname,
'--logcfg=res/config/logging.pycc.yml',
'--rel=%s' % self.rel,
"--config={'system': {'auto_bootstrap': True}}"]
debug.write('Starting cc process: %s\n' % ' '.join(ccargs))
newenv = os.environ.copy()
po = subprocess.Popen(ccargs, env=newenv, close_fds=True)
self.ccs.append(po)
# Wait for container to be ready
while not self.container_started:
time.sleep(0.2)
debug.write('Child container is ready...\n')
# Dump datastore
self.datastore_admin.dump_datastore(path='res/dd', compact=True)
debug.write('Dump child container state to file...\n')
# Clean again to make sure the first nosetest starts on a clean
# slate
self.datastore_admin.clear_datastore(prefix=self.sysname)
# Set PYCC env var in case CEI needs to skip tests in pycc mode
os.environ['PYCC_MODE'] = '1'
# Enable CEI mode for the tests
os.environ['CEI_LAUNCH_TEST'] = '1'
debug.write('Start nose tests now...\n')
except Exception as e:
self.container_shutdown()
raise e
def start_paster():
from gevent import monkey
monkey.patch_all()
from pyon.core.bootstrap import bootstrap_pyon
bootstrap_pyon()
paste.script.command.run()
#! /usr/bin/env python
"""Unit test base class and utils"""
from copy import deepcopy
from mock import Mock, mocksignature, patch, DEFAULT
import unittest
from unittest import SkipTest
from zope.interface import implementedBy
from pyon.core.bootstrap import IonObject, bootstrap_pyon, get_service_registry, CFG
from pyon.util.containers import dict_merge, DotDict
from pyon.util.file_sys import FileSystem
bootstrap_pyon()
def func_names(cls):
import types
return [name for name, value in cls.__dict__.items() if
isinstance(value, types.FunctionType)]
def pop_last_call(mock):
if not mock.call_count:
raise AssertionError('Cannot pop last call: call_count is 0')
mock.call_args_list.pop()
try:
mock.call_args = mock.call_args_list[-1]
except IndexError:
mock.call_args = None
mock.called = False
def prepare_container():
"""
Walks through pyon initialization in a deterministic way and initializes Container.
In particular make sure configuration is loaded in correct order and
pycc startup arguments are considered.
"""
# SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults
import pyon
import threading
threading.current_thread().name = "CC-Main"
import logging
global log
log = logging.getLogger('pycc')
from pyon.core import bootstrap, config
from pyon.util.containers import get_safe, dict_merge
# Set global testing flag to False. We are running as capability container, because
# we started through the pycc program.
bootstrap.testing = False
# Set sysname if provided in startup argument
if opts.sysname:
bootstrap.set_sys_name(opts.sysname)
# Trigger any initializing default logic in get_sys_name
bootstrap.get_sys_name()
command_line_config = kwargs
# This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
bootstrap_config = None
# This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
pyon_config = config.read_standard_configuration() # Initial pyon.yml + pyon.local.yml
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if opts.config:
if '{' in opts.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(opts.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" % opts.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([opts.config]).data
# Determine bootstrap_config
if opts.config_from_directory:
# Load minimal bootstrap config if option "config_from_directory"
bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
log.info("config_from_directory=True. Minimal bootstrap configuration: %s", bootstrap_config)
else:
# Otherwise: Set to standard set of local config files plus command line overrides
bootstrap_config = deepcopy(pyon_config)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not opts.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Force_clean - deletes sysname datastores
if opts.force_clean:
from pyon.datastore import clear_db_util
log.info("force_clean=True. DROP DATASTORES for sysname=%s", bootstrap.get_sys_name())
clear_db_util.clear_db(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name())
from pyon.core.interfaces.interfaces import InterfaceAdmin
iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)
# If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config
# WARNING: If multiple containers start concurrently, this may fail
if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True:
log.debug("auto_store_interfaces=True.")
stored_config = deepcopy(pyon_config)
config.apply_configuration(stored_config, config_override)
config.apply_configuration(stored_config, command_line_config)
iadm.create_core_datastores()
iadm.store_config(stored_config)
# Determine the final pyon_config:
# - Start from standard config already set (pyon.yml + local YML files)
# - Optionally load config from directory
if opts.config_from_directory:
config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config)
# - Apply container profile specific config
config.apply_profile_configuration(pyon_config, bootstrap_config)
# - Reapply pyon.local.yml here again for good measure
config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
# - Last apply any separate command line config overrides
config.apply_configuration(pyon_config, config_override)
config.apply_configuration(pyon_config, command_line_config)
iadm.set_config(pyon_config)
# Set the immediate flag when command line override specified
if opts.immediate:
dict_merge(pyon_config, {"system": {"immediate": True}}, inplace=True)
# Determine system bootmode for bootstrapping actions (unless explicitly specified)
if not pyon_config.get_safe("bootmode"):
set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode")
if set_bootmode == "auto":
if iadm.system_data_exists():
dict_merge(pyon_config, {"bootmode": "restart"}, inplace=True)
log.info("System bootmode auto-detection is ON. Determined bootmode=%s", pyon_config.get_safe("bootmode", "initial"))
elif set_bootmode == "secondary":
dict_merge(pyon_config, {"bootmode": "secondary"}, inplace=True)
log.info("System bootmode override. Set to bootmode=%s", pyon_config.get_safe("bootmode", ""))
log.info("System in bootmode=%s", pyon_config.get_safe("bootmode", "initial"))
# Bootstrap the pyon framework's core. Load configuration etc.
bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)
# Delete any queues/exchanges owned by sysname if option "broker_clean" is set
if opts.broker_clean:
log.info("broker_clean=True, sysname: %s", bootstrap.get_sys_name())
from putil.rabbitmq.rabbit_util import RabbitManagementUtil
rabbit_util = RabbitManagementUtil(pyon_config, sysname=bootstrap.get_sys_name())
deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname()
log.info("Exchanges deleted (%s): %s" % (len(deleted_exchanges), ", ".join(deleted_exchanges)))
log.info("Queues deleted (%s): %s" % (len(deleted_queues), ", ".join(deleted_queues)))
if opts.force_clean:
from pyon.util.file_sys import FileSystem
FileSystem._clean(pyon_config)
# If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging
if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True:
iadm.store_interfaces(idempotent=True)
iadm.declare_core_exchange_resources()
iadm.close()
if opts.no_container:
log.info("no_container=True. Stopping here.")
return None
# Create the container instance
from pyon.container.cc import Container
container = Container(*args, **command_line_config)
container.version = version
return container
def prepare_container():
"""
Walks through pyon initialization in a deterministic way and initializes Container.
In particular make sure configuration is loaded in correct order and
pycc startup arguments are considered.
"""
import threading
threading.current_thread().name = "CC-Main"
# SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
import pyon
from pyon.core import bootstrap, config
# Set global testing flag to False. We are running as capability container. This is NO TEST.
bootstrap.testing = False
# Set sysname if provided in startup argument
if opts.sysname:
bootstrap.set_sys_name(opts.sysname)
# Trigger any initializing default logic in get_sys_name
bootstrap.get_sys_name()
command_line_config = kwargs
# This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
bootstrap_config = None
# This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
pyon_config = config.read_standard_configuration(
) # Initial pyon.yml + pyon.local.yml
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if opts.config:
if '{' in opts.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(opts.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" %
opts.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([opts.config]).data
# Determine bootstrap_config
if opts.config_from_directory:
# Load minimal bootstrap config if option "config_from_directory"
bootstrap_config = config.read_local_configuration(
['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config,
pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
else:
# Otherwise: Set to standard set of local config files plus command line overrides
bootstrap_config = deepcopy(pyon_config)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not opts.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Delete sysname datastores if option "force_clean" is set
if opts.force_clean:
from pyon.datastore import clear_couch_util
print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name(
)
clear_couch_util.clear_couch(bootstrap_config,
prefix=bootstrap.get_sys_name())
pyon_config.container.filesystem.force_clean = True
from pyon.core.interfaces.interfaces import InterfaceAdmin
iadm = InterfaceAdmin(bootstrap.get_sys_name(),
config=bootstrap_config)
# If auto_bootstrap, load config and interfaces into directory
# Note: this is idempotent and will not alter anything if this is not the first container to run
if bootstrap_config.system.auto_bootstrap:
print "pycc: auto_bootstrap=True."
stored_config = deepcopy(pyon_config)
config.apply_configuration(stored_config, config_override)
config.apply_configuration(stored_config, command_line_config)
iadm.create_core_datastores()
iadm.store_config(stored_config)
# Determine the final pyon_config
# - Start from standard config already set (pyon.yml + local YML files)
# - Optionally load config from directory
if opts.config_from_directory:
config.apply_remote_config(bootstrap_cfg=bootstrap_config,
system_cfg=pyon_config)
config.apply_local_configuration(
pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS
) # apply pyon.local.yml again over top
# - Last apply any separate command line config overrides
config.apply_configuration(pyon_config, config_override)
config.apply_configuration(pyon_config, command_line_config)
# Also set the immediate flag, but only if specified - it is an override
if opts.immediate:
from pyon.util.containers import dict_merge
dict_merge(pyon_config, {'system': {'immediate': True}}, True)
# Bootstrap pyon's core. Load configuration etc.
bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)
# Delete any queues/exchanges owned by sysname if option "broker_clean" is set
if opts.broker_clean:
print "pycc: broker_clean=True, sysname:", bootstrap.get_sys_name()
# build connect str
connect_str = "-q -H %s -P 55672 -u %s -p %s -V %s" % (
pyon_config.get_safe(
'server.amqp_priv.host',
pyon_config.get_safe('server.amqp.host', 'localhost')),
pyon_config.get_safe('container.exchange.management.username',
'guest'),
pyon_config.get_safe('container.exchange.management.password',
'guest'), '/')
from putil.rabbithelper import clean_by_sysname
deleted_exchanges, deleted_queues = clean_by_sysname(
connect_str, bootstrap.get_sys_name())
print " exchanges deleted (%s): %s" % (
len(deleted_exchanges), ",".join(deleted_exchanges))
print " queues deleted (%s): %s" % (
len(deleted_queues), ",".join(deleted_queues))
# Auto-bootstrap interfaces
if bootstrap_config.system.auto_bootstrap:
iadm.store_interfaces(idempotent=True)
iadm.close()
if opts.no_container:
print "pycc: no_container=True. Stopping here."
return None
# Create the container instance
from pyon.container.cc import Container
container = Container(*args, **command_line_config)
return container
from pyon.datastore.datastore import DatastoreManager
from pyon.event.event import EventRepository
from pyon.ion.directory import Directory
from pyon.ion.state import StateRepository
from pyon.util.containers import DotDict, dict_merge, DictModifier
from pyon.util.log import log
from mock import patch
from pyon.public import CFG
from pyon.datastore.couchdb.couchdb_datastore import CouchDB_DataStore
from pyon.datastore.mockdb.mockdb_datastore import MockDB_DataStore
from contextlib import contextmanager
import unittest
import os
# Make this call more deterministic in time.
bootstrap_pyon()
scanned_services = False
class IonIntegrationTestCase(unittest.TestCase):
"""
Base test class to allow operations such as starting the container
TODO: Integrate with IonUnitTestCase
"""
def run(self, result=None):
unittest.TestCase.run(self, result)
def _start_container(self):
# hack to force queue auto delete on for int tests
self._turn_on_queue_auto_delete()
def prepare_container():
"""
Walks through pyon initialization in a deterministic way and initializes Container.
In particular make sure configuration is loaded in correct order and
pycc startup arguments are considered.
"""
# SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults
import pyon
import threading
threading.current_thread().name = "CC-Main"
import logging
global log
log = logging.getLogger('pycc')
from pyon.core import bootstrap, config
from pyon.util.containers import get_safe, dict_merge
# Set global testing flag to False. We are running as capability container, because
# we started through the pycc program.
bootstrap.testing = False
# Set sysname if provided in startup argument
if opts.sysname:
bootstrap.set_sys_name(opts.sysname)
# Trigger any initializing default logic in get_sys_name
bootstrap.get_sys_name()
command_line_config = kwargs
# This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
bootstrap_config = None
# This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
pyon_config = config.read_standard_configuration(
) # Initial pyon.yml + pyon.local.yml
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if opts.config:
if '{' in opts.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(opts.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" %
opts.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([opts.config]).data
# Determine bootstrap_config
if opts.config_from_directory:
# Load minimal bootstrap config if option "config_from_directory"
bootstrap_config = config.read_local_configuration(
['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config,
pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
log.info(
"config_from_directory=True. Minimal bootstrap configuration: %s",
bootstrap_config)
else:
# Otherwise: Set to standard set of local config files plus command line overrides
bootstrap_config = deepcopy(pyon_config)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not opts.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Force_clean - deletes sysname datastores
if opts.force_clean:
from pyon.datastore import clear_db_util
log.info("force_clean=True. DROP DATASTORES for sysname=%s",
bootstrap.get_sys_name())
clear_db_util.clear_db(bootstrap_config,
prefix=bootstrap.get_sys_name(),
sysname=bootstrap.get_sys_name())
from pyon.core.interfaces.interfaces import InterfaceAdmin
iadm = InterfaceAdmin(bootstrap.get_sys_name(),
config=bootstrap_config)
# If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config
# WARNING: If multiple containers start concurrently, this may fail
if get_safe(bootstrap_config,
"bootstrap.auto_store_interfaces") is True:
log.debug("auto_store_interfaces=True.")
stored_config = deepcopy(pyon_config)
config.apply_configuration(stored_config, config_override)
config.apply_configuration(stored_config, command_line_config)
iadm.create_core_datastores()
iadm.store_config(stored_config)
# Determine the final pyon_config:
# - Start from standard config already set (pyon.yml + local YML files)
# - Optionally load config from directory
if opts.config_from_directory:
config.apply_remote_config(bootstrap_cfg=bootstrap_config,
system_cfg=pyon_config)
# - Apply container profile specific config
config.apply_profile_configuration(pyon_config, bootstrap_config)
# - Reapply pyon.local.yml here again for good measure
config.apply_local_configuration(pyon_config,
pyon.DEFAULT_LOCAL_CONFIG_PATHS)
# - Last apply any separate command line config overrides
config.apply_configuration(pyon_config, config_override)
config.apply_configuration(pyon_config, command_line_config)
iadm.set_config(pyon_config)
# Set the immediate flag when command line override specified
if opts.immediate:
dict_merge(pyon_config, {"system": {
"immediate": True
}},
inplace=True)
# Determine system bootmode for bootstrapping actions (unless explicitly specified)
if not pyon_config.get_safe("bootmode"):
set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode")
if set_bootmode == "auto":
if iadm.system_data_exists():
dict_merge(pyon_config, {"bootmode": "restart"},
inplace=True)
log.info(
"System bootmode auto-detection is ON. Determined bootmode=%s",
pyon_config.get_safe("bootmode", "initial"))
elif set_bootmode == "secondary":
dict_merge(pyon_config, {"bootmode": "secondary"},
inplace=True)
log.info("System bootmode override. Set to bootmode=%s",
pyon_config.get_safe("bootmode", ""))
log.info("System in bootmode=%s",
pyon_config.get_safe("bootmode", "initial"))
# Bootstrap the pyon framework's core. Load configuration etc.
bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)
# Delete any queues/exchanges owned by sysname if option "broker_clean" is set
if opts.broker_clean:
log.info("broker_clean=True, sysname: %s",
bootstrap.get_sys_name())
from putil.rabbitmq.rabbit_util import RabbitManagementUtil
rabbit_util = RabbitManagementUtil(
pyon_config, sysname=bootstrap.get_sys_name())
deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname()
log.info("Exchanges deleted (%s): %s" %
(len(deleted_exchanges), ", ".join(deleted_exchanges)))
log.info("Queues deleted (%s): %s" %
(len(deleted_queues), ", ".join(deleted_queues)))
if opts.force_clean:
from pyon.util.file_sys import FileSystem
FileSystem._clean(pyon_config)
# If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging
if get_safe(bootstrap_config,
"bootstrap.auto_store_interfaces") is True:
iadm.store_interfaces(idempotent=True)
iadm.declare_core_exchange_resources()
iadm.close()
if opts.no_container:
log.info("no_container=True. Stopping here.")
return None
# Create the container instance
from pyon.container.cc import Container
container = Container(*args, **command_line_config)
container.version = version
return container
