def __init__(self, *args, **kwargs): BaseContainerAgent.__init__(self, *args, **kwargs) # set id and name (as they are set in base class call) self.id = string.replace('%s_%d' % (os.uname()[1], os.getpid()), ".", "_") self.name = "cc_agent_%s" % self.id Container.instance = self # TODO: Bug: Replacing CFG instance not work because references are already public. Update directly dict_merge(CFG, kwargs) from pyon.core import bootstrap bootstrap.sys_name = CFG.system.name or bootstrap.sys_name log.debug("Container (sysname=%s) initializing ..." % bootstrap.sys_name) # Keep track of the overrides from the command-line, so they can trump app/rel file data self.spawn_args = DictModifier(CFG, kwargs) # Load object and service registry etc. bootstrap_pyon() # Create this Container's specific ExchangeManager instance self.ex_manager = ExchangeManager(self) # Create this Container's specific ProcManager instance self.proc_manager = ProcManager(self) # Create this Container's specific AppManager instance self.app_manager = AppManager(self) # DatastoreManager - controls access to Datastores (both mock and couch backed) self.datastore_manager = DatastoreManager() log.debug("Container initialized, OK.")
def start_paster(): from gevent import monkey monkey.patch_all() from pyon.core.bootstrap import bootstrap_pyon bootstrap_pyon() paste.script.command.run()
def main(): parser = argparse.ArgumentParser(description="ScionCC Control") parser.add_argument( "pidfile", help="pidfile to use. If not specified, uses the first one found.") parser.add_argument("command", help="command to send to the container agent", choices=IContainerAgent.names()) parser.add_argument("commandargs", metavar="arg", nargs="*", help="arguments to the command being sent") opts = parser.parse_args() pidfile = opts.pidfile if not pidfile: raise Exception("No pidfile specified") parms = {} with open(pidfile, 'r') as pf: parms = msgpack.loads(pf.read()) assert parms, "No content in pidfile" bootstrap_pyon() node, ioloop = make_node(parms['messaging']) node.setup_interceptors(CFG.interceptor) cc = ContainerAgentClient(node=node, to_name=(parms['container-xp'], parms['container-agent'])) # make a manual call - this is to avoid having to have the IonObject for the call methdefs = [ x[1] for x in IContainerAgent.namesAndDescriptions() if x[0] == opts.command ] assert len(methdefs) == 1 arg_names = methdefs[0].positional # ('name', 'module', 'cls', 'config') msg_args = dict( zip(arg_names, opts.commandargs) ) # ('name', <usrinp1>, 'cls', <usrinp2>) -> { 'name' : <usrinp1>, 'cls': <usrinp2> } retval = cc.request(msg_args, op=opts.command) # special case: status if opts.command == "status": statstr = retval print "Status:", statstr if statstr != "RUNNING": node.client.close() sys.exit(2) else: print "Returned", retval node.client.close()
def __init__(self, *args, **kwargs): BaseContainerAgent.__init__(self, *args, **kwargs) self._is_started = False # set id and name (as they are set in base class call) self.id = string.replace('%s_%d' % (os.uname()[1], os.getpid()), ".", "_") self.name = "cc_agent_%s" % self.id Container.instance = self # TODO: Bug: Replacing CFG instance not work because references are already public. Update directly dict_merge(CFG, kwargs, inplace=True) from pyon.core import bootstrap bootstrap.container_instance = self bootstrap.assert_configuration(CFG) log.debug("Container (sysname=%s) initializing ..." % bootstrap.get_sys_name()) # Keep track of the overrides from the command-line, so they can trump app/rel file data self.spawn_args = kwargs # Load object and service registry etc. bootstrap_pyon() # Create this Container's specific ExchangeManager instance self.ex_manager = ExchangeManager(self) # Create this Container's specific ProcManager instance self.proc_manager = ProcManager(self) # Create this Container's specific AppManager instance self.app_manager = AppManager(self) # DatastoreManager - controls access to Datastores (both mock and couch backed) self.datastore_manager = DatastoreManager() # File System - Interface to the OS File System, using correct path names and setups self.file_system = FileSystem(CFG) # Governance Controller - manages the governance related interceptors self.governance_controller = GovernanceController(self) # sFlow manager - controls sFlow stat emission self.sflow_manager = SFlowManager(self) # Coordinates the container start self._is_started = False self._capabilities = [] self._status = "INIT" # protection for when the container itself is used as a Process for clients self.container = self log.debug("Container initialized, OK.")
def begin(self): from interface.services.cei.iprocess_dispatcher_service import ProcessDispatcherServiceClient from pyon.net.messaging import make_node from pyon.core import bootstrap from pyon.public import CFG self.base_pids = [] self.rpc_timeout = 2 self._procs_by_test = {} if not bootstrap.pyon_initialized: bootstrap.bootstrap_pyon() self.node, self.ioloop = make_node() self.node.setup_interceptors(CFG.interceptor) self.pd_cli = ProcessDispatcherServiceClient(node=self.node)
def __init__(self): bootstrap_pyon() dsm = DatastoreManager() self.datastore = dsm.get_datastore(ds_name='coverage') if self.datastore is None: raise RuntimeError("Unable to load datastore for coverage") else: self.entity_table_name = self.datastore._get_datastore_name() log.trace("Got datastore: %s type %s" % (self.datastore._get_datastore_name(), str(type(self.datastore)))) self.span_store = dsm.get_datastore(ds_name='coverage_spans') if self.span_store is None: raise RuntimeError("Unable to load datastore for coverage_spans") else: self.span_table_name = self.span_store._get_datastore_name() log.trace("Got datastore: %s type %s", self.span_store._get_datastore_name(), type(self.span_store))
def main(): parser = argparse.ArgumentParser(description="ScionCC Control") parser.add_argument("pidfile", help="pidfile to use. If not specified, uses the first one found.") parser.add_argument("command", help="command to send to the container agent", choices=IContainerAgent.names()) parser.add_argument("commandargs", metavar="arg", nargs="*", help="arguments to the command being sent") opts = parser.parse_args() pidfile = opts.pidfile if not pidfile: raise Exception("No pidfile specified") parms = {} with open(pidfile, 'r') as pf: parms = msgpack.loads(pf.read()) assert parms, "No content in pidfile" bootstrap_pyon() node, ioloop = make_node(parms['messaging']) node.setup_interceptors(CFG.interceptor) cc = ContainerAgentClient(node=node, to_name=(parms['container-xp'], parms['container-agent'])) # make a manual call - this is to avoid having to have the IonObject for the call methdefs = [x[1] for x in IContainerAgent.namesAndDescriptions() if x[0] == opts.command] assert len(methdefs) == 1 arg_names = methdefs[0].positional # ('name', 'module', 'cls', 'config') msg_args = dict(zip(arg_names, opts.commandargs)) # ('name', <usrinp1>, 'cls', <usrinp2>) -> { 'name' : <usrinp1>, 'cls': <usrinp2> } retval = cc.request(msg_args, op=opts.command) # special case: status if opts.command == "status": statstr = retval print "Status:", statstr if statstr != "RUNNING": node.client.close() sys.exit(2) else: print "Returned", retval node.client.close()
def arg_initialize(self): if bootstrap.pyon_initialized: raise BadRequest("Pyon already initalized") parser = argparse.ArgumentParser() parser.add_argument("-s", "--sysname", dest="sysname", help="System name") options, extra = parser.parse_known_args() args, command_line_config = parse_args(extra) # ------------------------------------------------------------------------- # Store config and interfaces # Set global testing flag to False. We are running as standalone script. This is NO TEST. bootstrap.testing = False # Set sysname if provided in startup argument if options.sysname: bootstrap.set_sys_name(options.sysname) # bootstrap_config - Used for running this store_interfaces script bootstrap_config = config.read_local_configuration( ['res/config/pyon_min_boot.yml']) config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) config.apply_configuration(bootstrap_config, command_line_config) # Override sysname from config file or command line if not options.sysname and bootstrap_config.get_safe( "system.name", None): new_sysname = bootstrap_config.get_safe("system.name") bootstrap.set_sys_name(new_sysname) # ion_config - Holds the new CFG object for the system (independent of this tool's config) ion_config = config.read_standard_configuration() config.apply_configuration(ion_config, command_line_config) # Bootstrap pyon's core. Load configuration etc. bootstrap.bootstrap_pyon(pyon_cfg=ion_config)
def begin(self): from interface.services.cei.iprocess_dispatcher_service import ProcessDispatcherServiceClient from pyon.net.messaging import make_node from pyon.core import bootstrap from pyon.public import CFG self.rpc_timeout = 2 self.base_pids = [] self._procs_by_test = {} if not bootstrap.pyon_initialized: bootstrap.bootstrap_pyon() self.node, self.ioloop = make_node() self.node.setup_interceptors(CFG.interceptor) self.pd_cli = ProcessDispatcherServiceClient(node=self.node) # Set base_pids once from pyon.core.exception import Timeout try: self.base_pids = [ proc.process_id for proc in self.pd_cli.list_processes(timeout=20) ] except Timeout: pass
def begin(self): self._active_queues = set() self._test_changes = {} self._queues_declared = [] # ordered list of queues declared self._queues = defaultdict(list) # queue name -> list of accesses # Make sure we initialize pyon before anything in this plugin executes from pyon.core import bootstrap if not bootstrap.pyon_initialized: bootstrap.bootstrap_pyon() from pyon.ion.exchange import ExchangeManager from pyon.util.containers import DotDict from pyon.core.bootstrap import CFG from mock import Mock containermock = Mock() containermock.resource_registry.find_resources.return_value = ([], None) self.ex_manager = ExchangeManager(containermock) # needs to be able to setattr self.ex_manager._nodes['priviledged'] = DotDict(client=DotDict(parameters=DotDict(host=CFG.get_safe('server.amqp.host', 'localhost'))))
def report(self, stream): """ all tests have completed but --with-pycc has not yet stopped external container. request that containers log statistics now """ # initialize pyon so we can get system name from pyon.core import bootstrap if not bootstrap.pyon_initialized: bootstrap.bootstrap_pyon() from pyon.public import get_sys_name, CFG # make request: bin/pycc --sysname mgmt -x ion.processes.test.manage_system.ReportStats null = open('/dev/null', 'w') cmd = ['bin/pycc', '--sysname', get_sys_name(), '-x', 'ion.processes.test.manage_system.ReportStats' ] status = subprocess.call(cmd, stdout=null, stderr=null) if status==0: stream.write('container statistics: a report request has been sent\n') time.sleep(5) # give time to handle before container shutdown begins else: stream.write('container statistics: failed to send report request (logging anyway -- who needs a container?)\n') from ooi.timer import get_accumulators for a in get_accumulators().values(): a.log()
def arg_initialize(self): if bootstrap.pyon_initialized: raise BadRequest("Pyon already initalized") parser = argparse.ArgumentParser() parser.add_argument("-s", "--sysname", dest="sysname", help="System name") options, extra = parser.parse_known_args() args, command_line_config = parse_args(extra) # ------------------------------------------------------------------------- # Store config and interfaces # Set global testing flag to False. We are running as standalone script. This is NO TEST. bootstrap.testing = False # Set sysname if provided in startup argument if options.sysname: bootstrap.set_sys_name(options.sysname) # bootstrap_config - Used for running this store_interfaces script bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml']) config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) config.apply_configuration(bootstrap_config, command_line_config) # Override sysname from config file or command line if not options.sysname and bootstrap_config.get_safe("system.name", None): new_sysname = bootstrap_config.get_safe("system.name") bootstrap.set_sys_name(new_sysname) # ion_config - Holds the new CFG object for the system (independent of this tool's config) ion_config = config.read_standard_configuration() config.apply_configuration(ion_config, command_line_config) # Bootstrap pyon's core. Load configuration etc. bootstrap.bootstrap_pyon(pyon_cfg=ion_config)
def prepare_container(): """ Walks through pyon initialization in a deterministic way and initializes Container. In particular make sure configuration is loaded in correct order and pycc startup arguments are considered. """ import threading threading.current_thread().name = "CC-Main" # SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults import pyon from pyon.core import bootstrap, config # Set global testing flag to False. We are running as capability container. This is NO TEST. bootstrap.testing = False # Set sysname if provided in startup argument if opts.sysname: bootstrap.set_sys_name(opts.sysname) # Trigger any initializing default logic in get_sys_name bootstrap.get_sys_name() command_line_config = kwargs # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores. bootstrap_config = None # This holds the new CFG object for pyon. Build it up in proper sequence and conditions. pyon_config = config.read_standard_configuration() # Load config override if provided. Supports variants literal and list of paths config_override = None if opts.config: if '{' in opts.config: # Variant 1: Dict of config values try: eval_value = ast.literal_eval(opts.config) config_override = eval_value except ValueError: raise Exception("Value error in config arg '%s'" % opts.config) else: # Variant 2: List of paths from pyon.util.config import Config config_override = Config([opts.config]).data # Determine bootstrap_config if opts.config_from_directory: # Load minimal bootstrap config if option "config_from_directory" bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml']) config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config else: # Otherwise: Set to standard set of local config files plus command line overrides bootstrap_config = pyon_config.copy() config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) # Override sysname from config file or command line if not opts.sysname and bootstrap_config.get_safe("system.name", None): new_sysname = bootstrap_config.get_safe("system.name") bootstrap.set_sys_name(new_sysname) # Delete sysname datastores if option "force_clean" is set if opts.force_clean: from pyon.datastore import clear_couch_util print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name() clear_couch_util.clear_couch(bootstrap_config, prefix=bootstrap.get_sys_name()) # If auto_bootstrap, load config and interfaces into directory # Note: this is idempotent and will not alter anything if this is not the first container to run if bootstrap_config.system.auto_bootstrap: print "pycc: auto_bootstrap=True." stored_config = pyon_config.copy() config.apply_configuration(stored_config, config_override) config.apply_configuration(stored_config, command_line_config) config.auto_bootstrap_config(bootstrap_config, system_cfg=stored_config) # Determine the final pyon_config # - Start from standard config already set (pyon.yml + local YML files) # - Optionally load config from directory if opts.config_from_directory: config.apply_remote_config(pyon_config) # - Last apply any separate command line config overrides config.apply_configuration(pyon_config, config_override) config.apply_configuration(pyon_config, command_line_config) # Load logging override config if provided. Supports variants literal and path. logging_config_override = None if opts.logcfg: if '{' in opts.logcfg: # Variant 1: Value is dict of config values try: eval_value = ast.literal_eval(opts.logcfg) logging_config_override = eval_value except ValueError: raise Exception("Value error in logcfg arg '%s'" % opts.logcfg) else: # Variant 2: Value is path to YAML file containing config values pyon.DEFAULT_LOGGING_PATHS.append(opts.logcfg) # Also set the immediate flag, but only if specified - it is an override if opts.immediate: dict_merge(pyon_config, {'system':{'immediate':True}}, True) # Bootstrap pyon's core. Load configuration etc. bootstrap.bootstrap_pyon( logging_config_override=logging_config_override, pyon_cfg=pyon_config) # Auto-bootstrap interfaces # @WARN: This currently imports ALL modules, executing ALL static initializers as side effect!!!!!!! if bootstrap_config.system.auto_bootstrap: config.auto_bootstrap_interfaces(bootstrap_config) if opts.no_container: print "pycc: no_container=True. Stopping here." return None # Create the container instance from pyon.container.cc import Container container = Container(*args, **command_line_config) return container
def generate_validation_report(self): # WARNING!!!! # At this point, all the code (py files) should be generated. Now we can bootstrap pyon # which reads these py files. # THEN we can load all the modules, which depend on pyon from pyon.core import bootstrap bootstrap.bootstrap_pyon() validation_results = "Report generated on " + self.currtime + "\n" self.load_mods("interface/services", True) base_subtypes = self.find_subtypes(BaseService) self.load_mods("ion", False) self.load_mods("examples", False) for base_subtype in base_subtypes: base_subtype_name = base_subtype.__module__ + "." + base_subtype.__name__ compare_methods = {} for method_tuple in inspect.getmembers(base_subtype, inspect.ismethod): method_name = method_tuple[0] method = method_tuple[1] # Ignore private methods if method_name.startswith("_"): continue # Ignore methods not implemented in the class if method_name not in base_subtype.__dict__: continue compare_methods[method_name] = method # Find implementing subtypes of each base interface impl_subtypes = self.find_subtypes(base_subtype) if len(impl_subtypes) == 0: validation_results += "\nBase service: %s \n" % base_subtype_name validation_results += " No impl subtypes found\n" for impl_subtype in self.find_subtypes(base_subtype): impl_subtype_name = impl_subtype.__module__ + "." + impl_subtype.__name__ # Compare parameters added_class_names = False found_error = False for key in compare_methods: if key not in impl_subtype.__dict__: found_error = True if not added_class_names: added_class_names = True validation_results += "\nBase service: %s\n" % base_subtype_name validation_results += "Impl subtype: %s\n" % impl_subtype_name validation_results += " Method '%s' not implemented\n" % key else: base_params = inspect.getargspec(compare_methods[key]) impl_params = inspect.getargspec( impl_subtype.__dict__[key]) if base_params != impl_params: found_error = True if not added_class_names: added_class_names = True validation_results += "\nBase service: %s\n" % base_subtype_name validation_results += "Impl subtype: %s\n" % impl_subtype_name validation_results += " Method '%s' implementation is out of sync\n" % key validation_results += " Base: %s\n" % str( base_params) validation_results += " Impl: %s\n" % str( impl_params) if found_error is False: validation_results += "\nBase service: %s\n" % base_subtype_name validation_results += "Impl subtype: %s\n" % impl_subtype_name validation_results += " OK\n" reportfile = os.path.join('interface', 'validation_report.txt') try: os.unlink(reportfile) except: pass print " Writing service implementation validation report to '" + reportfile + "'" with open(reportfile, 'w') as f: f.write(validation_results)
def initialize_ion_int_tests(): # Bootstrap pyon CFG, logging and object/resource interfaces bootstrap_pyon() if bootstrap.is_testing(): IonIntegrationTestCase._force_clean(False) pre_initialize_ion()
def begin(self): """Called before any tests are collected or run. Use this to perform any setup needed before testing begins. """ # Make sure we initialize pyon before anything in this plugin executes from pyon.core import bootstrap if not bootstrap.pyon_initialized: bootstrap.bootstrap_pyon() try: from pyon.public import get_sys_name, CFG self.sysname = get_sys_name() # Clean exchanges and system queues out there try: rmh = RabbitManagementHelper(make_parser(), '-H %s -P 55672 -u %s -p %s -V %s' % (CFG.server.amqp.host, CFG.server.amqp.username, CFG.server.amqp.password, CFG.server.amqp.vhost)) exchanges = rmh.list_names('exchanges') deleted = rmh.delete_names_with_prefix('exchange', exchanges, self.sysname) debug.write('Deleted exchanges:\n%s \n' % '\n'.join(deleted)) queues = rmh.list_names('queues') deleted = rmh.delete_names_with_prefix('queue', queues, self.sysname) debug.write('Deleted queues:\n%s \n' % '\n'.join(deleted)) except Exception as e: pass # Force datastore loader to use the same sysname from pyon.datastore.datastore_admin import DatastoreAdmin self.datastore_admin = DatastoreAdmin(config=CFG) self.datastore_admin.clear_datastore(prefix=self.sysname) def die(signum, frame): # For whatever reason, the parent doesn't die some times # when getting KeyboardInterrupt. Hence this signal # handler. # Signal is pass through. The child pycc gets # its own KeyboardInterrupt and will shut down accordingly. debug.write('Received Keyboard Interrupt. Exiting now.\n') os._exit(9) signal.signal(signal.SIGINT, die) def no_zombie(signum, frame): # Debug to figure out who's dying debug.write('SIGCHLD received\n') stack = [] while frame: stack.append(frame) frame =frame.f_back stack.reverse() for frame in stack: debug.write('Frame %s in %s at line %s\n' % (frame.f_code.co_name, frame.f_code.co_filename, frame.f_lineno)) debug.write('Child is dead...Clean up now so there is no zombie\n') (pid, status) = os.wait() exitstatus, signum = status & 0xff, (status & 0xff00) >> 8 debug.write('Child pid %d with exit status %d and signum %d\n' % (pid, exitstatus, signum)) # Could be dangerous. Comment this out. # signal.signal(signal.SIGCHLD, no_zombie) def container_started_cb(signum, frame): """Callback when child pycc service is ready""" self.container_started = True signal.signal(signal.SIGUSR1, container_started_cb) # Make sure the pycc process has the same sysname as the nose ccargs = ['bin/pycc', '--noshell', '-sp', '--sysname=%s' % self.sysname, '--logcfg=res/config/logging.pycc.yml', '--rel=%s' % self.rel, "--config={'system': {'auto_bootstrap': True}}"] debug.write('Starting cc process: %s\n' % ' '.join(ccargs)) newenv = os.environ.copy() po = subprocess.Popen(ccargs, env=newenv, close_fds=True) self.ccs.append(po) # Wait for container to be ready while not self.container_started: time.sleep(0.2) debug.write('Child container is ready...\n') # Dump datastore self.datastore_admin.dump_datastore(path='res/dd', compact=True) debug.write('Dump child container state to file...\n') # Clean again to make sure the first nosetest starts on a clean # slate self.datastore_admin.clear_datastore(prefix=self.sysname) # Enable CEI mode for the tests os.environ['CEI_LAUNCH_TEST'] = '1' debug.write('Start nose tests now...\n') except Exception as e: self.container_shutdown() raise e
def initialize_ion_int_tests(): # Bootstrap pyon CFG, logging and object/resource interfaces bootstrap_pyon() if bootstrap.is_testing(): IonIntegrationTestCase._force_clean(False, initial=True) pre_initialize_ion()
def prepare_container(): """ Walks through pyon initialization in a deterministic way and initializes Container. In particular make sure configuration is loaded in correct order and pycc startup arguments are considered. """ import threading threading.current_thread().name = "CC-Main" # SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults import pyon from pyon.core import bootstrap, config # Set global testing flag to False. We are running as capability container, because # we started through the pycc program. bootstrap.testing = False # Set sysname if provided in startup argument if opts.sysname: bootstrap.set_sys_name(opts.sysname) # Trigger any initializing default logic in get_sys_name bootstrap.get_sys_name() command_line_config = kwargs # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores. bootstrap_config = None # This holds the new CFG object for pyon. Build it up in proper sequence and conditions. pyon_config = config.read_standard_configuration() # Initial pyon.yml + pyon.local.yml # Load config override if provided. Supports variants literal and list of paths config_override = None if opts.config: if "{" in opts.config: # Variant 1: Dict of config values try: eval_value = ast.literal_eval(opts.config) config_override = eval_value except ValueError: raise Exception("Value error in config arg '%s'" % opts.config) else: # Variant 2: List of paths from pyon.util.config import Config config_override = Config([opts.config]).data # Determine bootstrap_config if opts.config_from_directory: # Load minimal bootstrap config if option "config_from_directory" bootstrap_config = config.read_local_configuration(["res/config/pyon_min_boot.yml"]) config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config else: # Otherwise: Set to standard set of local config files plus command line overrides bootstrap_config = deepcopy(pyon_config) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) # Override sysname from config file or command line if not opts.sysname and bootstrap_config.get_safe("system.name", None): new_sysname = bootstrap_config.get_safe("system.name") bootstrap.set_sys_name(new_sysname) # Delete sysname datastores if option "force_clean" is set if opts.force_clean: from pyon.datastore import clear_couch_util print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name() clear_couch_util.clear_couch( bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name() ) pyon_config.container.filesystem.force_clean = True from pyon.core.interfaces.interfaces import InterfaceAdmin iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config) # If auto_bootstrap, load config and interfaces into directory # Note: this is idempotent and will not alter anything if this is not the first container to run if bootstrap_config.system.auto_bootstrap: print "pycc: auto_bootstrap=True." stored_config = deepcopy(pyon_config) config.apply_configuration(stored_config, config_override) config.apply_configuration(stored_config, command_line_config) iadm.create_core_datastores() iadm.store_config(stored_config) # Determine the final pyon_config # - Start from standard config already set (pyon.yml + local YML files) # - Optionally load config from directory if opts.config_from_directory: config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config) # - Apply container profile specific config config.apply_profile_configuration(pyon_config, bootstrap_config) # - Reapply pyon.local.yml here again for good measure config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) # - Last apply any separate command line config overrides config.apply_configuration(pyon_config, config_override) config.apply_configuration(pyon_config, command_line_config) # Also set the immediate flag, but only if specified - it is an override if opts.immediate: from pyon.util.containers import dict_merge dict_merge(pyon_config, {"system": {"immediate": True}}, True) # Bootstrap pyon's core. Load configuration etc. bootstrap.bootstrap_pyon(pyon_cfg=pyon_config) # Delete any queues/exchanges owned by sysname if option "broker_clean" is set if opts.broker_clean: print "pycc: broker_clean=True, sysname:", bootstrap.get_sys_name() # build connect str connect_str = "-q -H %s -P %s -u %s -p %s -V %s" % ( pyon_config.get_safe("server.amqp_priv.host", pyon_config.get_safe("server.amqp.host", "localhost")), pyon_config.get_safe("container.exchange.management.port", "55672"), pyon_config.get_safe("container.exchange.management.username", "guest"), pyon_config.get_safe("container.exchange.management.password", "guest"), "/", ) from putil.rabbithelper import clean_by_sysname deleted_exchanges, deleted_queues = clean_by_sysname(connect_str, bootstrap.get_sys_name()) print " exchanges deleted (%s): %s" % (len(deleted_exchanges), ",".join(deleted_exchanges)) print " queues deleted (%s): %s" % (len(deleted_queues), ",".join(deleted_queues)) if opts.force_clean: path = os.path.join(pyon_config.get_safe("container.filesystem.root", "/tmp/ion"), bootstrap.get_sys_name()) print "force_clean: Removing", path FileSystem._clean(pyon_config) # Auto-bootstrap interfaces if bootstrap_config.system.auto_bootstrap: iadm.store_interfaces(idempotent=True) iadm.close() if opts.no_container: print "pycc: no_container=True. Stopping here." return None # Create the container instance from pyon.container.cc import Container container = Container(*args, **command_line_config) return container
def generate_validation_report (self): # WARNING!!!! # At this point, all the code (py files) should be generated. Now we can bootstrap pyon # which reads these py files. # THEN we can load all the modules, which depend on pyon from pyon.core import bootstrap bootstrap.bootstrap_pyon() validation_results = "Report generated on " + self.currtime + "\n" self.load_mods("interface/services", True) base_subtypes = self.find_subtypes(BaseService) self.load_mods("ion", False) self.load_mods("examples", False) for base_subtype in base_subtypes: base_subtype_name = base_subtype.__module__ + "." + base_subtype.__name__ compare_methods = {} for method_tuple in inspect.getmembers(base_subtype, inspect.ismethod): method_name = method_tuple[0] method = method_tuple[1] # Ignore private methods if method_name.startswith("_"): continue # Ignore methods not implemented in the class if method_name not in base_subtype.__dict__: continue compare_methods[method_name] = method # Find implementing subtypes of each base interface impl_subtypes = self.find_subtypes(base_subtype) if len(impl_subtypes) == 0: validation_results += "\nBase service: %s \n" % base_subtype_name validation_results += " No impl subtypes found\n" for impl_subtype in self.find_subtypes(base_subtype): impl_subtype_name = impl_subtype.__module__ + "." + impl_subtype.__name__ # Compare parameters added_class_names = False found_error = False for key in compare_methods: if key not in impl_subtype.__dict__: found_error = True if not added_class_names: added_class_names = True validation_results += "\nBase service: %s\n" % base_subtype_name validation_results += "Impl subtype: %s\n" % impl_subtype_name validation_results += " Method '%s' not implemented\n" % key else: base_params = inspect.getargspec(compare_methods[key]) impl_params = inspect.getargspec(impl_subtype.__dict__[key]) if base_params != impl_params: found_error = True if not added_class_names: added_class_names = True validation_results += "\nBase service: %s\n" % base_subtype_name validation_results += "Impl subtype: %s\n" % impl_subtype_name validation_results += " Method '%s' implementation is out of sync\n" % key validation_results += " Base: %s\n" % str(base_params) validation_results += " Impl: %s\n" % str(impl_params) if found_error == False: validation_results += "\nBase service: %s\n" % base_subtype_name validation_results += "Impl subtype: %s\n" % impl_subtype_name validation_results += " OK\n" reportfile = os.path.join('interface', 'validation_report.txt') try: os.unlink(reportfile) except: pass print "Writing validation report to '" + reportfile + "'" with open(reportfile, 'w') as f: f.write(validation_results)
def prepare_container(): """ Walks through pyon initialization in a deterministic way and initializes Container. In particular make sure configuration is loaded in correct order and pycc startup arguments are considered. """ import threading threading.current_thread().name = "CC-Main" # SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults import pyon from pyon.core import bootstrap, config # Set global testing flag to False. We are running as capability container. This is NO TEST. bootstrap.testing = False # Set sysname if provided in startup argument if opts.sysname: bootstrap.set_sys_name(opts.sysname) # Trigger any initializing default logic in get_sys_name bootstrap.get_sys_name() command_line_config = kwargs # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores. bootstrap_config = None # This holds the new CFG object for pyon. Build it up in proper sequence and conditions. pyon_config = config.read_standard_configuration() # Load config override if provided. Supports variants literal and list of paths config_override = None if opts.config: if '{' in opts.config: # Variant 1: Dict of config values try: eval_value = ast.literal_eval(opts.config) config_override = eval_value except ValueError: raise Exception("Value error in config arg '%s'" % opts.config) else: # Variant 2: List of paths from pyon.util.config import Config config_override = Config([opts.config]).data # Determine bootstrap_config if opts.config_from_directory: # Load minimal bootstrap config if option "config_from_directory" bootstrap_config = config.read_local_configuration( ['res/config/pyon_min_boot.yml']) config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config else: # Otherwise: Set to standard set of local config files plus command line overrides bootstrap_config = pyon_config.copy() config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) # Override sysname from config file or command line if not opts.sysname and bootstrap_config.get_safe("system.name", None): new_sysname = bootstrap_config.get_safe("system.name") bootstrap.set_sys_name(new_sysname) # Delete sysname datastores if option "force_clean" is set if opts.force_clean: from pyon.datastore import clear_couch_util print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name( ) clear_couch_util.clear_couch(bootstrap_config, prefix=bootstrap.get_sys_name()) # If auto_bootstrap, load config and interfaces into directory # Note: this is idempotent and will not alter anything if this is not the first container to run if bootstrap_config.system.auto_bootstrap: print "pycc: auto_bootstrap=True." stored_config = pyon_config.copy() config.apply_configuration(stored_config, config_override) config.apply_configuration(stored_config, command_line_config) config.auto_bootstrap_config(bootstrap_config, system_cfg=stored_config) # Determine the final pyon_config # - Start from standard config already set (pyon.yml + local YML files) # - Optionally load config from directory if opts.config_from_directory: config.apply_remote_config(pyon_config) # - Last apply any separate command line config overrides config.apply_configuration(pyon_config, config_override) config.apply_configuration(pyon_config, command_line_config) # Load logging override config if provided. Supports variants literal and path. logging_config_override = None if opts.logcfg: if '{' in opts.logcfg: # Variant 1: Value is dict of config values try: eval_value = ast.literal_eval(opts.logcfg) logging_config_override = eval_value except ValueError: raise Exception("Value error in logcfg arg '%s'" % opts.logcfg) else: # Variant 2: Value is path to YAML file containing config values pyon.DEFAULT_LOGGING_PATHS.append(opts.logcfg) # Also set the immediate flag, but only if specified - it is an override if opts.immediate: dict_merge(pyon_config, {'system': {'immediate': True}}, True) # Bootstrap pyon's core. Load configuration etc. bootstrap.bootstrap_pyon( logging_config_override=logging_config_override, pyon_cfg=pyon_config) # Auto-bootstrap interfaces # @WARN: This currently imports ALL modules, executing ALL static initializers as side effect!!!!!!! if bootstrap_config.system.auto_bootstrap: config.auto_bootstrap_interfaces(bootstrap_config) if opts.no_container: print "pycc: no_container=True. Stopping here." return None # Create the container instance from pyon.container.cc import Container container = Container(*args, **command_line_config) return container
def begin(self): """Called before any tests are collected or run. Use this to perform any setup needed before testing begins. """ # Make sure we initialize pyon before anything in this plugin executes from pyon.core import bootstrap if not bootstrap.pyon_initialized: bootstrap.bootstrap_pyon() try: from pyon.public import get_sys_name, CFG self.sysname = get_sys_name() # Clean exchanges and system queues out there try: connect_str = '-H %s -P 55672 -u %s -p %s -V %s' % (CFG.server.amqp.host, CFG.server.amqp.username, CFG.server.amqp.password, CFG.server.amqp.vhost) deleted_exchanges, deleted_queues = clean_by_sysname(connect_str, self.sysname) debug.write('Deleted exchanges:\n%s \n' % '\n'.join(deleted_exchanges)) debug.write('Deleted queues:\n%s \n' % '\n'.join(deleted_queues)) except Exception as e: pass # Force datastore loader to use the same sysname from pyon.datastore.datastore_admin import DatastoreAdmin self.datastore_admin = DatastoreAdmin(config=CFG) self.datastore_admin.clear_datastore(prefix=self.sysname) def die(signum, frame): # For whatever reason, the parent doesn't die some times # when getting KeyboardInterrupt. Hence this signal # handler. # Signal is pass through. The child pycc gets # its own KeyboardInterrupt and will shut down accordingly. debug.write('Received Keyboard Interrupt. Exiting now.\n') os._exit(9) signal.signal(signal.SIGINT, die) def no_zombie(signum, frame): # Debug to figure out who's dying debug.write('SIGCHLD received\n') stack = [] while frame: stack.append(frame) frame =frame.f_back stack.reverse() for frame in stack: debug.write('Frame %s in %s at line %s\n' % (frame.f_code.co_name, frame.f_code.co_filename, frame.f_lineno)) debug.write('Child is dead...Clean up now so there is no zombie\n') (pid, status) = os.wait() exitstatus, signum = status & 0xff, (status & 0xff00) >> 8 debug.write('Child pid %d with exit status %d and signum %d\n' % (pid, exitstatus, signum)) # Could be dangerous. Comment this out. # signal.signal(signal.SIGCHLD, no_zombie) def container_started_cb(signum, frame): """Callback when child pycc service is ready""" self.container_started = True signal.signal(signal.SIGUSR1, container_started_cb) # Make sure the pycc process has the same sysname as the nose ccargs = ['bin/pycc', '-o', '--noshell', '-sp', '--sysname=%s' % self.sysname, '--logcfg=res/config/logging.pycc.yml', '--rel=%s' % self.rel, "--config={'system': {'auto_bootstrap': True}}"] debug.write('Starting cc process: %s\n' % ' '.join(ccargs)) newenv = os.environ.copy() po = subprocess.Popen(ccargs, env=newenv, close_fds=True) self.ccs.append(po) # Wait for container to be ready while not self.container_started: time.sleep(0.2) debug.write('Child container is ready...\n') # Dump datastore self.datastore_admin.dump_datastore(path='res/dd', compact=True) debug.write('Dump child container state to file...\n') # Clean again to make sure the first nosetest starts on a clean # slate self.datastore_admin.clear_datastore(prefix=self.sysname) # Set PYCC env var in case CEI needs to skip tests in pycc mode os.environ['PYCC_MODE'] = '1' # Enable CEI mode for the tests os.environ['CEI_LAUNCH_TEST'] = '1' debug.write('Start nose tests now...\n') except Exception as e: self.container_shutdown() raise e
#! /usr/bin/env python """Unit test base class and utils""" from copy import deepcopy from mock import Mock, mocksignature, patch, DEFAULT import unittest from unittest import SkipTest from zope.interface import implementedBy from pyon.core.bootstrap import IonObject, bootstrap_pyon, get_service_registry, CFG from pyon.util.containers import dict_merge, DotDict from pyon.util.file_sys import FileSystem bootstrap_pyon() def func_names(cls): import types return [name for name, value in cls.__dict__.items() if isinstance(value, types.FunctionType)] def pop_last_call(mock): if not mock.call_count: raise AssertionError('Cannot pop last call: call_count is 0') mock.call_args_list.pop() try: mock.call_args = mock.call_args_list[-1] except IndexError: mock.call_args = None mock.called = False
def prepare_container(): """ Walks through pyon initialization in a deterministic way and initializes Container. In particular make sure configuration is loaded in correct order and pycc startup arguments are considered. """ # SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults import pyon import threading threading.current_thread().name = "CC-Main" import logging global log log = logging.getLogger('pycc') from pyon.core import bootstrap, config from pyon.util.containers import get_safe, dict_merge # Set global testing flag to False. We are running as capability container, because # we started through the pycc program. bootstrap.testing = False # Set sysname if provided in startup argument if opts.sysname: bootstrap.set_sys_name(opts.sysname) # Trigger any initializing default logic in get_sys_name bootstrap.get_sys_name() command_line_config = kwargs # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores. bootstrap_config = None # This holds the new CFG object for pyon. Build it up in proper sequence and conditions. pyon_config = config.read_standard_configuration() # Initial pyon.yml + pyon.local.yml # Load config override if provided. Supports variants literal and list of paths config_override = None if opts.config: if '{' in opts.config: # Variant 1: Dict of config values try: eval_value = ast.literal_eval(opts.config) config_override = eval_value except ValueError: raise Exception("Value error in config arg '%s'" % opts.config) else: # Variant 2: List of paths from pyon.util.config import Config config_override = Config([opts.config]).data # Determine bootstrap_config if opts.config_from_directory: # Load minimal bootstrap config if option "config_from_directory" bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml']) config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) log.info("config_from_directory=True. Minimal bootstrap configuration: %s", bootstrap_config) else: # Otherwise: Set to standard set of local config files plus command line overrides bootstrap_config = deepcopy(pyon_config) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) # Override sysname from config file or command line if not opts.sysname and bootstrap_config.get_safe("system.name", None): new_sysname = bootstrap_config.get_safe("system.name") bootstrap.set_sys_name(new_sysname) # Force_clean - deletes sysname datastores if opts.force_clean: from pyon.datastore import clear_db_util log.info("force_clean=True. DROP DATASTORES for sysname=%s", bootstrap.get_sys_name()) clear_db_util.clear_db(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name()) from pyon.core.interfaces.interfaces import InterfaceAdmin iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config) # If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config # WARNING: If multiple containers start concurrently, this may fail if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True: log.debug("auto_store_interfaces=True.") stored_config = deepcopy(pyon_config) config.apply_configuration(stored_config, config_override) config.apply_configuration(stored_config, command_line_config) iadm.create_core_datastores() iadm.store_config(stored_config) # Determine the final pyon_config: # - Start from standard config already set (pyon.yml + local YML files) # - Optionally load config from directory if opts.config_from_directory: config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config) # - Apply container profile specific config config.apply_profile_configuration(pyon_config, bootstrap_config) # - Reapply pyon.local.yml here again for good measure config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) # - Last apply any separate command line config overrides config.apply_configuration(pyon_config, config_override) config.apply_configuration(pyon_config, command_line_config) iadm.set_config(pyon_config) # Set the immediate flag when command line override specified if opts.immediate: dict_merge(pyon_config, {"system": {"immediate": True}}, inplace=True) # Determine system bootmode for bootstrapping actions (unless explicitly specified) if not pyon_config.get_safe("bootmode"): set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode") if set_bootmode == "auto": if iadm.system_data_exists(): dict_merge(pyon_config, {"bootmode": "restart"}, inplace=True) log.info("System bootmode auto-detection is ON. Determined bootmode=%s", pyon_config.get_safe("bootmode", "initial")) elif set_bootmode == "secondary": dict_merge(pyon_config, {"bootmode": "secondary"}, inplace=True) log.info("System bootmode override. Set to bootmode=%s", pyon_config.get_safe("bootmode", "")) log.info("System in bootmode=%s", pyon_config.get_safe("bootmode", "initial")) # Bootstrap the pyon framework's core. Load configuration etc. bootstrap.bootstrap_pyon(pyon_cfg=pyon_config) # Delete any queues/exchanges owned by sysname if option "broker_clean" is set if opts.broker_clean: log.info("broker_clean=True, sysname: %s", bootstrap.get_sys_name()) from putil.rabbitmq.rabbit_util import RabbitManagementUtil rabbit_util = RabbitManagementUtil(pyon_config, sysname=bootstrap.get_sys_name()) deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname() log.info("Exchanges deleted (%s): %s" % (len(deleted_exchanges), ", ".join(deleted_exchanges))) log.info("Queues deleted (%s): %s" % (len(deleted_queues), ", ".join(deleted_queues))) if opts.force_clean: from pyon.util.file_sys import FileSystem FileSystem._clean(pyon_config) # If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True: iadm.store_interfaces(idempotent=True) iadm.declare_core_exchange_resources() iadm.close() if opts.no_container: log.info("no_container=True. Stopping here.") return None # Create the container instance from pyon.container.cc import Container container = Container(*args, **command_line_config) container.version = version return container
def prepare_container(): """ Walks through pyon initialization in a deterministic way and initializes Container. In particular make sure configuration is loaded in correct order and pycc startup arguments are considered. """ import threading threading.current_thread().name = "CC-Main" # SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults import pyon from pyon.core import bootstrap, config # Set global testing flag to False. We are running as capability container. This is NO TEST. bootstrap.testing = False # Set sysname if provided in startup argument if opts.sysname: bootstrap.set_sys_name(opts.sysname) # Trigger any initializing default logic in get_sys_name bootstrap.get_sys_name() command_line_config = kwargs # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores. bootstrap_config = None # This holds the new CFG object for pyon. Build it up in proper sequence and conditions. pyon_config = config.read_standard_configuration( ) # Initial pyon.yml + pyon.local.yml # Load config override if provided. Supports variants literal and list of paths config_override = None if opts.config: if '{' in opts.config: # Variant 1: Dict of config values try: eval_value = ast.literal_eval(opts.config) config_override = eval_value except ValueError: raise Exception("Value error in config arg '%s'" % opts.config) else: # Variant 2: List of paths from pyon.util.config import Config config_override = Config([opts.config]).data # Determine bootstrap_config if opts.config_from_directory: # Load minimal bootstrap config if option "config_from_directory" bootstrap_config = config.read_local_configuration( ['res/config/pyon_min_boot.yml']) config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config else: # Otherwise: Set to standard set of local config files plus command line overrides bootstrap_config = deepcopy(pyon_config) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) # Override sysname from config file or command line if not opts.sysname and bootstrap_config.get_safe("system.name", None): new_sysname = bootstrap_config.get_safe("system.name") bootstrap.set_sys_name(new_sysname) # Delete sysname datastores if option "force_clean" is set if opts.force_clean: from pyon.datastore import clear_couch_util print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name( ) clear_couch_util.clear_couch(bootstrap_config, prefix=bootstrap.get_sys_name()) pyon_config.container.filesystem.force_clean = True from pyon.core.interfaces.interfaces import InterfaceAdmin iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config) # If auto_bootstrap, load config and interfaces into directory # Note: this is idempotent and will not alter anything if this is not the first container to run if bootstrap_config.system.auto_bootstrap: print "pycc: auto_bootstrap=True." stored_config = deepcopy(pyon_config) config.apply_configuration(stored_config, config_override) config.apply_configuration(stored_config, command_line_config) iadm.create_core_datastores() iadm.store_config(stored_config) # Determine the final pyon_config # - Start from standard config already set (pyon.yml + local YML files) # - Optionally load config from directory if opts.config_from_directory: config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config) config.apply_local_configuration( pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS ) # apply pyon.local.yml again over top # - Last apply any separate command line config overrides config.apply_configuration(pyon_config, config_override) config.apply_configuration(pyon_config, command_line_config) # Also set the immediate flag, but only if specified - it is an override if opts.immediate: from pyon.util.containers import dict_merge dict_merge(pyon_config, {'system': {'immediate': True}}, True) # Bootstrap pyon's core. Load configuration etc. bootstrap.bootstrap_pyon(pyon_cfg=pyon_config) # Delete any queues/exchanges owned by sysname if option "broker_clean" is set if opts.broker_clean: print "pycc: broker_clean=True, sysname:", bootstrap.get_sys_name() # build connect str connect_str = "-q -H %s -P 55672 -u %s -p %s -V %s" % ( pyon_config.get_safe( 'server.amqp_priv.host', pyon_config.get_safe('server.amqp.host', 'localhost')), pyon_config.get_safe('container.exchange.management.username', 'guest'), pyon_config.get_safe('container.exchange.management.password', 'guest'), '/') from putil.rabbithelper import clean_by_sysname deleted_exchanges, deleted_queues = clean_by_sysname( connect_str, bootstrap.get_sys_name()) print " exchanges deleted (%s): %s" % ( len(deleted_exchanges), ",".join(deleted_exchanges)) print " queues deleted (%s): %s" % ( len(deleted_queues), ",".join(deleted_queues)) # Auto-bootstrap interfaces if bootstrap_config.system.auto_bootstrap: iadm.store_interfaces(idempotent=True) iadm.close() if opts.no_container: print "pycc: no_container=True. Stopping here." return None # Create the container instance from pyon.container.cc import Container container = Container(*args, **command_line_config) return container
from pyon.datastore.datastore import DatastoreManager from pyon.event.event import EventRepository from pyon.ion.directory import Directory from pyon.ion.state import StateRepository from pyon.util.containers import DotDict, dict_merge, DictModifier from pyon.util.log import log from mock import patch from pyon.public import CFG from pyon.datastore.couchdb.couchdb_datastore import CouchDB_DataStore from pyon.datastore.mockdb.mockdb_datastore import MockDB_DataStore from contextlib import contextmanager import unittest import os # Make this call more deterministic in time. bootstrap_pyon() scanned_services = False class IonIntegrationTestCase(unittest.TestCase): """ Base test class to allow operations such as starting the container TODO: Integrate with IonUnitTestCase """ def run(self, result=None): unittest.TestCase.run(self, result) def _start_container(self): # hack to force queue auto delete on for int tests self._turn_on_queue_auto_delete()
def prepare_container(): """ Walks through pyon initialization in a deterministic way and initializes Container. In particular make sure configuration is loaded in correct order and pycc startup arguments are considered. """ # SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults import pyon import threading threading.current_thread().name = "CC-Main" import logging global log log = logging.getLogger('pycc') from pyon.core import bootstrap, config from pyon.util.containers import get_safe, dict_merge # Set global testing flag to False. We are running as capability container, because # we started through the pycc program. bootstrap.testing = False # Set sysname if provided in startup argument if opts.sysname: bootstrap.set_sys_name(opts.sysname) # Trigger any initializing default logic in get_sys_name bootstrap.get_sys_name() command_line_config = kwargs # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores. bootstrap_config = None # This holds the new CFG object for pyon. Build it up in proper sequence and conditions. pyon_config = config.read_standard_configuration( ) # Initial pyon.yml + pyon.local.yml # Load config override if provided. Supports variants literal and list of paths config_override = None if opts.config: if '{' in opts.config: # Variant 1: Dict of config values try: eval_value = ast.literal_eval(opts.config) config_override = eval_value except ValueError: raise Exception("Value error in config arg '%s'" % opts.config) else: # Variant 2: List of paths from pyon.util.config import Config config_override = Config([opts.config]).data # Determine bootstrap_config if opts.config_from_directory: # Load minimal bootstrap config if option "config_from_directory" bootstrap_config = config.read_local_configuration( ['res/config/pyon_min_boot.yml']) config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) log.info( "config_from_directory=True. Minimal bootstrap configuration: %s", bootstrap_config) else: # Otherwise: Set to standard set of local config files plus command line overrides bootstrap_config = deepcopy(pyon_config) config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) # Override sysname from config file or command line if not opts.sysname and bootstrap_config.get_safe("system.name", None): new_sysname = bootstrap_config.get_safe("system.name") bootstrap.set_sys_name(new_sysname) # Force_clean - deletes sysname datastores if opts.force_clean: from pyon.datastore import clear_db_util log.info("force_clean=True. DROP DATASTORES for sysname=%s", bootstrap.get_sys_name()) clear_db_util.clear_db(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name()) from pyon.core.interfaces.interfaces import InterfaceAdmin iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config) # If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config # WARNING: If multiple containers start concurrently, this may fail if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True: log.debug("auto_store_interfaces=True.") stored_config = deepcopy(pyon_config) config.apply_configuration(stored_config, config_override) config.apply_configuration(stored_config, command_line_config) iadm.create_core_datastores() iadm.store_config(stored_config) # Determine the final pyon_config: # - Start from standard config already set (pyon.yml + local YML files) # - Optionally load config from directory if opts.config_from_directory: config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config) # - Apply container profile specific config config.apply_profile_configuration(pyon_config, bootstrap_config) # - Reapply pyon.local.yml here again for good measure config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) # - Last apply any separate command line config overrides config.apply_configuration(pyon_config, config_override) config.apply_configuration(pyon_config, command_line_config) iadm.set_config(pyon_config) # Set the immediate flag when command line override specified if opts.immediate: dict_merge(pyon_config, {"system": { "immediate": True }}, inplace=True) # Determine system bootmode for bootstrapping actions (unless explicitly specified) if not pyon_config.get_safe("bootmode"): set_bootmode = get_safe(pyon_config, "bootstrap.set_bootmode") if set_bootmode == "auto": if iadm.system_data_exists(): dict_merge(pyon_config, {"bootmode": "restart"}, inplace=True) log.info( "System bootmode auto-detection is ON. Determined bootmode=%s", pyon_config.get_safe("bootmode", "initial")) elif set_bootmode == "secondary": dict_merge(pyon_config, {"bootmode": "secondary"}, inplace=True) log.info("System bootmode override. Set to bootmode=%s", pyon_config.get_safe("bootmode", "")) log.info("System in bootmode=%s", pyon_config.get_safe("bootmode", "initial")) # Bootstrap the pyon framework's core. Load configuration etc. bootstrap.bootstrap_pyon(pyon_cfg=pyon_config) # Delete any queues/exchanges owned by sysname if option "broker_clean" is set if opts.broker_clean: log.info("broker_clean=True, sysname: %s", bootstrap.get_sys_name()) from putil.rabbitmq.rabbit_util import RabbitManagementUtil rabbit_util = RabbitManagementUtil( pyon_config, sysname=bootstrap.get_sys_name()) deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname() log.info("Exchanges deleted (%s): %s" % (len(deleted_exchanges), ", ".join(deleted_exchanges))) log.info("Queues deleted (%s): %s" % (len(deleted_queues), ", ".join(deleted_queues))) if opts.force_clean: from pyon.util.file_sys import FileSystem FileSystem._clean(pyon_config) # If auto_store_interfaces (cont'd): Store interfaces if not yet existing; set up messaging if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True: iadm.store_interfaces(idempotent=True) iadm.declare_core_exchange_resources() iadm.close() if opts.no_container: log.info("no_container=True. Stopping here.") return None # Create the container instance from pyon.container.cc import Container container = Container(*args, **command_line_config) container.version = version return container