def load_logging_config(logging_config_override=None): """ Initialize pyon logging system """ from pyon.core import log log.configure_logging(pyon.DEFAULT_LOGGING_PATHS, logging_config_override=logging_config_override)
def main(): usage = \ """ %prog [options] prefix """ description = "Use this program to clear databases that match a given prefix" parser = OptionParser(usage=usage, description=description) parser.add_option("-P", "--port", dest="db_port", default=None, help="Port number for db", action="store", type=int, metavar="PORT") parser.add_option("-H", "--host", dest="db_host", default='localhost', help="The host name or ip address of the db server", action="store", type=str, metavar="HOST") parser.add_option("-u", "--username", dest="db_uname", default=None, help="Username for the db server", action="store", type=str, metavar="UNAME") parser.add_option("-p", "--password", dest="db_pword", default=None, help="Password for the db server", action="store", type=str, metavar="PWORD") parser.add_option("-s", "--sysname", dest="sysname", default=None, help="The sysname prefix to clear databases", action="store", type=str, metavar="SYSNAME") parser.add_option("-t", "--store_type", dest="db_type", default="postgresql", help="Datastore type", action="store", type=str, metavar="DSTYPE") parser.add_option("-v", "--verbose", help="More verbose output", action="store_true") parser.add_option("-d", "--dump", dest="dump_path", default=None, help="Dump sysname datastores to path", action="store", type=str, metavar="DPATH") parser.add_option("-l", "--load", dest="load_path", default=None, help="Load dumped datastore from path", action="store", type=str, metavar="LPATH") (options, args) = parser.parse_args() from pyon.core import log as logutil logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS) if options.dump_path: config = create_config(options.db_host, options.db_port, options.db_uname, options.db_pword) sysname = options.sysname or "scion" log.info("dumping %s datastores to %s", sysname, options.dump_path) from pyon.datastore.datastore_admin import DatastoreAdmin datastore_admin = DatastoreAdmin(config=config, sysname=sysname) datastore_admin.dump_datastore(path=options.dump_path) elif options.load_path: config = create_config(options.db_host, options.db_port, options.db_uname, options.db_pword) sysname = options.sysname or "scion" log.info("loading %s datastores from dumped content in %ss", sysname, options.dump_path) from pyon.datastore.datastore_admin import DatastoreAdmin datastore_admin = DatastoreAdmin(config=config, sysname=sysname) datastore_admin.load_datastore(path=options.load_path) else: if len(args) == 0: log.error("Error: no prefix argument specified") parser.print_help() sys.exit() if len(args) != 1: log.error("Error: You can not specify multiple prefixes. Received args: %s", str(args)) parser.print_help() sys.exit() prefix = args[0] if prefix == "": log.error("Error: You can not give the empty string as a prefix!") parser.print_help() sys.exit() config = create_config(options.db_host, options.db_port, options.db_uname, options.db_pword, options.db_type) _clear_db(config, prefix=prefix, sysname=options.sysname, verbose=bool(options.verbose))
def bootstrap_pyon(logging_config_override=None, pyon_cfg=None): """ This function initializes the core elements of the Pyon framework in a controlled way. It does not initialize the ION container or the ION system. @param logging_config_override A dict to initialize the Python logging subsystem (None loads default files) @param pyon_cfg A DotDict with the fully loaded pyon configuration to merge into CFG (None loads default files) """ log.info("pyon.bootstrap (bootstrap_pyon) executing...") # Make sure Pyon is only initialized only once global pyon_initialized if pyon_initialized: log.warn("WARNING -- bootstrap_pyon() called again!") return # ENVIRONMENT. Check we are called in an expected environment (files, directories, etc) assert_environment() # LOGGING. Initialize logging from config if not logutil.is_logging_configured(): logutil.configure_logging( logutil.DEFAULT_LOGGING_PATHS, logging_config_override=logging_config_override) # YAML patch: OrderedDicts instead of dicts from pyon.util.yaml_ordered_dict import apply_yaml_patch apply_yaml_patch() # CONFIG. Initialize pyon global configuration from local files set_config(pyon_cfg) log.debug("CFG set to %s", CFG) # OBJECTS. Object and message definitions. from pyon.core.registry import IonObjectRegistry global _obj_registry _obj_registry = IonObjectRegistry() # SERVICES. Service definitions # TODO: change the following to read service definitions from directory and import selectively from pyon.ion.service import IonServiceRegistry import interface.services global _service_registry _service_registry = IonServiceRegistry() _service_registry.load_service_mods(interface.services) _service_registry.build_service_map() # RESOURCES. Load and initialize definitions from pyon.ion import resource resource.load_definitions() # Fix a weird bug on Ubuntu that resets time.sleep to un-monkey patched version on import threading from gevent import monkey monkey.patch_time() # Set initialized flag pyon_initialized = True log.debug("Pyon initialized OK")
def bootstrap_pyon(logging_config_override=None, pyon_cfg=None): """ This function initializes the core elements of the Pyon framework in a controlled way. It does not initialize the ION container or the ION system. @param logging_config_override A dict to initialize the Python logging subsystem (None loads default files) @param pyon_cfg A DotDict with the fully loaded pyon configuration to merge into CFG (None loads default files) """ print "pyon: pyon.bootstrap (bootstrap_pyon) executing..." # Make sure Pyon is only initialized only once global pyon_initialized if pyon_initialized: print "pyon: WARNING -- bootstrap_pyon() called again!" return # ENVIRONMENT. Check we are called in an expected environment (files, directories, etc) assert_environment() # LOGGING. Initialize logging from config if not logutil.is_logging_configured(): logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS, logging_config_override=logging_config_override) # YAML patch: OrderedDicts instead of dicts from pyon.util.yaml_ordered_dict import apply_yaml_patch apply_yaml_patch() # CONFIG. Initialize pyon global configuration from local files set_config(pyon_cfg) log.debug("pyon: CFG set to %s", CFG) # OBJECTS. Object and message definitions. from pyon.core.registry import IonObjectRegistry global _obj_registry _obj_registry = IonObjectRegistry() # SERVICES. Service definitions # TODO: change the following to read service definitions from directory and import selectively from pyon.ion.service import IonServiceRegistry import interface.services global _service_registry _service_registry = IonServiceRegistry() _service_registry.load_service_mods(interface.services) _service_registry.build_service_map() # RESOURCES. Load and initialize definitions from pyon.ion import resource resource.load_definitions() # Fix a weird bug on Ubuntu that resets time.sleep to un-monkey patched version on import threading from gevent import monkey; monkey.patch_time() # Set initialized flag pyon_initialized = True log.debug("pyon: initialized OK")
def prepare_logging(): # Load logging override config if provided. Supports variants literal and path. logging_config_override = None if opts.logcfg: if "{" in opts.logcfg: # Variant 1: Value is dict of config values try: eval_value = ast.literal_eval(opts.logcfg) logging_config_override = eval_value except ValueError: raise Exception("Value error in logcfg arg '%s'" % opts.logcfg) else: # Variant 2: Value is path to YAML file containing config values logutil.DEFAULT_LOGGING_PATHS.append(opts.logcfg) logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS, logging_config_override=logging_config_override)
def prepare_logging(): # Load logging override config if provided. Supports variants literal and path. logging_config_override = None if opts.logcfg: if '{' in opts.logcfg: # Variant 1: Value is dict of config values try: eval_value = ast.literal_eval(opts.logcfg) logging_config_override = eval_value except ValueError: raise Exception("Value error in logcfg arg '%s'" % opts.logcfg) else: # Variant 2: Value is path to YAML file containing config values logutil.DEFAULT_LOGGING_PATHS.append(opts.logcfg) logutil.configure_logging( logutil.DEFAULT_LOGGING_PATHS, logging_config_override=logging_config_override)
def main(): usage = \ """ %prog [options] prefix """ description = "Use this program to clear databases that match a given prefix" parser = OptionParser(usage=usage, description=description) parser.add_option("-P", "--port", dest="db_port", default=None, help="Port number for db", action="store", type=int, metavar="PORT") parser.add_option("-H", "--host", dest="db_host", default='localhost', help="The host name or ip address of the db server", action="store", type=str, metavar="HOST") parser.add_option("-u", "--username", dest="db_uname", default=None, help="Username for the db server", action="store", type=str, metavar="UNAME") parser.add_option("-p", "--password", dest="db_pword", default=None, help="Password for the db server", action="store", type=str, metavar="PWORD") parser.add_option("-s", "--sysname", dest="sysname", default=None, help="The sysname prefix to clear databases", action="store", type=str, metavar="SYSNAME") parser.add_option("-t", "--store_type", dest="db_type", default="postgresql", help="Datastore type", action="store", type=str, metavar="DSTYPE") parser.add_option("-v", "--verbose", help="More verbose output", action="store_true") parser.add_option("-d", "--dump", dest="dump_path", default=None, help="Dump sysname datastores to path", action="store", type=str, metavar="DPATH") parser.add_option("-l", "--load", dest="load_path", default=None, help="Load dumped datastore from path", action="store", type=str, metavar="LPATH") (options, args) = parser.parse_args() from pyon.core import log as logutil logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS) if options.dump_path: config = create_config(options.db_host, options.db_port, options.db_uname, options.db_pword) sysname = options.sysname or "scion" log.info("dumping %s datastores to %s", sysname, options.dump_path) from pyon.datastore.datastore_admin import DatastoreAdmin datastore_admin = DatastoreAdmin(config=config, sysname=sysname) datastore_admin.dump_datastore(path=options.dump_path) elif options.load_path: config = create_config(options.db_host, options.db_port, options.db_uname, options.db_pword) sysname = options.sysname or "scion" log.info("loading %s datastores from dumped content in %ss", sysname, options.dump_path) from pyon.datastore.datastore_admin import DatastoreAdmin datastore_admin = DatastoreAdmin(config=config, sysname=sysname) datastore_admin.load_datastore(path=options.load_path) else: if len(args) == 0: log.error("Error: no prefix argument specified") parser.print_help() sys.exit() if len(args) != 1: log.error( "Error: You can not specify multiple prefixes. Received args: %s", str(args)) parser.print_help() sys.exit() prefix = args[0] if prefix == "": log.error("Error: You can not give the empty string as a prefix!") parser.print_help() sys.exit() config = create_config(options.db_host, options.db_port, options.db_uname, options.db_pword, options.db_type) _clear_db(config, prefix=prefix, sysname=options.sysname, verbose=bool(options.verbose))
def setUp(self): log_config.configure_logging( [ 'res/config/logging.yml', 'res/config/logging.local.yml'] )
def main(): """ Store configuration and interfaces into the datastore How to run this from command line: bin/store_interfaces -s system name [ -of filename | -sf filename | -fc true|false] -of Load object definition file -sf Load service definition file -fc Force clean the database Example: Load all object and service definitions bin/python bin/store_interfaces -s mysysname Load all object and service definitions with force clean the database bin/python bin/store_interfaces -s mysysname -fc Load object definition from a file bin/python bin/store_interfaces -s mysysname -of obj/data/core/org.yml Load service definition from a file bin/python bin/store_interfaces -s mysysname -sf obj/services/core/datastore_service.yml """ parser = argparse.ArgumentParser() parser.add_argument( '-c', '--config', type=str, help='Additional config files to load or dict config content.', default=[]) parser.add_argument('-fc', '--force_clean', action='store_true', help='Force clean.') parser.add_argument("-of", "--object", dest="fobject", help="Load object definition from a file") parser.add_argument("-s", "--sysname", dest="sysname", help="System name") parser.add_argument("-sf", "--service", dest="fservice", help="Load service definition from a file") options, extra = parser.parse_known_args() args, command_line_config = parse_args(extra) log.info("Storing SciON config and interfaces, with options: %s", str(options)) from pyon.core import log as logutil logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS) # ------------------------------------------------------------------------- # Store config and interfaces # Set global testing flag to False. We are running as standalone script. This is NO TEST. bootstrap.testing = False # Set sysname if provided in startup argument if options.sysname: bootstrap.set_sys_name(options.sysname) # Load config override if provided. Supports variants literal and list of paths config_override = None if options.config: if '{' in options.config: # Variant 1: Dict of config values try: eval_value = ast.literal_eval(options.config) config_override = eval_value except ValueError: raise Exception("Value error in config arg '%s'" % options.config) else: # Variant 2: List of paths from pyon.util.config import Config config_override = Config([options.config]).data # bootstrap_config - Used for running this store_interfaces script bootstrap_config = config.read_local_configuration( ['res/config/pyon_min_boot.yml']) config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) if config_override: config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) # Override sysname from config file or command line if not options.sysname and bootstrap_config.get_safe("system.name", None): new_sysname = bootstrap_config.get_safe("system.name") bootstrap.set_sys_name(new_sysname) # Delete sysname datastores if option "force_clean" is set if options.force_clean: from pyon.datastore import clear_db_util from pyon.util.file_sys import FileSystem log.info("force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name()) pyon_config = config.read_standard_configuration( ) # Initial pyon.yml + pyon.local.yml clear_db_util.clear_db(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name()) FileSystem._clean(pyon_config) # ion_config - Holds the new CFG object for the system (independent of this tool's config) ion_config = config.read_standard_configuration() if config_override: config.apply_configuration(ion_config, config_override) config.apply_configuration(ion_config, command_line_config) # ------------------------------------------------------------------------- # Store config and interfaces iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config) # Make sure core datastores exist iadm.create_core_datastores() # Store system CFG properties iadm.store_config(ion_config) # Store system interfaces iadm.store_interfaces(options.fobject, options.fservice) iadm.close()
def setUp(self): log_config.configure_logging( ['res/config/logging.yml', 'res/config/logging.local.yml'])
#!/usr/bin/env python """ @package coverage_model.base_test_classes @file coverage_model/base_test_classes.py @author Christopher Mueller @brief Base classes for Unit and Int testing within the coverage model """ from unittest import TestCase import os, shutil, tempfile from pyon.core import log as logutil if not logutil.is_logging_configured(): logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS) class CoverageModelUnitTestCase(TestCase): # Prevent test docstring from printing - uses test name instead # @see # http://www.saltycrane.com/blog/2012/07/how-prevent-nose-unittest-using-docstring-when-verbosity-2/ def shortDescription(self): return None # override __str__ and __repr__ behavior to show a copy-pastable nosetest name for ion tests # ion.module:TestClassName.test_function_name def __repr__(self): name = self.id() name = name.split('.') if name[0] not in ["coverage_model"]: return "%s (%s)" % (name[-1], '.'.join(name[:-1]))
#!/usr/bin/env python """ @package coverage_model.base_test_classes @file coverage_model/base_test_classes.py @author Christopher Mueller @brief Base classes for Unit and Int testing within the coverage model """ from unittest import TestCase import os, shutil, tempfile from pyon.core import log as logutil if not logutil.is_logging_configured(): logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS) class CoverageModelUnitTestCase(TestCase): # Prevent test docstring from printing - uses test name instead # @see # http://www.saltycrane.com/blog/2012/07/how-prevent-nose-unittest-using-docstring-when-verbosity-2/ def shortDescription(self): return None # override __str__ and __repr__ behavior to show a copy-pastable nosetest name for ion tests # ion.module:TestClassName.test_function_name def __repr__(self): name = self.id() name = name.split(".")
def main(): """ Store configuration and interfaces into the datastore How to run this from command line: bin/store_interfaces -s system name [ -of filename | -sf filename | -fc true|false] -of Load object definition file -sf Load service definition file -fc Force clean the database Example: Load all object and service definitions bin/python bin/store_interfaces -s mysysname Load all object and service definitions with force clean the database bin/python bin/store_interfaces -s mysysname -fc Load object definition from a file bin/python bin/store_interfaces -s mysysname -of obj/data/core/org.yml Load service definition from a file bin/python bin/store_interfaces -s mysysname -sf obj/services/core/datastore_service.yml """ parser = argparse.ArgumentParser() parser.add_argument('-c', '--config', type=str, help='Additional config files to load or dict config content.', default=[]) parser.add_argument('-fc', '--force_clean', action='store_true', help='Force clean.') parser.add_argument("-of", "--object", dest="fobject", help="Load object definition from a file") parser.add_argument("-s", "--sysname", dest="sysname", help="System name") parser.add_argument("-sf", "--service", dest="fservice", help="Load service definition from a file") options, extra = parser.parse_known_args() args, command_line_config = parse_args(extra) log.info("Storing SciON config and interfaces, with options: %s", str(options)) from pyon.core import log as logutil logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS) # ------------------------------------------------------------------------- # Store config and interfaces # Set global testing flag to False. We are running as standalone script. This is NO TEST. bootstrap.testing = False # Set sysname if provided in startup argument if options.sysname: bootstrap.set_sys_name(options.sysname) # Load config override if provided. Supports variants literal and list of paths config_override = None if options.config: if '{' in options.config: # Variant 1: Dict of config values try: eval_value = ast.literal_eval(options.config) config_override = eval_value except ValueError: raise Exception("Value error in config arg '%s'" % options.config) else: # Variant 2: List of paths from pyon.util.config import Config config_override = Config([options.config]).data # bootstrap_config - Used for running this store_interfaces script bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml']) config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS) if config_override: config.apply_configuration(bootstrap_config, config_override) config.apply_configuration(bootstrap_config, command_line_config) # Override sysname from config file or command line if not options.sysname and bootstrap_config.get_safe("system.name", None): new_sysname = bootstrap_config.get_safe("system.name") bootstrap.set_sys_name(new_sysname) # Delete sysname datastores if option "force_clean" is set if options.force_clean: from pyon.datastore import clear_db_util from pyon.util.file_sys import FileSystem log.info("force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name()) pyon_config = config.read_standard_configuration() # Initial pyon.yml + pyon.local.yml clear_db_util.clear_db(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name()) FileSystem._clean(pyon_config) # ion_config - Holds the new CFG object for the system (independent of this tool's config) ion_config = config.read_standard_configuration() if config_override: config.apply_configuration(ion_config, config_override) config.apply_configuration(ion_config, command_line_config) # ------------------------------------------------------------------------- # Store config and interfaces iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config) # Make sure core datastores exist iadm.create_core_datastores() # Store system CFG properties iadm.store_config(ion_config) # Store system interfaces iadm.store_interfaces(options.fobject, options.fservice) iadm.close()