コード例 #1
0
def load_logging_config(logging_config_override=None):
    """
    Initialize pyon logging system
    """
    from pyon.core import log
    log.configure_logging(pyon.DEFAULT_LOGGING_PATHS,
                          logging_config_override=logging_config_override)
コード例 #2
0
ファイル: clear_db_util.py プロジェクト: edwardhunter/scioncc
def main():

    usage = \
    """
    %prog [options] prefix
    """
    description = "Use this program to clear databases that match a given prefix"
    parser = OptionParser(usage=usage, description=description)
    parser.add_option("-P", "--port", dest="db_port", default=None, help="Port number for db", action="store", type=int, metavar="PORT")
    parser.add_option("-H", "--host", dest="db_host", default='localhost', help="The host name or ip address of the db server", action="store", type=str, metavar="HOST")
    parser.add_option("-u", "--username", dest="db_uname", default=None, help="Username for the db server", action="store", type=str, metavar="UNAME")
    parser.add_option("-p", "--password", dest="db_pword", default=None, help="Password for the db server", action="store", type=str, metavar="PWORD")
    parser.add_option("-s", "--sysname", dest="sysname", default=None, help="The sysname prefix to clear databases", action="store", type=str, metavar="SYSNAME")
    parser.add_option("-t", "--store_type", dest="db_type", default="postgresql", help="Datastore type", action="store", type=str, metavar="DSTYPE")
    parser.add_option("-v", "--verbose", help="More verbose output", action="store_true")
    parser.add_option("-d", "--dump", dest="dump_path", default=None, help="Dump sysname datastores to path", action="store", type=str, metavar="DPATH")
    parser.add_option("-l", "--load", dest="load_path", default=None, help="Load dumped datastore from path", action="store", type=str, metavar="LPATH")

    (options, args) = parser.parse_args()

    from pyon.core import log as logutil
    logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS)

    if options.dump_path:
        config = create_config(options.db_host, options.db_port, options.db_uname, options.db_pword)
        sysname = options.sysname or "scion"
        log.info("dumping %s datastores to %s", sysname, options.dump_path)
        from pyon.datastore.datastore_admin import DatastoreAdmin
        datastore_admin = DatastoreAdmin(config=config, sysname=sysname)
        datastore_admin.dump_datastore(path=options.dump_path)
    elif options.load_path:
        config = create_config(options.db_host, options.db_port, options.db_uname, options.db_pword)
        sysname = options.sysname or "scion"
        log.info("loading %s datastores from dumped content in %ss", sysname, options.dump_path)
        from pyon.datastore.datastore_admin import DatastoreAdmin
        datastore_admin = DatastoreAdmin(config=config, sysname=sysname)
        datastore_admin.load_datastore(path=options.load_path)
    else:
        if len(args) == 0:
            log.error("Error: no prefix argument specified")
            parser.print_help()
            sys.exit()

        if len(args) != 1:
            log.error("Error: You can not specify multiple prefixes. Received args: %s", str(args))
            parser.print_help()
            sys.exit()

        prefix = args[0]

        if prefix == "":
            log.error("Error: You can not give the empty string as a prefix!")
            parser.print_help()
            sys.exit()

        config = create_config(options.db_host, options.db_port, options.db_uname, options.db_pword, options.db_type)
        _clear_db(config, prefix=prefix, sysname=options.sysname, verbose=bool(options.verbose))
コード例 #3
0
def bootstrap_pyon(logging_config_override=None, pyon_cfg=None):
    """
    This function initializes the core elements of the Pyon framework in a controlled way.
    It does not initialize the ION container or the ION system.
    @param logging_config_override  A dict to initialize the Python logging subsystem (None loads default files)
    @param pyon_cfg   A DotDict with the fully loaded pyon configuration to merge into CFG (None loads default files)
    """
    log.info("pyon.bootstrap (bootstrap_pyon) executing...")

    # Make sure Pyon is only initialized only once
    global pyon_initialized
    if pyon_initialized:
        log.warn("WARNING -- bootstrap_pyon() called again!")
        return

    # ENVIRONMENT. Check we are called in an expected environment (files, directories, etc)
    assert_environment()

    # LOGGING. Initialize logging from config
    if not logutil.is_logging_configured():
        logutil.configure_logging(
            logutil.DEFAULT_LOGGING_PATHS,
            logging_config_override=logging_config_override)

    # YAML patch: OrderedDicts instead of dicts
    from pyon.util.yaml_ordered_dict import apply_yaml_patch
    apply_yaml_patch()

    # CONFIG. Initialize pyon global configuration from local files
    set_config(pyon_cfg)
    log.debug("CFG set to %s", CFG)

    # OBJECTS. Object and message definitions.
    from pyon.core.registry import IonObjectRegistry
    global _obj_registry
    _obj_registry = IonObjectRegistry()

    # SERVICES. Service definitions
    # TODO: change the following to read service definitions from directory and import selectively
    from pyon.ion.service import IonServiceRegistry
    import interface.services
    global _service_registry
    _service_registry = IonServiceRegistry()
    _service_registry.load_service_mods(interface.services)
    _service_registry.build_service_map()

    # RESOURCES. Load and initialize definitions
    from pyon.ion import resource
    resource.load_definitions()

    # Fix a weird bug on Ubuntu that resets time.sleep to un-monkey patched version on import threading
    from gevent import monkey
    monkey.patch_time()

    # Set initialized flag
    pyon_initialized = True
    log.debug("Pyon initialized OK")
コード例 #4
0
ファイル: bootstrap.py プロジェクト: crchemist/scioncc
def bootstrap_pyon(logging_config_override=None, pyon_cfg=None):
    """
    This function initializes the core elements of the Pyon framework in a controlled way.
    It does not initialize the ION container or the ION system.
    @param logging_config_override  A dict to initialize the Python logging subsystem (None loads default files)
    @param pyon_cfg   A DotDict with the fully loaded pyon configuration to merge into CFG (None loads default files)
    """
    print "pyon: pyon.bootstrap (bootstrap_pyon) executing..."

    # Make sure Pyon is only initialized only once
    global pyon_initialized
    if pyon_initialized:
        print "pyon: WARNING -- bootstrap_pyon() called again!"
        return

    # ENVIRONMENT. Check we are called in an expected environment (files, directories, etc)
    assert_environment()

    # LOGGING. Initialize logging from config
    if not logutil.is_logging_configured():
        logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS, logging_config_override=logging_config_override)

    # YAML patch: OrderedDicts instead of dicts
    from pyon.util.yaml_ordered_dict import apply_yaml_patch
    apply_yaml_patch()

    # CONFIG. Initialize pyon global configuration from local files
    set_config(pyon_cfg)
    log.debug("pyon: CFG set to %s", CFG)

    # OBJECTS. Object and message definitions.
    from pyon.core.registry import IonObjectRegistry
    global _obj_registry
    _obj_registry = IonObjectRegistry()

    # SERVICES. Service definitions
    # TODO: change the following to read service definitions from directory and import selectively
    from pyon.ion.service import IonServiceRegistry
    import interface.services
    global _service_registry
    _service_registry = IonServiceRegistry()
    _service_registry.load_service_mods(interface.services)
    _service_registry.build_service_map()

    # RESOURCES. Load and initialize definitions
    from pyon.ion import resource
    resource.load_definitions()

    # Fix a weird bug on Ubuntu that resets time.sleep to un-monkey patched version on import threading
    from gevent import monkey; monkey.patch_time()

    # Set initialized flag
    pyon_initialized = True
    log.debug("pyon: initialized OK")
コード例 #5
0
ファイル: pycc.py プロジェクト: ooici/pyon
 def prepare_logging():
     # Load logging override config if provided. Supports variants literal and path.
     logging_config_override = None
     if opts.logcfg:
         if "{" in opts.logcfg:
             # Variant 1: Value is dict of config values
             try:
                 eval_value = ast.literal_eval(opts.logcfg)
                 logging_config_override = eval_value
             except ValueError:
                 raise Exception("Value error in logcfg arg '%s'" % opts.logcfg)
         else:
             # Variant 2: Value is path to YAML file containing config values
             logutil.DEFAULT_LOGGING_PATHS.append(opts.logcfg)
     logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS, logging_config_override=logging_config_override)
コード例 #6
0
ファイル: pycc.py プロジェクト: pkediyal/pyon
 def prepare_logging():
     # Load logging override config if provided. Supports variants literal and path.
     logging_config_override = None
     if opts.logcfg:
         if '{' in opts.logcfg:
             # Variant 1: Value is dict of config values
             try:
                 eval_value = ast.literal_eval(opts.logcfg)
                 logging_config_override = eval_value
             except ValueError:
                 raise Exception("Value error in logcfg arg '%s'" %
                                 opts.logcfg)
         else:
             # Variant 2: Value is path to YAML file containing config values
             logutil.DEFAULT_LOGGING_PATHS.append(opts.logcfg)
     logutil.configure_logging(
         logutil.DEFAULT_LOGGING_PATHS,
         logging_config_override=logging_config_override)
コード例 #7
0
def main():

    usage = \
    """
    %prog [options] prefix
    """
    description = "Use this program to clear databases that match a given prefix"
    parser = OptionParser(usage=usage, description=description)
    parser.add_option("-P",
                      "--port",
                      dest="db_port",
                      default=None,
                      help="Port number for db",
                      action="store",
                      type=int,
                      metavar="PORT")
    parser.add_option("-H",
                      "--host",
                      dest="db_host",
                      default='localhost',
                      help="The host name or ip address of the db server",
                      action="store",
                      type=str,
                      metavar="HOST")
    parser.add_option("-u",
                      "--username",
                      dest="db_uname",
                      default=None,
                      help="Username for the db server",
                      action="store",
                      type=str,
                      metavar="UNAME")
    parser.add_option("-p",
                      "--password",
                      dest="db_pword",
                      default=None,
                      help="Password for the db server",
                      action="store",
                      type=str,
                      metavar="PWORD")
    parser.add_option("-s",
                      "--sysname",
                      dest="sysname",
                      default=None,
                      help="The sysname prefix to clear databases",
                      action="store",
                      type=str,
                      metavar="SYSNAME")
    parser.add_option("-t",
                      "--store_type",
                      dest="db_type",
                      default="postgresql",
                      help="Datastore type",
                      action="store",
                      type=str,
                      metavar="DSTYPE")
    parser.add_option("-v",
                      "--verbose",
                      help="More verbose output",
                      action="store_true")
    parser.add_option("-d",
                      "--dump",
                      dest="dump_path",
                      default=None,
                      help="Dump sysname datastores to path",
                      action="store",
                      type=str,
                      metavar="DPATH")
    parser.add_option("-l",
                      "--load",
                      dest="load_path",
                      default=None,
                      help="Load dumped datastore from path",
                      action="store",
                      type=str,
                      metavar="LPATH")

    (options, args) = parser.parse_args()

    from pyon.core import log as logutil
    logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS)

    if options.dump_path:
        config = create_config(options.db_host, options.db_port,
                               options.db_uname, options.db_pword)
        sysname = options.sysname or "scion"
        log.info("dumping %s datastores to %s", sysname, options.dump_path)
        from pyon.datastore.datastore_admin import DatastoreAdmin
        datastore_admin = DatastoreAdmin(config=config, sysname=sysname)
        datastore_admin.dump_datastore(path=options.dump_path)
    elif options.load_path:
        config = create_config(options.db_host, options.db_port,
                               options.db_uname, options.db_pword)
        sysname = options.sysname or "scion"
        log.info("loading %s datastores from dumped content in %ss", sysname,
                 options.dump_path)
        from pyon.datastore.datastore_admin import DatastoreAdmin
        datastore_admin = DatastoreAdmin(config=config, sysname=sysname)
        datastore_admin.load_datastore(path=options.load_path)
    else:
        if len(args) == 0:
            log.error("Error: no prefix argument specified")
            parser.print_help()
            sys.exit()

        if len(args) != 1:
            log.error(
                "Error: You can not specify multiple prefixes. Received args: %s",
                str(args))
            parser.print_help()
            sys.exit()

        prefix = args[0]

        if prefix == "":
            log.error("Error: You can not give the empty string as a prefix!")
            parser.print_help()
            sys.exit()

        config = create_config(options.db_host, options.db_port,
                               options.db_uname, options.db_pword,
                               options.db_type)
        _clear_db(config,
                  prefix=prefix,
                  sysname=options.sysname,
                  verbose=bool(options.verbose))
コード例 #8
0
 def setUp(self):
     log_config.configure_logging( [ 'res/config/logging.yml', 'res/config/logging.local.yml'] )
コード例 #9
0
def main():
    """
    Store configuration and interfaces into the datastore
    How to run this from command line:
        bin/store_interfaces  -s system name [ -of filename | -sf filename | -fc true|false]
        -of Load object definition file
        -sf Load service definition file
        -fc Force clean the database

     Example:
        Load all object and service definitions
        bin/python bin/store_interfaces  -s mysysname

        Load all object and service definitions with force clean the database
        bin/python bin/store_interfaces  -s mysysname -fc

        Load object definition from a file
        bin/python bin/store_interfaces  -s mysysname -of obj/data/core/org.yml

        Load service definition from a file
        bin/python bin/store_interfaces  -s mysysname -sf obj/services/core/datastore_service.yml
    """

    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-c',
        '--config',
        type=str,
        help='Additional config files to load or dict config content.',
        default=[])
    parser.add_argument('-fc',
                        '--force_clean',
                        action='store_true',
                        help='Force clean.')
    parser.add_argument("-of",
                        "--object",
                        dest="fobject",
                        help="Load object definition from a file")
    parser.add_argument("-s", "--sysname", dest="sysname", help="System name")
    parser.add_argument("-sf",
                        "--service",
                        dest="fservice",
                        help="Load service definition from a file")

    options, extra = parser.parse_known_args()
    args, command_line_config = parse_args(extra)

    log.info("Storing SciON config and interfaces, with options: %s",
             str(options))

    from pyon.core import log as logutil
    logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS)

    # -------------------------------------------------------------------------
    # Store config and interfaces

    # Set global testing flag to False. We are running as standalone script. This is NO TEST.
    bootstrap.testing = False

    # Set sysname if provided in startup argument
    if options.sysname:
        bootstrap.set_sys_name(options.sysname)

    # Load config override if provided. Supports variants literal and list of paths
    config_override = None
    if options.config:
        if '{' in options.config:
            # Variant 1: Dict of config values
            try:
                eval_value = ast.literal_eval(options.config)
                config_override = eval_value
            except ValueError:
                raise Exception("Value error in config arg '%s'" %
                                options.config)
        else:
            # Variant 2: List of paths
            from pyon.util.config import Config
            config_override = Config([options.config]).data

    # bootstrap_config - Used for running this store_interfaces script
    bootstrap_config = config.read_local_configuration(
        ['res/config/pyon_min_boot.yml'])
    config.apply_local_configuration(bootstrap_config,
                                     pyon.DEFAULT_LOCAL_CONFIG_PATHS)
    if config_override:
        config.apply_configuration(bootstrap_config, config_override)
    config.apply_configuration(bootstrap_config, command_line_config)

    # Override sysname from config file or command line
    if not options.sysname and bootstrap_config.get_safe("system.name", None):
        new_sysname = bootstrap_config.get_safe("system.name")
        bootstrap.set_sys_name(new_sysname)

    # Delete sysname datastores if option "force_clean" is set
    if options.force_clean:
        from pyon.datastore import clear_db_util
        from pyon.util.file_sys import FileSystem
        log.info("force_clean=True. DROP DATASTORES for sysname=%s" %
                 bootstrap.get_sys_name())
        pyon_config = config.read_standard_configuration(
        )  # Initial pyon.yml + pyon.local.yml
        clear_db_util.clear_db(bootstrap_config,
                               prefix=bootstrap.get_sys_name(),
                               sysname=bootstrap.get_sys_name())
        FileSystem._clean(pyon_config)

    # ion_config - Holds the new CFG object for the system (independent of this tool's config)
    ion_config = config.read_standard_configuration()
    if config_override:
        config.apply_configuration(ion_config, config_override)
    config.apply_configuration(ion_config, command_line_config)

    # -------------------------------------------------------------------------
    # Store config and interfaces

    iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)

    # Make sure core datastores exist
    iadm.create_core_datastores()

    # Store system CFG properties
    iadm.store_config(ion_config)

    # Store system interfaces
    iadm.store_interfaces(options.fobject, options.fservice)

    iadm.close()
コード例 #10
0
ファイル: bootstrap.py プロジェクト: swarbhanu/pyon
def load_logging_config(logging_config_override=None):
    """
    Initialize pyon logging system
    """
    from pyon.core import log
    log.configure_logging(pyon.DEFAULT_LOGGING_PATHS, logging_config_override=logging_config_override)
コード例 #11
0
 def setUp(self):
     log_config.configure_logging(
         ['res/config/logging.yml', 'res/config/logging.local.yml'])
コード例 #12
0
#!/usr/bin/env python
"""
@package coverage_model.base_test_classes
@file coverage_model/base_test_classes.py
@author Christopher Mueller
@brief Base classes for Unit and Int testing within the coverage model
"""

from unittest import TestCase
import os, shutil, tempfile

from pyon.core import log as logutil
if not logutil.is_logging_configured():
    logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS)


class CoverageModelUnitTestCase(TestCase):

    # Prevent test docstring from printing - uses test name instead
    # @see
    # http://www.saltycrane.com/blog/2012/07/how-prevent-nose-unittest-using-docstring-when-verbosity-2/
    def shortDescription(self):
        return None

    # override __str__ and __repr__ behavior to show a copy-pastable nosetest name for ion tests
    #  ion.module:TestClassName.test_function_name
    def __repr__(self):
        name = self.id()
        name = name.split('.')
        if name[0] not in ["coverage_model"]:
            return "%s (%s)" % (name[-1], '.'.join(name[:-1]))
コード例 #13
0
#!/usr/bin/env python

"""
@package coverage_model.base_test_classes
@file coverage_model/base_test_classes.py
@author Christopher Mueller
@brief Base classes for Unit and Int testing within the coverage model
"""

from unittest import TestCase
import os, shutil, tempfile

from pyon.core import log as logutil

if not logutil.is_logging_configured():
    logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS)


class CoverageModelUnitTestCase(TestCase):

    # Prevent test docstring from printing - uses test name instead
    # @see
    # http://www.saltycrane.com/blog/2012/07/how-prevent-nose-unittest-using-docstring-when-verbosity-2/
    def shortDescription(self):
        return None

    # override __str__ and __repr__ behavior to show a copy-pastable nosetest name for ion tests
    #  ion.module:TestClassName.test_function_name
    def __repr__(self):
        name = self.id()
        name = name.split(".")
コード例 #14
0
def main():
    """
    Store configuration and interfaces into the datastore
    How to run this from command line:
        bin/store_interfaces  -s system name [ -of filename | -sf filename | -fc true|false]
        -of Load object definition file
        -sf Load service definition file
        -fc Force clean the database

     Example:
        Load all object and service definitions
        bin/python bin/store_interfaces  -s mysysname

        Load all object and service definitions with force clean the database
        bin/python bin/store_interfaces  -s mysysname -fc

        Load object definition from a file
        bin/python bin/store_interfaces  -s mysysname -of obj/data/core/org.yml

        Load service definition from a file
        bin/python bin/store_interfaces  -s mysysname -sf obj/services/core/datastore_service.yml
    """

    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--config', type=str, help='Additional config files to load or dict config content.', default=[])
    parser.add_argument('-fc', '--force_clean', action='store_true', help='Force clean.')
    parser.add_argument("-of", "--object", dest="fobject", help="Load object definition from a file")
    parser.add_argument("-s", "--sysname", dest="sysname", help="System name")
    parser.add_argument("-sf", "--service", dest="fservice", help="Load service definition from a file")

    options, extra = parser.parse_known_args()
    args, command_line_config = parse_args(extra)

    log.info("Storing SciON config and interfaces, with options: %s", str(options))

    from pyon.core import log as logutil
    logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS)

    # -------------------------------------------------------------------------
    # Store config and interfaces

    # Set global testing flag to False. We are running as standalone script. This is NO TEST.
    bootstrap.testing = False

    # Set sysname if provided in startup argument
    if options.sysname:
        bootstrap.set_sys_name(options.sysname)

    # Load config override if provided. Supports variants literal and list of paths
    config_override = None
    if options.config:
        if '{' in options.config:
            # Variant 1: Dict of config values
            try:
                eval_value = ast.literal_eval(options.config)
                config_override = eval_value
            except ValueError:
                raise Exception("Value error in config arg '%s'" % options.config)
        else:
            # Variant 2: List of paths
            from pyon.util.config import Config
            config_override = Config([options.config]).data

    # bootstrap_config - Used for running this store_interfaces script
    bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
    config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
    if config_override:
        config.apply_configuration(bootstrap_config, config_override)
    config.apply_configuration(bootstrap_config, command_line_config)

    # Override sysname from config file or command line
    if not options.sysname and bootstrap_config.get_safe("system.name", None):
        new_sysname = bootstrap_config.get_safe("system.name")
        bootstrap.set_sys_name(new_sysname)

    # Delete sysname datastores if option "force_clean" is set
    if options.force_clean:
        from pyon.datastore import clear_db_util
        from pyon.util.file_sys import FileSystem
        log.info("force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name())
        pyon_config = config.read_standard_configuration()      # Initial pyon.yml + pyon.local.yml
        clear_db_util.clear_db(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name())
        FileSystem._clean(pyon_config)


    # ion_config - Holds the new CFG object for the system (independent of this tool's config)
    ion_config = config.read_standard_configuration()
    if config_override:
        config.apply_configuration(ion_config, config_override)
    config.apply_configuration(ion_config, command_line_config)


    # -------------------------------------------------------------------------
    # Store config and interfaces

    iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)

    # Make sure core datastores exist
    iadm.create_core_datastores()

    # Store system CFG properties
    iadm.store_config(ion_config)

    # Store system interfaces
    iadm.store_interfaces(options.fobject, options.fservice)

    iadm.close()