Example #1
0
    def setUp(self):
        config = {key: value for key, value in self.base_config.items()}
        config.update({
            'formatters': {
                'graylog': {
                    '()':
                    'graylog_json_formatter.GrayLogJSONFormatter',
                    'format':
                    '({levelname}) | {name} | [{asctime}]: '
                    'File {pathname}:{lineno}" - {funcName}() | '
                    '{message}',
                    'style':
                    '{',
                    'source':
                    'test',
                    'extra':
                    lambda record: {
                        'service': 'service-func'
                    }
                }
            },
        })
        logging.config.dictConfig(config)

        self.logger = logging.getLogger('test')
Example #2
0
    def cfg(self):
        """Load the application configuration.

        This method loads configuration from python module.
        """
        config = LStruct(self.defaults)
        module = config['CONFIG'] = os.environ.get(
            CONFIGURATION_ENVIRON_VARIABLE, config['CONFIG'])

        if module:
            try:
                module = import_module(module)
                config.update({
                    name: getattr(module, name)
                    for name in dir(module)
                    if name == name.upper() and not name.startswith('_')
                })

            except ImportError as exc:
                config.CONFIG = None
                self.logger.error("Error importing %s: %s", module, exc)

        # Patch configuration from ENV
        for name in config:
            if name.startswith(
                    '_') or name != name.upper() or name not in os.environ:
                continue
            try:
                config[name] = json.loads(os.environ[name])
            except ValueError:
                pass

        return config
Example #3
0
def load_config(settings_file='./test_settings.py'):
    """
    Loads the config files merging the defaults
    with the file defined in environ.PULLSBURY_SETTINGS if it exists.
    """
    config = Config(os.getcwd())

    if 'PULLSBURY_SETTINGS' in os.environ:
        config.from_envvar('PULLSBURY_SETTINGS')
    else:
        config.from_pyfile(settings_file)

    if config.get('LOGGING_CONFIG'):
        logging.config.fileConfig(
            config.get('LOGGING_CONFIG'),
            disable_existing_loggers=False)

    json_values = [
        'TEAMS',
        'HAPPY_SLACK_EMOJIS',
        'REPO_BLACKLIST',
        'SLACK_CUSTOM_EMOJI_MAPPING'
    ]

    for value in json_values:
        config.update({
            value: json.loads(config.get(value, '{}'))
        })

    return config
Example #4
0
def parse_config(filename='config.yaml') -> Dict:
    s = _read_file(filename)
    config = _parse_config(s)
    log_conf = get_config(config)
    config.update(update_config(config))
    logging.config.dictConfig(log_conf)
    return Dict(config)
    def __init__(self, args, options='', timestamp=True):
        # parse default and custom cli options
        for opt in options:
            args.add_argument(*opt.flags, default=None, type=opt.type)
        args = args.parse_args()

        if args.device:
            os.environ["CUDA_VISIBLE_DEVICES"] = args.device
        if args.resume is None:
            msg_no_cfg = "Configuration file need to be specified. Add '-c config.json', for example."
            assert args.config is not None, msg_no_cfg
            self.cfg_fname = Path(args.config)
            config = read_json(self.cfg_fname)
            self.resume = None
        else:
            self.resume = Path(args.resume)
            resume_cfg_fname = self.resume.parent / 'config.json'
            config = read_json(resume_cfg_fname)
            if args.config is not None:
                config.update(read_json(Path(args.config)))

        # load config file and apply custom cli options
        self._config = _update_config(config, options, args)

        # set save_dir where trained model and log will be saved.
        # save_dir = Path(self.config['trainer']['save_dir'])
        # timestamp = datetime.now().strftime(r'%m%d_%H%M%S') if timestamp else ''

        exper_name = self.config['name']
Example #6
0
def create(config):
    """http://flask.pocoo.org/docs/patterns/appfactories/"""

    app = Flask(__name__.split('.')[0])

    # Load YAML config into app.config
    if not isinstance(config, dict):
        with open(os.path.join(app.instance_path, config)) as f:
            config = yaml.load(f)
    config.update({k.upper(): v for k, v in config["flask"].items()})
    app.config.update(config)
    del config

    # Configure logging
    logging.config.dictConfig(app.config["logging"])

    # Initialize database
    app.database = Database(app)

    # Load blueprints
    from .views.submit import submit
    app.register_blueprint(submit)

    if False and app.debug:
        app.logger.warning("Debug mode is on. Do not use this in production.")
        from .views.debug import debug
        app.register_blueprint(debug)

    return app
Example #7
0
def create(config):
    """http://flask.pocoo.org/docs/patterns/appfactories/"""

    app = Flask(__name__.split('.')[0])

    # Load YAML config into app.config
    if not isinstance(config, dict):
        with open(os.path.join(app.instance_path, config)) as f:
            config = yaml.load(f)
    config.update({k.upper(): v for k, v in config["flask"].items()})
    app.config.update(config)
    del config
    
    # Configure logging
    logging.config.dictConfig(app.config["logging"])

    # Initialize database
    app.database = Database(app)

    # Load blueprints
    from .views.submit import submit
    app.register_blueprint(submit)

    if False and app.debug:
        app.logger.warning("Debug mode is on. Do not use this in production.")
        from .views.debug import debug
        app.register_blueprint(debug)

    return app
def load_config():
    src_root = realpath(dirname(__file__))
    project_root = realpath(dirname(src_root))
    filenames = [
        join(src_root, filename) for filename in ('config.yaml', 'local.yaml')
    ]
    config = {}
    for name in filenames:
        if not exists(name):
            continue
        with open(name) as f:
            data = yaml.load(f)
            config = merge_dicts(config, data)
    config.update({
        'src_root': src_root,
        'tmp_root': join(project_root, 'tmp/'),
        'logs_root': join(project_root, 'logs/'),
    })
    config['webhook_url'] = 'https://{hostname}/{token}'.format(
        hostname=config['host'], token=config['telegram']['token'])
    config['evernote'][
        'oauth_callback_url'] = 'https://{hostname}{path}'.format(
            hostname=config['host'], path=config['evernote']['oauth_path'])
    makedirs(config['logs_root'], exist_ok=True)
    makedirs(config['tmp_root'], exist_ok=True)
    logging_config = get_logging_config(config['logs_root'])
    logging.config.dictConfig(logging_config)
    return config
Example #9
0
    def __init__(self):
        # Config set up. Environment overrides app.yaml
        with open("cfg/app.yaml", "r") as f:
            config = yaml.safe_load(f)
            config.update(os.environ)

        # Logger set up
        self._init_logger(config["log_dir"])
        self._logger.info("Initializing the App")

        # Quik connector
        self._connector = WebQuikConnector(conn=config["conn"],
                                           passwd=config["passwd"],
                                           account=config["account"])

        # Feed2Csv just receive price and level2 for single configured asset and write to data folder
        web_quik_feed = WebQuikFeed(self._connector,
                                    rabbit_host=config["rabbit_host"])
        # self._feed = Feed2Csv(web_quik_feed, config.sec_class, config.sec_code)

        # Broker is not implemented, just a stub.
        web_quik_broker = WebQuikBroker(connector=self._connector,
                                        client_code=config["client_code"],
                                        trade_account=config["trade_account"],
                                        rabbit_host=config["rabbit_host"])

        # Create feed, subscribe events
        # Todo: support making orders
        self._strategy = Strategy(web_quik_feed, web_quik_broker,
                                  config["sec_class"], config["sec_code"])
Example #10
0
def read() -> None:
    """
    Read configuration into global variable 'config'.
    """
    with open('disy.yml') as f:
        y = yaml.load(f) or {}
        config.update(y.items())
Example #11
0
def get_config(args):
    config = {}

    if args.config:
        with open(args.config, 'r') as f:
            config.update(yaml.load(f.read()))

    return config
Example #12
0
def update_config(overlay_dict):
    """Apply a partial config dict to the singleton config dict

    Values for top-level keys will be overwritten with those in overlay_dict
    No recursion will take place for sub-dicts; the entire sub-dict will be overwritten
    """
    config.update(overlay_dict)
    new_config = config_schema.check_value(config)
    config.clear()
    config.update(new_config)
Example #13
0
def parse_commandline(args=None, config=None):
    """Construct commandline parser and then parse arguments.

    :param args: Command line arguments to parse. If none are
        given then ``sys.argv`` is used by default.

    :param config: ``configparser.ConfigParser`` object that
        contains the initial configuration of the system. The
        default is to use ``load_configuration`` to get the
        default.

    :return: A namespace that contains attributes with values
        determined from ``config`` and then command line
        arguments.
    """

    if config is None:
        config = load_configuration()
    parser = argparse.ArgumentParser()
    parser.add_argument("--version",
                        action="version",
                        version=f"%(prog)s {__version__}")
    parser.add_argument(
        "--verbose",
        type=str,
        default="warning",
        choices=["critical", "error", "warning", "info", "debug"],
        help="Change default logging verbosity.",
    )

    subparsers = parser.add_subparsers(help="sub-command help")
    init_subparser(subparsers)
    models_subparser(subparsers, config)
    upload_subparser(subparsers, config)
    train_subparser(subparsers, config)
    predict_subparser(subparsers, config)
    options = parser.parse_args(args=args)

    # Adjust root logger and console to match verbose
    rootlogger = logging.getLogger()
    rootlogger.setLevel(options.verbose.upper())
    rootconsole = [h for h in rootlogger.handlers if h.get_name() == "console"]
    if rootconsole:
        rootconsole = rootconsole[0]
        rootconsole.setLevel(options.verbose.upper())

    # Update configuration and set in options
    config.update(options)
    options.configuration = config
    if not hasattr(options, "func"):
        parser.print_help()
        sys.exit(1)
    if options.verbose == "debug":
        print("Working directory:", pathlib.Path.cwd(), file=sys.stderr)
    return options
Example #14
0
def _post_init_config(params):
    '''
    add additional parameters into config
    '''
    logger.warning('params: %s', params)

    for (k, v) in params.iteritems():
        if k in config:
            logger.warning('params will be overwrite: key: %s origin: %s new: %s', k, config[k], v)

    config.update(params)
Example #15
0
    def make_from_config_list(cls, config_paths):
        """
        Initialize the App by using the configuration list. This function must be executed only once.
        :param config_paths:
        :return: App Class
        """
        config = Config.get_empty()

        for path in config_paths:
            config.update(config.from_yaml(path))
        return cls.instance(config, update=True)
Example #16
0
def reload_config():
    """[Re]load the configuration

    * [Re-]reads the identified config file and replaces the contents of the singleton config dict with it
    * Any config variables in os.environ will be overlaid onto the dict [again]
    * Normalization and setting of defaults will occur [again]
    """
    new_config = read_config_file(config_filename)
    _env_config(new_config)
    new_config = config_schema.check_value(new_config)
    config.clear()
    config.update(new_config)
Example #17
0
def main(args):
    app_name = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
    config = load_config(app_name,
                         'credentials',
                         'site.conf',
                         verbose=True,
                         s3_bucket=os.getenv("SETTINGS_BUCKET", None),
                         passphrase=os.getenv("SETTINGS_CRYPT_KEY", None))
    config.update({
        'SESSION_COOKIE_NAME': 'sessionid',
        'ALLOWED_NO_HOST': ['/static/']
    })
    app.config.update(config)

    settings.update(DEBUG=app.debug, APP_NAME=app_name)

    logging.config.dictConfig({
        'version': 1,
        'disable_existing_loggers': False,
        'filters': {},
        'formatters': {
            'simple': {
                'format': 'X X %(levelname)s [%(asctime)s] %(message)s',
                'datefmt': '%d/%b/%Y:%H:%M:%S %z'
            },
        },
        'handlers': {
            'logfile': {
                'level': 'DEBUG',
                'formatter': 'simple',
                'filters': [],
                'class': 'logging.StreamHandler',
            },
        },
        'loggers': {
            'deployutils': {
                'handlers': [],
                'level': 'INFO',
            },
            # This is the root logger.
            # The level will only be taken into account if the record is not
            # propagated from a child logger.
            #https://docs.python.org/2/library/logging.html#logging.Logger.propagate
            '': {
                'handlers': ['logfile'],
                'level': 'INFO'
            },
        },
    })
    global session
    session = sessions.Session()
    session.init_app(app)
    app.run()
Example #18
0
    def reload(self):
        """Reload the configuration from disk returning True if the
        configuration has changed from the previous values.

        """
        config = self._default_configuration()
        if self._file_path:
            config.update(self._load_config_file())
        if config != self._values:
            self._values = config
            return True
        return False
Example #19
0
File: config.py Project: gmr/helper
    def reload(self):
        """Reload the configuration from disk returning True if the
        configuration has changed from the previous values.

        """
        config = self._default_configuration()
        if self._file_path:
            config.update(self._load_config_file())
        if config != self._values:
            self._values = config
            return True
        return False
Example #20
0
def _update_cli_config(dict_conf: Dict) -> None:
    """
    Update CLI config and write to yaml file.

    :param dict_conf: dict config to write.

    :return: None
    """
    config = _get_or_create_cli_config()
    config.update(dict_conf)
    with open(CLI_CONFIG_PATH, "w") as f:
        yaml.dump(config, f, default_flow_style=False)
Example #21
0
def get_baboond_config():
    """ Returns the baboond full dict configuration.
    """

    arg_attrs = get_config_args(PARSER)
    file_attrs = get_config_file(arg_attrs, 'baboondrc')
    init_config_log(arg_attrs, LOGGING)

    config = {}
    config.update(arg_attrs)
    config.update(file_attrs)

    return config
Example #22
0
 def supervisor(self, config):
     config.update({
         key: val
         for key, val in os.environ.items()
         if key.startswith("SUPERVISOR_")
     })
     rpc = supervisor.childutils.getRPCInterface(config)
     rpc.supervisor.getState()
     state = rpc.supervisor.getState().get("statename")
     if state != consts.RUNNING:
         raise ValueError("can not talk to supervisor server")
     self._supervisor = rpc.supervisor
     return self._supervisor
Example #23
0
def _post_init_config(params):
    '''
    add additional parameters into config
    '''
    logger.warning('params: %s', params)

    for (k, v) in params.iteritems():
        if k in config:
            logger.warning(
                'params will be overwrite: key: %s origin: %s new: %s', k,
                config[k], v)

    config.update(params)
Example #24
0
def configure(backend=backend.amqp_puka.AMQPHandler,
              # args=("amqp://*****:*****@localhost:5672/%2F", "lc-topic", "topic"),
              args=[],
              kwargs={}):
    '''
    Explicitly configure contexture with a handler.

    This will skip the logging-based configuration.
    '''
    global backend_handler
    backend_handler = backend(*args, **kwargs)
    config.update({k: v for k, v in kwargs.iteritems() if k in config})
    logging.basicConfig()
Example #25
0
def load_config(path=None):
    """Load configuration."""
    global config
    _path = abspath("." if path is None else path)
    try:
        with open(join(_path, "item_config.yaml")) as f:
            result = yaml.safe_load(f)
            config["_from_file"] = result
            config.update(result)
            config["_filename"] = "item_config.yaml"
    except FileNotFoundError:
        if path is not None:
            raise
Example #26
0
def setup(cfg=None):
    if cfg and isinstance(cfg, dict):
        config.update(cfg)

    if os.path.exists('logger.conf'):
        logging.config.fileConfig('logger.conf',
                                  disable_existing_loggers=False)

    logger_filter = LoggerFilter()
    logging.getLogger().addFilter(logger_filter)

    for logger_name in list(logging.getLogger().manager.loggerDict.keys()):
        if logger_name.startswith('es.') or logger_name.startswith('tornado'):
            logging.getLogger(logger_name).addFilter(logger_filter)
Example #27
0
def new_config(new_config):
    """
    Temporarily change configuration dictionary.
    """
    orig_config = config.copy()
    try:
        config.clear()
        config.update(new_config)
        initialize_logging(config)
        yield
    finally:
        config.clear()
        config.update(orig_config)
        initialize_logging(config)
    def get_oxauth_config(self):
        req = self.backend.exec_query(
            "SELECT oxRevision, oxAuthConfDynamic, oxAuthConfWebKeys "
            "FROM `gluu` "
            "USE KEYS 'configuration_oxauth'", )
        if not req.ok:
            return {}

        config = req.json()["results"][0]

        if not config:
            return {}

        config.update({"id": "configuration_oxauth"})
        return config
Example #29
0
    def get_configuration(self):
        req = self.backend.exec_query(
            "SELECT oxTrustCacheRefreshServerIpAddress, gluuVdsCacheRefreshEnabled "
            "FROM `gluu` "
            "USE KEYS 'configuration'")

        if not req.ok:
            return {}

        config = req.json()["results"][0]

        if not config:
            return {}

        config.update({"id": "configuration"})
        return config
Example #30
0
    def clean_stores(self):
        fleet = self._config_params.get('fleet')
        robot_proxy_store_config = self._config_params.get("robot_proxy_store")
        robot_store_config = self._config_params.get("robot_store")
        store_configs = {'robot_proxy_store': robot_proxy_store_config,
                         'robot_store': robot_store_config}

        for robot_id in fleet:
            for store_name, config in store_configs.items():
                config.update({'db_name': store_name + '_' + robot_id.split('_')[1]})
                store = MongoStore(**config)
                self.clean_store(store)

        ccu_store_config = self._config_params.get('ccu_store')
        store = MongoStore(**ccu_store_config)
        self.clean_store(store)
    def get_auth_config(self):
        bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans")
        req = self.client.exec_query(
            "SELECT jansRevision, jansConfDyn, jansConfWebKeys "
            f"FROM `{bucket}` "
            "USE KEYS 'configuration_jans-auth'", )
        if not req.ok:
            return {}

        config = req.json()["results"][0]

        if not config:
            return {}

        config.update({"id": "configuration_jans-auth"})
        return config
Example #32
0
def _load_config(config_filepaths):
    """ Load config files with inheritance

    Load each config file and update the result dictionary.

    """
    # Base Settings
    config = {}
    config['MODULE_ROOT_PATH'] = os.path.dirname(os.path.dirname(__file__))

    for fp in config_filepaths:
        with open(fp) as f:
            config.update(yaml.load(f))

    _update_relpaths(config, '_PATH')
    _update_relpaths(config, '_FILEPATH')
    return config
Example #33
0
def new_config(new_config):
    """
    Temporarily change configuration dictionary.
    """
    from .config import defaults
    config = dask.config.config
    orig_config = config.copy()
    try:
        config.clear()
        config.update(defaults.copy())
        dask.config.update(config, new_config)
        initialize_logging(config)
        yield
    finally:
        config.clear()
        config.update(orig_config)
        initialize_logging(config)
Example #34
0
def readXMLConfig(filename):
    config = {}
    ctx, root = resource_helper.getRoot(filename, safe=False)
    if root is None:
        return config
    else:
        sectionCtx, section = resource_helper.getSubSection(ctx,
                                                            root,
                                                            'common',
                                                            safe=False)
        if section is not None:
            config.update(resource_helper.readDict(sectionCtx, section).value)
        for key in ('filters', 'formatters', 'handlers', 'loggers'):
            config[key] = _readConfigItem(key, ctx, root)

        resource_helper.purgeResource(filename)
        return config
Example #35
0
def new_config(new_config):
    """
    Temporarily change configuration dictionary.
    """
    from .config import defaults
    config = dask.config.config
    orig_config = config.copy()
    try:
        config.clear()
        config.update(defaults.copy())
        dask.config.update(config, new_config)
        initialize_logging(config)
        yield
    finally:
        config.clear()
        config.update(orig_config)
        initialize_logging(config)
Example #36
0
def main(args=None):
    args = parse_args(args)

    main_proc = rpc_server.MainProcess()

    config = {}

    # load default configuration from file
    for path in (args.config_file, 'config.yaml'):
        if path and os.path.exists(path):
            config = read_config(path)
            break

    process_config(config)

    # allow overwritting any configuration setting via env vars
    for k, v in os.environ.items():
        if k.startswith('WORKER_'):
            config[k.replace("WORKER_", "").replace("_", ".").lower()] = v

    # make zmon worker compatible with old redis config vars
    if 'redis.host' in config:
        port = config.get('redis.port', 6379)
        config.update({"redis.servers":'{}:{}'.format(config["redis.host"], port)})

    # save config in our settings module
    settings.set_workers_log_level(config.get('loglevel', 'INFO'))
    settings.set_external_config(config)
    settings.set_rpc_server_port(config.get('server.port'))

    logging.config.dictConfig(settings.RPC_SERVER_CONF['LOGGING'])

    # start the process controller
    main_proc.start_proc_control()

    # start some processes per queue according to the config
    queues = config['zmon.queues']['local']
    for qn in queues.split(','):
        queue, N = (qn.rsplit('/', 1) + [DEFAULT_NUM_PROC])[:2]
        main_proc.proc_control.spawn_many(int(N), kwargs={"queue": queue, "flow": "simple_queue_processor"})

    if not args.no_rpc:
        main_proc.start_rpc_server()

    return main_proc
def init():
    config = {'logging': {'level': 'DEBUG'}}

    MULTINET_SERVER_CONFIG_DIR = os.environ.get(
        'MULTINET_SERVER_CONFIG_DIR', os.path.join('/', 'etc',
                                                   'gremlin-server'))
    default_config = os.path.join(MULTINET_SERVER_CONFIG_DIR, 'config.yml')
    default_logging_config = os.path.join(MULTINET_SERVER_CONFIG_DIR,
                                          'logging.conf')

    conf_file = (os.environ.get('MULTINET_SERVER_CONF') or default_config)

    if os.path.exists(conf_file):
        with open(conf_file) as stream:
            config.update(yaml.load(stream))

    logging.basicConfig(stream=sys.stdout,
                        level=logging.getLevelName(config['logging']['level']))
Example #38
0
def set_global_logging(logging_file: str = None,
                       logging_level: int = logging.INFO,
                       logging_config_file: Union[str, None] = None) -> None:
    import logging.config
    if not logging_config_file:
        config = dict(
            level=logging_level,
            format=
            "%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s"
        )
        if logging_file:
            config.update(filename=logging_file)
        logging.basicConfig(**config)
    else:
        if logging_config_file == 'default':
            logging_config_file = os.path.join(config_dir, 'logging.yaml')
        with open(logging_config_file, 'r') as f:
            config = yaml.load(f)
            logging.config.dictConfig(config)
Example #39
0
def init_from_ini(ini_file, games=None, overrides=None):
    import logging
    import logging.config
    
    logging.config.fileConfig(ini_file)

    # Reset log after logging has been configured
    global log
    log = logging.getLogger(__name__)

    full_config = ConfigParser.SafeConfigParser()
    full_config.readfp(open(ini_file))
    config = dict(full_config.items('DEFAULT'))

    if overrides:
        config.update(overrides)

    init(config, games)
    return (full_config, config)
Example #40
0
    def cfg(self):
        """ Load the application configuration. """
        config = LStruct(self.defaults)
        module = config['CONFIG'] = os.environ.get(
            CONFIGURATION_ENVIRON_VARIABLE, config['CONFIG'])

        if module:
            try:
                module = import_module(module)
                config.update({
                    name: getattr(module, name) for name in dir(module)
                    if name == name.upper() and not name.startswith('_')
                })

            except ImportError:
                config.CONFIG = None
                self.register_on_start(
                    lambda app: app.logger.warn("The configuration hasn't found: %s" % module))

        return config
Example #41
0
    def cfg(self):
        """Load the application configuration.

        This method loads configuration from python module.
        """
        config = utils.LStruct(self.defaults)
        module = config['CONFIG'] = os.environ.get(
            "CONFIG", config['CONFIG'])

        if module:
            try:
                module = import_module(module)
                config.update({
                    name: getattr(module, name) for name in dir(module)
                    if name == name.upper() and not name.startswith('_')
                })

            except ImportError as exc:
                config.CONFIG = None
                print("Error importing %s: %s" % (module, exc))

        return config
Example #42
0
def load_config():
    src_root = realpath(dirname(__file__))
    project_root = realpath(dirname(src_root))
    filenames = [join(src_root, filename) for filename in ('config.yaml', 'local.yaml')]
    config = {}
    for name in filenames:
        if not exists(name):
            continue
        with open(name) as f:
            data = yaml.load(f)
            config = merge_dicts(config, data)
    config.update({
        'src_root': src_root,
        'tmp_root': join(project_root, 'tmp/'),
        'logs_root': join(project_root, 'logs/'),
    })
    config['webhook_url'] = 'https://{hostname}/{token}'.format(hostname=config['host'], token=config['telegram']['token'])
    config['evernote']['oauth_callback_url'] = 'https://{hostname}{path}'.format(hostname=config['host'], path=config['evernote']['oauth_path'])
    makedirs(config['logs_root'], exist_ok=True)
    makedirs(config['tmp_root'], exist_ok=True)
    logging_config = get_logging_config(config['logs_root'])
    logging.config.dictConfig(logging_config)
    return config
Example #43
0
    def cfg(self):
        """Load the application configuration.

        This method loads configuration from python module.
        """
        config = LStruct(self.defaults)
        module = config['CONFIG'] = os.environ.get(
            CONFIGURATION_ENVIRON_VARIABLE, config['CONFIG'])

        if module:
            try:
                module = import_module(module)
                config.update({
                    name: getattr(module, name) for name in dir(module)
                    if name == name.upper() and not name.startswith('_')
                })

            except ImportError as exc:
                config.CONFIG = None
                message = "Error importing %s: %s" % (module, exc)
                self.register_on_start(lambda app: app.logger.error(message))

        return config
Example #44
0
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
from models import Material, Blueprint, Ingredient, Engineer
from utils import DotDict
from sqlalchemy import create_engine, distinct
from sqlalchemy.orm import sessionmaker, joinedload, subqueryload
import click
import yaml
import logging
import logging.config
import json


config = DotDict()
with open("config.yml", "r") as fp:
    config.update(yaml.load(fp))


logging.config.dictConfig(config["logging"])
log = logging.getLogger(__name__)
engine = create_engine(config["db.url"], echo=config["db.echo"])
Session = sessionmaker(bind=engine)


@click.command()
@click.argument("outfile")
def json_export(outfile):
    db = Session()
    data = dict()
    data["materials"] = [
        it.to_dict(["locations"])
Example #45
0
def raster2pyramid(
    input_file,
    output_dir,
    options
    ):
    """
    Creates a tile pyramid out of an input raster dataset.
    """
    pyramid_type = options["pyramid_type"]
    scale_method = options["scale_method"]
    output_format = options["output_format"]
    resampling = options["resampling"]
    zoom = options["zoom"]
    bounds = options["bounds"]
    overwrite = options["overwrite"]

    # Prepare process parameters
    minzoom, maxzoom = _get_zoom(zoom, input_file, pyramid_type)
    process_file = os.path.join(
        os.path.dirname(os.path.realpath(__file__)),
        "tilify.py"
    )

    with rasterio.open(input_file, "r") as input_raster:
        output_bands = input_raster.count
        input_dtype = input_raster.dtypes[0]
        output_dtype = input_raster.dtypes[0]
        nodataval = input_raster.nodatavals[0]
        if not nodataval:
            nodataval = 0
        if output_format == "PNG":
            if output_bands > 3:
                output_bands = 3
                output_dtype = 'uint8'
        scales_minmax = ()
        if scale_method == "dtype_scale":
            for index in range(1, output_bands+1):
                scales_minmax += (DTYPE_RANGES[input_dtype], )
        elif scale_method == "minmax_scale":
            for index in range(1, output_bands+1):
                band = input_raster.read(index)
                scales_minmax += ((band.min(), band.max()), )
        elif scale_method == "crop":
            for index in range(1, output_bands+1):
                scales_minmax += ((0, 255), )
        if input_dtype == "uint8":
            scale_method = None
            scales_minmax = ()
            for index in range(1, output_bands+1):
                scales_minmax += ((None, None), )

    # Create configuration
    config = {}
    config.update(
        process_file=process_file,
        output={
            "path": output_dir,
            "format": output_format,
            "type": pyramid_type,
            "bands": output_bands,
            "dtype": output_dtype
            },
        scale_method=scale_method,
        scales_minmax=scales_minmax,
        input_files={"raster": input_file},
        config_dir=os.getcwd(),
        process_minzoom=minzoom,
        process_maxzoom=maxzoom,
        nodataval=nodataval,
        resampling=resampling,
        bounds=bounds,
        pixelbuffer=5,
        baselevel={"zoom": maxzoom, "resampling": resampling}
    )

    LOGGER.info("preparing process ...")

    try:
        mapchete = Mapchete(
            MapcheteConfig(
                config,
                zoom=zoom,
                bounds=bounds
            )
        )
    except PyCompileError as error:
        print error
        return
    except:
        raise

    # Prepare output directory and logging
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    logging.config.dictConfig(get_log_config(mapchete))

    for zoom in reversed(range(minzoom, maxzoom+1)):
        # Determine work tiles and run
        work_tiles = mapchete.get_work_tiles(zoom)
        func = partial(_worker,
            mapchete=mapchete,
            overwrite=overwrite
        )
        pool = Pool()
        try:
            pool.map_async(func, work_tiles)
            pool.close()
        except KeyboardInterrupt:
            LOGGER.info(
                "Caught KeyboardInterrupt, terminating workers"
                )
            pool.terminate()
            break
        except:
            raise
        finally:
            pool.close()
            pool.join()
Example #46
0
#!/usr/bin/env python
# This file is part of HemeLB and is Copyright (C)
# the HemeLB team and/or their institutions, as detailed in the
# file AUTHORS. This software is provided under the terms of the
# license in the file LICENSE.
import yaml
import os
import logging
import logging.config


localroot = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
#Load and invoke the default non-machine specific config JSON dictionaries.
defaults_file = open(os.path.join(localroot, 'Tools', 'analysis', 'config_defaults.yml'))
config = yaml.load(defaults_file)
defaults_file.close()

user_file = open(os.path.join(localroot, 'Tools', 'analysis', 'config.yml'))
config.update(yaml.load(user_file))
user_file.close()

logging_settings_file = open(os.path.join(localroot, 'Tools', 'analysis', 'logging.yml'))
dc = yaml.load(logging_settings_file)
logging_settings_file.close()
dc['handlers']['parsing']['filename'] = os.path.expanduser(os.path.join(config['reports_path'], 'parsing.log'))
logging.config.dictConfig(dc)
Example #47
0
 def setup_logging(**kwargs):
     config = DEFAULT_BOT_SETTINGS["logging"]
     config.update(kwargs)
     logging.config.dictConfig(config)
Example #48
0
def test_config_update_context_converts_list():
    import math

    config = konch.Config()
    config.update({"context": [math]})
    assert config["context"] == {"math": math}
def main():
    '''Main(). Commandline parsing and stalker startup.'''

    parser = argparse.ArgumentParser()

    parser.add_argument("-p", "--posttroll_port", dest="posttroll_port",
                        help="Local port where messages are published")
    parser.add_argument("-t", "--topic", dest="topic",
                        help="Topic of the sent messages")
    parser.add_argument("-c", "--configuration_file",
                        help="Name of the config.ini configuration file")
    parser.add_argument("-C", "--config_item",
                        help="Name of the configuration item to use")
    parser.add_argument("-e", "--event_names",
                        help="Name of the pyinotify events to monitor")
    parser.add_argument("-f", "--filepattern",
                        help="Filepath pattern used to parse "
                        "satellite/orbit/date/etc information")
    parser.add_argument("-i", "--instrument",
                        help="Instrument name in the satellite")

    if len(sys.argv) <= 1:
        parser.print_help()
        sys.exit()
    else:
        args = parser.parse_args()

    # Parse commandline arguments.  If args are given, they override
    # the configuration file.

    args_dict = vars(args)
    args_dict = {k: args_dict[k]
                 for k in args_dict if args_dict[k] != None}

    config = {}

    if args.configuration_file is not None:
        config_fname = args.configuration_file

        if "template" in config_fname:
            print "Template file given as trollstalker logging config," \
                " aborting!"
            sys.exit()

        cparser = ConfigParser()
        cparser.read(config_fname)
        config = dict(cparser.items(args.config_item, vars=args_dict))

    config.update(args_dict)

    config.update({k: config[k].split(",")
                   for k in config if "," in config[k]})

    config.setdefault("posttroll_port", "0")

    try:
        log_config = config["stalker_log_config"]
    except KeyError:
        try:
            loglevel = getattr(logging, config.get("loglevel", "DEBUG"))
            if loglevel == "":
                raise AttributeError
        except AttributeError:
            loglevel = logging.DEBUG

        LOGGER.setLevel(loglevel)
        rootlogger = logging.getLogger("")
        rootlogger.setLevel(loglevel)
        strhndl = logging.StreamHandler()
        strhndl.setLevel(loglevel)
        log_format = "[%(asctime)s %(levelname)-8s %(name)s] %(message)s"
        formatter = logging.Formatter(log_format)

        strhndl.setFormatter(formatter)
        rootlogger.addHandler(strhndl)
    else:
        logging.config.fileConfig(log_config)

    LOGGER.debug("Logger started")

    # Start watching for new files
    notifier = FilePublisher(config)
    notifier.start()

    try:
        while True:
            time.sleep(6000000)
    except KeyboardInterrupt:
        LOGGER.info("Interrupting TrollStalker")
    finally:
        notifier.stop()
Example #50
0
__copyright__ = "Copyright (C) 2014-2017 Ivan Kravets"

config = dict(
    SQLALCHEMY_DATABASE_URI=None,
    GITHUB_LOGIN=None,
    GITHUB_PASSWORD=None,
    DL_PIO_DIR=None,
    DL_PIO_URL=None,
    MAX_DLFILE_SIZE=1024 * 1024 * 150,  # 150 Mb

    # Fuzzy search will not be applied to words shorter than the value below
    SOLR_FUZZY_MIN_WORD_LENGTH=3,
    LOGGING=dict(version=1)
)

assert "PIOAPI_CONFIG_PATH" in os.environ
with open(os.environ.get("PIOAPI_CONFIG_PATH")) as f:
    config.update(json.load(f))

# configure logging for packages
logging.basicConfig()
logging.config.dictConfig(config['LOGGING'])

# setup time zone to UTC globally
os.environ['TZ'] = "+00:00"
try:
    from time import tzset
    tzset()
except ImportError:
    pass
Example #51
0
import os
import jsmin
import json
import logging.config

current_path = os.path.dirname(__file__)

local_config_path = os.path.join(current_path, 'local.config.json')
config_path = os.path.join(current_path, 'config.json')



config = json.loads(jsmin.jsmin(open(config_path).read()))
if os.path.exists(local_config_path):
    config.update(json.loads(jsmin.jsmin(open(local_config_path).read())))

config["current_dir"] = os.path.dirname(os.path.abspath(__file__))

if not "template_dir" in config:
    config["template_dir"] = os.path.join(config["current_dir"], "templates")
if not "template_path" in config:
    config["template_path"] = os.path.join(config["current_dir"], "templates")


if not "staticfiles_dir" in config:
    config["staticfiles_dir"] = os.path.join(config["current_dir"], "images")
if not "static_path" in config:
    config["static_path"] = config["current_dir"]

Example #52
0
def test_config_shallow_merges_context():
    config = konch.Config()
    config.update({"context": {"foo": 42}, "banner": "bar"})
    config.update({"context": {"baz": 24}, "banner": "qux"})

    assert config["context"] == {"foo": 42, "baz": 24}
    assert config["banner"] == "qux"

    config = konch.Config()
    config.update({"context": {"foo": 42}})
    config.update({"context": {"foo": 24}})
    assert config["context"] == {"foo": 24}

    config = konch.Config()
    config.update({"context": {"foo": {"inner": 42}}})
    config.update({"context": {"foo": {"inner2": 24}}})
    assert config["context"] == {"foo": {"inner2": 24}}

    config = konch.Config()

    def bar():
        pass

    config.update({"context": {"foo": 42}, "banner": "bar"})
    config.update({"context": [bar], "banner": "bar"})
Example #53
0
def main(args=None):

    args = parse_args(args)

    main_proc = rpc_server.MainProcess()

    config = {}

    # load default configuration from file
    for path in (args.config_file, 'config.yaml'):
        if path and os.path.exists(path):
            config = read_config(path)
            break

    process_config(config)

    # allow overwritting any configuration setting via env vars
    for k, v in os.environ.items():
        if k.startswith('WORKER_'):
            config[k.replace("WORKER_", "").replace("_", ".").lower()] = v

    # make zmon worker compatible with old redis config vars
    if 'redis.host' in config:
        port = config.get('redis.port', 6379)
        config.update({"redis.servers": '{}:{}'.format(config["redis.host"], port)})

    # save config in our settings module
    settings.set_workers_log_level(config.get('loglevel', 'INFO'))
    settings.set_external_config(config)
    settings.set_rpc_server_port(config.get('server.port'))

    logging.config.dictConfig(settings.RPC_SERVER_CONF['LOGGING'])

    logger = logging.getLogger(__name__)

    # start the process controller
    main_proc.start_proc_control()

    # start worker processes per queue according to the config
    queues = config['zmon.queues']
    for qn in queues.split(','):
        queue, N = (qn.rsplit('/', 1) + [DEFAULT_NUM_PROC])[:2]
        main_proc.proc_control.spawn_many(int(N), kwargs={"queue": queue, "flow": "simple_queue_processor"},
                                          flags=MONITOR_RESTART | MONITOR_KILL_REQ | MONITOR_PING)

    # start web server process under supervision
    main_proc.proc_control.spawn_process(
        target=start_web,
        kwargs=dict(
            listen_on=config.get('webserver.listen_on', '0.0.0.0'),
            port=int(config.get('webserver.port', '8080')),
            log_conf=None,
            threaded=True,
            rpc_url='http://{host}:{port}{path}'.format(host='localhost', port=config.get('server.port'),
                                                        path=settings.RPC_SERVER_CONF['RPC_PATH']),
        ),
        flags=MONITOR_RESTART,  # web server will be restarted if dies
    )

    if not args.no_rpc:
        try:
            main_proc.start_rpc_server()
        except (KeyboardInterrupt, SystemExit):
            logger.info('RPC server stopped. Exiting main')

    return main_proc