def run(): """ Main function for command-line interface. """ # pylint: disable=too-many-locals # pylint: disable=too-many-statements # pylint: disable=too-many-branches args = ArgParser().get_args() if args.model == 'version': print("MyTardis Client v%s" % VERSION) sys.exit(0) if args.verbose and (not hasattr(args, 'json') or not args.json): print("MyTardis Client v%s" % VERSION) config_path = DEFAULT_CONFIG_PATH if not os.path.exists(config_path) or \ args.model == 'config': ConfigController(config_path).configure(args) if args.model == 'config': sys.exit(0) config = Config(config_path) config.validate() logging.config.fileConfig(config.logging_config_path, disable_existing_loggers=False) logging.getLogger("dogpile.core.dogpile").setLevel(logging.WARNING) logging.getLogger("requests").setLevel(logging.WARNING) logger = logging.getLogger(__name__) logger.info("MyTardis Client v%s", VERSION) if args.verbose and (not hasattr(args, 'json') or not args.json): print("General config: %s" % config_path) print("Logging config: %s" % config.logging_config_path) print("MyTardis URL: %s" % config.url) print("Username: %s" % config.username) if args.model == 'api': ApiController().run_command(args) elif args.model == 'facility': FacilityController().run_command(args) elif args.model == 'instrument': InstrumentController().run_command(args) elif args.model == 'experiment': ExperimentController().run_command(args) elif args.model == 'dataset': DatasetController().run_command(args) elif args.model == 'datafile': DataFileController().run_command(args) elif args.model == 'storagebox': StorageBoxController().run_command(args) elif args.model == 'schema': SchemaController().run_command(args)
def initialize_configuration(): global config global configfilename configfilename = "server.conf" config = configuration() print "Reading configuration from " + configfilename + "..." ret = config.read(configfilename) config.validate() return ret
def run(): """ Main function for command-line interface. """ # pylint: disable=too-many-branches args = ArgParser().get_args() if args.model == 'version': print("MyTardis Client v%s" % VERSION) sys.exit(0) if args.verbose and (not hasattr(args, 'json') or not args.json): print("MyTardis Client v%s" % VERSION) config_path = DEFAULT_CONFIG_PATH if not os.path.exists(config_path) or \ args.model == 'config': ConfigController(config_path).configure(args) if args.model == 'config': sys.exit(0) config = Config(config_path) config.validate() logging.config.fileConfig(config.logging_config_path, disable_existing_loggers=False) logging.getLogger("requests").setLevel(logging.WARNING) if args.version: print("MyTardis Client v%s" % VERSION) if args.verbose and (not hasattr(args, 'json') or not args.json): print("General config: %s" % config_path) print("Logging config: %s" % config.logging_config_path) print("MyTardis URL: %s" % config.url) print("Username: %s" % config.username) if args.model == 'api': ApiController().run_command(args) elif args.model == 'facility': FacilityController().run_command(args) elif args.model == 'instrument': InstrumentController().run_command(args) elif args.model == 'experiment': ExperimentController().run_command(args) elif args.model == 'dataset': DatasetController().run_command(args) elif args.model == 'datafile': DataFileController().run_command(args) elif args.model == 'storagebox': StorageBoxController().run_command(args) elif args.model == 'schema': SchemaController().run_command(args)
def load() -> ConfigDict: # language=rst """ Load and validate the configuration. """ config_path = _config_path() config = ConfigDict(_load_yaml(config_path)) if 'logging' not in config: raise ConfigError( "No 'logging' entry in config file {}".format(config_path)) logging.config.dictConfig(config['logging']) logger.info("Loaded configuration from '%s'", os.path.abspath(str(config_path))) config.validate(_config_schema()) return config
def __init__(self, config_file, error, initial_conf=None): conf = configobj.ConfigObj(config_file, configspec=configobj.ConfigObj(self.spec), list_values=False, interpolation='Template') conf.merge(initial_conf or {}) config.validate(config_file, conf, error) logging.addLevelName(10000, 'NONE') log_conf = '' for k, v in conf['logging'].items(): log_conf += '\n[%s]\n' % k log_conf += '\n'.join(map('='.join, v.items())) logging.config.fileConfig(StringIO.StringIO(log_conf)) self.manager = Manager(mqtt_conf=conf['mqtt'])
def start(fqdn, dynamo_region, dynamo_host, table_root, log_config, verbosity, environment, server_config): """ Starts an APScheduler job to periodically reload HAProxy config as well as run the API to register/deregister new services, target groups and backends. :param fqdn: The fully qualified domain name of Flyby - requests coming here will go to the API endpoints :param dynamo_region: The AWS region of the DynamoDB tables Flyby stores and reads config in :param dynamo_host: The hostname and port to use for DynamoDB connections. Useful for local testing with moto or DynamoDB Local. :param table_root: The root that will be used for table names in DynamboDB. This will be prefixed to all tables created. :param log_config: Location of python yaml config file. :param verbosity: Logging verbosity, defaults to INFO. :return: """ logging.getLogger().setLevel(level=getattr(logging, verbosity)) dynamo_manager = DynamoTableManagement() config = ConfigObj(infile=server_config, configspec='flyby/configspec.ini', stringify=True) res = config.validate(Validator(), preserve_errors=True) if res is not True: for section, key, msg in flatten_errors(config, res): click.echo("{}: {} in {}".format(key, msg, section)) raise click.ClickException('bad server config') # Create the DynamoDB tables if missing, update the DynamoDB read/write capacity if required dynamo_manager.update_capacity(dynamo_host, dynamo_region, table_root, logger, config) if log_config: with open(log_config, 'r') as conf: logging.config.dictConfig(yaml.load(conf)) scheduler = BackgroundScheduler(timezone=utc) scheduler.add_job(update, 'interval', seconds=10, args=(fqdn,)) scheduler.start() if environment == "development": app.run(host='0.0.0.0') else: serve(app, listen='*:5000')
def init_config(configfiles=None): """ Initialize the config object with values from the configuration files. :param configfiles: A list of config files (will default to DEFAULT_CONFIG_FILES module global) :type configfiles: list """ global config if configfiles is None: configfiles = DEFAULT_CONFIG_FILES for fname in configfiles: if exists(fname): # This is a huge hack, but basically because I can't pass in a dict of "default" # values to ConfigObj constructure, I have to modify the actual "file" (well, the # list of lines from file) to prepend the value for 'root' else interpolation fails. # -1 for ConfigObj with open(fname) as fp: filelines = [line for line in fp] partial = configobj.ConfigObj(["root=%s\n" % ROOT_DIR] + filelines, encoding='utf-8') config.merge(partial) validation = config.validate(Validator(), preserve_errors=True) if validation != True: raise ConfigurationError("configuration validation error(s) (): {0!r}".format(configobj.flatten_errors(config, validation)))
def main(): global _logger parser = ArgumentParser(usage='prog [options]') parser.add_argument('-c', '--config', dest='config_file', required=True, metavar='CONFIG_FILE', help='The full path to the config file to open') parser.add_argument('-l', '--log-config', default='logger.conf', dest='log_config_file', metavar='LOG_CONFIG_FILE', help='The full path to the log config file to open') parser.add_argument('--start-frame-time', default=-1, type=int, dest='start_frame_pos', help='Start frame time in seconds') parser.add_argument('--end-frame-time', default=-1, type=int, dest='end_frame_pos', help='End frame time in seconds') parser.add_argument('--smooth-filter-size', default=3, type=int, dest='gaussian_filter_size', help='Gaussian filter kernel size') parser.add_argument('--smooth-filter-sigma', default=0, type=int, dest='gaussian_filter_sigma', help='Gaussian filter sigma') parser.add_argument('--nthreads', default=1, type=int, dest='nthreads') parser.add_argument('--nprocesses', default=1, type=int, dest='nprocesses', help='Number of processes to run in parallel') args = parser.parse_args() # setup logger logging.config.fileConfig(args.log_config_file) _logger = logging.getLogger('tracker') if args.config_file is None: _logger.warning('Missing config file') parser.exit(1, 'Missing config file\n') # load config file config, errors = load_config(args.config_file) errors |= set(config.validate()) if len(errors) == 0: if args.nprocesses > 1: source = MovieFile(config.get_source(), start_msecs=args.start_frame_pos * 1000, end_msecs=args.end_frame_pos * 1000, resolution=config.get_image_size()) if not source.is_opened(): _logger.error('Error opening %s' % config.get_source()) return start_frame_pos = int(source.get_start_time_in_seconds()) end_frame_pos = int(source.get_end_time_in_seconds()) frame_interval = int((end_frame_pos - start_frame_pos) / args.nprocesses) tracker_args = [(config, s * 1000, (s + frame_interval) * 1000, args.gaussian_filter_size, args.gaussian_filter_sigma, args.nthreads, _get_run_interval(s, s + frame_interval)[1]) for s in range(start_frame_pos, end_frame_pos, frame_interval) ] with Pool(args.nprocesses) as p: p.starmap(_run_tracker, tracker_args) else: _run_tracker(config, args.start_frame_pos * 1000, args.end_frame_pos * 1000, args.gaussian_filter_size, args.gaussian_filter_sigma, args.nthreads) else: _logger.error('Config load error: %r' % errors)
def load_instance_config(instance_name): def b(unicode): """ convert unicode to bytestring """ return unicode.encode('utf-8') confspec = [] confspec.append(b(u'[instance]')) confspec.append(b(u'source-directory = string()')) confspec.append(b(u'backup-directory = string()')) confspec.append(b(u'target-file = string()')) confspec.append(b(u'exchange = string(default="navitia")')) confspec.append(b(u'synonyms_file = string(default="")')) confspec.append(b(u'aliases_file = string(default="")')) confspec.append(b(u'name = string()')) confspec.append(b(u'is-free = boolean(default=False)')) confspec.append(b(u'[database]')) confspec.append(b(u'host = string()')) confspec.append(b(u'dbname = string()')) confspec.append(b(u'username = string()')) confspec.append(b(u'password = string()')) confspec.append(b(u'port = string(default="5432")')) ini_file = os.path.join( os.path.realpath(current_app.config['INSTANCES_DIR']), instance_name + ".ini") if not os.path.isfile(ini_file): raise ValueError("File doesn't exists or is not a file %s" % ini_file) config = ConfigObj(ini_file, configspec=confspec, stringify=True) val = Validator() res = config.validate(val, preserve_errors=True) # validate retourne true, ou un dictionaire ... if type(res) is dict: error = build_error(config, res) raise ValueError("Config is not valid: %s in %s" % (error, ini_file)) instance = InstanceConfig() instance.source_directory = config['instance']['source-directory'] instance.backup_directory = config['instance']['backup-directory'] instance.target_file = config['instance']['target-file'] instance.exchange = config['instance']['exchange'] instance.synonyms_file = config['instance']['synonyms_file'] instance.aliases_file = config['instance']['aliases_file'] instance.name = config['instance']['name'] instance.is_free = config['instance']['is-free'] instance.pg_host = config['database']['host'] instance.pg_dbname = config['database']['dbname'] instance.pg_username = config['database']['username'] instance.pg_password = config['database']['password'] instance.pg_port = config['database']['port'] return instance
def read_config(resources, path): """ build and validate a ConfigObj using config file found at given filesystem path """ _locate_config_files(path) # grab contents of config file with open(os.path.join(path, "mediarover.conf"), "r") as f: file = f.readlines() version_check = re.compile("__version__") # TODO the next time the user is required to regenerate the config file this code will no longer # be needed. Between version 1 and version 2, __version__ was moved into the __SYSTEM__ subsection version = 0 if version_check.search(file[0]): value = file[0] elif version_check.search(file[-1]): value = file[-1] if value: (left, sep, right) = value.partition("=") version = right.strip(" \n") # check if users config file is current if version > 0: if int(version) < int(__config_version__.get('min', __config_version__['version'])): raise ConfigurationError("Configuration file is out of date and needs to be regenerated! See `python mediarover.py write-configs --help` for instructions") else: raise ConfigurationError("Out of date or corrupt configuration file! See `python mediarover.py write-configs --help` for instructions") spec = os.path.join(resources, "config.spec") config = ConfigObj(file, configspec=spec) # validate config options results = config.validate(_get_validator(), preserve_errors=True) if results != True: results = flatten_errors(config, results) message = ["ERROR: Encountered the following configuration error(s):"] for error in results: level = 1 section = [] dict = config for key in error[0]: section.append(("[" * level) + key + ("]" * level)) dict = dict[key] level += 1 message.append(" %s %s = %s" % (" ".join(section), error[1], error[2])) raise ConfigurationError("Invalid Data in configuration file\n\n%s\n" % "\n".join(message)) return config
def load_instance_config(instance_name): confspec = [] confspec.append('[instance]') confspec.append('source-directory = string()') confspec.append('backup-directory = string()') confspec.append('tmp-file = string()') confspec.append('target-file = string()') confspec.append('rt-topics = string_list()') confspec.append('exchange = string(default="navitia")') confspec.append('synonyms_file = string(default="")') confspec.append('aliases_file = string(default="")') confspec.append('name = string()') confspec.append('is-free = boolean(default=False)') confspec.append('[database]') confspec.append('host = string()') confspec.append('dbname = string()') confspec.append('username = string()') confspec.append('password = string()') ini_file = '%s/%s.ini' % \ (current_app.config['INSTANCES_DIR'], instance_name) if not os.path.isfile(ini_file): raise ValueError("File doesn't exists or is not a file %s" % ini_file) config = ConfigObj(ini_file, configspec=confspec, stringify=True) val = Validator() res = config.validate(val, preserve_errors=True) #validate retourne true, ou un dictionaire ... if type(res) is dict: error = build_error(config, res) raise ValueError("Config is not valid: %s in %s" % (error, ini_file)) instance = InstanceConfig() instance.source_directory = config['instance']['source-directory'] instance.backup_directory = config['instance']['backup-directory'] instance.tmp_file = config['instance']['tmp-file'] instance.target_file = config['instance']['target-file'] instance.rt_topics = config['instance']['rt-topics'] instance.exchange = config['instance']['exchange'] instance.synonyms_file = config['instance']['synonyms_file'] instance.aliases_file = config['instance']['aliases_file'] instance.name = config['instance']['name'] instance.is_free = config['instance']['is-free'] instance.pg_host = config['database']['host'] instance.pg_dbname = config['database']['dbname'] instance.pg_username = config['database']['username'] instance.pg_password = config['database']['password'] return instance
def _validate_config(config, validator, filename): """ validate the given config object using the given validator """ results = config.validate(validator, preserve_errors=True) if results != True: results = flatten_errors(config, results) message = ["ERROR: Encountered the following error(s) in %s:" % filename] for error in results: level = 1 section = [] dict = config for key in error[0]: section.append(("[" * level) + key + ("]" * level)) dict = dict[key] level += 1 message.append(" %s %s = %s" % (" ".join(section), error[1], error[2])) raise ConfigurationError("Invalid Data in configuration file\n\n%s\n" % "\n".join(message))
def LoadIni(FilePath,ConfigSpecPath=None): config = ConfigObj(FilePath,configspec=ConfigSpecPath) validator = Validator() if ConfigSpecPath!=None: result = config.validate(validator) return config
"handlers": ["file_scheduler", "console"], "level": logging.INFO, }, "debug": {"handlers": ["file_debug"], "level": logging.DEBUG}, "atelier": {"handlers": ["file", "console"], "level": logging.DEBUG}, "server": {"handlers": ["file_server"], "level": logging.INFO}, }, ) ) logger = logging.getLogger("atelier") server_logger = logging.getLogger("server") try: config.validate() except Exception as e: server_logger.error("Configuration invalide.", exc_info=e) raise e from server import alerts, arduino, db, scheduler, web def remove_logging_handler(logger, handler_name): for handler in logger.handlers: if handler.name == handler_name: logger.handlers.remove(handler) return