def test_config_strict_mode_errors(): config.register(Item, strict_mode=True) test_config_path = testing.relative_module_path(__file__, 'test_config.yaml') with pytest.raises(ConfigLoadError, match='missing type declaration'): config.load_config(test_config_path)
def test_config_unsafe_object_creation(): from ruamel import yaml import calendar test_config_path = testing.relative_module_path(__file__, 'test_config_unsafe.yaml') # Unsafe default loader allows any python object initialization data = yaml.load(open(test_config_path), Loader=yaml.Loader) assert 'time' in data assert isinstance(data['time'], calendar.Calendar) # With use safe version only to allow construct previously registered objects with pytest.raises(config.ConfigLoadError, match='could not determine a constructor'): config.load_config(test_config_path)
def main(config): # Initialize logging subsystem. log_levels = {} log_levels.setdefault(DEFAULT_MODULE, 'info') logger.configure_loggers_from_dict(log_levels) log.warning('This software is pre-production and should not be deployed to production servers!') log.debug('started PID=%r', os.getpid()) log.info('Starting wca-scheduler.') # Initialize all necessary objects. register_algorithms() register_dataproviders() try: configuration = load_config(config) except ConfigLoadError as e: log.error('Error: Cannot load config file! : %s', e) exit(1) # Configure loggers using configuration file. if 'loggers' in configuration: log_levels_config = configuration['loggers'] if not isinstance(log_levels, dict): log.error('Loggers configuration error: log levels are mapping from logger name to' 'log level!') exit(1) log_levels = dict(log_levels, **log_levels_config) logger.configure_loggers_from_dict(log_levels) server = Server(configuration) return server.app
def allocate( self, platform: Platform, tasks_data: TasksData, ) -> (TasksAllocations, List[Anomaly], List[Metric]): rules = [] if self.rules: rules.extend(self.rules) if self.config: if not os.path.exists(self.config): log.warning( 'StaticAllocator: cannot find config file %r - ignoring!', self.config) else: rules.extend(load_config(self.config)) if len(rules) == 0: log.warning('StaticAllocator: no rules were provided!') return {}, [], [] log.log(TRACE, 'StaticAllocator: handling allocations for %i tasks. ', len(tasks_data)) for task, data in tasks_data.items(): log.log( TRACE, '%s', ' '.join('%s=%s' % (k, v) for k, v in sorted(data.labels.items()))) tasks_allocations = _build_allocations_from_rules(tasks_data, rules) log.debug('StaticAllocator: final tasks allocations: \n %s', pprint.pformat(tasks_allocations)) return tasks_allocations, [], []
def test_config_with_env_class(): test_config_path = testing.relative_module_path(__file__, 'test_config_env.yaml') data = config.load_config(test_config_path) user = data['from_env'] assert user == os.environ["USER"] assert user.strip() != '' assert data['unset_env'] == ''
def __post_init__(self): init_logging('info', 'Tester') self.testcases = load_config(self.config)['tests'] self.testcases_count = len(self.testcases) self.current_iteration = 0 self.processes: Dict[str, subprocess.Popen] = {} self.tasks: List[Task] = [] # To keep between consecutive get_tasks calls. self.metrics = [] self.checks = []
def main(): # Parse file argument parser = argparse.ArgumentParser() parser.add_argument('-c', '--config', required=True, help='Configuration file') args = parser.parse_args() # Initialize logging subsystem. log_levels = {} log_levels.setdefault(DEFAULT_MODULE, 'info') logger.configure_loggers_from_dict(log_levels) log.warning( 'This software is pre-production and should not be deployed to production servers!' ) log.debug('started PID=%r', os.getpid()) log.info('Starting wca-admission-controller.') register_components() try: configuration = load_config(args.config) except ConfigLoadError as e: log.error('Cannot load config file! : %s', e) exit(1) if 'loggers' in configuration: log_levels_config = configuration['loggers'] if not isinstance(log_levels, dict): log.error( 'Loggers configuration error: log levels are mapping from logger name to' 'log level!') exit(1) log_levels = dict(log_levels, **log_levels_config) logger.configure_loggers_from_dict(log_levels) annotating_service = AnnotatingService(configuration) annotating_service.app.run( host="0.0.0.0", ssl_context=( './ssl/server-cert.pem', # nosec './ssl/server-key.pem')) # nosec
def test_dataclass(): test_config_path = testing.relative_module_path(__file__, 'test_dataclasses.yaml') data = config.load_config(test_config_path) dc1 = data['dc1'] assert dc1.x == 5 assert dc1.y is None assert dc1.z == [1, 2, 3] assert dc1.d == 'asd' assert dc1.e == '/abc/def' assert dc1.foo == 'foobaz' dc2 = data['dc2'] assert dc2.x == 1 assert dc2.y == 'newble' assert dc2.z == [3, 4, 5] assert dc2.d == 4 assert dc2.foo == FooEnum.BAR
def test_config_with_simple_classes(): # Another method for registering items (other than using decorator). config.register(Item, strict_mode=False) test_config_path = testing.relative_module_path(__file__, 'test_config.yaml') data = config.load_config(test_config_path) foo_with_defaults = data['foo_with_defaults'] assert foo_with_defaults.f == 1 empty_boo = data['empty_boo'] assert empty_boo.foo is None foo = data['foo'] boo = data['boo'] assert foo.s == 'some_string' assert foo.f == 2.5 assert boo.foo is foo assert len(boo.items) == 2 assert isinstance(boo.items[0], Item)
def allocate( self, platform: Platform, tasks_measurements: TasksMeasurements, tasks_resources: TasksResources, tasks_labels: TasksLabels, tasks_allocations: TasksAllocations, ) -> (TasksAllocations, List[Anomaly], List[Metric]): rules = [] if self.rules: rules.extend(self.rules) if self.config: if not os.path.exists(self.config): log.warning('StaticAllocator: cannot find config file %r - ignoring!', self.config) else: rules.extend(load_config(self.config)) if len(rules) == 0: log.warning('StaticAllocator: no rules were provided!') return {}, [], [] # Merge all tasks ids. all_tasks_ids = (set(tasks_labels.keys()) | set(tasks_resources.keys()) | set(tasks_allocations.keys())) log.log(TRACE, 'StaticAllocator: handling allocations for %i tasks. ', len(all_tasks_ids)) for task_id, labels in tasks_labels.items(): log.log(TRACE, '%s', ' '.join('%s=%s' % (k, v) for k, v in sorted(labels.items()))) tasks_allocations = _build_allocations_from_rules(all_tasks_ids, tasks_labels, rules) log.debug('StaticAllocator: final tasks allocations: \n %s', pprint.pformat(tasks_allocations)) return tasks_allocations, [], []
def test_dataclass_validation(): test_config_path = testing.relative_module_path( __file__, 'test_dataclasses_validation.yaml') data = config.load_config(test_config_path) assert data['dc1'].rr == [[2, 3], []]
def test_dataclass_exceed_maximum(): test_config_path = testing.relative_module_path( __file__, 'test_dataclasses_exceed_maximum.yaml') with pytest.raises(config.ConfigLoadError, match="field 'x'.*Maximum value is 5. Got 123."): config.load_config(test_config_path)
def test_dataclass_invalid_field(): test_config_path = testing.relative_module_path( __file__, 'test_dataclasses_invalid_field.yaml') with pytest.raises(config.ConfigLoadError, match="constructor signature"): config.load_config(test_config_path)
def test_invalid_dataclass(): test_config_path = testing.relative_module_path( __file__, 'test_dataclasses_invalid.yaml') with pytest.raises(config.ConfigLoadError, match="field 'x'.*Invalid type"): config.load_config(test_config_path)
def test_dataclass_validation_invalid_dict(): test_config_path = testing.relative_module_path( __file__, 'test_dataclasses_validation_invalid_dict.yaml') with pytest.raises(ConfigLoadError, match='wrong_value'): config.load_config(test_config_path)
def main(): parser = argparse.ArgumentParser() parser.add_argument('-c', '--config', help="Configuration", default=None, required=True) parser.add_argument( '-l', '--log-level', help='Log level for modules (by default for wca) in [module:]level form,' 'where level can be one of: CRITICAL,ERROR,WARNING,INFO,DEBUG,TRACE' 'Example -l debug -l example:debug. Defaults to wca:INFO.' 'Can be overridden at runtime with config.yaml "loggers" section.', default=[], action='append', dest='levels', ) parser.add_argument('-r', '--register', action='append', dest='components', help="Register additional components in config", default=[]) parser.add_argument('-v', '--version', action='version', version=platforms.get_wca_version(), help="Show version") parser.add_argument('-0', '--root', help="Allow WCA process to be run using root account", dest='is_root_allowed', action='store_true') args = parser.parse_args() # Do not allow to run WCA with root privileges unless user indicates that it is intended. uid = os.geteuid() if uid == 0 and not args.is_root_allowed: log.fatal( "Do not run WCA with root privileges. Consult documentation " "to understand what capabilities are required. If root account " "has to be used then set --root/-0 argument to override.") exit(2) # Initialize logging subsystem from command line options. log_levels = logger.parse_loggers_from_list(args.levels) log_levels_copy_with_default = dict(**log_levels) log_levels_copy_with_default.setdefault(logger.DEFAULT_MODULE, 'info') logger.configure_loggers_from_dict(log_levels_copy_with_default) log.warning( 'This software is pre-production and should not be deployed to production servers.' ) log.debug('started PID=%r', os.getpid()) log.info('Version wca: %s', platforms.get_wca_version()) # Register internal & external components. components.register_components(extra_components=args.components) valid_config_file(args.config) # Initialize all necessary objects. try: configuration = config.load_config(args.config) except config.ConfigLoadError as e: log.error('Error: Cannot load config file! : %s', e) if log.getEffectiveLevel() <= logging.DEBUG: log.exception('Detailed exception:') exit(1) for key in configuration: if key != 'loggers' and key != 'runner': log.error('Error: Unknown fields in configuration ' 'file! Possible are: \'loggers\', ' '\'runner\'') exit(1) assure_type(configuration, dict) assert 'runner' in configuration, 'Improper config - missing runner instance!' # Configure loggers using configuration file. if 'loggers' in configuration: log_levels_config = configuration['loggers'] if not isinstance(log_levels, dict): log.error( 'Loggers configuration error: log levels are mapping from logger name to' 'log level!') exit(1) # Merge config from cmd line and config file. # Overwrite config file values with values provided from command line. log_levels = dict(log_levels_config, **log_levels) logger.configure_loggers_from_dict(log_levels) # Dump loggers configurations to debug issues with loggers. if os.environ.get('WCA_DUMP_LOGGERS') == 'True': print( '------------------------------------ Logging tree ---------------------' ) import logging_tree logging_tree.printout() print( '------------------------------------ Logging tree END------------------' ) # Extract main loop component. runner = configuration['runner'] assure_type(runner, Runner) # Prepare and run the "main loop". exit_code = runner.run() exit(exit_code)
def test_invalid_dataclass_union(): test_config_path = testing.relative_module_path( __file__, 'test_dataclasses_invalid_union.yaml') with pytest.raises(config.ConfigLoadError, match="field 'd'.*improper type from union"): config.load_config(test_config_path)