class ValidateConfig(Validator): """Given a parsed config file (should be only basic literals and containers), return an immutable, fully populated series of namedtuples and FrozenDicts with all defaults filled in, all valid values, and no unused values. Throws a ConfigError if any part of the input dict is invalid. """ config_class = TronConfig defaults = { 'action_runner': {}, 'output_stream_dir': None, 'command_context': {}, 'ssh_options': ValidateSSHOptions.defaults, 'notification_options': None, 'time_zone': None, 'state_persistence': DEFAULT_STATE_PERSISTENCE, 'nodes': {'localhost': DEFAULT_NODE}, 'node_pools': {}, 'jobs': (), 'services': (), } node_pools = build_dict_name_validator(valid_node_pool, allow_empty=True) nodes = build_dict_name_validator(valid_node, allow_empty=True) validators = { 'action_runner': ValidateActionRunner(), 'output_stream_dir': valid_output_stream_dir, 'command_context': valid_command_context, 'ssh_options': valid_ssh_options, 'notification_options': valid_notification_options, 'time_zone': valid_time_zone, 'state_persistence': valid_state_persistence, 'nodes': nodes, 'node_pools': node_pools, } optional = False def validate_node_pool_nodes(self, config): """Validate that each node in a node_pool is in fact a node, and not another pool. """ all_node_names = set(config['nodes']) for node_pool in config['node_pools'].itervalues(): invalid_names = set(node_pool.nodes) - all_node_names if invalid_names: msg = "NodePool %s contains other NodePools: " % node_pool.name raise ConfigError(msg + ",".join(invalid_names)) def post_validation(self, config, _): """Validate a non-named config.""" node_names = config_utils.unique_names( 'Node and NodePool names must be unique %s', config['nodes'], config.get('node_pools', [])) if config.get('node_pools'): self.validate_node_pool_nodes(config) config_context = ConfigContext('config', node_names, config.get('command_context'), MASTER_NAMESPACE) validate_jobs_and_services(config, config_context)
def validate_jobs_and_services(config, config_context): """Validate jobs and services.""" valid_jobs = build_dict_name_validator(valid_job, allow_empty=True) valid_services = build_dict_name_validator(valid_service, allow_empty=True) validation = [('jobs', valid_jobs), ('services', valid_services)] for config_name, valid in validation: child_context = config_context.build_child_context(config_name) config[config_name] = valid(config.get(config_name, []), child_context) fmt_string = 'Job and Service names must be unique %s' config_utils.unique_names(fmt_string, config['jobs'], config['services'])
def validate_jobs_and_services(config, config_context): """Validate jobs and services.""" valid_jobs = build_dict_name_validator(valid_job, allow_empty=True) valid_services = build_dict_name_validator(valid_service, allow_empty=True) validation = [('jobs', valid_jobs), ('services', valid_services)] for config_name, valid in validation: child_context = config_context.build_child_context(config_name) config[config_name] = valid(config.get(config_name, []), child_context) fmt_string = 'Job and Service names must be unique %s' config_utils.unique_names(fmt_string, config['jobs'], config['services'])
class ValidateJob(Validator): """Validate jobs.""" config_class = ConfigJob defaults = { 'run_limit': 50, 'all_nodes': False, 'cleanup_action': None, 'enabled': True, 'queueing': True, 'allow_overlap': False, 'max_runtime': None, } validators = { 'name': valid_name_identifier, 'schedule': valid_schedule, 'run_limit': valid_int, 'all_nodes': valid_bool, 'actions': build_dict_name_validator(valid_action), 'cleanup_action': valid_cleanup_action, 'node': valid_node_name, 'queueing': valid_bool, 'enabled': valid_bool, 'allow_overlap': valid_bool, 'max_runtime': config_utils.valid_time_delta, } def cast(self, in_dict, config_context): in_dict['namespace'] = config_context.namespace return in_dict # TODO: extract common code to a util function def _validate_dependencies(self, job, actions, base_action, current_action=None, stack=None): """Check for circular or misspelled dependencies.""" stack = stack or [] current_action = current_action or base_action stack.append(current_action.name) for dep in current_action.requires: if dep == base_action.name and len(stack) > 0: msg = 'Circular dependency in job.%s: %s' raise ConfigError(msg % (job['name'], ' -> '.join(stack))) if dep not in actions: raise ConfigError('Action jobs.%s.%s has a dependency "%s"' ' that is not in the same job!' % (job['name'], current_action.name, dep)) self._validate_dependencies(job, actions, base_action, actions[dep], stack) stack.pop() def post_validation(self, job, config_context): """Validate actions for the job.""" for action in job['actions'].itervalues(): self._validate_dependencies(job, job['actions'], action)
def validate_jobs_and_services(config, config_context): """Validate jobs and services.""" if 'command_context' in config: sorted_context = [(key,config['command_context'][key]) for key in \ sorted(config['command_context'], key=len, reverse=True)] for job in config['jobs']: tron_cmd = 'enable' if ('enabled' not in job or job['enabled']) \ else 'disable' #namespace_job = '%s.%s' % (config_context.namespace, # job['name'].replace(" ", "_")) #disable_pr = subprocess.Popen(['tronctl', tron_cmd, # namespace_job], stdout=subprocess.PIPE, # stderr=subprocess.PIPE) #disable_pr.communicate() if tron_cmd == 'disable': print '%s %sd' % (job['name'], tron_cmd) if 'report' in job: if job['report'] is True: job['email'] = 'report-%s' % job['email'] del job['report'] if 'node' not in job: job['node'] = 'be-master' if 'actions' not in job: job['actions'] = [{}] job['actions'][0]['command'] = job['command'] job['actions'][0]['name'] = job['email'] \ if 'email' in job else 'none' for var, repl_var in sorted_context: old_cmd = job['actions'][0]['command'] if var in old_cmd: job['actions'][0]['command'] = old_cmd.replace( var, repl_var) continue valid_jobs = build_dict_name_validator(valid_job, allow_empty=True) valid_services = build_dict_name_validator(valid_service, allow_empty=True) validation = [('jobs', valid_jobs), ('services', valid_services)] for config_name, valid in validation: child_context = config_context.build_child_context(config_name) config[config_name] = valid(config.get(config_name, []), child_context) fmt_string = 'Job and Service names must be unique %s' config_utils.unique_names(fmt_string, config['jobs'], config['services'])
def validate_jobs_and_services(config, config_context): """Validate jobs and services.""" if 'command_context' in config: sorted_context = [(key,config['command_context'][key]) for key in \ sorted(config['command_context'], key=len, reverse=True)] for job in config['jobs']: tron_cmd = 'enable' if ('enabled' not in job or job['enabled']) \ else 'disable' #namespace_job = '%s.%s' % (config_context.namespace, # job['name'].replace(" ", "_")) #disable_pr = subprocess.Popen(['tronctl', tron_cmd, # namespace_job], stdout=subprocess.PIPE, # stderr=subprocess.PIPE) #disable_pr.communicate() if tron_cmd == 'disable': print '%s %sd' % (job['name'], tron_cmd) if 'report' in job: if job['report'] is True: job['email'] = 'report-%s' % job['email'] del job['report'] if 'node' not in job: job['node'] = 'be-master' if 'actions' not in job: job['actions'] = [{}] job['actions'][0]['command'] = job['command'] job['actions'][0]['name'] = job['email'] \ if 'email' in job else 'none' for var, repl_var in sorted_context: old_cmd = job['actions'][0]['command'] if var in old_cmd: job['actions'][0]['command'] = old_cmd.replace(var, repl_var) continue valid_jobs = build_dict_name_validator(valid_job, allow_empty=True) valid_services = build_dict_name_validator(valid_service, allow_empty=True) validation = [('jobs', valid_jobs), ('services', valid_services)] for config_name, valid in validation: child_context = config_context.build_child_context(config_name) config[config_name] = valid(config.get(config_name, []), child_context) fmt_string = 'Job and Service names must be unique %s' config_utils.unique_names(fmt_string, config['jobs'], config['services'])