def _collect_component_configs(self, client, component,
args, default_config=None):
config_files = []
p = index(args, lambda s: s == '--config-file')
if p != -1 and p + 1 < len(args):
config_path = args[p + 1]
else:
config_path = '/etc/keystone/keystone.conf'
if config_path:
config_files.append(self._collect_file(client, config_path))
p = index(args, lambda s: s == '--config-dir')
if p != -1 and p + 1 < len(args):
result = client.run(['ls', '%s/*.conf' % args[p + 1]])
if result.return_code == 0:
for config_path in result.output.split("\n"):
config_files.extend(
self._collect_file(client, config_path))
component.config_files = config_files
for i, arg in enumerate(args):
if arg.startswith('--'):
name = arg[2:]
if '=' in name:
name, value = name.split('=', 1)
elif i + 1 < len(args):
value = args[i + 1]
i += 1
else:
continue
component.config.set_cli(name, value)
def get_schema(self, project, version, configname=None):
if not configname:
configname = '%s.conf' % project
fullname = '%s/%s' % (project, configname)
version = Version(version)
if not fullname in self.__schemas:
return None
records = self.__schemas[fullname]
i = len(records) - 1
# Find latest checkpoint prior given version
while i >= 0 and not (records[i].checkpoint
and records[i].version <= version):
i -= 1
if i < 0:
return None
parameters = []
seen_parameters = set()
last_version = None
while i < len(records) and records[i].version <= version:
last_version = records[i].version
for param in records[i].adds:
if param.name in seen_parameters:
old_param_index = index(
parameters,
lambda p: p.name == param.name)
if old_param_index != -1:
parameters[old_param_index] = param
else:
parameters.append(param)
seen_parameters.add(param.name)
for param_name in records[i].removals:
param_index = index(
parameters,
lambda p: p.name == param_name)
if index != -1:
parameters.pop(param_index)
seen_parameters.remove(param_name)
i += 1
return ConfigSchema(fullname, last_version, 'ini', parameters)
def get_schema(self, project, version, configname=None):
if not configname:
configname = '%s.conf' % project
fullname = '%s/%s' % (project, configname)
version = Version(version)
if not fullname in self.__schemas:
return None
records = self.__schemas[fullname]
i = len(records) - 1
# Find latest checkpoint prior given version
while i >= 0 and not (records[i].checkpoint
and records[i].version <= version):
i -= 1
if i < 0:
return None
parameters = []
seen_parameters = set()
last_version = None
while i < len(records) and records[i].version <= version:
last_version = records[i].version
for param in records[i].adds:
if param.name in seen_parameters:
old_param_index = index(parameters,
lambda p: p.name == param.name)
if old_param_index != -1:
parameters[old_param_index] = param
else:
parameters.append(param)
seen_parameters.add(param.name)
for param_name in records[i].removals:
param_index = index(parameters, lambda p: p.name == param_name)
if index != -1:
parameters.pop(param_index)
seen_parameters.remove(param_name)
i += 1
return ConfigSchema(fullname, last_version, 'ini', parameters)
def collect_component_configs(driver, client, component,
command, default_config=None):
config_files = []
args = shlex.split(command)[1:]
p = index(args, lambda s: s == '--config-file')
if p != -1 and p + 1 < len(args):
config_path = args[p + 1]
else:
config_path = default_config
if config_path:
r = driver.discover('file', client.host, path=config_path)
if r:
config_files.append(r)
p = index(args, lambda s: s == '--config-dir')
if p != -1 and p + 1 < len(args):
files = driver.discover('file', client.host, path='%s/*.conf' % args[p + 1])
if files:
if not isinstance(files, list):
files = [files]
config_files.extend(files)
component.config_files = config_files
for i, arg in enumerate(args):
if arg.startswith('--'):
name = arg[2:]
if '=' in name:
name, value = name.split('=', 1)
elif i + 1 < len(args):
value = args[i + 1]
i += 1
else:
continue
component.config.set_cli(name, value)
def _collect_component_configs(self,
client,
component,
args,
default_config=None):
config_files = []
p = index(args, lambda s: s == '--config-file')
if p != -1 and p + 1 < len(args):
config_path = args[p + 1]
else:
config_path = '/etc/keystone/keystone.conf'
if config_path:
config_files.append(self._collect_file(client, config_path))
p = index(args, lambda s: s == '--config-dir')
if p != -1 and p + 1 < len(args):
result = client.run(['ls', '%s/*.conf' % args[p + 1]])
if result.return_code == 0:
for config_path in result.output.split("\n"):
config_files.extend(self._collect_file(
client, config_path))
component.config_files = config_files
for i, arg in enumerate(args):
if arg.startswith('--'):
name = arg[2:]
if '=' in name:
name, value = name.split('=', 1)
elif i + 1 < len(args):
value = args[i + 1]
i += 1
else:
continue
component.config.set_cli(name, value)
def validate_network_mask(s):
# TODO(someone): implement proper checking
result = validate_ipv4_address(s)
if isissue(result):
return result
parts = [int(p) for p in result.split(".", 3)]
x = index(parts, lambda p: p != 255)
if x == -1:
return result
if parts[x] not in [0, 128, 192, 224, 240, 248, 252, 254]:
return InvalidValueError("Invalid netmask")
x += 1
while x < 4:
if parts[x] != 0:
return InvalidValueError("Invalid netmask")
return result
def get_schema(self, project, version, configname=None, schema_loader=ConfigSchemaLoader()):
if not configname:
configname = "%s.conf" % project
fullname = "%s/%s" % (project, configname)
version = Version(version)
records = schema_loader.load(project, configname)
if not records:
return None
i = len(records) - 1
# Find latest checkpoint prior given version
while i >= 0 and not (records[i].get("checkpoint", False) and Version(records[i]["version"]) <= version):
i -= 1
if i < 0:
if Version(records[0]["version"]) > version:
# Reached the earliest record yet haven't found version
return None
# Haven't found checkpoint but yearliest version is less than given
# Assuming first record is checkpoint
i = 0
parameters = []
seen_parameters = set()
last_version = None
while i < len(records) and Version(records[i]["version"]) <= version:
last_version = records[i]["version"]
for param_data in records[i].get("added", []):
name = param_data["name"]
section = None
if "." in name:
section, name = name.split(".", 1)
param = ConfigParameterSchema(
name,
param_data["type"],
section=section,
type_args=param_data.get("type_args", {}),
default=param_data.get("default", None),
description=param_data.get("help", None),
required=param_data.get("required", False),
deprecation_message=param_data.get("deprecated", None),
)
if param.name in seen_parameters:
old_param_index = index(parameters, lambda p: p.name == param.name)
if old_param_index != -1:
parameters[old_param_index] = param
else:
parameters.append(param)
seen_parameters.add(param.name)
for param_name in records[i].get("removed", []):
param_index = index(parameters, lambda p: p.name == param_name)
if index != -1:
parameters.pop(param_index)
seen_parameters.discard(param_name)
i += 1
return ConfigSchema(fullname, last_version, "ini", parameters)
def get_schema(self, project, version, configname=None):
if not configname:
configname = '%s.conf' % project
fullname = '%s/%s' % (project, configname)
version = Version(version)
path = os.path.join(self.db_path, project, configname + '.yml')
if not os.path.exists(path):
return None
with open(path) as f:
records = yaml.load(f.read())
i = len(records) - 1
# Find latest checkpoint prior given version
while i >= 0 and not (records[i].get('checkpoint', False)
and Version(records[i]['version']) <= version):
i -= 1
if i < 0:
if Version(records[0]['version']) > version:
# Reached the earliest record yet haven't found version
return None
# Haven't found checkpoint but yearliest version is less than given
# Assuming first record is checkpoint
i = 0
parameters = []
seen_parameters = set()
last_version = None
while i < len(records) and Version(records[i]['version']) <= version:
last_version = records[i]['version']
for param_data in records[i].get('added', []):
name = param_data['name']
section = None
if '.' in name:
section, name = name.split('.', 1)
param = ConfigParameterSchema(
name, param_data['type'], section=section,
type_args=param_data.get('type_args', {}),
default=param_data.get('default', None),
description=param_data.get('help', None),
required=param_data.get('required', False),
deprecation_message=param_data.get('deprecated', None))
if param.name in seen_parameters:
old_param_index = index(
parameters,
lambda p: p.name == param.name)
if old_param_index != -1:
parameters[old_param_index] = param
else:
parameters.append(param)
seen_parameters.add(param.name)
for param_name in records[i].get('removed', []):
param_index = index(
parameters,
lambda p: p.name == param_name)
if index != -1:
parameters.pop(param_index)
seen_parameters.discard(param_name)
i += 1
return ConfigSchema(fullname, last_version, 'ini', parameters)
def generate_project_schema(project):
logger.info('Processing project %s' % project)
project_path = os.path.join(os.path.dirname(__file__), project)
files = glob.glob(os.path.join(project_path, '*.yml'))
if files == []:
logger.info("Found no YAML files in project %s. Skipping it" % project)
return
x = index(files, lambda f: f.endswith('.conf.yml'))
if x != -1:
database_file = files[x]
del files[x]
else:
database_file = os.path.join(project_path, project + '.conf.yml')
schema_records = []
if os.path.exists(database_file):
logger.debug("Processing database file %s" % database_file)
with open(database_file) as f:
schema_records.extend(yaml.load(f.read()))
schema_versions = []
for version_file in files:
logger.debug("Processing version file %s" % version_file)
with open(version_file) as f:
schema_versions.append(yaml.load(f.read()))
schema_versions = sorted(schema_versions,
key=lambda s: Version(s['version']))
parameters = OrderedDict()
for schema in schema_versions:
added = []
seen = set()
logger.debug('Processing schema version %s' % schema['version'])
for param in schema['parameters']:
# TODO(mkulkin): reduce the level of nesting
prev_param = parameters.get(param['name'], None)
if not prev_param:
logger.debug('Parameter %s does not exist yet,'
' adding it as new'
% param['name'])
added.append(param)
else:
seen.add(param['name'])
if param['type'] != prev_param['type']:
validator = TypeRegistry.get_validator(prev_param['type'])
if param['type'] == validator.base_type:
param['type'] = prev_param['type']
if param.get('default', None) is not None:
type_args = param.get('type_args', {})
value = validator.validate(param['default'], **type_args)
if not isinstance(value, Issue):
param['default'] = value
else:
logger.error("In project '%s' version %s"
" default value for parameter"
" '%s' is not valid value of"
" type %s: %s"
% (project, schema['version'],
param['name'], param['type'],
repr(param['default'])))
else:
logger.debug('Parameter %s type has'
' changed from %s to %s' %
(param['name'], prev_param['type'],
param['type']))
param['comment'] = 'Type has changed'
added.append(param)
continue
if param.get('default', None) != \
prev_param.get('default', None):
logger.debug('Parameter %s default value'
' has changed from %s to %s' %
(param['name'], prev_param['default'],
param['default']))
param['comment'] = 'Default value has changed'
added.append(param)
continue
if param.get('help', None) != prev_param.get('help', None):
param['comment'] = 'Help string has changed'
added.append(param)
removed = [name for name in parameters.keys() if name not in seen]
if len(removed) > 0:
logger.debug('Following parameters from previous'
' schema version are not present in'
' current version, marking as removed: %s'
% ','.join(removed))
# Decide either to use full schema update or incremental
changes_count = sum(map(len, [added, removed]))
logger.debug('Found %d change(s) from previous version schema'
% changes_count)
if changes_count > int(len(parameters) * DIFF_THRESHOLD):
logger.debug('Using full schema update')
new_parameters = parameters.copy()
for param in added:
new_parameters[param['name']] = param
for name in removed:
del new_parameters[name]
new_schema_record = dict(version=schema['version'],
added=new_parameters.values(),
removed=[],
checkpoint=True)
else:
logger.debug('Using incremental schema update')
new_schema_record = dict(version=schema['version'],
added=added, removed=removed)
# Place schema record either replacing existing one or appending as new
old_schema_record_idx = index(schema_records, lambda r:
str(r['version']) ==
str(new_schema_record['version']))
if old_schema_record_idx != -1:
old_schema_record = schema_records[old_schema_record_idx]
# Collect information from existing records
old_schema_parameters = {}
for param in old_schema_record.get('added', []):
old_schema_parameters[param['name']] = param
for param in added:
old_param = old_schema_parameters.get(param['name'], None)
if not old_param:
param.setdefault('comment', 'New param')
continue
extra_data = [(k, v) for k, v in old_param.items()
if k not in ['name', 'type', 'default', 'help']]
param.update(extra_data)
validator = TypeRegistry.get_validator(old_param['type'])
if param['type'] not in [old_param['type'],
validator.base_type]:
param['comment'] = 'Type has changed'
# Type has changed, enforcing old type to prevent
# accidental data loss
param['type'] = old_param['type']
if 'default' in old_param:
param['default'] = old_param['default']
if param.get('default', None) is not None:
type_args = old_param.get('type_args', {})
value = validator.validate(old_param['default'], **type_args)
if not isinstance(value, Issue):
param['default'] = value
else:
logger.error("In project '%s' version %s default value"
" for parameter '%s' is not valid value"
" of type %s: %s" %
(project, schema['version'],
param['name'], param['type'],
repr(param['default'])))
if param.get('default', None) != old_param.get('default',
None):
param['comment'] = 'Default value has changed'
continue
logger.debug('Replacing schema record %s'
% repr(new_schema_record))
schema_records[old_schema_record_idx] = new_schema_record
else:
for param in added:
param.setdefault('comment', 'New param')
logger.debug('Appending schema record %s'
% repr(new_schema_record))
schema_records.append(new_schema_record)
# Update parameter info
for param in new_schema_record.get('added', []):
parameters[param['name']] = param
for name in new_schema_record.get('removed', []):
del parameters[name]
schema_records = sorted(schema_records,
key=lambda r: Version(r['version']))
with open(database_file, 'w') as f:
f.write(yaml_dump_schema_records(schema_records))
