def process_files(dirname, files):
node_groups = []
clusters = []
plugin_name = get_plugin_name()
try:
for fname in files:
if os.path.splitext(fname)[1] == ".json":
fpath = os.path.join(dirname, fname)
with open(fpath, "r") as fp:
try:
template = json.load(fp)
except ValueError as e:
LOG.warning("Error processing {path}, {reason}".format(path=fpath, reason=e))
raise Handled("error processing files")
# If this file doesn't contain basic fields, skip it.
# If we are filtering on plugin and version make
# sure the file is one that we want
if not u.check_basic_fields(template) or (
not u.check_plugin_name_and_version(template, plugin_name, CONF.command.plugin_version)
):
continue
# Look through the sections in CONF and register
# options for this template if we find a section
# related to the template (ie, plugin, version, name)
section = add_config_section_for_template(template)
LOG.debug("Using config section {section} " "for {path}".format(section=section, path=fpath))
# Attempt to resolve substitutions using the config section
substitute_config_values(get_configs(section), template, fpath)
file_entry = {"template": template, "path": fpath}
if u.is_node_group(template):
# JSON validator
try:
ng_validator.validate(template)
except jsonschema.ValidationError as e:
LOG.warning("Validation for {path} failed, " "{reason}".format(path=fpath, reason=e))
raise Handled("node group template validation failed")
node_groups.append(file_entry)
LOG.debug("Added {path} to node group " "template files".format(path=fpath))
else:
clusters.append(file_entry)
LOG.debug("Added {path} to cluster template " "files".format(path=fpath))
except Handled as e:
log_skipping_dir(dirname, e.message)
node_groups = []
clusters = []
except Exception as e:
log_skipping_dir(dirname, "unhandled exception, {reason}".format(reason=e))
node_groups = []
clusters = []
return node_groups, clusters
def add_config_section_for_template(template):
'''Register a config section based on the template values
Check to see if the configuration files contain a section
that corresponds to the template. If an appropriate section
can be found, register options for the template so that the
config values can be read and applied to the template via
substitution (oslo supports registering groups and options
at any time, before or after the config files are parsed).
Corresponding section names may be of the following forms:
<template_name>, example "hdp-2.0.6-master"
This is useful when a template naming convention is being used,
so that the template name is already unambiguous
<plugin_name>_<hadoop_version>_<template_name>, example "hdp_2.0.6_master"
This can be used if there is a name collision between templates
<plugin_name>_<hadoop_version>, example "hdp_2.0.6"
<plugin_name>, example "hdp"
DEFAULT
Sections are tried in the order given above.
Since the first two section naming forms refer to a specific
template by name, options are added based on template type.
However, the other section naming forms may map to node group templates
or cluster templates, so options for both are added.
'''
sections = list(CONF.list_all_sections())
unique_name = "{name}".format(**template)
fullname = "{plugin_name}_{hadoop_version}_{name}".format(**template)
plugin_version = "{plugin_name}_{hadoop_version}".format(**template)
plugin = "{plugin_name}".format(**template)
section_name = None
if unique_name in sections:
section_name = unique_name
elif fullname in sections:
section_name = fullname
if section_name:
if u.is_node_group(template):
opts = node_group_template_opts
else:
opts = cluster_template_opts
else:
if plugin_version in sections:
section_name = plugin_version
elif plugin in sections:
section_name = plugin
opts = all_template_opts
add_config_section(section_name, opts)
return section_name
def add_config_section_for_template(template):
'''Register a config section based on the template values
Check to see if the configuration files contain a section
that corresponds to the template. If an appropriate section
can be found, register options for the template so that the
config values can be read and applied to the template via
substitution (oslo supports registering groups and options
at any time, before or after the config files are parsed).
Corresponding section names may be of the following forms:
<template_name>, example "hdp-2.0.6-master"
This is useful when a template naming convention is being used,
so that the template name is already unambiguous
<plugin_name>_<hadoop_version>_<template_name>, example "hdp_2.0.6_master"
This can be used if there is a name collision between templates
<plugin_name>_<hadoop_version>, example "hdp_2.0.6"
<plugin_name>, example "hdp"
DEFAULT
Sections are tried in the order given above.
Since the first two section naming forms refer to a specific
template by name, options are added based on template type.
However, the other section naming forms may map to node group templates
or cluster templates, so options for both are added.
'''
sections = list(CONF.list_all_sections())
unique_name = "{name}".format(**template)
fullname = "{plugin_name}_{hadoop_version}_{name}".format(**template)
plugin_version = "{plugin_name}_{hadoop_version}".format(**template)
plugin = "{plugin_name}".format(**template)
section_name = None
if unique_name in sections:
section_name = unique_name
elif fullname in sections:
section_name = fullname
if section_name:
if u.is_node_group(template):
opts = node_group_template_opts
else:
opts = cluster_template_opts
else:
if plugin_version in sections:
section_name = plugin_version
elif plugin in sections:
section_name = plugin
opts = all_template_opts
add_config_section(section_name, opts)
return section_name
def substitute_config_values(configs, template, path):
if u.is_node_group(template):
opt_names = node_group_template_opt_names
else:
opt_names = cluster_template_opt_names
for opt, value in six.iteritems(configs):
if opt in opt_names and opt in template:
template[opt] = value
def substitute_config_values(configs, template, path):
if u.is_node_group(template):
opt_names = node_group_template_opt_names
else:
opt_names = cluster_template_opt_names
for opt, value in six.iteritems(configs):
if opt in opt_names and opt in template:
template[opt] = value
def substitute_config_values(configs, template, path):
if u.is_node_group(template):
opt_names = node_group_template_opt_names
else:
opt_names = cluster_template_opt_names
for opt, value in six.iteritems(configs):
if opt in opt_names and opt in template:
if value is None:
template[opt] = None
else:
# Use args to allow for keyword arguments to format
args = {opt: value}
template[opt] = template[opt].format(**args)
def substitute_config_values(configs, template, path):
if u.is_node_group(template):
opt_names = node_group_template_opt_names
else:
opt_names = cluster_template_opt_names
for opt, value in six.iteritems(configs):
if opt in opt_names and opt in template:
if value is None:
# TODO(tmckay): someday if we support 'null' in JSON
# we should replace this value with None. json.load
# will replace 'null' with None, and sqlalchemy will
# accept None as a value for a nullable field.
del template[opt]
LOG.debug("No replacement value specified for {opt} in " "{path}, removing".format(opt=opt, path=path))
else:
# Use args to allow for keyword arguments to format
args = {opt: value}
template[opt] = template[opt].format(**args)
def substitute_config_values(configs, template, path):
if u.is_node_group(template):
opt_names = node_group_template_opt_names
else:
opt_names = cluster_template_opt_names
for opt, value in six.iteritems(configs):
if opt in opt_names and opt in template:
if value is None:
# TODO(tmckay): someday if we support 'null' in JSON
# we should replace this value with None. json.load
# will replace 'null' with None, and sqlalchemy will
# accept None as a value for a nullable field.
del template[opt]
LOG.debug("No replacement value specified for {opt} in "
"{path}, removing".format(opt=opt, path=path))
else:
# Use args to allow for keyword arguments to format
args = {opt: value}
template[opt] = template[opt].format(**args)
def process_files(dirname, files):
node_groups = []
clusters = []
plugin_name = get_plugin_name()
try:
for fname in files:
if os.path.splitext(fname)[1] == ".json":
fpath = os.path.join(dirname, fname)
with open(fpath, 'r') as fp:
try:
data = fp.read()
template = json.loads(data)
except ValueError as e:
LOG.warning(
_LW("Error processing {path}, "
"{reason}").format(path=fpath, reason=e))
raise Handled("error processing files")
# If this file doesn't contain basic fields, skip it.
# If we are filtering on plugin and version make
# sure the file is one that we want
if not u.check_basic_fields(template) or (
not u.check_plugin_name_and_version(
template, plugin_name,
CONF.command.plugin_version)):
continue
# Look through the sections in CONF and register
# options for this template if we find a section
# related to the template (ie, plugin, version, name)
section = add_config_section_for_template(template)
LOG.debug("Using config section {section} "
"for {path}".format(section=section, path=fpath))
# Attempt to resolve substitutions using the config section
substitute_config_values(get_configs(section), template,
fpath)
file_entry = {'template': template, 'path': fpath}
if u.is_node_group(template):
# JSON validator
try:
ng_validator.validate(template)
except jsonschema.ValidationError as e:
LOG.warning(
_LW("Validation for {path} failed, "
"{reason}").format(path=fpath, reason=e))
raise Handled(
"node group template validation failed")
node_groups.append(file_entry)
LOG.debug("Added {path} to node group "
"template files".format(path=fpath))
else:
clusters.append(file_entry)
LOG.debug("Added {path} to cluster template "
"files".format(path=fpath))
except Handled as e:
log_skipping_dir(dirname, str(e))
node_groups = []
clusters = []
except Exception as e:
log_skipping_dir(dirname,
"unhandled exception, {reason}".format(reason=e))
node_groups = []
clusters = []
return node_groups, clusters