def test_substitute_ng_ids(self):
cl = {
"node_groups": [
{"name": "worker", "node_group_template_id": "{vanilla-worker}", "count": 3},
{"name": "master", "node_group_template_id": "{vanilla-master}", "count": 1},
{"name": "secondary-name", "node_group_template_id": "some_id"},
]
}
ng_dict = {"vanilla-worker": 1, "vanilla-master": 2}
utils.substitute_ng_ids(cl, ng_dict)
self.assertEqual("1", cl["node_groups"][0]["node_group_template_id"])
self.assertEqual("2", cl["node_groups"][1]["node_group_template_id"])
self.assertEqual("some_id", cl["node_groups"][2]["node_group_template_id"])
def check_cluster_templates_valid(ng_templates, cl_templates):
# Check that if name references to node group templates
# are replaced with a uuid value that the cluster template
# passes JSON validation. We don't have the real uuid yet,
# but this will allow the validation test.
if ng_templates:
dummy_uuid = uuid.uuid4()
ng_ids = {ng["template"]["name"]: dummy_uuid for ng in ng_templates}
else:
ng_ids = {}
for cl in cl_templates:
template = copy.deepcopy(cl["template"])
u.substitute_ng_ids(template, ng_ids)
try:
ct_validator.validate(template)
except jsonschema.ValidationError as e:
LOG.warning("Validation for {path} failed, {reason}".format(path=cl["path"], reason=e))
return True
return False
def test_substitute_ng_ids(self):
cl = {"node_groups":
[{"name": "worker",
"node_group_template_id": "{vanilla-worker}",
"count": 3},
{"name": "master",
"node_group_template_id": "{vanilla-master}",
"count": 1},
{"name": "secondary-name",
"node_group_template_id": "some_id"}]}
ng_dict = {"vanilla-worker": 1,
"vanilla-master": 2}
utils.substitute_ng_ids(cl, ng_dict)
self.assertEqual("1", cl["node_groups"][0]["node_group_template_id"])
self.assertEqual("2", cl["node_groups"][1]["node_group_template_id"])
self.assertEqual("some_id",
cl["node_groups"][2]["node_group_template_id"])
def check_cluster_templates_valid(ng_templates, cl_templates):
# Check that if name references to node group templates
# are replaced with a uuid value that the cluster template
# passes JSON validation. We don't have the real uuid yet,
# but this will allow the validation test.
if ng_templates:
dummy_uuid = uuid.uuid4()
ng_ids = {ng["template"]["name"]: dummy_uuid for ng in ng_templates}
else:
ng_ids = {}
for cl in cl_templates:
template = copy.deepcopy(cl["template"])
u.substitute_ng_ids(template, ng_ids)
try:
ct_validator.validate(template)
except jsonschema.ValidationError as e:
LOG.warning("Validation for {path} failed, {reason}".format(
path=cl["path"], reason=e))
return True
return False
def add_cluster_templates(ctx, clusters, ng_dict):
'''Add cluster templates to the database.
The value of any node_group_template_id fields in cluster
templates which reference a node group template in ng_dict by name
will be changed to the id of the node group template.
If there is an error in creating or updating a template, any templates
that have already been created will be delete and any updates will
be reversed.
:param clusters: a list of dictionaries. Each dictionary
has a "template" entry holding the cluster template
and a "path" entry holding the path of the file
from which the template was read.
:param ng_dict: a dictionary of node group template ids keyed
by node group template names
'''
error = False
created = []
updated = []
def do_reversals(created, updated):
reverse_cluster_template_updates(ctx, updated)
reverse_cluster_template_creates(ctx, created)
return True
try:
for cl in clusters:
template = cl['template']
# Fix up node_group_template_id fields
u.substitute_ng_ids(template, ng_dict)
# Name + tenant_id is unique, so search by name
current = u.find_cluster_template_by_name(ctx, template['name'])
if current:
# Track what we see in the current template that is different
# from our update values. Save it for possible rollback.
# Note, this is not perfect because it does not recurse through
# nested structures to get an exact diff, but it ensures that
# we track only fields that are valid in the JSON schema
updated_fields = u.value_diff(current.to_dict(), template)
# Always attempt to update. Since the template value is a
# combination of JSON and config values, there is no useful
# timestamp we can use to skip an update.
# If sqlalchemy determines no change in fields, it will not
# mark it as updated.
# TODO(tmckay): why when I change the count in an
# entry in node_groups does it not count as an update?
# Probably a bug
try:
template = conductor.API.cluster_template_update(
ctx, current['id'], template, ignore_default=True)
except Exception as e:
LOG.warning(_LW("Update of cluster template {info} "
"failed, {reason}").format(
info=u.name_and_id(current), reason=e))
raise Handled()
if template['updated_at'] != current['updated_at']:
updated.append((template, updated_fields))
LOG.info(_LI("Updated cluster template {info} "
"from {path}").format(
info=u.name_and_id(template),
path=cl['path']))
else:
LOG.debug("No change to cluster template {info} "
"from {path}".format(info=u.name_and_id(current),
path=cl["path"]))
else:
template["is_default"] = True
try:
template = conductor.API.cluster_template_create(ctx,
template)
except Exception as e:
LOG.warning(_LW("Creation of cluster template "
"from {path} failed, {reason}").format(
path=cl['path'],
reason=e))
raise Handled()
created.append(template)
LOG.info(_LI("Created cluster template {info} "
"from {path}").format(info=u.name_and_id(template),
path=cl['path']))
except Handled:
error = do_reversals(created, updated)
except Exception as e:
LOG.warning(_LW("Unhandled exception while processing "
"cluster templates, {reason}").format(reason=e))
error = do_reversals(created, updated)
return error
def add_cluster_templates(ctx, clusters, ng_dict):
'''Add cluster templates to the database.
The value of any node_group_template_id fields in cluster
templates which reference a node group template in ng_dict by name
will be changed to the id of the node group template.
If there is an error in creating or updating a template, any templates
that have already been created will be delete and any updates will
be reversed.
:param clusters: a list of dictionaries. Each dictionary
has a "template" entry holding the cluster template
and a "path" entry holding the path of the file
from which the template was read.
:param ng_dict: a dictionary of node group template ids keyed
by node group template names
'''
error = False
created = []
updated = []
def do_reversals(created, updated):
reverse_cluster_template_updates(ctx, updated)
reverse_cluster_template_creates(ctx, created)
return True
try:
for cl in clusters:
template = cl['template']
# Fix up node_group_template_id fields
u.substitute_ng_ids(template, ng_dict)
# Name + tenant_id is unique, so search by name
current = u.find_cluster_template_by_name(ctx, template['name'])
if current:
# Track what we see in the current template that is different
# from our update values. Save it for possible rollback.
# Note, this is not perfect because it does not recurse through
# nested structures to get an exact diff, but it ensures that
# we track only fields that are valid in the JSON schema
updated_fields = u.value_diff(current.to_dict(), template)
# Always attempt to update. Since the template value is a
# combination of JSON and config values, there is no useful
# timestamp we can use to skip an update.
# If sqlalchemy determines no change in fields, it will not
# mark it as updated.
# TODO(tmckay): why when I change the count in an
# entry in node_groups does it not count as an update?
# Probably a bug
try:
template = conductor.API.cluster_template_update(
ctx, current['id'], template, ignore_prot_on_def=True)
except Exception as e:
LOG.warning(
_LW("Update of cluster template {info} "
"failed, {reason}").format(
info=u.name_and_id(current), reason=e))
raise Handled()
if template['updated_at'] != current['updated_at']:
updated.append((template, updated_fields))
LOG.info(
_LI("Updated cluster template {info} "
"from {path}").format(info=u.name_and_id(template),
path=cl['path']))
else:
LOG.debug("No change to cluster template {info} "
"from {path}".format(info=u.name_and_id(current),
path=cl["path"]))
else:
template["is_default"] = True
try:
template = conductor.API.cluster_template_create(
ctx, template)
except Exception as e:
LOG.warning(
_LW("Creation of cluster template "
"from {path} failed, {reason}").format(
path=cl['path'], reason=e))
raise Handled()
created.append(template)
LOG.info(
_LI("Created cluster template {info} "
"from {path}").format(info=u.name_and_id(template),
path=cl['path']))
except Handled:
error = do_reversals(created, updated)
except Exception as e:
LOG.warning(
_LW("Unhandled exception while processing "
"cluster templates, {reason}").format(reason=e))
error = do_reversals(created, updated)
return error