def test_add_cluster_templates(self):
self.logger.clear_log()
ctx = context.ctx()
# Create a record that will be updated in the db
existing = copy.copy(c.SAMPLE_CLT)
existing = self.api.cluster_template_create(ctx, existing)
# Create the update
update = copy.copy(c.SAMPLE_CLT)
update["hadoop_version"] = "1"
# Create a record that will be new in the db
new = copy.copy(c.SAMPLE_CLT)
new["name"] = "new_name"
clts = [{"template": update, "path": "foo"}, {"template": new, "path": "bar"}]
error = template_api.add_cluster_templates(ctx, clts, {})
self.assertFalse(error)
new = self.api.cluster_template_get_all(ctx, name=new["name"])[0]
self.assertIsNotNone(new)
msg = "Created cluster template {info} from bar".format(info=u.name_and_id(new))
self.assertIn(msg, self.logger.infos)
msg = "Updated cluster template {info} from foo".format(info=u.name_and_id(existing))
self.assertIn(msg, self.logger.infos)
self.api.cluster_template_destroy(ctx, new["id"], ignore_default=True)
self.api.cluster_template_destroy(ctx, existing["id"], ignore_default=True)
def delete_cluster_template(ctx, template, rollback=False):
rollback_msg = " on rollback" if rollback else ""
# If we are not deleting something that we just created,
# do usage checks to ensure that the template is not in
# use by a cluster
if not rollback:
clusters = conductor.API.cluster_get_all(ctx)
cluster_users = u.check_cluster_template_usage(template["id"],
clusters)
if cluster_users:
LOG.warning(_LW("Cluster template {info} "
"in use by clusters {clusters}").format(
info=u.name_and_id(template),
clusters=cluster_users))
LOG.warning(_LW("Deletion of cluster template "
"{info} failed").format(info=u.name_and_id(template)))
return
try:
conductor.API.cluster_template_destroy(ctx, template["id"],
ignore_default=True)
except Exception as e:
LOG.warning(_LW("Deletion of cluster template {info} failed{rollback}"
", {reason}").format(info=u.name_and_id(template),
reason=e,
rollback=rollback_msg))
else:
LOG.info(_LI("Deleted cluster template {info}{rollback}").format(
info=u.name_and_id(template), rollback=rollback_msg))
def delete_cluster_template(ctx, template, rollback=False):
rollback_msg = " on rollback" if rollback else ""
# If we are not deleting something that we just created,
# do usage checks to ensure that the template is not in
# use by a cluster
if not rollback:
clusters = conductor.API.cluster_get_all(ctx)
cluster_users = u.check_cluster_template_usage(template["id"],
clusters)
if cluster_users:
LOG.warning("Cluster template {info} "
"in use by clusters {clusters}".format(
info=u.name_and_id(template),
clusters=cluster_users))
LOG.warning("Deletion of cluster template "
"{info} failed".format(info=u.name_and_id(template)))
return
try:
conductor.API.cluster_template_destroy(ctx,
template["id"],
ignore_default=True)
except Exception as e:
LOG.warning("Deletion of cluster template {info} failed{rollback}"
", {reason}".format(info=u.name_and_id(template),
reason=e,
rollback=rollback_msg))
else:
LOG.info("Deleted cluster template {info}{rollback}".format(
info=u.name_and_id(template), rollback=rollback_msg))
def reverse_cluster_template_updates(ctx, update_info):
for template, values in update_info:
# values are the original values that we overwrote in the update
try:
conductor.API.cluster_template_update(ctx, template["id"], values, ignore_default=True)
except Exception as e:
LOG.warning(
"Rollback of update for cluster "
"template {info} failed, {reason}".format(info=u.name_and_id(template), reason=e)
)
else:
LOG.info("Rolled back update for " "cluster template {info}".format(info=u.name_and_id(template)))
def test_add_node_group_templates(self):
self.logger.clear_log()
ctx = context.ctx()
# Create a record that will be updated in the db
existing = copy.copy(c.SAMPLE_NGT)
existing = self.api.node_group_template_create(ctx, existing)
# Create the update
update = copy.copy(c.SAMPLE_NGT)
update["flavor_id"] = "6"
# Create a record that will be new in the db
new = copy.copy(c.SAMPLE_NGT)
new["name"] = "new_name"
ngts = [{"template": update, "path": "foo"},
{"template": new, "path": "bar"}]
ng_info, error = template_api.add_node_group_templates(ctx, ngts)
self.assertFalse(error)
new = self.api.node_group_template_get_all(ctx, name=new["name"])[0]
self.assertIsNotNone(new)
# ng_info["created"] is a list of templates that were created
self.assertEqual(1, len(ng_info["created"]))
self.assertEqual(new["id"], ng_info["created"][0]["id"])
# ng_info["updated"] is a list of tuples for templates that
# were updated. First element in the tuple is the template,
# second is a dictionary of fields that were updated.
self.assertEqual(1, len(ng_info["updated"]))
self.assertEqual(existing["id"], ng_info["updated"][0][0]["id"])
self.assertEqual({"flavor_id": "42"}, ng_info["updated"][0][1])
# ng_info["dict"] is a dictionary of name/id pairs
self.assertEqual({new["name"]: new["id"],
existing["name"]: existing["id"]}, ng_info["ids"])
msg = ("Created node group template {info} from bar".format(
info=u.name_and_id(new)))
self.assertIn(msg, self.logger.infos)
msg = ("Updated node group template {info} from foo".format(
info=u.name_and_id(existing)))
self.assertIn(msg, self.logger.infos)
self.api.node_group_template_destroy(ctx, new["id"],
ignore_default=True)
self.api.node_group_template_destroy(ctx, existing["id"],
ignore_default=True)
def test_node_group_template_delete_in_use(self):
self.logger.clear_log()
ctx = context.ctx()
t = self.api.node_group_template_create(ctx, c.SAMPLE_NGT)
# Make a cluster that references the node group template
cluster_values = copy.deepcopy(c.SAMPLE_CLUSTER)
cluster_values["node_groups"][0]["node_group_template_id"] = t["id"]
cl = self.api.cluster_create(ctx, cluster_values)
# Make a cluster template that references the node group template
cluster_temp_values = copy.deepcopy(c.SAMPLE_CLT)
cluster_temp_values["node_groups"] = cluster_values["node_groups"]
clt = self.api.cluster_template_create(ctx, cluster_temp_values)
# Set up the expected messages
msgs = [
"Node group template {info} in use "
"by clusters {clusters}".format(info=u.name_and_id(t),
clusters=[cl["name"]])
]
msgs += [
"Node group template {info} in use "
"by cluster templates {cluster_temps}".format(
info=u.name_and_id(t), cluster_temps=[clt["name"]])
]
msgs += [
"Deletion of node group template {info} failed".format(
info=u.name_and_id(t))
]
# Check delete by name
option_values = {
"tenant_id": t["tenant_id"],
"template_name": t["name"]
}
template_api.set_conf(Config(option_values))
template_api.do_node_group_template_delete()
for msg in msgs:
self.assertIn(msg, self.logger.warnings)
self.logger.clear_log()
# Check again with delete by id
option_values = {"tenant_id": t["tenant_id"], "id": t["id"]}
template_api.set_conf(Config(option_values))
template_api.do_node_group_template_delete_by_id()
for msg in msgs:
self.assertIn(msg, self.logger.warnings)
self.logger.clear_log()
def reverse_cluster_template_updates(ctx, update_info):
for template, values in update_info:
# values are the original values that we overwrote in the update
try:
conductor.API.cluster_template_update(ctx,
template["id"],
values,
ignore_prot_on_def=True)
except Exception as e:
LOG.warning("Rollback of update for cluster "
"template {info} failed, {reason}".format(
info=u.name_and_id(template), reason=e))
else:
LOG.info("Rolled back update for cluster "
"template {info}".format(info=u.name_and_id(template)))
def reverse_node_group_template_updates(ctx, update_info):
for template, values in update_info:
# values are the original values that we overwrote in the update
try:
conductor.API.node_group_template_update(ctx,
template["id"], values,
ignore_prot_on_def=True)
except Exception as e:
LOG.warning(_LW("Rollback of update for node group "
"template {info} failed, {reason}").format(
info=u.name_and_id(template),
reason=e))
else:
LOG.info(_LI("Rolled back update for "
"node group template {info}").format(
info=u.name_and_id(template)))
def test_add_cluster_templates_update_failed(self, reverse_updates, reverse_creates, cl_update):
self.logger.clear_log()
ctx = context.ctx()
cl_update.side_effect = Exception("mistake")
# Create a record that will be updated in the db
existing = copy.copy(c.SAMPLE_CLT)
existing = self.api.cluster_template_create(ctx, existing)
# Create the update
update = copy.copy(c.SAMPLE_CLT)
update["hadoop_version"] = "1"
# Create a record that will be new in the db
new = copy.copy(c.SAMPLE_CLT)
new["name"] = "new_name"
clts = [{"template": new, "path": "bar"}, {"template": update, "path": "foo"}]
error = template_api.add_cluster_templates(ctx, clts, {})
new = self.api.cluster_template_get_all(ctx, name=new["name"])[0]
self.assertTrue(error)
self.assertEqual(1, reverse_creates.call_count)
# call should have been (ctx, [new])
self.assertEqual(new["id"], reverse_creates.call_args[0][1][0]["id"])
self.assertEqual(1, reverse_updates.call_count)
msg = "Update of cluster template {info} failed, mistake".format(info=u.name_and_id(existing))
self.assertIn(msg, self.logger.warnings)
self.api.cluster_template_destroy(ctx, new["id"], ignore_default=True)
self.api.cluster_template_destroy(ctx, existing["id"], ignore_default=True)
def delete_node_group_template(ctx, template, rollback=False):
rollback_msg = " on rollback" if rollback else ""
# If we are not deleting something that we just created,
# do usage checks to ensure that the template is not in
# use by a cluster or a cluster template
if not rollback:
clusters = conductor.API.cluster_get_all(ctx)
cluster_templates = conductor.API.cluster_template_get_all(ctx)
cluster_users, template_users = u.check_node_group_template_usage(
template["id"], clusters, cluster_templates)
if cluster_users:
LOG.warning(
_LW("Node group template {info} "
"in use by clusters {clusters}").format(
info=u.name_and_id(template), clusters=cluster_users))
if template_users:
LOG.warning(
_LW("Node group template {info} "
"in use by cluster templates {templates}").format(
info=u.name_and_id(template),
templates=template_users))
if cluster_users or template_users:
LOG.warning(
_LW("Deletion of node group template "
"{info} failed").format(info=u.name_and_id(template)))
return
try:
conductor.API.node_group_template_destroy(ctx,
template["id"],
ignore_prot_on_def=True)
except Exception as e:
LOG.warning(
_LW("Deletion of node group template {info} "
"failed{rollback}, {reason}").format(
info=u.name_and_id(template),
reason=e,
rollback=rollback_msg))
else:
LOG.info(
_LI("Deleted node group template {info}{rollback}").format(
info=u.name_and_id(template), rollback=rollback_msg))
def test_node_group_template_delete_in_use(self):
self.logger.clear_log()
ctx = context.ctx()
t = self.api.node_group_template_create(ctx, c.SAMPLE_NGT)
# Make a cluster that references the node group template
cluster_values = copy.deepcopy(c.SAMPLE_CLUSTER)
cluster_values["node_groups"][0]["node_group_template_id"] = t["id"]
cl = self.api.cluster_create(ctx, cluster_values)
# Make a cluster template that references the node group template
cluster_temp_values = copy.deepcopy(c.SAMPLE_CLT)
cluster_temp_values["node_groups"] = cluster_values["node_groups"]
clt = self.api.cluster_template_create(ctx, cluster_temp_values)
# Set up the expected messages
msgs = ["Node group template {info} in use "
"by clusters {clusters}".format(
info=u.name_and_id(t), clusters=[cl["name"]])]
msgs += ["Node group template {info} in use "
"by cluster templates {cluster_temps}".format(
info=u.name_and_id(t), cluster_temps=[clt["name"]])]
msgs += ["Deletion of node group template {info} failed".format(
info=u.name_and_id(t))]
# Check delete by name
option_values = {"tenant_id": t["tenant_id"],
"template_name": t["name"]}
template_api.set_conf(Config(option_values))
template_api.do_node_group_template_delete()
for msg in msgs:
self.assertIn(msg, self.logger.warnings)
self.logger.clear_log()
# Check again with delete by id
option_values = {"tenant_id": t["tenant_id"],
"id": t["id"]}
template_api.set_conf(Config(option_values))
template_api.do_node_group_template_delete_by_id()
for msg in msgs:
self.assertIn(msg, self.logger.warnings)
self.logger.clear_log()
def test_add_cluster_templates(self):
self.logger.clear_log()
ctx = context.ctx()
# Create a record that will be updated in the db
existing = copy.copy(c.SAMPLE_CLT)
existing = self.api.cluster_template_create(ctx, existing)
# Create the update
update = copy.copy(c.SAMPLE_CLT)
update["hadoop_version"] = "1"
# Create a record that will be new in the db
new = copy.copy(c.SAMPLE_CLT)
new["name"] = "new_name"
clts = [{
"template": update,
"path": "foo"
}, {
"template": new,
"path": "bar"
}]
error = template_api.add_cluster_templates(ctx, clts, {})
self.assertFalse(error)
new = self.api.cluster_template_get_all(ctx, name=new["name"])[0]
self.assertIsNotNone(new)
msg = ("Created cluster template {info} from bar".format(
info=u.name_and_id(new)))
self.assertIn(msg, self.logger.infos)
msg = ("Updated cluster template {info} from foo".format(
info=u.name_and_id(existing)))
self.assertIn(msg, self.logger.infos)
self.api.cluster_template_destroy(ctx,
new["id"],
ignore_prot_on_def=True)
self.api.cluster_template_destroy(ctx,
existing["id"],
ignore_prot_on_def=True)
def delete_node_group_template(ctx, template, rollback=False):
rollback_msg = " on rollback" if rollback else ""
# If we are not deleting something that we just created,
# do usage checks to ensure that the template is not in
# use by a cluster or a cluster template
if not rollback:
clusters = conductor.API.cluster_get_all(ctx)
cluster_templates = conductor.API.cluster_template_get_all(ctx)
cluster_users, template_users = u.check_node_group_template_usage(
template["id"], clusters, cluster_templates)
if cluster_users:
LOG.warning("Node group template {info} "
"in use by clusters {clusters}".format(
info=u.name_and_id(template),
clusters=cluster_users))
if template_users:
LOG.warning("Node group template {info} "
"in use by cluster templates {templates}".format(
info=u.name_and_id(template),
templates=template_users))
if cluster_users or template_users:
LOG.warning("Deletion of node group template "
"{info} failed".format(info=u.name_and_id(template)))
return
try:
conductor.API.node_group_template_destroy(ctx, template["id"],
ignore_prot_on_def=True)
except Exception as e:
LOG.warning("Deletion of node group template {info} "
"failed{rollback}, {reason}".format(
info=u.name_and_id(template),
reason=e,
rollback=rollback_msg))
else:
LOG.info("Deleted node group template {info}{rollback}".format(
info=u.name_and_id(template), rollback=rollback_msg))
def test_cluster_template_delete_by_id(self):
self.logger.clear_log()
ctx = context.ctx()
t = self.api.cluster_template_create(ctx, c.SAMPLE_CLT)
option_values = {"tenant_id": t["tenant_id"], "id": t["id"]}
template_api.set_conf(Config(option_values))
template_api.do_cluster_template_delete_by_id()
msg = 'Deleted cluster template {info}'.format(info=u.name_and_id(t))
self.assertIn(msg, self.logger.infos)
t = self.api.cluster_template_get(ctx, t["id"])
self.assertIsNone(t)
def do_cluster_template_delete_by_id():
ctx = Context(is_admin=True)
# Make sure it's a default
t = conductor.API.cluster_template_get(ctx, CONF.command.id)
if t:
if t["is_default"]:
delete_cluster_template(ctx, t)
else:
LOG.warning("Deletion of cluster template {info} skipped, "
"not a default template".format(info=u.name_and_id(t)))
else:
LOG.warning("Deletion of cluster template {id} failed, "
"no such template".format(id=CONF.command.id))
def do_cluster_template_delete_by_id():
ctx = Context(is_admin=True)
# Make sure it's a default
t = conductor.API.cluster_template_get(ctx, CONF.command.id)
if t:
if t["is_default"]:
delete_cluster_template(ctx, t)
else:
LOG.warning("Deletion of cluster template {info} skipped, "
"not a default template".format(info=u.name_and_id(t)))
else:
LOG.warning("Deletion of cluster template {id} failed, "
"no such template".format(id=CONF.command.id))
def test_cluster_template_delete_by_id_skipped(self):
self.logger.clear_log()
ctx = context.ctx()
template_values = copy.copy(c.SAMPLE_CLT)
template_values["is_default"] = False
t = self.api.cluster_template_create(ctx, template_values)
option_values = {"tenant_id": t["tenant_id"], "id": t["id"]}
template_api.set_conf(Config(option_values))
template_api.do_cluster_template_delete_by_id()
msg = ("Deletion of cluster template {info} skipped, "
"not a default template".format(info=u.name_and_id(t)))
self.assertIn(msg, self.logger.warnings)
t = self.api.cluster_template_get(ctx, t["id"])
self.assertIsNotNone(t)
def test_node_group_template_delete_by_name(self):
self.logger.clear_log()
ctx = context.ctx()
t = self.api.node_group_template_create(ctx, c.SAMPLE_NGT)
option_values = {"tenant_id": t["tenant_id"],
"template_name": t["name"]}
template_api.set_conf(Config(option_values))
template_api.do_node_group_template_delete()
msg = 'Deleted node group template {info}'.format(
info=u.name_and_id(t))
self.assertIn(msg, self.logger.infos)
t = self.api.node_group_template_get(ctx, t["id"])
self.assertIsNone(t)
def test_add_node_group_templates_update_failed(self, reverse_updates,
reverse_creates,
ng_update):
self.logger.clear_log()
ctx = context.ctx()
ng_update.side_effect = Exception("mistake")
# Create a record that will be updated in the db
existing = copy.copy(c.SAMPLE_NGT)
existing = self.api.node_group_template_create(ctx, existing)
# Create the update
update = copy.copy(c.SAMPLE_NGT)
update["flavor_id"] = "6"
# Create a record that will be new in the db
new = copy.copy(c.SAMPLE_NGT)
new["name"] = "new_name"
ngts = [{
"template": new,
"path": "bar"
}, {
"template": update,
"path": "foo"
}]
ng_info, error = template_api.add_node_group_templates(ctx, ngts)
new = self.api.node_group_template_get_all(ctx, name=new["name"])[0]
self.assertTrue(error)
self.assertEqual(1, reverse_creates.call_count)
# call should have been (ctx, [new])
self.assertEqual(new["id"], reverse_creates.call_args[0][1][0]["id"])
self.assertEqual(1, reverse_updates.call_count)
msg = ("Update of node group template {info} failed, mistake".format(
info=u.name_and_id(existing)))
self.assertIn(msg, self.logger.warnings)
self.api.node_group_template_destroy(ctx,
new["id"],
ignore_prot_on_def=True)
self.api.node_group_template_destroy(ctx,
existing["id"],
ignore_prot_on_def=True)
def test_cluster_template_delete_by_id_skipped(self):
self.logger.clear_log()
ctx = context.ctx()
template_values = copy.copy(c.SAMPLE_CLT)
template_values["is_default"] = False
t = self.api.cluster_template_create(ctx, template_values)
option_values = {"tenant_id": t["tenant_id"],
"id": t["id"]}
template_api.set_conf(Config(option_values))
template_api.do_cluster_template_delete_by_id()
msg = ("Deletion of cluster template {info} skipped, "
"not a default template".format(info=u.name_and_id(t)))
self.assertIn(msg, self.logger.warnings)
t = self.api.cluster_template_get(ctx, t["id"])
self.assertIsNotNone(t)
def test_cluster_template_delete_by_id(self):
self.logger.clear_log()
self.setup_context(tenant_id=None)
ctx = context.ctx()
t = self.api.cluster_template_create(ctx, c.SAMPLE_CLT)
option_values = {"tenant_id": t["tenant_id"],
"id": t["id"]}
template_api.set_conf(Config(option_values))
template_api.do_cluster_template_delete_by_id()
msg = 'Deleted cluster template {info}'.format(
info=u.name_and_id(t))
self.assertIn(msg, self.logger.infos)
t = self.api.cluster_template_get(ctx, t["id"])
self.assertIsNone(t)
def test_add_cluster_templates_update_failed(self, reverse_updates,
reverse_creates, cl_update):
self.logger.clear_log()
ctx = context.ctx()
cl_update.side_effect = Exception("mistake")
# Create a record that will be updated in the db
existing = copy.copy(c.SAMPLE_CLT)
existing = self.api.cluster_template_create(ctx, existing)
# Create the update
update = copy.copy(c.SAMPLE_CLT)
update["hadoop_version"] = "1"
# Create a record that will be new in the db
new = copy.copy(c.SAMPLE_CLT)
new["name"] = "new_name"
clts = [{
"template": new,
"path": "bar"
}, {
"template": update,
"path": "foo"
}]
error = template_api.add_cluster_templates(ctx, clts, {})
new = self.api.cluster_template_get_all(ctx, name=new["name"])[0]
self.assertTrue(error)
reverse_creates.assert_called_once()
# call should have been (ctx, [new])
self.assertEqual(new["id"], reverse_creates.call_args[0][1][0]["id"])
reverse_updates.assert_called_once()
msg = ("Update of cluster template {info} failed, mistake".format(
info=u.name_and_id(existing)))
self.assertIn(msg, self.logger.warnings)
self.api.cluster_template_destroy(ctx, new["id"], ignore_default=True)
self.api.cluster_template_destroy(ctx,
existing["id"],
ignore_default=True)
def test_add_node_group_templates_update_failed(self,
reverse_updates,
reverse_creates,
ng_update):
self.logger.clear_log()
ctx = context.ctx()
ng_update.side_effect = Exception("mistake")
# Create a record that will be updated in the db
existing = copy.copy(c.SAMPLE_NGT)
existing = self.api.node_group_template_create(ctx, existing)
# Create the update
update = copy.copy(c.SAMPLE_NGT)
update["flavor_id"] = "6"
# Create a record that will be new in the db
new = copy.copy(c.SAMPLE_NGT)
new["name"] = "new_name"
ngts = [{"template": new, "path": "bar"},
{"template": update, "path": "foo"}]
ng_info, error = template_api.add_node_group_templates(ctx, ngts)
new = self.api.node_group_template_get_all(ctx, name=new["name"])[0]
self.assertTrue(error)
reverse_creates.assert_called_once()
# call should have been (ctx, [new])
self.assertEqual(new["id"], reverse_creates.call_args[0][1][0]["id"])
reverse_updates.assert_called_once()
msg = ("Update of node group template {info} failed, mistake".format(
info=u.name_and_id(existing)))
self.assertIn(msg, self.logger.warnings)
self.api.node_group_template_destroy(ctx, new["id"],
ignore_default=True)
self.api.node_group_template_destroy(ctx, existing["id"],
ignore_default=True)
def test_do_delete(self):
self.logger.clear_log()
ctx = context.ctx()
# Make some plugins to delete
ngt, clt = self._make_templates(ctx, "first", "plugin", "v1")
# Make some more for the same plugin, different version
ngt2, clt2 = self._make_templates(ctx, "second", "plugin", "v2")
# Make another set for a different plugin, overlapping version
safe_ngt, safe_clt = self._make_templates(ctx, "third", "plugin2",
"v1")
# Run a delete by plugin name/version for the first set
option_values = {"tenant_id": ngt["tenant_id"],
"plugin_name": [ngt["plugin_name"]],
"plugin_version": [ngt["hadoop_version"]]}
template_api.set_conf(Config(option_values))
# Should delete clt and then ngt, check for messages in order
template_api.do_delete()
msgs = ["Deleted cluster template {info}".format(
info=u.name_and_id(clt))]
msgs += ["Deleted node group template {info}".format(
info=u.name_and_id(ngt))]
self.assertEqual(msgs, self.logger.infos)
self.assertIsNone(self.api.node_group_template_get(ctx, ngt["id"]))
self.assertIsNone(self.api.cluster_template_get(ctx, clt["id"]))
# Make sure the other templates are still there
self.assertIsNotNone(self.api.node_group_template_get(ctx,
ngt2["id"]))
self.assertIsNotNone(self.api.cluster_template_get(ctx,
clt2["id"]))
self.assertIsNotNone(self.api.node_group_template_get(ctx,
safe_ngt["id"]))
self.assertIsNotNone(self.api.cluster_template_get(ctx,
safe_clt["id"]))
# Run delete again for the plugin but with no version specified
self.logger.clear_log()
option_values = {"tenant_id": ngt2["tenant_id"],
"plugin_name": [ngt2["plugin_name"]],
"plugin_version": None}
template_api.set_conf(Config(option_values))
# Should delete clt2 and then ngt2, check for messages in order
template_api.do_delete()
msgs = ["Deleted cluster template {info}".format(
info=u.name_and_id(clt2))]
msgs += ["Deleted node group template {info}".format(
info=u.name_and_id(ngt2))]
self.assertEqual(msgs, self.logger.infos)
self.assertIsNone(self.api.node_group_template_get(ctx, ngt2["id"]))
self.assertIsNone(self.api.cluster_template_get(ctx, clt2["id"]))
# Make sure the other templates are still there
self.assertIsNotNone(self.api.node_group_template_get(ctx,
safe_ngt["id"]))
self.assertIsNotNone(self.api.cluster_template_get(ctx,
safe_clt["id"]))
def add_node_group_templates(ctx, node_groups):
error = False
ng_info = {"ids": {}, "created": [], "updated": []}
def do_reversals(ng_info):
reverse_node_group_template_updates(ctx, ng_info["updated"])
reverse_node_group_template_creates(ctx, ng_info["created"])
return {}, True
try:
for ng in node_groups:
template = ng['template']
current = u.find_node_group_template_by_name(ctx, template['name'])
if current:
# Track what we see in the current template that is different
# from our update values. Save it for possible rollback.
# Note, this is not perfect because it does not recurse through
# nested structures to get an exact diff, but it ensures that
# we track only fields that are valid in the JSON schema
updated_fields = u.value_diff(current.to_dict(), template)
# Always attempt to update. Since the template value is a
# combination of JSON and config values, there is no useful
# timestamp we can use to skip an update.
# If sqlalchemy determines no change in fields, it will not
# mark it as updated.
try:
template = conductor.API.node_group_template_update(
ctx, current['id'], template, ignore_prot_on_def=True)
except Exception as e:
LOG.warning(
_LW("Update of node group template {info} "
"failed, {reason}").format(
info=u.name_and_id(current), reason=e))
raise Handled()
if template['updated_at'] != current['updated_at']:
ng_info["updated"].append((template, updated_fields))
LOG.info(
_LI("Updated node group template {info} "
"from {path}").format(info=u.name_and_id(template),
path=ng["path"]))
else:
LOG.debug("No change to node group template {info} "
"from {path}".format(info=u.name_and_id(current),
path=ng['path']))
else:
template['is_default'] = True
try:
template = conductor.API.node_group_template_create(
ctx, template)
except Exception as e:
LOG.warning(
_LW("Creation of node group template "
"from {path} failed, {reason}").format(
path=ng['path'], reason=e))
raise Handled()
ng_info["created"].append(template)
LOG.info(
_LI("Created node group template {info} "
"from {path}").format(info=u.name_and_id(template),
path=ng["path"]))
# For the purposes of substitution we need a dict of id by name
ng_info["ids"][template['name']] = template['id']
except Handled:
ng_info, error = do_reversals(ng_info)
except Exception as e:
LOG.warning(
_LW("Unhandled exception while processing "
"node group templates, {reason}").format(reason=e))
ng_info, error = do_reversals(ng_info)
return ng_info, error
def add_cluster_templates(ctx, clusters, ng_dict):
'''Add cluster templates to the database.
The value of any node_group_template_id fields in cluster
templates which reference a node group template in ng_dict by name
will be changed to the id of the node group template.
If there is an error in creating or updating a template, any templates
that have already been created will be delete and any updates will
be reversed.
:param clusters: a list of dictionaries. Each dictionary
has a "template" entry holding the cluster template
and a "path" entry holding the path of the file
from which the template was read.
:param ng_dict: a dictionary of node group template ids keyed
by node group template names
'''
error = False
created = []
updated = []
def do_reversals(created, updated):
reverse_cluster_template_updates(ctx, updated)
reverse_cluster_template_creates(ctx, created)
return True
try:
for cl in clusters:
template = cl['template']
# Fix up node_group_template_id fields
u.substitute_ng_ids(template, ng_dict)
# Name + tenant_id is unique, so search by name
current = u.find_cluster_template_by_name(ctx, template['name'])
if current:
# Track what we see in the current template that is different
# from our update values. Save it for possible rollback.
# Note, this is not perfect because it does not recurse through
# nested structures to get an exact diff, but it ensures that
# we track only fields that are valid in the JSON schema
updated_fields = u.value_diff(current.to_dict(), template)
# Always attempt to update. Since the template value is a
# combination of JSON and config values, there is no useful
# timestamp we can use to skip an update.
# If sqlalchemy determines no change in fields, it will not
# mark it as updated.
# TODO(tmckay): why when I change the count in an
# entry in node_groups does it not count as an update?
# Probably a bug
try:
template = conductor.API.cluster_template_update(
ctx, current['id'], template, ignore_prot_on_def=True)
except Exception as e:
LOG.warning(
_LW("Update of cluster template {info} "
"failed, {reason}").format(
info=u.name_and_id(current), reason=e))
raise Handled()
if template['updated_at'] != current['updated_at']:
updated.append((template, updated_fields))
LOG.info(
_LI("Updated cluster template {info} "
"from {path}").format(info=u.name_and_id(template),
path=cl['path']))
else:
LOG.debug("No change to cluster template {info} "
"from {path}".format(info=u.name_and_id(current),
path=cl["path"]))
else:
template["is_default"] = True
try:
template = conductor.API.cluster_template_create(
ctx, template)
except Exception as e:
LOG.warning(
_LW("Creation of cluster template "
"from {path} failed, {reason}").format(
path=cl['path'], reason=e))
raise Handled()
created.append(template)
LOG.info(
_LI("Created cluster template {info} "
"from {path}").format(info=u.name_and_id(template),
path=cl['path']))
except Handled:
error = do_reversals(created, updated)
except Exception as e:
LOG.warning(
_LW("Unhandled exception while processing "
"cluster templates, {reason}").format(reason=e))
error = do_reversals(created, updated)
return error
def test_do_delete(self):
self.logger.clear_log()
ctx = context.ctx()
# Make some plugins to delete
ngt, clt = self._make_templates(ctx, "first", "plugin", "v1")
# Make some more for the same plugin, different version
ngt2, clt2 = self._make_templates(ctx, "second", "plugin", "v2")
# Make another set for a different plugin, overlapping version
safe_ngt, safe_clt = self._make_templates(ctx, "third", "plugin2",
"v1")
# Run a delete by plugin name/version for the first set
option_values = {"tenant_id": ngt["tenant_id"],
"plugin_name": [ngt["plugin_name"]],
"plugin_version": [ngt["hadoop_version"]]}
template_api.set_conf(Config(option_values))
# Should delete clt and then ngt, check for messages in order
template_api.do_delete()
msgs = ["Deleted cluster template {info}".format(
info=u.name_and_id(clt))]
msgs += ["Deleted node group template {info}".format(
info=u.name_and_id(ngt))]
self.assertEqual(msgs, self.logger.infos)
self.assertIsNone(self.api.node_group_template_get(ctx, ngt["id"]))
self.assertIsNone(self.api.cluster_template_get(ctx, clt["id"]))
# Make sure the other templates are still there
self.assertIsNotNone(self.api.node_group_template_get(ctx,
ngt2["id"]))
self.assertIsNotNone(self.api.cluster_template_get(ctx,
clt2["id"]))
self.assertIsNotNone(self.api.node_group_template_get(ctx,
safe_ngt["id"]))
self.assertIsNotNone(self.api.cluster_template_get(ctx,
safe_clt["id"]))
# Run delete again for the plugin but with no version specified
self.logger.clear_log()
option_values = {"tenant_id": ngt2["tenant_id"],
"plugin_name": [ngt2["plugin_name"]],
"plugin_version": None}
template_api.set_conf(Config(option_values))
# Should delete clt2 and then ngt2, check for messages in order
template_api.do_delete()
msgs = ["Deleted cluster template {info}".format(
info=u.name_and_id(clt2))]
msgs += ["Deleted node group template {info}".format(
info=u.name_and_id(ngt2))]
self.assertEqual(msgs, self.logger.infos)
self.assertIsNone(self.api.node_group_template_get(ctx, ngt2["id"]))
self.assertIsNone(self.api.cluster_template_get(ctx, clt2["id"]))
# Make sure the other templates are still there
self.assertIsNotNone(self.api.node_group_template_get(ctx,
safe_ngt["id"]))
self.assertIsNotNone(self.api.cluster_template_get(ctx,
safe_clt["id"]))
def test_add_node_group_templates(self):
self.logger.clear_log()
ctx = context.ctx()
# Create a record that will be updated in the db
existing = copy.copy(c.SAMPLE_NGT)
existing = self.api.node_group_template_create(ctx, existing)
# Create the update
update = copy.copy(c.SAMPLE_NGT)
update["flavor_id"] = "6"
# Create a record that will be new in the db
new = copy.copy(c.SAMPLE_NGT)
new["name"] = "new_name"
ngts = [{
"template": update,
"path": "foo"
}, {
"template": new,
"path": "bar"
}]
ng_info, error = template_api.add_node_group_templates(ctx, ngts)
self.assertFalse(error)
new = self.api.node_group_template_get_all(ctx, name=new["name"])[0]
self.assertIsNotNone(new)
# ng_info["created"] is a list of templates that were created
self.assertEqual(1, len(ng_info["created"]))
self.assertEqual(new["id"], ng_info["created"][0]["id"])
# ng_info["updated"] is a list of tuples for templates that
# were updated. First element in the tuple is the template,
# second is a dictionary of fields that were updated.
self.assertEqual(1, len(ng_info["updated"]))
self.assertEqual(existing["id"], ng_info["updated"][0][0]["id"])
self.assertEqual({"flavor_id": "42"}, ng_info["updated"][0][1])
# ng_info["dict"] is a dictionary of name/id pairs
self.assertEqual(
{
new["name"]: new["id"],
existing["name"]: existing["id"]
}, ng_info["ids"])
msg = ("Created node group template {info} from bar".format(
info=u.name_and_id(new)))
self.assertIn(msg, self.logger.infos)
msg = ("Updated node group template {info} from foo".format(
info=u.name_and_id(existing)))
self.assertIn(msg, self.logger.infos)
self.api.node_group_template_destroy(ctx,
new["id"],
ignore_prot_on_def=True)
self.api.node_group_template_destroy(ctx,
existing["id"],
ignore_prot_on_def=True)
def add_node_group_templates(ctx, node_groups):
error = False
ng_info = {"ids": {},
"created": [],
"updated": []}
def do_reversals(ng_info):
reverse_node_group_template_updates(ctx, ng_info["updated"])
reverse_node_group_template_creates(ctx, ng_info["created"])
return {}, True
try:
for ng in node_groups:
template = ng['template']
current = u.find_node_group_template_by_name(ctx, template['name'])
if current:
# Track what we see in the current template that is different
# from our update values. Save it for possible rollback.
# Note, this is not perfect because it does not recurse through
# nested structures to get an exact diff, but it ensures that
# we track only fields that are valid in the JSON schema
updated_fields = u.value_diff(current.to_dict(), template)
# Always attempt to update. Since the template value is a
# combination of JSON and config values, there is no useful
# timestamp we can use to skip an update.
# If sqlalchemy determines no change in fields, it will not
# mark it as updated.
try:
template = conductor.API.node_group_template_update(
ctx, current['id'], template, ignore_default=True)
except Exception as e:
LOG.warning(_LW("Update of node group template {info} "
"failed, {reason}").format(
info=u.name_and_id(current),
reason=e))
raise Handled()
if template['updated_at'] != current['updated_at']:
ng_info["updated"].append((template, updated_fields))
LOG.info(_LI("Updated node group template {info} "
"from {path}").format(
info=u.name_and_id(template),
path=ng["path"]))
else:
LOG.debug("No change to node group template {info} "
"from {path}".format(
info=u.name_and_id(current),
path=ng['path']))
else:
template['is_default'] = True
try:
template = conductor.API.node_group_template_create(
ctx, template)
except Exception as e:
LOG.warning(_LW("Creation of node group template "
"from {path} failed, {reason}").format(
path=ng['path'], reason=e))
raise Handled()
ng_info["created"].append(template)
LOG.info(_LI("Created node group template {info} "
"from {path}").format(info=u.name_and_id(template),
path=ng["path"]))
# For the purposes of substituion we need a dict of id by name
ng_info["ids"][template['name']] = template['id']
except Handled:
ng_info, error = do_reversals(ng_info)
except Exception as e:
LOG.warning(_LW("Unhandled exception while processing "
"node group templates, {reason}").format(reason=e))
ng_info, error = do_reversals(ng_info)
return ng_info, error
def add_cluster_templates(ctx, clusters, ng_dict):
'''Add cluster templates to the database.
The value of any node_group_template_id fields in cluster
templates which reference a node group template in ng_dict by name
will be changed to the id of the node group template.
If there is an error in creating or updating a template, any templates
that have already been created will be delete and any updates will
be reversed.
:param clusters: a list of dictionaries. Each dictionary
has a "template" entry holding the cluster template
and a "path" entry holding the path of the file
from which the template was read.
:param ng_dict: a dictionary of node group template ids keyed
by node group template names
'''
error = False
created = []
updated = []
def do_reversals(created, updated):
reverse_cluster_template_updates(ctx, updated)
reverse_cluster_template_creates(ctx, created)
return True
try:
for cl in clusters:
template = cl['template']
# Fix up node_group_template_id fields
u.substitute_ng_ids(template, ng_dict)
# Name + tenant_id is unique, so search by name
current = u.find_cluster_template_by_name(ctx, template['name'])
if current:
# Track what we see in the current template that is different
# from our update values. Save it for possible rollback.
# Note, this is not perfect because it does not recurse through
# nested structures to get an exact diff, but it ensures that
# we track only fields that are valid in the JSON schema
updated_fields = u.value_diff(current.to_dict(), template)
# Always attempt to update. Since the template value is a
# combination of JSON and config values, there is no useful
# timestamp we can use to skip an update.
# If sqlalchemy determines no change in fields, it will not
# mark it as updated.
# TODO(tmckay): why when I change the count in an
# entry in node_groups does it not count as an update?
# Probably a bug
try:
template = conductor.API.cluster_template_update(
ctx, current['id'], template, ignore_default=True)
except Exception as e:
LOG.warning(_LW("Update of cluster template {info} "
"failed, {reason}").format(
info=u.name_and_id(current), reason=e))
raise Handled()
if template['updated_at'] != current['updated_at']:
updated.append((template, updated_fields))
LOG.info(_LI("Updated cluster template {info} "
"from {path}").format(
info=u.name_and_id(template),
path=cl['path']))
else:
LOG.debug("No change to cluster template {info} "
"from {path}".format(info=u.name_and_id(current),
path=cl["path"]))
else:
template["is_default"] = True
try:
template = conductor.API.cluster_template_create(ctx,
template)
except Exception as e:
LOG.warning(_LW("Creation of cluster template "
"from {path} failed, {reason}").format(
path=cl['path'],
reason=e))
raise Handled()
created.append(template)
LOG.info(_LI("Created cluster template {info} "
"from {path}").format(info=u.name_and_id(template),
path=cl['path']))
except Handled:
error = do_reversals(created, updated)
except Exception as e:
LOG.warning(_LW("Unhandled exception while processing "
"cluster templates, {reason}").format(reason=e))
error = do_reversals(created, updated)
return error
