def test_add_cluster_templates(self): self.logger.clear_log() ctx = context.ctx() # Create a record that will be updated in the db existing = copy.copy(c.SAMPLE_CLT) existing = self.api.cluster_template_create(ctx, existing) # Create the update update = copy.copy(c.SAMPLE_CLT) update["hadoop_version"] = "1" # Create a record that will be new in the db new = copy.copy(c.SAMPLE_CLT) new["name"] = "new_name" clts = [{"template": update, "path": "foo"}, {"template": new, "path": "bar"}] error = template_api.add_cluster_templates(ctx, clts, {}) self.assertFalse(error) new = self.api.cluster_template_get_all(ctx, name=new["name"])[0] self.assertIsNotNone(new) msg = "Created cluster template {info} from bar".format(info=u.name_and_id(new)) self.assertIn(msg, self.logger.infos) msg = "Updated cluster template {info} from foo".format(info=u.name_and_id(existing)) self.assertIn(msg, self.logger.infos) self.api.cluster_template_destroy(ctx, new["id"], ignore_default=True) self.api.cluster_template_destroy(ctx, existing["id"], ignore_default=True)
def delete_cluster_template(ctx, template, rollback=False): rollback_msg = " on rollback" if rollback else "" # If we are not deleting something that we just created, # do usage checks to ensure that the template is not in # use by a cluster if not rollback: clusters = conductor.API.cluster_get_all(ctx) cluster_users = u.check_cluster_template_usage(template["id"], clusters) if cluster_users: LOG.warning(_LW("Cluster template {info} " "in use by clusters {clusters}").format( info=u.name_and_id(template), clusters=cluster_users)) LOG.warning(_LW("Deletion of cluster template " "{info} failed").format(info=u.name_and_id(template))) return try: conductor.API.cluster_template_destroy(ctx, template["id"], ignore_default=True) except Exception as e: LOG.warning(_LW("Deletion of cluster template {info} failed{rollback}" ", {reason}").format(info=u.name_and_id(template), reason=e, rollback=rollback_msg)) else: LOG.info(_LI("Deleted cluster template {info}{rollback}").format( info=u.name_and_id(template), rollback=rollback_msg))
def delete_cluster_template(ctx, template, rollback=False): rollback_msg = " on rollback" if rollback else "" # If we are not deleting something that we just created, # do usage checks to ensure that the template is not in # use by a cluster if not rollback: clusters = conductor.API.cluster_get_all(ctx) cluster_users = u.check_cluster_template_usage(template["id"], clusters) if cluster_users: LOG.warning("Cluster template {info} " "in use by clusters {clusters}".format( info=u.name_and_id(template), clusters=cluster_users)) LOG.warning("Deletion of cluster template " "{info} failed".format(info=u.name_and_id(template))) return try: conductor.API.cluster_template_destroy(ctx, template["id"], ignore_default=True) except Exception as e: LOG.warning("Deletion of cluster template {info} failed{rollback}" ", {reason}".format(info=u.name_and_id(template), reason=e, rollback=rollback_msg)) else: LOG.info("Deleted cluster template {info}{rollback}".format( info=u.name_and_id(template), rollback=rollback_msg))
def reverse_cluster_template_updates(ctx, update_info): for template, values in update_info: # values are the original values that we overwrote in the update try: conductor.API.cluster_template_update(ctx, template["id"], values, ignore_default=True) except Exception as e: LOG.warning( "Rollback of update for cluster " "template {info} failed, {reason}".format(info=u.name_and_id(template), reason=e) ) else: LOG.info("Rolled back update for " "cluster template {info}".format(info=u.name_and_id(template)))
def test_add_node_group_templates(self): self.logger.clear_log() ctx = context.ctx() # Create a record that will be updated in the db existing = copy.copy(c.SAMPLE_NGT) existing = self.api.node_group_template_create(ctx, existing) # Create the update update = copy.copy(c.SAMPLE_NGT) update["flavor_id"] = "6" # Create a record that will be new in the db new = copy.copy(c.SAMPLE_NGT) new["name"] = "new_name" ngts = [{"template": update, "path": "foo"}, {"template": new, "path": "bar"}] ng_info, error = template_api.add_node_group_templates(ctx, ngts) self.assertFalse(error) new = self.api.node_group_template_get_all(ctx, name=new["name"])[0] self.assertIsNotNone(new) # ng_info["created"] is a list of templates that were created self.assertEqual(1, len(ng_info["created"])) self.assertEqual(new["id"], ng_info["created"][0]["id"]) # ng_info["updated"] is a list of tuples for templates that # were updated. First element in the tuple is the template, # second is a dictionary of fields that were updated. self.assertEqual(1, len(ng_info["updated"])) self.assertEqual(existing["id"], ng_info["updated"][0][0]["id"]) self.assertEqual({"flavor_id": "42"}, ng_info["updated"][0][1]) # ng_info["dict"] is a dictionary of name/id pairs self.assertEqual({new["name"]: new["id"], existing["name"]: existing["id"]}, ng_info["ids"]) msg = ("Created node group template {info} from bar".format( info=u.name_and_id(new))) self.assertIn(msg, self.logger.infos) msg = ("Updated node group template {info} from foo".format( info=u.name_and_id(existing))) self.assertIn(msg, self.logger.infos) self.api.node_group_template_destroy(ctx, new["id"], ignore_default=True) self.api.node_group_template_destroy(ctx, existing["id"], ignore_default=True)
def test_node_group_template_delete_in_use(self): self.logger.clear_log() ctx = context.ctx() t = self.api.node_group_template_create(ctx, c.SAMPLE_NGT) # Make a cluster that references the node group template cluster_values = copy.deepcopy(c.SAMPLE_CLUSTER) cluster_values["node_groups"][0]["node_group_template_id"] = t["id"] cl = self.api.cluster_create(ctx, cluster_values) # Make a cluster template that references the node group template cluster_temp_values = copy.deepcopy(c.SAMPLE_CLT) cluster_temp_values["node_groups"] = cluster_values["node_groups"] clt = self.api.cluster_template_create(ctx, cluster_temp_values) # Set up the expected messages msgs = [ "Node group template {info} in use " "by clusters {clusters}".format(info=u.name_and_id(t), clusters=[cl["name"]]) ] msgs += [ "Node group template {info} in use " "by cluster templates {cluster_temps}".format( info=u.name_and_id(t), cluster_temps=[clt["name"]]) ] msgs += [ "Deletion of node group template {info} failed".format( info=u.name_and_id(t)) ] # Check delete by name option_values = { "tenant_id": t["tenant_id"], "template_name": t["name"] } template_api.set_conf(Config(option_values)) template_api.do_node_group_template_delete() for msg in msgs: self.assertIn(msg, self.logger.warnings) self.logger.clear_log() # Check again with delete by id option_values = {"tenant_id": t["tenant_id"], "id": t["id"]} template_api.set_conf(Config(option_values)) template_api.do_node_group_template_delete_by_id() for msg in msgs: self.assertIn(msg, self.logger.warnings) self.logger.clear_log()
def reverse_cluster_template_updates(ctx, update_info): for template, values in update_info: # values are the original values that we overwrote in the update try: conductor.API.cluster_template_update(ctx, template["id"], values, ignore_prot_on_def=True) except Exception as e: LOG.warning("Rollback of update for cluster " "template {info} failed, {reason}".format( info=u.name_and_id(template), reason=e)) else: LOG.info("Rolled back update for cluster " "template {info}".format(info=u.name_and_id(template)))
def reverse_node_group_template_updates(ctx, update_info): for template, values in update_info: # values are the original values that we overwrote in the update try: conductor.API.node_group_template_update(ctx, template["id"], values, ignore_prot_on_def=True) except Exception as e: LOG.warning(_LW("Rollback of update for node group " "template {info} failed, {reason}").format( info=u.name_and_id(template), reason=e)) else: LOG.info(_LI("Rolled back update for " "node group template {info}").format( info=u.name_and_id(template)))
def test_add_cluster_templates_update_failed(self, reverse_updates, reverse_creates, cl_update): self.logger.clear_log() ctx = context.ctx() cl_update.side_effect = Exception("mistake") # Create a record that will be updated in the db existing = copy.copy(c.SAMPLE_CLT) existing = self.api.cluster_template_create(ctx, existing) # Create the update update = copy.copy(c.SAMPLE_CLT) update["hadoop_version"] = "1" # Create a record that will be new in the db new = copy.copy(c.SAMPLE_CLT) new["name"] = "new_name" clts = [{"template": new, "path": "bar"}, {"template": update, "path": "foo"}] error = template_api.add_cluster_templates(ctx, clts, {}) new = self.api.cluster_template_get_all(ctx, name=new["name"])[0] self.assertTrue(error) self.assertEqual(1, reverse_creates.call_count) # call should have been (ctx, [new]) self.assertEqual(new["id"], reverse_creates.call_args[0][1][0]["id"]) self.assertEqual(1, reverse_updates.call_count) msg = "Update of cluster template {info} failed, mistake".format(info=u.name_and_id(existing)) self.assertIn(msg, self.logger.warnings) self.api.cluster_template_destroy(ctx, new["id"], ignore_default=True) self.api.cluster_template_destroy(ctx, existing["id"], ignore_default=True)
def delete_node_group_template(ctx, template, rollback=False): rollback_msg = " on rollback" if rollback else "" # If we are not deleting something that we just created, # do usage checks to ensure that the template is not in # use by a cluster or a cluster template if not rollback: clusters = conductor.API.cluster_get_all(ctx) cluster_templates = conductor.API.cluster_template_get_all(ctx) cluster_users, template_users = u.check_node_group_template_usage( template["id"], clusters, cluster_templates) if cluster_users: LOG.warning( _LW("Node group template {info} " "in use by clusters {clusters}").format( info=u.name_and_id(template), clusters=cluster_users)) if template_users: LOG.warning( _LW("Node group template {info} " "in use by cluster templates {templates}").format( info=u.name_and_id(template), templates=template_users)) if cluster_users or template_users: LOG.warning( _LW("Deletion of node group template " "{info} failed").format(info=u.name_and_id(template))) return try: conductor.API.node_group_template_destroy(ctx, template["id"], ignore_prot_on_def=True) except Exception as e: LOG.warning( _LW("Deletion of node group template {info} " "failed{rollback}, {reason}").format( info=u.name_and_id(template), reason=e, rollback=rollback_msg)) else: LOG.info( _LI("Deleted node group template {info}{rollback}").format( info=u.name_and_id(template), rollback=rollback_msg))
def test_node_group_template_delete_in_use(self): self.logger.clear_log() ctx = context.ctx() t = self.api.node_group_template_create(ctx, c.SAMPLE_NGT) # Make a cluster that references the node group template cluster_values = copy.deepcopy(c.SAMPLE_CLUSTER) cluster_values["node_groups"][0]["node_group_template_id"] = t["id"] cl = self.api.cluster_create(ctx, cluster_values) # Make a cluster template that references the node group template cluster_temp_values = copy.deepcopy(c.SAMPLE_CLT) cluster_temp_values["node_groups"] = cluster_values["node_groups"] clt = self.api.cluster_template_create(ctx, cluster_temp_values) # Set up the expected messages msgs = ["Node group template {info} in use " "by clusters {clusters}".format( info=u.name_and_id(t), clusters=[cl["name"]])] msgs += ["Node group template {info} in use " "by cluster templates {cluster_temps}".format( info=u.name_and_id(t), cluster_temps=[clt["name"]])] msgs += ["Deletion of node group template {info} failed".format( info=u.name_and_id(t))] # Check delete by name option_values = {"tenant_id": t["tenant_id"], "template_name": t["name"]} template_api.set_conf(Config(option_values)) template_api.do_node_group_template_delete() for msg in msgs: self.assertIn(msg, self.logger.warnings) self.logger.clear_log() # Check again with delete by id option_values = {"tenant_id": t["tenant_id"], "id": t["id"]} template_api.set_conf(Config(option_values)) template_api.do_node_group_template_delete_by_id() for msg in msgs: self.assertIn(msg, self.logger.warnings) self.logger.clear_log()
def test_add_cluster_templates(self): self.logger.clear_log() ctx = context.ctx() # Create a record that will be updated in the db existing = copy.copy(c.SAMPLE_CLT) existing = self.api.cluster_template_create(ctx, existing) # Create the update update = copy.copy(c.SAMPLE_CLT) update["hadoop_version"] = "1" # Create a record that will be new in the db new = copy.copy(c.SAMPLE_CLT) new["name"] = "new_name" clts = [{ "template": update, "path": "foo" }, { "template": new, "path": "bar" }] error = template_api.add_cluster_templates(ctx, clts, {}) self.assertFalse(error) new = self.api.cluster_template_get_all(ctx, name=new["name"])[0] self.assertIsNotNone(new) msg = ("Created cluster template {info} from bar".format( info=u.name_and_id(new))) self.assertIn(msg, self.logger.infos) msg = ("Updated cluster template {info} from foo".format( info=u.name_and_id(existing))) self.assertIn(msg, self.logger.infos) self.api.cluster_template_destroy(ctx, new["id"], ignore_prot_on_def=True) self.api.cluster_template_destroy(ctx, existing["id"], ignore_prot_on_def=True)
def delete_node_group_template(ctx, template, rollback=False): rollback_msg = " on rollback" if rollback else "" # If we are not deleting something that we just created, # do usage checks to ensure that the template is not in # use by a cluster or a cluster template if not rollback: clusters = conductor.API.cluster_get_all(ctx) cluster_templates = conductor.API.cluster_template_get_all(ctx) cluster_users, template_users = u.check_node_group_template_usage( template["id"], clusters, cluster_templates) if cluster_users: LOG.warning("Node group template {info} " "in use by clusters {clusters}".format( info=u.name_and_id(template), clusters=cluster_users)) if template_users: LOG.warning("Node group template {info} " "in use by cluster templates {templates}".format( info=u.name_and_id(template), templates=template_users)) if cluster_users or template_users: LOG.warning("Deletion of node group template " "{info} failed".format(info=u.name_and_id(template))) return try: conductor.API.node_group_template_destroy(ctx, template["id"], ignore_prot_on_def=True) except Exception as e: LOG.warning("Deletion of node group template {info} " "failed{rollback}, {reason}".format( info=u.name_and_id(template), reason=e, rollback=rollback_msg)) else: LOG.info("Deleted node group template {info}{rollback}".format( info=u.name_and_id(template), rollback=rollback_msg))
def test_cluster_template_delete_by_id(self): self.logger.clear_log() ctx = context.ctx() t = self.api.cluster_template_create(ctx, c.SAMPLE_CLT) option_values = {"tenant_id": t["tenant_id"], "id": t["id"]} template_api.set_conf(Config(option_values)) template_api.do_cluster_template_delete_by_id() msg = 'Deleted cluster template {info}'.format(info=u.name_and_id(t)) self.assertIn(msg, self.logger.infos) t = self.api.cluster_template_get(ctx, t["id"]) self.assertIsNone(t)
def do_cluster_template_delete_by_id(): ctx = Context(is_admin=True) # Make sure it's a default t = conductor.API.cluster_template_get(ctx, CONF.command.id) if t: if t["is_default"]: delete_cluster_template(ctx, t) else: LOG.warning("Deletion of cluster template {info} skipped, " "not a default template".format(info=u.name_and_id(t))) else: LOG.warning("Deletion of cluster template {id} failed, " "no such template".format(id=CONF.command.id))
def do_cluster_template_delete_by_id(): ctx = Context(is_admin=True) # Make sure it's a default t = conductor.API.cluster_template_get(ctx, CONF.command.id) if t: if t["is_default"]: delete_cluster_template(ctx, t) else: LOG.warning("Deletion of cluster template {info} skipped, " "not a default template".format(info=u.name_and_id(t))) else: LOG.warning("Deletion of cluster template {id} failed, " "no such template".format(id=CONF.command.id))
def test_cluster_template_delete_by_id_skipped(self): self.logger.clear_log() ctx = context.ctx() template_values = copy.copy(c.SAMPLE_CLT) template_values["is_default"] = False t = self.api.cluster_template_create(ctx, template_values) option_values = {"tenant_id": t["tenant_id"], "id": t["id"]} template_api.set_conf(Config(option_values)) template_api.do_cluster_template_delete_by_id() msg = ("Deletion of cluster template {info} skipped, " "not a default template".format(info=u.name_and_id(t))) self.assertIn(msg, self.logger.warnings) t = self.api.cluster_template_get(ctx, t["id"]) self.assertIsNotNone(t)
def test_node_group_template_delete_by_name(self): self.logger.clear_log() ctx = context.ctx() t = self.api.node_group_template_create(ctx, c.SAMPLE_NGT) option_values = {"tenant_id": t["tenant_id"], "template_name": t["name"]} template_api.set_conf(Config(option_values)) template_api.do_node_group_template_delete() msg = 'Deleted node group template {info}'.format( info=u.name_and_id(t)) self.assertIn(msg, self.logger.infos) t = self.api.node_group_template_get(ctx, t["id"]) self.assertIsNone(t)
def test_add_node_group_templates_update_failed(self, reverse_updates, reverse_creates, ng_update): self.logger.clear_log() ctx = context.ctx() ng_update.side_effect = Exception("mistake") # Create a record that will be updated in the db existing = copy.copy(c.SAMPLE_NGT) existing = self.api.node_group_template_create(ctx, existing) # Create the update update = copy.copy(c.SAMPLE_NGT) update["flavor_id"] = "6" # Create a record that will be new in the db new = copy.copy(c.SAMPLE_NGT) new["name"] = "new_name" ngts = [{ "template": new, "path": "bar" }, { "template": update, "path": "foo" }] ng_info, error = template_api.add_node_group_templates(ctx, ngts) new = self.api.node_group_template_get_all(ctx, name=new["name"])[0] self.assertTrue(error) self.assertEqual(1, reverse_creates.call_count) # call should have been (ctx, [new]) self.assertEqual(new["id"], reverse_creates.call_args[0][1][0]["id"]) self.assertEqual(1, reverse_updates.call_count) msg = ("Update of node group template {info} failed, mistake".format( info=u.name_and_id(existing))) self.assertIn(msg, self.logger.warnings) self.api.node_group_template_destroy(ctx, new["id"], ignore_prot_on_def=True) self.api.node_group_template_destroy(ctx, existing["id"], ignore_prot_on_def=True)
def test_cluster_template_delete_by_id_skipped(self): self.logger.clear_log() ctx = context.ctx() template_values = copy.copy(c.SAMPLE_CLT) template_values["is_default"] = False t = self.api.cluster_template_create(ctx, template_values) option_values = {"tenant_id": t["tenant_id"], "id": t["id"]} template_api.set_conf(Config(option_values)) template_api.do_cluster_template_delete_by_id() msg = ("Deletion of cluster template {info} skipped, " "not a default template".format(info=u.name_and_id(t))) self.assertIn(msg, self.logger.warnings) t = self.api.cluster_template_get(ctx, t["id"]) self.assertIsNotNone(t)
def test_cluster_template_delete_by_id(self): self.logger.clear_log() self.setup_context(tenant_id=None) ctx = context.ctx() t = self.api.cluster_template_create(ctx, c.SAMPLE_CLT) option_values = {"tenant_id": t["tenant_id"], "id": t["id"]} template_api.set_conf(Config(option_values)) template_api.do_cluster_template_delete_by_id() msg = 'Deleted cluster template {info}'.format( info=u.name_and_id(t)) self.assertIn(msg, self.logger.infos) t = self.api.cluster_template_get(ctx, t["id"]) self.assertIsNone(t)
def test_add_cluster_templates_update_failed(self, reverse_updates, reverse_creates, cl_update): self.logger.clear_log() ctx = context.ctx() cl_update.side_effect = Exception("mistake") # Create a record that will be updated in the db existing = copy.copy(c.SAMPLE_CLT) existing = self.api.cluster_template_create(ctx, existing) # Create the update update = copy.copy(c.SAMPLE_CLT) update["hadoop_version"] = "1" # Create a record that will be new in the db new = copy.copy(c.SAMPLE_CLT) new["name"] = "new_name" clts = [{ "template": new, "path": "bar" }, { "template": update, "path": "foo" }] error = template_api.add_cluster_templates(ctx, clts, {}) new = self.api.cluster_template_get_all(ctx, name=new["name"])[0] self.assertTrue(error) reverse_creates.assert_called_once() # call should have been (ctx, [new]) self.assertEqual(new["id"], reverse_creates.call_args[0][1][0]["id"]) reverse_updates.assert_called_once() msg = ("Update of cluster template {info} failed, mistake".format( info=u.name_and_id(existing))) self.assertIn(msg, self.logger.warnings) self.api.cluster_template_destroy(ctx, new["id"], ignore_default=True) self.api.cluster_template_destroy(ctx, existing["id"], ignore_default=True)
def test_add_node_group_templates_update_failed(self, reverse_updates, reverse_creates, ng_update): self.logger.clear_log() ctx = context.ctx() ng_update.side_effect = Exception("mistake") # Create a record that will be updated in the db existing = copy.copy(c.SAMPLE_NGT) existing = self.api.node_group_template_create(ctx, existing) # Create the update update = copy.copy(c.SAMPLE_NGT) update["flavor_id"] = "6" # Create a record that will be new in the db new = copy.copy(c.SAMPLE_NGT) new["name"] = "new_name" ngts = [{"template": new, "path": "bar"}, {"template": update, "path": "foo"}] ng_info, error = template_api.add_node_group_templates(ctx, ngts) new = self.api.node_group_template_get_all(ctx, name=new["name"])[0] self.assertTrue(error) reverse_creates.assert_called_once() # call should have been (ctx, [new]) self.assertEqual(new["id"], reverse_creates.call_args[0][1][0]["id"]) reverse_updates.assert_called_once() msg = ("Update of node group template {info} failed, mistake".format( info=u.name_and_id(existing))) self.assertIn(msg, self.logger.warnings) self.api.node_group_template_destroy(ctx, new["id"], ignore_default=True) self.api.node_group_template_destroy(ctx, existing["id"], ignore_default=True)
def test_do_delete(self): self.logger.clear_log() ctx = context.ctx() # Make some plugins to delete ngt, clt = self._make_templates(ctx, "first", "plugin", "v1") # Make some more for the same plugin, different version ngt2, clt2 = self._make_templates(ctx, "second", "plugin", "v2") # Make another set for a different plugin, overlapping version safe_ngt, safe_clt = self._make_templates(ctx, "third", "plugin2", "v1") # Run a delete by plugin name/version for the first set option_values = {"tenant_id": ngt["tenant_id"], "plugin_name": [ngt["plugin_name"]], "plugin_version": [ngt["hadoop_version"]]} template_api.set_conf(Config(option_values)) # Should delete clt and then ngt, check for messages in order template_api.do_delete() msgs = ["Deleted cluster template {info}".format( info=u.name_and_id(clt))] msgs += ["Deleted node group template {info}".format( info=u.name_and_id(ngt))] self.assertEqual(msgs, self.logger.infos) self.assertIsNone(self.api.node_group_template_get(ctx, ngt["id"])) self.assertIsNone(self.api.cluster_template_get(ctx, clt["id"])) # Make sure the other templates are still there self.assertIsNotNone(self.api.node_group_template_get(ctx, ngt2["id"])) self.assertIsNotNone(self.api.cluster_template_get(ctx, clt2["id"])) self.assertIsNotNone(self.api.node_group_template_get(ctx, safe_ngt["id"])) self.assertIsNotNone(self.api.cluster_template_get(ctx, safe_clt["id"])) # Run delete again for the plugin but with no version specified self.logger.clear_log() option_values = {"tenant_id": ngt2["tenant_id"], "plugin_name": [ngt2["plugin_name"]], "plugin_version": None} template_api.set_conf(Config(option_values)) # Should delete clt2 and then ngt2, check for messages in order template_api.do_delete() msgs = ["Deleted cluster template {info}".format( info=u.name_and_id(clt2))] msgs += ["Deleted node group template {info}".format( info=u.name_and_id(ngt2))] self.assertEqual(msgs, self.logger.infos) self.assertIsNone(self.api.node_group_template_get(ctx, ngt2["id"])) self.assertIsNone(self.api.cluster_template_get(ctx, clt2["id"])) # Make sure the other templates are still there self.assertIsNotNone(self.api.node_group_template_get(ctx, safe_ngt["id"])) self.assertIsNotNone(self.api.cluster_template_get(ctx, safe_clt["id"]))
def add_node_group_templates(ctx, node_groups): error = False ng_info = {"ids": {}, "created": [], "updated": []} def do_reversals(ng_info): reverse_node_group_template_updates(ctx, ng_info["updated"]) reverse_node_group_template_creates(ctx, ng_info["created"]) return {}, True try: for ng in node_groups: template = ng['template'] current = u.find_node_group_template_by_name(ctx, template['name']) if current: # Track what we see in the current template that is different # from our update values. Save it for possible rollback. # Note, this is not perfect because it does not recurse through # nested structures to get an exact diff, but it ensures that # we track only fields that are valid in the JSON schema updated_fields = u.value_diff(current.to_dict(), template) # Always attempt to update. Since the template value is a # combination of JSON and config values, there is no useful # timestamp we can use to skip an update. # If sqlalchemy determines no change in fields, it will not # mark it as updated. try: template = conductor.API.node_group_template_update( ctx, current['id'], template, ignore_prot_on_def=True) except Exception as e: LOG.warning( _LW("Update of node group template {info} " "failed, {reason}").format( info=u.name_and_id(current), reason=e)) raise Handled() if template['updated_at'] != current['updated_at']: ng_info["updated"].append((template, updated_fields)) LOG.info( _LI("Updated node group template {info} " "from {path}").format(info=u.name_and_id(template), path=ng["path"])) else: LOG.debug("No change to node group template {info} " "from {path}".format(info=u.name_and_id(current), path=ng['path'])) else: template['is_default'] = True try: template = conductor.API.node_group_template_create( ctx, template) except Exception as e: LOG.warning( _LW("Creation of node group template " "from {path} failed, {reason}").format( path=ng['path'], reason=e)) raise Handled() ng_info["created"].append(template) LOG.info( _LI("Created node group template {info} " "from {path}").format(info=u.name_and_id(template), path=ng["path"])) # For the purposes of substitution we need a dict of id by name ng_info["ids"][template['name']] = template['id'] except Handled: ng_info, error = do_reversals(ng_info) except Exception as e: LOG.warning( _LW("Unhandled exception while processing " "node group templates, {reason}").format(reason=e)) ng_info, error = do_reversals(ng_info) return ng_info, error
def add_cluster_templates(ctx, clusters, ng_dict): '''Add cluster templates to the database. The value of any node_group_template_id fields in cluster templates which reference a node group template in ng_dict by name will be changed to the id of the node group template. If there is an error in creating or updating a template, any templates that have already been created will be delete and any updates will be reversed. :param clusters: a list of dictionaries. Each dictionary has a "template" entry holding the cluster template and a "path" entry holding the path of the file from which the template was read. :param ng_dict: a dictionary of node group template ids keyed by node group template names ''' error = False created = [] updated = [] def do_reversals(created, updated): reverse_cluster_template_updates(ctx, updated) reverse_cluster_template_creates(ctx, created) return True try: for cl in clusters: template = cl['template'] # Fix up node_group_template_id fields u.substitute_ng_ids(template, ng_dict) # Name + tenant_id is unique, so search by name current = u.find_cluster_template_by_name(ctx, template['name']) if current: # Track what we see in the current template that is different # from our update values. Save it for possible rollback. # Note, this is not perfect because it does not recurse through # nested structures to get an exact diff, but it ensures that # we track only fields that are valid in the JSON schema updated_fields = u.value_diff(current.to_dict(), template) # Always attempt to update. Since the template value is a # combination of JSON and config values, there is no useful # timestamp we can use to skip an update. # If sqlalchemy determines no change in fields, it will not # mark it as updated. # TODO(tmckay): why when I change the count in an # entry in node_groups does it not count as an update? # Probably a bug try: template = conductor.API.cluster_template_update( ctx, current['id'], template, ignore_prot_on_def=True) except Exception as e: LOG.warning( _LW("Update of cluster template {info} " "failed, {reason}").format( info=u.name_and_id(current), reason=e)) raise Handled() if template['updated_at'] != current['updated_at']: updated.append((template, updated_fields)) LOG.info( _LI("Updated cluster template {info} " "from {path}").format(info=u.name_and_id(template), path=cl['path'])) else: LOG.debug("No change to cluster template {info} " "from {path}".format(info=u.name_and_id(current), path=cl["path"])) else: template["is_default"] = True try: template = conductor.API.cluster_template_create( ctx, template) except Exception as e: LOG.warning( _LW("Creation of cluster template " "from {path} failed, {reason}").format( path=cl['path'], reason=e)) raise Handled() created.append(template) LOG.info( _LI("Created cluster template {info} " "from {path}").format(info=u.name_and_id(template), path=cl['path'])) except Handled: error = do_reversals(created, updated) except Exception as e: LOG.warning( _LW("Unhandled exception while processing " "cluster templates, {reason}").format(reason=e)) error = do_reversals(created, updated) return error
def test_do_delete(self): self.logger.clear_log() ctx = context.ctx() # Make some plugins to delete ngt, clt = self._make_templates(ctx, "first", "plugin", "v1") # Make some more for the same plugin, different version ngt2, clt2 = self._make_templates(ctx, "second", "plugin", "v2") # Make another set for a different plugin, overlapping version safe_ngt, safe_clt = self._make_templates(ctx, "third", "plugin2", "v1") # Run a delete by plugin name/version for the first set option_values = {"tenant_id": ngt["tenant_id"], "plugin_name": [ngt["plugin_name"]], "plugin_version": [ngt["hadoop_version"]]} template_api.set_conf(Config(option_values)) # Should delete clt and then ngt, check for messages in order template_api.do_delete() msgs = ["Deleted cluster template {info}".format( info=u.name_and_id(clt))] msgs += ["Deleted node group template {info}".format( info=u.name_and_id(ngt))] self.assertEqual(msgs, self.logger.infos) self.assertIsNone(self.api.node_group_template_get(ctx, ngt["id"])) self.assertIsNone(self.api.cluster_template_get(ctx, clt["id"])) # Make sure the other templates are still there self.assertIsNotNone(self.api.node_group_template_get(ctx, ngt2["id"])) self.assertIsNotNone(self.api.cluster_template_get(ctx, clt2["id"])) self.assertIsNotNone(self.api.node_group_template_get(ctx, safe_ngt["id"])) self.assertIsNotNone(self.api.cluster_template_get(ctx, safe_clt["id"])) # Run delete again for the plugin but with no version specified self.logger.clear_log() option_values = {"tenant_id": ngt2["tenant_id"], "plugin_name": [ngt2["plugin_name"]], "plugin_version": None} template_api.set_conf(Config(option_values)) # Should delete clt2 and then ngt2, check for messages in order template_api.do_delete() msgs = ["Deleted cluster template {info}".format( info=u.name_and_id(clt2))] msgs += ["Deleted node group template {info}".format( info=u.name_and_id(ngt2))] self.assertEqual(msgs, self.logger.infos) self.assertIsNone(self.api.node_group_template_get(ctx, ngt2["id"])) self.assertIsNone(self.api.cluster_template_get(ctx, clt2["id"])) # Make sure the other templates are still there self.assertIsNotNone(self.api.node_group_template_get(ctx, safe_ngt["id"])) self.assertIsNotNone(self.api.cluster_template_get(ctx, safe_clt["id"]))
def test_add_node_group_templates(self): self.logger.clear_log() ctx = context.ctx() # Create a record that will be updated in the db existing = copy.copy(c.SAMPLE_NGT) existing = self.api.node_group_template_create(ctx, existing) # Create the update update = copy.copy(c.SAMPLE_NGT) update["flavor_id"] = "6" # Create a record that will be new in the db new = copy.copy(c.SAMPLE_NGT) new["name"] = "new_name" ngts = [{ "template": update, "path": "foo" }, { "template": new, "path": "bar" }] ng_info, error = template_api.add_node_group_templates(ctx, ngts) self.assertFalse(error) new = self.api.node_group_template_get_all(ctx, name=new["name"])[0] self.assertIsNotNone(new) # ng_info["created"] is a list of templates that were created self.assertEqual(1, len(ng_info["created"])) self.assertEqual(new["id"], ng_info["created"][0]["id"]) # ng_info["updated"] is a list of tuples for templates that # were updated. First element in the tuple is the template, # second is a dictionary of fields that were updated. self.assertEqual(1, len(ng_info["updated"])) self.assertEqual(existing["id"], ng_info["updated"][0][0]["id"]) self.assertEqual({"flavor_id": "42"}, ng_info["updated"][0][1]) # ng_info["dict"] is a dictionary of name/id pairs self.assertEqual( { new["name"]: new["id"], existing["name"]: existing["id"] }, ng_info["ids"]) msg = ("Created node group template {info} from bar".format( info=u.name_and_id(new))) self.assertIn(msg, self.logger.infos) msg = ("Updated node group template {info} from foo".format( info=u.name_and_id(existing))) self.assertIn(msg, self.logger.infos) self.api.node_group_template_destroy(ctx, new["id"], ignore_prot_on_def=True) self.api.node_group_template_destroy(ctx, existing["id"], ignore_prot_on_def=True)
def add_node_group_templates(ctx, node_groups): error = False ng_info = {"ids": {}, "created": [], "updated": []} def do_reversals(ng_info): reverse_node_group_template_updates(ctx, ng_info["updated"]) reverse_node_group_template_creates(ctx, ng_info["created"]) return {}, True try: for ng in node_groups: template = ng['template'] current = u.find_node_group_template_by_name(ctx, template['name']) if current: # Track what we see in the current template that is different # from our update values. Save it for possible rollback. # Note, this is not perfect because it does not recurse through # nested structures to get an exact diff, but it ensures that # we track only fields that are valid in the JSON schema updated_fields = u.value_diff(current.to_dict(), template) # Always attempt to update. Since the template value is a # combination of JSON and config values, there is no useful # timestamp we can use to skip an update. # If sqlalchemy determines no change in fields, it will not # mark it as updated. try: template = conductor.API.node_group_template_update( ctx, current['id'], template, ignore_default=True) except Exception as e: LOG.warning(_LW("Update of node group template {info} " "failed, {reason}").format( info=u.name_and_id(current), reason=e)) raise Handled() if template['updated_at'] != current['updated_at']: ng_info["updated"].append((template, updated_fields)) LOG.info(_LI("Updated node group template {info} " "from {path}").format( info=u.name_and_id(template), path=ng["path"])) else: LOG.debug("No change to node group template {info} " "from {path}".format( info=u.name_and_id(current), path=ng['path'])) else: template['is_default'] = True try: template = conductor.API.node_group_template_create( ctx, template) except Exception as e: LOG.warning(_LW("Creation of node group template " "from {path} failed, {reason}").format( path=ng['path'], reason=e)) raise Handled() ng_info["created"].append(template) LOG.info(_LI("Created node group template {info} " "from {path}").format(info=u.name_and_id(template), path=ng["path"])) # For the purposes of substituion we need a dict of id by name ng_info["ids"][template['name']] = template['id'] except Handled: ng_info, error = do_reversals(ng_info) except Exception as e: LOG.warning(_LW("Unhandled exception while processing " "node group templates, {reason}").format(reason=e)) ng_info, error = do_reversals(ng_info) return ng_info, error
def add_cluster_templates(ctx, clusters, ng_dict): '''Add cluster templates to the database. The value of any node_group_template_id fields in cluster templates which reference a node group template in ng_dict by name will be changed to the id of the node group template. If there is an error in creating or updating a template, any templates that have already been created will be delete and any updates will be reversed. :param clusters: a list of dictionaries. Each dictionary has a "template" entry holding the cluster template and a "path" entry holding the path of the file from which the template was read. :param ng_dict: a dictionary of node group template ids keyed by node group template names ''' error = False created = [] updated = [] def do_reversals(created, updated): reverse_cluster_template_updates(ctx, updated) reverse_cluster_template_creates(ctx, created) return True try: for cl in clusters: template = cl['template'] # Fix up node_group_template_id fields u.substitute_ng_ids(template, ng_dict) # Name + tenant_id is unique, so search by name current = u.find_cluster_template_by_name(ctx, template['name']) if current: # Track what we see in the current template that is different # from our update values. Save it for possible rollback. # Note, this is not perfect because it does not recurse through # nested structures to get an exact diff, but it ensures that # we track only fields that are valid in the JSON schema updated_fields = u.value_diff(current.to_dict(), template) # Always attempt to update. Since the template value is a # combination of JSON and config values, there is no useful # timestamp we can use to skip an update. # If sqlalchemy determines no change in fields, it will not # mark it as updated. # TODO(tmckay): why when I change the count in an # entry in node_groups does it not count as an update? # Probably a bug try: template = conductor.API.cluster_template_update( ctx, current['id'], template, ignore_default=True) except Exception as e: LOG.warning(_LW("Update of cluster template {info} " "failed, {reason}").format( info=u.name_and_id(current), reason=e)) raise Handled() if template['updated_at'] != current['updated_at']: updated.append((template, updated_fields)) LOG.info(_LI("Updated cluster template {info} " "from {path}").format( info=u.name_and_id(template), path=cl['path'])) else: LOG.debug("No change to cluster template {info} " "from {path}".format(info=u.name_and_id(current), path=cl["path"])) else: template["is_default"] = True try: template = conductor.API.cluster_template_create(ctx, template) except Exception as e: LOG.warning(_LW("Creation of cluster template " "from {path} failed, {reason}").format( path=cl['path'], reason=e)) raise Handled() created.append(template) LOG.info(_LI("Created cluster template {info} " "from {path}").format(info=u.name_and_id(template), path=cl['path'])) except Handled: error = do_reversals(created, updated) except Exception as e: LOG.warning(_LW("Unhandled exception while processing " "cluster templates, {reason}").format(reason=e)) error = do_reversals(created, updated) return error