def test_substitute_ng_ids(self): cl = { "node_groups": [ {"name": "worker", "node_group_template_id": "{vanilla-worker}", "count": 3}, {"name": "master", "node_group_template_id": "{vanilla-master}", "count": 1}, {"name": "secondary-name", "node_group_template_id": "some_id"}, ] } ng_dict = {"vanilla-worker": 1, "vanilla-master": 2} utils.substitute_ng_ids(cl, ng_dict) self.assertEqual("1", cl["node_groups"][0]["node_group_template_id"]) self.assertEqual("2", cl["node_groups"][1]["node_group_template_id"]) self.assertEqual("some_id", cl["node_groups"][2]["node_group_template_id"])
def check_cluster_templates_valid(ng_templates, cl_templates): # Check that if name references to node group templates # are replaced with a uuid value that the cluster template # passes JSON validation. We don't have the real uuid yet, # but this will allow the validation test. if ng_templates: dummy_uuid = uuid.uuid4() ng_ids = {ng["template"]["name"]: dummy_uuid for ng in ng_templates} else: ng_ids = {} for cl in cl_templates: template = copy.deepcopy(cl["template"]) u.substitute_ng_ids(template, ng_ids) try: ct_validator.validate(template) except jsonschema.ValidationError as e: LOG.warning("Validation for {path} failed, {reason}".format(path=cl["path"], reason=e)) return True return False
def test_substitute_ng_ids(self): cl = {"node_groups": [{"name": "worker", "node_group_template_id": "{vanilla-worker}", "count": 3}, {"name": "master", "node_group_template_id": "{vanilla-master}", "count": 1}, {"name": "secondary-name", "node_group_template_id": "some_id"}]} ng_dict = {"vanilla-worker": 1, "vanilla-master": 2} utils.substitute_ng_ids(cl, ng_dict) self.assertEqual("1", cl["node_groups"][0]["node_group_template_id"]) self.assertEqual("2", cl["node_groups"][1]["node_group_template_id"]) self.assertEqual("some_id", cl["node_groups"][2]["node_group_template_id"])
def check_cluster_templates_valid(ng_templates, cl_templates): # Check that if name references to node group templates # are replaced with a uuid value that the cluster template # passes JSON validation. We don't have the real uuid yet, # but this will allow the validation test. if ng_templates: dummy_uuid = uuid.uuid4() ng_ids = {ng["template"]["name"]: dummy_uuid for ng in ng_templates} else: ng_ids = {} for cl in cl_templates: template = copy.deepcopy(cl["template"]) u.substitute_ng_ids(template, ng_ids) try: ct_validator.validate(template) except jsonschema.ValidationError as e: LOG.warning("Validation for {path} failed, {reason}".format( path=cl["path"], reason=e)) return True return False
def add_cluster_templates(ctx, clusters, ng_dict): '''Add cluster templates to the database. The value of any node_group_template_id fields in cluster templates which reference a node group template in ng_dict by name will be changed to the id of the node group template. If there is an error in creating or updating a template, any templates that have already been created will be delete and any updates will be reversed. :param clusters: a list of dictionaries. Each dictionary has a "template" entry holding the cluster template and a "path" entry holding the path of the file from which the template was read. :param ng_dict: a dictionary of node group template ids keyed by node group template names ''' error = False created = [] updated = [] def do_reversals(created, updated): reverse_cluster_template_updates(ctx, updated) reverse_cluster_template_creates(ctx, created) return True try: for cl in clusters: template = cl['template'] # Fix up node_group_template_id fields u.substitute_ng_ids(template, ng_dict) # Name + tenant_id is unique, so search by name current = u.find_cluster_template_by_name(ctx, template['name']) if current: # Track what we see in the current template that is different # from our update values. Save it for possible rollback. # Note, this is not perfect because it does not recurse through # nested structures to get an exact diff, but it ensures that # we track only fields that are valid in the JSON schema updated_fields = u.value_diff(current.to_dict(), template) # Always attempt to update. Since the template value is a # combination of JSON and config values, there is no useful # timestamp we can use to skip an update. # If sqlalchemy determines no change in fields, it will not # mark it as updated. # TODO(tmckay): why when I change the count in an # entry in node_groups does it not count as an update? # Probably a bug try: template = conductor.API.cluster_template_update( ctx, current['id'], template, ignore_default=True) except Exception as e: LOG.warning(_LW("Update of cluster template {info} " "failed, {reason}").format( info=u.name_and_id(current), reason=e)) raise Handled() if template['updated_at'] != current['updated_at']: updated.append((template, updated_fields)) LOG.info(_LI("Updated cluster template {info} " "from {path}").format( info=u.name_and_id(template), path=cl['path'])) else: LOG.debug("No change to cluster template {info} " "from {path}".format(info=u.name_and_id(current), path=cl["path"])) else: template["is_default"] = True try: template = conductor.API.cluster_template_create(ctx, template) except Exception as e: LOG.warning(_LW("Creation of cluster template " "from {path} failed, {reason}").format( path=cl['path'], reason=e)) raise Handled() created.append(template) LOG.info(_LI("Created cluster template {info} " "from {path}").format(info=u.name_and_id(template), path=cl['path'])) except Handled: error = do_reversals(created, updated) except Exception as e: LOG.warning(_LW("Unhandled exception while processing " "cluster templates, {reason}").format(reason=e)) error = do_reversals(created, updated) return error
def add_cluster_templates(ctx, clusters, ng_dict): '''Add cluster templates to the database. The value of any node_group_template_id fields in cluster templates which reference a node group template in ng_dict by name will be changed to the id of the node group template. If there is an error in creating or updating a template, any templates that have already been created will be delete and any updates will be reversed. :param clusters: a list of dictionaries. Each dictionary has a "template" entry holding the cluster template and a "path" entry holding the path of the file from which the template was read. :param ng_dict: a dictionary of node group template ids keyed by node group template names ''' error = False created = [] updated = [] def do_reversals(created, updated): reverse_cluster_template_updates(ctx, updated) reverse_cluster_template_creates(ctx, created) return True try: for cl in clusters: template = cl['template'] # Fix up node_group_template_id fields u.substitute_ng_ids(template, ng_dict) # Name + tenant_id is unique, so search by name current = u.find_cluster_template_by_name(ctx, template['name']) if current: # Track what we see in the current template that is different # from our update values. Save it for possible rollback. # Note, this is not perfect because it does not recurse through # nested structures to get an exact diff, but it ensures that # we track only fields that are valid in the JSON schema updated_fields = u.value_diff(current.to_dict(), template) # Always attempt to update. Since the template value is a # combination of JSON and config values, there is no useful # timestamp we can use to skip an update. # If sqlalchemy determines no change in fields, it will not # mark it as updated. # TODO(tmckay): why when I change the count in an # entry in node_groups does it not count as an update? # Probably a bug try: template = conductor.API.cluster_template_update( ctx, current['id'], template, ignore_prot_on_def=True) except Exception as e: LOG.warning( _LW("Update of cluster template {info} " "failed, {reason}").format( info=u.name_and_id(current), reason=e)) raise Handled() if template['updated_at'] != current['updated_at']: updated.append((template, updated_fields)) LOG.info( _LI("Updated cluster template {info} " "from {path}").format(info=u.name_and_id(template), path=cl['path'])) else: LOG.debug("No change to cluster template {info} " "from {path}".format(info=u.name_and_id(current), path=cl["path"])) else: template["is_default"] = True try: template = conductor.API.cluster_template_create( ctx, template) except Exception as e: LOG.warning( _LW("Creation of cluster template " "from {path} failed, {reason}").format( path=cl['path'], reason=e)) raise Handled() created.append(template) LOG.info( _LI("Created cluster template {info} " "from {path}").format(info=u.name_and_id(template), path=cl['path'])) except Handled: error = do_reversals(created, updated) except Exception as e: LOG.warning( _LW("Unhandled exception while processing " "cluster templates, {reason}").format(reason=e)) error = do_reversals(created, updated) return error