def dereference(self, reference_type, reference_value, token, indent): if self.trace: uprint("%sdereference (%s, %s, %s)" % (indent, reference_type, reference_value, token)) indent += " " if reference_type == "resource": return self.dereference_resource(reference_value, token, indent + " ") elif reference_type == "resource-section": value_resource_name, value_section_name = ( reference_value.split(".")) return self.dereference_resource(value_resource_name, value_section_name + "_" + token, indent + " ") elif reference_type == "value": if not isinstance(reference_value, dict): if self.trace: uprint("%sCan't dereference '%s' from a %s" % (indent, token, type(reference_value))) return False, None, None if token in reference_value: return True, "value", reference_value[token] elif str(token) in reference_value: return True, "value", reference_value[str(token)] else: if self.trace: uprint("%sToken '%s' not found in %s reference '%s'" % (indent, token, reference_type, reference_value)) return False, None, None if self.trace: uprint("%svalue not present: %s" % (indent, token)) return False, None, None
def resolve_resource_values(self): num_resolved = None pass_count = 0 while num_resolved is None or num_resolved > 0: if self.trace: uprint("resolve_resource_values () pass %s" % (pass_count + 1)) num_resolved = 0 for resource \ in self.resources.values (): num_resolved += (self.resolve_resource_values_one(resource)) pass_count += 1
def resolve_resource_values (self): num_resolved = None pass_count = 0 while num_resolved is None or num_resolved > 0: if self.trace: uprint ( "resolve_resource_values () pass %s" % ( pass_count + 1)) num_resolved = 0 for resource \ in self.resources.values (): num_resolved += ( self.resolve_resource_values_one ( resource)) pass_count += 1
def resolve_resource_values_one (self, resource): num_resolved = 0 for prefix, data \ in resource.not_yet_resolved.items (): if isinstance (data, dict): for name, value in data.items (): if self.trace: uprint ( " resolve %s.%s.%s" % ( resource.unique_name, prefix, name)) full_name = "_".join ([ prefix, name, ]) if full_name in resource.resolved: continue success, resolved = ( self.resolve_value_real ( resource, value, " ")) if not success: if self.trace: uprint ( " failed to resolve") continue resource.resolve ( [ prefix, name ], resolved) if self.trace: uprint ( " resolved successfully") num_resolved += 1 elif prefix not in resource.resolved: if self.trace: uprint ( " resolve %s.%s" % ( resource.unique_name, prefix)) success, resolved = ( self.resolve_value_real ( resource, data, " ")) if not success: continue resource.resolve ( [ prefix ], resolved) if self.trace: uprint ( " resolved successfully") num_resolved += 1 return num_resolved
def dereference_resource ( self, resource_source, token, indent): resource = ( self.find_resource ( resource_source)) if self.trace: uprint ( "%sdereference_resource (%s, %s)" % ( indent, resource.unique_name, token)) indent = indent + " " for reference \ in resource.resource_class.references: if token != reference ["name"]: continue if reference ["type"] == "resource": target_success, target_name = ( self.resolve_value_real ( resource, reference ["value"], indent)) if not target_success: return False, None, None if not target_name in self.resources: raise Exception () if "section" in reference: target_resource = ( self.resources [ target_name]) target_data = ( target_resource.get ( reference ["section"])) if self.trace: uprint ( "%sfound resource class section reference: %s" % ( indent, token)) target_combined_name = ".".join ([ target_name, reference ["section"], ]) return True, "resource-section", target_combined_name else: target_resource = ( self.find_resource ( target_name)) if self.trace: uprint ( "%sfound resource class reference: %s" % ( indent, token)) return True, "resource", target_resource.unique_name else: raise Exception () if resource.has_resolved (token): if self.trace: uprint ( "%sfound in resource: %s" % ( indent, token)) return True, "value", resource.get_resolved (token) if token == "parent": parent_name = ( "%s/%s" % ( resource.resource_class.parent_namespace, resource.identity_parent)) parent = ( self.resources [ parent_name]) if self.trace: uprint ( "%srecurse parent: %s" % ( indent, parent_name)) return True, "resource", parent_name if token in self.all: if self.trace: uprint ( "%sfound in globals: %s" % ( indent, token)) unresolved_value = ( self.all [token]) success, resolved_value = ( self.resolve_value_real ( resource, unresolved_value, indent)) if success: return True, "value", resolved_value else: return False, None, None return False, None, None
def dereference ( self, reference_type, reference_value, token, indent): if self.trace: uprint ( "%sdereference (%s, %s, %s)" % ( indent, reference_type, reference_value, token)) indent += " " if reference_type == "resource": return self.dereference_resource ( reference_value, token, indent) elif reference_type == "resource-section": value_resource_name, value_section_name = ( reference_value.split (".")) return self.dereference_resource ( value_resource_name, value_section_name + "_" + token, indent) elif reference_type == "value": if not isinstance (reference_value, dict): raise Exception ( "Can't dereference '%s' from a %s" % ( token, type (reference_value))) if token in reference_value: return True, "value", reference_value [token] elif str (token) in reference_value: return True, "value", reference_value [str (token)] else: raise Exception ( "Token '%s' not found in %s reference '%s'" % ( token, reference_type, reference_value)) elif self.trace: uprint ( "%svalue not present: %s" % ( indent, token)) return False, None, None
def parse_simple ( self, tokens, token_index, resource_source, indent): token = ( tokens [token_index]) resource = ( self.find_resource ( resource_source)) if token [0] == "'": string_value = ( re.sub ( r"\\(.)", lambda match: match.group (1), token [1 : -1])) return True, token_index + 1, "value", string_value if token == "[": new_token_index = ( token_index + 1) items = [] while tokens [new_token_index] != "]": success, new_token_index, item = ( self.parse_expression ( tokens, new_token_index, resource, indent + " ")) if not success: return False, token_index, None, None items.append ( item) if tokens [new_token_index] == ",": new_token_index += 1 elif tokens [new_token_index] != "]": return False, token_index, None, None return True, new_token_index + 1, "value", items if token == "hostvars": if self.trace: uprint ( "%srecurse hostvars" % ( indent)) return True, token_index + 1, "hostvars", None if token in self.context.project_metadata ["project_data"]: if self.trace: uprint ( "%sfound in project data: %s" % ( indent, token)) unresolved_value = ( self.context.local_data [ self.context.project_metadata ["project_data"] [token]]) success, resolved_value = ( self.resolve_value_real ( resource, unresolved_value, indent + " ")) if not success: return False, None, None, None else: return True, token_index + 1, "value", resolved_value success, value_type, value = ( self.dereference_resource ( resource, token, indent)) if success: return True, token_index + 1, value_type, value if self.trace: uprint ( "%sunable to resolve: %s" % ( indent, token)) return False, token_index, None, None
def parse_expression ( self, tokens, token_index, resource_source, indent): resource = ( self.find_resource ( resource_source)) if self.trace: uprint ( "%sparse_expression ([ '%s' ], %s, %s)" % ( indent, "', '".join (tokens), token_index, resource.unique_name)) indent = indent + " " success, token_index, value_type, value = ( self.parse_simple ( tokens, token_index, resource, indent)) if not success: return False, None, None while token_index < len (tokens): if tokens [token_index] == ".": token_index += 1 token = tokens [token_index] token_index += 1 success, value_type, value = ( self.dereference ( value_type, value, token, indent)) if success: continue else: if self.trace: uprint ( "%svalue not present: %s" % ( indent, token)) return False, None, None elif tokens [token_index] == "|": token_index += 1 if tokens [token_index] == "join" \ and value_type == "value" \ and isinstance (value, list): token_index += 1 value = "".join (value) continue if tokens [token_index] == "keys" \ and value_type == "value": token_index += 1 value = value.keys () continue if tokens [token_index] == "values" \ and value_type == "value": token_index += 1 value = value.values () continue if tokens [token_index + 0] == "substring_before" \ and tokens [token_index + 1] == "(" \ and tokens [token_index + 2] [0] == "'" \ and tokens [token_index + 3] == ")": separator = ( re.sub ( r"\\(.)", lambda match: match.group (1), tokens [token_index + 2] [1 : -1])) token_index += 4 if value_type == "value": value = ( value.partition (separator) [0]) continue raise Exception () return False, None, None elif tokens [token_index] == "[": token_index += 1 success, token_index, unresolved_value = ( self.parse_expression ( tokens, token_index, resource, indent)) if not success: return False, None, None if tokens [token_index] != "]": return False, None, None token_index += 1 success, resolved_value = ( self.resolve_value_real ( resource, unresolved_value, indent)) if not success: return False, None, None if value_type == "resource": success, value_type, value = ( self.dereference_resource ( value, resolved_value, indent)) if success: continue elif value_type == "value": if resolved_value in value: value = value [resolved_value] continue elif str (resolved_value) in value: value = value [str (resolved_value)] continue elif value_type == "hostvars": if resolved_value in self.resources: value_type = "resource" value = resolved_value continue if self.trace: uprint ( "%svalue not present: %s" % ( indent, token)) return False, None, None else: break return True, token_index, value
def resolve_expression ( self, resource_source, name, indent): resource = ( self.find_resource ( resource_source)) if self.trace: uprint ( "%sresolve_expression (%s, %s)" % ( indent, resource.unique_name, name)) indent = indent + " " success, tokens = ( self.tokenize (name)) if not success: if self.trace: uprint ( "%stokenize failed" % ( indent)) return False, None if self.trace: uprint ( "%stokens: '%s'" % ( indent, "', '".join (tokens))) token_index = 0 success, token_index, value = ( self.parse_expression ( tokens, token_index, resource, indent)) if not success: if self.trace: uprint ( "%sparse failed" % ( indent)) return False, None if token_index < len (tokens): if self.trace: uprint ( "%sonly used %s/%s tokens" % ( indent, token_index, len (tokens))) return False, None if self.trace: uprint ( "%ssuccess: %s" % ( indent, value)) return True, value
def resolve_value_real ( self, resource_source, value, indent): resource = ( self.find_resource ( resource_source)) if self.trace: uprint (type (value)) uprint ( "%sresolve_value (%s, %s)" % ( indent, resource.unique_name, value)) indent = indent + " " if isinstance (value, list): ret = [] for item in value: success, resolved = ( self.resolve_value_real ( resource, item, indent)) if not success: return False, None ret.append (resolved) return True, ret elif isinstance (value, dict): ret = collections.OrderedDict () for key, item in value.items (): success, resolved = self.resolve_value_real ( resource, item, indent) if not success: return False, None ret [key] = resolved return True, ret elif isinstance (value, str) \ or isinstance (value, unicode): match = ( re.search ( r"^\{\{\s*([^{}]*\S)\s*\}\}$", value)) if match: return self.resolve_expression ( resource, match.group (1), indent) else: ret = "" last_pos = 0 for match in re.finditer (r"\{\{\s*(.*?)\s*\}\}", value): ret += value [last_pos : match.start ()] success, resolved = ( self.resolve_expression ( resource, match.group (1), indent)) if not success: return False, None ret += unicode (resolved) last_pos = match.end () ret += value [last_pos :] return True, ret else: return False, None
def parse_expression(self, tokens, token_index, resource_source, indent): resource = (self.find_resource(resource_source)) if self.trace: uprint("%sparse_expression ([ '%s' ], %s, %s)" % (indent, "', '".join(tokens), token_index, resource.name())) indent = indent + " " success, token_index, value_type, value = (self.parse_simple( tokens, token_index, resource, indent + " ")) if not success: return False, None, None while token_index < len(tokens): if tokens[token_index] == ".": if self.trace: uprint("%sparse simple attribute - x.y" % (indent)) token_index += 1 token = tokens[token_index] token_index += 1 success, value_type, value = (self.dereference( value_type, value, token, indent + " ")) if success: if self.trace: uprint("%sresult - .%s = %s: %s" % (indent, token, value_type, value)) continue else: if self.trace: uprint("%svalue not present: %s" % (indent, token)) return False, None, None elif tokens[token_index] == "|": token_index += 1 if tokens [token_index] == "keys" \ and value_type == "value": token_index += 1 value = value.keys() continue if tokens [token_index] == "values" \ and value_type == "value": token_index += 1 value = value.values() continue if tokens [token_index] == "hash" \ and value_type == "value" \ and ( isinstance (value, str) or isinstance (value, unicode) ) and tokens [token_index + 1] == "(": token_index += 2 success, token_index, hash_type = (self.parse_expression( tokens, token_index, resource, indent + " ")) if not success: return False, None, None if hash_type == "sha1": value = hashlib.sha1(value).hexdigest() else: return False, None, None if tokens[token_index] != ")": return False, None, None token_index += 1 continue if tokens [token_index] == "join" \ and value_type == "value" \ and isinstance (value, list): token_index += 1 value = "".join(value) continue if tokens [token_index + 0] == "union" \ and tokens [token_index + 1] == "(" \ and value_type == "value" \ and isinstance (value, list): token_index += 2 success, token_index, union_value = (self.parse_expression( tokens, token_index, resource, indent + " ")) if not success: return False, None, None if not isinstance(union_value, list): return False, None, None if tokens[token_index] != ")": return False, None, None token_index += 1 item_set = set() new_value = list() for item in value + union_value: if isinstance(item, dict): new_value.append(item) elif item not in item_set: new_value.append(item) item_set.add(item) value = new_value continue if tokens [token_index + 0] == "substring_before" \ and tokens [token_index + 1] == "(" \ and tokens [token_index + 2] [0] == "'" \ and tokens [token_index + 3] == ")": separator = (re.sub(r"\\(.)", lambda match: match.group(1), tokens[token_index + 2][1:-1])) token_index += 4 if value_type == "value": value = (value.partition(separator)[0]) continue raise Exception() if tokens [token_index + 0] == "not_empty_string" \ and value_type == "value": token_index += 1 string_value = (unicode(value)) if string_value != "": value = string_value else: return False, None, None continue if tokens [token_index + 0] == "default" \ and tokens [token_index + 1] == "(" \ and value_type == "value": token_index += 2 success, token_index, default_value = ( self.parse_expression(tokens, token_index, resource, indent + " ")) if not success: return False, None, None if tokens[token_index] != ")": return False, None, None token_index += 1 if value is None: value = default_value continue return False, None, None elif tokens [token_index + 0] == "if" \ and value_type == "value": token_index += 1 success, token_index, test_value = (self.parse_expression( tokens, token_index, resource, indent + " ")) if not success: return False, None, None if tokens[token_index] != "else": return False, None, None token_index += 1 success, token_index, false_value = (self.parse_expression( tokens, token_index, resource, indent + " ")) if not success: return False, None, None if not test_value: value = false_value continue elif tokens [token_index + 0] == "==" \ and value_type == "value": token_index += 1 success, token_index, right_value = (self.parse_expression( tokens, token_index, resource, indent + " ")) if not success: return False, None, None value = (value == right_value) continue elif tokens [token_index + 0] == "!=" \ and value_type == "value": token_index += 1 success, token_index, right_value = (self.parse_expression( tokens, token_index, resource, indent + " ")) if not success: return False, None, None value = (value != right_value) continue elif tokens [token_index + 0] == "+" \ and value_type == "value": if self.trace: uprint("%sparse addition - %s + ?" % (indent, value)) token_index += 1 success, token_index, right_value = (self.parse_expression( tokens, token_index, resource, indent + " ")) if not success: return False, None, None value = (value + right_value) if self.trace: uprint("%sresult: + %s = %s" % (indent, right_value, value)) continue elif tokens[token_index] == "[": if self.trace: uprint("%sparse dynamic attribute - x [y]" % (indent)) token_index += 1 success, token_index, resolved_value = (self.parse_expression( tokens, token_index, resource, indent + " ")) if not success: return False, None, None if tokens[token_index] != "]": return False, None, None token_index += 1 if self.trace: uprint("%sresolved index: %s" % (indent, resolved_value)) if value_type == "resource": success, value_type, value = (self.dereference_resource( value, resolved_value, indent + " ")) if success: if self.trace: uprint("%sresult: [%s] = %s" % (indent, resolved_value, value)) continue elif value_type == "resource-data": project_resource_data = ( self.context.project_metadata["resource_data"][value]) if project_resource_data["key"] != "{{ identity_name }}": continue if project_resource_data ["group"] not in self.namespaces \ and project_resource_data ["group"] not in self.classes: continue if project_resource_data["group"] not in self.namespaces: target_class = ( self.classes[project_resource_data["group"]]) target_namespace = (target_class.namespace_name()) else: target_namespace = (project_resource_data["group"]) success, value_type, value = (self.dereference_resource( "%s/%s" % (target_namespace, resolved_value), project_resource_data["section"], indent + " ")) if success: if self.trace: uprint("%sresult: [%s] = %s" % (indent, resolved_value, value)) continue elif value_type == "value": if resolved_value in value: value = value[resolved_value] if self.trace: uprint("%sresult: [%s] = %s" % (indent, resolved_value, value)) continue elif str(resolved_value) in value: value = value[str(resolved_value)] if self.trace: uprint("%sresult: [%s] = %s" % (indent, resolved_value, value)) continue elif value_type == "hostvars": if resolved_value in self.resources: value_type = "resource" value = resolved_value if self.trace: uprint("%sresult: [%s] = resource: %s" % (indent, resolved_value, value)) continue if self.trace: uprint("%svalue not present: %s" % (indent, resolved_value)) return False, None, None else: break return True, token_index, value
def resolve_expression(self, resource_source, name, indent): resource = (self.find_resource(resource_source)) if self.trace: uprint("%sresolve_expression (%s, %s)" % (indent, resource.name(), name)) indent = indent + " " success, tokens = (self.tokenize(name)) if not success: if self.trace: uprint("%stokenize failed" % (indent)) return False, None if self.trace: uprint("%stokens: '%s'" % (indent, "', '".join(tokens))) token_index = 0 success, token_index, value = (self.parse_expression( tokens, token_index, resource, indent + " ")) if not success: if self.trace: uprint("%sparse failed" % (indent)) return False, None if token_index < len(tokens): if self.trace: uprint("%sonly used %s/%s tokens" % (indent, token_index, len(tokens))) return False, None if self.trace: uprint("%ssuccess: %s" % (indent, value)) return True, value
def resolve_value_real(self, resource_source, value, indent): resource = (self.find_resource(resource_source)) if self.trace: uprint("%sresolve_value (%s, %s)" % (indent, resource.name(), value)) indent = indent + " " if isinstance(value, list): ret = [] for item in value: success, resolved = (self.resolve_value_real( resource, item, indent + " ")) if not success: return False, None ret.append(resolved) return True, ret elif isinstance(value, dict): ret = collections.OrderedDict() for key, item in value.items(): success, resolved = (self.resolve_value_real( resource, item, indent + " ")) if not success: return False, None ret[key] = resolved return True, ret elif isinstance (value, str) \ or isinstance (value, unicode): match = (re.search(r"^\{\{\s*([^{}]*\S)\s*\}\}$", value)) if match: return self.resolve_expression(resource, match.group(1), indent + " ") else: ret = "" last_pos = 0 for match in re.finditer(r"\{\{\s*(.*?)\s*\}\}", value): ret += value[last_pos:match.start()] success, resolved = (self.resolve_expression( resource, match.group(1), indent + " ")) if not success: return False, None ret += unicode(resolved) last_pos = match.end() ret += value[last_pos:] return True, ret else: return False, None
def resolve_resource_values_one(self, resource): num_resolved = 0 for prefix, data \ in resource._not_yet_resolved.items (): if isinstance(data, dict): for name, value in data.items(): if self.trace: uprint(" resolve %s.%s.%s" % (resource.name(), prefix, name)) full_name = "_".join([ prefix, name, ]) if full_name in resource._resolved: continue success, resolved = (self.resolve_value_real( resource, value, " ")) if not success: if self.trace: uprint(" failed to resolve") continue resource.resolve([prefix, name], resolved) if self.trace: uprint(" resolved successfully") num_resolved += 1 elif prefix not in resource._resolved: if self.trace: uprint(" resolve %s.%s" % (resource.name(), prefix)) success, resolved = (self.resolve_value_real( resource, data, " ")) if not success: continue resource.resolve([prefix], resolved) if self.trace: uprint(" resolved successfully") num_resolved += 1 return num_resolved
def dereference_resource(self, resource_source, token, indent): resource = (self.find_resource(resource_source)) if self.trace: uprint("%sdereference_resource (%s, %s)" % (indent, resource.name(), token)) indent = indent + " " for reference \ in itertools.chain ( resource.class_references (), resource.namespace_references (), ): if token != reference["name"]: continue if reference["type"] == "resource": target_success, target_name = (self.resolve_value_real( resource, reference["value"], indent + " ")) if not target_success: return False, None, None if not target_name in self.resources: raise Exception() if "section" in reference: target_resource = (self.resources[target_name]) target_data = (target_resource.get(reference["section"])) if self.trace: uprint("%sfound resource class section reference: %s" % (indent, token)) target_combined_name = ".".join([ target_name, reference["section"], ]) return True, "resource-section", target_combined_name else: target_resource = (self.find_resource(target_name)) if self.trace: uprint("%sfound resource class reference: %s" % (indent, token)) return True, "resource", target_resource.name() elif reference["type"] == "simple": value_success, value = (self.resolve_value_real( resource, reference["value"], indent + " ")) if not value_success: return False, None, None return True, "value", value else: raise Exception() if resource.has_resolved(token): if self.trace: uprint("%sfound in resource: %s" % (indent, token)) return True, "value", resource.get_resolved(token) if token == "parent": parent = (self.resources[resource.parent_name()]) if self.trace: uprint("%srecurse parent: %s" % (indent, resource.parent_name())) return True, "resource", resource.parent_name() if token in self.all: if self.trace: uprint("%sfound in globals: %s" % (indent, token)) unresolved_value = (self.all[token]) success, resolved_value = (self.resolve_value_real( resource, unresolved_value, indent + " ")) if success: return True, "value", resolved_value else: return False, None, None return False, None, None
def parse_simple(self, tokens, token_index, resource_source, indent): resource = (self.find_resource(resource_source)) if self.trace: uprint("%sparse_simple ([ '%s' ], %s, %s)" % (indent, "', '".join(tokens), token_index, resource.name())) indent = indent + " " token = (tokens[token_index]) if token[0] == "'": string_value = (re.sub(r"\\(.)", lambda match: match.group(1), token[1:-1])) return True, token_index + 1, "value", string_value if token == "(": new_token_index = (token_index + 1) success, new_token_index, value = (self.parse_expression( tokens, new_token_index, resource, indent + " ")) if not success: return False, token_index, None, None if tokens[new_token_index] != ")": return False, token_index, None, None return True, new_token_index + 1, "value", value if token == "[": if self.trace: uprint("%sparse dynamic lookup - x [y]" % (indent)) new_token_index = (token_index + 1) items = [] while tokens[new_token_index] != "]": success, new_token_index, item = (self.parse_expression( tokens, new_token_index, resource, indent + " ")) if not success: return False, token_index, None, None items.append(item) if tokens[new_token_index] == ",": new_token_index += 1 elif tokens[new_token_index] != "]": return False, token_index, None, None return True, new_token_index + 1, "value", items if token == "hostvars": if self.trace: uprint("%srecurse hostvars" % (indent)) return True, token_index + 1, "hostvars", None if token in self.context.project_metadata["project_data"]: if self.trace: uprint("%sfound in project data: %s" % (indent, token)) unresolved_value = (self.context.local_data[ self.context.project_metadata["project_data"][token]]) success, resolved_value = (self.resolve_value_real( resource, unresolved_value, indent + " ")) if not success: return False, None, None, None else: return True, token_index + 1, "value", resolved_value if token in self.context.project_metadata["resource_data"]: if self.trace: uprint("%sfound in resource data: %s" % (indent, token)) resource_definition = ( self.context.project_metadata["resource_data"][token]) return True, token_index + 1, "resource-data", token success, value_type, value = (self.dereference_resource( resource, token, indent)) if success: return True, token_index + 1, value_type, value if self.trace: uprint("%sunable to resolve: %s" % (indent, token)) return False, token_index, None, None