def __init__(self, content, solution_template='',
default_vals=None, code=''):
# public properties
self.content = content
self.output = ''
self.default_vals = default_vals
self.code = code
self.output_vals = {}
# private properties
self._env = Environment()
self._templateerror = False
self.solution_template = solution_template
try:
self._allvars = find_undeclared_variables(
self._env.parse(self.content))
except TemplateSyntaxError:
self._allvars = set([])
self._templateerror = True
if not bool(self._allvars):
self._allvars = set([])
self._template_output_error = False
try:
self._all_output_vars = find_undeclared_variables(
self._env.parse(self.solution_template))
except TemplateSyntaxError:
self._all_output_vars = {}
self._template_output_error = True
def test_find_undeclared_variables(self):
ast = env.parse('{% set foo = 42 %}{{ bar + foo }}')
x = meta.find_undeclared_variables(ast)
ast = env.parse(
'{% set foo = 42 %}{{ bar + foo }}{% macro meh(x) %}{{ x }}{% endmacro %}{% for item in seq %}{{ muh(item) + meh(seq) }}{% endfor %}'
)
x = meta.find_undeclared_variables(ast)
def test_find_undeclared_variables(self):
ast = env.parse('{% set foo = 42 %}{{ bar + foo }}')
x = meta.find_undeclared_variables(ast)
assert x == set(['bar'])
ast = env.parse('{% set foo = 42 %}{{ bar + foo }}'
'{% macro meh(x) %}{{ x }}{% endmacro %}'
'{% for item in seq %}{{ muh(item) + meh(seq) }}{% endfor %}')
x = meta.find_undeclared_variables(ast)
assert x == set(['bar', 'seq', 'muh'])
def test_find_undeclared_variables(self):
ast = env.parse("{% set foo = 42 %}{{ bar + foo }}")
x = meta.find_undeclared_variables(ast)
assert x == set(["bar"])
ast = env.parse(
"{% set foo = 42 %}{{ bar + foo }}"
"{% macro meh(x) %}{{ x }}{% endmacro %}"
"{% for item in seq %}{{ muh(item) + meh(seq) }}{% endfor %}"
)
x = meta.find_undeclared_variables(ast)
assert x == set(["bar", "seq", "muh"])
def get_tags_in_str(s):
"""
Returns tags (e.g. {{variable}}) in a given string as a set, returns an
empty set for None
"""
# render placeholders
vars_render = meta.find_undeclared_variables(env_render.parse(s))
# runtime placeholders
vars_runtime = meta.find_undeclared_variables(env_runtime.parse(s))
return vars_render | vars_runtime
def test_find_undeclared_variables(self, env):
ast = env.parse('{% set foo = 42 %}{{ bar + foo }}')
x = meta.find_undeclared_variables(ast)
assert x == set(['bar'])
ast = env.parse('{% set foo = 42 %}{{ bar + foo }}'
'{% macro meh(x) %}{{ x }}{% endmacro %}'
'{% for item in seq %}{{ muh(item) + meh(seq) }}'
'{% endfor %}')
x = meta.find_undeclared_variables(ast)
assert x == set(['bar', 'seq', 'muh'])
ast = env.parse('{% for x in range(5) %}{{ x }}{% endfor %}{{ foo }}')
x = meta.find_undeclared_variables(ast)
assert x == set(['foo'])
def test_find_undeclared_variables(self, env):
ast = env.parse("{% set foo = 42 %}{{ bar + foo }}")
x = meta.find_undeclared_variables(ast)
assert x == {"bar"}
ast = env.parse("{% set foo = 42 %}{{ bar + foo }}"
"{% macro meh(x) %}{{ x }}{% endmacro %}"
"{% for item in seq %}{{ muh(item) + meh(seq) }}"
"{% endfor %}")
x = meta.find_undeclared_variables(ast)
assert x == {"bar", "seq", "muh"}
ast = env.parse("{% for x in range(5) %}{{ x }}{% endfor %}{{ foo }}")
x = meta.find_undeclared_variables(ast)
assert x == {"foo"}
def main():
chosen_template = choose_options(load.list_templates())
_out_map = {}
# Jinja magic to get all variables from the template
template = env.loader.get_source(env, chosen_template)
variables = meta.find_undeclared_variables(env.parse(template))
# Iterate over the jinja variables and ask user to set them
# Based on the restrictions in `options.yaml`
for i in variables:
if i in list(options.keys()):
v = get_input(inn=i, option=options[i], type=options[i]["type"])
_out_map[i] = v
# Get any recommended options, based on the variables set or
# Template name.
recommended = recommend_options(_out_map, chosen_template)
print("\nForce set recommended options")
missingo = {}
for k, v in recommended.items():
val = get_input(inn=k, option=v, type=v["type"])
if val:
missingo[k] = val
# Better solution to add missing options would be better
temp = open("templates/" + chosen_template, "r").read()
if missingo:
for i in missingo.items():
temp += "\n{0} {1}".format(*i)
template = Template(temp)
print("")
return template.render(**_out_map)
def generate_edges(resources):
""" The connecting line between two resource is called an edge.
Edges are directed from one vertex (resource) to another,
the graph is called a directed graph. This function is
used to construct the graph.
Args:
resources (list): resources list
"""
for resource in resources:
# convert to raw string
raw = json.dumps(resource)
env = Environment(loader=FileSystemLoader('templates'))
vertex = resource['name']
# direct dependency
neighbors = []
# jinja function find_undeclared_variables found not resolved vars
for neighbour in meta.find_undeclared_variables(env.parse(raw)):
if not neighbour in EDGES_IGNORES and neighbour != vertex\
and not neighbour in neighbors:
neighbors.append(neighbour)
for neighbour in neighbors:
yield (vertex, neighbour,)
# isolated node
if len(neighbors) == 0:
yield (vertex, vertex,)
def __GetIncludeFilesCandidateMessage(self, categolies: list,
env: jinja2.Environment,
template: jinja2.Template):
with pathlib.Path(template.filename).open() as f:
source = f.read()
includes = TemplateIncludeFiles(self.__cmdfile.TemplateDir,
env).Get(source)
print('****************************************')
print('テンプレート変数が不足しているか誤りがあります。')
print('たとえば以下のように入力してください。\n')
tpl_var_names = sorted(meta.find_undeclared_variables(
env.parse(source)),
key=str.lower)
print('$ do', ' '.join(categolies),
self.__GetExampleCommand(tpl_var_names, includes) + '\n')
print('テンプレート変数は以下のコマンドで指定します。')
print(' '.join([self.__tpl_var_prefix + v
for v in tpl_var_names]) + '\n')
if 0 < len(includes):
print('include用テンプレ引数とその値の候補は以下のとおり。')
for i in includes:
print(self.__tpl_var_prefix + i[0], i[1])
print('\n対象テンプレートは以下です。')
print(template.filename)
print('****************************************')
def parseVariables(env, source):
"""
Return the variables referenced in the source Jinja2 template string.
"""
ast = env.parse(source)
variables = list(meta.find_undeclared_variables(ast))
return sorted(variables, cmp=lambda a,b: cmp(source.find(a), source.find(b)))
def verify_template_odt(template_path):
if not os.path.isfile(template_path):
raise FileNotFoundError("File {} not found".format(template_path))
counter = 0
test_string = "a"
print("Verifying template...")
renderer = CustomOdtRenderer()
template = open(template_path, 'rb')
no_params = renderer.render_content_to_xml(template)
# check if each of the fields in csv file exist in template
# print("Checking fields in csv")
for field in fieldnames:
# print("\tChecking '"+field + "'...")
w_param = renderer.render_content_to_xml(template,
**{field: test_string})
if w_param == no_params:
print("\t'{}' not defined in the template".format(field))
counter += 1
# else:
# print("\t\tOK")
# check if any field(s) used in the template is not defined in the csv
# print("Checking variables in template")
parse_content = Environment().parse(renderer.content_original)
undeclared_template_variables = meta.find_undeclared_variables(
parse_content)
for variable in undeclared_template_variables:
# print("\tChecking '"+variable + "'...")
if variable not in fieldnames:
print("\t'{}' not defined in the csv".format(variable))
counter += 1
# else:
# print("\t\tOK")
return counter
def update_from_yaml_template(template_path, parser_env, data):
"""
Given a path to a yaml file and a template parser environment
add all data in the yaml file to the provided data dictionary
"""
import yaml
from jinja2 import meta, UndefinedError
# Find variables in template
content = parser_env.loader.get_source(parser_env, template_path)
parsed_template = parser_env.parse(content[0])
template_variables = meta.find_undeclared_variables(parsed_template)
# Find undeclared variables not already in context
undefined_variables = list(set(template_variables) - set(data))
# Parse template
template = parser_env.get_template(template_path)
yaml_content = template.render(data)
new_variables = yaml.load(yaml_content)
data.update(new_variables)
# If any new variables were in the undeclared variables
list_intersection = filter(
lambda x: x in undefined_variables, new_variables
)
if list_intersection:
data = update_from_yaml_template(template_path, parser_env, data)
elif undefined_variables:
raise UndefinedError('Undefined variables: ' + str(undefined_variables))
return data
def extract_variables(filename):
"""Extract text from template file and get jinja variables"""
extracted = textract.process(filename)
env = Environment(loader=BaseLoader())
content = env.parse(extracted)
variables = meta.find_undeclared_variables(content)
return variables
def analyze_dependency(template_dict):
context = {}
to_render = {}
for k, v in template_dict.items():
if not is_template(str(v)):
context[k] = v
else:
to_render[k] = v
print 'to_render : ', to_render
print 'context : ', context
dependencies = {}
for k, v in to_render.items():
ast = env.parse(v)
dependencies[k] = meta.find_undeclared_variables(ast)
def check_dependencies(dep_chain):
last_idx = len(dep_chain) - 1
last_value = dep_chain[last_idx]
for dependency in dependencies.get(last_value, []):
if dependency in dep_chain:
dep_chain.append(dependency)
return True
dep_chain.append(dependency)
if check_dependencies(dep_chain):
return True
dep_chain.pop()
for k in dependencies:
dep_chain = []
dep_chain.append(k)
if check_dependencies(dep_chain):
print 'Circular dependecy found - ', dep_chain
return True
def convert():
dummy_values = [
"Lorem",
"Ipsum",
"Amet",
"Elit",
"Expositum",
"Dissimile",
"Superiori",
"Laboro",
"Torquate",
"sunt",
]
tpl = Template(request.form["template"])
values = {}
if bool(int(request.form["dummyvalues"])):
# List variables (introspection)
env = Environment()
vars_to_fill = meta.find_undeclared_variables(env.parse(request.form["template"]))
for v in vars_to_fill:
values[v] = choice(dummy_values)
else:
values = json.loads(request.form["values"])
rendered_tpl = tpl.render(values)
if bool(int(request.form["showwhitespaces"])):
# Replace whitespaces with a visible character (will be grayed with javascript)
rendered_tpl = rendered_tpl.replace(" ", u"•")
return rendered_tpl.replace("\n", "<br />")
def find_undeclared_variables_in_order(ast):
"""
Returns a list of undeclared variables **IN ORDER**,
unlike the same function from jinjia.meta
"""
undeclared_set = meta.find_undeclared_variables(
ast) # undeclared, unordered
ordered_nodes = [
node for node in ast.find_all(
nodes.Name) # including declared, but in order
if node.name in undeclared_set # filter out declared
]
result = []
seen = set()
# remove duplicates
for node in ordered_nodes:
name = node.name
if name in seen: # pragma: no cover
continue
seen.add(name)
result.append(name)
return result
def render(obj, param_dict):
""" Use values found in obj, param_dict, and kwargs to populate jinja2
templates in the obj. This operation works in-place on the given obj.
"""
if isinstance(obj, list):
for i, item in enumerate(obj):
obj[i] = render(item, param_dict)
elif isinstance(obj, dict):
for item in obj:
obj[item] = render(obj[item], param_dict)
elif isinstance(obj, basestring):
# Ensure that we have all the key/value pairs we need in the
# mapping object we are applying to the key.
env = jinja2.Environment()
ast = env.parse(obj)
undeclared = meta.find_undeclared_variables(ast)
missing_vars = [var for var in undeclared if var not in param_dict]
if missing_vars:
raise errors.MissingTemplateVariableError(missing_vars, obj)
template = jinja2.Template(obj)
return template.render(param_dict)
return obj
def createMadlib(cls):
template = input("Type your story with double curly braces around each variable name (e.g. {{Name}} is going to {{country}}.)")
env = Environment()
ast = env.parse(template)
variables = list(meta.find_undeclared_variables(ast))
print(variables)
print()
originalStory = {}
for var in variables:
value = input("What is the value of {{" + var + "}} in your original story?")
originalStory[var] = value
print(originalStory)
print()
prompts = {}
for var in variables:
prompt = input("What is the description of {{" + var + "}}?")
prompts[var] = prompt
print(prompts)
print()
madlib = {"template": template,
"originalStory": originalStory,
"prompts": prompts,
"stories": {}
}
madlibName = input("What do you want to name this madlib?")
filename = './madlibs/' + madlibName + ".json"
with open(filename, 'w') as f:
json.dump(madlib, f, indent=4)
print("madlib saved successfully")
def fill_template(param_list):
"""Here we setup the core template engine, we will need to declare any filters we add inside this module, see the 'mine_salt' example below."""
env = Environment(loader=FileSystemLoader(('nginx-templates/', 'mysql-templates/', 'temp/', 'misc-templates/')))
env.filters['mine_salt'] = mine_salt
try:
template_source = env.loader.get_source(env, param_list[1])[0]
except:
raise NameError('Template Name invalid, please check recipe')
template = env.get_template(param_list[1])
parsed_content = env.parse(template_source)
tags = meta.find_undeclared_variables(parsed_content)
tags_input = {}
for tag in tags:
if tag in values_used:
tags_input[tag] = values_used[tag]
else :
print 'Please enter value for ' + tag
temp = raw_input()
if temp.count(' ') :
value_list = temp.split(' ')
tags_input[tag] = value_list
values_used[tag] = value_list
else :
tags_input[tag] = [temp]
values_used[tag] = [temp]
print(template.render(tags_input))
def save_related(self, request, form, formsets, change):
"""
After saving the related objects, remove and add
SnippetTemplateVariables depending on how the template code changed.
"""
super(SnippetTemplateAdmin, self).save_related(request, form, formsets,
change)
# Parse the template code and find any undefined variables.
ast = models.JINJA_ENV.env.parse(form.instance.code)
new_vars = find_undeclared_variables(ast)
var_manager = form.instance.variable_set
# Filter out reserved variable names.
new_vars = [x for x in new_vars if x not in RESERVED_VARIABLES]
# Delete variables not in the new set.
var_manager.filter(~Q(name__in=new_vars)).delete()
# Create variables that don't exist.
for i, variable in enumerate(new_vars, start=1):
obj, _ = models.SnippetTemplateVariable.objects.get_or_create(
template=form.instance, name=variable)
if obj.order == 0:
obj.order = i * 10
obj.save()
def __init__(self, source):
super().__init__()
ast = self.environment.parse(source)
self.source = source
self.variables = find_undeclared_variables(ast)
def evalute_field(field, info):
# return info.content.get(field)
template = Template(field)
ast = Environment().parse(field)
variables = meta.find_undeclared_variables(ast)
d = {key: info.get(key) for key in variables}
return template.render(d)
def analyze_dependency(template_dict):
context = {}
to_render = {}
for k, v in template_dict.items():
if not is_template(str(v)):
context[k] = v
else:
to_render[k] = v
print 'to_render : ', to_render
print 'context : ', context
dependencies = {}
for k, v in to_render.items():
ast = env.parse(v)
dependencies[k] = meta.find_undeclared_variables(ast)
def check_dependencies(dep_chain):
last_idx = len(dep_chain) - 1
last_value = dep_chain[last_idx]
for dependency in dependencies.get(last_value, []):
if dependency in dep_chain:
dep_chain.append(dependency)
return True
dep_chain.append(dependency)
if check_dependencies(dep_chain):
return True
dep_chain.pop()
for k in dependencies:
dep_chain = []
dep_chain.append(k)
if check_dependencies(dep_chain):
print 'Circular dependecy found - ', dep_chain
return True
def get_template_variables(filename):
path = os.path.join(current_app.root_path, "templates")
env = Environment(loader=FileSystemLoader(path))
template_source = env.loader.get_source(env, filename)[0]
parsed_content = env.parse(template_source)
return meta.find_undeclared_variables(parsed_content)
def _validate_dependencies(renderable_params, context):
'''
Validates dependencies between the parameters.
e.g.
{
'a': '{{b}}',
'b': '{{a}}'
}
In this example 'a' requires 'b' for template rendering and vice-versa. There is no way for
these templates to be rendered and will be flagged with an ActionRunnerException.
'''
env = Environment(undefined=StrictUndefined)
dependencies = {}
for k, v in six.iteritems(renderable_params):
template_ast = env.parse(v)
dependencies[k] = meta.find_undeclared_variables(template_ast)
for k, v in six.iteritems(dependencies):
if not _check_availability(k, v, renderable_params, context):
msg = 'Dependecy unsatisfied - %s: %s.' % (k, v)
raise actionrunner.ActionRunnerException(msg)
dep_chain = []
dep_chain.append(k)
if not _check_cyclic(dep_chain, dependencies):
msg = 'Cyclic dependecy found - %s.' % dep_chain
raise actionrunner.ActionRunnerException(msg)
def get_required_vars_from_template(template_name):
"""
Parse the template and return a list of all the variables defined therein
template path is usually something like 'templates/panos/bootstrap.xml'
:param template_name: relative path to the application root to a jinja2 template
:return: set of variable named defined in the template
"""
template_variables = set()
try:
t = Template.query.filter(Template.name == template_name).first()
if t is None:
print('Could not load template %s' % template_name)
return template_variables
# get the jinja environment to use it's parse function
env = jinja2.Environment()
# parse returns an AST that can be send to the meta module
ast = env.parse(t.template)
# return a set of all variable defined in the template
template_variables = meta.find_undeclared_variables(ast)
except TemplateSyntaxError as tse:
print('Could not parse template')
print(tse)
except SQLAlchemyError as sqe:
print('Could not load template variables')
print(sqe)
finally:
return template_variables
def extract_variable(file):
# Load Base Template File
templateloader = FileSystemLoader(searchpath="./Base_Template_Files")
env = Environment(loader=templateloader, autoescape=False)
jinja_filters_func = dir(_jinja_custom_filters)
# Register custom filters into Jinja2 Environment
for func in jinja_filters_func:
if 'jinj_' in func:
func_ref = getattr(_jinja_custom_filters, func)
filter_name = func.split('_')[1]
env.filters[filter_name] = func_ref
try:
template = env.get_template(file)
except TemplateNotFound:
print("\033[91m", "Template File", file,
"not found in Base_Template_Files directory", "\033[0m")
traceback.print_exc()
sys.exit(2)
# Fetch all variables declared in template file
template_source = env.loader.get_source(env, file)
parsed_content = env.parse(template_source)
template_vars = meta.find_undeclared_variables(parsed_content)
print("\n", "\u001b[36;1m", str(len(template_vars)),
"Variables found in template:", file, "\u001b[0m", "\n")
for var in template_vars:
print("\u001b[36;1m", '---------------------------------------------',
"\u001b[0m")
print('\033[93m', var, '\033[0m')
print("\n")
return template
def get_var_templates(template):
env = Environment()
ast = env.parse(template.content)
exceptions = ("vidly_tokenize", "edgecast_tokenize", "akamai_tokenize", "popcorn_url", "event", "poster_url")
undeclared_variables = [x for x in meta.find_undeclared_variables(ast) if x not in exceptions]
return ["%s=" % v for v in undeclared_variables]
def find_undeclared_variables_in_order(ast):
"""
Returns a list of undeclared variables **IN ORDER**,
unlike the same function from jinjia.meta
"""
undeclaredSet = meta.find_undeclared_variables(ast) # undeclared, unordered
orderedNodes = [
node
for node in ast.find_all(nodes.Name) # including declared, but in order
if node.name in undeclaredSet # filter out declared
]
result = []
seen = set()
# remove duplicates
for node in orderedNodes:
name = node.name
if name in seen:
continue
seen.add(name)
result.append(name)
return result
def save_related(self, request, form, formsets, change):
"""
After saving the related objects, remove and add
SnippetTemplateVariables depending on how the template code changed.
"""
super(SnippetTemplateAdmin, self).save_related(request, form, formsets,
change)
# Parse the template code and find any undefined variables.
ast = models.JINJA_ENV.env.parse(form.instance.code)
new_vars = find_undeclared_variables(ast)
var_manager = form.instance.variable_set
# Filter out reserved variable names.
new_vars = [x for x in new_vars if x not in RESERVED_VARIABLES]
# Delete variables not in the new set.
var_manager.filter(~Q(name__in=new_vars)).delete()
# Create variables that don't exist.
for i, variable in enumerate(new_vars, start=1):
obj, _ = models.SnippetTemplateVariable.objects.get_or_create(
template=form.instance, name=variable)
if obj.order == 0:
obj.order = i * 10
obj.save()
def convert():
dummy_values = [ 'Lorem', 'Ipsum', 'Amet', 'Elit', 'Expositum',
'Dissimile', 'Superiori', 'Laboro', 'Torquate', 'sunt',
]
tpl = app.jinja_env.from_string(request.form['template'])
values = {}
if bool(int(request.form['dummyvalues'])):
# List variables (introspection)
env = Environment()
vars_to_fill = meta.find_undeclared_variables(env.parse(request.form['template']))
for v in vars_to_fill:
values[v] = choice(dummy_values)
else:
if request.form['optiontype'] == 'yaml':
values = json.loads(json.dumps(yaml.load(request.form['values'])))
else:
values = json.loads(request.form['values'])
rendered_tpl = tpl.render(values)
if bool(int(request.form['showwhitespaces'])):
# Replace whitespaces with a visible character (will be grayed with javascript)
rendered_tpl = rendered_tpl.replace(' ', u'•')
return rendered_tpl.replace('\n', '<br />')
def validate(self, template_file):
"""
Find all available and overridden vars in a template. Recursively checks all
super templates.
:param template_file: The name of the template file
:return: the set and unset variables
:rtype: tuple
"""
# Get all the info for the CURRENT template
# Get the source of the template
template_source = self.env.loader.get_source(self.env, template_file)[0]
# parse it into an abstract syntax tree
ast = self.env.parse(template_source)
# the UNSET variables in the current template
unset_vars = meta.find_undeclared_variables(ast)
# the SET variables in the current template
set_vars = self.find_set_vars(ast)
# validate the super templates
super_templates = meta.find_referenced_templates(ast)
for template in super_templates:
# Get all the information about the super template recursively
super_unset, super_set = self.validate(template)
# We do it this way so values in derived templates override those in base templates
super_set.update(set_vars)
set_vars = super_set
unset_vars = unset_vars.union(super_unset)
return unset_vars, set_vars
def process_file(env, sections, filename, output_filename):
log("Parsing input template: %s" % filename)
with open(filename, "r") as file_stream:
temp_str = file_stream.read().decode("utf-8")
# do sanity checking on the template to make sure they aren't reffing things
# which will never be replaced with a section.
ast = env.parse(temp_str)
template_vars = meta.find_undeclared_variables(ast)
unused_vars = [var for var in template_vars if var not in sections]
if len(unused_vars) > 0:
raise Exception(
"You have {{ variables }} which are not found in sections: %s" %
(unused_vars, ))
# process the template
temp = Template(temp_str)
output = create_from_template(temp, sections)
# Do these substitutions outside of the ordinary templating system because
# we want them to apply to things like the underlying swagger used to
# generate the templates, not just the top-level sections.
for old, new in substitutions.items():
output = output.replace(old, new)
with open(output_filename, "w") as f:
f.write(output.encode("utf-8"))
log("Output file for: %s" % output_filename)
def get_undeclared(self, root_path, env, filename, context):
with open(os.path.join(root_path, filename)) as f:
t = f.read()
ast = env.parse(t)
undeclared = meta.find_undeclared_variables(ast) - \
set(env.globals.keys()) - context.keys()
return undeclared
def process_file(env, sections, filename, output_filename):
log("Parsing input template: %s" % filename)
with open(filename, "r") as file_stream:
temp_str = file_stream.read().decode("utf-8")
# do sanity checking on the template to make sure they aren't reffing things
# which will never be replaced with a section.
ast = env.parse(temp_str)
template_vars = meta.find_undeclared_variables(ast)
unused_vars = [var for var in template_vars if var not in sections]
if len(unused_vars) > 0:
raise Exception("You have {{ variables }} which are not found in sections: %s" % (unused_vars,))
# process the template
temp = Template(temp_str)
output = create_from_template(temp, sections)
# Do these substitutions outside of the ordinary templating system because
# we want them to apply to things like the underlying swagger used to
# generate the templates, not just the top-level sections.
for old, new in substitutions.items():
output = output.replace(old, new)
with open(output_filename, "w") as f:
f.write(output.encode("utf-8"))
log("Output file for: %s" % output_filename)
def main():
chosen_template = choose_options(load.list_templates())
_out_map = {}
# Jinja magic to get all variables from the template
template = env.loader.get_source(env, chosen_template)
variables = meta.find_undeclared_variables(env.parse(template))
# Iterate over the jinja variables and ask user to set them
# Based on the restrictions in `options.yaml`
for i in variables:
if i in list(options.keys()):
v = get_input(inn=i, option=options[i], type=options[i]["type"])
_out_map[i] = v
# Get any recommended options, based on the variables set or
# Template name.
recommended = recommend_options(_out_map, chosen_template)
print("\nForce set recommended options")
missingo = {}
for k, v in recommended.items():
val = get_input(inn=k, option=v, type=v["type"])
if val:
missingo[k] = val
# Better solution to add missing options would be better
temp = open("templates/" + chosen_template, "r").read()
if missingo:
for i in missingo.items():
temp += "\n{0} {1}".format(*i)
template = Template(temp)
print("")
return template.render(**_out_map)
def template_text(self, dict):
reqs = []
for element in self.elements:
for object in element.children:
if object.text and any([key in object.text for key in dict]):
text = object.text
text = text.replace("’", "'")
text = text.replace("‘", "'")
env = Environment()
ast = env.parse(text)
vars = meta.find_undeclared_variables(ast)
if vars:
bullets = False
font_size = 8
render_dict = {key: dict[key] for key in dict if key in vars}
for var in render_dict:
# turn it into paragraphs if list
# and add bullets
if isinstance(render_dict[var], FancyFont):
ff_item = render_dict[var]
bullets = ff_item.bullets
font_size = ff_item.font_size
render_dict[var] = "\n".join(ff_item.lines)
new_text = Template(text).render(**render_dict)
if new_text != object.text:
reqs.append(
object.update_text(new_text, bullets=bullets, font_size=font_size)
)
self._presentation._mutation(reqs=reqs)
def jinja_find_required_variables(content, name='default_name'):
myenv = jinja2.Environment(loader=jinja2.DictLoader({name: content}))
myenv.filters['bool'] = jinja_filter_bool
myenv.filters['regex_replace'] = jinja_regex_replace
template_source = myenv.loader.get_source(myenv, name)[0]
parsed_content = myenv.parse(template_source)
return meta.find_undeclared_variables(parsed_content)
def _handle_preconditions(configs):
success = True
c_files, messages = _get_filtered_config_files(configs)
if not c_files:
return (False, messages)
variables, messages = _get_variables()
if not variables:
return (False, messages)
for file_location, _ in c_files:
try:
file_path = os.path.join(projectpath,
file_location)
source = loader.get_source(env, file_location)
ast = env.parse(loader.get_source(env, file_location))
used_vars = meta.find_undeclared_variables(ast)
ud_vars = set()
for v in used_vars:
if not v in variables:
ud_vars.add(v)
if ud_vars:
messages.append('Undeclared variables '
'found in %s: %s' % (file_path,
ud_vars))
success = False
except TemplateNotFound:
messages.append('Template not found: %s' % (file_path))
success = False
return (success, messages)
def get_template_variables(config, upload_path):
with open(upload_path + "/test_template.txt", 'w') as f:
f.write(config)
env = Environment()
parsed_config = env.parse(config)
result = list(meta.find_undeclared_variables(parsed_config))
return result
def __init__(self, source:str, environment:Environment) -> None:
self.source = source
parsed = environment.parse(source)
self.variables = meta.find_undeclared_variables(parsed)
self.template = environment.from_string(source)
def parse_string(string, params, autoescape=True):
"""Parses a string using Jinja templating.
Args:
string: the string to be parsed.
params: the parameters to parse the string with.
autoescape: whether to enable autoescaping when parsing.
Returns:
the parsed string, or None if the string could not be parsed.
"""
env = jinja2.Environment(autoescape=autoescape)
env.filters.update(JINJA_FILTERS)
try:
parsed_string = env.parse(string)
except Exception:
raise Exception('Unable to parse string with Jinja: %s' % string)
variables = meta.find_undeclared_variables(parsed_string)
if any([var not in params for var in variables]):
logging.info('Cannot parse %s fully using %s', string, params)
try:
return env.from_string(string).render(params)
except Exception:
logging.error(
'jinja_utils.parse_string() failed with args: %s, %s, %s' %
(string, params, autoescape))
return env.from_string('[CONTENT PARSING ERROR]').render({})
def _insert_eval_data(self, tmp_dict):
"""
Args:
tmp_dict: dictionary of key value from a iteration of view
Returns: if there is any eval expression in view, does the eval using
tmp_dictionary. For example:
---
XMChipStatsTable:
command: show xmchip {{ instance }} pt stats
args:
instance: 0
target: fpc2
item: '*'
view: XMChipStatsView
XMChipStatsView:
regex:
pct_wi_1: 'PCT entries used by all WI-1 streams\s+:\s?(\d+)'
pct_wi_0: 'PCT entries used by all WI-0 streams\s+:\s?(\d+)'
eval:
total_pct: '{{ pct_wi_1 }} + {{ pct_wi_0 }}'
which returns
{'pct_wi_1': 0, 'pct_wi_0': 0, 'total_pct': 0}
"""
if self._view and len(self._view.EVAL) > 0:
for name, expression in self._view.EVAL.items():
variables = meta.find_undeclared_variables(expression)
t = Template(expression)
expression = t.render({k: tmp_dict.get(k) for k in variables})
val = eval(expression)
tmp_dict[name] = val
def _expand(p, context, all_vars, client, getenv, getshell):
if isinstance(p, dict):
return {k: _expand(v, context, all_vars, client, getenv, getshell)
for k, v in p.items()}
elif isinstance(p, (list, tuple, set)):
return type(p)(_expand(v, context, all_vars, client, getenv, getshell)
for v in p)
elif isinstance(p, str):
jinja = Environment()
if getenv and not client:
jinja.globals['env'] = _j_getenv
else:
jinja.globals['env'] = lambda x: _j_passthrough(x, funcname='env')
if getenv and client:
jinja.globals['client_env'] = _j_getenv
else:
jinja.globals['client_env'] = lambda x: _j_passthrough(x, funcname='client_env')
if getshell and not client:
jinja.globals['shell'] = _j_getshell
else:
jinja.globals['shell'] = lambda x: _j_passthrough(x, funcname='shell')
if getshell and client:
jinja.globals['client_shell'] = _j_getshell
else:
jinja.globals['client_shell'] = lambda x: _j_passthrough(x, funcname='client_shell')
ast = jinja.parse(p)
all_vars -= meta.find_undeclared_variables(ast)
return jinja.from_string(p).render(context)
else:
# no expansion
return p
def _validate_dependencies(renderable_params, context):
'''
Validates dependencies between the parameters.
e.g.
{
'a': '{{b}}',
'b': '{{a}}'
}
In this example 'a' requires 'b' for template rendering and vice-versa. There is no way for
these templates to be rendered and will be flagged with an ActionRunnerException.
'''
env = Environment(undefined=StrictUndefined)
dependencies = {}
for k, v in six.iteritems(renderable_params):
template_ast = env.parse(v)
dependencies[k] = meta.find_undeclared_variables(template_ast)
for k, v in six.iteritems(dependencies):
if not _check_availability(k, v, renderable_params, context):
msg = 'Dependecy unsatisfied - %s: %s.' % (k, v)
raise actionrunner.ActionRunnerException(msg)
dep_chain = []
dep_chain.append(k)
if not _check_cyclic(dep_chain, dependencies):
msg = 'Cyclic dependecy found - %s.' % dep_chain
raise actionrunner.ActionRunnerException(msg)
def parse_string(string, params, autoescape=True):
"""Parses a string using Jinja templating.
Args:
string: the string to be parsed.
params: the parameters to parse the string with.
autoescape: whether to enable autoescaping when parsing.
Returns:
the parsed string, or None if the string could not be parsed.
"""
env = jinja2.Environment(autoescape=autoescape)
env.filters.update(JINJA_FILTERS)
try:
parsed_string = env.parse(string)
except Exception:
raise Exception('Unable to parse string with Jinja: %s' % string)
variables = meta.find_undeclared_variables(parsed_string)
if any([var not in params for var in variables]):
logging.info('Cannot parse %s fully using %s', string, params)
try:
return env.from_string(string).render(params)
except Exception:
logging.error(
'jinja_utils.parse_string() failed with args: %s, %s, %s' %
(string, params, autoescape))
return env.from_string('[CONTENT PARSING ERROR]').render({})
def _process(G, name, value):
'''
Determines whether parameter is a template or a value. Adds graph nodes and edges accordingly.
'''
# Jinja defaults to ascii parser in python 2.x unless you set utf-8 support on per module level
# Instead we're just assuming every string to be a unicode string
if isinstance(value, str):
value = to_unicode(value)
complex_value_str = None
if isinstance(value, list) or isinstance(value, dict):
complex_value_str = str(value)
is_jinja_expr = (
jinja_utils.is_jinja_expression(value) or jinja_utils.is_jinja_expression(
complex_value_str
)
)
if is_jinja_expr:
G.add_node(name, template=value)
template_ast = ENV.parse(value)
LOG.debug('Template ast: %s', template_ast)
# Dependencies of the node represent jinja variables used in the template
# We're connecting nodes with an edge for every depencency to traverse them
# in the right order and also make sure that we don't have missing or cyclic
# dependencies upfront.
dependencies = meta.find_undeclared_variables(template_ast)
LOG.debug('Dependencies: %s', dependencies)
if dependencies:
for dependency in dependencies:
G.add_edge(dependency, name)
else:
G.add_node(name, value=value)
def get_tempalte(self, path):
with open(path, "r", encoding='UTF-8') as f:
contents = f.readlines()
contents = ''.join(contents)
dom = xml.dom.minidom.parseString(contents)
dom.normalize()
self.template_t = dom.toxml()
soup = BeautifulSoup(self.template_t, 'xml')
for group in dom.getElementsByTagName('Delta'):
self.delta = int(group.childNodes[0].data)
new_1 = list(map(str, soup.find_all('Block')))
self.bloks_data = ''.join(new_1)
new_1 = list(map(str, soup.find_all('Link')))
self.links_data = ''.join(new_1)
env = Environment(loader=FileSystemLoader(os.path.dirname(path)))
template_source = env.loader.get_source(env, os.path.basename(path))[0]
template_source = template_source.replace('dict.', '')
parsed_content = env.parse(template_source)
list_atrib = list(meta.find_undeclared_variables(parsed_content))
list_atrib.remove('X')
list_atrib.remove('Y')
list_atrib.remove('index')
# list_atrib.sort()
list_atrib = sorted(list_atrib, key=lambda x: (len(x), x))
self.atr_dict = {}
for atr in list_atrib:
self.atr_dict[atr] = 'None'
def _process(G, name, value):
'''
Determines whether parameter is a template or a value. Adds graph nodes and edges accordingly.
'''
# Jinja defaults to ascii parser in python 2.x unless you set utf-8 support on per module level
# Instead we're just assuming every string to be a unicode string
if isinstance(value, str):
value = to_unicode(value)
complex_value_str = None
if isinstance(value, list) or isinstance(value, dict):
complex_value_str = str(value)
is_jinja_expr = (jinja_utils.is_jinja_expression(value)
or jinja_utils.is_jinja_expression(complex_value_str))
if is_jinja_expr:
G.add_node(name, template=value)
template_ast = ENV.parse(value)
LOG.debug('Template ast: %s', template_ast)
# Dependencies of the node represent jinja variables used in the template
# We're connecting nodes with an edge for every depencency to traverse them
# in the right order and also make sure that we don't have missing or cyclic
# dependencies upfront.
dependencies = meta.find_undeclared_variables(template_ast)
LOG.debug('Dependencies: %s', dependencies)
if dependencies:
for dependency in dependencies:
G.add_edge(dependency, name)
else:
G.add_node(name, value=value)
def _get_unknown_instack_tags(env, src):
found_tags = set(meta.find_undeclared_variables(env.parse(src)))
known_tags = set(INSTACK_NETCONF_MAPPING.keys())
if found_tags <= known_tags:
return (', ').join(found_tags - known_tags)
else:
return None
def read(self):
abspath = os.path.abspath(os.getcwd())
loc = self.get_config_location(abspath)
os.chdir(loc[0])
print('Running on \'' + loc[0] + '\'')
cwd = os.path.abspath(os.getcwd())
self.env = self.get_env(cwd)
p_cwd = pathlib.Path(cwd)
targets = get_retriever_targets(cwd)
target_dict = {}
for target in targets:
buf = {}
for f in target.files:
try:
p_f = pathlib.Path(f)
source = self.env.loader.get_source(self.env, str(p_f.relative_to(p_cwd)))
parsed_content = self.env.parse(source)
variables = meta.find_undeclared_variables(parsed_content)
keys = {}
for key in variables:
keys[key] = None
buf[f] = keys
except TemplateNotFound:
pass
target_dict[target.path] = buf
print(yaml.dump(target_dict, default_flow_style=False))
def get_var_templates(template):
env = Environment()
ast = env.parse(template.content)
exceptions = ('vidly_tokenize', 'edgecast_tokenize', 'popcorn_url')
undeclared_variables = [x for x in meta.find_undeclared_variables(ast)
if x not in exceptions]
return ["%s=" % v for v in undeclared_variables]
def parse_template(template):
""" Parses the abstract syntax tree of a template
and returns a set of all context variables in the template.
"""
env = Environment()
ast = env.parse(template)
return meta.find_undeclared_variables(ast)
def _get(obj, key):
if not obj.get(key):
return
try:
ast = env.parse(obj[key])
except:
return
var.update(meta.find_undeclared_variables(ast))
def jinja_find_required_variables(fullpath):
myenv = jinja2.Environment(loader=jinja2.FileSystemLoader(
os.path.dirname(fullpath)))
myenv.filters['bool'] = type_utils.str_to_bool
template_source = myenv.loader.get_source(myenv,
os.path.basename(fullpath))[0]
parsed_content = myenv.parse(template_source)
return meta.find_undeclared_variables(parsed_content)
def _config_file_build_check(self, applicationGroup, configFile, server):
''' This function will check to ensure all variables in the configuration file have some type of value set.
'''
for key in meta.find_undeclared_variables(self.envTemplates.parse # Parse the template data
(self.envTemplates.loader.get_source # Load the source of the template data
(self.envTemplates, configFile)[0])):
if key not in self.configurationFileSettings:
raise Exception (("ERROR: Variable %s not set in configuration file %s for group %s for server %s") % (key, applicationGroup, configFile, server))
def get_template_vars(self, ignorevars=[]):
env = Environment(loader=FileSystemLoader(self.templatepath), undefined=StrictUndefined)
tplsrc = env.loader.get_source(env, self.templatename)[0]
ast = env.parse(tplsrc)
tplvars = meta.find_undeclared_variables(ast)
# TODO Option for returing data unordered?
return sorted([e for e in tplvars if not e in ignorevars])
def _check_missing_vars(env, tpl_file, config):
"""Check for missing variables in a template string"""
tpl_str = tpl_file.read()
ast = env.parse(tpl_str)
required_properties = meta.find_undeclared_variables(ast)
missing_properties = required_properties - config.keys() - set(dir(builtins))
if len(missing_properties) > 0:
print('Required properties not set: {}'.format(','.join(missing_properties)))
sys.exit(1)
