def getOptions():
try:
options, args = getopt.getopt(
sys.argv[1:], "y:j:n:sw",
["yaml=", "jinja=", "notrim", "strict", "warning"])
except getopt.GetoptError as err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
sys.exit(2)
global yamlfile, jinjafile, trim, undefined
trim = True
opts = 0
for opt, arg in options:
opts = opts + 1
if opt in ("-y", "-yaml"):
yamlfile = arg
elif opt in ("-j", "-jinja"):
jinjafile = arg
elif opt in ("-n", "-notrim"):
trim = False
elif opt in ("-w", "-warning"):
undefined = make_logging_undefined(base=Undefined)
elif opt in ("-s", "-strict"):
undefined = make_logging_undefined(base=StrictUndefined)
return opts > 0
def getOptions():
try:
options, args = getopt.getopt(sys.argv[1:], "y:j:n:sw", ["yaml=", "jinja=", "notrim", "strict", "warning"])
except getopt.GetoptError as err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
sys.exit(2)
global yamlfile,jinjafile,trim,undefined
trim = True
opts = 0
for opt,arg in options:
opts = opts + 1
if opt in ("-y","-yaml"):
yamlfile = arg
elif opt in ("-j","-jinja"):
jinjafile = arg
elif opt in ("-n","-notrim"):
trim = False
elif opt in ("-w","-warning"):
undefined = make_logging_undefined (base = Undefined)
elif opt in ("-s","-strict"):
undefined = make_logging_undefined (base = StrictUndefined)
return opts > 0
def compile_template(template, tmpl_dirs, debug=False, **options):
"""Generate command string from Jinja2 template and options"""
loader = FileSystemLoader(searchpath=tmpl_dirs)
if debug == False:
undefined_t = Undefined
elif debug == True or not issubclass(debug, Undefined):
# LoggingUndefined w/ it's own logger
_logger = logging.getLogger(f"genomegenie.batch.templates.{template}")
undefined_t = make_logging_undefined(logger=_logger)
if debug != True:
# warn when debug was unknown custom undefined
logger.warning(f"Ignoring {debug}, not a subclass of Undefined")
else:
# custom undefined, e.g. DebugUndefined
undefined_t = debug
env = Environment(loader=loader,
trim_blocks=True,
lstrip_blocks=True,
undefined=undefined_t)
# add custom filters to env
filters = import_module(".filters", "genomegenie.batch")
env.filters.update((k, getattr(filters, k)) for k in dir(filters)
if not k.startswith("_"))
try:
template = env.get_template(template)
return template.render(options)
except TemplateError:
logger.error(f"Failed to render '{template}' from '{tmpl_dirs}'",
exc_info=True)
return ""
def test_logging_undefined(self):
_messages = []
class DebugLogger:
def warning(self, msg, *args):
_messages.append("W:" + msg % args)
def error(self, msg, *args):
_messages.append("E:" + msg % args)
logging_undefined = make_logging_undefined(DebugLogger())
env = Environment(undefined=logging_undefined)
assert env.from_string("{{ missing }}").render() == ""
pytest.raises(UndefinedError,
env.from_string("{{ missing.attribute }}").render)
assert env.from_string("{{ missing|list }}").render() == "[]"
assert env.from_string(
"{{ missing is not defined }}").render() == "True"
assert env.from_string("{{ foo.missing }}").render(foo=42) == ""
assert env.from_string("{{ not missing }}").render() == "True"
assert _messages == [
"W:Template variable warning: 'missing' is undefined",
"E:Template variable error: 'missing' is undefined",
"W:Template variable warning: 'missing' is undefined",
"W:Template variable warning: 'int object' has no attribute 'missing'",
"W:Template variable warning: 'missing' is undefined",
]
def JinjaEnv(template_locations=[]):
LoggingUndefined = jinja2.make_logging_undefined(logger=logger,
base=Undefined)
jenv = jinja2.Environment(
trim_blocks=True,
lstrip_blocks=True,
undefined=LoggingUndefined,
extensions=['jinja2.ext.loopcontrols', 'jinja2.ext.do'])
if template_locations:
jinja_loaders = []
for _dir in template_locations:
jinja_loaders.append(jinja2.FileSystemLoader(str(_dir)))
jenv.loader = jinja2.ChoiceLoader(jinja_loaders)
else:
jenv.loader = jinja2.BaseLoader()
jenv.globals['include_raw'] = include_raw_gz
jenv.globals['raise'] = raise_helper
jenv.globals['option'] = option
jenv.filters['sub'] = sub
jenv.filters['pyminify'] = pyminify
jenv.filters['inline_yaml'] = inline_yaml
jenv.tests['match'] = match
jenv.tests['regex'] = regex
jenv.tests['search'] = search
return jenv
def test_logging_undefined(self):
_messages = []
class DebugLogger(object):
def warning(self, msg, *args):
_messages.append('W:' + msg % args)
def error(self, msg, *args):
_messages.append('E:' + msg % args)
logging_undefined = make_logging_undefined(DebugLogger())
env = Environment(undefined=logging_undefined)
assert env.from_string('{{ missing }}').render() == u''
pytest.raises(UndefinedError,
env.from_string('{{ missing.attribute }}').render)
assert env.from_string('{{ missing|list }}').render() == '[]'
assert env.from_string('{{ missing is not defined }}').render() \
== 'True'
assert env.from_string('{{ foo.missing }}').render(foo=42) == ''
assert env.from_string('{{ not missing }}').render() == 'True'
assert _messages == [
'W:Template variable warning: missing is undefined',
"E:Template variable error: 'missing' is undefined",
'W:Template variable warning: missing is undefined',
'W:Template variable warning: int object has no attribute missing',
'W:Template variable warning: missing is undefined',
]
def jinja_environment(
templates_folder: Path,
shell_command_working_directory: Path,
) -> Environment:
"""Return a jinja Environment instance for templates in a folder."""
logger = logging.getLogger(__name__)
LoggingUndefined = make_logging_undefined(
logger=logger,
base=Undefined,
)
env = Environment(
loader=FileSystemLoader(
str(templates_folder),
followlinks=True,
),
autoescape=False,
auto_reload=True,
optimized=True,
finalize=finalize_variable_expression,
undefined=LoggingUndefined,
)
# Add env context containing all environment variables
env.globals['env'] = os.environ
# Add run shell command filter
run_shell_from_working_directory = partial(
utils.run_shell,
working_directory=shell_command_working_directory,
log_success=False,
)
env.filters['shell'] = run_shell_from_working_directory
return env
def __init__(self, os_target: str):
self.log = jinja2.make_logging_undefined(logger=log)
self.location = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
self.os_target = os_target
self.templates_folders = get_folder_structure_recursively(os.path.join(self.location, 'templates', os_target),
('.*j2',))
self.env = jinja2.Environment(loader=jinja2.FileSystemLoader(self.templates_folders), autoescape=True)
def gen_email_body(f, **kwargs) -> str:
"""
given an opened jinja template file and any number of kwargs (key=value),
will render the jinja template file and return the resulting string
:param f: jinja template file, for example request_emails.html (regrading)
see https://palletsprojects.com/p/jinja/
:type f: file
:param **kwargs: variables that will be passed to the jinja renderer
for example if you pass login='******', you can use
{{ login }} somewhere in `f` to have eberkowi rendered
into the html.
:returns: the rendered html template
:rtype: str
:raises: ValueError if you do not pass in the
"""
# catch undefined errors using https://stackoverflow.com/questions/46619830
content = f.read()
LoggingUndefined = make_logging_undefined(base=StrictUndefined)
templateEnv = Environment(undefined=LoggingUndefined)
template = templateEnv.from_string(content)
try:
body = template.render(kwargs).replace('\n', '')
print(body)
return body
except UndefinedError as err:
msg = f'Error rendering template {f.name!r}: {err.args[0]}'
raise ValueError(msg)
def test_logging_undefined(self):
_messages = []
class DebugLogger(object):
def warning(self, msg, *args):
_messages.append('W:' + msg % args)
def error(self, msg, *args):
_messages.append('E:' + msg % args)
logging_undefined = make_logging_undefined(DebugLogger())
env = Environment(undefined=logging_undefined)
assert env.from_string('{{ missing }}').render() == u''
pytest.raises(UndefinedError,
env.from_string('{{ missing.attribute }}').render)
assert env.from_string('{{ missing|list }}').render() == '[]'
assert env.from_string('{{ missing is not defined }}').render() \
== 'True'
assert env.from_string('{{ foo.missing }}').render(foo=42) == ''
assert env.from_string('{{ not missing }}').render() == 'True'
assert _messages == [
'W:Template variable warning: missing is undefined',
"E:Template variable error: 'missing' is undefined",
'W:Template variable warning: missing is undefined',
'W:Template variable warning: int object has no attribute missing',
'W:Template variable warning: missing is undefined',
]
def take_action(self, parsed_args):
se = self.app.secrets
if parsed_args.no_env:
template_vars = dict()
else:
se.requires_environment()
se.read_secrets_and_descriptions()
template_vars = se.items()
template_loader = FileSystemLoader('.')
base = Undefined if parsed_args.check_defined is True \
else StrictUndefined
LoggingUndefined = make_logging_undefined(
logger=self.logger,
base=base)
template_env = Environment(
loader=template_loader,
autoescape=select_autoescape(
disabled_extensions=('txt',),
default_for_string=True,
default=True,
),
undefined=LoggingUndefined)
template = template_env.get_template(parsed_args.source)
output_text = template.render(template_vars)
if parsed_args.check_defined is False:
if parsed_args.dest == "-":
print(output_text)
else:
with open(parsed_args.dest, 'w') as f:
f.writelines(output_text)
def generate():
base_path = os.path.abspath(os.path.dirname(__file__))
template_paths = [os.path.join(base_path, '.')]
if os.path.exists(self._options.custom_template_path):
template_paths.append(self._options.custom_template_path)
file_loader = jinja2.FileSystemLoader(template_paths)
logging_undefined = jinja2.make_logging_undefined(
logger=self._options.logger, base=jinja2.Undefined)
env = jinja2.Environment(loader=file_loader,
undefined=logging_undefined,
lstrip_blocks=True,
trim_blocks=True)
env.filters['strptime'] = strptime
env.filters['strftime'] = strftime
env.filters['to_url'] = URLFilter(self._options, self._config)
if Barcode:
env.filters['barcode'] = Barcode(self._options, self._config)
if QRCode:
env.filters['qrcode'] = QRCode(self._options, self._config)
return env
def __init__(self, dispatch_loader, filter_path):
self.filter_path = filter_path
template_loader = DispatchJinjaLoader(dispatch_loader)
# Make access to undefined context variables generate logs.
undef = jinja2.make_logging_undefined(logger=logger)
self.env = jinja2.Environment(loader=template_loader,
trim_blocks=True,
undefined=undef)
def gen_langs():
langdata = json.load(open("assets/data/langs.json", "r"))
jinja_env = jinja2.Environment(
loader=templateLoader,
undefined=jinja2.make_logging_undefined(base=jinja2.DebugUndefined))
template = jinja_env.get_template("_index.html")
for lang in langdata:
os.system("mkdir %s" % (lang["lang_code"]))
print(lang["data"])
print(template.render(lang_code=lang["lang_code"], **lang["data"]),
file=open("%s/index.html" % (lang["lang_code"]), "w"))
def template_with_jinja2(text, variables):
"""
Template using jinja2 with the variables dictionary unpacked as keyword
arguments.
"""
jinja_environment = jinja2.Environment(
loader=jinja2.BaseLoader(),
undefined=jinja2.make_logging_undefined(logging.getLogger()),
comment_start_string='{##',
comment_end_string='##}',
)
template = jinja_environment.from_string(text)
return template.render(**variables)
def generateHTML(template, output, **kwargs):
# print("Starting {0}".format(output))
undefined = jinja2.make_logging_undefined(DebugLogger(output))
env = jinja2.Environment(loader=jinja2.FileSystemLoader(template),
undefined=undefined)
output_from_parsed_template = env.get_template('').render(kwargs)
with open(output, "wb") as f:
try:
f.write(output_from_parsed_template.encode('utf8'))
except Exception as exc:
print(exc)
print(output_from_parsed_template)
def read_template_file(path):
templates = os.path.join(os.path.dirname(__file__), "templates")
loader = jinja2.FileSystemLoader(searchpath=templates)
logging.basicConfig()
logger = logging.getLogger('logger')
logger = jinja2.make_logging_undefined(logger=logger,
base=jinja2.Undefined)
env = jinja2.Environment(loader=loader, undefined=logger)
env.lstrip_blocks = True
env.trim_blocks = True
return env.get_template(path)
def replace_jinja_variables_string(self, field_name, field_content):
parsed_content = field_content
template_environment = jinja2.Environment(
undefined=jinja2.make_logging_undefined())
try:
parsed_content = template_environment.from_string(
str(field_content)).render(self.as_dictionary)
# logging.info('string "{}" into "{}" using jinja'.format(stringToReplace, stringReplaced))
except Exception as exception:
logging.error(
'when processing variable "%s" (value %s) using jinja2' %
(field_name, field_content))
logging.error(exception)
return parsed_content
def template_with_jinja2(text, variables):
"""
Template using jinja2 with the variables dictionary unpacked as keyword
arguments.
"""
jinja_environment = jinja2.Environment(
loader=jinja2.BaseLoader(),
undefined=jinja2.make_logging_undefined(logging.getLogger()),
autoescape=False,
comment_start_string="{##",
comment_end_string="##}",
extensions=["jinja2.ext.do", "jinja2.ext.loopcontrols"],
)
template = jinja_environment.from_string(text)
return template.render(**variables)
def parse_variable(self, new_variable: ConfigurationVariableData):
if self.count > 0:
template_environment = jinja2.Environment(
undefined=jinja2.make_logging_undefined())
try:
parsed_value = template_environment.from_string(
str(new_variable.value)).render(self.as_dictionary)
new_variable.value = parsed_value
return new_variable
except Exception as exception:
logging.error(
'when processing variable "{}" (value {}) using jinja2'.
format(new_variable.name, new_variable.value))
logging.error(exception)
return new_variable
def configure_jinja():
"""This:
- configures jinja's policy on undefined variables;
- install any globals needed by the templates;
- install any extensions used by the templates."""
# Configure jinja to warn undefined variables.
# For more information and possibilities, see:
# http://jinja.pocoo.org/docs/2.10/api/#jinja2.make_logging_undefined
undefined_logger = jinja2.make_logging_undefined(app.logger,
base=jinja2.Undefined)
app.jinja_env.undefined = undefined_logger
# install pgettext for i18n and generate_csrf for non-standard forms
app.jinja_env.globals['pgettext'] = pgettext
app.jinja_env.globals['generate_csrf'] = generate_csrf
def _append_report_header(self):
template_file = os.path.join(self._config.get('site_path'), self._REPORT_TEMPLATE)
if not os.path.exists(template_file):
LOGGER.info('Skipping missing report header template %s', template_file)
self._writeln(self._DEFAULT_HEADER % self._clean_mac)
return
LOGGER.info('Adding templated report header from %s', template_file)
try:
undefined_logger = jinja2.make_logging_undefined(logger=LOGGER, base=jinja2.Undefined)
environment = jinja2.Environment(loader=jinja2.FileSystemLoader('.'),
undefined=undefined_logger)
self._writeln(environment.get_template(template_file).render(self._module_config))
except Exception as e:
self._writeln('Report generation error: %s' % e)
self._writeln('Failing data model:\n%s' % str(self._module_config))
LOGGER.error('Report generation failed: %s', e)
def create_env(site: "miyadaiku.site.Site", themes: List[str],
paths: List[Path]) -> Environment:
loaders: List[Any] = [PackagesLoader()]
for path in paths:
loaders.append(FileSystemLoader(os.fspath(path)))
for theme in themes:
try:
loader = PackageLoader(theme)
except ValueError:
# PackageLoader raises ValueError if the theme does not contain
# templates directory
continue
loaders.append(loader)
loaders.append(PackageLoader("miyadaiku.themes.base"))
env = Environment(
undefined=make_logging_undefined(logger, DebugUndefined),
# undefined=make_logging_undefined(logger, StrictUndefined),
loader=ChoiceLoader(loaders),
autoescape=select_autoescape(["html", "xml", "j2"]),
extensions=EXTENSIONS,
)
env.globals["str"] = str
env.globals["list"] = list
env.globals["tuple"] = tuple
env.globals["dict"] = dict
env.globals["site"] = site
env.globals["len"] = len
env.globals["repr"] = repr
env.globals["print"] = print
env.globals["type"] = type
env.globals["dir"] = dir
env.globals["isinstance"] = isinstance
env.globals["setattr"] = setattr
env.globals["getattr"] = getattr
env.filters["urlquote"] = urlquote
env.filters["safepath"] = safepath
return env
def gen_static_pages():
jinja_env = jinja2.Environment(
loader=templateLoader,
undefined=jinja2.make_logging_undefined(base=jinja2.DebugUndefined))
template = jinja_env.get_template("templates/who.html")
for frmt in URLS.keys():
json_data_mb = requests.get(URLS[frmt]).json()
sheet_data = json_data_mb["feed"]["entry"]
data = []
for row in sheet_data:
data.append({
"question": row["gsx$q"]['$t'],
"answer": row["gsx$a"]["$t"]
})
print(template.render(data=data,
title=frmt.title(),
source=SOURCES[frmt]),
file=open(frmt + ".html", "w"))
def http_jinja_env(**options):
"""
Jinja2 settings for http output.
"""
env = jinja2.Environment(**options)
env.globals.update({
'static': static,
'url': reverse,
'urlarg': urlarg,
'DEBUG_PRINT': (lambda x: str(x) + ""),
})
# This makes errors in templates louder.
class MyDebugUndefined(jinja2.DebugUndefined):
def __str__(self):
return "Undefined:" + super().__str__()
logger = logging.getLogger('chargen.undefined_templates')
env.undefined = jinja2.make_logging_undefined(logger=logger,
base=MyDebugUndefined)
return env
def render_jinja2_template(secret_data,
namespace,
config_map=None,
template_file=None):
if config_map:
templateData = read_data_from_configmap(config_map,
namespace=namespace)
elif template_file:
log.info("Reading jinja2 template from: {}".format(template_file))
with open(template_file, "r") as file:
templateData = file.read()
# Display undefined variables as WARNING
LoggingUndefined = jinja2.make_logging_undefined(logger=log,
base=jinja2.Undefined)
templateEnv = jinja2.Environment(undefined=LoggingUndefined)
configTemplate = templateEnv.from_string(templateData)
try:
rendered_template = configTemplate.render(values=secret_data)
return rendered_template
except Exception as error:
log.error("Error while rendering template")
log.error(error)
log.error(
"Please check your jinja2 templating variables in configmap: {}".
format(config_map))
log.error(
"Please avoid to use characters other than \"[a-zA-Z0-9_]\" in jinja2 template variables"
)
log.error(
"Or convert your jinja2 template variables to wrap it with \"values\""
)
log.error(
"For example - instead of {{ user-name }}, wrap it as {{ values['user-name'] }}"
)
sys.exit(1)
def __init__(
self,
undefined_variables_mode_behaviour='strict',
j2_env_params={},
):
UndefinedHandler = StrictUndefined
m = undefined_variables_mode_behaviour
if m in ['empty', 'Undefined']:
UndefinedHandler = Undefined
elif m in ['keep', 'DebugUndefined']:
UndefinedHandler = DebugUndefined
# Setup debug logging on STDERR to have the jinja2 engine emit
# its activities
root = logging.getLogger(__name__)
root.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stderr)
handler.setFormatter(
logging.Formatter('%(name)s %(levelname)s: %(message)s'))
root.addHandler(handler)
UndefinedHandler = make_logging_undefined(logger=root,
base=UndefinedHandler)
j2_env_params.setdefault('undefined', UndefinedHandler)
j2_env_params.setdefault('trim_blocks', True)
j2_env_params.setdefault('keep_trailing_newline', False)
j2_env_params.setdefault('extensions', [
'jinja2.ext.i18n',
'jinja2.ext.do',
'jinja2.ext.loopcontrols',
])
self.j2_env_params = j2_env_params
self.filters = get_symbols(filters)
self.tests = get_symbols(tests)
self.globals = get_symbols(globals)
def main():
parser = argparse.ArgumentParser()
# qa-reports parameters
parser.add_argument(
"--environment",
help=
"User specified the environment name, prefix or suffix won't be used",
dest="environment",
default="",
)
parser.add_argument(
"--env-prefix",
help="Prefix for the environment name",
dest="env_prefix",
default="",
)
parser.add_argument(
"--env-suffix",
help="Suffix for the environment name",
dest="env_suffix",
default="",
)
parser.add_argument("--build-id",
"--build-number",
help="ID for the build",
dest="build_id")
parser.add_argument("--qa-server-team",
help="Team in QA Reports service",
dest="qa_server_team")
parser.add_argument(
"--qa-server-project",
help="Project in QA Reports service",
dest="qa_server_project",
)
parser.add_argument(
"--qa-server",
help="QA Reports server",
dest="qa_server",
default="https://qa-reports.linaro.org",
)
parser.add_argument(
"--qa-token",
help="QA Reports token",
dest="qa_token",
default=os.environ.get("QA_REPORTS_TOKEN"),
)
# lava parameters
parser.add_argument("--lava-server",
help="LAVA server URL",
dest="lava_server")
parser.add_argument("--lava-username",
help="LAVA username",
dest="lava_username")
parser.add_argument("--lava-token",
help="LAVA authentication token",
dest="lava_token")
# rendering parameters (test plan)
parser.add_argument(
"--variables",
help="Path to file(s) with variable values",
dest="variables",
required=True,
nargs="+",
)
parser.add_argument(
"--overwrite-variables",
help="Key-value pairs overwriting variables from the file",
nargs="+",
dest="overwrite_variables",
default=[],
)
parser.add_argument("--device-type",
help="Device type in LAVA",
dest="device_type",
required=True)
parser.add_argument(
"--template-path",
help="Path to LAVA job templates",
dest="template_path",
default=template_base_path,
)
parser.add_argument(
"--testplan-path",
help="Path to directory containing all test plans",
dest="testplan_path",
default=testplan_base_path,
)
parser.add_argument(
"--testcase-path",
help="Path to directory containing all test cases",
dest="testcase_path",
default=testcase_base_path,
)
parser.add_argument(
"--testplan-device-path",
help="Relative path to Jinja2 device deployment fragments",
dest="testplan_device_path",
default=testplan_device_path,
)
parser.add_argument(
"--test-plan",
help="""Directories containing Jinja2 templates to submit for testing.
It is assumed that the templates produce valid LAVA job
definitions. All variables are substituted using Jinja2
engine""",
dest="test_plan",
nargs="*",
required=False,
)
parser.add_argument(
"--test-case",
help="""Specific test cases (as Jinja2 templates) to submit.
It is assumed that the templates produce valid LAVA job
definitions. All variables are substituted using Jinja2
engine. Multiple --test-case variables are allowed:
--test-case test1.yaml test2.yaml --test-case test3.yaml""",
dest="test_case",
nargs="*",
action="append",
required=False,
)
parser.add_argument(
"--dry-run",
help="""Prepare and write templates to tmp/.
Don't submit to actual servers.""",
action="store_true",
dest="dryrun",
)
parser.add_argument(
"--test-lava-validity",
help="""Test generated templates using LAVA container validator""",
action="store_true",
dest="test_lava_validity",
)
parser.add_argument(
"--verbose",
help="""Verbosity level. Follows logging levels:
CRITICAL: 50
ERROR: 40
WARNING: 30
INFO: 20
DEBUG: 10
NOTSET: 0""",
dest="verbose",
type=int,
default=logging.INFO,
)
args = parser.parse_args()
logger.setLevel(args.verbose)
exit_code = 0
output_path = os.path.join(script_dirname, "tmp")
if args.qa_server_project:
if "/" in args.qa_server_project:
logger.error(
"--qa-server-project can not contain of a slash in the name")
sys.exit(1)
if args.dryrun:
if not os.path.exists(output_path):
os.mkdir(output_path)
if args.qa_token is None and args.lava_token is None and not args.dryrun:
logger.error("QA_REPORTS_TOKEN and LAVA_TOKEN are missing")
sys.exit(1)
lava_jobs = []
template_dirs = [
os.path.abspath(template_base_path),
os.path.abspath(args.testplan_path),
os.path.abspath(args.testcase_path),
os.path.abspath(args.testplan_device_path),
]
# prevent creating templates when variables are missing
j2_env = Environment(
loader=FileSystemLoader(template_dirs, followlinks=True),
undefined=StrictUndefined,
)
if args.dryrun:
LoggingUndefined = make_logging_undefined(logger=logger,
base=StrictUndefined)
j2_env = Environment(
loader=FileSystemLoader(template_dirs, followlinks=True),
undefined=LoggingUndefined,
)
context = {}
for variables in args.variables:
if not os.path.exists(variables):
variables = os.path.join(script_dirname, variables)
try:
context.update(ConfigObj(variables).dict())
except ConfigObjError:
logger.info("Unable to parse .ini file")
logger.info("Trying YAML")
with open(variables, "r") as vars_file:
try:
yaml = YAML(typ="safe")
context.update(yaml.load(vars_file))
except ParserError as e:
logger.error(e)
except ComposerError as e:
logger.error(e)
for variable in args.overwrite_variables:
key, value = variable.split("=")
context.update({key: value})
context.update({"device_type": args.device_type})
test_list = []
if args.test_plan:
for test_plan in args.test_plan:
test_plan_path = os.path.abspath(
os.path.join(args.testplan_path, test_plan))
for test in _get_test_plan_list(test_plan_path):
test_list.append(test)
if args.test_case:
test_list = test_list + list(
itertools.chain.from_iterable(args.test_case))
if len(test_list) == 0:
logger.error("No tests matched the given criteria.")
sys.exit(1)
# convert test_list to set to remove potential duplicates
for test in set(test_list):
"""Prepare lava jobs"""
lava_job = None
try:
lava_job = j2_env.get_template(test).render(context)
lava_job = parse_template(lava_job)
lava_jobs.append(lava_job)
logger.debug(lava_job)
except DuplicateKeyError as e:
logger.error(e)
exit_code = 1
except ConstructorError as e:
logger.error(e)
exit_code = 1
except DuplicateKeyFutureWarning as e:
logger.error(e)
exit_code = 1
except ScannerError as e:
logger.error(e)
exit_code = 1
except ParserError as e:
testpath = os.path.join(output_path, args.device_type, test)
logger.error("Failed to parse: %s" % testpath)
logger.error(e)
exit_code = 1
except TemplateSyntaxError as e:
testpath = os.path.join(output_path, args.device_type, test)
logger.error("Trying to render: %s" % testpath)
logger.error("Error in file: %s" % e.name)
logger.error("\tline: %s" % e.lineno)
logger.error("\tissue: %s" % e.message)
exit_code = 1
except UndefinedError as e:
testpath = os.path.join(output_path, args.device_type, test)
logger.error("Trying to render: %s" % testpath)
logger.error("\tissue: %s" % e.message)
exit_code = 1
if args.dryrun and lava_job is not None:
testpath = os.path.join(output_path, args.device_type,
os.path.basename(test))
logger.info(testpath)
if not os.path.exists(os.path.dirname(testpath)):
os.makedirs(os.path.dirname(testpath))
with open(os.path.join(testpath), "w") as f:
f.write(lava_job)
if args.test_lava_validity:
import docker
client = docker.from_env(version="1.38")
logger.debug("Checking for LAVA validity")
for test in set(test_list):
testpath = os.path.join(os.getcwd(), output_path, args.device_type)
logger.debug(testpath)
logger.debug(test)
container = client.containers.run(
image="lavasoftware/lava-server:latest",
command="/usr/share/lava-common/lava-schema.py job /data/%s" %
test,
volumes={"%s" % testpath: {
"bind": "/data",
"mode": "rw"
}},
detach=True,
)
container_exit_code = container.wait()
logger.debug(exit_code)
if container_exit_code["StatusCode"] != 0:
logger.error("LAVA validation of %s/%s failed" %
(testpath, test))
logger.error(container.logs())
exit_code = 1
if not args.dryrun:
if not args.qa_server:
logger.error("QA-reports server not specified")
exit_code = 1
if not args.lava_server:
logger.error("Lava server not specified")
exit_code = 1
if exit_code != 0:
sys.exit(exit_code)
qa_server_base = args.qa_server
if not (qa_server_base.startswith("http://")
or qa_server_base.startswith("https://")):
qa_server_base = "https://" + qa_server_base
qa_server_team = args.qa_server_team
qa_server_project = args.qa_server_project
qa_server_build = args.build_id
if not args.environment:
# when user not specify value for the environment option,
# use the device_type as before
qa_server_env = args.env_prefix + args.device_type + args.env_suffix
else:
# when user specified value for the environment option,
# use the user specified value
qa_server_env = args.environment
qa_server_api = "%s/api/submitjob/%s/%s/%s/%s" % (
qa_server_base,
qa_server_team,
qa_server_project,
qa_server_build,
qa_server_env,
)
lava_server = args.lava_server
if not (lava_server.startswith("http://")
or lava_server.startswith("https://")):
lava_server = "https://" + lava_server
lava_url_base = "%s://%s/" % (
urlsplit(lava_server).scheme,
urlsplit(lava_server).netloc,
)
for lava_job in lava_jobs:
"""Submit lava jobs"""
if args.qa_token:
_submit_to_squad(
lava_job,
lava_url_base,
qa_server_api,
qa_server_base,
args.qa_token,
)
if args.lava_token:
_submit_to_lava(lava_job, lava_url_base, lava_username,
lava_token)
else:
sys.exit(exit_code)
def __init__(self, app, *args, **kwargs):
self.app = app
# Before we create the base Environement, let's figure out the options
# we want to pass to it.
#
# Disable auto-reload when we're baking.
if app.config.get('baker/is_baking'):
kwargs.setdefault('auto_reload', False)
# Don't unload templates from the cache.
kwargs.setdefault('cache_size', -1)
# Let the user override most Jinja options via the site config.
for name in ['block_start_string', 'block_end_string',
'variable_start_string', 'variable_end_string',
'comment_start_string', 'comment_end_string',
'line_statement_prefix', 'line_comment_prefix',
'trim_blocks', 'lstrip_blocks',
'newline_sequence', 'keep_trailing_newline']:
val = app.config.get('jinja/' + name)
if val is not None:
kwargs.setdefault(name, val)
# Undefined behaviour.
undef = app.config.get('jinja/undefined')
if undef == 'logging':
from jinja2 import make_logging_undefined
kwargs.setdefault('undefined',
make_logging_undefined(logger))
elif undef == 'strict':
from jinja2 import StrictUndefined
kwargs.setdefault('undefined', StrictUndefined)
# All good! Create the Environment.
super(PieCrustEnvironment, self).__init__(*args, **kwargs)
# Now add globals and filters.
self.globals.update({
'now': get_now_date(),
'fail': raise_exception,
'highlight_css': get_highlight_css})
self.filters.update({
'keys': get_dict_keys,
'values': get_dict_values,
'paginate': self._paginate,
'formatwith': self._formatWith,
'markdown': lambda v: self._formatWith(v, 'markdown'),
'textile': lambda v: self._formatWith(v, 'textile'),
'nocache': add_no_cache_parameter,
'wordcount': get_word_count,
'stripoutertag': strip_outer_tag,
'stripslash': strip_slash,
'titlecase': title_case,
'md5': make_md5,
'atomdate': get_xml_date,
'xmldate': get_xml_date,
'emaildate': get_email_date,
'date': get_date})
self.filters['raw'] = self.filters['safe']
def setup(self, subtype, app_name, dest):
super().setup(subtype, app_name, dest)
self.source_len = len(self.source)
self.env = Environment(loader=FunctionLoader(self.jinja_loader),
undefined=make_logging_undefined())
def __init__(self, hass, limited=False):
"""Initialise template environment."""
super().__init__(undefined=jinja2.make_logging_undefined(
logger=_LOGGER))
self.hass = hass
self.template_cache = weakref.WeakValueDictionary()
self.filters["round"] = forgiving_round
self.filters["multiply"] = multiply
self.filters["log"] = logarithm
self.filters["sin"] = sine
self.filters["cos"] = cosine
self.filters["tan"] = tangent
self.filters["asin"] = arc_sine
self.filters["acos"] = arc_cosine
self.filters["atan"] = arc_tangent
self.filters["atan2"] = arc_tangent2
self.filters["sqrt"] = square_root
self.filters["as_timestamp"] = forgiving_as_timestamp
self.filters["as_local"] = dt_util.as_local
self.filters["timestamp_custom"] = timestamp_custom
self.filters["timestamp_local"] = timestamp_local
self.filters["timestamp_utc"] = timestamp_utc
self.filters["to_json"] = to_json
self.filters["from_json"] = from_json
self.filters["is_defined"] = fail_when_undefined
self.filters["max"] = max
self.filters["min"] = min
self.filters["random"] = random_every_time
self.filters["base64_encode"] = base64_encode
self.filters["base64_decode"] = base64_decode
self.filters["ordinal"] = ordinal
self.filters["regex_match"] = regex_match
self.filters["regex_replace"] = regex_replace
self.filters["regex_search"] = regex_search
self.filters["regex_findall_index"] = regex_findall_index
self.filters["bitwise_and"] = bitwise_and
self.filters["bitwise_or"] = bitwise_or
self.filters["ord"] = ord
self.globals["log"] = logarithm
self.globals["sin"] = sine
self.globals["cos"] = cosine
self.globals["tan"] = tangent
self.globals["sqrt"] = square_root
self.globals["pi"] = math.pi
self.globals["tau"] = math.pi * 2
self.globals["e"] = math.e
self.globals["asin"] = arc_sine
self.globals["acos"] = arc_cosine
self.globals["atan"] = arc_tangent
self.globals["atan2"] = arc_tangent2
self.globals["float"] = forgiving_float
self.globals["as_local"] = dt_util.as_local
self.globals["as_timestamp"] = forgiving_as_timestamp
self.globals["relative_time"] = relative_time
self.globals["timedelta"] = timedelta
self.globals["strptime"] = strptime
self.globals["urlencode"] = urlencode
if hass is None:
return
# We mark these as a context functions to ensure they get
# evaluated fresh with every execution, rather than executed
# at compile time and the value stored. The context itself
# can be discarded, we only need to get at the hass object.
def hassfunction(func):
"""Wrap function that depend on hass."""
@wraps(func)
def wrapper(*args, **kwargs):
return func(hass, *args[1:], **kwargs)
return contextfunction(wrapper)
self.globals["device_entities"] = hassfunction(device_entities)
self.filters["device_entities"] = contextfilter(
self.globals["device_entities"])
if limited:
# Only device_entities is available to limited templates, mark other
# functions and filters as unsupported.
def unsupported(name):
def warn_unsupported(*args, **kwargs):
raise TemplateError(
f"Use of '{name}' is not supported in limited templates"
)
return warn_unsupported
hass_globals = [
"closest",
"distance",
"expand",
"is_state",
"is_state_attr",
"state_attr",
"states",
"utcnow",
"now",
]
hass_filters = ["closest", "expand"]
for glob in hass_globals:
self.globals[glob] = unsupported(glob)
for filt in hass_filters:
self.filters[filt] = unsupported(filt)
return
self.globals["expand"] = hassfunction(expand)
self.filters["expand"] = contextfilter(self.globals["expand"])
self.globals["closest"] = hassfunction(closest)
self.filters["closest"] = contextfilter(hassfunction(closest_filter))
self.globals["distance"] = hassfunction(distance)
self.globals["is_state"] = hassfunction(is_state)
self.globals["is_state_attr"] = hassfunction(is_state_attr)
self.globals["state_attr"] = hassfunction(state_attr)
self.globals["states"] = AllStates(hass)
self.globals["utcnow"] = hassfunction(utcnow)
self.globals["now"] = hassfunction(now)
def template(j2, data, path):
ENV = Environment(loader=PackageLoader('netsim',path), \
trim_blocks=True,lstrip_blocks=True, \
undefined=make_logging_undefined(base=StrictUndefined))
template = ENV.get_template(j2)
return template.render(**data)
def install_jinja_logger():
"""
Inits a logger to log undefined values in jinja template expansions
"""
logger = logging.getLogger(__name__)
return jinja2.make_logging_undefined(logger=logger, base=jinja2.Undefined)
