def __init__(self, name, nexpected, ngiven):
"""
"""
mark = name.mark
TemplateError.__init__(self, self.message.format(
name, mark.name, mark.line + 1, mark.column + 1, nexpected, ngiven
))
def check_duplicate_attr(self, d, attr=None, mandatory=False):
seen = {}
stat = []
def get_value(value):
def get_single_value(v, k):
if not (k in v):
if mandatory:
raise TemplateError(
"Missing mandatory attribute %s in %s" % (k, v))
else:
return None
return v[k]
if type(attr) is list:
retval = ""
for a in attr:
item = get_single_value(value, a)
retval += " " if retval else ""
retval += "%s=%s" % (a, item)
return retval
else:
return get_single_value(value, attr)
def check_unique_value(key, value):
if key is not None:
value['key'] = key
v = get_value(value)
if v in seen:
stat.append(
"Duplicate value %s of attribute %s found in %s and %s" %
(v, attr, seen[v]['key'] if
('key' in seen[v]) else seen[v], value['key'] if
('key' in value) else value))
else:
seen[v] = value
# sanity check: do we know which attribute to check?
#
if attr is None:
raise TemplateError("You have to specify attr=name in checkunique")
# iterate over a list or a dictionary, fail otherwise
#
if type(d) is list:
for value in d:
check_unique_value(None, value)
elif type(d) is dict:
for key in d:
check_unique_value(key, d[key])
else:
raise TemplateError("")
if len(stat) == 0:
return None
else:
return stat
def _get_templated_body(self) -> str:
"""Assemble and return the body of the message using the template.
Any captcha data is automatically removed, if present.
"""
tmpl_path = settings.MAIL_TEMPLATE_PATH
data = self.data.copy()
if self.captcha_client.key:
data.pop(self.captcha_client.key)
if tmpl_path:
try:
with open(tmpl_path) as fd:
tmpl = Template(fd.read())
try:
return tmpl.render(data=data, metadata=self.metadata)
except UndefinedError as exc:
raise TemplateError(
f"The template did not define the required fields:"
f" {exc.message}")
except IOError:
err = (f"Cannot open template file. "
f"Check if it exists and is readable.")
log.error(err)
raise exceptions.ConfigError(err)
else:
return json.dumps(self.data)
def markdown(env, value):
"""
Markdown filter with support for extensions.
"""
try:
import markdown as md
except ImportError:
log.error(u"Cannot load the markdown library.")
raise TemplateError(u"Cannot load the markdown library")
output = value
extensions = [
'markdown.extensions.meta', 'markdown.extensions.headerid',
'markdown.extensions.attr_list', 'markdown.extensions.toc',
'markdown.extensions.def_list', 'digital_logic.ext.passthrough',
'digital_logic.ext.sections'
]
d = dict()
d['extensions'] = list()
d['extensions'].extend(extensions)
marked = md.Markdown(**d)
return marked.convert(output)
def generate_file(self, p: Path):
try:
template = Template(p.read_text())
text = template.render(**self.ctx)
except TemplateError as e:
raise TemplateError('error in {}'.format(p)) from e
text = text.strip('\n\t ')
new_path = self.project_root / p.relative_to(self.template_dir)
if len(text) < 3:
# empty files don't get created, in case a few characters get left behind
logger.debug('not creating "%s", as it would be empty', new_path)
return
logger.debug('creating "%s"', new_path)
if p.name == 'requirements.txt':
packages = {
p.strip()
for p in text.split('\n') if p.strip() and p.strip() != '#'
}
text = '\n'.join(sorted(packages))
else:
# helpful when debugging: print(text.replace(' ', '·').replace('\n', '⏎\n'))
if p.suffix == '.py':
text = SortImports(file_contents=text).output
regexes = FILES_REGEXES.get(p.suffix, [])
for regex, repl in regexes:
text = regex.sub(repl, text)
# re-add a trailing newline accounting for newlines added by PY_REGEXES
text = re.sub('\n*$', '\n', text)
new_path.parent.mkdir(parents=True, exist_ok=True)
new_path.write_text(text)
self.files_created += 1
def get_source(self, environment, template):
parts = template.split(":")
module = None
templatePath = None
if len(parts) == 1:
module = self.baseModule
templatePath = parts[0]
elif len(parts) == 2:
module = parts[0]
templatePath = parts[1]
else:
raise TemplateError("Invalid syntax " + template)
if module == "__main__":
module = self.baseModule
decode = 'utf-8'
parts = templatePath.split("#")
if len(parts) > 1:
templatePath = parts[0]
decode = parts[1]
path = self.path + "/" + templatePath
data = None
if decode == "base64":
myLogger.debug("Loading base64 resource from {0}:{1}".format(
module, path))
with pkg_resources.resource_stream(module, path) as res:
data = base64.b64encode(res.read()).decode("utf-8")
else:
data = pkg_resources.resource_string(module, path).decode(decode)
return data, templatePath, lambda: False
def device_type(self, show_ver, *deftype):
if "Cisco IOS" in show_ver: return "ios"
if "NX-OS" in show_ver: return "nx-os"
if len(deftype) > 0: return deftype[0]
raise TemplateError(
"Unknown device type - did you pass me show version?")
def status():
if request.args.get('json'):
payload = this_node.abridged_node_details()
response = jsonify(payload)
return response
else:
headers = {"Content-Type": "text/html", "charset": "utf-8"}
previous_states = list(reversed(this_node.known_nodes.states.values()))[:5]
# Mature every known node before rendering.
for node in this_node.known_nodes:
node.mature()
try:
content = status_template.render(this_node=this_node,
known_nodes=this_node.known_nodes,
previous_states=previous_states,
domains=serving_domains,
version=nucypher.__version__,
checksum_address=this_node.checksum_address)
except Exception as e:
log.debug("Template Rendering Exception: ".format(str(e)))
raise TemplateError(str(e)) from e
return Response(response=content, headers=headers)
def get_single_value(v, k):
if not (k in v):
if mandatory:
raise TemplateError(
"Missing mandatory attribute %s in %s" % (k, v))
else:
return None
return v[k]
def dict_to_list(self,o):
if type(o) is not dict:
raise TemplateError("dict_to_list can only be used on dictionaries")
l = []
for k,v in o.items():
v['id'] = k
l.append(v)
return l
def list_append(self,l,*argv):
if type(l) is not list:
raise TemplateError("First argument of append filter must be a list")
for element in argv:
if type(element) is list:
l.extend(element)
else:
l.append(element)
return l
def render_to_pdf(template_name,
params,
font_resolver=font_resolver,
image_resolver=image_resolver):
"""Renders PDF from RML, which is rendered from a Django template.
"""
rml = render_to_string(template_name, params).encode('utf-8')
try:
pdf = rml2pdf(rml, font_resolver, image_resolver)
except Exception, e:
raise TemplateError(str(e))
def status():
headers = {"Content-Type": "text/html", "charset": "utf-8"}
previous_states = list(reversed(this_node.known_nodes.states.values()))[:5]
try:
content = status_template.render(this_node=this_node,
known_nodes=this_node.known_nodes,
previous_states=previous_states)
except Exception as e:
log.debug("Template Rendering Exception: ".format(str(e)))
raise TemplateError(str(e)) from e
return Response(response=content, headers=headers)
def list_flatten(self, l):
if type(l) is not list:
raise TemplateError("flatten filter takes a list")
def recurse_flatten(l):
if type(l) is not list:
return [l]
r = []
for i in l:
r.extend(recurse_flatten(i))
return r
return recurse_flatten(l)
def Jinja2Process(flines, dir, inputs=[], inputs_file=None, verbose=False):
env = Environment(loader=FileSystemLoader(dir),
undefined=StrictUndefined,
extensions=['jinja2.ext.do'])
# Load any custom Jinja2 filters in the suite definition directory
# Example: a filter to pad integer values some fill character:
#|(file SUITE_DEFINIION_DIRECTORY/Jinja2/foo.py)
#| #!/usr/bin/env python
#| def foo( value, length, fillchar ):
#| return str(value).rjust( int(length), str(fillchar) )
fdirs = [
os.path.join(os.environ['CYLC_DIR'], 'lib', 'Jinja2Filters'),
os.path.join(dir, 'Jinja2Filters'),
os.path.join(os.path.join(os.environ['HOME'], '.cylc',
'Jinja2Filters'))
]
usedfdirs = []
for fdir in fdirs:
if os.path.isdir(fdir):
usedfdirs.append(fdir)
for filterdir in usedfdirs:
sys.path.append(os.path.abspath(filterdir))
for f in glob.glob(os.path.join(filterdir, '*.py')):
fname = os.path.basename(f).rstrip('.py')
# TODO - EXCEPTION HANDLING FOR LOADING CUSTOM FILTERS
m = __import__(fname)
env.filters[fname] = getattr(m, fname)
# Import SUITE HOST USER ENVIRONMENT into template:
# (usage e.g.: {{environ['HOME']}}).
env.globals['environ'] = os.environ
# load file lines into a template, excluding '#!jinja2' so
# that '#!cylc-x.y.z' rises to the top.
# CALLERS SHOULD HANDLE JINJA2 TEMPLATESYNTAXERROR AND TEMPLATEERROR
# try:
template = env.from_string('\n'.join(flines[1:]))
# except Exception, x:
# # This happens if we use an unknown Jinja2 filter, for example.
## # TODO - THIS IS CAUGHT BY VALIDATE BUT NOT BY VIEW COMMAND...
# raise TemplateError( x )
try:
template_vars = load_template_vars(inputs, inputs_file, verbose)
except Exception, x:
raise TemplateError(x)
def _validate_randomizable_source(
name: str, source: Union[str, List[Union[str, int, float]]]) -> None:
"""Validate that a randomizable template source has the proper format.
Raises a TemplateError when one of the items of a list source has an
invalid format, which can easily happen when one forgets to quote a complex
string.
"""
for val in source:
if not isinstance(val, str):
if not (isinstance(val, list) and len(val) == 2
and isinstance(val[1], (int, float))):
raise TemplateError(
f'Invalid format on randomizable template {name}. The '
f'individual items of a randomizable template must be '
f'strings or 2-item lists, where the second item is a '
f'number. Perhaps you forgot to enclose a complex string '
f'in quotes?')
def status():
# TODO: Seems very strange to deserialize *this node* when we can just pass it in.
# Might be a sign that we need to rethnk this composition.
headers = {"Content-Type": "text/html", "charset": "utf-8"}
this_node = _node_class.from_bytes(node_bytes_caster(), federated_only=federated_only)
previous_states = list(reversed(node_tracker.states.values()))[:5]
try:
content = status_template.render(this_node=this_node,
known_nodes=node_tracker,
previous_states=previous_states)
except Exception as e:
log.debug("Template Rendering Exception: ".format(str(e)))
raise TemplateError(str(e)) from e
return Response(response=content, headers=headers)
def raise_template_error(message):
raise TemplateError(message)
def render(self,
template=None,
output="./out/spec.html",
foreign_sources={},
local=None,
local_sources=None):
"""
With no parameters render will update the info.json with the transformation information
with params we'll render a html inline.
"""
# base data sources
raw_data = self._create_output_dict()
info_json = self._update_transform_stage(raw_data)
# we also need some basic jenkins details
# Jenkins is case sensitive, and the pattern si different for covid
family = info_json["families"][0].lower()
if "COVID" in family.upper():
family = family.upper()
else:
family = "-".join([x.capitalize() for x in family.split("-")])
# Jenkins Job ---------
# TODO - this bits very unlikely to be robust, but only matters for running
# locaally.... sometimes is better than never
jenkins_title = info_json["title"].replace(",", "").replace(" ", "-")
pub_name = info_json["publisher"].split(" ")
try:
jenkins_title = pub_name[0] + "-" + pub_name[
1][:1] + "-" + jenkins_title
except IndexError:
jenkins_title = pub_name[0] + jenkins_title
jenkins_job = pathify(
os.environ.get('JOB_NAME',
f'replace-with-family-name/job/')) + jenkins_title
jenkins_job = jenkins_job.replace("replace-with-family-name", family)
jenkins = {"job": jenkins_job, "family": family}
# Jenkins Build Status ---------
jenkins_build = f"https://ci.floop.org.uk/buildStatus/icon?job=GSS_data%2F{family}%2F{jenkins_title}/"
jenkins["build"] = jenkins_build
if local is not None and template is not None:
raise Exception(
"You can only pass EITHER local= or a remote template url/name, not both"
)
# Templating, going to wrap this in case something goes bang
try:
if local is not None:
try:
with open(local, "r") as f:
localTemplate = f.read()
except Exception as e:
raise Exception(
"Unable to open template from location {}.".format(
local)) from e
elif template is None:
# Exit immediately if no local template and we don't want to render
return
elif "/" not in template:
# If it's just a file name so it's a template from the standard library
template = "https://raw.githubusercontent.com/GSS-Cogs/frontend-template-resources/master/templates/jinja2/" + template
else:
pass
"""
A foreign source is where we want to call in a little more data for the renderer,
We'll add them to the two existing data sources we always make availible, which are:
info_json: the info.json
raw_data: the dictionary that is the tracer store (similar but with a little extra info)
"""
kwargs = {
"info_json": info_json,
"raw_data": raw_data,
"jenkins": jenkins
}
for template_referal, url in foreign_sources.items():
# template_referal is how we want to refer to this exta data
# in the template. Make sure it's not overwriting a standard data key
if template_referal in ["info_json", "raw_data"]:
raise Exception("Aborting, you cannot pass a data source with the protected " \
"label of {}.".format(template_referal))
r = requests.get(url)
if r.status_code != 200:
raise Exception(
"Unable to additional data from: {}, status code {}".
format(url, r.status_code))
kwargs[template_referal] = r.json()
# the user might also pass in some additional dictionaries
if local_sources is not None:
for template_referal, extra_dict in local_sources.items():
# template_referal is how we want to refer to this exta data
# in the template. Make sure it's not overwriting a standard data key
if template_referal in ["info_json", "raw_data"]:
raise Exception("Aborting, you cannot pass a data source with the protected " \
"label of {}.".format(template_referal))
kwargs[template_referal] = extra_dict
if local is None:
r = requests.get(template)
if r.status_code != 200:
raise Exception(
"Unable to http get template from: {}, status code {}".
format(template, r.status_code))
templateRenderer = Template(r.text)
else:
templateRenderer = Template(localTemplate)
outputText = templateRenderer.render(**kwargs)
with open(output, "w") as f:
f.write(outputText)
print("Template {} rendered as {}".format(template, output))
except Exception as e:
raise TemplateError(
"Problem encountered attmepting to render template") from e
def Jinja2Process( flines, dir, inputs=[], inputs_file=None ):
env = Environment( loader=FileSystemLoader(dir), undefined=StrictUndefined, extensions=['jinja2.ext.do'] )
# Load any custom Jinja2 filters in the suite definition directory
# Example: a filter to pad integer values some fill character:
#|(file SUITE_DEFINIION_DIRECTORY/Jinja2/foo.py)
#| #!/usr/bin/env python
#| def foo( value, length, fillchar ):
#| return str(value).rjust( int(length), str(fillchar) )
fdirs = [os.path.join( os.environ['CYLC_DIR'], 'lib', 'Jinja2Filters' ),
os.path.join( dir, 'Jinja2Filters' ),
os.path.join( os.path.join( os.environ['HOME'], '.cylc', 'Jinja2Filters' ))]
usedfdirs = []
for fdir in fdirs:
if os.path.isdir( fdir ):
usedfdirs.append( fdir )
for filterdir in usedfdirs:
sys.path.append( os.path.abspath( filterdir ))
for f in glob.glob( os.path.join( filterdir, '*.py' )):
fname = os.path.splitext(os.path.basename(f))[0]
# TODO - EXCEPTION HANDLING FOR LOADING CUSTOM FILTERS
m = __import__( fname )
env.filters[ fname ] = getattr( m, fname )
# Import SUITE HOST USER ENVIRONMENT into template:
# (usage e.g.: {{environ['HOME']}}).
env.globals['environ'] = os.environ
# load file lines into a template, excluding '#!jinja2' so
# that '#!cylc-x.y.z' rises to the top.
# CALLERS SHOULD HANDLE JINJA2 TEMPLATESYNTAXERROR AND TEMPLATEERROR
# try:
template = env.from_string( '\n'.join(flines[1:]) )
# except Exception as exc:
# # This happens if we use an unknown Jinja2 filter, for example.
## # TODO - THIS IS CAUGHT BY VALIDATE BUT NOT BY VIEW COMMAND...
# raise TemplateError(exc)
try:
template_vars = load_template_vars( inputs, inputs_file )
except Exception as exc:
if isinstance(exc, TemplateError):
raise
raise TemplateError(exc)
# CALLERS SHOULD HANDLE JINJA2 TEMPLATESYNTAXERROR AND TEMPLATEERROR
# AND TYPEERROR (e.g. for not using "|int" filter on number inputs.
# Convert unicode to plain str, ToDo - still needed for parsec?)
#try:
rendered = str( template.render( template_vars ) )
#except Exception, x:
# raise TemplateError( x )
xlines = rendered.split('\n')
suiterc = []
for line in xlines:
# Jinja2 leaves blank lines where source lines contain
# only Jinja2 code; this matters if line continuation
# markers are involved, so we remove blank lines here.
if re.match( '^\s*$', line ):
continue
# restoring newlines here is only necessary for display by
# the cylc view command:
###suiterc.append(line + '\n')
suiterc.append(line)
return suiterc