def get_html_tissue(self):
env = Environment()
env.loader = jinja2.FileSystemLoader(
get_package_location("gdsctools") + "/gdsctools/data/templates/")
template = env.get_template("boxplot_tissue.html")
data = self._get_boxplot_data("tissue")
if data is None:
return ""
# Show from bottom to top
labels = data[1][::-1]
data = data[0][::-1]
jinja = {}
N = len(self.odof.negatives) + len(self.odof.positives)
jinja["title"] = "FEATURE/MS-instability interactions"
jinja["subtitle"] = "%s versus %s" % (self.drug, self.feature)
factor = []
for i, thisdata in enumerate(data):
factor.extend([labels[i]] * len(thisdata))
jinja['sign'] = [x.split()[1] for x in factor]
jinja['status'] = [x.split()[0] for x in factor]
jinja["data"] = list(pylab.flatten([list(this) for this in data]))
jinja['xlabel'] = '"logIC50"'
if len(labels) / 2 >= 10:
jinja["minTextSize"] = 10
html = template.render(jinja)
return html
def get_html_association(self):
env = Environment()
env.loader = jinja2.FileSystemLoader(
get_package_location("gdsctools") + "/gdsctools/data/templates/")
template = env.get_template("boxplot_association.html")
jinja = {}
N = len(self.odof.negatives) + len(self.odof.positives)
jinja["title"] = "Individual Association"
if self.odof.drug_name:
drug_name = "%s (%s)" % (self.drug, self.odof.drug_name)
jinja["subtitle"] = "%s versus %s" % (drug_name, self.feature)
else:
jinja["subtitle"] = "%s versus %s" % (self.drug, self.feature)
jinja['factor'] = ["neg"] * len(self.odof.negatives) + ["pos"] * len(
self.odof.positives)
jinja["data"] = self.odof.negatives.tolist(
) + self.odof.positives.tolist()
jinja["smps"] = [str(this) for this in self.odof.indices]
jinja["subject"] = [str(this) for this in self.odof.indices]
jinja['ylabel'] = '"logIC50"'
html = template.render(jinja)
return html
def prepare_environment(env=None):
try:
loader = PackageLoader("glypy", "search/results_template")
loader.list_templates()
except:
loader = FileSystemLoader(os.path.join(os.path.dirname(__file__), 'results_template'))
if env is None:
env = Environment(loader=loader, extensions=[FragmentCacheExtension])
else:
env.loader = loader
env.add_extension(FragmentCacheExtension)
env.filters["collect_fragments"] = collect_fragments
env.filters["all_matches"] = fetch_all_matches
env.filters["strip_derivatize"] = strip_derivatize_glycoct
env.filters["scientific_notation"] = scientific_notation
env.filters["cfg_plot"] = cfg_plot
env.filters["simple_cfg_plot"] = simple_plot
env.filters["min"] = min
env.filters["max"] = max
env.filters["unique"] = unique
env.filters["limit_sigfig"] = limit_sigfig
env.filters['css_escape'] = css_escape
env.filters['greek_fragment_name'] = greek_fragment_names
env.fragment_cache = dict()
return env
def test_cache_loader_change(self):
loader1 = loaders.DictLoader({"foo": "one"})
loader2 = loaders.DictLoader({"foo": "two"})
env = Environment(loader=loader1, cache_size=2)
assert env.get_template("foo").render() == "one"
env.loader = loader2
assert env.get_template("foo").render() == "two"
def warning(message):
"""Print a warning message on the rich text view"""
env = Environment()
env.loader = FileSystemLoader(osp.join(CONFDIR_PATH, 'templates'))
warning = env.get_template("warning.html")
return warning.render(css_path=CSS_PATH, text=message)
def _render_html_file(self, template_fh, context, output_fh):
"""
Renders the HTML file using the configured template. Separated as a
method to be able to easily test.
:param context: A dict containing target urls, enabled plugins, etc.
:return: True on successful rendering
"""
severity_icon = functools.partial(get_severity_icon, self.template_root)
jinja2_env = Environment(undefined=StrictUndefined,
trim_blocks=True,
autoescape=True,
lstrip_blocks=True)
jinja2_env.filters['render_markdown'] = render_markdown
jinja2_env.filters['request'] = request_dump
jinja2_env.filters['response'] = response_dump
jinja2_env.filters['severity_icon'] = severity_icon
jinja2_env.filters['severity_text'] = get_severity_text
jinja2_env.globals['get_current_date'] = get_current_date
jinja2_env.loader = FileSystemLoader(self.template_root)
template = jinja2_env.from_string(template_fh.read())
try:
rendered_output = template.render(context)
except Exception, e:
msg = u'Failed to render html report template. Exception: "%s"'
om.out.error(msg % e)
return False
def generate_pages():
"""
Generates all static pages from templates.
"""
print('Generating pages...')
cfg = config()
env = Environment()
env.loader = FileSystemLoader(['pages', 'templates'])
for page in listdir('pages'):
print('Generating page {0}...'.format(page))
pagepath = path.join(cfg['output'], page.rstrip('.html'))
template = env.get_template(page)
html = template.render({
'page': page,
'author': cfg['author'],
'sitename': cfg['sitename'],
'license': cfg['license'],
'logo': cfg['logo'],
'rooturl': cfg['rooturl'],
'link': '{0}/'.format(page.rstrip('.html'))
})
try:
makedirs(path.join(pagepath))
except OSError:
pass
with open('{0}/index.html'.format(pagepath), 'w') as file:
file.write(html)
with open('{0}/{1}'.format(cfg['output'], page), 'w') as file:
file.write(html)
def main(data, template, output_file):
env = Environment()
env.loader = FileSystemLoader('.')
t = env.get_template(template)
t.globals['counter'] = counter
t.globals['reset_counter'] = reset_counter
open(output_file, 'w').write(t.render(json.load(open(data))))
def generate_archive(posts, tag_set):
print('Generating blog archive...')
env = Environment()
env.loader = FileSystemLoader('templates')
tpl = env.get_template('blog.html')
html = tpl.render(dict(
sitename=cfg['sitename'],
license=cfg['license'],
title='blog',
posts=posts
))
with open('blog.html', 'w') as file:
file.write(html)
for tag in tag_set:
print('Generating {0} archive page...'.format(tag))
post_list = []
for post in posts:
if tag in post['tags']:
post_list.append(post)
tpl = env.get_template('blog.html')
html = tpl.render(dict(
sitename=cfg['sitename'],
title='blog: #{0}'.format(tag),
posts=post_list
))
tagpath = path.join('tag', tag)
try:
mkdir(tagpath)
except OSError:
pass
with open('{0}/index.html'.format(tagpath), 'w') as file:
file.write(html)
def _render_html_file(self, template_fh, context, output_fh):
"""
Renders the HTML file using the configured template. Separated as a
method to be able to easily test.
:param context: A dict containing target urls, enabled plugins, etc.
:return: True on successful rendering
"""
severity_icon = functools.partial(get_severity_icon,
self.template_root)
jinja2_env = Environment(undefined=StrictUndefined,
trim_blocks=True,
autoescape=True,
lstrip_blocks=True)
jinja2_env.filters['render_markdown'] = render_markdown
jinja2_env.filters['request'] = request_dump
jinja2_env.filters['response'] = response_dump
jinja2_env.filters['severity_icon'] = severity_icon
jinja2_env.filters['severity_text'] = get_severity_text
jinja2_env.globals['get_current_date'] = get_current_date
jinja2_env.loader = FileSystemLoader(self.template_root)
template = jinja2_env.from_string(template_fh.read())
try:
rendered_output = template.render(context)
except Exception, e:
msg = u'Failed to render html report template. Exception: "%s"'
om.out.error(msg % e)
return False
def generate_archive(posts, tag_set):
print('Generating blog archive...')
env = Environment()
env.loader = FileSystemLoader('templates')
tpl = env.get_template('blog.html')
html = tpl.render(
dict(sitename=cfg['sitename'],
license=cfg['license'],
logo=cfg['logo'],
title='blog',
posts=posts))
with open('blog.html', 'w') as file:
file.write(html)
for tag in tag_set:
print('Generating {0} archive page...'.format(tag))
post_list = []
for post in posts:
if tag in post['tags']:
post_list.append(post)
tpl = env.get_template('blog.html')
html = tpl.render(
dict(sitename=cfg['sitename'],
license=cfg['license'],
logo=cfg['logo'],
title='blog: #{0}'.format(tag),
posts=post_list))
tagpath = path.join('tag', tag)
try:
mkdir(tagpath)
except OSError:
pass
with open('{0}/index.html'.format(tagpath), 'w') as file:
file.write(html)
def prepare_environment(env=None):
try:
raise Exception()
loader = PackageLoader("glycresoft_app", "html")
loader.list_templates()
except Exception:
loader = FileSystemLoader(
os.path.join(os.path.dirname(__file__), 'html'))
if env is None:
env = Environment(loader=loader)
else:
env.loader = loader
env.fragment_cache = dict()
env.filters["n_per_row"] = n_per_row
env.filters['highlight_sequence_site'] = highlight_sequence_site
env.filters['svg_plot'] = svg_plot
env.filters['png_plot'] = png_plot
env.filters['svguri_plot'] = svguri_plot
env.filters['glycopeptide_string'] = glycopeptide_string
env.filters['glycan_composition_string'] = glycan_composition_string
env.filters["formula"] = formula
env.filters["sort_peak_match_pairs"] = sort_peak_match_pairs
env.filters["mass_charge_ratio"] = mass_charge_ratio
env.filters["logit"] = logit
return env
def usage(title, message, tutorial_message, tutorial):
"""Print a usage message on the rich text view"""
env = Environment()
env.loader = FileSystemLoader(osp.join(CONFDIR_PATH, 'templates'))
usage = env.get_template("usage.html")
return usage.render(css_path=CSS_PATH, title=title, intro_message=message,
tutorial_message=tutorial_message, tutorial=tutorial)
def warning(message):
"""Print a warning message on the rich text view"""
env = Environment()
env.loader = FileSystemLoader(osp.join(CONFDIR_PATH, 'templates'))
warning = env.get_template("warning.html")
return warning.render(css_path=CSS_PATH, text=message)
def get_html_tissue(self):
env = Environment()
env.loader = jinja2.FileSystemLoader(
get_package_location("gdsctools")
+ "/gdsctools/data/templates/")
template = env.get_template("boxplot_tissue.html")
data = self._get_boxplot_data("tissue")
if data is None:
return ""
# Show from bottom to top
labels = data[1][::-1]
data = data[0][::-1]
jinja = {}
N = len(self.odof.negatives) + len(self.odof.positives)
jinja["title"] = "FEATURE/MS-instability interactions"
jinja["subtitle"] = "%s versus %s" % (self.drug, self.feature)
factor = []
for i, thisdata in enumerate(data):
factor.extend( [labels[i]] * len(thisdata))
jinja['sign'] = [x.split()[1] for x in factor]
jinja['status'] = [x.split()[0] for x in factor]
jinja["data"] = list(pylab.flatten([list(this) for this in data]))
jinja['xlabel'] = '"logIC50"'
if len(labels)/2 >= 10:
jinja["minTextSize"] = 10
html = template.render(jinja)
return html
def test_cache_loader_change(self):
loader1 = loaders.DictLoader({'foo': 'one'})
loader2 = loaders.DictLoader({'foo': 'two'})
env = Environment(loader=loader1, cache_size=2)
assert env.get_template('foo').render() == 'one'
env.loader = loader2
assert env.get_template('foo').render() == 'two'
def render_invoice(id, client, data):
if 'hours' in data['services'][0]:
tmpl_file = 'hourly.html'
else:
tmpl_file = 'flat.html'
dir = os.path.dirname(os.path.abspath(__file__))
templ_dir = os.path.join(dir, 'assets')
env = Environment()
env.loader = FileSystemLoader(templ_dir)
tmpl = env.get_template(tmpl_file)
html = tmpl.render(id=id, client=client, **data)
fname = 'invoices/{}.html'.format(id)
pdfname = 'invoices/{}.pdf'.format(id)
with open(fname, 'w') as f:
f.write(html)
subprocess.call([
'wkhtmltopdf',
'-q', # quiet
'-T',
'2', # top margin
'-L',
'0', # left margin
'-R',
'0', # right margin
'--zoom',
'0.8',
fname,
pdfname
])
return pdfname
def get_html_association(self):
env = Environment()
env.loader = jinja2.FileSystemLoader(
get_package_location("gdsctools")
+ "/gdsctools/data/templates/")
template = env.get_template("boxplot_association.html")
jinja = {}
N = len(self.odof.negatives) + len(self.odof.positives)
jinja["title"] = "Individual Association"
if self.odof.drug_name:
drug_name = "%s (%s)" % (self.drug, self.odof.drug_name)
jinja["subtitle"] = "%s versus %s" % (drug_name, self.feature)
else:
jinja["subtitle"] = "%s versus %s" % (self.drug, self.feature)
jinja['factor'] = ["neg"] * len(self.odof.negatives) + ["pos"] * len(self.odof.positives)
jinja["data"] = self.odof.negatives.tolist() + self.odof.positives.tolist()
jinja["smps"] = [str(this) for this in self.odof.indices]
jinja["subject"] = [str(this) for this in self.odof.indices]
jinja['ylabel'] = '"logIC50"'
html = template.render(jinja)
return html
def usage(title, message, tutorial_message, tutorial):
"""Print a usage message on the rich text view"""
env = Environment()
env.loader = FileSystemLoader(osp.join(CONFDIR_PATH, 'templates'))
usage = env.get_template("usage.html")
return usage.render(css_path=CSS_PATH, title=title, intro_message=message,
tutorial_message=tutorial_message, tutorial=tutorial)
def render(template, folder="templates", **kwargs):
env = Environment()
env.loader = FileSystemLoader(folder)
template = env.get_template(template)
return template.render(**kwargs)
def prepare_environment(env=None):
try:
loader = PackageLoader("glycresoft_sqlalchemy.web_app", "html")
loader.list_templates()
except:
loader = FileSystemLoader(os.path.join(os.path.dirname(__file__), 'html'))
if env is None:
env = Environment(loader=loader, extensions=[FragmentCacheExtension])
else:
env.loader = loader
env.add_extension(FragmentCacheExtension)
env.fragment_cache = dict()
env.filters["n_per_row"] = n_per_row
env.filters['highlight_sequence_site'] = highlight_sequence_site
env.filters['plot_glycoforms'] = plot_glycoforms
env.filters['chromatogram'] = plot_chromatogram
env.filters['svg_plot'] = svg_plot
env.filters['png_plot'] = png_plot
env.filters['fsort'] = fsort
env.filters['glycopeptide_string'] = glycopeptide_string
env.filters['glycan_composition_string'] = glycan_composition_string
env.filters["glycopeptide_match_logo"] = glycopeptide_match_logo
env.filters["formula"] = formula
env.globals
return env
def test_cache_loader_change(self):
loader1 = loaders.DictLoader({'foo': 'one'})
loader2 = loaders.DictLoader({'foo': 'two'})
env = Environment(loader=loader1, cache_size=2)
assert env.get_template('foo').render() == 'one'
env.loader = loader2
assert env.get_template('foo').render() == 'two'
def prepare_env(cnt, file_name=True, inline=False):
this = {}
if file_name:
this['file_name_json'] = os.path.join(SAMPLES_PATH, '{}.json'.format(cnt))
this['file_name_rst'] = os.path.join(SAMPLES_PATH,
'{}{inline}.rst'.format(cnt, inline='_inline' if inline else '')
)
with codecs.open(this['file_name_json'], 'r', encoding='utf-8') as _file:
doc = json.load(_file)
else:
this['file_name_json'] = False
this['file_name_rst'] = False
doc = json.load(cnt)
this['swagger_doc'] = rst.SwaggerObject(doc)
doc_module = importlib.import_module('swg2rst.utils.rst')
jinja_env = Environment(lstrip_blocks=True, trim_blocks=True)
jinja_env.loader = PackageLoader('swg2rst')
for name, function in inspect.getmembers(doc_module, inspect.isfunction):
jinja_env.filters[name] = function
jinja_env.filters['sorted'] = sorted
template = jinja_env.get_template('main.rst')
this['raw_rst'] = template.render(doc=this['swagger_doc'], inline=inline)
this['pattern'] = re.compile(r'[idm]_\w{32}')
this['normalize'] = lambda x: x[:-1] if x[-1] == '\n' else x
return this
def get_html(self):
# We have 3 colors but sometimes you may have only one or 2.
# This may be an issue with canvasXpress. It seems essential
# to sort the color column so that names are sorted alphabetically
# and to include colors that are present in the sale order
"""try:
self.data.sort_values(by='color', inplace=True)
except:
self.data.sort("color", inplace=True)
"""
# Jinja related
from easydev import get_package_location
env = Environment()
env.loader = jinja2.FileSystemLoader(
get_package_location("gdsctools") + "/gdsctools/data/templates/")
template = env.get_template("scatter.html")
# We need to cireate 20 different colors
from colormap import Colormap
c = Colormap()
cmap = c.cmap_linear("red", "blue", "yellow", N=20)
colors = self.data[self._colname_color]
colors = ["red" for x in self.data[self._colname_color]]
jinja = {}
jinja["colors"] = ["rgba(205,0,0,0.5)"]
jinja["Group"] = colors
jinja['xlabel'] = '"%s"' % self.xlabel
jinja['ylabel'] = '"%s"' % self.ylabel
selection = [self._colname_x, self._colname_y]
text = []
for index in zip(self.data.index):
text.append("<pre>%s</pre>" %
self.data.ix[index][selection].to_string())
jinja['vars'] = text
#self.data.markersize /= (self.data.markersize.max()/3.)
self.data['markersize'] = 20
selection = [self._colname_x, self._colname_y, self._colname_size]
#First value is Y, second is X, following will be used in the
try: # introduced in pandas > 0.16.2
jinja['data'] = self.data[selection].round(3).values.tolist()
except: #for py3.3 on travis
jinja['data'] = np.around(self.data[selection]).values.tolist()
jinja[
'title'] = '"Regression coefficient vs Bayes factor for all drugs"'
jinja['minX'] = self.minX
jinja['minY'] = self.minY
jinja['maxX'] = self.maxX
jinja['maxY'] = self.maxY
self.html = template.render(jinja)
return self.html
def prepare_template(flags, module):
jinja_env = Environment(lstrip_blocks=True, trim_blocks=True)
for name, function in inspect.getmembers(module, inspect.isfunction):
jinja_env.filters[name] = function
if flags.template:
jinja_env.loader = FileSystemLoader(os.path.dirname(flags.template))
template = jinja_env.get_template(os.path.basename(flags.template))
else:
jinja_env.loader = PackageLoader('swg2rst')
try:
template = jinja_env.get_template('main.{}'.format(flags.format))
except TemplateError as err:
sys.exit(u'Template Error: {}'.format(err.message))
return template
def prepare_template(flags, module):
jinja_env = Environment(lstrip_blocks=True, trim_blocks=True)
for name, function in inspect.getmembers(module, inspect.isfunction):
jinja_env.filters[name] = function
if flags.template:
jinja_env.loader = FileSystemLoader(os.path.dirname(flags.template))
template = jinja_env.get_template(os.path.basename(flags.template))
else:
jinja_env.loader = PackageLoader('swg2rst')
try:
template = jinja_env.get_template('main.{}'.format(flags.format))
except TemplateError as err:
sys.exit(u'Template Error: {}'.format(err.message))
return template
def loading(message, loading_img, css_path=CSS_PATH):
"""Print loading message on the rich text view."""
env = Environment()
env.loader = FileSystemLoader(osp.join(CONFDIR_PATH, 'templates'))
loading = env.get_template("loading.html")
return loading.render(css_path=css_path,
loading_img=loading_img,
message=message)
def get_html(self):
# We have 3 colors but sometimes you may have only one or 2.
# This may be an issue with canvasXpress. It seems essential
# to sort the color column so that names are sorted alphabetically
# and to include colors that are present in the sale order
"""try:
self.data.sort_values(by='color', inplace=True)
except:
self.data.sort("color", inplace=True)
"""
# Jinja related
from easydev import get_package_location
env = Environment()
env.loader = jinja2.FileSystemLoader(
get_package_location("gdsctools")
+ "/gdsctools/data/templates/")
template = env.get_template("scatter.html")
# We need to cireate 20 different colors
from colormap import Colormap
c = Colormap()
cmap = c.cmap_linear("red", "blue", "yellow", N=20)
colors = self.data[self._colname_color]
colors = ["red" for x in self.data[self._colname_color]]
jinja = {}
jinja["colors"] = ["rgba(205,0,0,0.5)"]
jinja["Group"] = colors
jinja['xlabel'] = '"%s"' % self.xlabel
jinja['ylabel'] = '"%s"' % self.ylabel
text = []
for index in zip(self.data.index):
text.append("<pre>%s</pre>" % self.data.ix[index].to_string())
jinja['vars'] = text
#self.data.markersize /= (self.data.markersize.max()/3.)
self.data['markersize'] = 20
selection = [self._colname_x, self._colname_y, self._colname_size]
#First value is Y, second is X, following will be used in the
try: # introduced in pandas > 0.16.2
jinja['data'] = self.data[selection].round(3).values.tolist()
except: #for py3.3 on travis
jinja['data'] = np.around(self.data[selection]).values.tolist()
jinja['title'] = '"Regression coefficient/ttest/bayes factor for all drugs"'
jinja['minX'] = 0
jinja['minY'] = 0
jinja['maxX'] = 1
jinja['maxY'] = self.data[self._colname_y].max() * 1.1
self.html = template.render(jinja)
return self.html
def run(self, wrapper):
self.wrapper = wrapper
self.remove_reports_dir(wrapper)
self.copy_template_files()
# If not too large, copy the log so it can be viewed in HTML
self.wrapper.flush_logs()
if os.path.getsize(self.wrapper.log_path()) < 500000:
with codecs.open(self.wrapper.log_path(), 'r', encoding='UTF-8') as f:
log_contents = f.read()
else:
log_contents = "Log file is too large to include in HTML. Look in %s" % self.wrapper.log_path()
env_data = self.template_data()
# add additional env elements - should these also be in plugins?
env_data['wrapper'] = wrapper
env_data['batch'] = wrapper.batch
env_data['log_contents'] = log_contents
def printable_args(args):
return dict((k, v) for k, v in args.iteritems() if not k in ('contents', 'wrapper'))
def print_children(node, indent=0, extra=""):
rand_id = random.randint(10000000,99999999)
spaces = " " * 4 * indent
nbspaces = " " * 4 * indent
content = ""
node_div = """%s<div data-toggle="collapse" data-target="#%s">%s%s%s%s</div>"""
node_div_args = (spaces, rand_id, nbspaces, extra,
node.key_with_class(), link_to_doc_if_doc(node),)
content += node_div % node_div_args
content += """ %s<div id="%s" class="collapse">""" % (spaces, rand_id)
for child in list(node.inputs):
if not "Artifact" in child.__class__.__name__:
content += print_children(child, indent+1, "→")
for child in node.children:
if not "Artifact" in child.__class__.__name__:
content += print_children(child, indent+1)
content += " %s</div>" % spaces
return content
env_data['print_children'] = print_children
env_data['printable_args'] = printable_args
env = Environment()
env.loader = FileSystemLoader(os.path.dirname(__file__))
template = env.get_template('template.html')
template.stream(env_data).dump(self.report_file(), encoding="utf-8")
def summary(self):
env = Environment(trim_blocks=True, lstrip_blocks=True)
env.filters['column_display_name'] = column_display_name
env.loader = FileSystemLoader(
os.path.join(os.path.dirname(__file__), 'templates'))
template = env.get_template('{0}.markdown.jinja'.format(
self.__class__.__name__.lower()))
rendered = template.render(result=self.result,
titles=self._format_titles,
is_empty=self._is_empty)
return rendered
def render(self, form_submission_action='https://workersandbox.mturk.com/mturk/externalSubmit'):
env = Environment(trim_blocks=True, lstrip_blocks=True, extensions=['jinja2.ext.do'])
env.loader = FileSystemLoader(TEMPLATES_DIR + 'base')
template = env.get_template('HIT.html')
self.questions = self.__get_questions()
with open(BASE_DIR + "/render.html", "w+") as f:
f.write(template.render(HIT=self, jinja_form_action=form_submission_action))
try:
render = template.render(HIT=self, jinja_form_action=form_submission_action)
except Exception as e:
HIT_logger.error("Unable to get opencrowd question {}. An exception of type {} occurred. Arguments:\n{!r}".format(type(e).__name__, e.args))
return render
def generate_pages():
env = Environment()
env.loader = FileSystemLoader(['pages', 'templates'])
for page in listdir('pages'):
print('Generating page {0}...'.format(page))
template = env.get_template(page)
html = template.render({'page': page,
'author': cfg['author'],
'sitename': cfg['sitename'],
'license': cfg['license']})
with open(page, 'w') as file:
file.write(html)
def render(template_name, folder='templates', **kwars):
"""
Функция рендеринга шаблона html.
:param template_name имя шаблона
:param folder папка с шаблонами
:param kwars параметры для рендеринга
:return Рендер шаблона
"""
environ = Environment()
environ.loader = FileSystemLoader(folder)
template = environ.get_template(template_name)
return template.render(**kwars)
def __setup():
"""Caffe helper set up function"""
import os
import sys
from jinja2 import Environment, FileSystemLoader
def j2filter_slice_list(value, slices):
try:
return [value[i] for i in slices]
except TypeError:
return value[:slices]
def j2filter_bool2str(value):
if value:
return 'true'
else:
return 'false'
def j2filter_to_int(value):
return int(value)
j2filters = {
'max': max,
'min': min,
'slice_list': j2filter_slice_list,
'bool2str': j2filter_bool2str,
'to_int': j2filter_to_int,
}
pj = os.path.join
caffe_root = os.getenv("CAFFE_ROOT")
if caffe_root is None:
raise ValueError(
"Before calling this module, you should set `CAFFE_ROOT` as "
"your environmental variable.")
caffe_bin = pj(caffe_root, 'build/tools/caffe')
dir_log = pj(caffe_root, 'tmp', 'log')
if not os.path.isdir(dir_log):
os.makedirs(dir_log)
# Setting path to Caffe python
sys.path.append(pj(caffe_root, 'python'))
# Setting up jinja2
path_jinja2 = os.path.abspath(
pj(os.path.dirname(__file__), '..', '..', 'model_templates')
)
env = Environment(extensions=['jinja2.ext.do'])
env.filters.update(j2filters)
env.loader = FileSystemLoader(path_jinja2)
dir_proto_out = pj(caffe_root, 'tmp', 'prototxt')
dir_template = path_jinja2
if not os.path.isdir(dir_proto_out):
os.makedirs(dir_proto_out)
return caffe_root, caffe_bin, dir_log, env, dir_proto_out, dir_template,
def __setup():
"""Caffe helper set up function"""
import os
import sys
from jinja2 import Environment, FileSystemLoader
def j2filter_slice_list(value, slices):
try:
return [value[i] for i in slices]
except TypeError:
return value[:slices]
def j2filter_bool2str(value):
if value:
return 'true'
else:
return 'false'
def j2filter_to_int(value):
return int(value)
j2filters = {
'max': max,
'min': min,
'slice_list': j2filter_slice_list,
'bool2str': j2filter_bool2str,
'to_int': j2filter_to_int,
}
pj = os.path.join
caffe_root = os.getenv("CAFFE_ROOT")
if caffe_root is None:
raise ValueError(
"Before calling this module, you should set `CAFFE_ROOT` as "
"your environmental variable.")
caffe_bin = pj(caffe_root, 'build/tools/caffe')
dir_log = pj(caffe_root, 'tmp', 'log')
if not os.path.isdir(dir_log):
os.makedirs(dir_log)
# Setting path to Caffe python
sys.path.append(pj(caffe_root, 'python'))
# Setting up jinja2
path_jinja2 = os.path.abspath(
pj(os.path.dirname(__file__), '..', '..', 'model_templates'))
env = Environment(extensions=['jinja2.ext.do'])
env.filters.update(j2filters)
env.loader = FileSystemLoader(path_jinja2)
dir_proto_out = pj(caffe_root, 'tmp', 'prototxt')
dir_template = path_jinja2
if not os.path.isdir(dir_proto_out):
os.makedirs(dir_proto_out)
return caffe_root, caffe_bin, dir_log, env, dir_proto_out, dir_template,
def create_page ():
env = Environment()
env.loader = FileSystemLoader('/templates/')
template = env.get_template('slider.html')
#template = Template('<html> Hello {{ name }}!</html>')
templateVars = { "title" : "Photo Gallery",
"allURL" : allURL
}
site = template.render(templateVars)
#print site[1:]
make_sure_path_exists("/templates/")
with open ('/templates/gallery.html','w+') as file:
file.write(site.encode('utf-8'))
def build_undercloud_on_libvirt(self, image_path,
rhsm=None, repositories=[]):
"""Build the Undercloud by using instack-virt-setup script."""
self.run('sysctl net.ipv4.ip_forward=1')
self.fetch_image(path=image_path, dest='/home/stack/guest_image.qcow2',
user='******')
# NOTE(Gonéri): this is a hack for our OpenStack, the MTU of its outgoing route
# is 1400 and libvirt do not provide a mechanism to adjust the guests MTU.
self.run("LIBGUESTFS_BACKEND=direct virt-customize -a /home/stack/guest_image.qcow2 --run-command 'echo MTU=\"1400\" >> /etc/sysconfig/network-scripts/ifcfg-eth0'")
env = Environment()
env.loader = FileSystemLoader(pkg_data_filename('template'))
template = env.get_template('virt-setup-env.j2')
self.run('mkdir -p /home/stack/DIB', user='******')
self.run('cp -v /etc/yum.repos.d/*.repo /home/stack/DIB', user='******')
# NOTE(Gonéri): Hack to be sure DIB won't complain because of missing gpg files
# self.run('sed -i "s,gpgcheck=1,gpgcheck=0," /home/stack/DIB/*.repo', user='******')
dib_yum_repo_conf = self.run('find /home/stack/DIB -type f', user='******')[0].split()
virt_setup_template = {
'dib_yum_repo_conf': dib_yum_repo_conf,
'node': {
'count': 2,
'mem': 6144,
'cpu': 2
},
'undercloud_node_mem': 8192,
'guest_image_name': '/home/stack/guest_image.qcow2'
}
if rhsm is not None:
virt_setup_template['rhsm'] = {
'login': rhsm.get('login'),
'password': rhsm.get('password', os.environ.get('RHN_PW')),
'pool_id': rhsm.get('pool_id', ''),
'repositories': [i['name'] for i in repositories if i['type'] == 'rhsm_channel']
}
virt_setup_env = template.render(virt_setup_template)
self.create_file('virt-setup-env', virt_setup_env, user='******')
self.run('virsh destroy instack', ignore_error=True)
self.run('virsh undefine instack --remove-all-storage', ignore_error=True)
self.run('source virt-setup-env; instack-virt-setup', user='******')
undercloud_ip = self.run(
'/sbin/ip n | grep $(tripleo get-vm-mac instack) | awk \'{print $1;}\'',
user='******')[0]
assert undercloud_ip, 'undercloud should have an IP'
undercloud = Undercloud(hostname=undercloud_ip,
via_ip=self.hostname,
user='******',
key_filename=self._key_filename)
return undercloud
def generate_pages():
env = Environment()
env.loader = FileSystemLoader(['pages', 'templates'])
for page in listdir('pages'):
print('Generating page {0}...'.format(page))
template = env.get_template(page)
html = template.render({
'page': page,
'author': cfg['author'],
'sitename': cfg['sitename'],
'license': cfg['license']
})
with open(page, 'w') as file:
file.write(html)
def get_env(self, extensions = None):
default_extensions = [ "jinja2.ext.do", "jinja2.ext.with_" ]
if extensions:
extensions.extend(default_extensions)
else:
extensions = default_extensions
kwargs = dict(trim_blocks = True,
lstrip_blocks = True,
undefined = StrictUndefined,
extensions = extensions)
#if not template_path.startswith("/"): kwargs["loader"] = PackageLoader("onering", "data/templates")
env = Environment(**kwargs)
env.loader = self
return env
def jinja_environment(self, template_path, additional_args=None):
"""
Returns jinja Environment object.
"""
args = {'undefined': jinja2.StrictUndefined}
if additional_args:
args.update(additional_args)
env = Environment(**args)
dirs = [".", os.path.dirname(__file__), os.path.dirname(template_path)]
env.loader = FileSystemLoader(dirs)
return env
def render_template(template_file, template_paths, data, filters={}, **kwargs):
# Create Jinja environment
environment = Environment()
environment.loader = FileSystemLoader(template_paths)
if filters:
environment.filters = dict(environment.filters.items() + filters.items())
# Render template, passing data in as Config dictionary
try:
template = environment.get_template(template_file)
rendered = template.render({'Config': data})
return yaml.load(rendered)
except TemplateNotFound as e:
raise AnsibleError("Could not locate template %s in the supplied template paths %s" % (template_file,template_paths))
except Exception as e:
raise AnsibleError("An error occurred: %s" % e)
def render_templates(ctxt, dom_path, dom_templates, local_path,
local_templates):
env = Environment()
env.loader = PackageLoader('basejmpr.domain', 'templates')
# Expect to fail if exists
os.makedirs(dom_path)
for t in dom_templates:
rt = env.get_template(t).render(**ctxt)
with open(os.path.join(dom_path, t), 'w') as fd:
fd.write(rt)
for t in local_templates:
rt = env.get_template(t).render(**ctxt)
with open(os.path.join(local_path, t), 'w') as fd:
fd.write(rt)
def _render_html_file(self, template_fh, context, output_fh):
"""
Renders the HTML file using the configured template. Separated as a
method to be able to easily test.
:param context: A dict containing target urls, enabled plugins, etc.
:return: True on successful rendering
"""
severity_icon = functools.partial(get_severity_icon,
self.template_root)
env_config = {
'undefined': StrictUndefined,
'trim_blocks': True,
'autoescape': True,
'lstrip_blocks': True
}
try:
jinja2_env = Environment(**env_config)
except TypeError:
# Kali uses a different jinja2 version, which doesn't have the same
# Environment kwargs, so we first try with the version we expect
# to have available, and then if it doesn't work apply this
# workaround for Kali
#
# https://github.com/andresriancho/w3af/issues/9552
env_config.pop('lstrip_blocks')
jinja2_env = Environment(**env_config)
jinja2_env.filters['render_markdown'] = render_markdown
jinja2_env.filters['request'] = request_dump
jinja2_env.filters['response'] = response_dump
jinja2_env.filters['severity_icon'] = severity_icon
jinja2_env.filters['severity_text'] = get_severity_text
jinja2_env.globals['get_current_date'] = get_current_date
jinja2_env.loader = FileSystemLoader(self.template_root)
template = jinja2_env.from_string(template_fh.read())
report_stream = template.stream(context)
report_stream.enable_buffering(5)
for report_section in report_stream:
output_fh.write(report_section.encode('utf-8'))
return True
def __init__(self):
with open('atomfeed.xml') as f:
atom_template = f.read()
env = Environment()
env.loader = DictLoader({'atom': atom_template})
env.filters['article'] = self.render_article
env.filters['isodate'] = self.isodate
self.env = env
self.bbcode_parser = bbcode.Parser(escape_html=False, replace_links=False)
self.bbcode_parser.add_simple_formatter('img', '<img src="%(value)s">')
for i in range(1, 7):
tag = 'h%d' % i
self.bbcode_parser.add_simple_formatter(tag, '<{t}>%(value)s</{t}>'.format(t=tag))
def jinja_environment(self, template_path, additional_args=None):
"""
Returns jinja Environment object.
"""
args = {
'undefined' : jinja2.StrictUndefined
}
if additional_args:
args.update(additional_args)
env = Environment(**args)
dirs = [".", os.path.dirname(__file__), os.path.dirname(template_path)]
env.loader = FileSystemLoader(dirs)
return env
def render(template_name, folder='templates', **kwargs):
"""
:param template_name: имя шаблона
:param folder: папка в которой ищем шаблон
:param kwargs: параметры
:return:
"""
# file_path = os.path.join(folder, template_name)
# # Открываем шаблон по имени
# with open(file_path, encoding='utf-8') as f:
# # Читаем
# template = Template(f.read())
# # рендерим шаблон с параметрами
env = Environment()
env.loader = FileSystemLoader(folder)
template = env.get_template(template_name)
return template.render(**kwargs)
def generate_feeds(posts, tag_set):
"""
Generates atom feeds
"""
print('Generating atom feed...')
cfg = _config()
updated = str(time.strftime('%Y-%m-%dT%H:%M:%SZ'))
env = Environment()
env.loader = FileSystemLoader('templates')
xml = env.get_template('feed.xml').render(
items=posts[:10],
sitename=cfg['sitename'],
author=cfg['author'],
rooturl=cfg['rooturl'],
license=cfg['license'],
logo=cfg['logo'],
updated=updated
)
with open(cfg['output'] + '/feed.xml', 'w') as file:
file.write(xml)
for tag in tag_set:
print('Generating {0} atom feed...'.format(tag))
post_list = []
for post in posts[:10]:
if tag in post['tags']:
post_list.append(post)
xml = env.get_template('feed.xml').render(
items=post_list,
sitename=cfg['sitename'],
author=cfg['author'],
rooturl=cfg['rooturl'],
tagtitle=' - {0}'.format(tag),
updated=updated
)
tagpath = path.join(cfg['output'], 'tag', tag)
try:
makedirs(tagpath)
except OSError:
pass
with open('{0}/feed.xml'.format(tagpath), 'w') as file:
file.write(xml)
def _get_jinja2_env(self):
"""
Creates the jinja2 environment which will be used to render all templates
The same environment is used in order to take advantage of jinja's template
cache.
:return: A jinja2 environment
"""
env_config = {'undefined': StrictUndefined,
'trim_blocks': True,
'autoescape': True,
'lstrip_blocks': True}
jinja2_env = Environment(**env_config)
jinja2_env.loader = FileSystemLoader(TEMPLATE_ROOT)
jinja2_env.filters['escape_attr_val'] = jinja2_attr_value_escape_filter
return jinja2_env
def render_template(template_file, template_paths, data, filters={}, **kwargs):
# Create Jinja environment
environment = Environment()
environment.loader = FileSystemLoader(template_paths)
if filters:
environment.filters = dict(environment.filters.items() +
filters.items())
# Render template, passing data in as Config dictionary
try:
template = environment.get_template(template_file)
rendered = template.render({'Config': data})
return yaml.load(rendered)
except TemplateNotFound as e:
raise AnsibleError(
"Could not locate template %s in the supplied template paths %s" %
(template_file, template_paths))
except Exception as e:
raise AnsibleError("An error occurred: %s" % e)
def instack_virt_setup(self,
guest_image_path,
guest_image_checksum,
rhsm_login=None,
rhsm_password=None):
self.run('sysctl net.ipv4.ip_forward=1')
self.fetch_image(path=guest_image_path,
checksum=guest_image_checksum,
dest='/home/stack/guest_image.qcow2',
user='******')
# NOTE(Gonéri): this is a hack for our OpenStack, the MTU of its outgoing route
# is 1400 and libvirt do not provide a mechanism to adjust the guests MTU.
self.run(
"LIBGUESTFS_BACKEND=direct virt-customize -a /home/stack/guest_image.qcow2 --run-command 'echo MTU=\"1400\" >> /etc/sysconfig/network-scripts/ifcfg-eth0'"
)
env = Environment()
env.loader = FileSystemLoader('templates')
template = env.get_template('virt-setup-env.j2')
virt_setup_env = template.render({
'dib_dir': '/home/stack/DIB',
'node': {
'count': 3,
'mem': 4096,
'cpu': 2
},
'undercloud_node_mem': 4096,
'guest_image_name': '/home/stack/guest_image.qcow2',
'rhsm': {
'user': rhsm_login,
'password': rhsm_password
}
})
self.create_file('virt-setup-env', virt_setup_env, user='******')
self.run('source virt-setup-env; instack-virt-setup', user='******')
undercloud_ip = self.run(
'/sbin/ip n | grep $(tripleo get-vm-mac instack) | awk \'{print $1;}\'',
user='******')[0]
undercloud = Undercloud(undercloud_ip,
via_ip=self._hostname,
user='******',
key_filename=self._key_filename)
return undercloud
def _render_html_file(self, template_fh, context, output_fh):
"""
Renders the HTML file using the configured template. Separated as a
method to be able to easily test.
:param context: A dict containing target urls, enabled plugins, etc.
:return: True on successful rendering
"""
severity_icon = functools.partial(get_severity_icon, self.template_root)
env_config = {'undefined': StrictUndefined,
'trim_blocks': True,
'autoescape': True,
'lstrip_blocks': True}
try:
jinja2_env = Environment(**env_config)
except TypeError:
# Kali uses a different jinja2 version, which doesn't have the same
# Environment kwargs, so we first try with the version we expect
# to have available, and then if it doesn't work apply this
# workaround for Kali
#
# https://github.com/andresriancho/w3af/issues/9552
env_config.pop('lstrip_blocks')
jinja2_env = Environment(**env_config)
jinja2_env.filters['render_markdown'] = render_markdown
jinja2_env.filters['request'] = request_dump
jinja2_env.filters['response'] = response_dump
jinja2_env.filters['severity_icon'] = severity_icon
jinja2_env.filters['severity_text'] = get_severity_text
jinja2_env.globals['get_current_date'] = get_current_date
jinja2_env.loader = FileSystemLoader(self.template_root)
template = jinja2_env.from_string(template_fh.read())
report_stream = template.stream(context)
report_stream.enable_buffering(5)
for report_section in report_stream:
output_fh.write(report_section.encode('utf-8'))
return True
def run(self, wrapper):
self.wrapper = wrapper
self.remove_reports_dir(wrapper)
self.copy_template_files()
env_data = self.run_plugins()
graphs = {
"name" : "",
"contents" : [self.node_info(node) for node in wrapper.roots]
}
env_data['graph_json'] = json.dumps(graphs)
env = Environment()
env.loader = FileSystemLoader(os.path.dirname(__file__))
template = env.get_template('template.js')
template.stream(env_data).dump(self.report_file(), encoding="utf-8")
def run(self):
self.load_batch_artifacts()
# Create a directory to hold the report.
report_dir = os.path.join(self.logsdir, "run-%s" %
datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S"))
latest_report_dir = os.path.join(self.logsdir, "run-latest")
report_filename = os.path.join(report_dir, 'index.html')
shutil.rmtree(report_dir, ignore_errors=True)
template_dir = os.path.join(os.path.dirname(__file__), 'run_reporter')
shutil.copytree(template_dir, report_dir)
formatter = HtmlFormatter()
js_lexer = JavascriptLexer()
env_data = {}
# highlight config
j = json.dumps(self.batch_info['config'], sort_keys = True, indent=4)
dexy_config = highlight(j, js_lexer, formatter)
def highlight_json(d):
j = json.dumps(d, sort_keys=True, indent=4)
return highlight(j, js_lexer, formatter)
env_data['artifacts'] = self.artifacts
env_data['batch_info'] = self.batch_info
env_data['dexy_config'] = dexy_config
env_data['docs'] = self.batch_info['docs']
env_data['sorted'] = sorted
env_data['highlight_json'] = highlight_json
env = Environment()
env.loader = FileSystemLoader(os.path.dirname(__file__))
template = env.get_template('run_reporter_template.html')
template.stream(env_data).dump(report_filename)
shutil.rmtree(latest_report_dir, ignore_errors=True)
shutil.copytree(report_dir, latest_report_dir)
def generate_pages():
"""
Generates all static pages from templates.
"""
print('Generating pages...')
cfg = _config()
env = Environment()
env.loader = FileSystemLoader(['pages', 'templates'])
for page in listdir('pages'):
print('Generating page {0}...'.format(page))
template = env.get_template(page)
html = template.render({'page': page,
'author': cfg['author'],
'sitename': cfg['sitename'],
'license': cfg['license'],
'logo': cfg['logo'],
'rooturl': cfg['rooturl'],
'link': page})
with open(cfg['output'] + '/' + page, 'w') as file:
file.write(html)
def instack_virt_setup(self, guest_image_path, guest_image_checksum,
rhsm_login=None, rhsm_password=None):
self.run('sysctl net.ipv4.ip_forward=1')
self.fetch_image(path=guest_image_path, checksum=guest_image_checksum, dest='/home/stack/guest_image.qcow2',
user='******')
# NOTE(Gonéri): this is a hack for our OpenStack, the MTU of its outgoing route
# is 1400 and libvirt do not provide a mechanism to adjust the guests MTU.
self.run("LIBGUESTFS_BACKEND=direct virt-customize -a /home/stack/guest_image.qcow2 --run-command 'echo MTU=\"1400\" >> /etc/sysconfig/network-scripts/ifcfg-eth0'")
env = Environment()
env.loader = FileSystemLoader('templates')
template = env.get_template('virt-setup-env.j2')
virt_setup_env = template.render(
{
'dib_dir': '/home/stack/DIB',
'node': {
'count': 3,
'mem': 4096,
'cpu': 2
},
'undercloud_node_mem': 4096,
'guest_image_name': '/home/stack/guest_image.qcow2',
'rhsm': {
'user': rhsm_login,
'password': rhsm_password
}})
self.create_file('virt-setup-env', virt_setup_env, user='******')
self.run('source virt-setup-env; instack-virt-setup', user='******')
undercloud_ip = self.run(
'/sbin/ip n | grep $(tripleo get-vm-mac instack) | awk \'{print $1;}\'',
user='******')[0]
undercloud = Undercloud(undercloud_ip,
via_ip=self._hostname,
user='******',
key_filename=self._key_filename)
return undercloud
def purify(indent=False):
global env
env = Environment(extensions=[WithGenericView, FieldShortcut, FieldsListShortcut, HamlishExtension])
called_from = sys._getframe(1).f_globals["__file__"]
module_directory = os.path.split(called_from)[0]
if not os.path.exists(os.path.join(module_directory, "__openerp__.py")):
return
env.loader = FileSystemLoader(module_directory)
module_settings = eval(open(os.path.join(module_directory, "__openerp__.py")).read())
xml_files = list(set(filter(lambda x: x.strip().endswith(".xml"), module_settings.get("init_xml", []) + module_settings.get("data", []) + module_settings.get("update_xml", []))))
for xml_file in xml_files:
haml_file = xml_file[:-3] + "haml"
haml_file_path = os.path.join(module_directory, xml_file[:-3] + "haml")
if not os.path.exists(haml_file_path):
continue
to_write = env.get_template(haml_file).render()
if not re.match("^\s*<\s*openerp\s*>\s*<\s*data ", to_write):
to_write = env.hamlish_from_string("%openerp\n %data\n =body").render(body=to_write)
to_write = format_xml(xml_file, to_write)
open(os.path.join(module_directory, xml_file), "w").write(to_write)
def run(self):
report_dir = os.path.join(self.controller.logs_dir, "run-%s" %
datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S"))
latest_report_dir = os.path.join(self.controller.logs_dir, "run-latest")
report_filename = os.path.join(report_dir, 'index.html')
shutil.rmtree(report_dir, ignore_errors=True)
template_dir = os.path.join(os.path.dirname(__file__), 'run_reporter')
shutil.copytree(template_dir, report_dir)
formatter = HtmlFormatter()
js_lexer = JavascriptLexer()
for doc in self.controller.docs:
if len(doc.args) > 0:
doc.args_html = highlight(json.dumps(doc.args, sort_keys=True, indent=4), js_lexer, formatter)
for a in doc.artifacts:
if hasattr(a, 'stdout'):
html = ansi_output_to_html(a.stdout, self.controller.log)
a.stdout_html = """stdout:<br />%s""" % html
env_data = {}
j = json.dumps(self.controller.config, sort_keys = True, indent=4)
html = highlight(j, js_lexer, formatter)
env_data['dexy_config'] = html
env_data['docs'] = self.controller.docs
env_data['controller'] = self.controller
env = Environment()
env.loader = FileSystemLoader(os.path.dirname(__file__))
template = env.get_template('run_reporter_template.html')
template.stream(env_data).dump(report_filename)
shutil.rmtree(latest_report_dir, ignore_errors=True)
shutil.copytree(report_dir, latest_report_dir)
