def run_pdflatex(context, outputfilename, overwrite=True):
if context.has_key('footer_text') and context.has_key('footer_image'):
context['footer_text'] = publish_parts(context['footer_text'], writer_name='latex')['body']
context['extra_head'] = ("\\setbeamertemplate{footline}{\\hskip1cm " +
"" + context['footer_text'] + "\\hfill\n" +
"\\includegraphics[width=.2\\textwidth]{support/" + context['footer_image'] + "}\n" +
"\\hskip.5cm~\n" +
"\\vskip.5cm\n" +
"} ")
else:
context['extra_head'] = ''
if not context.has_key('textemplate'):
context['textemplate'] = "text-image-quer.tex"
genshitex = TemplateLoader([config.textemplatedir])
template = genshitex.load(
context['textemplate'], cls=NewTextTemplate, encoding='utf8')
if not overwrite and os.path.isfile(outputfilename) and os.path.getmtime(template.filepath) < os.path.getmtime(outputfilename):
return
if context['markup'] == 'rst':
context['text'] = publish_parts(context['text'], writer_name='latex')['body']
#context['headline'] = publish_parts(context['headline'], writer_name='latex')['body']
tmpdir = tempfile.mkdtemp(dir=config.tmpdir)
if context.has_key('img') and context['img'] and context['img'] != '__none':
try:
shutil.copy(os.path.join(config.imagedir, context['img']),
os.path.join(tmpdir, context['img']))
except:
raise IOError("COULD NOT COPY")
else:
# print "MEH No image"
pass
tmptexfile = os.path.join(tmpdir, 'output.tex')
tmppdffile = os.path.join(tmpdir, 'output.pdf')
with open(tmptexfile, 'w') as texfile:
texfile.write(template.generate(form=context).render(encoding='utf8'))
cwd = os.getcwd()
os.chdir(tmpdir)
os.symlink(config.texsupportdir, os.path.join(tmpdir, 'support'))
try:
texlog = check_output(
['pdflatex', '--halt-on-error', tmptexfile], stderr=STDOUT)
except CalledProcessError as e:
if overwrite:
try:
flash(Markup("<p>PDFLaTeX Output:</p><pre>%s</pre>" % e.output), 'log')
except:
print(e.output)
raise SyntaxWarning("PDFLaTeX bailed out")
finally:
os.chdir(cwd)
if overwrite:
try:
flash(Markup("<p>PDFLaTeX Output:</p><pre>%s</pre>" % texlog), 'log')
except:
print(texlog)
shutil.copy(tmppdffile, outputfilename)
shutil.rmtree(tmpdir)
def save( self, *args, **kwargs ): if self.id: self.default_price = self.get_default_price() self.short_configuration_str = self.short_configuration() self.description_html = publish_parts( html_escape( self.description ), writer_name="html4css1" )["fragment"] self.description_html_en = publish_parts( html_escape( self.description_en ), writer_name="html4css1" )["fragment"] self.last_modified = datetime.datetime.now() super( ComputerModel, self ).save(*args, **kwargs)
def save(self, *args, **kwargs):
if self.content_format == 'html':
self.summary_html = self.summary
self.body_html = self.body
elif self.content_format == 'reST':
self.summary_html = publish_parts(source=smart_str(self.summary),
writer_name="html",
settings_overrides=BLOG_DOCUTILS_SETTINGS)['fragment']
self.body_html = publish_parts(source=smart_str(self.body),
writer_name="html",
settings_overrides=BLOG_DOCUTILS_SETTINGS)['fragment']
super(Entry, self).save(*args, **kwargs)
def save(self, *args, **kwargs):
if self.content_format == "html":
self.summary_html = self.summary
self.body_html = self.body
elif self.content_format == "reST":
self.summary_html = publish_parts(
source=smart_str(self.summary), writer_name="html", settings_overrides=BLOG_DOCUTILS_SETTINGS
)["fragment"]
self.body_html = publish_parts(
source=smart_str(self.body), writer_name="html", settings_overrides=BLOG_DOCUTILS_SETTINGS
)["fragment"]
super(Entry, self).save(*args, **kwargs)
self.invalidate_cached_entry()
def get_help(request):
'''Get the requested help file if it exists'''
help_dir = get_help_dir_from_config(request)
requested_dir = (urllib.unquote(sep.join(request.matchdict['dir']))
.encode('utf8'))
requested_file = join(help_dir, requested_dir)
if isfile(requested_file + '.rst'):
# a single help file was requested
html = ''
with open(requested_file + '.rst', 'r') as f:
html = publish_parts(f.read(), writer_name='html')['html_body']
return {'path': requested_file, 'html': html}
elif isdir(requested_file) and requested_dir is not '':
# a directory was requested
# aggregate the files contained with in the given directory
# and sub dirs.
for path, dirnames, filenames in walk(requested_file):
html = ''
for fname in filenames:
with open(join(path, fname), 'r') as f:
html += publish_parts(f.read(),
writer_name='html')['html_body']
return {'path': requested_file, 'html': html}
elif isdir(requested_file) and requested_dir is '':
aggregate = []
for path, dirnames, filenames in walk(requested_file):
for fname in filenames:
text = ''
with open(join(path, fname), 'r') as f:
text = f.read()
parts_whole = publish_parts(text)
parts = publish_parts(text, writer_name='html')
html = parts['html_body']
keywords = iter_keywords(parts_whole['whole'])
aggregate.append({'path': join(path,
fname.replace('.rst', '')),
'html': html,
'keywords': keywords})
return aggregate
else:
raise cors_exception(request, HTTPNotFound)
def refresh_html(sender, instance, **kwargs):
instance.about_html = publish_parts(source=instance.about, writer_name="html4css1")['body']
instance.content_html = publish_parts(
source=instance.content,
writer_name="html4css1",
settings_overrides={'initial_header_level': 3},
)['body']
if instance.created:
if not instance.draft and Post.objects.get(pk=instance.pk).draft:
#going public
instance.created = datetime.now()
else:
#a new object
instance.created = datetime.now()
def rst(cls, source, safe=True):
source = safe_unicode(source)
try:
from docutils.core import publish_parts
from docutils.parsers.rst import directives
docutils_settings = dict([(alias, None) for alias in
cls.RESTRUCTUREDTEXT_DISALLOWED_DIRECTIVES])
docutils_settings.update({'input_encoding': 'unicode',
'report_level': 4})
for k, v in docutils_settings.iteritems():
directives.register_directive(k, v)
parts = publish_parts(source=source,
writer_name="html4css1",
settings_overrides=docutils_settings)
return parts['html_title'] + parts["fragment"]
except ImportError:
log.warning('Install docutils to use this function')
return cls.plain(source)
except Exception:
log.error(traceback.format_exc())
if safe:
return source
else:
raise
def _format_rest(self, doc):
try:
from docutils.core import publish_parts
except ImportError:
raise DataError("reST format requires 'docutils' module to be installed.")
parts = publish_parts(doc, writer_name="html")
return self._format_html(parts["html_body"])
def html_parts(input_string, source_path=None, destination_path=None,
input_encoding='unicode', doctitle=1, initial_header_level=1):
"""
Given an input string, returns a dictionary of HTML document parts.
Dictionary keys are the names of parts, and values are Unicode strings;
encoding is up to the client.
Parameters:
- `input_string`: A multi-line text string; required.
- `source_path`: Path to the source file or object. Optional, but useful
for diagnostic output (system messages).
- `destination_path`: Path to the file or object which will receive the
output; optional. Used for determining relative paths (stylesheets,
source links, etc.).
- `input_encoding`: The encoding of `input_string`. If it is an encoded
8-bit string, provide the correct encoding. If it is a Unicode string,
use "unicode", the default.
- `doctitle`: Disable the promotion of a lone top-level section title to
document title (and subsequent section title to document subtitle
promotion); enabled by default.
- `initial_header_level`: The initial level for header elements (e.g. 1
for "<h1>").
"""
overrides = {'input_encoding': input_encoding,
'doctitle_xform': doctitle,
'initial_header_level': initial_header_level}
parts = core.publish_parts(
source=input_string, source_path=source_path,
destination_path=destination_path,
writer_name='html', settings_overrides=overrides)
return parts
def rst2html(source, initial_header_level=2, inline=False, part='body'):
source = '.. default-role:: math\n\n' + source
writer_name = 'html'
settings_overrides = {
'compact_lists' : True,
'footnote_references' : 'superscript',
'math_output' : 'MathJax',
'stylesheet_path' : None,
'initial_header_level' : initial_header_level,
# 'doctitle_xform' : 0,
}
html = publish_parts(
source=source,
writer_name=writer_name,
settings_overrides=settings_overrides,
)[part].strip()
if inline:
if html[:3] == '<p>' and html[-4:] == '</p>':
html = html[3:-4]
html = html.replace('...','…')
html = html.replace('---','—')
html = html.replace('--','–')
# oops ... need to reverse these back
html = html.replace('<!–','<!--')
html = html.replace('–>','-->')
return mark_safe(html)
def render_rest(content):
# start by h2 and ignore invalid directives and so on
# (most likely from Sphinx)
settings = {'initial_header_level': 2, 'report_level': 'quiet'}
return publish_parts(content,
writer=Writer(),
settings_overrides=settings).get('html_body')
def test_math_output_html(self):
# There should be no MathJax script when math_output is not MathJax
mysettings = self.settings_overrides.copy()
mysettings.update({'math_output': 'HTML'})
head = core.publish_parts(self.data, writer_name='html4css1',
settings_overrides=mysettings)['head']
self.assertNotIn('MathJax.js', head)
def parse_rst(rst):
#overrides = {'initial_header_level': 2}
#return Markup(publish_parts(rst,
#writer_name='html',
#settings_overrides=overrides)['body'])
return Markup(publish_parts(rst,
writer_name='html')['body'])
def render_rst(ctx, blob_obj):
parts = publish_parts(blob_obj.data, writer=Writer(),
settings_overrides={'initial_header_level': 2,
'syntax_highlight': 'short',
'ctx': ctx, 'obj': blob_obj})
parts['description'] = parts['title']
return Document(**parts)
def blog_post(kword_or_id):
session = DBSession()
post = session.query(Post).filter(Post.url_kword == kword_or_id).first()
if not post and kword_or_id.isdigit():
post = session.query(Post).get(int(kword_or_id))
if not post:
raise HTTPNotFound
post.content_title = post.title or _title_from_content(post.content)
parts = publish_parts(post.content, writer_name="html")
html_body = parts["html_body"]
prev_post = session.query(Post).filter(Post.id < post.id).order_by(desc(Post.id))
if prev_post.count() > 0:
prev_post = prev_post[0]
prev_post.content_title = prev_post.title or _title_from_content(prev_post.content)
else:
prev_post = None
next_post = session.query(Post).filter(Post.id > post.id).order_by(asc(Post.id))
if next_post.count() > 0:
next_post = next_post[0]
next_post.content_title = next_post.title or _title_from_content(next_post.content)
else:
next_post = None
post.html_content = html_body
return {"post": post, "recent_posts": recent_posts(), "prev_post": prev_post, "next_post": next_post}
def method_help(http_request, method_name):
"""
Display automatic help about an XML-RPC method.
"""
if len(http_request.POST):
raise Http404 # Don't POST here, only GET documentation
if method_name not in dispatcher.list_methods(http_request):
raise Http404 # Method not found
signatures = dispatcher.method_signature(http_request, method_name)
signature_lines = []
for signature in signatures:
result = signature[0]
params = signature[1:]
signature_lines.append('%s(%s) => %s' % (
method_name, ', '.join(params), result))
docstring = dispatcher.method_help(http_request, method_name)
try:
from docutils import core
parts = core.publish_parts(
source=docstring, writer_name='html',
settings_overrides=RST_SETTINGS)
docstring = parts['html_body']
except ImportError:
docstring = u'<pre>\n%s\n</pre>\n' % docstring
for method in dispatcher.funcs:
docstring = docstring.replace(
method, u'<a href="../%s/">%s</a>' % (method, method))
docstring = mark_safe(docstring)
return render_to_response('xmlrpc/method_help.html', locals(),
context_instance=RequestContext(http_request))
def rst2html(source, overrides={}):
'''
Wrapper for docutils ``publish_parts`` HTML writer.
'''
if source:
writer = 'html'
settings_overrides = {
'compact_lists' : True,
'footnote_references' : 'superscript',
'math_output' : 'MathJax',
'stylesheet_path' : None,
'initial_header_level' : 2,
'doctitle_xform' : 0,
}
settings_overrides.update(overrides)
html = publish_parts(
source=source,
writer_name=writer,
settings_overrides=settings_overrides,
)['body']
html = html.replace('---','—')
html = html.replace('--','–')
html = html.replace('...','…')
else:
html = ''
return html.strip()
def view_page(request):
pagename = request.matchdict['pagename']
page = DBSession.query(Page).filter_by(name=pagename).first()
if page is None:
return HTTPNotFound('No such page')
def check(match):
# match.group() the entire matched string.
# match.group(0) the entire matched string.
# match.group(1) the first matched string.
# match.groups() the matched list.
word = match.group(1)
exists = DBSession.query(Page).filter_by(name=word).all()
if exists:
view_url = request.route_url('view_page', pagename=word)
return '<a href="%s">%s</a>' % (view_url, word)
else:
add_url = request.route_url('add_page', pagename=word)
return '<a href="%s">%s</a>' % (add_url, word)
content = publish_parts(page.data, writer_name='html')['html_body']
# jiawzhang:
# check function will be invoked the same times as the content is matched. e.g. if content is 'ZHANG JIAWEI', it will be invoked twice.
# first time, word = match.group(1) above is 'ZHANG', the second time, word = match.group(1) above is 'JIAWEI'.
# Hence, the content will be '<a href="/add_page/ZHANG">ZHANG</a> <a href="/add_page/JIAWEI">JIAWEI</a>' (suppose, "exists" above is False)
content = wikiwords.sub(check, content)
# the edit_url for current page.
edit_url = request.route_url('edit_page', pagename=pagename)
# logged_in is available if you've invoking "headers = remember(request, login)" in login.py
return dict(page=page, content=content, edit_url=edit_url, logged_in=authenticated_userid(request))
def _save_post(request, rst):
"Code for saving a blog post, used by save_blog and add_blog"
if rst:
post = Post(title=request.POST['title'],
slug=request.POST['slug'],
keywords=request.POST['keywords'],
rst_source=request.POST['body'],
body=publish_parts(request.POST['body'],
writer_name='html')['html_body'])
else:
post = Post(title=request.POST['title'],
slug=request.POST['slug'],
keywords=request.POST['keywords'],
body=request.POST['body'])
if 'y' != request.POST.get('comments_allowed', 'n'):
post.comments_allowed = False
if '' != request.POST.get('category_id', ''):
post.category_id = int(request.POST['category_id'])
if 'publish' == request.POST['blog_action']:
post.published = True
# TODO, add user ID
db.add(post)
def format(self, formatter):
# Create our simple parser
parser = MoinDirectives(self.request)
parts = publish_parts(
source=self.raw,
writer=MoinWriter(formatter, self.request),
settings_overrides={
'halt_level': 5,
'traceback': True,
'file_insertion_enabled': 0,
'raw_enabled': 0,
'stylesheet_path': '',
'template': '',
}
)
html = []
if parts['title']:
# Document title.
html.append(formatter.rawHTML('<h1>%s</h1>' % parts['title']))
# If there is only one subtitle it is propagated by Docutils
# to a document subtitle and is held in parts['subtitle'].
# However, if there is more than one subtitle then this is
# empty and fragment contains all of the subtitles.
if parts['subtitle']:
html.append(formatter.rawHTML('<h2>%s</h2>' % parts['subtitle']))
if parts['docinfo']:
html.append(parts['docinfo'])
html.append(parts['fragment'])
self.request.write(html_escape_unicode('\n'.join(html)))
def test_default_stylesheet(self):
# default style sheet, embedded
mysettings = {'_disable_config': True,}
styles = core.publish_parts(self.data, writer_name='html4css1',
settings_overrides=mysettings)['stylesheet']
self.assertIn('Default cascading style sheet '
'for the HTML output of Docutils.', styles)
def test_default_stylesheet_linked(self):
# default style sheet, linked
mysettings = {'_disable_config': True,
'embed_stylesheet': False}
styles = core.publish_parts(self.data, writer_name='html4css1',
settings_overrides=mysettings)['stylesheet']
self.assertIn('docutils/writers/html4css1/html4css1.css', styles)
def restructuredtext(value, short=False):
try:
from docutils.core import publish_parts
except ImportError:
if settings.DEBUG:
raise template.TemplateSyntaxError(
"Error in 'restructuredtext' filter: "
"The Python docutils library isn't installed."
)
return force_text(value)
else:
docutils_settings = {
'raw_enabled': False,
'file_insertion_enabled': False,
}
docutils_settings.update(getattr(settings, 'RESTRUCTUREDTEXT_FILTER_SETTINGS', {}))
parts = publish_parts(source=force_bytes(value), writer_name="html4css1",
settings_overrides=docutils_settings)
out = force_text(parts["fragment"])
try:
if short:
out = out.split("\n")[0]
except IndexError:
pass
finally:
return mark_safe(out)
def test_math_output_html(self):
mysettings = {'_disable_config': True,
'math_output': 'HTML'}
head = core.publish_parts(self.data, writer_name='html4css1',
settings_overrides=mysettings)['head']
# There should be no MathJax script when math_output is not MathJax
self.assertNotIn('MathJax.js', head)
def process_rst(self):
""" read the contents of the post, process as rst """
with open(self.fname) as f:
return publish_parts(
source=f.read(),
writer_name='html',
settings_overrides=self.rst_overrides)
def render_flatpage(content):
parsed = ""
path = getattr(content, "path", "")
if isinstance(content, basestring):
content = content.split("\n")
for line in content:
match = re.match("^(\.\. *(?:_[^:]*:|(?:\|\w+\|)? *image::) *)([^ ]+) *$", line)
if match:
directive, urlname = match.groups()
line = directive
try:
i = urlname.index("telemeta-")
except ValueError:
i = -1
if i == 0:
line += reverse(urlname)
elif urlname[:1] != "/" and urlname[:4] != "http":
line += reverse("telemeta-flatpage", args=[path + "/../" + urlname])
else:
line += urlname
parsed += line + "\n"
parts = publish_parts(source=smart_str(parsed), writer_name="html4css1", settings_overrides={})
return mark_safe('<div class="rst-content">\n' + force_unicode(parts["html_body"]) + "</div>")
def generate_documentation(data):
toc = '\n\n..\n TOC\n\n.. contents::\n\n..\n /TOC'
parts = publish_parts(data + toc,
writer=DocumentationWriter(),
settings_overrides=dict(
initial_header_level=2,
field_name_limit=50
)
)
toc = None
body = parts['body']
match = _toc_re.search(body)
body = body[:match.start()] + body[match.end():]
match = _toc_contents_re.search(match.group(1))
if match is not None:
toc = match.group(1)
# just add the toc if there are at least two entries.
if toc.count('</li>') < 2:
toc = None
return {
'title': parts['title'],
'body': body,
'toc': toc
}
def generate(self):
if not os.path.isdir(self.target_path):
os.makedirs(self.target_path)
contents = {}
for name,sourcefile in self.sources.items():
content = codecs.open(sourcefile, encoding='utf-8').read()
if fnmatch.fnmatch(sourcefile, '*.rst'):
contents[name] = body=publish_parts(
content,
writer_name='html',
settings_overrides=dict(
input_encoding='utf-8',
output_encoding='utf-8',
doctitle_xform=False,
syntax_highlight="short",
)
)['html_body']
else:
contents[name] = content
logging.info("Creating: %s -(%s)-> %s" % (", ".join(self.sources.values()), self.template, self.target))
taltemplate = PageTemplateFile(self.template)
result = taltemplate(
encoding='utf-8',
cms=self.cms,
root=self.cms.navigation.root,
content=contents,
section=self.section,
)
codecs.open(self.target, "w", encoding='utf-8').write(result)
def read(self):
f = open(self.filepath)
logger.info('read ' + self.filepath)
content = f.read()
f.close()
extra_setting = {'initial_header_level': '2'}
parts = publish_parts(
content, writer_name='html',
settings_overrides=extra_setting,
)
# get docinfo
docinfo = []
content = parts['docinfo'].replace('\n', '')
if not content:
return parts
dom = minidom.parseString(content.encode('utf-8'))
nodes = dom.getElementsByTagName('tr')
for node in nodes:
docinfo.append(self._node_to_pairs(node))
parts['docinfo'] = docinfo
return parts
def release_detail(request, package_name, version):
pkg = get_object_or_404(Pkg, name=package_name)
release = get_object_or_404(Release, pkg=pkg, version=version)
release_data = get_object_or_404(ReleaseData, release=release)
release_urls = release.releaseurl_set.all()
license_url = _LICENSE_URLS.get(release_data.license, '')
settings_overrides = {
'raw_enabled': 0, # no raw HTML code
'file_insertion_enabled': 0, # no file/URL access
'halt_level': 2, # at warnings or errors, raise an exception
'report_level': 5, # never report problems with the reST code
}
try:
parts = publish_parts(source=release_data.description,
writer_name='html',
settings_overrides=settings_overrides)
description = parts['body']
except:
description = '<pre>%s</pre>' % release_data.description
description = mark_safe(description)
return { 'pkg': pkg, 'release': release,
'release_data': release_data,
'license_url': license_url,
'release_urls': release_urls,
'description': description }
def doc_parts(input_string):
parts = core.publish_parts(source=input_string, writer_name='html')
return parts
def text2html(text):
return '<div class="mainsphinx">' + publish_parts(
text, writer_name='html')['fragment'] + '</div>'
def test_math_output_default(self):
# HTML with math.css stylesheet (since 0.11)
mysettings = {'_disable_config': True,}
styles = core.publish_parts(self.data, writer_name='html5_polyglot',
settings_overrides=mysettings)['stylesheet']
self.assertIn('convert LaTeX equations to HTML output.', styles)
def visit_literal_block(self, node):
"No classes please"
self.body.append(self.starttag(node, 'pre', ''))
def visit_literal(self, node):
self.body.append(self.starttag(node, 'code', ''))
def depart_literal(self, node):
self.body.append('</code>')
def inline_roles(role, raw, text, *args):
if role == 'kbd':
return [nodes.literal('kbd', text)], []
elif role == 'var':
return [nodes.literal('var', text)], []
elif role == 'abbr':
return [nodes.literal('abbr', text)], []
roles.register_local_role('kbd', inline_roles)
roles.register_local_role('var', inline_roles)
roles.register_local_role('abbr', inline_roles)
if __name__ == '__main__':
import sys
parts = publish_parts(writer=MyHTMLWriter(),
source=open(sys.argv[1]).read())
print parts['html_body']
def auto_report(dag, path, stylesheet=None):
try:
from jinja2 import Template, Environment, PackageLoader, UndefinedError
except ImportError as e:
raise WorkflowError(
"Python package jinja2 must be installed to create reports.")
mode_embedded = True
if path.endswith(".zip"):
mode_embedded = False
elif not path.endswith(".html"):
raise WorkflowError("Report file does not end with .html or .zip")
custom_stylesheet = None
if stylesheet is not None:
try:
with open(stylesheet) as s:
custom_stylesheet = s.read()
except (Exception, BaseException) as e:
raise WorkflowError("Unable to read custom report stylesheet.", e)
logger.info("Creating report...")
env = Environment(
loader=PackageLoader("snakemake", "report/template"),
trim_blocks=True,
lstrip_blocks=True,
)
env.filters["get_resource_as_string"] = get_resource_as_string
persistence = dag.workflow.persistence
results = defaultdict(lambda: defaultdict(list))
records = defaultdict(JobRecord)
recorded_files = set()
for job in dag.jobs:
for f in itertools.chain(job.expanded_output, job.input):
if is_flagged(f, "report") and f not in recorded_files:
if not f.exists:
raise WorkflowError("File {} marked for report but does "
"not exist.".format(f))
report_obj = get_flag_value(f, "report")
def register_file(f,
wildcards_overwrite=None,
aux_files=None,
name_overwrite=None):
wildcards = wildcards_overwrite or job.wildcards
category = Category(report_obj.category,
wildcards=wildcards,
job=job)
subcategory = Category(report_obj.subcategory,
wildcards=wildcards,
job=job)
labels = expand_labels(report_obj.labels, wildcards, job)
results[category][subcategory].append(
FileRecord(
f,
job,
report_obj.caption,
env,
category,
dag.workflow,
wildcards_overwrite=wildcards_overwrite,
mode_embedded=mode_embedded,
aux_files=aux_files,
name_overwrite=name_overwrite,
labels=labels,
))
recorded_files.add(f)
if os.path.isfile(f):
register_file(f)
elif os.path.isdir(f):
if report_obj.htmlindex:
if mode_embedded:
raise WorkflowError(
"Directory marked for report specifies htmlindex. "
"This is unsupported when requesting a pure HTML report. "
"Please use store as zip instead (--report report.zip)."
)
aux_files = []
index_found = False
for root, dirs, files in os.walk(f):
for name in files:
if name != ".snakemake_timestamp":
filepath = os.path.join(root, name)
if (os.path.relpath(filepath, f) !=
report_obj.htmlindex):
aux_files.append(filepath)
else:
index_found = True
if not index_found:
raise WorkflowError(
"Given htmlindex {} not found in directory "
"marked for report".format(
report_obj.htmlindex))
register_file(
os.path.join(f, report_obj.htmlindex),
aux_files=aux_files,
name_overwrite="{}.html".format(
os.path.basename(f)),
)
elif report_obj.patterns:
if not isinstance(report_obj.patterns, list):
raise WorkflowError(
"Invalid patterns given for report. Must be list.",
rule=job.rule,
)
for pattern in report_obj.patterns:
pattern = os.path.join(f, pattern)
wildcards = glob_wildcards(pattern)._asdict()
names = wildcards.keys()
for w in zip(*wildcards.values()):
w = dict(zip(names, w))
w.update(job.wildcards_dict)
w = Wildcards(fromdict=w)
f = apply_wildcards(pattern, w)
register_file(f, wildcards_overwrite=w)
else:
raise WorkflowError(
"Directory marked for report but neither file patterns "
"given via patterns=[...], nor htmlindex given. "
"See report documentation.",
rule=job.rule,
)
for f in job.expanded_output:
meta = persistence.metadata(f)
if not meta:
logger.warning("Missing metadata for file {}. Maybe metadata "
"was deleted or it was created using an older "
"version of Snakemake. This is a non critical "
"warning.".format(f))
continue
def get_time(rectime, metatime, sel_func):
if metatime is None:
return rectime
return sel_func(metatime, rectime)
try:
job_hash = meta["job_hash"]
rule = meta["rule"]
rec = records[(job_hash, rule)]
rec.rule = rule
rec.job = job
rec.starttime = get_time(rec.starttime, meta["starttime"], min)
rec.endtime = get_time(rec.endtime, meta["endtime"], max)
rec.conda_env_file = None
rec.conda_env = meta["conda_env"]
rec.container_img_url = meta["container_img_url"]
rec.output.append(f)
except KeyError as e:
logger.warning("Metadata for file {} was created with a too "
"old Snakemake version.".format(f))
for subcats in results.values():
for catresults in subcats.values():
catresults.sort(key=lambda res: res.name)
# prepare runtimes
runtimes = [{
"rule": rec.rule,
"runtime": rec.endtime - rec.starttime
} for rec in sorted(records.values(), key=lambda rec: rec.rule)]
def get_datetime(rectime):
try:
return datetime.datetime.fromtimestamp(rectime).isoformat()
except OSError:
return None
# prepare end times
timeline = [{
"rule": rec.rule,
"starttime": get_datetime(rec.starttime),
"endtime": get_datetime(rec.endtime),
} for rec in sorted(records.values(), key=lambda rec: rec.rule)]
# prepare per-rule information
rules = defaultdict(list)
for rec in records.values():
rule = RuleRecord(rec.job, rec)
if rec.rule not in rules:
rules[rec.rule].append(rule)
else:
merged = False
for other in rules[rec.rule]:
if rule == other:
other.add(rec)
merged = True
break
if not merged:
rules[rec.rule].append(rule)
# In theory there could be more than one rule with the same name kept from above.
# For now, we just keep the first.
rules = {rulename: items[0] for rulename, items in rules.items()}
# rulegraph
rulegraph, xmax, ymax = rulegraph_spec(dag)
# configfiles
configfiles = [ConfigfileRecord(f) for f in dag.workflow.configfiles]
seen = set()
files = [
seen.add(res.target) or res for cat in results.values()
for subcat in cat.values() for res in subcat if res.target not in seen
]
rst_links = textwrap.dedent("""
.. _Workflow: javascript:show_panel('workflow')
.. _Statistics: javascript:show_panel('statistics')
{% for cat, catresults in categories|dictsort %}
.. _{{ cat.name }}: javascript:app.showCategory('{{ cat.name|urlencode }}')
{% endfor %}
{% for res in files %}
.. _{{ res.target }}: javascript:app.showResultInfo('{{ res.path|urlencode }}')
{% endfor %}
""")
for cat, subcats in results.items():
for subcat, catresults in subcats.items():
for res in catresults:
res.render(env, rst_links, results, files)
# global description
text = ""
if dag.workflow.report_text:
with dag.workflow.sourcecache.open(dag.workflow.report_text) as f:
class Snakemake:
config = dag.workflow.config
text = f.read() + rst_links
try:
text = publish_parts(
env.from_string(text).render(snakemake=Snakemake,
categories=results,
files=files),
writer_name="html",
)["body"]
except UndefinedError as e:
raise WorkflowError(
"Error rendering global report caption {}:".format(
dag.workflow.report_text.get_path_or_uri()),
e,
)
# record time
now = "{} {}".format(datetime.datetime.now().ctime(), time.tzname[0])
results_size = sum(res.size for cat in results.values()
for subcat in cat.values() for res in subcat)
try:
from pygments.formatters import HtmlFormatter
except ImportError:
raise WorkflowError(
"Python package pygments must be installed to create reports.")
categories = data.render_categories(results)
rendered_results = data.render_results(results)
rulegraph = data.render_rulegraph(rulegraph["nodes"], rulegraph["links"],
rulegraph["links_direct"])
rules = data.render_rules(rules)
runtimes = data.render_runtimes(runtimes)
timeline = data.render_timeline(timeline)
packages = data.get_packages()
template = env.get_template("index.html.jinja2")
logger.info("Downloading resources and rendering HTML.")
rendered = template.render(
results=rendered_results,
categories=categories,
rulegraph=rulegraph,
rules=rules,
workflow_desc=json.dumps(text),
runtimes=runtimes,
timeline=timeline,
packages=packages,
pygments_css=HtmlFormatter(
style="stata-dark").get_style_defs(".source"),
custom_stylesheet=custom_stylesheet,
logo=data_uri_from_file(
Path(__file__).parent / "template" / "logo.svg"),
now=now,
)
# TODO look into supporting .WARC format, also see (https://webrecorder.io)
if not mode_embedded:
with ZipFile(path, compression=ZIP_DEFLATED, mode="w") as zipout:
folder = Path(Path(path).stem)
# store results in data folder
for subcats in results.values():
for catresults in subcats.values():
for result in catresults:
# write raw data
zipout.write(result.path,
str(folder.joinpath(result.data_uri)))
# write aux files
parent = folder.joinpath(result.data_uri).parent
for aux_path in result.aux_files:
# print(aux_path, parent, str(parent.joinpath(os.path.relpath(aux_path, os.path.dirname(result.path)))))
zipout.write(
aux_path,
str(
parent.joinpath(
os.path.relpath(
aux_path,
os.path.dirname(result.path)))),
)
# write report html
zipout.writestr(str(folder.joinpath("report.html")), rendered)
else:
with open(path, "w", encoding="utf-8") as htmlout:
htmlout.write(rendered)
logger.info("Report created: {}.".format(path))
def test_math_output_mathjax_no_math(self):
mysettings = {'_disable_config': True, 'math_output': 'MathJax'}
# There should be no math script when text does not contain math
head = core.publish_parts('No math.',
writer_name='html5_polyglot')['head']
self.assertNotIn('MathJax', head)
def restructuredtext(value):
return mark_safe(
publish_parts(value, writer_name='html').get('html_body') or u'')
def _reSt_to_html(source):
# from https://wiki.python.org/moin/reStructuredText
from docutils import core
parts = core.publish_parts(source=source, writer_name='html')
return parts['body_pre_docinfo'] + parts['fragment']
def is_sphinx_markup(s):
return False
if is_sphinx_markup(s):
try:
return sphinxify(s)
except:
pass
# Not in ReST format, so use docutils
# to process the preamble ("**File:**" etc.) and put
# everything else in a <pre> block.
i = s.find("**Docstring:**")
if i != -1:
preamble = s[:i + 14]
from docutils.core import publish_parts
preamble = publish_parts(s[:i + 14], writer_name='html')['body']
s = s[i + 14:]
else:
preamble = ""
return '<div class="docstring">' + preamble + '<pre>' + s + '</pre></div>'
def source_code(s, globs, system='sage'):
r"""
Format an object's source code to process and display in the
Sage notebook.
INPUT:
- ``s`` - a string; a name of an object
def _getentrybody(format, entry):
if format == 'rst':
body = publish_parts(entry, writer_name='html')['fragment']
else:
body = entry
return body
def generate_full_config(config_file_path,
slides,
gen_expanded=False,
verbose=False):
''' Generates a full configuration from a simplified configuration
'''
global current_module
global current_module_base
global expanded
expanded = gen_expanded
register()
conf_data = read_conf_file(config_file_path)
validate_glob_config(conf_data)
full_config = OrderedDict()
full_config = conf_data.copy()
full_config['chapters'] = OrderedDict()
if 'glob_ka_options' in full_config:
del full_config['glob_ka_options']
if 'glob_ss_options' in full_config:
del full_config['glob_ss_options']
if 'glob_ff_options' in full_config:
del full_config['glob_ff_options']
if 'glob_pe_options' in full_config:
del full_config['glob_pe_options']
if 'glob_ae_options' in full_config:
del full_config['glob_ae_options']
if 'glob_extr_options' in full_config:
del full_config['glob_extr_options']
mod_files = get_chapter_module_files(conf_data)
for chapter, files in mod_files.items():
full_config['chapters'][chapter] = OrderedDict()
for x in files:
rst_dir_name = x.split(os.sep)[-2]
rst_fname = os.path.basename(x).partition('.')[0]
if rst_dir_name == conf_data['lang']:
mod_path = rst_fname
else:
mod_path = rst_dir_name + '/' + rst_fname
current_module = mod_path
if verbose:
print(("Processing module " + mod_path))
current_module_base = os.path.basename(mod_path)
if not os.path.isfile(x):
print_err(
"ERROR: '{0}' is not a valid module path".format(mod_path))
sys.exit(1)
with open(x, 'rt', encoding='utf-8') as rstfile:
source = rstfile.read()
source = remove_markup(source)
rst_parts = publish_parts(source,
settings_overrides={
'output_encoding': 'utf8',
'initial_header_level': 2
},
writer_name="xml",
source_path=mod_path)
mod_json = xmltodict.parse(rst_parts['whole'])
mod_config = extract_mod_config(mod_json)
full_config['chapters'][chapter][mod_path] = mod_config
if not slides:
for mod_name, exercises in ex_options.items():
for exer in exercises:
print_err(
'WARNING: the exercise "{0}" does not exist in module "{1}"'
.format(exer, mod_name))
for mod_name, sections in sect_options.items():
for sect in sections:
print_err(
'WARNING: the section "{0}" does not exist in module "{1}"'
.format(sect, mod_name))
return full_config
def _render(self, file_pointer, url=None, **kwargs):
return publish_parts(file_pointer.read(),
writer_name='html')['html_body']
rst_fname = os.path.basename(x).partition('.')[0]
if rst_fname in execluded_files:
continue
rst_dir_name = x.split(os.sep)[-2]
if rst_dir_name not in folder_names.keys():
continue
if not current_dir == rst_dir_name:
current_dir = rst_dir_name
print('Processing directory ' + rst_dir_name)
rst_parts = publish_parts(source,
settings_overrides={
'output_encoding': 'utf8',
'initial_header_level': 2
},
writer_name="xml")
mod_json = xmltodict.parse(rst_parts['whole'])
mod_config = extract_mod_config(mod_json)
if folder_names[rst_dir_name] not in everything_config[
'chapters'].keys():
everything_config['chapters'][
folder_names[rst_dir_name]] = OrderedDict()
everything_config['chapters'][folder_names[rst_dir_name]][
rst_dir_name + '/' + rst_fname] = mod_config
if options.dev_mode:
save_debug_files(rst_parts['whole'], mod_json, rst_fname)
source = """\
This is some text.
.. video:: divid
:controls:
:thumb: _static/turtlestill.png
:loop:
http://knuth.luther.edu/~bmiller/foo.mov
http://knuth.luther.edu/~bmiller/foo.webm
This is some more text.
"""
if __name__ == "__main__":
from docutils.core import publish_parts
directives.register_directive("video", Video)
doc_parts = publish_parts(
source,
settings_overrides={
"output_encoding": "utf8",
"initial_header_level": 2
},
writer_name="html",
)
print(doc_parts["html_body"])
def render_restructuredtext(text):
from docutils.core import publish_parts
parts = publish_parts(text, writer_name='html')
# Want to return the html_body without the <div class="document" id="rst"> part
return parts['body_pre_docinfo'] + parts['body'] + parts['footer']
def render(self, context, mimetype, content, filename=None, rev=None):
# Minimize visual impact of errors
class TracHTMLTranslator(html4css1.HTMLTranslator):
"""Specialized translator with unobtrusive error reporting
and some extra security features"""
def __init__(self, *args, **kwargs):
self._render_unsafe_content = wikisys.render_unsafe_content
self._safe_schemes = set(wikisys.safe_schemes)
html4css1.HTMLTranslator.__init__(self, *args, **kwargs)
def visit_system_message(self, node):
paragraph = node.children.pop(0)
message = escape(paragraph.astext()) if paragraph else ''
backrefs = node['backrefs']
if backrefs:
span = ('<span class="system-message">%s</span>' %
(''.join('<a href="#%s" title="%s">?</a>' %
(backref, message)
for backref in backrefs)))
else:
span = (
'<span class="system-message" title="%s">?</span>' %
message)
self.body.append(span)
def depart_system_message(self, node):
pass
def visit_image(self, node):
html4css1.HTMLTranslator.visit_image(self, node)
uri = node.attributes.get('uri')
if not wikisys.is_safe_origin(uri, context.req):
self.body[-1] = self.body[-1].replace(
'<img ', '<img crossorigin="anonymous" ')
def visit_reference(self, node):
if self._is_safe_uri(node.get('refuri')):
html4css1.HTMLTranslator.visit_reference(self, node)
def depart_reference(self, node):
if self._is_safe_uri(node.get('refuri')):
html4css1.HTMLTranslator.depart_reference(self, node)
def _is_safe_uri(self, uri):
if self._render_unsafe_content or not uri:
return True
else:
pos = uri.find(':')
return pos < 0 or uri[0:pos] in self._safe_schemes
wikisys = WikiSystem(self.env)
writer = html4css1.Writer()
writer.translator_class = TracHTMLTranslator
inliner = rst.states.Inliner()
inliner.trac = (self.env, context)
parser = rst.Parser(inliner=inliner)
content = content_to_unicode(self.env, content, mimetype)
# The default Reader is explicitly passed as a workaround for #11248
parts = publish_parts(content,
writer=writer,
parser=parser,
reader=standalone.Reader(parser),
settings_overrides={
'halt_level': 6,
'file_insertion_enabled': 0,
'raw_enabled': 0,
'warning_stream': False
})
return parts['html_body']
def rst2html(rst):
return publish_parts(rst, writer_name="html")["fragment"]
def write(rst_fragment):
# How do I convert a docutils document tree into an HTML string?
# https://stackoverflow.com/a/32168938/598057
html = publish_parts(rst_fragment, writer_name="html")["html_body"]
return html
def rst_to_html(self, text):
"""Convert the .rst file to html code"""
parts = core.publish_parts(source=text, writer_name='html')
return parts['body_pre_docinfo'] + parts['fragment']
def parse(self, path, source, dest):
content = Content.load(self.read(path))
html = publish_parts(content.body, writer_name="html5")
self.write(path, dest, html["html_body"])
sys.stdout.write("\x1b[1;32m{} converted to HTML. Metadata: {}\n".format(path.name, content))
def rst_to_html(value):
parts = publish_parts(source=value,
writer=HTMLWriter(),
settings_overrides={"initial_header_level": 2})
return parts["fragment"]
def apply_markup_filter(text):
"""Applies a text-to-HTML conversion function to a piece of text and
returns the generated HTML.
The function to use is derived from the value of the setting
``POOTLE_MARKUP_FILTER``, which should be a 2-tuple:
* The first element should be the name of a markup filter --
e.g., "markdown" -- to apply. If no markup filter is desired,
set this to None.
* The second element should be a dictionary of keyword
arguments which will be passed to the markup function. If no
extra arguments are desired, set this to an empty
dictionary; some arguments may still be inferred as needed,
however.
So, for example, to use Markdown with bleach cleaning turned on (cleaning
removes non-whitelisted HTML), put this in your settings file::
POOTLE_MARKUP_FILTER = ('markdown', {})
Currently supports Textile, Markdown and reStructuredText, using
names identical to the template filters found in
``django.contrib.markup``.
Borrowed from http://djangosnippets.org/snippets/104/
"""
markup_filter_name, markup_kwargs = get_markup_filter()
if not text.strip():
return text
html = text
if markup_filter_name is not None:
if markup_filter_name == 'textile':
import textile
if 'encoding' not in markup_kwargs:
markup_kwargs.update(encoding=settings.DEFAULT_CHARSET)
if 'output' not in markup_kwargs:
markup_kwargs.update(output=settings.DEFAULT_CHARSET)
html = textile.textile(text, **markup_kwargs)
elif markup_filter_name == 'markdown':
import bleach
import markdown
# See ALLOWED_TAGS in
# https://github.com/mozilla/bleach/blob/master/bleach/__init__.py
tags = bleach.ALLOWED_TAGS + [
u'h1',
u'h2',
u'h3',
u'h4',
u'h5',
u'p',
u'pre',
u'img',
u'hr',
]
tags_provided = ('clean' in markup_kwargs
and 'extra_tags' in markup_kwargs['clean'])
if tags_provided:
tags += markup_kwargs['clean']['extra_tags']
html = bleach.clean(markdown.markdown(text, **markup_kwargs),
tags=tags)
elif markup_filter_name == 'restructuredtext':
from docutils import core
if 'settings_overrides' not in markup_kwargs:
markup_kwargs.update(settings_overrides=getattr(
settings,
"RESTRUCTUREDTEXT_FILTER_SETTINGS",
{},
))
if 'writer_name' not in markup_kwargs:
markup_kwargs.update(writer_name='html4css1')
parts = core.publish_parts(source=text, **markup_kwargs)
html = parts['html_body']
return rewrite_links(html, rewrite_internal_link)
def parse_rst(source):
return publish_parts(source,
writer_name='html',
settings_overrides={'initial_header_level': 2})
root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
print
"Main..."
tpl = open(os.path.join(root, 'index.html.tpl'), 'r').read()
pat = re.compile(r'^.*(Basic usage)', re.DOTALL)
txt = nose.__doc__.replace(':: python', '::')
txt = pat.sub(r'\1', txt)
# cut from 'about the name' down (goes to end of page)
pat = re.compile(r'^(.*?)(About the name.*$)', re.DOTALL)
txt, coda = pat.search(txt).groups()
docs = publish_parts(txt, reader=DocReader(), writer_name='html')
docs.update({'version': nose.__version__, 'date': time.ctime()})
docs['coda'] = publish_parts(coda, writer_name='html')['body']
# print "Tools..."
# tools = publish_parts(nose.tools.__doc__, writer_name='html')
# docs['tools'] = tools['body']
print
"Commands..."
cmds = publish_parts(nose.commands.__doc__,
reader=DocReader(),
writer_name='html')
docs['commands'] = cmds['body']
print
def _inside():
content = pkg_resources.resource_string(
'pylonshq', 'templates/home/inside.rst')
body = publish_parts(content, writer_name='html')['html_body']
return body
def inspect_wheel(wheel: str):
"""Inspeggtor reporting for duty!
This thing opens up Python Eggs to analyze their content in an isomeric way.
Some data pulled out is specific to isomer modules but could be used for general
structures. Specifically, a `docs/README.rst` and a `docs/preview.png` is looked
for.
:param wheel: Filename of the python egg to inspect
:return: Dictionary with
`info` - content of docs/README.rst if found
`preview` - content of docs/preview.png if found
`date` - publishing date according to EGG-INFO/PKG-INFO timestamp
`requires` - any required external python packages
:rtype: dict
"""
archive = zipfile.ZipFile(wheel)
meta_name = os.path.basename(wheel).split("-py")[0] + ".dist-info/METADATA"
try:
info = archive.read("docs/README.rst").decode('utf-8')
info = core.publish_parts(info, writer_name="html")["html_body"]
except KeyError:
info = "No information provided"
try:
preview = base64.b64encode(
archive.read("docs/preview.png")).decode('utf-8')
except KeyError:
preview = ""
pkg_info_info = archive.getinfo(meta_name)
date = std_datetime(pkg_info_info.date_time)
requires = []
homepage = ""
author = ""
contact = ""
package_license = ""
try:
lines = str(archive.read(meta_name), encoding="ascii").split("\n")
for line in lines:
if line.startswith("Home-page:"):
homepage = line.split("Home-page: ")[1]
if line.startswith("Author:"):
author = line.split("Author: ")[1]
if line.startswith("Author-email:"):
contact = line.split("Author-email: ")[1]
if line.startswith("License:"):
package_license = line.split("License: ")[1]
if line.startswith("Requires-Dist:"):
req = line.split("Requires-Dist: ")[1]
req = req.replace(" (", "").replace(")", "")
requires.append(req)
except KeyError:
log("No metadata found")
result = {
'info': info,
'preview': preview,
'date': date,
'requires': requires,
'homepage': homepage,
'author': author,
'contact': contact,
'license': package_license,
'downloads': "-",
'stars': "-"
}
log(result, pretty=True)
return result
def render_rest(markup):
parts = publish_parts(source=markup, writer_name="html4css1")
return parts["fragment"]
def get(self):
self.render("index.html",
motd=publish_parts(self.application._ctx.config.motd,
writer_name="html",
settings_overrides={"initial_header_level": 2})["fragment"],
clients=sorted(self.application._ctx.bot.clients.items()))
def get_formatted_full_description(self):
return publish_parts(self.full_description, writer_name='html')['body']
def distribution(dist_name=None):
""" The Distribution entry-point (/distribution/<dist_name>)
for the Cilantropy server.
:param dist_name: the package name
"""
try:
pkg_dist = get_pkg_res().get_distribution(dist_name)
except _pkg_resources.DistributionNotFound:
abort(404)
data = {}
data.update(get_shared_data())
data['dist'] = pkg_dist
data['breadpath'] = [
Crumb('Main', url_for('index')),
Crumb('Package'),
Crumb(pkg_dist.project_name)
]
settings_overrides = {
'raw_enabled': 0, # no raw HTML code
'file_insertion_enabled': 0, # no file/URL access
'halt_level': 2, # at warnings or errors, raise an exception
'report_level': 5, # never report problems with the reST code
}
try:
pkg_metadata = pkg_dist.get_metadata(metadata.METADATA_NAME[0])
except FileNotFoundError:
pkg_metadata = pkg_dist.get_metadata(metadata.METADATA_NAME[1])
except FileNotFoundError:
pass
parsed, key_known = metadata.parse_metadata(pkg_metadata)
distinfo = metadata.metadata_to_dict(parsed, key_known)
parts = None
""" get description from pep-0426 """
if distinfo['metadata-version'] == '2.0':
distinfo['description'] = pkg_dist.get_metadata(
metadata.DESCRIPTION_2_0[0])
try:
parts = publish_parts(source=distinfo['description'],
writer_name='html',
settings_overrides=settings_overrides)
except:
pass
try:
parts = publish_parts(source=distinfo['description'],
writer_name='html',
settings_overrides=settings_overrides)
except:
pass
data['distinfo'] = distinfo
data['entry_map'] = pkg_dist.get_entry_map()
data['location'] = '{}/{}'.format(pkg_dist.location, dist_name)
if parts is not None:
data['description_render'] = parts['body']
return render_template('distribution.html', **data)