def SyntaxStyleSheet(req):
#Displays the css data for Pygments' syntax highlighting
req.content_type = "text/css"
req.write(HtmlFormatter().get_style_defs('.highlight'))
return
def test_styles():
# minimal style test
from pygments.formatters import HtmlFormatter
HtmlFormatter(style="pastie")
#!/usr/bin/env python3
import sys
import pygments
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter
lexer = get_lexer_by_name("python", stripall=True)
formatter = HtmlFormatter(cssclass="source")
def highlight(raw_code):
code = pygments.highlight(raw_code, lexer, formatter)
if len(raw_code) > 0:
if raw_code[-1] == '\n':
code = code.split("</pre></div>")[0] + "\n</pre></div>"
if raw_code[0] == ' ':
indent = 0
for i, char in enumerate(raw_code):
if char != ' ':
break
indent += 1
parts = code.split("<span class")
parts[0] += ' ' * indent
code = "<span class".join(parts)
if code.startswith("""<div class="source">""") and code.endswith("</div>"):
code = code[20:-6]
return code
import os
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter
totlinen = 0
totsize = 0
totstrlist = []
lexer = get_lexer_by_name("python", stripall=True)
formatter = HtmlFormatter(linenos=True)
for root, dirs, files in os.walk('eudplib'):
for f in files:
if f[-3:] == '.py':
finalpath = os.path.join(root, f)
code = open(finalpath, 'r').read()
totsize += len(code)
linen = code.count('\n') + 1
print("%-40s : %4d" % (finalpath, linen))
totlinen += linen
highlighted_code = highlight(code, lexer, formatter)
totstrlist.append('<p><h2> {finalpath} : {linen} lines </h2></p>\n'
'{highlighted_code}<br><br>'.format(**locals()))
print('Total lines: %d' % totlinen)
print('Total size: %d' % totsize)
def highlight(self):
formatter = HtmlFormatter(linenos=True)
return highlight(self.original_code, self.language.get_lexer(),
formatter)
def pygments_css(style):
formatter = HtmlFormatter(style=style)
return formatter.get_style_defs('.highlight')
key = key.lower().rstrip(':')
valueNode = node.lastChild
tag = valueNode.firstChild.nodeName
if 'ul' == tag or 'ol' == tag:
value = []
for node in valueNode.getElementsByTagName('li'):
value.append(self._plain_text(node))
else:
value = self._plain_text(valueNode)
return key, value
INLINESTYLES = settings.get('highlight_inline', False)
DEFAULT = HtmlFormatter(noclasses=INLINESTYLES)
VARIANTS = {
'linenos': HtmlFormatter(noclasses=INLINESTYLES, linenos=True),
}
class Pygments(Directive):
""" Source code syntax hightlighting.
"""
required_arguments = 1
optional_arguments = 0
final_argument_whitespace = True
option_spec = dict([(key, directives.flag) for key in VARIANTS])
has_content = True
def run(self):
def _repr_html_(self):
from pygments import highlight
from pygments.formatters import HtmlFormatter
fmt = HtmlFormatter()
style = '<style>{}</style>'.format(fmt.get_style_defs('.output_html'))
return style + highlight(self.data, self._get_lexer(), fmt)
def generate_style(css):
css2 = HtmlFormatter(style='trac').get_style_defs()
write_file(join(DIRS['build'], STYLESHEET), ''.join([css, "\n\n", css2]))
def highlight(self, code):
Lexer = find_lexer_class_by_name(self.syntax)
formatter = HtmlFormatter()
highlighted = highlight(code, Lexer(), formatter)
return highlighted
def get_file_content(self, file_path):
file_content = open(file_path).read()
lexer = get_lexer_for_filename(file_path)
return highlight(file_content, lexer, HtmlFormatter(linenos='table'))
def pygment_html_render(s, lexer=lexers.TextLexer):
return highlight(
s,
lexer(),
HtmlFormatter(linenos=True),
)
def traceback(self, tb):
"""Highlight text of a Python traceback. """
return pygments.highlight(tb, PythonTracebackLexer(), HtmlFormatter())
def python(self, code):
"""Highlight a piece of Python source code. """
return pygments.highlight(code, PythonLexer(), HtmlFormatter())
https://docutils.sourceforge.io/docs/howto/rst-directives.html
:copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
# Options
# ~~~~~~~
# Set to True if you want inline CSS styles instead of classes
INLINESTYLES = False
from pygments.formatters import HtmlFormatter
# The default formatter
DEFAULT = HtmlFormatter(noclasses=INLINESTYLES)
# Add name -> formatter pairs for every variant you want to use
VARIANTS = {
# 'linenos': HtmlFormatter(noclasses=INLINESTYLES, linenos=True),
}
from docutils import nodes
from docutils.parsers.rst import directives, Directive
from pygments import highlight
from pygments.lexers import get_lexer_by_name, TextLexer
class Pygments(Directive):
""" Source code syntax highlighting.
def highlighter(m):
lexer = _get_lexer(codesyntax)
code = m.group().replace("```", "")
return pygments.highlight(code, lexer, HtmlFormatter())
def pygments_highlight(text, lang, style):
lexer = get_lexer_by_name(lang, stripall=False)
formatter = HtmlFormatter(nowrap=True, style=style)
return pygments.highlight(text, lexer, formatter)
def run(out, dry_run=False, check_built=False):
snippets = []
py_lexer = lexers.PythonLexer()
go_lexer = lexers.GoLexer()
html_formatter = HtmlFormatter()
for snippet_dir in _dirs():
go_codes = []
for fn in glob(os.path.join(snippet_dir, "*.go")):
with open(fn) as f:
go_codes.append((fn, f.read()))
if not go_codes:
print(snippet_dir, "has no .go code")
continue
py_codes = []
for fn in glob(os.path.join(snippet_dir, "*.py")):
with open(fn) as f:
py_codes.append((fn, f.read()))
if not py_codes:
print(snippet_dir, "has no .py code")
continue
id = os.path.basename(snippet_dir)
if os.path.isfile(os.path.join(snippet_dir, "title")):
with open(os.path.join(snippet_dir, "title")) as f:
title = f.read().strip()
else:
title = id.replace("_", " ").title()
readme = None
for fn in glob(os.path.join(snippet_dir, "*.md")):
if readme is not None:
raise SnippetError("%s contains multiple .md files")
with open(fn) as f:
readme = special_markdown(f.read())
snippets.append(
{
"id": id,
"title": title,
"readme": readme,
"go_codes": [
(filename, pygments.highlight(code, go_lexer, html_formatter))
for filename, code in go_codes
],
"py_codes": [
(filename, pygments.highlight(code, py_lexer, html_formatter))
for filename, code in py_codes
],
}
)
template = env.get_template("build.html")
html = template.render(
snippets=snippets,
highlight_css=html_formatter.get_style_defs(".highlight"),
bootstrap_css=env.get_template("bootstrap.min.css").render(),
)
if dry_run:
# Everything worked!
return 0
if check_built:
# Raise an error if the newly created HTML isn't the same as what
# was created before.
with open(out) as f:
before = f.read()
lines_before = [x.strip() for x in before.strip().splitlines() if x.strip()]
lines_html = [x.strip() for x in html.strip().splitlines() if x.strip()]
if lines_before != lines_html:
import difflib
print(
"".join(
difflib.unified_diff(
before.splitlines(True),
html.splitlines(True),
fromfile="HTML before",
tofile="New HTML",
)
)
)
raise CheckBuiltFailed(
"The generated HTML is different from what it was before. "
"That means that the HTML made from the snippets doesn't match "
"the build HTML. Run this script without --check-built and check "
"in the changes to the dist HTML."
)
elif out:
with open(out, "w") as f:
f.write(html)
f.write("\n")
else:
print(html)
def show_script(str_script):
html_vers = highlight(str_script, BashLexer(), HtmlFormatter(full=True))
display(HTML(html_vers))
def pretty_format(text, name=None):
name = name or 'XML'
return highlight(text, get_lexer_by_name(name),
HtmlFormatter(linenos='table'))
def pythonprettyprint(val):
s = pprint.pformat(val)
lexer = lexers.get_lexer_by_name('python')
formatter = HtmlFormatter()
return mark_safe(highlight(s, lexer, formatter))
def __init__(self, **kwargs):
super(InspectorMagics, self).__init__(**kwargs)
self.formatter = HtmlFormatter()
self.lexer = PythonLexer()
self.style_name = "default"
def json(cls, value) -> json:
response = json.dumps(value, sort_keys=True, indent=4, default=str)
formatter = HtmlFormatter(style="colorful")
response = highlight(response, JsonLexer(), formatter)
style = "<style>" + formatter.get_style_defs() + "</style>"
return mark_safe(style + response)
def block_code(self, code, lang):
if not lang:
return "\n<pre><code>{}</code></pre>\n".format(
mistune.escape(code))
if lang == 'javascript':
try:
obj = json.loads(code)
except Exception as e:
pass
else:
if type(obj) == dict and 'uri' in obj:
return self.format_wamp_code(obj)
lexer = get_lexer_by_name(lang, stripall=True)
formatter = HtmlFormatter()
return highlight(code, lexer, formatter)
if __name__ == '__main__':
renderer = MyRenderer()
md = mistune.Markdown(renderer=renderer)
with open('test2.md', 'r') as f:
source = f.read()
content = md.render(source)
css = HtmlFormatter().get_style_defs('.highlight')
generated = TEMPLATE.format(css, content)
with open('test2.html', 'w') as t:
t.write(generated)
def custom_highlighter(source, language='ipython', metadata=None):
formatter = HtmlFormatter(cssclass='highlight-ipynb')
if not language:
language = 'ipython'
output = _pygments_highlight(source, formatter, language)
return output.replace('<pre>', '<pre class="ipynb">')
def pygment_html_render(s, lexer=lexers.TextLexer): # noqa pylint: disable=no-member
"""Highlight text using a given Lexer"""
return highlight(s, lexer(), HtmlFormatter(linenos=True))
def show_tx(txid, more_details=False):
lmq, oxend = lmq_connection()
info = FutureJSON(lmq, oxend, 'rpc.get_info', 1)
txs = tx_req(lmq, oxend, [txid]).get()
if 'txs' not in txs or not txs['txs']:
return flask.render_template(
'not_found.html',
info=info.get(),
type='tx',
id=txid,
)
tx = parse_txs(txs)[0]
# If this is a state change, see if we have the quorum stored to provide context
testing_quorum = None
if tx['info']['version'] >= 4 and 'sn_state_change' in tx['extra']:
testing_quorum = FutureJSON(
lmq,
oxend,
'rpc.get_quorum_state',
60,
cache_key='tx_state_change',
args={
'quorum_type': 0,
'start_height': tx['extra']['sn_state_change']['height']
})
kindex_info = {} # { amount => { keyindex => {output-info} } }
block_info_req = None
if 'vin' in tx['info']:
if len(tx['info']['vin']) == 1 and 'gen' in tx['info']['vin'][0]:
tx['coinbase'] = True
elif tx['info']['vin'] and config.enable_mixins_details:
# Load output details for all outputs contained in the inputs
outs_req = []
for inp in tx['info']['vin']:
# Key positions are stored as offsets from the previous index rather than indices,
# so de-delta them back into indices:
if 'key_offsets' in inp['key'] and 'key_indices' not in inp[
'key']:
kis = []
inp['key']['key_indices'] = kis
kbase = 0
for koff in inp['key']['key_offsets']:
kbase += koff
kis.append(kbase)
del inp['key']['key_offsets']
outs_req = [{
"amount": inp['key']['amount'],
"index": ki
} for inp in tx['info']['vin'] for ki in inp['key']['key_indices']]
outputs = FutureJSON(lmq,
oxend,
'rpc.get_outs',
args={
'get_txid': True,
'outputs': outs_req,
}).get()
if outputs and 'outs' in outputs and len(
outputs['outs']) == len(outs_req):
outputs = outputs['outs']
# Also load block details for all of those outputs:
block_info_req = FutureJSON(
lmq,
oxend,
'rpc.get_block_header_by_height',
args={'heights': [o["height"] for o in outputs]})
i = 0
for inp in tx['info']['vin']:
amount = inp['key']['amount']
if amount not in kindex_info:
kindex_info[amount] = {}
ki = kindex_info[amount]
for ko in inp['key']['key_indices']:
ki[ko] = outputs[i]
i += 1
if more_details:
formatter = HtmlFormatter(cssclass="syntax-highlight",
style="paraiso-dark")
more_details = {
'details_css':
formatter.get_style_defs('.syntax-highlight'),
'details_html':
highlight(json.dumps(tx, indent="\t", sort_keys=True), JsonLexer(),
formatter),
}
else:
more_details = {}
block_info = {} # { height => {block-info} }
if block_info_req:
bi = block_info_req.get()
if 'block_headers' in bi:
for bh in bi['block_headers']:
block_info[bh['height']] = bh
if testing_quorum:
testing_quorum = testing_quorum.get()
if testing_quorum:
if 'quorums' in testing_quorum and testing_quorum['quorums']:
testing_quorum = testing_quorum['quorums'][0]['quorum']
else:
testing_quorum = None
return flask.render_template(
'tx.html',
info=info.get(),
tx=tx,
kindex_info=kindex_info,
block_info=block_info,
testing_quorum=testing_quorum,
**more_details,
)
def auto_report(dag, path, stylesheet=None):
try:
from jinja2 import Template, Environment, PackageLoader
except ImportError as e:
raise WorkflowError(
"Python package jinja2 must be installed to create reports.")
mode_embedded = True
if path.endswith(".zip"):
mode_embedded = False
elif not path.endswith(".html"):
raise WorkflowError("Report file does not end with .html or .zip")
custom_stylesheet = None
if stylesheet is not None:
try:
with open(stylesheet) as s:
custom_stylesheet = s.read()
except (Exception, BaseException) as e:
raise WorkflowError("Unable to read custom report stylesheet.", e)
logger.info("Creating report...")
env = Environment(
loader=PackageLoader("snakemake", "report"),
trim_blocks=True,
lstrip_blocks=True,
)
env.filters["get_resource_as_string"] = get_resource_as_string
persistence = dag.workflow.persistence
results = defaultdict(lambda: defaultdict(list))
records = defaultdict(JobRecord)
recorded_files = set()
for job in dag.jobs:
for f in itertools.chain(job.expanded_output, job.input):
if is_flagged(f, "report") and f not in recorded_files:
if not f.exists:
raise WorkflowError("File {} marked for report but does "
"not exist.".format(f))
report_obj = get_flag_value(f, "report")
def register_file(f, wildcards_overwrite=None, aux_files=None):
wildcards = wildcards_overwrite or job.wildcards
category = Category(report_obj.category,
wildcards=wildcards,
job=job)
subcategory = Category(report_obj.subcategory,
wildcards=wildcards,
job=job)
results[category][subcategory].append(
FileRecord(
f,
job,
report_obj.caption,
env,
category,
wildcards_overwrite=wildcards_overwrite,
mode_embedded=mode_embedded,
aux_files=aux_files,
))
recorded_files.add(f)
if os.path.isfile(f):
register_file(f)
elif os.path.isdir(f):
if report_obj.htmlindex:
if mode_embedded:
raise WorkflowError(
"Directory marked for report specifies htmlindex. "
"This is unsupported when requesting a pure HTML report. "
"Please use store as zip instead (--report report.zip)."
)
aux_files = []
index_found = False
for root, dirs, files in os.walk(f):
for name in files:
if name != ".snakemake_timestamp":
filepath = os.path.join(root, name)
if (os.path.relpath(filepath, f) !=
report_obj.htmlindex):
aux_files.append(filepath)
else:
index_found = True
if not index_found:
raise WorkflowError(
"Given htmlindex {} not found in directory "
"marked for report".format(
report_obj.htmlindex))
register_file(os.path.join(f, report_obj.htmlindex),
aux_files=aux_files)
elif report_obj.patterns:
if not isinstance(report_obj.patterns, list):
raise WorkflowError(
"Invalid patterns given for report. Must be list.",
rule=job.rule,
)
for pattern in report_obj.patterns:
pattern = os.path.join(f, pattern)
wildcards = glob_wildcards(pattern)._asdict()
names = wildcards.keys()
for w in zip(*wildcards.values()):
w = dict(zip(names, w))
w.update(job.wildcards_dict)
w = Wildcards(fromdict=w)
f = apply_wildcards(pattern, w)
register_file(f, wildcards_overwrite=w)
else:
raise WorkflowError(
"Directory marked for report but neither file patterns "
"given via patterns=[...], nor htmlindex given. "
"See report documentation.",
rule=job.rule,
)
for f in job.expanded_output:
meta = persistence.metadata(f)
if not meta:
logger.warning("Missing metadata for file {}. Maybe metadata "
"was deleted or it was created using an older "
"version of Snakemake. This is a non critical "
"warning.".format(f))
continue
try:
job_hash = meta["job_hash"]
rule = meta["rule"]
rec = records[(job_hash, rule)]
rec.rule = rule
rec.job = job
rec.starttime = min(rec.starttime, meta["starttime"])
rec.endtime = max(rec.endtime, meta["endtime"])
rec.conda_env_file = None
rec.conda_env = meta["conda_env"]
rec.container_img_url = meta["container_img_url"]
rec.output.append(f)
except KeyError as e:
print(e)
logger.warning("Metadata for file {} was created with a too "
"old Snakemake version.".format(f))
for subcats in results.values():
for catresults in subcats.values():
catresults.sort(key=lambda res: res.name)
# prepare runtimes
runtimes = [{
"rule": rec.rule,
"runtime": rec.endtime - rec.starttime
} for rec in sorted(records.values(), key=lambda rec: rec.rule)]
# prepare end times
timeline = [{
"rule":
rec.rule,
"starttime":
datetime.datetime.fromtimestamp(rec.starttime).isoformat(),
"endtime":
datetime.datetime.fromtimestamp(rec.endtime).isoformat(),
} for rec in sorted(records.values(), key=lambda rec: rec.rule)]
# prepare per-rule information
rules = defaultdict(list)
for rec in records.values():
rule = RuleRecord(rec.job, rec)
if rec.rule not in rules:
rules[rec.rule].append(rule)
else:
merged = False
for other in rules[rec.rule]:
if rule == other:
other.add(rec)
merged = True
break
if not merged:
rules[rec.rule].append(rule)
# rulegraph
rulegraph, xmax, ymax = rulegraph_d3_spec(dag)
# configfiles
configfiles = [ConfigfileRecord(f) for f in dag.workflow.configfiles]
seen = set()
files = [
seen.add(res.target) or res for cat in results.values()
for subcat in cat.values() for res in subcat if res.target not in seen
]
rst_links = textwrap.dedent("""
.. _Workflow: javascript:show_panel('workflow')
.. _Statistics: javascript:show_panel('statistics')
{% for cat, catresults in categories|dictsort %}
.. _{{ cat.name }}: javascript:show_panel("{{ cat.id }}")
{% endfor %}
{% for res in files %}
.. _{{ res.target }}: javascript:show_panel("{{ res.category.id }}")
{% endfor %}
""")
for cat, subcats in results.items():
for subcat, catresults in subcats.items():
for res in catresults:
res.render(env, rst_links, results, files)
# global description
text = ""
if dag.workflow.report_text:
with open(dag.workflow.report_text) as f:
class Snakemake:
config = dag.workflow.config
text = f.read() + rst_links
text = publish_parts(
env.from_string(text).render(snakemake=Snakemake,
categories=results,
files=files),
writer_name="html",
)["body"]
# record time
now = "{} {}".format(datetime.datetime.now().ctime(), time.tzname[0])
results_size = sum(res.size for cat in results.values()
for subcat in cat.values() for res in subcat)
try:
from pygments.formatters import HtmlFormatter
except ImportError:
raise WorkflowError(
"Python package pygments must be installed to create reports.")
template = env.get_template("report.html")
logger.info("Downloading resources and rendering HTML.")
rendered = template.render(
results=results,
results_size=results_size,
configfiles=configfiles,
text=text,
rulegraph_nodes=rulegraph["nodes"],
rulegraph_links=rulegraph["links"],
rulegraph_width=xmax + 20,
rulegraph_height=ymax + 20,
runtimes=runtimes,
timeline=timeline,
rules=[rec for recs in rules.values() for rec in recs],
version=__version__,
now=now,
pygments_css=HtmlFormatter(style="trac").get_style_defs(".source"),
custom_stylesheet=custom_stylesheet,
mode_embedded=mode_embedded,
)
# TODO look into supporting .WARC format, also see (https://webrecorder.io)
if not mode_embedded:
with ZipFile(path, mode="w") as zipout:
folder = Path(Path(path).stem)
# store results in data folder
for subcats in results.values():
for catresults in subcats.values():
for result in catresults:
# write raw data
if result.table_content is not None:
zipout.writestr(
str(folder.joinpath(result.data_uri)),
result.table_content,
)
else:
zipout.write(result.path,
str(folder.joinpath(result.data_uri)))
# write thumbnail
if result.is_img and result.png_content:
zipout.writestr(
str(folder.joinpath(result.png_uri)),
result.png_content)
# write aux files
parent = folder.joinpath(result.data_uri).parent
for path in result.aux_files:
# print(path, parent, str(folder.joinpath(os.path.relpath(path, parent))))
zipout.write(
path,
str(
parent.joinpath(
os.path.relpath(
path,
os.path.dirname(result.path)))),
)
# write report html
zipout.writestr(str(folder.joinpath("report.html")), rendered)
else:
with open(path, "w", encoding="utf-8") as htmlout:
htmlout.write(rendered)
logger.info("Report created: {}.".format(path))
def _highlight_code_html(self, code):
from pygments import highlight
from pygments.lexers import PythonLexer
from pygments.formatters import HtmlFormatter
return highlight(code, PythonLexer(), HtmlFormatter())
'linespans': str,
'anchorlinenos': bool
}
argmap = {'true': True, 'false': False}
if len(command) == 4:
for opt in command[3].split(','):
opt = opt.split('=')
if opt[0] not in params.keys():
eprint('ignoring:', opt[0])
continue
if len(opt) >= 2:
if params[opt[0]] == bool:
opts[opt[0]] = argmap.get(opt[1].lower(), True)
elif params[opt[0]] == int:
try:
opts[opt[0]] = int(opt[1])
except ValueError:
eprint('Ignoring {}, expected an integer, got {}' \
.format(opt[0], opt[1]))
else:
opts[opt[0]] = opt[1]
else:
opts[opt[0]] = True
formatter = HtmlFormatter(**opts)
highlighted = highlight(code, lexer, formatter)
output.write('colorized|{}\n'.format(len(highlighted)))
output.write(highlighted)
output.flush()
