def replaceFunc(match):
if match.group(1) == 'python':
return highlight(match.group(2), PythonLexer(), HtmlFormatter())
elif match.group(1) in ['html', 'javascript']:
return highlight(match.group(2), HtmlSmartyLexer(), HtmlFormatter())
else:
raise Warning("Invalid lexer argument!")
def nice_body(body, content=None, cssclass=None, encoding='utf-8'):
if not body:
return None
cssclasses = ['codehilite']
if cssclass:
cssclasses.append(cssclass)
classes = ' '.join(cssclasses)
content = get_body_content_type(body, content)
if content is not None:
if 'x-www-form-urlencoded' in content:
lex = IniLexer()
elif 'json' in content:
lex = JsonLexer()
else:
try:
lex = get_lexer_for_mimetype(content, encoding=encoding)
except ClassNotFound as e:
return body
if isinstance(lex, IniLexer):
parsedbody = urlparse.parse_qsl(body, keep_blank_values=True)
if body and not parsedbody:
return tornado.escape.xhtml_escape(body)
parsedbody = [(x.strip(), y) for [x, y] in parsedbody]
args = collections.OrderedDict(sorted(parsedbody))
params = "\n".join([k.strip() + "=" + v for k, v in args.iteritems()])
return highlight(params, IniLexer(), HtmlFormatter(cssclass=classes, encoding=encoding))
elif isinstance(lex, JsonLexer):
try:
return highlight(json.dumps(json.loads(body), indent=4), JsonLexer(), HtmlFormatter(cssclass=classes))
except ValueError as e:
pass
return highlight(body, lex, HtmlFormatter(cssclass=classes, encoding=encoding))
def process(self):
self.artifact.setup_kv_storage()
loader = nose.loader.TestLoader()
for module_name in self.artifact.input_text().split():
self.log.debug("Starting to process module '%s'" % module_name)
tests = loader.loadTestsFromName(module_name)
self.log.debug("Loaded tests.")
for test in tests:
self.log.debug("Running test suite %s" % test)
test_passed = nose.core.run(suite=test, argv=['nosetests'])
self.log.debug("Passed: %s" % test_passed)
for x in dir(test.context):
xx = test.context.__dict__[x]
if inspect.ismethod(xx) or inspect.isfunction(xx):
test_context_name = test.context.__name__
qualified_test_name = "%s.%s" % (test_context_name, xx.__name__)
source = inspect.getsource(xx.__code__)
html_source = highlight(source, self.LEXER, self.HTML_FORMATTER)
latex_source = highlight(source, self.LEXER, self.LATEX_FORMATTER)
if test_passed:
html_result = """ <div class="test-passed"> %s PASSED </div> """ % qualified_test_name
else:
html_result = """ <div class="test-failed"> %s FAILED </div> """ % qualified_test_name
self.artifact.output_data.append("%s:source" % qualified_test_name, source)
self.artifact.output_data.append("%s:html-source" % qualified_test_name, html_source)
self.artifact.output_data.append("%s:latex-source" % qualified_test_name, latex_source)
self.artifact.output_data.append("%s:test-passed" % qualified_test_name, test_passed)
self.artifact.output_data.append("%s:html-result" % qualified_test_name, html_result)
self.artifact.output_data.append("%s:html-source+result" % qualified_test_name, "%s\n%s" % (html_source, html_result))
self.artifact._storage.save()
def pygments2xpre(s, language="python"):
"Return markup suitable for XPreformatted"
try:
from pygments import highlight
from pygments.formatters import HtmlFormatter
except ImportError:
return s
from pygments.lexers import get_lexer_by_name
l = get_lexer_by_name(language)
h = HtmlFormatter()
# XXX: Does not work in Python 2, since pygments creates non-unicode
# outpur snippets.
# from io import StringIO
from six import StringIO
out = StringIO()
highlight(s, l, h, out)
styles = [
(cls, style.split(";")[0].split(":")[1].strip())
for cls, (style, ttype, level) in h.class2style.items()
if cls and style and style.startswith("color:")
]
from reportlab.lib.pygments2xpre import _2xpre
return _2xpre(out.getvalue(), styles)
def highlightBlock(self, text):
"""Takes a block, applies format to the document.
according to what's in it.
"""
# I need to know where in the document we are,
# because our formatting info is global to
# the document
cb = self.currentBlock()
p = cb.position()
# The \n is not really needed, but sometimes
# you are in an empty last block, so your position is
# **after** the end of the document.
text=unicode(self.document().toPlainText())+'\n'
# Yes, re-highlight the whole document.
# There **must** be some optimizacion possibilities
# but it seems fast enough.
highlight(text,self.lexer,self.formatter)
# Just apply the formatting to this block.
# For titles, it may be necessary to backtrack
# and format a couple of blocks **earlier**.
for i in range(len(unicode(text))):
try:
self.setFormat(i,1,self.formatter.data[p+i])
except IndexError:
pass
# I may need to do something about this being called
# too quickly.
self.tstamp=time.time()
def do_pygments(self, request, doc, options):
'''Use pygments to create and save the html text.'''
if not ENABLE_PYGMENTS:
m = 'hauto60: Library needed for pygments is not available.'
messages.error(request, m)
return ''
if not options: options = 'xml, linenos' #defaults
custom = options.split(',')
# The first option sets the lexer to use. For available
# lexers goto http://pygments.org/docs/lexers/
if custom[0] == 'xml':
lexer = XmlLexer
elif custom[0] == 'xslt':
lexer = XsltLexer
elif custom[0] == 'guess':
lexer, example = self.pyg_guess(custom)
m = 'hauto61: Lexer guessed:%s using example: %s'%(lexer, example)
messages.info(request, m)
else:
lexer = TextLexer # Have a safe default if mistake made.
# The second controls if line number be shown on the output?
has_num = False
if len(custom)> 1 and str(custom[1]).strip() == 'linenos':
has_num = 'inline'
# Now we can do the task and save it.
try:
return highlight(doc.text, lexer(), HtmlFormatter(linenos=has_num))
except TypeError: # this is caused by the guess feature
try:
return highlight(doc.text, lexer, HtmlFormatter(linenos=has_num))
except: # TODO: what exceptions do lexer highlighter cause?
return ''
def send_message(self, mess):
if self.demo_mode:
print(self.md_ansi.convert(mess.body))
else:
bar = '\n╌╌[{mode}]' + ('╌' * 60)
super().send_message(mess)
print(bar.format(mode='MD '))
if ANSI:
print(highlight(mess.body, self.md_lexer, self.terminal_formatter))
else:
print(mess.body)
print(bar.format(mode='HTML'))
html = self.md_html.convert(mess.body)
if ANSI:
print(highlight(html, self.html_lexer, self.terminal_formatter))
else:
print(html)
print(bar.format(mode='TEXT'))
print(self.md_text.convert(mess.body))
print(bar.format(mode='IM '))
print(self.md_im.convert(mess.body))
if ANSI:
print(bar.format(mode='ANSI'))
print(self.md_ansi.convert(mess.body))
print(bar.format(mode='BORDERLESS'))
print(self.md_borderless_ansi.convert(mess.body))
print('\n\n')
def process(self, ctx, m):
if (not ctx.quiet):
if (self.eval):
obj = ctx.interpolate(m, self.eval)
else:
obj = m
res = self._prepare_res(obj)
if (self.truncate_line):
truncated = []
for line in res.split("\n"):
if (len(line) > self.truncate_line):
line = line[:self.truncate_line - 2] + ".."
truncated.append(line)
res = "\n".join(truncated)
if sys.stdout.isatty():
print highlight(res, self._lexer, self._formatter) #[:-1]
#print res
else:
print res
yield m
def print_action_exception(e):
if isinstance(e.inner_exception, (ExecCommandFailed, QueryException)):
print_exception(e.inner_exception)
else:
print '-'*79
print highlight(e.traceback, PythonTracebackLexer(), Formatter())
print '-'*79
def testExternalMemFuncDestructor( self ):
className = "myclassname"
funcName = "~myDesctructorName"
code = """
void %s::%s( char* param1 ) {
return param1;
};
""" % ( className, funcName )
highlight( code, self.lexer, self.formatter)
myRTags = FakeRTags()
test = CppSemantics( myRTags )
for t,v,num in self.formatter.current_line:
test.Feed( t,v,num )
self.assertEqual( myRTags.GetClass()[0].name, className,
"Class Name did not match %s" % myRTags.GetClass()[0].name )
mf = myRTags.GetClass()[0].GetElements()[0]
self.assertEqual( mf['method'][0].GetName(), funcName,
"Function Name %s did not match %s" % (mf['method'][0].GetName(), funcName) )
self.assertEqual( test.state, 'start',
"Semantics not in correct state '%s'" % ( test.state ))
def testExternalMultipleMemFuncs( self ):
className = "ClassName"
funcName = "FuncName"
funcName2 = "FuncName2"
code = """
int %s::%s (int count, char* mess ) { {} {} }
int %s::%s (int count, char* mess ) { }
""" % ( className, funcName, className, funcName2 )
highlight( code, self.lexer, self.formatter)
myRTags = FakeRTags()
test = CppSemantics( myRTags )
for t,v,num in self.formatter.current_line:
test.Feed( t,v,num )
self.assertEqual( myRTags.GetClass()[0].name, className,
"Class Name did not match %s" % myRTags.GetClass()[0].name )
mf = myRTags.GetClass()[0].GetElements()[0]
self.assertEqual( len(myRTags.GetClass()), 1,
"Too many classes (%d) created." % len(myRTags.GetClass()) )
self.assertEqual( mf['method'][0].GetName(), funcName,
"Function Name %s did not match %s" % (mf['method'][0].GetName(), funcName) )
self.assertEqual( test.state, 'start',
"Semantics not in correct state '%s'" % ( test.state ))
def setError(self, err_type=None, err_value=None, err_traceback=None):
"""Translates the given error object into an HTML string and places it
it the message panel
:param error: an error object (typically an exception object)
:type error: object"""
msgbox = self._msgbox
html_orig = '<html><head><style type="text/css">{style}</style>' "</head><body>"
style, formatter = "", None
if pygments is not None:
formatter = HtmlFormatter()
style = formatter.get_style_defs()
html = html_orig.format(style=style)
for de in err_value:
e_html = """<pre>{reason}: {desc}</pre>{origin}<hr>"""
origin, reason, desc = de.origin, de.reason, de.desc
if reason.startswith("PyDs_") and pygments is not None:
origin = highlight(origin, PythonTracebackLexer(), formatter)
else:
origin = "<pre>%s</pre>" % origin
html += e_html.format(desc=desc, origin=origin, reason=reason)
html += "</body></html>"
msgbox.setText(err_value[0].desc)
msgbox.setDetailedHtml(html)
exc_info = "".join(traceback.format_exception(err_type, err_value, err_traceback))
html = html_orig.format(style=style)
if pygments is None:
html += "<pre>%s</pre>" % exc_info
else:
html += highlight(exc_info, PythonTracebackLexer(), formatter)
html += "</body></html>"
msgbox.setOriginHtml(html)
def testMemFuncs( self ):
className = "myclassname"
funcName = "myFuncName"
code = """
class %s {
int %s (int count, char* mess ) { {} {} }
};
"""% ( className, funcName )
highlight( code, self.lexer, self.formatter)
myRTags = FakeRTags()
test = CppSemantics( myRTags )
for t,v,num in self.formatter.current_line:
test.Feed( t,v,num )
self.assertEqual( myRTags.GetClass()[0].name, className,
"Class Name did not match %s" % myRTags.GetClass()[0].name )
mf = myRTags.GetClass()[0].GetElements()[0]
self.assertEqual( mf['method'][0].GetName(), funcName,
"Function Name %s did not match %s" % (mf['method'][0].GetName(), funcName) )
self.assertEqual( test.state, 'start',
"Semantics not in correct state '%s'" % ( test.state ))
def main(server, files_to_upload, log_level, ftp_dir, finish, user, passwd):
api = dsapi.DataStreamAPI(None, None, None, ftp_server=server,ftp_dir=ftp_dir, ftp_user=user, ftp_pass=passwd)
for file in files_to_upload:
api.ftp_upload(file)
if(finish):
finish_return = api.ftp_finish()
print pygments.highlight(json.dumps(finish_return.json()),JsonLexer(),TerminalFormatter(bg="dark"))
def pygments2xpre(s, language="python"):
"Return markup suitable for XPreformatted"
try:
from pygments import highlight
from pygments.formatters import HtmlFormatter
except ImportError:
return s
from pygments.lexers import get_lexer_by_name
rconv = lambda x: x
if isPy3:
out = getStringIO()
else:
if isUnicode(s):
s = asBytes(s)
rconv = asUnicode
out = getBytesIO()
l = get_lexer_by_name(language)
h = HtmlFormatter()
highlight(s,l,h,out)
styles = [(cls, style.split(';')[0].split(':')[1].strip())
for cls, (style, ttype, level) in h.class2style.items()
if cls and style and style.startswith('color:')]
return rconv(_2xpre(out.getvalue(),styles))
def __init__(self,options,code):
lexer = pygments.lexers.web.JavascriptLexer(ensurenl=False)
lexer.encoding = 'utf8'
self.filter = JSFilter(options)
lexer.add_filter(self.filter)
fmter = pygments.formatters.get_formatter_by_name(options.format)
fmter.encoding = 'utf8'
#print "check_JS",repr(code)
self.code = re.sub(r'@{{(!?[A-Za-z_][A-Za-z_0-9]*)}}',self.filter.addXXX,code)
#workaround for pygments adding/removing "/n" to last token
add_nl = self.code.endswith(u'\n')
self.out = StringIO()
pygments.highlight(self.code, lexer, fmter, self.out)
self.out = self.out.getvalue()
#workaround for pygments adding/removing "\n" to last token
if add_nl:
i = len(self.filter.tokens)-1
v = self.filter.value(i)+u'\n'
self.filter.set_value(i,v)
if self.filter.errors:
print self.out
def pretty_print (src, s, out, with_formatter, ext=None):
""" `src' is a filepath to be formatted. `out' is a file object
to be written."""
if ext is None:
ext = guess_ext_by_filename(src)
if ext == "":
ext = guess_ext_by_contents(s)
# format
if with_formatter:
f = _load_formatter(ext)
if f is not None:
ext,s = f.format(s)
# highlight
h = _load_highlighter(ext)
if h is None:
try:
lexer = pygments.lexers.get_lexer_by_name(ext)
except pygments.util.ClassNotFound:
lexer = pygments.lexers.get_lexer_for_mimetype("text/plain")
fmt = pygments.formatters.Terminal256Formatter(encoding="utf-8")
pygments.highlight(s, lexer, fmt, out)
else:
h.highlight(out, s)
out.close()
def get_paste(cache, paste_id, raw=False):
'''
Return page with <paste_id>.
'''
paste_id = paste_id
if not cache.exists('paste:' + paste_id):
bottle.redirect('/')
data = json.loads(cache.get('paste:' + paste_id))
if not raw:
# Syntax hilighting
try:
lexer = get_lexer_by_name(data['syntax'], stripall=False)
except:
lexer = get_lexer_by_name('text', stripall=False)
formatter = HtmlLineFormatter(linenos=True, cssclass="paste")
linker = kwlinker.get_linker_by_name(data['syntax'])
if linker is not None:
lexer.add_filter(linker)
data['code'] = highlight(data['code'], lexer, formatter)
data['code'] = kwlinker.replace_markup(data['code'])
else:
data['code'] = highlight(data['code'], lexer, formatter)
data['css'] = HtmlLineFormatter().get_style_defs('.code')
return data
def main():
headers = []
content = []
active_list = headers
headers_done = False
for line in stdin:
if not headers_done and not line.strip():
active_list = content
continue
active_list.append(line)
if not content:
# no headers found, so the content had been loaded in the wrong list.
# Let's exchange them.
content, headers = headers, content
print ''.join(headers)
output = format_json(''.join(content))
if PYGMENTS_AVAILABLE:
print highlight(output, JSONLexer(), TerminalFormatter())
else:
print output
print >>stderr, ("NOTE: If you have the python package "
"`pygments` available for import, you'll get nice "
"syntax highlighting ^_^")
def view_paste(paste_id):
"""
Return page with paste_id
"""
paste_id = paste_id
if not cache.exists('paste:' + paste_id):
bottle.redirect('/')
p = json.loads(cache.get('paste:' + paste_id))
# Syntax hilighting
try:
lexer = get_lexer_by_name(p['syntax'], stripall=False)
except:
lexer = get_lexer_by_name('text', stripall=False)
formatter = HtmlFormatter(linenos=True, cssclass="paste")
linker = modules.kwlinker.get_linker_by_name(p['syntax'])
if linker is not None:
lexer.add_filter(linker)
p['code'] = highlight(p['code'], lexer, formatter)
p['code'] = modules.kwlinker.replace_markup(p['code'])
else:
p['code'] = highlight(p['code'], lexer, formatter)
p['css'] = HtmlFormatter().get_style_defs('.code')
return bottle.jinja2_template('view.html', paste=p, pid=paste_id)
def highlight(self,
code: "A string specifying the code to highlight",
language: "Optionally, the name of the language"=None):
if language:
return highlight(code, get_lexer_by_name(language), self.formatter)
return highlight(code, guess_lexer(code), self.formatter)
def print_colored_text(texts, format_name):
if pygments_available:
highlight(
texts, get_lexer_by_name(format_name),
TerminalFormatter(), sys.stdout)
else:
print(texts)
def pcat(filename, target='ipython'):
code = read_file_or_url(filename)
HTML_TEMPLATE = """<style>
{}
</style>
{}
"""
from pygments.lexers import get_lexer_for_filename
lexer = get_lexer_for_filename(filename, stripall=True)
from pygments.formatters import HtmlFormatter, TerminalFormatter
from pygments import highlight
try:
assert(target=='ipython')
from IPython.display import HTML, display
from pygments.formatters import HtmlFormatter
formatter = HtmlFormatter(linenos=True, cssclass="source")
html_code = highlight(code, lexer, formatter)
css = formatter.get_style_defs()
html = HTML_TEMPLATE.format(css, html_code)
htmlres = HTML(html)
return htmlres
except Exception as e:
print(e)
pass
formatter = TerminalFormatter()
output = highlight(code,lexer,formatter)
print(output)
def format_codechunks(self, chunk):
from pygments import highlight
from pygments.lexers import PythonLexer, TextLexer, PythonConsoleLexer
# from IPythonLexer import IPythonLexer
from pygments.formatters import LatexFormatter
chunk["content"] = highlight(
chunk["content"],
PythonLexer(),
LatexFormatter(verboptions="frame=single,fontsize=\small, xleftmargin=0.5em"),
)
if len(chunk["result"].strip()) > 0 and chunk["results"] == "verbatim":
if chunk["term"]:
chunk["result"] = highlight(
chunk["result"],
PythonLexer(),
LatexFormatter(verboptions="frame=single,fontsize=\small, xleftmargin=0.5em"),
)
else:
chunk["result"] = highlight(
chunk["result"],
TextLexer(),
LatexFormatter(verboptions="frame=leftline,fontsize=\small, xleftmargin=0.5em"),
)
return PwebFormatter.format_codechunks(self, chunk)
def pretty_print (src, out):
""" `src' is a filepath to be formatted. `out' is a file object
to be written. """
f = os.path.basename(src)
ext = os.path.splitext(src)[1]
if ext != "json":
s = open(src).read()
if ext == "json":
s = json.dumps(json.load(open(src)), indent=2)
elif ext == "":
# is executable script?
if os.access(f, os.X_OK) and s.startswith("#!"):
# this is a script without extention. read shebang
shebang = s.split("\n", 1)[0]
try:
interpreter = shebang.split("env ")[1]
except IndexError:
interpreter = os.path.basename(shebang[2:])
ext = _interpreter2ext(interpreter)
f += "." + ext
# colorful!
try:
lexer = pygments.lexers.get_lexer_for_filename(f)
except pygments.util.ClassNotFound:
lexer = pygments.lexers.get_lexer_for_mimetype("text/plain")
fmt = pygments.formatters.Terminal256Formatter(encoding="utf-8")
pygments.highlight(s, lexer, fmt, out)
def pretty_print_body(fmt, body):
try:
if fmt.lower() == 'json':
d = json.loads(body.strip())
s = json.dumps(d, indent=4, sort_keys=True)
print pygments.highlight(s, JsonLexer(), TerminalFormatter())
elif fmt.lower() == 'form':
qs = repeatable_parse_qs(body)
for k, v in qs.all_pairs():
s = Colors.GREEN
s += '%s: ' % urllib.unquote(k)
s += Colors.ENDC
s += urllib.unquote(v)
print s
elif fmt.lower() == 'text':
print body
elif fmt.lower() == 'xml':
import xml.dom.minidom
xml = xml.dom.minidom.parseString(body)
print pygments.highlight(xml.toprettyxml(), XmlLexer(), TerminalFormatter())
else:
raise PappyException('"%s" is not a valid format' % fmt)
except PappyException as e:
raise e
except:
raise PappyException('Body could not be parsed as "%s"' % fmt)
def inner():
assert re.match(r'^[a-zA-Z._-]+$', script)
exec_path = "scripts/" + script + ".py"
cmd = ["python3", "-u", exec_path] # -u: don't buffer output
error = False
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
for line in proc.stdout:
yield highlight(line, BashLexer(), HtmlFormatter())
# Maybe there is more stdout after an error...
for line in proc.stderr:
error = True
yield highlight(line, BashLexer(), HtmlFormatter())
if error:
yield "<script>parent.stream_error()</script>"
else:
yield "<script>parent.stream_success()</script>"
def template(template_id):
"""
The view where we display the result
in syntax-highlighted HTML and CSS
"""
template = Template.query.filter(Template.id == long(template_id)).first()
if not template:
return "The requested template doesn't exist", 404
hashtml = len(template.html.strip()) > 0
cssperma = 'http://%s/static/cssserve/%s.css'%(SITE_DOMAIN, str(template.id))
pygmented_css_link = highlight('<link rel="stylesheet" type="text/css" href="%s">'%cssperma,
CssLexer(),
HtmlFormatter(style = 'bw',
linenos = 'table'))
return render_template('saved_template.html',
template = template,
pygmented_css_link_code = pygmented_css_link,
pygmented_html_code = highlight(template.html,
HtmlLexer(),
HtmlFormatter(style = 'bw',
linenos = 'table')),
pygmented_css_code = highlight(template.css,
CssLexer(),
HtmlFormatter(style = 'bw',
linenos = 'table')),
pygments_style = HtmlFormatter(style = 'bw').get_style_defs('.highlight'),
hashtml = hashtml,
)
def get(self):
self.values["project"] = "http://www.proven-corporation.com/software/app-engine-console/"
if self.values["subpage"] == "usage":
for exampleNum in range(len(self.examples)):
key = "example%d" % (exampleNum + 1)
val = util.trim(self.examples[exampleNum])
val = pygments.highlight(val, self.resultLexer, self.outputFormatter).strip()
self.values[key] = val
elif self.values["subpage"] == "integration":
self.values["example1"] = pygments.highlight(
util.trim(
"""
def is_dev():
import os
return os.environ['SERVER_SOFTWARE'].startswith('Dev')
"""
),
self.pythonLexer,
self.outputFormatter,
).strip()
self.values["example2"] = pygments.highlight(
util.trim(
"""
>>> is_dev()
True
"""
),
self.resultLexer,
self.outputFormatter,
).strip()
def colorize(language, title, text):
"""Colorize the text syntax.
Guess the language of the text and colorize it.
Returns a tuple containing the colorized text and the language name.
"""
formatter = HtmlFormatter(
linenos=True, style=PygmentsStyle, noclasses=True, nobackground=True)
#Try to get the lexer by name
try:
lexer = get_lexer_by_name(language.lower())
return highlight(text, lexer, formatter), lexer.name
except LexerNotFound:
pass
#Try to get the lexer by filename
try:
lexer = get_lexer_for_filename(title.lower())
return highlight(text, lexer, formatter), lexer.name
except LexerNotFound:
pass
#Try to guess the lexer from the text
try:
lexer = guess_lexer(text)
if lexer.analyse_text(text) > .3:
return highlight(text, lexer, formatter), lexer.name
except LexerNotFound:
pass
#Fallback to the plain/text lexer
lexer = get_lexer_by_name('text')
return highlight(text, lexer, formatter), lexer.name
def main_inner(popts, args, usage):
opts = {}
O_opts = []
P_opts = []
F_opts = []
for opt, arg in popts:
if opt == '-O':
O_opts.append(arg)
elif opt == '-P':
P_opts.append(arg)
elif opt == '-F':
F_opts.append(arg)
opts[opt] = arg
if opts.pop('-h', None) is not None:
print(usage)
return 0
if opts.pop('-V', None) is not None:
print('Pygments version %s, (c) 2006-2019 by Georg Brandl.' %
__version__)
return 0
# handle ``pygmentize -L``
L_opt = opts.pop('-L', None)
if L_opt is not None:
if opts:
print(usage, file=sys.stderr)
return 2
# print version
main(['', '-V'])
if not args:
args = ['lexer', 'formatter', 'filter', 'style']
for arg in args:
_print_list(arg.rstrip('s'))
return 0
# handle ``pygmentize -H``
H_opt = opts.pop('-H', None)
if H_opt is not None:
if opts or len(args) != 2:
print(usage, file=sys.stderr)
return 2
what, name = args # pylint: disable=unbalanced-tuple-unpacking
if what not in ('lexer', 'formatter', 'filter'):
print(usage, file=sys.stderr)
return 2
return _print_help(what, name)
# parse -O options
parsed_opts = _parse_options(O_opts)
opts.pop('-O', None)
# parse -P options
for p_opt in P_opts:
try:
name, value = p_opt.split('=', 1)
except ValueError:
parsed_opts[p_opt] = True
else:
parsed_opts[name] = value
opts.pop('-P', None)
# encodings
inencoding = parsed_opts.get('inencoding', parsed_opts.get('encoding'))
outencoding = parsed_opts.get('outencoding', parsed_opts.get('encoding'))
# handle ``pygmentize -N``
infn = opts.pop('-N', None)
if infn is not None:
lexer = find_lexer_class_for_filename(infn)
if lexer is None:
lexer = TextLexer
print(lexer.aliases[0])
return 0
# handle ``pygmentize -S``
S_opt = opts.pop('-S', None)
a_opt = opts.pop('-a', None)
if S_opt is not None:
f_opt = opts.pop('-f', None)
if not f_opt:
print(usage, file=sys.stderr)
return 2
if opts or args:
print(usage, file=sys.stderr)
return 2
try:
parsed_opts['style'] = S_opt
fmter = get_formatter_by_name(f_opt, **parsed_opts)
except ClassNotFound as err:
print(err, file=sys.stderr)
return 1
print(fmter.get_style_defs(a_opt or ''))
return 0
# if no -S is given, -a is not allowed
if a_opt is not None:
print(usage, file=sys.stderr)
return 2
# parse -F options
F_opts = _parse_filters(F_opts)
opts.pop('-F', None)
allow_custom_lexer_formatter = False
# -x: allow custom (eXternal) lexers and formatters
if opts.pop('-x', None) is not None:
allow_custom_lexer_formatter = True
# select lexer
lexer = None
# given by name?
lexername = opts.pop('-l', None)
if lexername:
# custom lexer, located relative to user's cwd
if allow_custom_lexer_formatter and '.py' in lexername:
try:
filename = None
name = None
if ':' in lexername:
filename, name = lexername.rsplit(':', 1)
if '.py' in name:
# This can happen on Windows: If the lexername is
# C:\lexer.py -- return to normal load path in that case
name = None
if filename and name:
lexer = load_lexer_from_file(filename, name, **parsed_opts)
else:
lexer = load_lexer_from_file(lexername, **parsed_opts)
except ClassNotFound as err:
print('Error:', err, file=sys.stderr)
return 1
else:
try:
lexer = get_lexer_by_name(lexername, **parsed_opts)
except (OptionError, ClassNotFound) as err:
print('Error:', err, file=sys.stderr)
return 1
# read input code
code = None
if args:
if len(args) > 1:
print(usage, file=sys.stderr)
return 2
if '-s' in opts:
print('Error: -s option not usable when input file specified',
file=sys.stderr)
return 2
infn = args[0]
try:
with open(infn, 'rb') as infp:
code = infp.read()
except Exception as err:
print('Error: cannot read infile:', err, file=sys.stderr)
return 1
if not inencoding:
code, inencoding = guess_decode(code)
# do we have to guess the lexer?
if not lexer:
try:
lexer = get_lexer_for_filename(infn, code, **parsed_opts)
except ClassNotFound as err:
if '-g' in opts:
try:
lexer = guess_lexer(code, **parsed_opts)
except ClassNotFound:
lexer = TextLexer(**parsed_opts)
else:
print('Error:', err, file=sys.stderr)
return 1
except OptionError as err:
print('Error:', err, file=sys.stderr)
return 1
elif '-s' not in opts: # treat stdin as full file (-s support is later)
# read code from terminal, always in binary mode since we want to
# decode ourselves and be tolerant with it
code = sys.stdin.buffer.read() # use .buffer to get a binary stream
if not inencoding:
code, inencoding = guess_decode_from_terminal(code, sys.stdin)
# else the lexer will do the decoding
if not lexer:
try:
lexer = guess_lexer(code, **parsed_opts)
except ClassNotFound:
lexer = TextLexer(**parsed_opts)
else: # -s option needs a lexer with -l
if not lexer:
print('Error: when using -s a lexer has to be selected with -l',
file=sys.stderr)
return 2
# process filters
for fname, fopts in F_opts:
try:
lexer.add_filter(fname, **fopts)
except ClassNotFound as err:
print('Error:', err, file=sys.stderr)
return 1
# select formatter
outfn = opts.pop('-o', None)
fmter = opts.pop('-f', None)
if fmter:
# custom formatter, located relative to user's cwd
if allow_custom_lexer_formatter and '.py' in fmter:
try:
filename = None
name = None
if ':' in fmter:
# Same logic as above for custom lexer
filename, name = fmter.rsplit(':', 1)
if '.py' in name:
name = None
if filename and name:
fmter = load_formatter_from_file(filename, name,
**parsed_opts)
else:
fmter = load_formatter_from_file(fmter, **parsed_opts)
except ClassNotFound as err:
print('Error:', err, file=sys.stderr)
return 1
else:
try:
fmter = get_formatter_by_name(fmter, **parsed_opts)
except (OptionError, ClassNotFound) as err:
print('Error:', err, file=sys.stderr)
return 1
if outfn:
if not fmter:
try:
fmter = get_formatter_for_filename(outfn, **parsed_opts)
except (OptionError, ClassNotFound) as err:
print('Error:', err, file=sys.stderr)
return 1
try:
outfile = open(outfn, 'wb')
except Exception as err:
print('Error: cannot open outfile:', err, file=sys.stderr)
return 1
else:
if not fmter:
if '256' in os.environ.get('TERM', ''):
fmter = Terminal256Formatter(**parsed_opts)
else:
fmter = TerminalFormatter(**parsed_opts)
outfile = sys.stdout.buffer
# determine output encoding if not explicitly selected
if not outencoding:
if outfn:
# output file? use lexer encoding for now (can still be None)
fmter.encoding = inencoding
else:
# else use terminal encoding
fmter.encoding = terminal_encoding(sys.stdout)
# provide coloring under Windows, if possible
if not outfn and sys.platform in ('win32', 'cygwin') and \
fmter.name in ('Terminal', 'Terminal256'): # pragma: no cover
# unfortunately colorama doesn't support binary streams on Py3
outfile = UnclosingTextIOWrapper(outfile, encoding=fmter.encoding)
fmter.encoding = None
try:
import colorama.initialise
except ImportError:
pass
else:
outfile = colorama.initialise.wrap_stream(outfile,
convert=None,
strip=None,
autoreset=False,
wrap=True)
# When using the LaTeX formatter and the option `escapeinside` is
# specified, we need a special lexer which collects escaped text
# before running the chosen language lexer.
escapeinside = parsed_opts.get('escapeinside', '')
if len(escapeinside) == 2 and isinstance(fmter, LatexFormatter):
left = escapeinside[0]
right = escapeinside[1]
lexer = LatexEmbeddedLexer(left, right, lexer)
# ... and do it!
if '-s' not in opts:
# process whole input as per normal...
highlight(code, lexer, fmter, outfile)
return 0
else:
# line by line processing of stdin (eg: for 'tail -f')...
try:
while 1:
line = sys.stdin.buffer.readline()
if not line:
break
if not inencoding:
line = guess_decode_from_terminal(line, sys.stdin)[0]
highlight(line, lexer, fmter, outfile)
if hasattr(outfile, 'flush'):
outfile.flush()
return 0
except KeyboardInterrupt: # pragma: no cover
return 0
def highlight(src):
return pygments.highlight(src, CppLexer(), HtmlFormatter())
def do_install(self, url, name):
data = self.get_json(url)
if name in data:
utils.makedirs(self.output_dir)
LOGGER.info('Downloading: ' + data[name])
zip_file = io.BytesIO()
zip_file.write(requests.get(data[name]).content)
LOGGER.info('Extracting: {0} into {1}/'.format(
name, self.output_dir))
utils.extract_all(zip_file, self.output_dir)
dest_path = os.path.join(self.output_dir, name)
else:
try:
plugin_path = utils.get_plugin_path(name)
except:
LOGGER.error("Can't find plugin " + name)
return False
utils.makedirs(self.output_dir)
dest_path = os.path.join(self.output_dir, name)
if os.path.exists(dest_path):
LOGGER.error("{0} is already installed".format(name))
return False
LOGGER.info('Copying {0} into plugins'.format(plugin_path))
shutil.copytree(plugin_path, dest_path)
reqpath = os.path.join(dest_path, 'requirements.txt')
if os.path.exists(reqpath):
LOGGER.notice('This plugin has Python dependencies.')
LOGGER.info('Installing dependencies with pip...')
try:
subprocess.check_call(('pip', 'install', '-r', reqpath))
except subprocess.CalledProcessError:
LOGGER.error('Could not install the dependencies.')
print('Contents of the requirements.txt file:\n')
with io.open(reqpath, 'r', encoding='utf-8') as fh:
print(indent(fh.read(), 4 * ' '))
print('You have to install those yourself or through a '
'package manager.')
else:
LOGGER.info('Dependency installation succeeded.')
reqnpypath = os.path.join(dest_path, 'requirements-nonpy.txt')
if os.path.exists(reqnpypath):
LOGGER.notice('This plugin has third-party '
'dependencies you need to install '
'manually.')
print('Contents of the requirements-nonpy.txt file:\n')
with io.open(reqnpypath, 'r', encoding='utf-8') as fh:
for l in fh.readlines():
i, j = l.split('::')
print(indent(i.strip(), 4 * ' '))
print(indent(j.strip(), 8 * ' '))
print()
print('You have to install those yourself or through a package '
'manager.')
confpypath = os.path.join(dest_path, 'conf.py.sample')
if os.path.exists(confpypath):
LOGGER.notice(
'This plugin has a sample config file. Integrate it with yours in order to make this plugin work!'
)
print('Contents of the conf.py.sample file:\n')
with io.open(confpypath, 'r', encoding='utf-8') as fh:
if self.site.colorful:
print(
indent(
pygments.highlight(fh.read(), PythonLexer(),
TerminalFormatter()), 4 * ' '))
else:
print(indent(fh.read(), 4 * ' '))
return True
def source_code_to_html(func):
"""Present function source code as HTML"""
code = inspect.getsource(func)
return highlight(code, PythonLexer(),
HtmlFormatter(style='algol', lineseparator='<br />'))
def _run(handle_data, initialize, before_trading_start, analyze, algofile,
algotext, defines, data_frequency, capital_base, data, bundle,
bundle_timestamp, start, end, output, print_algo, local_namespace,
environ, broker, state_filename):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`zipline.run_algo`.
"""
if algotext is not None:
if local_namespace:
ip = get_ipython() # noqa
namespace = ip.user_ns
else:
namespace = {}
for assign in defines:
try:
name, value = assign.split('=', 2)
except ValueError:
raise ValueError(
'invalid define %r, should be of the form name=value' %
assign, )
try:
# evaluate in the same namespace so names may refer to
# eachother
namespace[name] = eval(value, namespace)
except Exception as e:
raise ValueError(
'failed to execute definition for name %r: %s' %
(name, e), )
elif defines:
raise _RunAlgoError(
'cannot pass define without `algotext`',
"cannot pass '-D' / '--define' without '-t' / '--algotext'",
)
else:
namespace = {}
if algofile is not None:
algotext = algofile.read()
if print_algo:
if PYGMENTS:
highlight(
algotext,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout,
)
else:
click.echo(algotext)
if bundle is not None:
bundle_data = load(
bundle,
environ,
bundle_timestamp,
)
prefix, connstr = re.split(
r'sqlite:///',
str(bundle_data.asset_finder.engine.url),
maxsplit=1,
)
if prefix:
raise ValueError(
"invalid url %r, must begin with 'sqlite:///'" %
str(bundle_data.asset_finder.engine.url), )
env = TradingEnvironment(asset_db_path=connstr, environ=environ)
first_trading_day =\
bundle_data.equity_minute_bar_reader.first_trading_day
DataPortalClass = (partial(DataPortalLive, broker)
if broker else DataPortal)
data = DataPortalClass(
env.asset_finder,
get_calendar("NYSE"),
first_trading_day=first_trading_day,
equity_minute_reader=bundle_data.equity_minute_bar_reader,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader)
pipeline_loader = USEquityPricingLoader(
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader,
)
def choose_loader(column):
if column in USEquityPricing.columns:
return pipeline_loader
raise ValueError("No PipelineLoader registered for column %s." %
column)
else:
env = TradingEnvironment(environ=environ)
choose_loader = None
emission_rate = 'daily'
if broker:
emission_rate = 'minute'
start = pd.Timestamp.utcnow()
end = start + pd.Timedelta('2 day')
TradingAlgorithmClass = (partial(
LiveTradingAlgorithm, broker=broker, state_filename=state_filename)
if broker else TradingAlgorithm)
perf = TradingAlgorithmClass(
namespace=namespace,
env=env,
get_pipeline_loader=choose_loader,
sim_params=create_simulation_parameters(
start=start,
end=end,
capital_base=capital_base,
emission_rate=emission_rate,
data_frequency=data_frequency,
),
**{
'initialize': initialize,
'handle_data': handle_data,
'before_trading_start': before_trading_start,
'analyze': analyze,
} if algotext is None else {
'algo_filename': getattr(algofile, 'name', '<algorithm>'),
'script': algotext,
}).run(
data,
overwrite_sim_params=False,
)
if output == '-':
click.echo(str(perf))
elif output != os.devnull: # make the zipline magic not write any data
perf.to_pickle(output)
return perf
if args.emails:
print()
pb_search_results_emails(ix, search)
else:
print()
pb_search_results(ix, search)
if args.download:
fname = args.download + ".bin"
if args.name:
fname = args.name
if (ix.FILE_READ(args.download, filename=fname)):
print(
colored(
f"[{rightnow()}] Successfully downloaded the file '{fname}'.\n",
'green'))
else:
print(
colored(
f"[{rightnow()}] Failed to download item {args.download}.\n",
'red'))
if args.capabilities:
print(
colored(f"[{rightnow()}] Getting your API capabilities.\n",
'green'))
capabilities = ix.GET_CAPABILITIES()
print(
highlight(json.dumps(capabilities, indent=4), JsonLexer(),
TerminalFormatter()))
def highlightEvent(self, code, style, lang, show_copy_button):
# Build the style and lexer object
lang = self.all_langs[lang]
style = styles.get_style_by_name(style)
lexer = lexers.get_lexer_by_name(lang)
formatter = HtmlFormatter(full=True, style=style)
# Highlight the code
results = highlight(code, lexer, formatter)
print(results)
# Finished
class_style = re.findall('body \.(.*) { (.+) }', results)
highlight_code = re.findall('<div class="highlight">(<pre>.*</pre></div>)', results, re.DOTALL)[0]
background_color = re.findall('body \{ background: (.+); \}', results)
if background_color:
background_color = background_color[0]
else:
background_color = "#f8f8f8"
for item in class_style:
class_pattern = '<span class="{}"'.format(item[0])
color_pattern = '<span style="{}"'.format(item[1])
highlight_code = highlight_code.replace(class_pattern, color_pattern)
highlight_code = re.sub('<pre><span></span>', '<pre class="ccode" style="margin: 0; line-height: 125%; font-size:15px;"><span></span>', highlight_code)
print(highlight_code)
results = \
'<!-- More information can refer: https://clay-atlas.com/blog/2020/03/05/python-english-tutorial-package-pygments-highlight-code/ -->' \
'\n<div style="background: {}; ' \
'overflow:auto; width:auto; border:solid gray; ' \
'border-width:.1em .1em .1em .8em; ' \
'padding:.2em .5em;">\n '.format(background_color) + highlight_code
# Add button
if show_copy_button:
# Random ID
random_id = "".join([str(random.randint(0, 9)) for _ in range(8)])
results += """
<textarea readonly id="{}" style="position:absolute;left:-9999px">""".format(random_id) +\
code +\
"""</textarea>
<button type="button" onclick="copyEvent('""" + random_id + """')" style="float: right">COPY</button>
<script>
function copyEvent(id) {
let textarea;
let result;
try {
textarea = document.createElement('textarea');
textarea.setAttribute('readonly', true);
textarea.setAttribute('contenteditable', true);
textarea.style.position = 'fixed'; // prevent scroll from jumping to the bottom when focus is set.
textarea.value = document.getElementById(id).textContent;
document.body.appendChild(textarea);
textarea.focus();
textarea.select();
const range = document.createRange();
range.selectNodeContents(textarea);
const sel = window.getSelection();
sel.removeAllRanges();
sel.addRange(range);
textarea.setSelectionRange(0, textarea.value.length);
result = document.execCommand('copy');
} catch (err) {
console.error(err);
result = null;
} finally {
document.body.removeChild(textarea);
}
// manual copy fallback using prompt
if (!result) {
const isMac = navigator.platform.toUpperCase().indexOf('MAC') >= 0;
const copyHotkey = isMac ? '⌘C' : 'CTRL+C';
result = prompt(`Press ${copyHotkey}`, string); // eslint-disable-line no-alert
if (!result) {
return false;
}
}
return true;
}
</script>"""
return results
def log_info_dict_json(data_str, colored=True):
data_str = json.dumps(data_str, indent=4, sort_keys=True)
if colored and os.name != "nt":
data_str = highlight(data_str, lexers.JsonLexer(),
formatters.Terminal256Formatter(style="monokai"))
logging.info(data_str)
def format_source(self, src):
if not self._init_pygments():
return src
from pygments import highlight
src = self.try_to_decode(src)
return highlight(src, self._lexer, self._fmt)
def paste(request, id):
paste = Paste.objects.get(pk=id)
highlighted = highlight(paste.content, PythonLexer(), HtmlFormatter())
paste.content = highlighted
ctx = {'paste': paste}
return render(request, 'pastebin/paste-detail.jinja2', ctx)
def _colorLog(self, log: str) -> str:
color_log = highlight(
log, SASLogLexer(),
HtmlFormatter(full=True, style=SASLogStyle, lineseparator="<br>"))
return color_log
def highlight_text(no_color, text, style, lexer=TextLexer()):
formatter = TerminalTrueColorFormatter(style=style)
if no_color:
formatter = TerminalTrueColorFormatter(style=NoColorStyle)
return highlight(text, lexer, formatter)[:-1]
def highlight_snippet(snippet):
return highlight(snippet, PythonLexer(), HtmlFormatter())
def view_file(repo, identifier, filename, username=None):
""" Displays the content of a file or a tree for the specified repo.
"""
repo = pagure.lib.get_project(SESSION, repo, user=username)
if not repo:
flask.abort(404, 'Project not found')
reponame = pagure.get_repo_path(repo)
repo_obj = pygit2.Repository(reponame)
if repo_obj.is_empty:
flask.abort(404, 'Empty repo cannot have a file')
if identifier in repo_obj.listall_branches():
branchname = identifier
branch = repo_obj.lookup_branch(identifier)
commit = branch.get_object()
else:
try:
commit = repo_obj.get(identifier)
branchname = identifier
except ValueError:
if 'master' not in repo_obj.listall_branches():
flask.abort(404, 'Branch no found')
# If it's not a commit id then it's part of the filename
commit = repo_obj[repo_obj.head.target]
branchname = 'master'
if commit and not isinstance(commit, pygit2.Blob):
content = __get_file_in_tree(
repo_obj, commit.tree, filename.split('/'), bail_on_tree=True)
if not content:
flask.abort(404, 'File not found')
content = repo_obj[content.oid]
else:
content = commit
if not content:
flask.abort(404, 'File not found')
if isinstance(content, pygit2.Blob):
if content.is_binary or not pagure.lib.could_be_text(content.data):
ext = filename[filename.rfind('.'):]
if ext in (
'.gif', '.png', '.bmp', '.tif', '.tiff', '.jpg',
'.jpeg', '.ppm', '.pnm', '.pbm', '.pgm', '.webp', '.ico'):
try:
Image.open(StringIO(content.data))
output_type = 'image'
except IOError as err:
LOG.debug(
'Failed to load image %s, error: %s', filename, err
)
output_type = 'binary'
else:
output_type = 'binary'
else:
try:
lexer = guess_lexer_for_filename(
filename,
content.data
)
except (ClassNotFound, TypeError):
lexer = TextLexer()
content = highlight(
content.data,
lexer,
HtmlFormatter(
noclasses=True,
style="tango",)
)
output_type = 'file'
else:
content = sorted(content, key=lambda x: x.filemode)
output_type = 'tree'
return flask.render_template(
'file.html',
select='tree',
repo=repo,
username=username,
branchname=branchname,
filename=filename,
content=content,
output_type=output_type,
repo_admin=is_repo_admin(repo),
)
def highlight_code(ed):
addon_conf = mw.col.conf['syntax_highlighting_conf']
# Do we want line numbers? linenos is either true or false according
# to the user's preferences
linenos = addon_conf['linenos']
centerfragments = addon_conf['centerfragments']
# Do we want to use css classes or have formatting directly in HTML?
# Using css classes takes up less space and gives the user more
# customization options, but is less self-contained as it requires
# setting the styling on every note type where code is used
noclasses = not addon_conf['cssclasses']
selected_text = ed.web.selectedText()
if selected_text:
# Sometimes, self.web.selectedText() contains the unicode character
# '\u00A0' (non-breaking space). This character messes with the
# formatter for highlighted code. To correct this, we replace all
# '\u00A0' characters with regular space characters
code = selected_text.replace('\u00A0', ' ')
else:
clipboard = QApplication.clipboard()
# Get the code from the clipboard
code = clipboard.text()
langAlias = ed.codeHighlightLangAlias
# Select the lexer for the correct language
try:
my_lexer = get_lexer_by_name(langAlias, stripall=True)
except ClassNotFound as e:
print(e)
showError(ERR_LEXER, parent=ed.parentWindow)
return False
# Create html formatter object including flags for line nums and css classes
try:
my_formatter = HtmlFormatter(linenos=linenos,
noclasses=noclasses,
font_size=16,
style=STYLE)
except ClassNotFound as e:
print(e)
showError(ERR_STYLE, parent=ed.parentWindow)
return False
if linenos:
if centerfragments:
pretty_code = "".join([
"<center>",
highlight(code, my_lexer, my_formatter), "</center><br>"
])
else:
pretty_code = "".join(
[highlight(code, my_lexer, my_formatter), "<br>"])
# TODO: understand why this is neccessary
else:
if centerfragments:
pretty_code = "".join([
"<center>",
highlight(code, my_lexer, my_formatter), "</center><br>"
])
else:
pretty_code = "".join(
[highlight(code, my_lexer, my_formatter), "<br>"])
pretty_code = process_html(pretty_code)
# These two lines insert a piece of HTML in the current cursor position
ed.web.eval("document.execCommand('inserthtml', false, %s);" %
json.dumps(pretty_code))
def highlight_machine_code(mnemonics, snippet):
lexer = PatchedGasLexer
if mnemonics == 'intel':
lexer = PatchedNasmLexer
return highlight(snippet, lexer(), HtmlFormatter())
def render_listing(in_name,
out_name,
input_folder,
output_folder,
folders=[],
files=[]):
needs_ipython_css = False
if in_name and in_name.endswith('.ipynb'):
# Special handling: render ipynbs in listings (Issue #1900)
ipynb_plugin = self.site.plugin_manager.getPluginByName(
"ipynb", "PageCompiler")
if ipynb_plugin is None:
msg = "To use .ipynb files as listings, you must set up the Jupyter compiler in COMPILERS and POSTS/PAGES."
utils.LOGGER.error(msg)
raise ValueError(msg)
ipynb_compiler = ipynb_plugin.plugin_object
with open(in_name, "r", encoding="utf-8-sig") as in_file:
nb_json = ipynb_compiler._nbformat_read(in_file)
code = ipynb_compiler._compile_string(nb_json)
title = os.path.basename(in_name)
needs_ipython_css = True
elif in_name:
with open(in_name, 'r', encoding='utf-8-sig') as fd:
try:
lexer = get_lexer_for_filename(in_name)
except Exception:
try:
lexer = guess_lexer(fd.read())
except Exception:
lexer = TextLexer()
fd.seek(0)
code = highlight(
fd.read(), lexer,
utils.NikolaPygmentsHTML(in_name, linenos='table'))
title = os.path.basename(in_name)
else:
code = ''
title = os.path.split(os.path.dirname(out_name))[1]
crumbs = utils.get_crumbs(os.path.relpath(
out_name, self.kw['output_folder']),
is_file=True)
permalink = self.site.link(
'listing',
os.path.join(
input_folder,
os.path.relpath(
out_name[:-5], # remove '.html'
os.path.join(self.kw['output_folder'],
output_folder))))
if in_name:
source_link = permalink[:-5] # remove '.html'
else:
source_link = None
context = {
'code':
code,
'title':
title,
'crumbs':
crumbs,
'permalink':
permalink,
'lang':
self.kw['default_lang'],
'folders':
natsort.natsorted(folders, alg=natsort.ns.F | natsort.ns.IC),
'files':
natsort.natsorted(files, alg=natsort.ns.F | natsort.ns.IC),
'description':
title,
'source_link':
source_link,
'pagekind': ['listing'],
}
if needs_ipython_css:
# If someone does not have ipynb posts and only listings, we
# need to enable ipynb CSS for ipynb listings.
context['needs_ipython_css'] = True
self.site.render_template('listing.tmpl', out_name, context)
def handle(self, *connections, **options):
self.verbosity = options['verbosity']
if not connections:
self.stdout.write("\n".join(settings.DATABASES.keys()))
return
for conn_name in connections:
engine = settings.DATABASES[conn_name]['ENGINE']
if options['check_engine'] and engine not in [
'django.contrib.gis.db.backends.postgis',
'django.contrib.gis.db.backends.postgresql',
'etools_datamart.apps.multitenant.postgresql'
]:
raise CommandError("Engine not supported: '%s'" % engine)
dbname = settings.DATABASES[conn_name]['NAME']
uname = settings.DATABASES[conn_name]['USER']
passw = settings.DATABASES[conn_name]['PASSWORD']
host = settings.DATABASES[conn_name]['HOST']
port = settings.DATABASES[conn_name]['PORT']
cmds = [
'pg_dump',
dbname,
'--schema',
'public',
'--schema-only',
'--no-owner',
'--no-security-labels',
'--no-synchronized-snapshots',
'--no-tablespaces',
'--no-privileges',
'-U',
uname,
'-p',
str(port),
'-h',
host,
]
if passw:
cmds.extend(['-W', passw])
if options['models']:
requested = RegexList(options['models'])
models = apps.get_models()
names = [m._meta.db_table for m in models]
selection = [n for n in names if n in requested]
for tablename in selection:
cmds.extend(['-t', tablename])
if options['tables']:
for tablename in options['tables']:
cmds.extend(['-t', tablename])
if self.verbosity > 1 or options['dry_run']:
self.stdout.write(" ".join(cmds))
if not options['dry_run']:
p = subprocess.Popen(cmds,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, errors = p.communicate()
if errors:
self.stderr.write(errors.decode())
else:
if options['format']:
output = sqlparse.format(output,
strip_comments=True,
strip_whitespace=True,
indent_width=2,
output_format='sql',
reindent=True,
keyword_case='upper')
lexer = SqlLexer()
formatter = Terminal256Formatter(
style=options['style'])
if options['highlight']:
output = highlight(output, lexer, formatter)
else:
output = output.decode('utf8')
self.stdout.write(output)
def pygments_highlight(text, lang, style):
lexer = get_lexer_by_name(lang, stripall=False)
formatter = HtmlFormatter(nowrap=True, style=style)
return pygments.highlight(text, lexer, formatter)
def export_json_object(dict_obj, filename=None, logging=True):
"""
Summary:
exports object to block filesystem object
Args:
:dict_obj (dict): dictionary object
:filename (str): name of file to be exported (optional)
Returns:
True | False Boolean export status
"""
def is_tty():
"""
Summary:
Determines if output is displayed to the screen or redirected
Returns:
True if tty terminal | False is redirected, TYPE: bool
"""
return sys.stdout.isatty()
try:
if filename:
try:
with open(filename, 'w') as handle:
handle.write(json.dumps(dict_obj, indent=4,
sort_keys=True))
logger.info('%s: Wrote %s to local filesystem location' %
(inspect.stack()[0][3], filename))
handle.close()
except TypeError as e:
logger.warning('%s: object in dict not serializable: %s' %
(inspect.stack()[0][3], str(e)))
elif is_tty():
json_str = json.dumps(dict_obj, indent=4, sort_keys=True)
print(
highlight(json_str, lexers.JsonLexer(),
formatters.TerminalFormatter()).strip())
if logging:
logger.info('%s: successful export to stdout' %
inspect.stack()[0][3])
return True
else:
# print output, but not to stdout; possibly commandline redirect
print(json.dumps(dict_obj, indent=4, sort_keys=True))
except OSError as e:
logger.critical(
'%s: export_file_object: error writing to %s to filesystem. Error: %s'
% (inspect.stack()[0][3], filename, str(e)))
return False
if logging:
logger.info('export_file_object: successful export to %s' % filename)
return True
f = open(file, "r")
for line in f:
requests.packages.urllib3.disable_warnings()
try:
line = line.rstrip("\n")
http = "http://" + line
https = "https://" + line
r1 = requests.get(http, timeout=2)
r2 = requests.get(https, verify=False, timeout=2)
r1h = dict(r1.headers)
r2h = dict(r2.headers)
dict1 = json.dumps(r1h, indent=4)
dict2 = json.dumps(r2h, indent=4)
print("\n")
print("-" * 50)
print("\n")
print("[*] IP: " + line)
print("[*] HTTP Response Headers")
print("\n")
print(highlight(dict1, JsonLexer(), TerminalFormatter()))
print("\n")
print("[*] HTTPS Response Headers")
print("\n")
print(highlight(dict2, JsonLexer(), TerminalFormatter()))
print("\n\n")
except requests.exceptions.Timeout:
print("[!] Request Timed Out")
pass
else:
print("[!] Error! Must Input File Parameter")
def pylight(code):
return highlight(code, PythonLexer(), HtmlFormatter(noclasses=True))
def pygment_html_render(s, lexer=lexers.TextLexer): # noqa pylint: disable=no-member
"""Highlight text using a given Lexer"""
return highlight(s, lexer(), HtmlFormatter(linenos=True))
def historyView(id):
if request.method == 'GET':
global ConfigParser
if ConfigParser is None:
import ConfigParser
config = ConfigParser.ConfigParser()
# SVN 설정 정보 로딩
config.read('config.ini')
pSvnId = request.args.get('svnid', '')
pFlag = request.args.get('flag', '')
pOffset = int(request.args.get('offset', 0))
if pOffset < 0:
pOffset = 0
result = mongo.db.tblBranch.find_one({"_id": ObjectId(id)})
print result
# path_info = g.db.execute('select * from svn_info where s_path_id=?', [pSID]).fetchall()
svnrooturl = ''
if result['product'] == 'MF2' or result['product'] == 'AE':
svnrooturl = config.get('svn', result['product'] + '.url')
svnurl = svnrooturl + result['url']
else:
svnrooturl = config.get('svn', result['product'] + '.url')
svnurl = svnrooturl + result['url']
s = svncheck(svnrooturl, result['url'], '', '', config.get('svn', 'uid'),
config.get('svn', 'upass'))
# 마지막 리비전 조회
last_revision = s.getLastRevision()
resultlist = []
print int(result['trev']), last_revision
# DB에 저장된 최종 리비전이 리파지토리 최종 리비전 보다 작을 경우 변경 로그 조회
if int(result['trev']) < last_revision:
# cur = g.db.cursor()
# cur.execute('update svn_info set s_last_revision=? where s_path_url=?', [last_revision, path_info[0][1]])
log = s.diffrence(int(result['trev']) + 1, last_revision)
tmpUpdateRevision = result['trev']
for info in log:
print info
tmpUpdateRevision = info.revision.number
result = mongo.db.tblHistory.insert_one({
'tblBranchKey':
ObjectId(id),
'revision':
info.revision.number,
'svnid':
info.author,
'date':
time.ctime(info.date),
'comment':
info.message.decode('utf8'),
'paths': []
})
pathlist = []
for pathinfo in info.changed_paths:
difftext = None
# 파일 수정
if pathinfo.action == 'M':
# 인코딩 에러가 발생할 경우 빈값으로 셋팅
try:
difftext = s.getDiffText(info.revision.number,
pathinfo.path).decode('utf-8')
except:
difftext = ''
# 파일 추가
elif pathinfo.action == 'A':
# 인코딩 에러가 발생할 경우 빈값으로 셋팅
try:
difftext = pathinfo.action + ' ' + s.getText(
info.revision.number,
pathinfo.path).decode('utf-8')
except:
difftext = ''
# 파일 삭제
elif pathinfo.action == 'D':
difftext = pathinfo.action + ' ' + pathinfo.path
if difftext:
difftext = highlight(difftext.decode("utf-8"), DiffLexer(),
HtmlFormatter())
tmpPath = pathinfo.path.decode('utf-8')
mongo.db.tblHistory.update_one(
{'_id': ObjectId(result.inserted_id)}, {
'$push': {
'paths': {
'action': pathinfo.action,
'file': tmpPath,
'diff': difftext
}
}
})
result = mongo.db.tblBranch.find_one_and_update(
{"_id": ObjectId(id)}, {"$set": {
"trev": last_revision
}})
if pFlag == 'n':
pOffset += 10
elif pFlag == 'p':
pOffset -= 10
else:
pOffset = 0
print pSvnId
if pSvnId:
result = mongo.db.tblHistory.find({
'tblBranchKey': ObjectId(id),
"svnid": {
'$regex': pSvnId
}
}).skip(pOffset).limit(10)
else:
result = mongo.db.tblHistory.find({
'tblBranchKey': ObjectId(id)
}).skip(pOffset).limit(10)
param = {'svnurl': svnurl, 'sid': id, 'offset': pOffset}
return render_template('history/historyView.html',
data=result,
param=param)
def printer(arg):
print(
highlight(arg, lexer(),
Terminal256Formatter(style=by_colorscheme())))
def pygment_html_render(s, lexer=lexers.TextLexer):
return highlight(
s,
lexer(),
HtmlFormatter(linenos=True),
)
def _run(handle_data, initialize, before_trading_start, analyze, algofile,
algotext, defines, data_frequency, capital_base, data, bundle,
bundle_timestamp, start, end, output, print_algo, local_namespace,
environ):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`catalyst.run_algo`.
"""
if algotext is not None:
if local_namespace:
ip = get_ipython() # noqa
namespace = ip.user_ns
else:
namespace = {}
for assign in defines:
try:
name, value = assign.split('=', 2)
except ValueError:
raise ValueError(
'invalid define %r, should be of the form name=value' %
assign, )
try:
# evaluate in the same namespace so names may refer to
# eachother
namespace[name] = eval(value, namespace)
except Exception as e:
raise ValueError(
'failed to execute definition for name %r: %s' %
(name, e), )
elif defines:
raise _RunAlgoError(
'cannot pass define without `algotext`',
"cannot pass '-D' / '--define' without '-t' / '--algotext'",
)
else:
namespace = {}
if algofile is not None:
algotext = algofile.read()
if print_algo:
if PYGMENTS:
highlight(
algotext,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout,
)
else:
click.echo(algotext)
if bundle is not None:
bundles = bundle.split(',')
def get_trading_env_and_data(bundles):
env = data = None
b = 'poloniex'
if len(bundles) == 0:
return env, data
elif len(bundles) == 1:
b = bundles[0]
bundle_data = load(
b,
environ,
bundle_timestamp,
)
prefix, connstr = re.split(
r'sqlite:///',
str(bundle_data.asset_finder.engine.url),
maxsplit=1,
)
if prefix:
raise ValueError(
"invalid url %r, must begin with 'sqlite:///'" %
str(bundle_data.asset_finder.engine.url), )
open_calendar = get_calendar('OPEN')
env = TradingEnvironment(
load=partial(load_crypto_market_data,
bundle=b,
bundle_data=bundle_data,
environ=environ),
bm_symbol='USDT_BTC',
trading_calendar=open_calendar,
asset_db_path=connstr,
environ=environ,
)
first_trading_day = bundle_data.minute_bar_reader.first_trading_day
data = DataPortal(
env.asset_finder,
open_calendar,
first_trading_day=first_trading_day,
minute_reader=bundle_data.minute_bar_reader,
five_minute_reader=bundle_data.five_minute_bar_reader,
daily_reader=bundle_data.daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
)
return env, data
def get_loader_for_bundle(b):
bundle_data = load(
b,
environ,
bundle_timestamp,
)
if b == 'poloniex':
return CryptoPricingLoader(
bundle_data,
data_frequency,
CryptoPricing,
)
elif b == 'quandl':
return USEquityPricingLoader(
bundle_data,
data_frequency,
USEquityPricing,
)
raise ValueError("No PipelineLoader registered for bundle %s." % b)
loaders = [get_loader_for_bundle(b) for b in bundles]
env, data = get_trading_env_and_data(bundles)
def choose_loader(column):
for loader in loaders:
if column in loader.columns:
return loader
raise ValueError("No PipelineLoader registered for column %s." %
column)
else:
env = TradingEnvironment(environ=environ)
choose_loader = None
perf = TradingAlgorithm(
namespace=namespace,
env=env,
get_pipeline_loader=choose_loader,
sim_params=create_simulation_parameters(
start=start,
end=end,
capital_base=capital_base,
data_frequency=data_frequency,
emission_rate=data_frequency,
),
**{
'initialize': initialize,
'handle_data': handle_data,
'before_trading_start': before_trading_start,
'analyze': analyze,
} if algotext is None else {
'algo_filename': getattr(algofile, 'name', '<algorithm>'),
'script': algotext,
}).run(
data,
overwrite_sim_params=False,
)
if output == '-':
click.echo(str(perf))
elif output != os.devnull: # make the catalyst magic not write any data
perf.to_pickle(output)
return perf
def colorize_table(value,arg=None):
try:
return mark_safe(highlight(value, get_lexer(value, arg), HtmlFormatter(linenos='table')))
except ClassNotFound:
return mark_safe("<pre>%s</pre>" % escape(value))
def _code(self):
if self.__code is None:
with open(self.path) as fh:
formatted = highlight(fh.read(), CLexer(), TheFormatter)
self.__code = formatted.splitlines()
return self.__code
def run_pipeline(print_algo=True, **kwargs):
"""Runs a full zipline pipeline given configuration keyword
arguments.
1. Load data (start and end dates can be provided a strings as
well as the source and symobls).
2. Instantiate algorithm (supply either algo_text or algofile
kwargs containing initialize() and handle_data() functions). If
algofile is supplied, will try to look for algofile_analyze.py and
append it.
3. Run algorithm (supply capital_base as float).
4. Return performance dataframe.
:Arguments:
* print_algo : bool <default=True>
Whether to print the algorithm to command line. Will use
pygments syntax coloring if pygments is found.
"""
start = kwargs['start']
end = kwargs['end']
# Compare against None because strings/timestamps may have been given
if start is not None:
start = pd.Timestamp(start, tz='UTC')
if end is not None:
end = pd.Timestamp(end, tz='UTC')
# Fail out if only one bound is provided
if ((start is None) or (end is None)) and (start != end):
raise PipelineDateError(start=start, end=end)
# Check if start and end are provided, and if the sim_params need to read
# a start and end from the DataSource
if start is None:
overwrite_sim_params = True
else:
overwrite_sim_params = False
symbols = kwargs['symbols'].split(',')
asset_identifier = kwargs['metadata_index']
# Pull asset metadata
asset_metadata = kwargs.get('asset_metadata', None)
asset_metadata_path = kwargs['metadata_path']
# Read in a CSV file, if applicable
if asset_metadata_path is not None:
if os.path.isfile(asset_metadata_path):
asset_metadata = pd.read_csv(asset_metadata_path,
index_col=asset_identifier)
source_arg = kwargs['source']
source_time_column = kwargs['source_time_column']
if source_arg is None:
raise NoSourceError()
elif source_arg == 'yahoo':
source = zipline.data.load_bars_from_yahoo(stocks=symbols,
start=start,
end=end)
elif os.path.isfile(source_arg):
source = zipline.data.load_prices_from_csv(
filepath=source_arg, identifier_col=source_time_column)
elif os.path.isdir(source_arg):
source = zipline.data.load_prices_from_csv_folder(
folderpath=source_arg, identifier_col=source_time_column)
else:
raise NotImplementedError('Source %s not implemented.' %
kwargs['source'])
algo_text = kwargs.get('algo_text', None)
if algo_text is None:
# Expect algofile to be set
algo_fname = kwargs['algofile']
with open(algo_fname, 'r') as fd:
algo_text = fd.read()
analyze_fname = os.path.splitext(algo_fname)[0] + '_analyze.py'
if os.path.exists(analyze_fname):
with open(analyze_fname, 'r') as fd:
# Simply append
algo_text += fd.read()
if print_algo:
if PYGMENTS:
highlight(algo_text,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout)
else:
print_(algo_text)
algo = zipline.TradingAlgorithm(script=algo_text,
namespace=kwargs.get('namespace', {}),
capital_base=float(kwargs['capital_base']),
algo_filename=kwargs.get('algofile'),
asset_metadata=asset_metadata,
identifiers=symbols,
start=start,
end=end)
perf = algo.run(source, overwrite_sim_params=overwrite_sim_params)
output_fname = kwargs.get('output', None)
if output_fname is not None:
perf.to_pickle(output_fname)
return perf