def pretty_print_body(fmt, body):
try:
if fmt.lower() == 'json':
d = json.loads(body.strip())
s = json.dumps(d, indent=4, sort_keys=True)
print pygments.highlight(s, JsonLexer(), TerminalFormatter())
elif fmt.lower() == 'form':
qs = repeatable_parse_qs(body)
for k, v in qs.all_pairs():
s = Colors.GREEN
s += '%s: ' % urllib.unquote(k)
s += Colors.ENDC
s += urllib.unquote(v)
print s
elif fmt.lower() == 'text':
print body
elif fmt.lower() == 'xml':
import xml.dom.minidom
xml = xml.dom.minidom.parseString(body)
print pygments.highlight(xml.toprettyxml(), XmlLexer(),
TerminalFormatter())
else:
raise PappyException('"%s" is not a valid format' % fmt)
except PappyException as e:
raise e
except:
raise PappyException('Body could not be parsed as "%s"' % fmt)
def pretty_print_body(fmt, body):
try:
bstr = body.decode()
if fmt.lower() == 'json':
d = json.loads(bstr.strip())
s = json.dumps(d, indent=4, sort_keys=True)
print(pygments.highlight(s, JsonLexer(), TerminalFormatter()))
elif fmt.lower() == 'form':
qs = parse_qs(bstr, keep_blank_values=True)
for k, vs in qs.items():
for v in vs:
s = Colors.GREEN
s += '%s: ' % unquote(k)
s += Colors.ENDC
if v == '':
s += Colors.RED
s += 'EMPTY'
s += Colors.ENDC
else:
s += unquote(v)
print(s)
elif fmt.lower() == 'text':
print(bstr)
elif fmt.lower() == 'xml':
import xml.dom.minidom
xml = xml.dom.minidom.parseString(bstr)
print(pygments.highlight(xml.toprettyxml(), XmlLexer(), TerminalFormatter()))
else:
raise CommandError('"%s" is not a valid format' % fmt)
except CommandError as e:
raise e
except Exception as e:
raise CommandError('Body could not be parsed as "{}": {}'.format(fmt, e))
def createLexers(self):
lex = {}
lex['.c'] = CFamilyLexer()
lex['.h'] = CFamilyLexer()
lex['.cpp'] = CppLexer()
lex['.hpp'] = CppLexer()
lex['.css'] = CssLexer()
lex['.sass'] = SassLexer()
lex['.yaml'] = YamlLexer()
lex['.yml'] = YamlLexer()
lex['.json'] = JsonLexer()
lex['.cs'] = CSharpLexer()
lex['.fs'] = FSharpLexer()
lex['.e'] = EiffelLexer()
lex['.erl'] = ErlangLexer()
lex['.hrl'] = ErlangLexer()
lex['.es'] = ErlangLexer()
lex['.f03'] = FortranLexer()
lex['.f90'] = FortranLexer()
lex['.F03'] = FortranLexer()
lex['.F90'] = FortranLexer()
lex['.go'] = GoLexer()
lex['.hs'] = HaskellLexer()
lex['.v'] = VerilogLexer()
lex['.vhdl'] = VhdlLexer()
lex['.vhd'] = VhdlLexer()
lex['.html'] = HtmlLexer()
lex['.htm'] = HtmlLexer()
lex['.xhtml'] = HtmlLexer()
lex['.xml'] = XmlLexer()
lex['.js'] = JavascriptLexer()
lex['.tex'] = TypeScriptLexer()
lex['.coffee'] = CoffeeScriptLexer()
lex['.java'] = JavaLexer()
lex['.scala'] = ScalaLexer()
lex['.kt'] = KotlinLexer()
lex['.ktm'] = KotlinLexer()
lex['.kts'] = KotlinLexer()
lex['.lisp'] = CommonLispLexer()
lex['make'] = MakefileLexer()
lex['Make'] = MakefileLexer()
lex['CMake'] = CMakeLexer()
lex['cmake'] = CMakeLexer()
lex['.m'] = MatlabLexer()
lex['.mat'] = MatlabLexer()
lex['.dpr'] = DelphiLexer()
lex['.perl'] = PerlLexer()
lex['.php'] = PhpLexer()
lex['.pr'] = PrologLexer()
lex['.py'] = Python3Lexer()
lex['.rb'] = RubyLexer()
lex['.sh'] = BashLexer()
lex['.sql'] = MySqlLexer()
lex['.mysql'] = MySqlLexer()
lex['.tcl'] = TclLexer()
lex['.awk'] = AwkLexer()
return lex
def nice_xml(self):
"""returns html formatted report xml"""
xml_out = self.report_xml
# if it's not got many newlines
# it's probably just on one, so prettyfy it.
if self.report_xml.count("\n") < 2:
dom = xml.dom.minidom.parseString(xml_out)
xml_out = dom.toprettyxml(indent=" ")
xml_html = highlight(xml_out, XmlLexer(), HtmlFormatter(noclasses=True))
return mark_safe(xml_html)
def apply_xpath(infile, xpath_query=None, colorize=False):
try:
parsed = etree.parse(infile, etree.XMLParser(remove_blank_text=True))
except etree.XMLSyntaxError:
parsed = etree.parse(infile, etree.HTMLParser(remove_blank_text=True))
if xpath_query:
matches = parsed.xpath(xpath_query)
results = wrap_in_results(matches)
output = etree.tostring(results, pretty_print=True, encoding='unicode')
else:
output = etree.tostring(parsed, pretty_print=True, encoding='unicode')
formatter = TerminalFormatter() if colorize else NullFormatter()
return highlight(output, XmlLexer(), formatter)
def lexer(self):
def strip(lex):
return [x[2:] for x in lex.filenames]
if self.extension in strip(RenpyLexer):
return RenpyLexer()
elif self.extension in strip(DiffLexer):
return DiffLexer()
elif self.extension in strip(JsonLexer):
return JsonLexer()
elif self.extension in strip(YamlLexer):
return YamlLexer()
elif self.extension in strip(MarkdownLexer):
return MarkdownLexer()
elif self.extension in strip(XmlLexer):
return XmlLexer()
else:
return NullLexer()
def show(uuid):
"""Show a validation error in its XML context."""
validation_error = ValidationError.query.get_or_404(str(uuid))
filepath = join(current_app.config['MEDIA_FOLDER'],
validation_error.supplied_data.original_file)
if not exists(filepath):
flash('That dataset is no longer available', 'danger')
return redirect(url_for('public.home'))
match = re.search(r'/iati-(?:activity|organisation)\[(\d+)\]',
validation_error.path)
act_num = int(match.group(1)) - 1
dataset = iatikit.Dataset(filepath)
dataset.unminify_xml()
activity = dataset.activities[act_num]
start_line = activity.etree.sourceline
line = validation_error.line - start_line + 1
highlighted_xml = highlight(
activity.xml, XmlLexer(),
HtmlFormatter(linenos='inline',
lineanchors='L',
hl_lines=[line],
linenostart=start_line))
return render_template('public/show_error.html', code=highlighted_xml)
def run(self):
while True:
# wait for next element
element = self.queue.get()
if element is None or element.tag is Comment:
self.return_signal.emit("")
continue
element = XML(tostring(element))
# process the element
deannotate(element, cleanup_namespaces=True)
code = tostring(element,
encoding="Unicode",
pretty_print=True,
xml_declaration=False)
self.return_signal.emit(
highlight(
code, XmlLexer(),
HtmlFormatter(noclasses=True,
style="autumn",
linenos="table")))
def highlight_xml(xml_str):
# Highlights a string containing XML, using terminal color codes
return highlight(xml_str, XmlLexer(), TerminalFormatter())
def highlight_xml(xml_str):
"""Highlight a string containing XML, using terminal color codes."""
return highlight(xml_str, XmlLexer(), TerminalFormatter()).rstrip()
def pretty_print_xml(self, xml):
print(highlight(xml.toprettyxml(), XmlLexer(), TerminalFormatter()))
from gui.scrollbar_autohide import AutoHideScrollbar
from collections import deque
from console import ConsoleUi, Processing
# python3 -m pip download --only-binary=wheel pygments
# sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)) + '/vendor/pygments')
# pylint: disable=too-many-ancestors
# TODO: Legg inn sjekk på shebang også: https://kite.com/python/docs/pygments.lexers.ruby.shebang_matches
lexer_from_ext = { # WAIT: Denne bør hentes fra configfil heller
'py': PythonLexer(),
'pyw': PythonLexer(),
'htm': HtmlLexer(),
'html': HtmlLexer(),
'xml': XmlLexer(), # WAIT: Se om finnes bedre
'xsl': XmlLexer(),
'rss': XmlLexer(),
'xslt': XmlLexer(),
'xsd': XmlLexer(),
'wsdl': XmlLexer(),
'php': HtmlPhpLexer(),
'php5': HtmlPhpLexer(),
'pl': Perl6Lexer(),
'pm': Perl6Lexer(),
'nqp': Perl6Lexer(),
'p6': Perl6Lexer(),
'6pl': Perl6Lexer(),
'p6l': Perl6Lexer(),
'pl6': Perl6Lexer(),
'p6m': Perl6Lexer(),
def ashtml(self):
"""Contenido del informe como HTML resaltado"""
return highlight(self.xml,
XmlLexer(),
HtmlFormatter(noclasses=True))
def highlight_xform(xml, **options):
formatter = XFormFormatter(style='friendly', **options)
return highlight(xml, XmlLexer(), formatter)
class Timex3Parser(object):
_formatter = HtmlFormatter(encoding='utf-8',
nowrap=False,
style='emacs',
linenos=False,
prestyles='font-size: 12px;')
_lexer = XmlLexer()
_css = _formatter.get_style_defs()
_re_hash = re.compile(r'(?:\A|\s)(?P<hash>#(?:\w|\.|_)+)(?:\s|\Z)',
re.IGNORECASE + re.MULTILINE)
def __init__(self):
pass
@classmethod
def parse_task(cls,
task,
include_dates=True,
include_title=True,
include_labels=True,
include_code=False):
if include_code:
task.code = cls.highlight(task.content) if task.content else ''
if include_dates:
task.dates = cls.parse_dates(task.content) if task.content else []
if include_labels:
task.labels = ','.join(cls.parse_hash_tags(task.text))
if include_title: task.title = cls.parse_text(task.title) or task.title
return task
@classmethod
def parse_text(cls, text):
simple_text = None
return simple_text
@classmethod
def parse_hash_tags(cls, text):
"""
extract hash tags ("#" leading words) from given text
:param text:
:return:
"""
tags = []
if text:
for m in cls._re_hash.findall(text):
tags.append(m[1:].lower()) # ignore leading "#"
return tags
@staticmethod
def parse_dates(timex3_xml):
"""
extract all mentioned dates.
:param timex3_xml:
:return:
"""
dates = []
cal = parsedatetime.Calendar()
xml = ET.fromstring(timex3_xml)
# base time
node = xml.findtext("./DATE")
base_time = datetime.strptime(node, '%a %b %d %H:%M:%S %z %Y')
'''
Fail dates:
DURATION PT24H 24 hours
DURATION P1D day
SET P1D each day
DURATION P1D day
SET P1D Daily
DURATION PT1M a Minute
DURATION P1D day
DURATION P1W Week
DURATION P1W Week
DURATION P1D day
DURATION P1D day
DURATION P1D Day
SET P1D Daily
DURATION P1D Day
SET P1D Daily
SET P1W Weekly
SET P1W Weekly
DURATION P22Y 22-year-old
SET P1D daily
DURATION P1W week
SET TMO every morning
DURATION P1W week
SET TMO every morning
DURATION P1W week
SET TMO every morning
DURATION P1D a day
TIME T19:00 7pm
DURATION PXY years
SET P1D Daily
SET P1D daily
DURATION P1D One day
DURATION P1D day
DURATION P1D a day
DURATION P1W a week
DURATION P1M a month
DURATION P1M a month
DURATION PXY years
DURATION P1D DAY
TIME T19:00 7pm
TIME TMO morning
DURATION P21D Less than 21 days
DURATION PT19S the last 19 seconds
SET P1D Daily
DURATION P1D day
DURATION PXY years
DURATION P1D Day
SET P1D Daily
DURATION P1D Day
SET P1D Daily
DURATION PXD days
DURATION PT3M 3 mins
DURATION P1W a week
SET P1D daily
SET P1D daily
TIME TMO Morning
DURATION P1W week
DURATION PT40S more than 40 seconds
SET P1D daily
DURATION P1W Week
SET P1D daily
SET P1D daily
DURATION P1W Week
DURATION P9W 9 weeks
DURATION P9W 9 weeks
DURATION P1D one day
DURATION P1D day
DURATION P1D Day
SET P1D Daily
DURATION P1D day
DURATION PXD few days
SET P1D every day
SET P1D every day
SET P1W Weekly
TIME TMO the next morning
TIME TEV evening
DURATION P1D day
TIME TMO morning
'''
for node in xml.findall("./TEXT/TIMEX3"):
dt = {}
dt['text'] = node.text
for k, v in node.items():
dt[k] = v
tp = dt['type']
if dt['text']:
if tp == 'DATE':
try:
# TODO: need to be parsed depends on date (when crawling)
value = dt['value']
# if value == 'PRESENT_REF':
# v = dateparser.parse(dt['date'])
# elif value == 'PAST_REF':
# v = None
# pass
# elif value == 'FUTURE_REF':
# v = None
# pass
# elif value == 'XXXX':
# v = dateparser.parse(dt['text'])
if value.startswith('XXXX-'):
# token = value.split('-')[1]
v = None
pass
else:
v = datetime(*cal.parse(
dt['text'], sourceTime=base_time)[0][:6])
dt['v'] = v
except Exception as err:
log.warn(err)
dt['v'] = None
elif tp == 'TIME':
dt['v'] = datetime(
*cal.parse(dt['text'], sourceTime=base_time)[0][:6])
elif tp == 'DURATION':
pass
elif tp == 'SET':
pass
else:
pass
if 'v' in dt and dt['v']:
dates.append(dt)
else:
log.warn('Fail/ignored date: %s' % dt)
return dates
@classmethod
def highlight(cls, src):
code = highlight(src, cls._lexer, cls._formatter)
return code
@classmethod
def get_highlight_css(cls):
return cls._css
if __name__ == "__main__":
extens = ""
try:
extens = sys.argv[1].split('.')[1]
except IndexError:
pass
if extens == "py" or extens == "pyw" or extens == "sc" or extens == "sage" or extens == "tac":
ui_core = CoreUI(lexer=PythonLexer())
elif extens == "txt" or extens == "README" or extens == "text":
ui_core = CoreUI(lexer=TextLexer())
elif extens == "htm" or extens == "html" or extens == "css" or extens == "js" or extens == "md":
ui_core = CoreUI(lexer=HtmlLexer())
elif extens == "xml" or extens == "xsl" or extens == "rss" or extens == "xslt" or extens == "xsd" or extens == "wsdl" or extens == "wsf":
ui_core = CoreUI(lexer=XmlLexer())
elif extens == "php" or extens == "php5":
ui_core = CoreUI(lexer=HtmlPhpLexer())
elif extens == "pl" or extens == "pm" or extens == "nqp" or extens == "p6" or extens == "6pl" or extens == "p6l" or extens == "pl6" or extens == "pm" or extens == "p6m" or extens == "pm6" or extens == "t":
ui_core = CoreUI(lexer=Perl6Lexer())
elif extens == "rb" or extens == "rbw" or extens == "rake" or extens == "rbx" or extens == "duby" or extens == "gemspec":
ui_core = CoreUI(lexer=RubyLexer())
elif extens == "ini" or extens == "init":
ui_core = CoreUI(lexer=IniLexer())
elif extens == "conf" or extens == "cnf" or extens == "config":
ui_core = CoreUI(lexer=ApacheConfLexer())
elif extens == "sh" or extens == "cmd" or extens == "bashrc" or extens == "bash_profile":
ui_core = CoreUI(lexer=BashLexer())
elif extens == "diff" or extens == "patch":
ui_core = CoreUI(lexer=DiffLexer())
elif extens == "cs":