def test_tab_indentation(self):
# Mixed tabs and spaces...
self.assertEqual(list(source_token_lines(MIXED_WS)),
[
[('key', 'def'), ('ws', ' '), ('nam', 'hello'), ('op', '('),
('op', ')'), ('op', ':')],
[('ws', ' '), ('nam', 'a'), ('op', '='),
('str', '"Hello world!"')],
[('ws', ' '), ('nam', 'b'), ('op', '='),
('str', '"indented"')],
])
def test_simple(self):
self.assertEqual(list(source_token_lines(SIMPLE)),
[
[('com', "# yay!")],
[('key', 'def'), ('ws', ' '), ('nam', 'foo'), ('op', '('),
('op', ')'), ('op', ':')],
[('ws', ' '), ('nam', 'say'), ('op', '('),
('str', "'two = %d'"), ('ws', ' '), ('op', '%'),
('ws', ' '), ('num', '2'), ('op', ')')]
])
self.check_tokenization(SIMPLE)
def check_tokenization(self, source):
"""Tokenize `source`, then put it back together, should be the same."""
tokenized = ""
for line in source_token_lines(source):
text = "".join([t for _,t in line])
tokenized += text + "\n"
# source_token_lines doesn't preserve trailing spaces, so trim all that
# before comparing.
source = source.replace('\r\n', '\n')
source = re.sub("(?m)[ \t]+$", "", source)
tokenized = re.sub("(?m)[ \t]+$", "", tokenized)
self.assertMultiLineEqual(source, tokenized)
def html_file(self, cu, analysis):
"""Generate an HTML file for one source file."""
source = cu.source_file().read()
nums = analysis.numbers
missing_branch_arcs = analysis.missing_branch_arcs()
n_par = 0 # accumulated below.
arcs = self.arcs
# These classes determine which lines are highlighted by default.
c_run = " run hide_run"
c_exc = " exc"
c_mis = " mis"
c_par = " par" + c_run
lines = []
for lineno, line in enumerate(source_token_lines(source)):
lineno += 1 # 1-based line numbers.
# Figure out how to mark this line.
line_class = ""
annotate_html = ""
annotate_title = ""
if lineno in analysis.statements:
line_class += " stm"
if lineno in analysis.excluded:
line_class += c_exc
elif lineno in analysis.missing:
line_class += c_mis
elif self.arcs and lineno in missing_branch_arcs:
line_class += c_par
n_par += 1
annlines = []
for b in missing_branch_arcs[lineno]:
if b == -1:
annlines.append("exit")
else:
annlines.append(str(b))
annotate_html = " ".join(annlines)
if len(annlines) > 1:
annotate_title = "no jumps to these line numbers"
elif len(annlines) == 1:
annotate_title = "no jump to this line number"
elif lineno in analysis.statements:
line_class += c_run
# Build the HTML for the line
html = ""
for tok_type, tok_text in line:
if tok_type == "ws":
html += escape(tok_text)
else:
tok_html = escape(tok_text) or ' '
html += "<span class='%s'>%s</span>" % (tok_type, tok_html)
lines.append({
'html': html,
'number': lineno,
'class': line_class.strip() or "pln",
'annotate': annotate_html,
'annotate_title': annotate_title,
})
# Write the HTML page for this file.
html_filename = cu.flat_rootname() + ".html"
html_path = os.path.join(self.directory, html_filename)
html = spaceless(self.source_tmpl.render(locals()))
fhtml = open(html_path, 'w')
fhtml.write(html)
fhtml.close()
# Save this file's information for the index file.
self.files.append({
'nums': nums,
'par': n_par,
'html_filename': html_filename,
'cu': cu,
})
def source_token_lines(self, source):
return source_token_lines(source)
def html_file(self, cu, analysis):
"""Generate an HTML file for one source file."""
source_file = cu.source_file()
try:
source = source_file.read()
finally:
source_file.close()
# Find out if the file on disk is already correct.
flat_rootname = cu.flat_rootname()
this_hash = self.file_hash(source, cu)
that_hash = self.status.file_hash(flat_rootname)
if this_hash == that_hash:
# Nothing has changed to require the file to be reported again.
self.files.append(self.status.index_info(flat_rootname))
return
self.status.set_file_hash(flat_rootname, this_hash)
# If need be, determine the encoding of the source file. We use it
# later to properly write the HTML.
if sys.version_info < (3, 0):
encoding = source_encoding(source)
# Some UTF8 files have the dreaded UTF8 BOM. If so, junk it.
if encoding.startswith("utf-8") and source[:3] == "\xef\xbb\xbf":
source = source[3:]
encoding = "utf-8"
# Get the numbers for this file.
nums = analysis.numbers
if self.arcs:
missing_branch_arcs = analysis.missing_branch_arcs()
# These classes determine which lines are highlighted by default.
c_run = "run hide_run"
c_exc = "exc"
c_mis = "mis"
c_par = "par " + c_run
lines = []
for lineno, line in enumerate(source_token_lines(source)):
lineno += 1 # 1-based line numbers.
# Figure out how to mark this line.
line_class = []
annotate_html = ""
annotate_title = ""
if lineno in analysis.statements:
line_class.append("stm")
if lineno in analysis.excluded:
line_class.append(c_exc)
elif lineno in analysis.missing:
line_class.append(c_mis)
elif self.arcs and lineno in missing_branch_arcs:
line_class.append(c_par)
annlines = []
for b in missing_branch_arcs[lineno]:
if b < 0:
annlines.append("exit")
else:
annlines.append(str(b))
annotate_html = " ".join(annlines)
if len(annlines) > 1:
annotate_title = "no jumps to these line numbers"
elif len(annlines) == 1:
annotate_title = "no jump to this line number"
elif lineno in analysis.statements:
line_class.append(c_run)
# Build the HTML for the line
html = []
for tok_type, tok_text in line:
if tok_type == "ws":
html.append(escape(tok_text))
else:
tok_html = escape(tok_text) or ' '
html.append(
"<span class='%s'>%s</span>" % (tok_type, tok_html)
)
lines.append({
'html': ''.join(html),
'number': lineno,
'class': ' '.join(line_class) or "pln",
'annotate': annotate_html,
'annotate_title': annotate_title,
})
# Write the HTML page for this file.
html = spaceless(self.source_tmpl.render({
'c_exc': c_exc, 'c_mis': c_mis, 'c_par': c_par, 'c_run': c_run,
'arcs': self.arcs, 'extra_css': self.extra_css,
'cu': cu, 'nums': nums, 'lines': lines,
}))
if sys.version_info < (3, 0):
html = html.decode(encoding)
html_filename = flat_rootname + ".html"
html_path = os.path.join(self.directory, html_filename)
self.write_html(html_path, html)
# Save this file's information for the index file.
index_info = {
'nums': nums,
'html_filename': html_filename,
'name': cu.name,
}
self.files.append(index_info)
self.status.set_index_info(flat_rootname, index_info)
def test_missing_final_newline(self):
# We can tokenize source that is missing the final newline.
assert list(source_token_lines(SIMPLE.rstrip())) == SIMPLE_TOKENS
def test_simple(self):
assert list(source_token_lines(SIMPLE)) == SIMPLE_TOKENS
self.check_tokenization(SIMPLE)
def test_missing_final_newline(self):
# We can tokenize source that is missing the final newline.
self.assertEqual(list(source_token_lines(SIMPLE.rstrip())), SIMPLE_TOKENS)
def test_tab_indentation(self):
# Mixed tabs and spaces...
self.assertEqual(list(source_token_lines(MIXED_WS)), MIXED_WS_TOKENS)
def test_simple(self):
self.assertEqual(list(source_token_lines(SIMPLE)), SIMPLE_TOKENS)
self.check_tokenization(SIMPLE)
def html_file(self, cu, analysis):
source_file = cu.source_file()
try:
source = source_file.read()
finally:
source_file.close()
flat_rootname = cu.flat_rootname()
this_hash = self.file_hash(source, cu)
that_hash = self.status.file_hash(flat_rootname)
if this_hash == that_hash:
self.files.append(self.status.index_info(flat_rootname))
return
self.status.set_file_hash(flat_rootname, this_hash)
if sys.version_info < (3, 0):
encoding = source_encoding(source)
if encoding.startswith('utf-8') and source[:3] == '\xef\xbb\xbf':
source = source[3:]
encoding = 'utf-8'
nums = analysis.numbers
missing_branch_arcs = analysis.missing_branch_arcs()
c_run = 'run hide_run'
c_exc = 'exc'
c_mis = 'mis'
c_par = 'par ' + c_run
lines = []
for lineno, line in enumerate(source_token_lines(source)):
lineno += 1
line_class = []
annotate_html = ''
annotate_title = ''
if lineno in analysis.statements:
line_class.append('stm')
if lineno in analysis.excluded:
line_class.append(c_exc)
elif lineno in analysis.missing:
line_class.append(c_mis)
elif self.arcs and lineno in missing_branch_arcs:
line_class.append(c_par)
annlines = []
for b in missing_branch_arcs[lineno]:
if b < 0:
annlines.append('exit')
else:
annlines.append(str(b))
annotate_html = ' '.join(annlines)
if len(annlines) > 1:
annotate_title = 'no jumps to these line numbers'
elif len(annlines) == 1:
annotate_title = 'no jump to this line number'
elif lineno in analysis.statements:
line_class.append(c_run)
html = []
for tok_type, tok_text in line:
if tok_type == 'ws':
html.append(escape(tok_text))
else:
tok_html = escape(tok_text) or ' '
html.append("<span class='%s'>%s</span>" % (tok_type, tok_html))
lines.append({'html': ''.join(html),
'number': lineno,
'class': ' '.join(line_class) or 'pln',
'annotate': annotate_html,
'annotate_title': annotate_title})
html = spaceless(self.source_tmpl.render({'c_exc': c_exc,
'c_mis': c_mis,
'c_par': c_par,
'c_run': c_run,
'arcs': self.arcs,
'extra_css': self.extra_css,
'cu': cu,
'nums': nums,
'lines': lines}))
if sys.version_info < (3, 0):
html = html.decode(encoding)
html_filename = flat_rootname + '.html'
html_path = os.path.join(self.directory, html_filename)
self.write_html(html_path, html)
index_info = {'nums': nums,
'html_filename': html_filename,
'name': cu.name}
self.files.append(index_info)
self.status.set_index_info(flat_rootname, index_info)
def html_file(self, cu, analysis):
"""Generate an HTML file for one source file."""
source_file = cu.source_file()
try:
source = source_file.read()
finally:
source_file.close()
# Find out if the file on disk is already correct.
flat_rootname = cu.flat_rootname()
this_hash = self.file_hash(source, cu)
that_hash = self.status.file_hash(flat_rootname)
if this_hash == that_hash:
# Nothing has changed to require the file to be reported again.
self.files.append(self.status.index_info(flat_rootname))
return
self.status.set_file_hash(flat_rootname, this_hash)
# If need be, determine the encoding of the source file. We use it
# later to properly write the HTML.
if sys.version_info < (3, 0):
encoding = source_encoding(source)
# Some UTF8 files have the dreaded UTF8 BOM. If so, junk it.
if encoding.startswith("utf-8") and source[:3] == "\xef\xbb\xbf":
source = source[3:]
encoding = "utf-8"
# Get the numbers for this file.
nums = analysis.numbers
if self.arcs:
missing_branch_arcs = analysis.missing_branch_arcs()
# These classes determine which lines are highlighted by default.
c_run = "run hide_run"
c_exc = "exc"
c_mis = "mis"
c_par = "par " + c_run
lines = []
for lineno, line in enumerate(source_token_lines(source)):
lineno += 1 # 1-based line numbers.
# Figure out how to mark this line.
line_class = []
annotate_html = ""
annotate_title = ""
if lineno in analysis.statements:
line_class.append("stm")
if lineno in analysis.excluded:
line_class.append(c_exc)
elif lineno in analysis.missing:
line_class.append(c_mis)
elif self.arcs and lineno in missing_branch_arcs:
line_class.append(c_par)
annlines = []
for b in missing_branch_arcs[lineno]:
if b < 0:
annlines.append("exit")
else:
annlines.append(str(b))
annotate_html = " ".join(annlines)
if len(annlines) > 1:
annotate_title = "no jumps to these line numbers"
elif len(annlines) == 1:
annotate_title = "no jump to this line number"
elif lineno in analysis.statements:
line_class.append(c_run)
# Build the HTML for the line
html = []
for tok_type, tok_text in line:
if tok_type == "ws":
html.append(escape(tok_text))
else:
tok_html = escape(tok_text) or ' '
html.append("<span class='%s'>%s</span>" %
(tok_type, tok_html))
lines.append({
'html': ''.join(html),
'number': lineno,
'class': ' '.join(line_class) or "pln",
'annotate': annotate_html,
'annotate_title': annotate_title,
})
# Write the HTML page for this file.
html = spaceless(
self.source_tmpl.render({
'c_exc': c_exc,
'c_mis': c_mis,
'c_par': c_par,
'c_run': c_run,
'arcs': self.arcs,
'extra_css': self.extra_css,
'cu': cu,
'nums': nums,
'lines': lines,
}))
if sys.version_info < (3, 0):
html = html.decode(encoding)
html_filename = flat_rootname + ".html"
html_path = os.path.join(self.directory, html_filename)
self.write_html(html_path, html)
# Save this file's information for the index file.
index_info = {
'nums': nums,
'html_filename': html_filename,
'name': cu.name,
}
self.files.append(index_info)
self.status.set_index_info(flat_rootname, index_info)
def html_file(self, cu, analysis):
"""Generate an HTML file for one source file."""
source = cu.source_file().read()
nums = analysis.numbers
missing_branch_arcs = analysis.missing_branch_arcs()
n_par = 0 # accumulated below.
arcs = self.arcs
# These classes determine which lines are highlighted by default.
c_run = "run hide_run"
c_exc = "exc"
c_mis = "mis"
c_par = "par " + c_run
lines = []
for lineno, line in enumerate(source_token_lines(source)):
lineno += 1 # 1-based line numbers.
# Figure out how to mark this line.
line_class = []
annotate_html = ""
annotate_title = ""
if lineno in analysis.statements:
line_class.append("stm")
if lineno in analysis.excluded:
line_class.append(c_exc)
elif lineno in analysis.missing:
line_class.append(c_mis)
elif self.arcs and lineno in missing_branch_arcs:
line_class.append(c_par)
n_par += 1
annlines = []
for b in missing_branch_arcs[lineno]:
if b < 0:
annlines.append("exit")
else:
annlines.append(str(b))
annotate_html = " ".join(annlines)
if len(annlines) > 1:
annotate_title = "no jumps to these line numbers"
elif len(annlines) == 1:
annotate_title = "no jump to this line number"
elif lineno in analysis.statements:
line_class.append(c_run)
# Build the HTML for the line
html = []
for tok_type, tok_text in line:
if tok_type == "ws":
html.append(escape(tok_text))
else:
tok_html = escape(tok_text) or ' '
html.append("<span class='%s'>%s</span>" %
(tok_type, tok_html))
lines.append({
'html': ''.join(html),
'number': lineno,
'class': ' '.join(line_class) or "pln",
'annotate': annotate_html,
'annotate_title': annotate_title,
})
# Write the HTML page for this file.
html_filename = cu.flat_rootname() + ".html"
html_path = os.path.join(self.directory, html_filename)
html = spaceless(self.source_tmpl.render(locals()))
fhtml = open(html_path, 'w')
fhtml.write(html)
fhtml.close()
# Save this file's information for the index file.
self.files.append({
'nums': nums,
'par': n_par,
'html_filename': html_filename,
'cu': cu,
})
def html_file(self, cu, analysis):
"""Generate an HTML file for one source file."""
source_file = cu.source_file()
try:
source = source_file.read()
finally:
source_file.close()
# Find out if the file on disk is already correct.
flat_rootname = cu.flat_rootname()
this_hash = self.file_hash(source, cu)
that_hash = self.status.file_hash(flat_rootname)
if this_hash == that_hash:
# Nothing has changed to require the file to be reported again.
self.files.append(self.status.index_info(flat_rootname))
return
self.status.set_file_hash(flat_rootname, this_hash)
nums = analysis.numbers
missing_branch_arcs = analysis.missing_branch_arcs()
n_par = 0 # accumulated below.
arcs = self.arcs
# These classes determine which lines are highlighted by default.
c_run = "run hide_run"
c_exc = "exc"
c_mis = "mis"
c_par = "par " + c_run
lines = []
for lineno, line in enumerate(source_token_lines(source)):
lineno += 1 # 1-based line numbers.
# Figure out how to mark this line.
line_class = []
annotate_html = ""
annotate_title = ""
if lineno in analysis.statements:
line_class.append("stm")
if lineno in analysis.excluded:
line_class.append(c_exc)
elif lineno in analysis.missing:
line_class.append(c_mis)
elif self.arcs and lineno in missing_branch_arcs:
line_class.append(c_par)
n_par += 1
annlines = []
for b in missing_branch_arcs[lineno]:
if b < 0:
annlines.append("exit")
else:
annlines.append(str(b))
annotate_html = " ".join(annlines)
if len(annlines) > 1:
annotate_title = "no jumps to these line numbers"
elif len(annlines) == 1:
annotate_title = "no jump to this line number"
elif lineno in analysis.statements:
line_class.append(c_run)
# Build the HTML for the line
html = []
for tok_type, tok_text in line:
if tok_type == "ws":
html.append(escape(tok_text))
else:
tok_html = escape(tok_text) or ' '
html.append(
"<span class='%s'>%s</span>" % (tok_type, tok_html)
)
lines.append({
'html': ''.join(html),
'number': lineno,
'class': ' '.join(line_class) or "pln",
'annotate': annotate_html,
'annotate_title': annotate_title,
})
# Write the HTML page for this file.
html_filename = flat_rootname + ".html"
html_path = os.path.join(self.directory, html_filename)
html = spaceless(self.source_tmpl.render(locals()))
self.write_html(html_path, html)
# Save this file's information for the index file.
index_info = {
'nums': nums,
'par': n_par,
'html_filename': html_filename,
'name': cu.name,
}
self.files.append(index_info)
self.status.set_index_info(flat_rootname, index_info)
def test_tab_indentation(self):
# Mixed tabs and spaces...
self.assertEqual(list(source_token_lines(MIXED_WS)), MIXED_WS_TOKENS)
def html_file(self, cu, analysis):
"""Generate an HTML file for one source file."""
source_file = cu.source_file()
try:
source = source_file.read()
finally:
source_file.close()
flat_rootname = cu.flat_rootname()
this_hash = self.file_hash(source, cu)
that_hash = self.status.file_hash(flat_rootname)
if this_hash == that_hash:
self.files.append(self.status.index_info(flat_rootname))
return
self.status.set_file_hash(flat_rootname, this_hash)
if sys.version_info < (3, 0):
encoding = source_encoding(source)
if encoding.startswith('utf-8') and source[:3] == '\xef\xbb\xbf':
source = source[3:]
encoding = 'utf-8'
nums = analysis.numbers
missing_branch_arcs = analysis.missing_branch_arcs()
c_run = 'run hide_run'
c_exc = 'exc'
c_mis = 'mis'
c_par = 'par ' + c_run
lines = []
for lineno, line in enumerate(source_token_lines(source)):
lineno += 1
line_class = []
annotate_html = ''
annotate_title = ''
if lineno in analysis.statements:
line_class.append('stm')
if lineno in analysis.excluded:
line_class.append(c_exc)
elif lineno in analysis.missing:
line_class.append(c_mis)
elif self.arcs and lineno in missing_branch_arcs:
line_class.append(c_par)
annlines = []
for b in missing_branch_arcs[lineno]:
if b < 0:
annlines.append('exit')
else:
annlines.append(str(b))
annotate_html = ' '.join(annlines)
if len(annlines) > 1:
annotate_title = 'no jumps to these line numbers'
elif len(annlines) == 1:
annotate_title = 'no jump to this line number'
elif lineno in analysis.statements:
line_class.append(c_run)
html = []
for tok_type, tok_text in line:
if tok_type == 'ws':
html.append(escape(tok_text))
else:
tok_html = escape(tok_text) or ' '
html.append("<span class='%s'>%s</span>" %
(tok_type, tok_html))
lines.append({
'html': ''.join(html),
'number': lineno,
'class': ' '.join(line_class) or 'pln',
'annotate': annotate_html,
'annotate_title': annotate_title
})
html = spaceless(
self.source_tmpl.render({
'c_exc': c_exc,
'c_mis': c_mis,
'c_par': c_par,
'c_run': c_run,
'arcs': self.arcs,
'extra_css': self.extra_css,
'cu': cu,
'nums': nums,
'lines': lines
}))
if sys.version_info < (3, 0):
html = html.decode(encoding)
html_filename = flat_rootname + '.html'
html_path = os.path.join(self.directory, html_filename)
self.write_html(html_path, html)
index_info = {
'nums': nums,
'html_filename': html_filename,
'name': cu.name
}
self.files.append(index_info)
self.status.set_index_info(flat_rootname, index_info)
def source_token_lines(self):
return source_token_lines(self.source())
def source_token_lines(self):
return source_token_lines(self.source())
def test_simple(self):
self.assertEqual(list(source_token_lines(SIMPLE)), SIMPLE_TOKENS)
self.check_tokenization(SIMPLE)