def get_diff_chunks(self):
# https://stackoverflow.com/questions/9505822/getting-line-numbers-that-were-changed
# + for add - for delete maybe use ? ??
if not self.DiffInAuxiliary:
return
base_all_lines = UtilMethods.text_from_file(self.code_file.file_path).splitlines()
base_preformatted_lines = self.code_file.preformated_all_lines
base_line_wrap_length = self.code_file.line_wrap_length
cmp_all_lines = UtilMethods.text_from_file(self.DiffFilePath).splitlines()
cmp_preformatted_lines = UtilMethods.get_preformated_innerhtml('\n'.join(cmp_all_lines), self.code_file.language)
cmp_line_wrap_length = self.code_file.wrap_lines_process(cmp_all_lines, self.code_file.MAX_LINE_LENGTH)
lno1 = 0
lno2 = 0
file_diff = difflib.ndiff(base_all_lines, cmp_all_lines)
base_chunk = []
base_align = None
cmp_chunk = []
cmp_align = None
for line in file_diff:
code = line[:2]
# Updating line numbers
lno1 += 1 if (code == ' ' or code == '- ') else 0
lno2 += 1 if (code == ' ' or code == '+ ') else 0
# Add the change chunk
if code == ' ':
if base_chunk:
base_line_nos = base_chunk
base_source_code_lines = list(map(lambda x: base_all_lines[x-1], base_line_nos))
base_format_code_lines = list(map(lambda x: self.get_diff_rem_line(base_preformatted_lines[x-1]), base_line_nos))
base_part_length = sum([base_line_wrap_length[l] for l in base_line_nos])
base_code_part = CodePartition(base_line_nos, base_source_code_lines, base_format_code_lines, base_part_length, Node)
self.diff_auxiliary_partition_dict[self.get_nearest_base_line(base_align)] = base_code_part
base_chunk = []
base_align = None
if cmp_chunk:
cmp_line_nos = cmp_chunk
cmp_source_code_lines = list(map(lambda x: cmp_all_lines[x-1], cmp_line_nos))
cmp_format_code_lines = list(map(lambda x: self.get_diff_add_line(cmp_preformatted_lines[x-1]), cmp_line_nos))
cmp_part_length = sum([cmp_line_wrap_length[l] for l in cmp_line_nos])
cmp_code_part = CodePartition(cmp_line_nos, cmp_source_code_lines, cmp_format_code_lines, cmp_part_length, Node)
self.diff_auxiliary_partition_dict[self.get_nearest_base_line(cmp_align)] = cmp_code_part
cmp_chunk = []
cmp_align = None
if code == '+ ':
cmp_chunk.append(lno2)
if not cmp_align:
cmp_align = lno1
elif code == '- ':
base_chunk.append(lno1)
if not base_align:
base_align = lno1
def get_table_for_sidebar(self, soup, partitions):
side_table = soup.new_tag('table')
for p in partitions:
_, code_p = UtilMethods.get_pre_formated_text(p)
pcode = code_p.splitlines()[:p['length']]
pcode = '\n'.join(pcode)
line_str = '<pre>' + '\n'.join(str(lno)
for lno in p['line_nos']) + '</pre>'
# Generating the table row
line_no_div = soup.new_tag('div')
line_no_div['class'] = ['linenodiv']
line_no_div.append(BeautifulSoup(line_str, 'html.parser'))
line_table_data = soup.new_tag('td')
line_table_data['class'] = ['linenos']
line_table_data.append(line_no_div)
highlight_div = soup.new_tag('div')
highlight_div['class'] = ['highlight']
highlight_div.append(BeautifulSoup(pcode, 'html.parser'))
code_table_data = soup.new_tag('td')
code_table_data['class'] = ['code']
code_table_data.append(highlight_div)
table_row = soup.new_tag('tr')
table_row.append(line_table_data)
table_row.append(code_table_data)
side_table.append(table_row)
return side_table
def get_table_for_sidebar(self, soup, partitions):
side_table = soup.new_tag('table')
side_table['class'] = ['sidebartable']
side_table['cellspacing'] = "0"
for p in partitions:
_, pcode = UtilMethods.get_pre_formated_text(p, self.code_file.language)
line_str = '<pre>' + '\n'.join(str(lno) for lno in p.line_nos) + '</pre>'
# Generating the table row
line_no_div = soup.new_tag('div')
line_no_div['class'] = ['linenodiv']
line_no_div.append(BeautifulSoup(line_str, 'html.parser'))
line_table_data = soup.new_tag('td')
line_table_data['class'] = ['sidelinenos']
line_table_data.append(line_no_div)
highlight_div = soup.new_tag('div')
highlight_div['class'] = ['highlight']
highlight_div.append(BeautifulSoup(pcode, 'html.parser'))
code_table_data = soup.new_tag('td')
code_table_data['class'] = ['code']
code_table_data.append(highlight_div)
table_row = soup.new_tag('tr')
table_row.append(line_table_data)
table_row.append(code_table_data)
side_table.append(table_row)
return side_table
def runts():
file_path = abspath('../PaperCode/papertsc/test/project1/pytutor.ts')
project_path = abspath('../PaperCode/papertsc/test/project1/**/*.ts')
pdf_file_path = abspath('papercode/temp/pup2.pdf')
html_path = abspath('papercode/temp/high3.html')
code_file = TsCodeFile(file_path, project_path)
# return
code_file.process()
# code_file.print_tree(code_file.syntax_tree)
base_div = ConfigurableBaseDiv(code_file)
sidebar_div = None
# sidebar_div = ReferencesSidebarDiv(base_div, code_file, 3)
code_printer = ConfigurableCodePrinter(pdf_file_path, code_file, base_div,
sidebar_div)
html_code = code_printer.print_code_file()
# html_code = code_printer.get_html()
UtilMethods.write_text_to_file(html_path, html_code)
def get_html(self):
# todo: paper options
# start with the syntax tree
template_text = UtilMethods.text_from_file(self.html_template_path)
soup = BeautifulSoup(template_text, 'html.parser')
self.base_div.generate_html(soup, self.paper)
if self.sidebar_div is not None:
self.sidebar_div.generate_html(soup)
return str(soup)
def get_html(self):
# todo: paper options
# start with the syntax tree
template_text = UtilMethods.text_from_file(self.paper.template_path)
soup = BeautifulSoup(template_text, 'html.parser')
self.base_div.generate_html(soup, self.paper)
self.base_div.get_auxiliary_pages(soup)
self.base_div.get_diff_auxiliary_pages(soup)
return str(soup)
def __init__(self,
file_path: str,
project_path: str = None,
lang: Language = Language.Python):
self.MAX_LINE_LENGTH = 89
self.MAX_SIDEBAR_LINE_LENGTH = 50
self.file_path = file_path
self.project_path = project_path
self.source_code = UtilMethods.text_from_file(file_path)
self.all_lines = self.source_code.splitlines()
self.all_lines_process()
self.syntax_tree = None
self.language = lang
self.preformated_all_lines = UtilMethods.get_preformated_innerhtml(
'\n'.join(self.all_lines), self.language)
self.line_wrap_length = self.wrap_lines_process(
self.all_lines, self.MAX_LINE_LENGTH)
self.sidebar_line_wrap_length = self.wrap_lines_process(
self.all_lines, self.MAX_SIDEBAR_LINE_LENGTH)
self.is_sort_topo_sort = True
def get_pre_formated_text(self, partition):
partition_code = '\n'.join(partition['source_code_lines'])
partition_html = UtilMethods.highlight(partition_code)
soup = BeautifulSoup(partition_html, 'html.parser')
res = soup.find('table')
td_line_nos = res.find('td')
td_code = td_line_nos.find_next_sibling()
code_preformated_text = td_code.find('pre')
#generating line_preformated_text
line_soup = soup.new_tag('pre')
line_soup.string = '' + '\n'.join(
str(lno) for lno in partition['line_nos'])
return line_soup, str(code_preformated_text)
def get_ts_parsed_json(self):
temp_filename = tempfile.mktemp()
command = [
'node', self.tsc_path, '-p', self.project_path, '-f',
self.file_path, '-o', temp_filename
]
process = subprocess.Popen(command, stdout=subprocess.PIPE)
process.wait()
# print(process.returncode)
if process.returncode != 0:
print(process.returncode)
print('-------------------\n\n\n\n')
raise Exception('Cannot parse typescript file')
return UtilMethods.text_from_file(temp_filename)
def generate_partitions(self):
partitions = []
flat_tree = []
UtilMethods.flatten_syntax_tree(self.syntax_tree, [CallNode],
flat_tree)
if len(flat_tree) == 0:
return None
# Include everython before the first non-Node
elif len(flat_tree) == 1:
# Only Node is present - include all the html
partitions.append(
self.get_partition(1,
len(self.all_lines) + 1, Node))
else:
# Find the first Non-root node, and include everything before it
first_non_root = flat_tree[1]
start_line = 1
end_line = first_non_root.start_pos.line
partitions.append(
self.get_partition(start_line, end_line - 1, Node))
tail_nodes = flat_tree[1:]
# get HTML for all the other nodes.
for tnode in tail_nodes:
# for each node, if it gets borken within the page, then start from a new page
if type(tnode) is ClassNode:
partitions.append(
self.get_partition(tnode.start_pos.line,
tnode.body_start_pos.line - 1,
ClassNode))
elif type(tnode) is FunctionNode:
partitions.append(
self.get_partition(tnode.start_pos.line,
tnode.end_pos.line, FunctionNode))
return partitions
def runpy():
# file_path = 'papercode/temp/temp.py'
# file_path = 'papercode/temp/temp2.py'
html_path = 'papercode/temp/high.html'
file_path = 'papercode/temp/fpdf.py'
# file_path = 'D:/PV/Research/PaperCode/papertsc/temp/project1/pytutor.ts'
# template_path = 'papercode/templates/template2.html'
pdf_file_path = 'papercode/temp/pup.pdf'
code_file = PyCodeFile(file_path)
code_file.process()
# code_file.print_tree(code_file.syntax_tree)
base_div = ConfigurableBaseDiv(code_file)
# base_div = EverythingBaseDiv(code_file)
# sidebar_div = SmallFunctionSidebarDiv(base_div, code_file, 3)
# base_div = BigFunctionBaseDiv(code_file)
# sidebar_div = ReferencesSidebarDiv(base_div, code_file, 3)
sidebar_div = None
# code_printer = SidebarCodePrinter(pdf_file_path, code_file, base_div, sidebar_div)
code_printer = ConfigurableCodePrinter(pdf_file_path, code_file, base_div,
sidebar_div)
# print(code_printer.get_html())
html_code = code_printer.print_code_file()
# print(html_code)
UtilMethods.write_text_to_file(html_path, html_code)
def post_process_syntax_tree(self):
if self.syntax_tree is None:
return
functions = []
UtilMethods.flatten_syntax_tree(
self.syntax_tree,
[Node, ClassNode, CallNode, CommentNode, InterfaceNode], functions)
function_line_dict = {f.start_pos.line: f for f in functions}
# print(function_line_dict)
for func in functions:
if type(func) is not FunctionNode:
continue
# get all the calls and point it to the actual function
calls = func.function_calls
for fc in calls:
# Get the function call location and find the function node
func_defs = self.jedi_script.goto(fc.ref_pos.line,
fc.ref_pos.column)
if len(func_defs) == 0:
continue
if func_defs[0].line not in function_line_dict:
continue
qf = function_line_dict[func_defs[0].line]
fc.function_node = qf
fc.from_function_node = func
qf.refs.append(fc)
# Topological sort for each node
class_nodes = []
UtilMethods.flatten_syntax_tree(
self.syntax_tree,
[Node, FunctionNode, CallNode, CommentNode, InterfaceNode],
class_nodes)
for cNode in class_nodes:
self.topo_sort(cNode)
def get_html_from_partitions(self):
# todo: paper options
template_text = UtilMethods.text_from_file(self.html_template_path)
soup = BeautifulSoup(template_text, 'html.parser')
highlight_table = soup.find('table', {'class': 'highlighttable'})
partitions = self.code_file.generate_partitions()
for p in partitions:
_, code_p = self.get_pre_formated_text(p)
pcode = code_p.splitlines()[:p['length']]
pcode = '\n'.join(pcode)
line_str = '<pre>' + '\n'.join(str(lno)
for lno in p['line_nos']) + '</pre>'
# Generating the table row
line_no_div = soup.new_tag('div')
line_no_div['class'] = ['linenodiv']
line_no_div.append(BeautifulSoup(line_str, 'html.parser'))
line_table_data = soup.new_tag('td')
line_table_data['class'] = ['linenos']
line_table_data.append(line_no_div)
highlight_div = soup.new_tag('div')
highlight_div['class'] = ['highlight']
highlight_div.append(BeautifulSoup(pcode, 'html.parser'))
code_table_data = soup.new_tag('td')
code_table_data['class'] = ['code']
code_table_data.append(highlight_div)
table_row = soup.new_tag('tr')
table_row.append(line_table_data)
table_row.append(code_table_data)
highlight_table.append(table_row)
return str(soup)
def getFunctionPartition(self, node: FunctionNode):
# Print evenrything except the children
line_nos = []
source_code_lines = []
sidebar_line_nos = []
sidebar_code = []
current_line = node.start_pos.line - 1
end_line = min(node.end_pos.line, len(self.code_file.all_lines))
children = node.children
for child in children:
# Adding intermediate lines
if current_line < child.start_pos.line - 1:
# Add a new partition
# print('partition (', current_line + 1, ',', child.start_pos.line - 1, '):', child.start_pos.line)
part = self.code_file.get_partition(current_line + 1, child.start_pos.line - 1, type(node))
line_nos.extend(part.line_nos)
source_code_lines.extend(part.source_code_lines)
# Add the comment partition
commentPart = self.getCommentPartition(child)
if not(commentPart.base is None): # Comment should be included in the base
line_nos.extend(commentPart.base.line_nos)
source_code_lines.extend(commentPart.base.source_code_lines)
else: # Comment should be a sidebar
remaining_lines = len(line_nos) - len(sidebar_line_nos)
if remaining_lines > 0:
sidebar_line_nos.extend([' ' for i in range(remaining_lines)])
sidebar_code.extend([' ' for i in range(remaining_lines)])
# adding to sidebar
sidebar_line_nos.extend(commentPart.sidebar.line_nos)
sidebar_code.extend(commentPart.sidebar.source_code_lines)
current_line = child.end_pos.line
if current_line < end_line:
# print('partition (', current_line + 1, ',', end_line, ')')
part = self.code_file.get_partition(current_line + 1 , end_line, type(node))
line_nos.extend(part.line_nos)
source_code_lines.extend(part.source_code_lines)
current_line = end_line
base_length = sum([self.code_file.line_wrap_length[l] for l in line_nos])
# Matching sidebar to the function length
remaining_lines = base_length - len(sidebar_line_nos)
if remaining_lines > 0:
sidebar_line_nos.extend([' ' for i in range(remaining_lines)])
sidebar_code.extend([' ' for i in range(remaining_lines)])
sidebar_length = len(sidebar_line_nos)
sidebar_part = CodePartition(
sidebar_line_nos,
sidebar_code,
UtilMethods.get_preformated_innerhtml('\n'.join(sidebar_code), self.code_file.language),
sidebar_length,
CommentNode
)
base_part = CodePartition(
line_nos,
source_code_lines,
UtilMethods.get_preformated_innerhtml('\n'.join(source_code_lines), self.code_file.language),
max(base_length, sidebar_length),
FunctionNode
)
return PagePartition(base_part, node, sidebar_part)
def generate_html(self, soup: BeautifulSoup, paper: Paper):
# Given soup, Insert the main table
highlight_table = soup.find('table', {'class': 'highlighttable'})
partitions = []
# Start from the root and recursively generate partitions
root_node: Node = self.code_file.syntax_tree
# Edge case for typescript
if root_node.start_pos.line > 1:
part = self.code_file.get_partition(1,
root_node.start_pos.line - 1,
type(root_node))
partitions.append({'base': part, 'node': root_node})
self.getPartitions(root_node, partitions)
for p in partitions:
# base, side bar : create tables seperately for both
# Making the base
base_part = p['base']
part_uuid = uuid.uuid4().hex
_, code_base = UtilMethods.get_pre_formated_text(
base_part, self.code_file.language)
# code_base = code_base.splitlines()[:base_part['length']]
# code_base = '\n'.join(code_base)
line_str = '<pre>' + '\n'.join(
str(lno) for lno in base_part['line_nos']) + '</pre>'
# Generating the line no div
line_no_div = soup.new_tag('div')
line_no_div['class'] = ['linenodiv']
line_no_div.append(BeautifulSoup(line_str, 'html.parser'))
# Generating the line no td
line_table_data = soup.new_tag('td')
if p['node'] not in self.elements_line_td:
self.elements_line_td[p['node']] = part_uuid + '-line'
line_table_data['id'] = self.elements_line_td[p['node']]
line_table_data['class'] = ['linenos']
line_table_data.append(line_no_div)
# Generating the code highlight div
highlight_div = soup.new_tag('div')
highlight_div['class'] = ['highlight']
highlight_div.append(BeautifulSoup(code_base, 'html.parser'))
# Generating the code td
code_table_data = soup.new_tag('td')
if p['node'] not in self.elements_code_td:
self.elements_code_td[p['node']] = part_uuid + '-code'
code_table_data['id'] = self.elements_code_td[p['node']]
code_table_data['class'] = ['code']
code_table_data.append(highlight_div)
# Generating the side bar table
# side_table = self.get_table_for_sidebar(soup, p['sidebar'])
sidebar_table_data = soup.new_tag('td')
if p['node'] not in self.elements_sidebar_td:
self.elements_sidebar_td[p['node']] = part_uuid + '-sidebar'
sidebar_table_data['id'] = self.elements_sidebar_td[p['node']]
sidebar_table_data['class'] = ['sidebar']
# sidebar_table_data.append(side_table)
table_row = soup.new_tag('tr')
table_row.append(line_table_data)
table_row.append(code_table_data)
table_row.append(sidebar_table_data)
highlight_table.append(table_row)
def add_pages_html(self, soup: BeautifulSoup, pages: dict):
# Generate a table for each page
for page in pages:
# highlight_table = self.generate_highlight_table(soup)
highlight_table, sidebar_table = self.setup_page(soup)
cur_page: Page = pages[page]
# Generate QR-Code and header/footer elements
qr_td_1 = soup.new_tag('td')
qr_td_1['style'] = "position:relative;"
qr_td_2 = soup.new_tag('td')
qr_td_2['style'] = "position:relative;"
qr_td_3 = soup.new_tag('td')
qr_td_3['style'] = "position:relative;"
qrcode = soup.new_tag('img')
qrcode['class'] = ['qrcode']
qrcode_binary = UtilMethods.getQRCodeFromData(cur_page.page_no)
qrcode_str = "data:image/png;base64," + qrcode_binary
qrcode['src'] = qrcode_str
qr_td_3.append(qrcode)
qr_td_4 = soup.new_tag('td')
qr_td_4['style'] = "position:relative;"
qr_table_row = soup.new_tag('tr')
qr_table_row.append(qr_td_1)
qr_table_row.append(qr_td_2)
qr_table_row.append(qr_td_3)
qr_table_row.append(qr_td_4)
highlight_table.append(qr_table_row)
for idx in range(len(cur_page.page_line_nos)):
# new table row for each line
line_str = '<pre>' + cur_page.page_line_nos[idx] + '</pre>'
code_str = '<pre>' + cur_page.page_code_lines[idx] + '</pre>'
# Generating the line no div
line_no_div = soup.new_tag('div')
line_no_div['class'] = ['linenodiv']
line_no_div.append(BeautifulSoup(line_str, 'html.parser'))
# Generating the line no td
line_table_data = soup.new_tag('td')
line_table_data['class'] = ['linenos']
line_table_data.append(line_no_div)
# Generating the code highlight div
highlight_div = soup.new_tag('div')
highlight_div['class'] = ['highlight']
highlight_div.append(BeautifulSoup(code_str, 'html.parser'))
# Generating the code td
code_table_data = soup.new_tag('td')
code_table_data['class'] = ['code']
code_table_data.append(highlight_div)
table_row = soup.new_tag('tr')
table_row.append(line_table_data)
table_row.append(code_table_data)
highlight_table.append(table_row)
# End of for
# generating table for sidebar
self.get_pages_sidebar(cur_page.page_sidebar_line_nos, cur_page.page_sidebar_code_lines, sidebar_table, soup)
def print_code_file(self):
html_code = self.get_html_from_partitions()
UtilMethods.get_pdf_sync(html_code, self.pdf_file_path)
return html_code
def generate_html(self, soup: BeautifulSoup, paper: Paper):
# Given soup, Insert the main table
highlight_table = soup.find('table', {'class': 'highlighttable'})
partitions = []
# Parition for the initial module
part = self.code_file.get_partition_from_node(
self.code_file.syntax_tree)
partitions.append({'base': part, 'node': self.code_file.syntax_tree})
# Todo what if there are functions before the node?
for cnode in self.code_file.syntax_tree.children:
if type(cnode) is Node:
part = self.code_file.get_partition_from_node(cnode)
partitions.append({'base': part, 'node': cnode})
continue
elif type(cnode) is FunctionNode:
part = self.code_file.get_partition_from_node(cnode)
partitions.append({'base': part, 'node': cnode})
continue
elif type(cnode) is ClassNode:
part = self.code_file.get_partition_from_node(cnode)
partitions.append({'base': part, 'node': cnode})
# for each function, do something
functions = [
n for n in cnode.children if type(n) is FunctionNode
]
big_function = [
f for f in functions if f.size > self.size_limit
]
# go through all the bif functions
for bf in big_function:
bf_part = self.code_file.get_partition_from_node(bf)
partitions.append({'base': bf_part, 'node': bf})
for p in partitions:
# base, side bar : create tables seperately for both
# Making the base
base_part = p['base']
part_uuid = uuid.uuid4().hex
_, code_base = UtilMethods.get_pre_formated_text(
base_part, self.code_file.language)
code_base = code_base.splitlines()[:base_part['length']]
code_base = '\n'.join(code_base)
line_str = '<pre>' + '\n'.join(
str(lno) for lno in base_part['line_nos']) + '</pre>'
# Generating the line no div
line_no_div = soup.new_tag('div')
line_no_div['class'] = ['linenodiv']
line_no_div.append(BeautifulSoup(line_str, 'html.parser'))
# Generating the line no td
line_table_data = soup.new_tag('td')
self.elements_line_td[p['node']] = part_uuid + '-line'
line_table_data['id'] = self.elements_line_td[p['node']]
line_table_data['class'] = ['linenos']
line_table_data.append(line_no_div)
# Generating the code highlight div
highlight_div = soup.new_tag('div')
highlight_div['class'] = ['highlight']
highlight_div.append(BeautifulSoup(code_base, 'html.parser'))
# Generating the code td
code_table_data = soup.new_tag('td')
self.elements_code_td[p['node']] = part_uuid + '-code'
code_table_data['id'] = self.elements_code_td[p['node']]
code_table_data['class'] = ['code']
code_table_data.append(highlight_div)
# Generating the side bar table
# side_table = self.get_table_for_sidebar(soup, p['sidebar'])
sidebar_table_data = soup.new_tag('td')
self.elements_sidebar_td[p['node']] = part_uuid + '-sidebar'
sidebar_table_data['id'] = self.elements_sidebar_td[p['node']]
sidebar_table_data['class'] = ['sidebar']
# sidebar_table_data.append(side_table)
table_row = soup.new_tag('tr')
table_row.append(line_table_data)
table_row.append(code_table_data)
table_row.append(sidebar_table_data)
highlight_table.append(table_row)