def new_parser(filename, log=None):
if not log:
log = LogWriterNull()
def ast_parser(filename):
with open(filename,'r') as f:
source = f.read()
node = ast.parse(source)
#print ast.dump(node)
return node
node = ast_parser(filename)
model, debuginfo = convert_ast_to_old_parser(node, filename, log)
return model, debuginfo
def new_parser(filename, log=None, options={}):
if not log:
log = LogWriterNull()
pmodel, debuginfo = parse(filename, log, options)
return pmodel, debuginfo
def _convert_ast_to_old_parser(node, filename, log, options={}):
"""
Args:
node: ast node after parsing by python's ast library
filename:
_log:
options:
Returns:
"""
logh = log
if not logh:
logh = LogWriterNull()
qp = QuickParse(filename, logh)
v = Visitor(qp, logh, options)
try:
v.visit(node)
except Exception as err:
source_code_line = _extract_source_code_line(filename, v.latest_lineno)
v.model.errors += f" Pynsource couldn't handle AST area in file {filename} at lineno {v.latest_lineno} coloffset {v.latest_col_offset} - source code:\n\n \"{source_code_line}\""
v.model.errors += f"\n\nPlease report this to https://github.com/abulka/pynsource/issues"
v.model.errors += f"\n\nThere may be more detail in the log file {LOG_FILENAME}"
if DEBUG_TO_LOG_PROPER: # output goes to regular python logging
log_proper.error(
f"html version of debug info may exist at: '{logh.out_filename}'"
)
log_proper.error(f"model.errors: '{v.model.errors}'")
# log_proper.error(f"Parsing Visit error: {err}, Debug info: {v.result_for_log_proper_cleaned}") # very verbose
# debug output goes to special html formatted log file controlled by 'logh' object
logh.out("Parsing Visit error: {0}".format(err), force_print=True)
logh.out_wrap_in_html(traceback.format_exc(), style_class="stacktrace")
if DEBUGINFO():
debuginfo = "<br>".join(v.result)
logh.out(debuginfo)
logh.out(f"<br>model.errors: '{v.model.errors}'<br>")
logh.out_html_footer()
logh.finish()
if STOP_ON_EXCEPTION:
raise
if DEBUG_TO_LOG_PROPER and DEBUG_TO_LOG_PROPER_PARSE_HISTORY:
log_proper.debug(v.result_for_log_proper_cleaned)
debuginfo = "<br>".join(v.result)
return v.model, debuginfo
def _convert_ast_to_old_parser(node, filename, log, options={}):
"""
Args:
node: ast node after parsing by python's ast library
filename:
_log:
options:
Returns:
"""
logh = log
if not logh:
logh = LogWriterNull()
qp = QuickParse(filename, logh)
v = Visitor(qp, logh, options)
# Give visitor access to source code, for diagnostic purposes
with open(filename) as f:
v.source_code_lines = f.readlines()
try:
v.visit(node)
except Exception as err:
# traceback.print_exception(type(ex), ex, ex.__traceback__)
source_code_line = _extract_source_code_line(filename, v.latest_lineno)
v.model.errors += f"Pynsource couldn't handle traversing AST in file {filename} corresponding to approx. lineno {v.latest_lineno} coloffset {v.latest_col_offset} - source code:\n\n {source_code_line}"
v.model.errors += f"\n\nPlease report this to https://github.com/abulka/pynsource/issues"
v.model.errors += f"\n\nLog file location:\n{LOG_FILENAME}"
v.model.errors += f"\n\nActual exception:\n\n{err}"
if DEBUG_TO_LOG_PROPER: # output goes to regular python logging
log_proper.error(
f"html version of debug info may exist at: '{logh.out_filename}'"
)
log_proper.error(f"model.errors: '{v.model.errors}'")
# IMPORTANT TO LEAVE THIS IN because it reports the actual exception in the proper log
log_proper.error(
f"Parsing Visit error: {err}, Debug info: {v.result_for_log_proper_cleaned}"
) # very verbose
log_proper.error(f"Full traceback: {traceback.format_exc()}")
# debug output goes to special html formatted log file controlled by 'logh' object
logh.out("Parsing Visit error: {0}".format(err), force_print=True)
logh.out_wrap_in_html(traceback.format_exc(), style_class="stacktrace")
if DEBUGINFO():
debuginfo = "<br>".join(v.result)
logh.out(debuginfo)
logh.out(f"<br>model.errors: '{v.model.errors}'<br>")
logh.out_html_footer()
logh.finish()
if STOP_ON_EXCEPTION:
raise
if DEBUG_TO_LOG_PROPER and DEBUG_TO_LOG_PROPER_PARSE_HISTORY:
log_proper.debug(v.result_for_log_proper_cleaned)
debuginfo = "<br>".join(v.result)
return v.model, debuginfo
def parse_old_and_new(in_filename, print_diffs=True, options={}):
"""
Runs old and new parsers and compares the result.
Logs everything to a special HTML log file using LogWriter instance.
The ability of the old parser to treat modules as pseudo classes is ignored,
since the new parser does not have this mode of operation yet.
Returns t/f
"""
global log
if DEBUGINFO:
log = LogWriter(in_filename, print_to_console=LOG_TO_CONSOLE)
else:
log = LogWriterNull()
def oldparse():
model, debuginfo = old_parser(in_filename)
d1 = dump_old_structure(model)
log.out_wrap_in_html(d1, style_class='dump1')
return d1
def newparse(options):
model, debuginfo = new_parser(in_filename, log, options)
d2 = dump_old_structure(model)
log.out_wrap_in_html(d2, style_class='dump1')
return d2, debuginfo
def dodiff(d1, d2):
diff = difflib.ndiff(d1.splitlines(1), d2.splitlines(1))
diff_s = ''.join(diff)
return diff_s
try:
log.out_html_header()
log.out("PARSING: %s *****\n" % in_filename)
d1 = oldparse()
log.out_divider()
d2, debuginfo = newparse(options)
comparedok = (d1 == d2)
log.out("** old vs new method comparison = %s" % comparedok)
diff_s = dodiff(d1, d2)
log.out_wrap_in_html(diff_s, style_class='dumpdiff')
if not comparedok and print_diffs:
print diff_s
#dd = difflib.HtmlDiff()
#diff_table = dd.make_table(d1.splitlines(1),d2.splitlines(1))
#log.out(diff_table)
if not comparedok:
global last_diff_s
log.out("<hr>")
delta = difflib.unified_diff(d1.splitlines(1),
d2.splitlines(1),
n=0,
fromfile='before.py',
tofile='after.py')
last_diff_s = "".join(delta)
log.out_wrap_in_html(last_diff_s, style_class='dumpdiff')
if DEBUGINFO:
log.out(debuginfo)
log.out_html_footer()
finally:
log.finish()
return comparedok
def _convert_ast_to_old_parser(node, filename, log, options={}):
"""
Args:
node: ast node after parsing by python's ast library
filename:
_log:
options:
Returns:
"""
logh = log
if not logh:
logh = LogWriterNull()
qp = QuickParse(filename, logh)
v = Visitor(qp, logh, options)
try:
v.visit(node)
except Exception as err:
logh.out("Parsing Visit error: {0}".format(err), force_print=True)
logh.out_wrap_in_html(traceback.format_exc(), style_class="stacktrace")
if STOP_ON_EXCEPTION:
if DEBUGINFO:
debuginfo = "<br>".join(v.result)
logh.out(debuginfo)
if DEBUG_TO_LOG_PROPER:
log_proper.error("Parsing Visit error: {0}".format(err) + v.result_as_compact_string)
logh.out_html_footer()
logh.finish()
raise
if DEBUG_TO_LOG_PROPER and DEBUG_TO_LOG_PROPER_PARSE_HISTORY:
log_proper.debug(v.result_as_compact_string)
debuginfo = "<br>".join(v.result)
return v.model, debuginfo
def parse_old_and_new(in_filename, print_diffs=True):
global log
if DEBUGINFO:
log = LogWriter(in_filename, print_to_console=LOG_TO_CONSOLE)
else:
log = LogWriterNull()
def oldparse():
model, debuginfo = old_parser(in_filename)
d1 = dump_old_structure(model)
log.out_wrap_in_html(d1, style_class='dump1')
return d1
def newparse():
model, debuginfo = new_parser(in_filename, log)
d2 = dump_old_structure(model)
log.out_wrap_in_html(d2, style_class='dump1')
return d2, debuginfo
def dodiff(d1, d2):
diff = difflib.ndiff(d1.splitlines(1),d2.splitlines(1))
diff_s = ''.join(diff)
return diff_s
try:
log.out_html_header()
log.out("PARSING: %s *****\n" % in_filename)
d1 = oldparse()
log.out_divider()
d2, debuginfo = newparse()
comparedok = (d1 == d2)
log.out("** old vs new method comparison = %s" % comparedok)
diff_s = dodiff(d1, d2)
log.out_wrap_in_html(diff_s, style_class='dumpdiff')
if not comparedok and print_diffs:
print diff_s
#dd = difflib.HtmlDiff()
#diff_table = dd.make_table(d1.splitlines(1),d2.splitlines(1))
#log.out(diff_table)
if not comparedok:
global last_diff_s
log.out("<hr>")
delta = difflib.unified_diff(d1.splitlines(1),d2.splitlines(1), n=0,
fromfile='before.py', tofile='after.py')
last_diff_s = "".join(delta)
log.out_wrap_in_html(last_diff_s, style_class='dumpdiff')
if DEBUGINFO:
log.out(debuginfo)
log.out_html_footer()
finally:
log.finish()
return comparedok