def main():
args = parse_args()
returndict = detect(args.files,
ast_checks=args.testast,
modules_checks=args.testmodules,
modsyms_checks=args.testmodulesyms,
stop_on_ok_ast=not args.asttestboth,
verbosity=args.verbosity)
if not args.showast:
for fdata in returndict:
del returndict[fdata][
'py2ast'] # not json serializable in the current form
del returndict[fdata][
'py3ast'] # not json serializable in the current form
pprint(returndict)
if args.verbosity:
py2_count = py3_count = pyany_count = 0
for key in returndict:
version = returndict[key]['version']
if version == 2 or (version == 6 and args.defaultversion == 2):
py2_count += 1
elif version == 3 or (version == 6 and args.defaultversion == 3):
py3_count += 1
else:
pyany_count += 1
print('%d files parsed, py2: %d, py3: %d any: %d' %
(len(returndict), py2_count, py3_count, pyany_count))
def test_detect_py3(self):
code = dedent("""
import sys
print('new', file=sys.stderr)
def func(arg1, arg2, arg3):
print(arg1, arg2, arg3)
func(1, 2, 3)
""")
self._check_res(detect(codestr=code,
modsyms_checks=True)['<code_string>'],
version=3,
ast3check=True)
def test_detect_both(self):
# avoid pyc and pyo extensions
f = __file__[:-1] if not __file__.endswith('.py') else __file__
with open(f) as this:
# this code should remain compatible with Python 2 and 3
# so this test actually also check this
code = this.read()
self._check_res(detect(codestr=code,
modsyms_checks=True)['<code_string>'],
version=6,
ast2check=True,
ast3check=True)
def test_detect_py2(self):
code = "print 'old'"
self._check_res(detect(codestr=code,
modsyms_checks=True)['<code_string>'],
version=2,
ast2check=True)
elif isinstance(node, list) or isinstance(node, tuple):
return [self.visit(x) for x in node]
else:
# string attribute
return node
if __name__ == '__main__':
import sys
import importlib.util
from pprint import pprint
if len(sys.argv) > 1:
from pydetector import detector
codestr = open(sys.argv[1]).read()
resdict = detector.detect(codestr=codestr, stop_on_ok_ast=True)
codeinfo = resdict['<code_string>']
version = codeinfo['version']
failed = False
testdict = None
if version in (3, 6) and codeinfo['py3ast']:
testdict = codeinfo['py3ast']["PY3AST"]
print("Using Python3")
elif version in (1, 2) and codeinfo['py2ast']:
testdict = codeinfo['py2ast']["PY2AST"]
print("Using Python2")
else:
failed = True
errors = [
def process_request(self, request: RawRequest) -> None:
"""
Main function doing the work of processing a single request. It will
do its best effort to detect the code Python version(s), extract the AST,
prepare the reply package and sent it on the output buffer.
Any error will cause an error or fatal package to be submitted.
:param request: the deserialized request dictionary.
"""
filepath = ''
ast = None
self.errors = []
try:
str_request = self._tostr_request(request)
code = asstr(str_request.get('content', ''))
if code:
# We want the code detection to be fast and we prefer Python3 AST so using
# the stop_on_ok_ast will avoid running a Python2 subprocess to check the
# AST with Python2 if the Python3 version (which is a better AST anyway) worked
resdict = detector.detect(codestr=code, stop_on_ok_ast=True)
codeinfo = resdict['<code_string>']
version = codeinfo['version']
if version in (3, 6) and codeinfo['py3ast']:
orig_ast = codeinfo['py3ast']["PY3AST"]
elif version in (1, 2) and codeinfo['py2ast']:
orig_ast = codeinfo['py2ast']["PY2AST"]
else:
raise Exception(
'Errors produced trying to get an AST for both Python versions' +
'\n------ Python2 errors:\n%s' % codeinfo['py2_ast_errors'] +
'\n------ Python3 errors:\n%s' % codeinfo['py3_ast_errors']
)
if not orig_ast:
raise Exception('Empty AST generated from non empty code')
ast = AstImprover(code, orig_ast).parse()
if not ast:
raise Exception('Empty AST generated from non empty code')
else:
# Module with empty code (like __init__.py) return a module-only AST
# since this would still have semantic meaning for Python
ast = {
"ast_type" : "Module",
"lineno" : 1,
"col_offset" : 1,
}
version = 3
response = Response({
'status' : 'ok',
'errors' : self.errors,
'ast' : {"PY%dAST" % version: ast},
'metadata' : {
'language' : 'python',
'language_version' : version,
'driver' : 'python23:%s' % __version__,
}
})
if filepath:
response['filepath'] = filepath
self._send_response(response)
except EmptyCodeException:
self.errors.append('Code field empty')
self._return_error(filepath, status='error', ast=ast)
except:
status = 'fatal' if ast is None else 'error'
self.errors.append(format_exc())
self._return_error(filepath, status=status)
