def test_positions_fstring(self):
code = "def func(): pass\nvar = f'Im a fstring {func()} string end'"
res = check_ast(code, try_other_on_sucess=True)
self.assertEqual(res[PYVERIDX], 3)
ast = res[PY3AST]
# f"
self.assertEqual(ast["body"][1]["value"]["col_offset"], 6)
self.assertEqual(ast["body"][1]["value"]["lineno"], 2)
# "Im a fstring "
self.assertEqual(ast["body"][1]["value"]["values"][0]["col_offset"], 6)
self.assertEqual(ast["body"][1]["value"]["values"][0]["lineno"], 2)
# FormattedValue node (virtual, should be the same as the child below
self.assertEqual(ast["body"][1]["value"]["values"][1]["col_offset"], 6)
self.assertEqual(ast["body"][1]["value"]["values"][1]["lineno"], 2)
# func() insde the braces
self.assertEqual(ast["body"][1]["value"]["values"][1]["value"]["col_offset"], 22)
self.assertEqual(ast["body"][1]["value"]["values"][1]["value"]["lineno"], 2)
# " string end"
self.assertEqual(ast["body"][1]["value"]["values"][2]["col_offset"], 6)
self.assertEqual(ast["body"][1]["value"]["values"][2]["lineno"], 2)
def test_neither_ast(self):
code = "Fail like a boss"
res = check_ast(code, try_other_on_sucess=True)
self.assertEqual(res[PYVERIDX], 0)
self.assertEqual(res[PY2AST], None)
self.assertEqual(res[PY3AST], None)
self.assertNotEqual(len(res[PY3ERR]), 0)
self.assertNotEqual(len(res[PY2ERR]), 0)
def test_both_ast(self):
code = "pass"
res = check_ast(code, try_other_on_sucess=True)
self.assertEqual(res[PYVERIDX], 6)
self.assertIsInstance(res[PY2AST], dict)
self.assertIsInstance(res[PY3AST], dict)
self.assertEqual(len(res[PY3ERR]), 0)
self.assertEqual(len(res[PY2ERR]), 0)
def test_py2_ast(self):
code = "print 'hello old world'"
res = check_ast(code, try_other_on_sucess=True)
self.assertEqual(res[PYVERIDX], 2)
self.assertIsInstance(res[PY2AST], dict)
self.assertEqual(res[PY3AST], None)
self.assertEqual(len(res[PY2ERR]), 0)
self.assertNotEqual(len(res[PY3ERR]), 0)
def test_py3_ast(self):
code = dedent("""
import sys
print('hello not so new world', file=sys.stderr)
""")
res = check_ast(code, try_other_on_sucess=True)
self.assertEqual(res[PYVERIDX], 3)
self.assertEqual(res[PY2AST], None)
self.assertIsInstance(res[PY3AST], dict)
self.assertEqual(len(res[PY3ERR]), 0)
self.assertNotEqual(len(res[PY2ERR]), 0)
def test_positions_args(self):
code = "func(aaa, bbb)"
res = check_ast(code, try_other_on_sucess=True)
self.assertEqual(res[PYVERIDX], 6)
self.assertEqual(len(res[PY3ERR]), 0)
ast = res[PY3AST]
self.assertEqual(ast["body"][0]["value"]["args"][0]["col_offset"], 5)
self.assertEqual(ast["body"][0]["value"]["args"][0]["lineno"], 1)
self.assertEqual(ast["body"][0]["value"]["args"][1]["col_offset"], 10)
self.assertEqual(ast["body"][0]["value"]["args"][1]["lineno"], 1)
def test_positions_attribute(self):
code = "import sys\nsys.stdout.write('foo')\nprint('py2', file=sys.stderr)"
res = check_ast(code, try_other_on_sucess=False)
self.assertEqual(res[PYVERIDX], 3)
self.assertEqual(len(res[PY3ERR]), 0)
ast = res[PY3AST]
self.assertIsInstance(ast, dict)
self.assertEqual(ast["body"][1]['value']['args'][0]['col_offset'], 17)
self.assertEqual(ast["body"][1]['value']['args'][0]['lineno'], 2)
self.assertEqual(ast["body"][1]['value']['func']['col_offset'], 0)
self.assertEqual(ast["body"][1]['value']['func']['lineno'], 2)
self.assertEqual(ast["body"][1]['value']['func']['value']['col_offset'], 0)
self.assertEqual(ast["body"][1]['value']['func']['value']['lineno'], 2)
self.assertEqual(ast["body"][1]['value']['func']['value']['value']['col_offset'], 0)
self.assertEqual(ast["body"][1]['value']['func']['value']['value']['lineno'], 2)
def detect(files=None, codestr=None, ast_checks=True, modules_checks=True,
modsyms_checks=False, stop_on_ok_ast=False, modules_score=150,
symbols_score=100, verbosity=0):
"""
Try to detect if a source file is Python 2 or 3. It uses a combination of
tests based on AST extraction and regular expressions.
Args: files (List[str], optional): list of files. You can omit this parameter
if you pass codestr.
codestr (str, optional): source code of a single module to parse. You can
omit this parameter if you pass "files".
ast_checks (bool): enable checking if the AST parses with both Python
versions
modules_checks (bool): enable checking version-specific module imports
modsyms_checks (bool): enable checking version-specific module symbols.
Please note that this test can be much slower than the others.
stop_on_ok_ast (bool): if the first AST tested works, don't even try with
the other version
modules_score (int): score given to specific-module matches
symbols_score (int): score given to symbol-specific matches
verbosity (int): verbosity level from 0 (quiet) to 2
Return:
Dictionary where each key is the filename and the value another dictionary
with the keys "py2ast" and "py3ast" that will hold the AST if sucessfully
parser for that version or "None", "version" with the version number (2 or
3) or 6 is the module seems to be compatible with both versions, "matches"
that will hold a list of the matched rules and scores and
"py2_score/py3_score" with the specific score.
"""
returndict = {}
if not files:
if not codestr:
raise Exception('files or codestr parameters are required')
files = ['<code_string>']
for filename in files:
returndict[filename] = {
'py2ast': None,
'py3ast': None,
'version': 0,
'matches': [],
'py2_score': 0,
'py3_score': 0,
'py2_ast_errors': [],
'py3_ast_errors': [],
}
retdict = returndict[filename] # alias
# helper for lazy bastards
def apply_score(py2_score, py3_score):
retdict['py2_score'] += py2_score
retdict['py3_score'] += py3_score
if verbosity:
print('Checking file %s: ' % filename)
input_code = None
if filename == '<code_string>':
input_code = codestr
else:
# this have problems if the file is not encoding in utf8 input_code = sys.stdin.read()
# From most to less common, this should cover 99.9% of the encodings used
for encoding in ('utf_8', 'iso8859_15', 'iso8859_1', 'gb2313',
'cp1251', 'cp1252', 'cp1250', 'shift-jis', 'gbk', 'cp1256',
'iso8859-2', 'euc_jp', 'big5', 'cp874', 'euc_kr', 'iso8859_7'
'cp1255'):
with open(filename, encoding=encoding) as infile:
try:
input_code = infile.read()
break
except UnicodeDecodeError:
continue
else:
raise Exception('Could not determine file encoding')
if ast_checks:
# Test the AST. This doesnt give points: either both pass, both fails
# or one is correct and the other dont in which case we shortcircuit the return
astversion, py2astroot, py3astroot, py2_err, py3_err = check_ast(
input_code, try_other_on_sucess=not stop_on_ok_ast,
verbosity=verbosity
)
retdict.update({
'py2ast': {'PY2AST': py2astroot} if py2astroot else None,
'py3ast': {'PY3AST': py3astroot} if py3astroot else None,
'matches': [('PY%dASTOK' % astversion, ())]
})
if py2_err:
retdict['py2_ast_errors'].append(py2_err)
if py3_err:
retdict['py3_ast_errors'].append(py3_err)
# One parsed and the other didnt, no need to continue checking
if astversion in (2, 3):
retdict['version'] = astversion
continue
# Remove comments and emptyfy strings before doing the regex tests,
# this will remove most fase positives
cleaned_code = remove_str_comments(input_code)
if modules_checks:
apply_score(*check_syntax_regex(cleaned_code, retdict['matches']))
apply_score(*check_modules_regex(cleaned_code, retdict['matches'],
match_score = modules_score))
# This one is SLOOOOOW
if modsyms_checks:
apply_score(
*check_modulesymbols_regex(cleaned_code, retdict['matches'], symbols_score)
)
if retdict['py2_score'] > retdict['py3_score']:
retdict['version'] = 2
elif retdict['py3_score'] > retdict['py2_score']:
retdict['version'] = 3
else:
retdict['version'] = 6
if verbosity:
print('Python 2 score: %d' % retdict['py2_score'])
print('Python 3 score: %d' % retdict['py3_score'])
print('\n')
return returndict