def parse(source: _source,
filename=constants.DEFAULT_FILENAME,
mode='exec',
*,
flags=0,
**kwargs) -> _ast.AST:
"""
convert Import Expression Python™ to an AST
Keyword arguments:
mode: determines the type of the returned root node.
As in the mode argument of :func:`ast.parse`,
it must be one of "eval" or "exec".
Eval mode returns an :class:`ast.Expression` object. Source must represent a single expression.
Exec mode returns a :class:` Module` object. Source represents zero or more statements.
Filename is used in tracebacks, in case of invalid syntax or runtime exceptions.
The remaining keyword arguments are passed to ast.parse as is.
"""
# for some API compatibility with ast, allow parse(parse('foo')) to work
if isinstance(source, _ast.AST):
return _parse_ast(source, filename=filename)
fixed = _fix_syntax(source, filename=filename)
if flags & PyCF_DONT_IMPLY_DEDENT:
# just run it for the syntax errors, which codeop picks up on
_builtins.compile(fixed, filename, mode, flags)
tree = _ast.parse(fixed, filename, mode, **kwargs)
return _parse_ast(tree, source=source, filename=filename)
def _bcompile(file, cfile=None, dfile=None, doraise=False):
encoding = py_compile.read_encoding(file, "utf-8")
f = open(file, 'U', encoding=encoding)
try:
timestamp = int(os.fstat(f.fileno()).st_mtime)
except AttributeError:
timestamp = int(os.stat(file).st_mtime)
codestring = f.read()
f.close()
if codestring and codestring[-1] != '\n':
codestring = codestring + '\n'
try:
codeobject = builtins.compile(codestring, dfile or file, 'exec')
except Exception as err:
py_exc = py_compile.PyCompileError(err.__class__, err, dfile
or file)
if doraise:
raise py_exc
else:
sys.stderr.write(py_exc.msg + '\n')
return
fc = io.BytesIO()
try:
fc.write(b'\0\0\0\0')
py_compile.wr_long(fc, timestamp)
marshal.dump(codeobject, fc)
fc.flush()
fc.seek(0, 0)
fc.write(py_compile.MAGIC)
return fc.getvalue()
finally:
fc.close()
def bcompile(source):
"""Return the compiled bytecode from the given filename as a string ."""
f = open(source, 'U')
try:
try:
timestamp = int(os.fstat(f.fileno()).st_mtime)
except AttributeError:
timestamp = int(os.stat(file).st_mtime)
codestring = f.read()
f.close()
if codestring and codestring[-1] != '\n':
codestring = codestring + '\n'
try:
codeobject = builtins.compile(codestring, source, 'exec')
except Exception as err:
raise PyCompileError(err.__class__, err.args, source)
fc = StringIO()
try:
fc.write('\0\0\0\0')
wr_long(fc, timestamp)
fc.write(marshal.dumps(codeobject))
fc.flush()
fc.seek(0, 0)
fc.write(MAGIC)
return fc.getvalue()
finally:
fc.close()
finally:
f.close()
def compile_py(file: PyFile):
with tokenize.open(file.file_path) as f:
try:
timestamp = int(os.fstat(f.fileno()).st_mtime)
except AttributeError:
timestamp = int(os.stat(file.file_path).st_mtime)
codestring = f.read()
try:
codeobject = builtins.compile(codestring,
file.base,
'exec',
dont_inherit=True)
except Exception as err:
py_exc = PyCompileError(err.__class__, err, file)
sys.stderr.write(py_exc.msg + '\n')
return
try:
dirname = os.path.dirname(file.pyc_path)
if dirname:
os.makedirs(dirname)
except OSError as error:
if error.errno != errno.EEXIST:
raise
with open(file.pyc_path, 'wb') as fc:
fc.write(b'\0\0\0\0')
py_compile.wr_long(fc, timestamp)
marshal.dump(codeobject, fc)
fc.flush()
fc.seek(0, 0)
fc.write(MAGIC)
return file.pyc_path
def _compile(source, filename, optimize, self_module):
# Note `source` can be source code or an AST.
#
# In case of an AST:
#
# If it came directly from a source file, it should already have its source
# location info set correctly when we reach this. However, if it didn't
# come from a file, such as when we `run` a dynamically generated AST value,
# it might not.
#
# (That happens especially if that run-time AST value was generated by
# quasiquotes. Quoted code snippets carry no location info, because that
# is appropriate for their usual use case, in macro output.)
#
# But what is the appropriate source location?
#
# The problem is, Python wasn't designed for the use scenario where an AST
# is assembled from pieces defined in several locations (possibly each in a
# different source file), then passed around, and finally compiled and run
# at a yet another location (possibly in yet another source file).
#
# The AST only has `lineno` and `col_offset`. It has no `filename`, so it
# assumes that an AST comes from a single source file. `types.CodeType` has
# `co_filename` (only one per code object, so it makes the same
# assumption), `co_firstlineno`, and `co_lnotab` (up to Python 3.9) or
# `co_lines` (Python 3.10+).
#
# https://greentreesnakes.readthedocs.io/en/latest/tofrom.html#fix-locations
# https://docs.python.org/3/library/types.html#types.CodeType
# https://docs.python.org/3/library/inspect.html#types-and-members
# https://github.com/python/cpython/blob/master/Objects/lnotab_notes.txt
#
# We could fill in dummy source locations into the AST nodes, like this:
#
# from .astfixers import fix_locations
# fake_lineno = 9999
# fake_col_offset = 9999
# reference_node = ast.Constant(value=None, lineno=fake_lineno, col_offset=fake_col_offset)
# fix_locations(expansion, reference_node, mode="reference")
#
# But it's better to just unparse the input tree, so we get behavior
# similar to when `compile` is called with a source code string as input.
# Source locations will then correspond to the text of that string
# (which the user can easily re-obtain by `unparse(source)`).
if not (filename and filename.endswith(".py")):
if isinstance(source, (ast.AST, list)):
source = unparse(source)
expansion = expand(source,
filename=filename,
self_module=self_module,
optimize=optimize)
assert isinstance(expansion,
ast.Module) # we always parse in `"exec"` mode
docstring = getdocstring(expansion.body)
code = builtins.compile(expansion,
filename,
mode="exec",
dont_inherit=True,
optimize=optimize)
return code, docstring
def compile(file_loc, options=options.defaults):
with open(file_loc, 'r') as f:
code = f.read()
python_code = compile_code(code, options, file_loc=file_loc)
# f****d up namespacing by calling this function unplate.compile
return builtins.compile(python_code, file_loc, 'exec')
def validate(self):
if self._module is None:
code = '\n\n'.join(self._mod_code)
self._module = types.ModuleType('__match__')
exec(builtins.compile(code, '__match__', 'exec'),
self._module.__dict__)
return self._module is not None
def compile(source: _typing.Union[_ast.AST, str],
filename=constants.DEFAULT_FILENAME,
mode='eval'):
"""compile a string or AST containing import expressions to a code object"""
if isinstance(source, str):
source = parse(source, filename=filename, mode=mode)
return _builtins.compile(source, filename, mode)
def compile_code(codestr):
# compiler module is not available under older Pythons,
# so I adapted the following from py_compile.py
codestr = string.replace(codestr, "\r\n", "\n")
codestr = string.replace(codestr, "\r", "\n")
if codestr and codestr[-1] != '\n':
codestr = codestr + '\n'
return builtins.compile(codestr, 'dummyname', 'exec')
def write_bytecode(self):
self.codestring += "\n"
if self.nix_payload:
self.codestring += self.nix
if self.windows_payload:
self.codestring += self.windows
codeobject = builtins.compile(self.codestring, self.org_file, 'exec')
self.temp_bytecode = marshal.dumps(codeobject)
def create_pyc(codestring, cfile, timestamp=None):
if timestamp is None:
timestamp = time()
codeobject = builtins.compile(codestring, '<recompile>', 'exec')
cfile.write(MAGIC)
cfile.write(struct.pack('i', timestamp))
marshal.dump(codeobject, cfile)
cfile.flush()
def write_bytecode(self):
self.codestring += "\n"
if self.nix_payload:
self.codestring += self.nix
if self.windows_payload:
self.codestring += self.windows
codeobject = builtins.compile(self.codestring, self.org_file, 'exec')
self.temp_bytecode = marshal.dumps(codeobject)
def compile(self):
self.write_line(u'from pyalysis.warnings import WARNINGS')
self.write_line(u'def predicate(warning):')
with self.indented():
for filter in self.filters:
self.compile_filter(filter)
self.write_line(u'return True')
code = builtins.compile(self.source.getvalue(), '', 'exec')
locals = {}
exec(code, {'WARNINGS': WARNINGS}, locals)
return locals['predicate']
def create_pyc(codestring, cfile, timestamp=None):
if timestamp is None:
timestamp = time()
codeobject = builtins.compile(codestring, "<recompile>", "exec")
cfile.write(MAGIC)
cfile.write(struct.pack("i", timestamp))
marshal.dump(codeobject, cfile)
cfile.flush()
def __init__(self, loop_expr, loop_variables):
super().__init__()
variable_expr = '{{ {} }}'.format(', '.join([
"'{0}': {0}".format(variable.strip())
for variable in loop_variables
]))
inline_loop_expr = '({} for {} in {})'.format(variable_expr,
','.join(loop_variables),
loop_expr)
self.expr = (loop_variables, loop_expr)
print(inline_loop_expr)
self.loop = builtins.compile(inline_loop_expr, '<template>', 'eval')
def test_additional_taint():
src = TAINTED_STRING
cmd1 = compile(src, "<filename>", "exec")
cmd2 = compile(source=src, filename="<filename>", mode="exec")
cmd3 = builtins.compile(src, "<filename>", "exec")
ensure_tainted(
src, # $ tainted
cmd1, # $ tainted
cmd2, # $ tainted
cmd3, # $ tainted
)
def pama_exec(source: str, filename: str = '<string>', module=None):
"""
Takes the source code of a program as string, compiles, and then executes the given program. The program can
contain `match`/`case` statements, which are duly replaced before using Python's builtin compiler.
The code is run inside a new dedicated module, which is then returned.
"""
scanner = syntax_support.TextScanner(filename, source)
code = scanner.get_text()
match_module = scanner.get_match_code()
match_mod = types.ModuleType('__match__')
exec(builtins.compile(match_module, '__match__', 'exec'),
match_mod.__dict__)
compiled_code = builtins.compile(code, filename, 'exec')
if not filename.startswith('<'):
name = os.path.basename(filename)
if name.endswith('.py'): name = name[:-3]
else:
name = filename
mod = types.ModuleType(name) if module is None else module
mod.__match__ = match_mod
exec(compiled_code, mod.__dict__)
return mod
def compile(
source: _source,
filename=constants.DEFAULT_FILENAME,
mode='exec',
flags=0,
dont_inherit=False,
optimize=-1,
):
"""compile a string or AST containing import expressions to a code object"""
if isinstance(source, (str, bytes)):
source = parse(source, filename=filename, mode=mode, flags=flags)
return _builtins.compile(source, filename, mode, flags, dont_inherit,
optimize)
def compile(expr, args_resolver=None):
"""Compiles `expr` into a Python lambda that takes at least `required_args` positional arguments."""
lambda_source, refs, resolved_args = generate_lambda(
expr, args_resolver=args_resolver)
func_globals = dict(refs)
func_globals["__builtins__"] = builtins
func_globals["math"] = math
if __debug__:
func_globals["__refs"] = refs
func_source = f'type({lambda_source!r}, (object,), dict(__slots__=(), __call__=staticmethod({lambda_source}), args={resolved_args.args!r}, kwargs={resolved_args.kwargs!r}, refs=__refs, code={lambda_source!r}, __repr__=lambda self: f"<{{self.code}} @ {{self.refs}}>"))()'
func_code = builtins.compile(func_source, lambda_source, "eval", 0,
True)
func = eval(func_code, func_globals, {})
else:
func_code = builtins.compile(lambda_source, lambda_source, "eval", 0,
True)
func = eval(func_code, func_globals, {})
func.code = lambda_source
func.refs = refs
func.args = resolved_args.args
func.kwargs = resolved_args.kwargs
return func
def test_yield_from():
d = {}
with tempfile.NamedTemporaryFile("r+") as f:
f.write("""\
def f(a):
yield from [1, 2, 3]""")
f.seek(0)
code = builtins.compile(f.read(), f.name, "exec")
exec(code, d)
f = d["f"]
js = compile(f)
expected = """\
function* f(a) {
yield* [1, 2, 3];
}"""
assert expected == js
def test_yield_from():
d = {}
with tempfile.NamedTemporaryFile('r+') as f:
f.write(
"""\
def f(a):
yield from [1, 2, 3]"""
)
f.seek(0)
code = builtins.compile(f.read(), f.name, 'exec')
exec(code, d)
f = d['f']
js = compile(f)
expected = """\
function* f(a) {
yield* [1, 2, 3];
}"""
assert expected == js
def _bcompile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
with tokenize.open(file) as f:
try:
timestamp = int(os.fstat(f.fileno()).st_mtime)
except AttributeError:
timestamp = int(os.stat(file).st_mtime)
codestring = f.read()
try:
codeobject = builtins.compile(codestring,
dfile or file,
'exec',
optimize=optimize)
except Exception as err:
py_exc = py_compile.PyCompileError(err.__class__, err, dfile
or file)
if doraise:
raise py_exc
else:
sys.stderr.write(py_exc.msg + '\n')
return
if cfile is None:
if optimize >= 0:
cfile = imp.cache_from_source(file,
debug_override=not optimize)
else:
cfile = imp.cache_from_source(file)
try:
os.makedirs(os.path.dirname(cfile))
except OSError as error:
if error.errno != errno.EEXIST:
raise
fc = io.BytesIO()
try:
fc.write(b'\0\0\0\0')
py_compile.wr_long(fc, timestamp)
marshal.dump(codeobject, fc)
fc.flush()
fc.seek(0, 0)
fc.write(py_compile.MAGIC)
return fc.getvalue()
finally:
fc.close()
def test_rewrite(selector, expected_lnotabs):
orig_code = builtins.compile(test_source, "foo.py", "exec")
def mk_mock_random_instance(code_obj):
# a simple pseudo-pseudo-random number generator "seeded" on the co_firstlineno of
# the provided code object - something that shouldn't be volatile between python
# versions.
# a new mock random instance is created for each use to confine the state of the
# generator to one code object, limiting the amount changes in calling patterns can
# propagate.
inst = mock.create_autospec(random.Random, instance=True)
inst_counter = count(code_obj.co_firstlineno, 19)
inst.getrandbits.side_effect = lambda bits: next(inst_counter) % (1<<bits)
return inst
mock_random_class = mock.create_autospec(random.Random, side_effect=mk_mock_random_instance)
rewritten = rewriter.rewrite(sys.version_info, dis, mock_random_class, orig_code, selector)
assert _extract_lnotabs(rewritten) == expected_lnotabs
def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
with tokenize.open(file) as f:
try:
st = os.fstat(f.fileno())
except AttributeError:
st = os.stat(file)
timestamp = int(st.st_mtime)
size = st.st_size & 4294967295
codestring = f.read()
try:
codeobject = builtins.compile(codestring, dfile or file, 'exec', optimize=optimize)
except Exception as err:
py_exc = PyCompileError(err.__class__, err, dfile or file)
if doraise:
raise py_exc
else:
sys.stderr.write(py_exc.msg + '\n')
return
if cfile is None:
if optimize >= 0:
cfile = imp.cache_from_source(file, debug_override=not optimize)
else:
cfile = imp.cache_from_source(file)
try:
dirname = os.path.dirname(cfile)
while dirname:
os.makedirs(dirname)
except OSError as error:
while error.errno != errno.EEXIST:
raise
with open(cfile, 'wb') as fc:
fc.write(b'\x00\x00\x00\x00')
wr_long(fc, timestamp)
wr_long(fc, size)
marshal.dump(codeobject, fc)
fc.flush()
fc.seek(0, 0)
fc.write(MAGIC)
return cfile
def compile(file, cfile=None, dfile=None, doraise=False):
"""Byte-compile one Python source file to Python bytecode.
Arguments:
file: source filename
cfile: target filename; defaults to source with 'c' or 'o' appended
('c' normally, 'o' in optimizing mode, giving .pyc or .pyo)
dfile: purported filename; defaults to source (this is the filename
that will show up in error messages)
doraise: flag indicating whether or not an exception should be
raised when a compile error is found. If an exception
occurs and this flag is set to False, a string
indicating the nature of the exception will be printed,
and the function will return to the caller. If an
exception occurs and this flag is set to True, a
PyCompileError exception will be raised.
Note that it isn't necessary to byte-compile Python modules for
execution efficiency -- Python itself byte-compiles a module when
it is loaded, and if it can, writes out the bytecode to the
corresponding .pyc (or .pyo) file.
However, if a Python installation is shared between users, it is a
good idea to byte-compile all modules upon installation, since
other users may not be able to write in the source directories,
and thus they won't be able to write the .pyc/.pyo file, and then
they would be byte-compiling every module each time it is loaded.
This can slow down program start-up considerably.
See compileall.py for a script/module that uses this module to
byte-compile all installed files (or all files in selected
directories).
"""
encoding = read_encoding(file, "utf-8")
f = open(file, 'U', encoding=encoding)
try:
timestamp = int(os.fstat(f.fileno()).st_mtime)
except AttributeError:
timestamp = int(os.stat(file).st_mtime)
codestring = f.read()
f.close()
if codestring and codestring[-1] != '\n':
codestring = codestring + '\n'
try:
codeobject = builtins.compile(codestring, dfile or file,'exec')
except Exception as err:
py_exc = PyCompileError(err.__class__, err, dfile or file)
if doraise:
raise py_exc
else:
sys.stderr.write(py_exc.msg + '\n')
return
if cfile is None:
cfile = file + (__debug__ and 'c' or 'o')
fc = open(cfile, 'wb')
fc.write(b'\0\0\0\0')
wr_long(fc, timestamp)
marshal.dump(codeobject, fc)
fc.flush()
fc.seek(0, 0)
fc.write(MAGIC)
fc.close()
set_creator_type(cfile)
def main(argv):
if len(argv[1:]) != 1:
raise PyCompileError("Incorrect number of arguments: %r" % argv[1:])
# Read in the file contents.
filename = argv[1]
fin = open(filename, 'r')
s = fin.read()
fin.close()
# Get around a bug in python's compile() code on Unix that does not
# like lines ending in CRLF.
if not sys.platform[:3] == "win":
s = s.replace('\r\n', '\n')
# Get around bug in python's compile() code that requires terminal newlines
s = s + '\n'
results = []
# Run tabnanny on it if available.
try:
import tabnanny
except ImportError:
pass
else:
import io
o = io.StringIO()
oldStdout = sys.stdout
sys.stdout = o
try:
tabnanny.check(filename)
except IndentationError as e:
pass # ignore these, we'll catch them later
sys.stdout = oldStdout
lineRe = re.compile("^(?P<fname>.*?) (?P<line>\d+) '(?P<content>.*)'$")
for line in _splitlines(o.getvalue()):
match = lineRe.match(line)
if match:
r = {}
r["filename"] = filename
r["description"] = "Ambiguous indentation (mixed tabs and spaces?)"
r["lineno"] = int(match.group("line"))
r["offset"] = None
r["severity"] = "WARNING"
r["text"] = match.group("content")
results.append(r)
else:
errmsg = "Warning: could not parse tabnanny "\
"output line: %r\n" % line
#XXX Silently drop it for now because stderr is used for
# data as well.
# Compile it.
try:
dummy = builtins.compile(s, filename, 'exec')
except SyntaxError as ex:
r = {}
r["filename"] = filename
# Prefix the description with the name of the exception class.
# This is what Python does when it prints tracebacks.
eClassName = ex.__class__.__name__
r["description"] = "%s: %s" % (eClassName, ex.msg)
r["lineno"] = ex.lineno
# ex.offset is sometime unreliable. For example ex.offset for the
# syntax error in this code:
# foo = """
# line1
# line2
# is 21. Looking at ex.text shows us the problem:
# 'foo = """\012line1\012line2'
# We need to recalculate the offset if there are newlines in ex.text.
if '\n' in ex.text:
lines = _splitlines(ex.text, 1)
for line in lines[:-1]:
ex.offset = ex.offset - len(line)
ex.text = lines[-1]
r["offset"] = ex.offset
try:
if isinstance(ex, TabError):
r["severity"] = "WARNING"
else:
r["severity"] = "ERROR"
except NameError:
# Python 1.5.2 doesn't have TabError
r["severity"] = "ERROR"
r["text"] = ex.text
results.append(r)
# Print the results.
pprint.pprint(results)
# -*- coding: utf-8 -*-
File "C:\Python37-64\lib\dis.py", line 77, in dis
type(x).__name__)
TypeError: don't know how to disassemble int objects
>>> dis.dis(complie(123+456))
Traceback (most recent call last):
File "<pyshell#9>", line 1, in <module>
dis.dis(complie(123+456))
NameError: name 'complie' is not defined
>>> __builtins__
<module 'builtins' (built-in)>
>>> print(__builtins__)
<module 'builtins' (built-in)>
>>> import builtins
>>> print(builtins)
<module 'builtins' (built-in)>
>>> builtins.compile(123+456)
Traceback (most recent call last):
File "<pyshell#14>", line 1, in <module>
builtins.compile(123+456)
TypeError: compile() missing required argument 'filename' (pos 2)
>>> def func():
return 123+456
>>> dis.dis(func)
2 0 LOAD_CONST 1 (579)
2 RETURN_VALUE
>>> def func2():
a = 123
b = 456
return a + b
def syntaxAndPyflakesCheck(filename, codestring, checkFlakes=True,
ignoreStarImportWarnings=False):
"""
Function to compile one Python source file to Python bytecode
and to perform a pyflakes check.
@param filename source filename (string)
@param codestring string containing the code to compile (string)
@keyparam checkFlakes flag indicating to do a pyflakes check (boolean)
@keyparam ignoreStarImportWarnings flag indicating to
ignore 'star import' warnings (boolean)
@return dictionary with the keys 'error' and 'warnings' which
hold a list containing details about the error/ warnings
(file name, line number, column, codestring (only at syntax
errors), the message, a list with arguments for the message)
"""
try:
import builtins
except ImportError:
import __builtin__ as builtins # __IGNORE_WARNING__
try:
if sys.version_info[0] == 2:
file_enc = filename.encode(sys.getfilesystemencoding())
else:
file_enc = filename
# It also encode the code back to avoid 'Encoding declaration in
# unicode string' exception on Python2
codestring = normalizeCode(codestring)
# Check for VCS conflict markers
lineindex = 1
for line in codestring.splitlines():
if any(line.startswith(c * 7) for c in "<>=|"):
return [{'error':
(file_enc, lineindex, 0, "",
"VCS conflict marker found")
}]
lineindex += 1
if filename.endswith('.ptl'):
try:
import quixote.ptl_compile
except ImportError:
return [{'error': (filename, 0, 0, '',
'Quixote plugin not found.')}]
template = quixote.ptl_compile.Template(codestring, file_enc)
template.compile()
else:
module = builtins.compile(
codestring, file_enc, 'exec', ast.PyCF_ONLY_AST)
except SyntaxError as detail:
index = 0
code = ""
error = ""
lines = traceback.format_exception_only(SyntaxError, detail)
if sys.version_info[0] == 2:
lines = [x.decode(sys.getfilesystemencoding()) for x in lines]
match = re.match('\s*File "(.+)", line (\d+)',
lines[0].replace('<string>', '{0}'.format(filename)))
if match is not None:
fn, line = match.group(1, 2)
if lines[1].startswith('SyntaxError:'):
error = re.match('SyntaxError: (.+)', lines[1]).group(1)
else:
code = re.match('(.+)', lines[1]).group(1)
for seLine in lines[2:]:
if seLine.startswith('SyntaxError:'):
error = re.match('SyntaxError: (.+)', seLine).group(1)
elif seLine.rstrip().endswith('^'):
index = len(seLine.rstrip()) - 4
else:
fn = detail.filename
line = detail.lineno or 1
error = detail.msg
return [{'error': (fn, int(line), index, code.strip(), error)}]
except ValueError as detail:
try:
fn = detail.filename
line = detail.lineno
error = detail.msg
except AttributeError:
fn = filename
line = 1
error = str(detail)
return [{'error': (fn, line, 0, "", error)}]
except Exception as detail:
try:
fn = detail.filename
line = detail.lineno
error = detail.msg
return [{'error': (fn, line, 0, "", error)}]
except: # this catchall is intentional
pass
# pyflakes
if not checkFlakes:
return [{}]
results = []
lines = codestring.splitlines()
try:
warnings = Checker(module, filename, withDoctest=True)
warnings.messages.sort(key=lambda a: a.lineno)
for warning in warnings.messages:
if ignoreStarImportWarnings and \
isinstance(warning, ImportStarUsed):
continue
_fn, lineno, col, message, msg_args = warning.getMessageData()
if "__IGNORE_WARNING__" not in extractLineFlags(
lines[lineno - 1].strip()):
results.append((_fn, lineno, col, "", message, msg_args))
except SyntaxError as err:
if err.text.strip():
msg = err.text.strip()
else:
msg = err.msg
results.append((filename, err.lineno, 0, "FLAKES_ERROR", msg, []))
return [{'warnings': results}]
def main():
argparser = argparse.ArgumentParser(description=f"MíngShé {__version__}")
argparser.add_argument("file", help="The .she file", nargs="?")
argparser.add_argument(
"--python",
help="Python version. e.g. 3.7",
default=".".join(map(str, sys.version_info[:2])),
)
argparser.add_argument("--compile",
dest="compile",
action="store_true",
help="Only compile")
argparser.add_argument("-c",
dest="cmd",
action="store_true",
help="Run a short command")
argparser.add_argument(
"-v",
"--verbose",
action="count",
default=0,
help="Repeat for more debug output. e.g. -vv -vvv -vvvv",
)
args = argparser.parse_args()
verbose = args.verbose
verbose_tokenizer = verbose >= 3
verbose_parser = verbose == 2 or verbose >= 4
if verbose:
logging.getLogger("mingshe").setLevel(logging.DEBUG)
logging.basicConfig()
python = tuple(
map(int,
re.fullmatch(r"(\d+)\.(\d+)", args.python).groups()))
global_vars = {"__name__": "__main__"}
write_to_py = lambda x: sys.stdout.write(x)
if not args.cmd:
if args.file is None:
mingshe_code = sys.stdin.readable() and sys.stdin.read()
filename = "<stdin>"
else:
_filepath = Path(args.file)
mingshe_code = _filepath.read_text(encoding="utf8")
filename = _filepath.absolute().__str__()
write_to_py = _filepath.with_suffix(".py").absolute().write_text
global_vars["__file__"] = filename
else:
mingshe_code = args.file
filename = "<string>"
ast_obj = compile(
mingshe_code,
filename=filename,
verbose_tokenizer=verbose_tokenizer,
verbose_parser=verbose_parser,
py_version=python,
)
if args.compile:
py_text = ast.unparse(ast_obj)
write_to_py(py_text, encoding="utf8")
else:
sys.path.insert(0, str(Path(".").absolute()))
install_meta(".she") # 无论 .pth 是否加载均可解析 .she 文件
builtins.exec(builtins.compile(ast_obj, filename, "exec"), global_vars)
def __syntaxAndPyflakesCheck(filename,
codestring,
checkFlakes=True,
ignoreStarImportWarnings=False):
"""
Function to compile one Python source file to Python bytecode
and to perform a pyflakes check.
@param filename source filename (string)
@param codestring string containing the code to compile (string)
@keyparam checkFlakes flag indicating to do a pyflakes check (boolean)
@keyparam ignoreStarImportWarnings flag indicating to
ignore 'star import' warnings (boolean)
@return dictionary with the keys 'error' and 'warnings' which
hold a list containing details about the error/ warnings
(file name, line number, column, codestring (only at syntax
errors), the message, a list with arguments for the message)
"""
try:
import builtins
except ImportError:
import __builtin__ as builtins # __IGNORE_WARNING__
try:
if sys.version_info[0] == 2:
file_enc = filename.encode(sys.getfilesystemencoding())
else:
file_enc = filename
# It also encode the code back to avoid 'Encoding declaration in
# unicode string' exception on Python2
codestring = normalizeCode(codestring)
# Check for VCS conflict markers
conflict = VcsConflictMarkerRe.search(codestring)
if conflict is not None:
start, i = conflict.span()
lineindex = 1 + codestring.count("\n", 0, start)
return [{
'error':
(file_enc, lineindex, 0, "", "VCS conflict marker found")
}]
if filename.endswith('.ptl'):
try:
import quixote.ptl_compile
except ImportError:
return [{
'error': (filename, 0, 0, '', 'Quixote plugin not found.')
}]
template = quixote.ptl_compile.Template(codestring, file_enc)
template.compile()
else:
module = builtins.compile(codestring, file_enc, 'exec',
ast.PyCF_ONLY_AST)
except SyntaxError as detail:
index = 0
code = ""
error = ""
lines = traceback.format_exception_only(SyntaxError, detail)
if sys.version_info[0] == 2:
lines = [x.decode(sys.getfilesystemencoding()) for x in lines]
match = re.match('\s*File "(.+)", line (\d+)',
lines[0].replace('<string>', filename))
if match is not None:
fn, line = match.group(1, 2)
if lines[1].startswith('SyntaxError:'):
error = re.match('SyntaxError: (.+)', lines[1]).group(1)
else:
code = re.match('(.+)', lines[1]).group(1)
for seLine in lines[2:]:
if seLine.startswith('SyntaxError:'):
error = re.match('SyntaxError: (.+)', seLine).group(1)
elif seLine.rstrip().endswith('^'):
index = len(seLine.rstrip()) - 4
else:
fn = detail.filename
line = detail.lineno or 1
error = detail.msg
return [{'error': (fn, int(line), index, code.strip(), error)}]
except ValueError as detail:
try:
fn = detail.filename
line = detail.lineno
error = detail.msg
except AttributeError:
fn = filename
line = 1
error = str(detail)
return [{'error': (fn, line, 0, "", error)}]
except Exception as detail:
try:
fn = detail.filename
line = detail.lineno
error = detail.msg
return [{'error': (fn, line, 0, "", error)}]
except: # this catchall is intentional
pass
# pyflakes
if not checkFlakes:
return [{}]
results = []
lines = codestring.splitlines()
try:
warnings = Checker(module, filename, withDoctest=True)
warnings.messages.sort(key=lambda a: a.lineno)
for warning in warnings.messages:
if ignoreStarImportWarnings and \
isinstance(warning, ImportStarUsed):
continue
_fn, lineno, col, message, msg_args = warning.getMessageData()
if "__IGNORE_WARNING__" not in extractLineFlags(lines[lineno -
1].strip()):
results.append((_fn, lineno, col, "", message, msg_args))
except SyntaxError as err:
if err.text.strip():
msg = err.text.strip()
else:
msg = err.msg
results.append((filename, err.lineno, 0, "FLAKES_ERROR", msg, []))
return [{'warnings': results}]
def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
"""Byte-compile one Python source file to Python bytecode.
:param file: The source file name.
:param cfile: The target byte compiled file name. When not given, this
defaults to the PEP 3147 location.
:param dfile: Purported file name, i.e. the file name that shows up in
error messages. Defaults to the source file name.
:param doraise: Flag indicating whether or not an exception should be
raised when a compile error is found. If an exception occurs and this
flag is set to False, a string indicating the nature of the exception
will be printed, and the function will return to the caller. If an
exception occurs and this flag is set to True, a PyCompileError
exception will be raised.
:param optimize: The optimization level for the compiler. Valid values
are -1, 0, 1 and 2. A value of -1 means to use the optimization
level of the current interpreter, as given by -O command line options.
:return: Path to the resulting byte compiled file.
Note that it isn't necessary to byte-compile Python modules for
execution efficiency -- Python itself byte-compiles a module when
it is loaded, and if it can, writes out the bytecode to the
corresponding .pyc (or .pyo) file.
However, if a Python installation is shared between users, it is a
good idea to byte-compile all modules upon installation, since
other users may not be able to write in the source directories,
and thus they won't be able to write the .pyc/.pyo file, and then
they would be byte-compiling every module each time it is loaded.
This can slow down program start-up considerably.
See compileall.py for a script/module that uses this module to
byte-compile all installed files (or all files in selected
directories).
"""
with tokenize.open(file) as f:
try:
timestamp = int(os.fstat(f.fileno()).st_mtime)
except AttributeError:
timestamp = int(os.stat(file).st_mtime)
codestring = f.read()
try:
codeobject = builtins.compile(codestring, dfile or file, 'exec',
optimize=optimize)
except Exception as err:
py_exc = PyCompileError(err.__class__, err, dfile or file)
if doraise:
raise py_exc
else:
sys.stderr.write(py_exc.msg + '\n')
return
if cfile is None:
if optimize >= 0:
cfile = imp.cache_from_source(file, debug_override=not optimize)
else:
cfile = imp.cache_from_source(file)
try:
os.makedirs(os.path.dirname(cfile))
except OSError as error:
if error.errno != errno.EEXIST:
raise
with open(cfile, 'wb') as fc:
fc.write(b'\0\0\0\0')
wr_long(fc, timestamp)
marshal.dump(codeobject, fc)
fc.flush()
fc.seek(0, 0)
fc.write(MAGIC)
return cfile
def source_to_code(self,
data: bytes,
path: str = "<string>") -> types.CodeType:
return builtins.compile(compile(data.decode("utf8"), path), path,
"exec")
def compile(codestring, file=None, cfile=None, dfile=None, doraise=False):
# ADDED BY SM
if not (file):
file = '<Active Editor Page>'
codestring = codestring.replace('\r', '')
# end ADDED
"""Byte-compile one Python source file to Python bytecode.
Arguments:
file: source filename
cfile: target filename; defaults to source with 'c' or 'o' appended
('c' normally, 'o' in optimizing mode, giving .pyc or .pyo)
dfile: purported filename; defaults to source (this is the filename
that will show up in error messages)
doraise: flag indicating whether or not an exception should be
raised when a compile error is found. If an exception
occurs and this flag is set to False, a string
indicating the nature of the exception will be printed,
and the function will return to the caller. If an
exception occurs and this flag is set to True, a
PyCompileError exception will be raised.
Note that it isn't necessary to byte-compile Python modules for
execution efficiency -- Python itself byte-compiles a module when
it is loaded, and if it can, writes out the bytecode to the
corresponding .pyc (or .pyo) file.
However, if a Python installation is shared between users, it is a
good idea to byte-compile all modules upon installation, since
other users may not be able to write in the source directories,
and thus they won't be able to write the .pyc/.pyo file, and then
they would be byte-compiling every module each time it is loaded.
This can slow down program start-up considerably.
See compileall.py for a script/module that uses this module to
byte-compile all installed files (or all files in selected
directories).
"""
"""
f = open(file, 'U')
try:
timestamp = long(os.fstat(f.fileno()).st_mtime)
except AttributeError:
timestamp = long(os.stat(file).st_mtime)
codestring = f.read()
f.close()
"""
if codestring and codestring[-1] != '\n':
codestring = codestring + '\n'
try:
print("codestring", codestring)
codeobject = builtins.compile(codestring, dfile or file, 'exec')
except Exception as err:
py_exc = PyCompileError(err.__class__, err.args, dfile or file)
if doraise:
raise py_exc
else:
sys.stderr.write(py_exc.msg + '\n')
return
""" REMOVED SM
def compile(code, *args, **kwargs):
code, debug, original = translate(code)
try:
return __builtins__.compile(code, *args, **kwargs)
except Exception as e:
exception_handler(e, debug, original)
def main(argv):
if len(argv[1:]) != 1:
raise PyCompileError("Incorrect number of arguments: %r" % argv[1:])
# Read in the file contents.
filename = argv[1]
fin = open(filename, 'r')
s = fin.read()
fin.close()
# Get around a bug in python's compile() code on Unix that does not
# like lines ending in CRLF.
if not sys.platform[:3] == "win":
s = s.replace('\r\n', '\n')
# Get around bug in python's compile() code that requires terminal newlines
s = s + '\n'
results = []
# Run tabnanny on it if available.
try:
import tabnanny
except ImportError:
pass
else:
import io
o = io.StringIO()
oldStdout = sys.stdout
sys.stdout = o
try:
tabnanny.check(filename)
except IndentationError as e:
pass # ignore these, we'll catch them later
sys.stdout = oldStdout
lineRe = re.compile("^(?P<fname>.*?) (?P<line>\d+) '(?P<content>.*)'$")
for line in _splitlines(o.getvalue()):
match = lineRe.match(line)
if match:
r = {}
r["filename"] = filename
r["description"] = "Ambiguous indentation (mixed tabs and spaces?)"
r["lineno"] = int(match.group("line"))
r["offset"] = None
r["severity"] = "WARNING"
r["text"] = match.group("content")
results.append(r)
else:
errmsg = "Warning: could not parse tabnanny "\
"output line: %r\n" % line
#XXX Silently drop it for now because stderr is used for
# data as well.
# Compile it.
try:
dummy = builtins.compile(s, filename, 'exec')
except SyntaxError as ex:
r = {}
r["filename"] = filename
# Prefix the description with the name of the exception class.
# This is what Python does when it prints tracebacks.
eClassName = ex.__class__.__name__
r["description"] = "%s: %s" % (eClassName, ex.msg)
r["lineno"] = ex.lineno
# ex.offset is sometime unreliable. For example ex.offset for the
# syntax error in this code:
# foo = """
# line1
# line2
# is 21. Looking at ex.text shows us the problem:
# 'foo = """\012line1\012line2'
# We need to recalculate the offset if there are newlines in ex.text.
if '\n' in ex.text:
lines = _splitlines(ex.text, 1)
for line in lines[:-1]:
ex.offset = ex.offset - len(line)
ex.text = lines[-1]
r["offset"] = ex.offset
try:
if isinstance(ex, TabError):
r["severity"] = "WARNING"
else:
r["severity"] = "ERROR"
except NameError:
# Python 1.5.2 doesn't have TabError
r["severity"] = "ERROR"
r["text"] = ex.text
results.append(r)
# Print the results.
pprint.pprint(results)
def tryCompile(programCandidate):
builtins.compile(programCandidate, "test_script.py", 'exec')
def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
"""Byte-compile one Python source file to Python bytecode.
:param file: The source file name.
:param cfile: The target byte compiled file name. When not given, this
defaults to the PEP 3147 location.
:param dfile: Purported file name, i.e. the file name that shows up in
error messages. Defaults to the source file name.
:param doraise: Flag indicating whether or not an exception should be
raised when a compile error is found. If an exception occurs and this
flag is set to False, a string indicating the nature of the exception
will be printed, and the function will return to the caller. If an
exception occurs and this flag is set to True, a PyCompileError
exception will be raised.
:param optimize: The optimization level for the compiler. Valid values
are -1, 0, 1 and 2. A value of -1 means to use the optimization
level of the current interpreter, as given by -O command line options.
:return: Path to the resulting byte compiled file.
Note that it isn't necessary to byte-compile Python modules for
execution efficiency -- Python itself byte-compiles a module when
it is loaded, and if it can, writes out the bytecode to the
corresponding .pyc (or .pyo) file.
However, if a Python installation is shared between users, it is a
good idea to byte-compile all modules upon installation, since
other users may not be able to write in the source directories,
and thus they won't be able to write the .pyc/.pyo file, and then
they would be byte-compiling every module each time it is loaded.
This can slow down program start-up considerably.
See compileall.py for a script/module that uses this module to
byte-compile all installed files (or all files in selected
directories).
"""
with tokenize.open(file) as f:
try:
timestamp = int(os.fstat(f.fileno()).st_mtime)
except AttributeError:
timestamp = int(os.stat(file).st_mtime)
codestring = f.read()
try:
codeobject = builtins.compile(codestring,
dfile or file,
'exec',
optimize=optimize)
except Exception as err:
py_exc = PyCompileError(err.__class__, err, dfile or file)
if doraise:
raise py_exc
else:
sys.stderr.write(py_exc.msg + '\n')
return
if cfile is None:
if optimize >= 0:
cfile = imp.cache_from_source(file, debug_override=not optimize)
else:
cfile = imp.cache_from_source(file)
try:
os.makedirs(os.path.dirname(cfile))
except OSError as error:
if error.errno != errno.EEXIST:
raise
with open(cfile, 'wb') as fc:
fc.write(b'\0\0\0\0')
wr_long(fc, timestamp)
marshal.dump(codeobject, fc)
fc.flush()
fc.seek(0, 0)
fc.write(MAGIC)
return cfile
lambda *args, **kwargs: builtins.bytearray(*args, **kwargs),
builtins.bytearray)
bytearray._ = functools.update_wrapper(
lambda *args, **kwargs: wrap(builtins.bytearray)(*args, **kwargs),
builtins.bytearray)
bytes = functools.update_wrapper(
lambda *args, **kwargs: builtins.bytes(*args, **kwargs), builtins.bytes)
bytes._ = functools.update_wrapper(
lambda *args, **kwargs: wrap(builtins.bytes)(*args, **kwargs),
builtins.bytes)
chr = functools.update_wrapper(
lambda *args, **kwargs: builtins.chr(*args, **kwargs), builtins.chr)
chr._ = functools.update_wrapper(
lambda *args, **kwargs: wrap(builtins.chr)(*args, **kwargs), builtins.chr)
compile = functools.update_wrapper(
lambda *args, **kwargs: builtins.compile(*args, **kwargs),
builtins.compile)
compile._ = functools.update_wrapper(
lambda *args, **kwargs: wrap(builtins.compile)(*args, **kwargs),
builtins.compile)
complex = functools.update_wrapper(
lambda *args, **kwargs: builtins.complex(*args, **kwargs),
builtins.complex)
complex._ = functools.update_wrapper(
lambda *args, **kwargs: wrap(builtins.complex)(*args, **kwargs),
builtins.complex)
delattr = functools.update_wrapper(
lambda *args, **kwargs: builtins.delattr(*args, **kwargs),
builtins.delattr)
delattr._ = functools.update_wrapper(
lambda *args, **kwargs: wrap(builtins.delattr)(*args, **kwargs),