def verify_file(real_source_filename, real_bytecode_filename):
"""Compile *real_source_filename* using
the running Python interpreter. Then
write bytecode out to a new place again using
Python's routines.
Next load it in using two of our routines.
Compare that the code objects there are equal.
Next write out the bytecode (using the same Python
bytecode writin routine as in step 1.
Finally compare the bytecode files.
"""
tempdir = tempfile.gettempdir()
source_filename = os.path.join(tempdir, "testing.py")
if not os.path.exists(real_source_filename):
return
if PYTHON_VERSION < 3.0:
f = open(real_source_filename, 'U')
elif PYTHON_VERSION == 3.0:
# Too hard to get working on 3.0
return
elif 3.1 <= PYTHON_VERSION <= 3.4:
f = open(real_source_filename, "rb")
else:
f = open(real_source_filename, newline=None, errors='backslashreplace')
codestring = f.read()
f.close()
codeobject1 = compile(codestring, source_filename, 'exec')
(version, timestamp, magic_int, codeobject2, is_pypy,
source_size) = load_module(real_bytecode_filename)
# A hack for PyPy 3.2
if magic_int == 3180 + 7:
magic_int = 48
assert magics.MAGIC == magics.int2magic(magic_int), \
("magic_int %d vs %d in %s/%s" %
(magic_int, magics.PYTHON_MAGIC_INT, os.getcwd(), real_bytecode_filename))
bytecode_filename1 = os.path.join(tempdir, "testing1.pyc")
dump_compile(codeobject1, bytecode_filename1, timestamp, magics.MAGIC)
(version, timestamp, magic_int, codeobject3, is_pypy,
source_size) = load_module(real_bytecode_filename, fast_load=not is_pypy)
# compare_code(codeobject1, codeobject2)
# compare_code(codeobject2, codeobject3)
bytecode_filename2 = os.path.join(tempdir, "testing2.pyc")
dump_compile(codeobject1, bytecode_filename2, timestamp,
magics.int2magic(magic_int))
compare_bytecode_files(bytecode_filename1, bytecode_filename2)
return
def verify_file(real_source_filename, real_bytecode_filename):
"""Compile *real_source_filename* using
the running Python interpreter. Then
write bytecode out to a new place again using
Python's routines.
Next load it in using two of our routines.
Compare that the code objects there are equal.
Next write out the bytecode (using the same Python
bytecode writin routine as in step 1.
Finally compare the bytecode files.
"""
tempdir = tempfile.gettempdir()
source_filename = os.path.join(tempdir, "testing.py")
if not os.path.exists(real_source_filename):
return
try:
f = open(real_source_filename, 'U')
except:
return
codestring = f.read()
f.close()
codeobject1 = compile(codestring, source_filename, 'exec')
(version, timestamp, magic_int, codeobject2, is_pypy,
source_size) = load_module(real_bytecode_filename)
# A hack for PyPy 3.2
if magic_int == 3180 + 7:
magic_int = 48
assert MAGIC == magics.int2magic(magic_int), \
("magic_int %d vs %d in %s/%s" %
(magic_int, magics.magic2int(MAGIC), os.getcwd(), real_bytecode_filename))
bytecode_filename1 = os.path.join(tempdir, "testing1.pyc")
dump_compile(codeobject1, bytecode_filename1, timestamp, MAGIC)
(version, timestamp, magic_int, codeobject3, is_pypy,
source_size) = load_module(real_bytecode_filename, fast_load=not is_pypy)
# compare_code(codeobject1, codeobject2)
# compare_code(codeobject2, codeobject3)
bytecode_filename2 = os.path.join(tempdir, "testing2.pyc")
dump_compile(codeobject1, bytecode_filename2, timestamp,
magics.int2magic(magic_int))
compare_bytecode_files(bytecode_filename1, bytecode_filename2)
return
def verify_file(real_source_filename, real_bytecode_filename):
"""Compile *real_source_filename* using
the running Python interpreter. Then
write bytecode out to a new place again using
Python's routines.
Next load it in using two of our routines.
Compare that the code objects there are equal.
Next write out the bytecode (using the same Python
bytecode writin routine as in step 1.
Finally compare the bytecode files.
"""
tempdir = tempfile.gettempdir()
source_filename = os.path.join(tempdir, "testing.py")
if not os.path.exists(real_source_filename):
return
try:
f = open(real_source_filename, 'U')
except:
return
codestring = f.read()
f.close()
codeobject1 = compile(codestring, source_filename,'exec')
(version, timestamp, magic_int, codeobject2, is_pypy,
source_size) = load_module(real_bytecode_filename)
# A hack for PyPy 3.2
if magic_int == 3180+7:
magic_int = 48
assert MAGIC == magics.int2magic(magic_int), \
("magic_int %d vs %d in %s/%s" %
(magic_int, magics.magic2int(MAGIC), os.getcwd(), real_bytecode_filename))
bytecode_filename1 = os.path.join(tempdir, "testing1.pyc")
dump_compile(codeobject1, bytecode_filename1, timestamp, MAGIC)
(version, timestamp, magic_int, codeobject3, is_pypy,
source_size) = load_module(real_bytecode_filename, fast_load=not is_pypy)
# compare_code(codeobject1, codeobject2)
# compare_code(codeobject2, codeobject3)
bytecode_filename2 = os.path.join(tempdir, "testing2.pyc")
dump_compile(codeobject1, bytecode_filename2, timestamp, magics.int2magic(magic_int))
compare_bytecode_files(bytecode_filename1, bytecode_filename2)
return
def test_basic(self):
"""Basic test of magic numbers"""
if hasattr(sys, 'version_info'):
version = version_tuple_to_str()
if IS_PYPY:
version += 'pypy'
self.assertTrue(
version in magics.magics.keys(),
"version %s is not in magic.magics.keys: %s" %
(version, magics.magics.keys()))
self.assertEqual(magics.MAGIC,
magics.int2magic(magics.magic2int(magics.MAGIC)))
lookup = str(PYTHON_VERSION)
if IS_PYPY:
lookup += 'pypy'
self.assertTrue(
lookup in magics.magics.keys(),
"PYTHON VERSION %s is not in magic.magics.keys: %s" %
(lookup, magics.magics.keys()))
if not (3, 5, 2) <= sys.version_info < (3, 6, 0):
self.assertEqual(
magics.sysinfo2magic(), magics.MAGIC,
"magic from imp.get_magic() for %s "
"should be sysinfo2magic()" % lookup)
def test_basic(self):
"""Basic test of magic numbers"""
current = imp.get_magic()
if hasattr(sys, 'version_info'):
version = '.'.join([str(v) for v in sys.version_info[0:3]])
if IS_PYPY:
version += 'pypy'
self.assertTrue(
version in magics.magics.keys(),
"version %s is not in magic.magics.keys: %s" %
(version, magics.magics.keys()))
self.assertEqual(current, magics.int2magic(magics.magic2int(current)))
lookup = str(PYTHON_VERSION)
if IS_PYPY:
lookup += 'pypy'
self.assertTrue(
lookup in magics.magics.keys(),
"PYTHON VERSION %s is not in magic.magics.keys: %s" %
(lookup, magics.magics.keys()))
if not (3, 5, 2) <= sys.version_info < (3, 6, 0):
self.assertEqual(
magics.sysinfo2magic(), current,
"magic from imp.get_magic() for %s "
"should be sysinfo2magic()" % lookup)
def test_basic(self):
"""Basic test of magic numbers"""
current = imp.get_magic()
if hasattr(sys, 'version_info'):
version = '.'.join([str(v) for v in sys.version_info[0:3]])
self.assertTrue(version in magics.magics.keys(),
"version %s is not in magic.magics.keys: %s" %
(version, magics.magics.keys()))
self.assertEqual(current, magics.int2magic(magics.magic2int(current)))
self.assertTrue(str(PYTHON_VERSION) in magics.magics.keys(),
"PYTHON VERSION %s is not in magic.magics.keys: %s" %
(PYTHON_VERSION, magics.magics.keys()))
def test_basic(self):
"""Basic test of magic numbers"""
current = imp.get_magic()
if hasattr(sys, 'version_info'):
version = '.'.join([str(v) for v in sys.version_info[0:3]])
if IS_PYPY:
version += 'pypy'
self.assertTrue(version in magics.magics.keys(),
"version %s is not in magic.magics.keys: %s" %
(version, magics.magics.keys()))
self.assertEqual(current, magics.int2magic(magics.magic2int(current)))
lookup = str(PYTHON_VERSION)
if IS_PYPY:
lookup += 'pypy'
self.assertTrue(lookup in magics.magics.keys(),
"PYTHON VERSION %s is not in magic.magics.keys: %s" %
(lookup, magics.magics.keys()))
if not (3, 5, 2) <= sys.version_info < (3, 6, 0):
self.assertEqual(magics.sysinfo2magic(), current,
"magic from imp.get_magic() for %s "
"should be sysinfo2magic()" % lookup)
def load_module_from_file_object(fp,
filename="<unknown>",
code_objects=None,
fast_load=False,
get_code=True):
"""load a module from a file object without importing it.
See :func:load_module for a list of return values.
"""
if code_objects is None:
code_objects = {}
timestamp = 0
try:
magic = fp.read(4)
magic_int = magic2int(magic)
# For reasons I don't understand, PyPy 3.2 stores a magic
# of '0'... The two values below are for Python 2.x and 3.x respectively
if magic[0:1] in ["0", b"0"]:
magic = int2magic(3180 + 7)
try:
# FIXME: use the internal routine below
float_version = magic_int2float(magic_int)
except KeyError:
if magic_int in (2657, 22138):
raise ImportError(
"This smells like Pyston which is not supported.")
if len(magic) >= 2:
raise ImportError(
"Unknown magic number %s in %s" %
(ord(magic[0:1]) + 256 * ord(magic[1:2]), filename))
else:
raise ImportError("Bad magic number: '%s'" % magic)
if magic_int in (
3010,
3020,
3030,
3040,
3050,
3060,
3061,
3071,
3361,
3091,
3101,
3103,
3141,
3270,
3280,
3290,
3300,
3320,
3330,
3371,
62071,
62071,
62081,
62091,
62092,
62111,
):
raise ImportError("%s is interim Python %s (%d) bytecode which is "
"not supported.\nFinal released versions are "
"supported." %
(filename, versions[magic], magic2int(magic)))
elif magic_int == 62135:
fp.seek(0)
return fix_dropbox_pyc(fp)
elif magic_int == 62215:
raise ImportError(
"%s is a dropbox-hacked Python %s (bytecode %d).\n"
"See https://github.com/kholia/dedrop for how to "
"decrypt." % (filename, versions[magic], magic2int(magic)))
try:
# print version
my_magic_int = PYTHON_MAGIC_INT
magic_int = magic2int(magic)
version = magic_int2float(magic_int)
timestamp = None
source_size = None
sip_hash = None
ts = fp.read(4)
if version >= 3.7:
# PEP 552. https://www.python.org/dev/peps/pep-0552/
pep_bits = ts[-1]
if PYTHON_VERSION <= 2.7:
pep_bits = ord(pep_bits)
if (pep_bits & 1) or magic_int == 3393: # 3393 is 3.7.0beta3
# SipHash
sip_hash = unpack("<Q", fp.read(8))[0]
else:
# Uses older-style timestamp and size
timestamp = unpack("<I", fp.read(4))[0] # pep552_bits
source_size = unpack("<I", fp.read(4))[0] # size mod 2**32
pass
else:
timestamp = unpack("<I", ts)[0]
# Note: a higher magic number doesn't necessarily mean a later
# release. At Python 3.0 the magic number decreased
# significantly. Hence the range below. Also note inclusion of
# the size info, occurred within a Python major/minor
# release. Hence the test on the magic value rather than
# PYTHON_VERSION, although PYTHON_VERSION would probably work.
if ((3200 <= magic_int < 20121) and version >= 1.5
or magic_int in IS_PYPY3):
source_size = unpack("<I", fp.read(4))[0] # size mod 2**32
if get_code:
if my_magic_int == magic_int:
bytecode = fp.read()
co = marshal.loads(bytecode)
elif fast_load:
co = xdis.marsh.load(fp, magicint2version[magic_int])
else:
co = xdis.unmarshal.load_code(fp, magic_int, code_objects)
pass
else:
co = None
except:
kind, msg = sys.exc_info()[0:2]
import traceback
traceback.print_exc()
raise ImportError("Ill-formed bytecode file %s\n%s; %s" %
(filename, kind, msg))
finally:
fp.close()
return (
float_version,
timestamp,
magic_int,
co,
is_pypy(magic_int),
source_size,
sip_hash,
)
def load_module_from_file_object(fp,
filename='<unknown>',
code_objects=None,
fast_load=False,
get_code=True):
"""load a module from a file object without importing it.
See :func:load_module for a list of return values.
"""
if code_objects is None:
code_objects = {}
timestamp = 0
try:
magic = fp.read(4)
magic_int = magics.magic2int(magic)
# For reasons I don't understand, PyPy 3.2 stores a magic
# of '0'... The two values below are for Python 2.x and 3.x respectively
if magic[0:1] in ['0', b'0']:
magic = magics.int2magic(3180 + 7)
try:
# FIXME: use the internal routine below
float_version = float(magics.versions[magic][:3])
# float_version = magics.magic_int2float(magic_int)
except KeyError:
if magic_int in (2657, 22138):
raise ImportError(
"This smells like Pyston which is not supported.")
if len(magic) >= 2:
raise ImportError(
"Unknown magic number %s in %s" %
(ord(magic[0:1]) + 256 * ord(magic[1:2]), filename))
else:
raise ImportError("Bad magic number: '%s'" % magic)
if magic_int in (3010, 3020, 3030, 3040, 3050, 3060, 3061, 3361, 3371):
raise ImportError(
"%s is interim Python %s (%d) bytecode which is "
"not supported.\nFinal released versions are "
"supported." %
(filename, magics.versions[magic], magics.magic2int(magic)))
elif magic_int == 62135:
fp.seek(0)
return fix_dropbox_pyc(fp)
elif magic_int == 62215:
raise ImportError(
"%s is a dropbox-hacked Python %s (bytecode %d).\n"
"See https://github.com/kholia/dedrop for how to "
"decrypt." %
(filename, magics.versions[magic], magics.magic2int(magic)))
try:
# print version
ts = fp.read(4)
my_magic_int = magics.magic2int(imp.get_magic())
magic_int = magics.magic2int(magic)
if magic_int == 3393:
timestamp = 0
_ = unpack("<I", ts)[0] # hash word 1
_ = unpack("<I", fp.read(4))[0] # hash word 2
elif magic_int in (3394, 3401):
timestamp = 0
_ = unpack("<I", fp.read(4))[0] # pep552_bits
else:
timestamp = unpack("<I", ts)[0]
# Note: a higher magic number doesn't necessarily mean a later
# release. At Python 3.0 the magic number decreased
# significantly. Hence the range below. Also note inclusion of
# the size info, occurred within a Python major/minor
# release. Hence the test on the magic value rather than
# PYTHON_VERSION, although PYTHON_VERSION would probably work.
if (((3200 <= magic_int < 20121) and
(magic_int not in (5892, 11913, 39170, 39171)))
or (magic_int in magics.IS_PYPY3)):
source_size = unpack("<I", fp.read(4))[0] # size mod 2**32
else:
source_size = None
if get_code:
if my_magic_int == magic_int:
bytecode = fp.read()
co = marshal.loads(bytecode)
elif fast_load:
co = xdis.marsh.load(fp,
magics.magicint2version[magic_int])
else:
co = xdis.unmarshal.load_code(fp, magic_int, code_objects)
pass
else:
co = None
except:
kind, msg = sys.exc_info()[0:2]
import traceback
traceback.print_exc()
raise ImportError("Ill-formed bytecode file %s\n%s; %s" %
(filename, kind, msg))
finally:
fp.close()
return float_version, timestamp, magic_int, co, is_pypy(
magic_int), source_size
def load_module(filename, code_objects=None, fast_load=False,
get_code=True):
"""load a module without importing it.
load_module(filename: string): version, magic_int, code_object
filename: name of file containing Python byte-code object
(normally a .pyc)
code_object: code_object from this file
version: Python major/minor value e.g. 2.7. or 3.4
magic_int: more specific than version. The actual byte code version of the
code object
Parsing the code object takes a bit of parsing time, but
sometimes all you want is the module info, time string, code size,
python version, etc. For that, set get_code=False.
"""
if code_objects is None:
code_objects = {}
timestamp = 0
fp = open(filename, 'rb')
try:
magic = fp.read(4)
magic_int = magics.magic2int(magic)
# For reasons I don't understand PyPy 3.2 stores a magic
# of '0'... The two values below are for Python 2.x and 3.x respectively
if magic[0:1] in ['0', b'0']:
magic = magics.int2magic(3180+7)
try:
version = float(magics.versions[magic][:3])
except KeyError:
if len(magic) >= 2:
raise ImportError("Unknown magic number %s in %s" %
(ord(magic[0])+256*ord(magic[1]), filename))
else:
raise ImportError("Bad magic number: '%s'" % magic)
if magic_int in (3361,):
raise ImportError("%s is interim Python %s (%d) bytecode which is "
"not supported.\nFinal released versions are "
"supported." % (
filename, magics.versions[magic],
magics.magic2int(magic)))
elif magic_int == 62135:
fp.seek(0)
return fix_dropbox_pyc(fp)
elif magic_int == 62215:
raise ImportError("%s is a dropbox-hacked Python %s (bytecode %d).\n"
"See https://github.com/kholia/dedrop for how to "
"decrypt." % (
filename, version, magics.magic2int(magic)))
try:
# print version
ts = fp.read(4)
timestamp = unpack("I", ts)[0]
my_magic_int = magics.magic2int(imp.get_magic())
magic_int = magics.magic2int(magic)
# Note: a higher magic number doesn't necessarily mean a later
# release. At Python 3.0 the magic number decreased
# significantly. Hence the range below. Also note inclusion of
# the size info, occurred within a Python major/minor
# release. Hence the test on the magic value rather than
# PYTHON_VERSION, although PYTHON_VERSION would probably work.
if 3200 <= magic_int < 20121:
source_size = unpack("I", fp.read(4))[0] # size mod 2**32
else:
source_size = None
if get_code:
if my_magic_int == magic_int:
bytecode = fp.read()
co = marshal.loads(bytecode)
elif fast_load:
co = xdis.marsh.load(fp, magic_int, code_objects)
else:
co = xdis.unmarshal.load_code(fp, magic_int, code_objects)
pass
else:
co = None
except:
import traceback
traceback.print_exc()
raise ImportError("Ill-formed bytecode file %s" % filename)
finally:
fp.close()
return version, timestamp, magic_int, co, is_pypy(magic_int), source_size
def load_module_from_file_object(fp, filename='<unknown>', code_objects=None, fast_load=False,
get_code=True):
"""load a module from a file object without importing it.
See :func:load_module for a list of return values.
"""
if code_objects is None:
code_objects = {}
timestamp = 0
try:
magic = fp.read(4)
magic_int = magics.magic2int(magic)
# For reasons I don't understand, PyPy 3.2 stores a magic
# of '0'... The two values below are for Python 2.x and 3.x respectively
if magic[0:1] in ['0', b'0']:
magic = magics.int2magic(3180+7)
try:
# FIXME: use the internal routine below
float_version = float(magics.versions[magic][:3])
# float_version = magics.magic_int2float(magic_int)
except KeyError:
if magic_int in (2657, 22138):
raise ImportError("This smells like Pyston which is not supported.")
if len(magic) >= 2:
raise ImportError("Unknown magic number %s in %s" %
(ord(magic[0:1])+256*ord(magic[1:2]), filename))
else:
raise ImportError("Bad magic number: '%s'" % magic)
if magic_int in (3010, 3020, 3030, 3040, 3050, 3060, 3061, 3361, 3371):
raise ImportError("%s is interim Python %s (%d) bytecode which is "
"not supported.\nFinal released versions are "
"supported." % (
filename, magics.versions[magic],
magics.magic2int(magic)))
elif magic_int == 62135:
fp.seek(0)
return fix_dropbox_pyc(fp)
elif magic_int == 62215:
raise ImportError("%s is a dropbox-hacked Python %s (bytecode %d).\n"
"See https://github.com/kholia/dedrop for how to "
"decrypt." % (
filename, magics.versions[magic],
magics.magic2int(magic)))
try:
# print version
ts = fp.read(4)
my_magic_int = magics.magic2int(imp.get_magic())
magic_int = magics.magic2int(magic)
if magic_int == 3393:
timestamp = 0
_ = unpack("<I", ts)[0] # hash word 1
_ = unpack("<I", fp.read(4))[0] # hash word 2
elif magic_int in (3394, 3401):
timestamp = 0
_ = unpack("<I", fp.read(4))[0] # pep552_bits
else:
timestamp = unpack("<I", ts)[0]
# Note: a higher magic number doesn't necessarily mean a later
# release. At Python 3.0 the magic number decreased
# significantly. Hence the range below. Also note inclusion of
# the size info, occurred within a Python major/minor
# release. Hence the test on the magic value rather than
# PYTHON_VERSION, although PYTHON_VERSION would probably work.
if 3200 <= magic_int < 20121 and magic_int not in (5892, 11913, 39170, 39171):
source_size = unpack("<I", fp.read(4))[0] # size mod 2**32
else:
source_size = None
if get_code:
if my_magic_int == magic_int:
bytecode = fp.read()
co = marshal.loads(bytecode)
elif fast_load:
co = xdis.marsh.load(fp, magics.magicint2version[magic_int])
else:
co = xdis.unmarshal.load_code(fp, magic_int, code_objects)
pass
else:
co = None
except:
kind, msg = sys.exc_info()[0:2]
import traceback
traceback.print_exc()
raise ImportError("Ill-formed bytecode file %s\n%s; %s"
% (filename, kind, msg))
finally:
fp.close()
return float_version, timestamp, magic_int, co, is_pypy(magic_int), source_size
