def verify_file(real_source_filename, real_bytecode_filename): """Compile *real_source_filename* using the running Python interpreter. Then write bytecode out to a new place again using Python's routines. Next load it in using two of our routines. Compare that the code objects there are equal. Next write out the bytecode (using the same Python bytecode writin routine as in step 1. Finally compare the bytecode files. """ tempdir = tempfile.gettempdir() source_filename = os.path.join(tempdir, "testing.py") if not os.path.exists(real_source_filename): return if PYTHON_VERSION < 3.0: f = open(real_source_filename, 'U') elif PYTHON_VERSION == 3.0: # Too hard to get working on 3.0 return elif 3.1 <= PYTHON_VERSION <= 3.4: f = open(real_source_filename, "rb") else: f = open(real_source_filename, newline=None, errors='backslashreplace') codestring = f.read() f.close() codeobject1 = compile(codestring, source_filename, 'exec') (version, timestamp, magic_int, codeobject2, is_pypy, source_size) = load_module(real_bytecode_filename) # A hack for PyPy 3.2 if magic_int == 3180 + 7: magic_int = 48 assert magics.MAGIC == magics.int2magic(magic_int), \ ("magic_int %d vs %d in %s/%s" % (magic_int, magics.PYTHON_MAGIC_INT, os.getcwd(), real_bytecode_filename)) bytecode_filename1 = os.path.join(tempdir, "testing1.pyc") dump_compile(codeobject1, bytecode_filename1, timestamp, magics.MAGIC) (version, timestamp, magic_int, codeobject3, is_pypy, source_size) = load_module(real_bytecode_filename, fast_load=not is_pypy) # compare_code(codeobject1, codeobject2) # compare_code(codeobject2, codeobject3) bytecode_filename2 = os.path.join(tempdir, "testing2.pyc") dump_compile(codeobject1, bytecode_filename2, timestamp, magics.int2magic(magic_int)) compare_bytecode_files(bytecode_filename1, bytecode_filename2) return
def verify_file(real_source_filename, real_bytecode_filename): """Compile *real_source_filename* using the running Python interpreter. Then write bytecode out to a new place again using Python's routines. Next load it in using two of our routines. Compare that the code objects there are equal. Next write out the bytecode (using the same Python bytecode writin routine as in step 1. Finally compare the bytecode files. """ tempdir = tempfile.gettempdir() source_filename = os.path.join(tempdir, "testing.py") if not os.path.exists(real_source_filename): return try: f = open(real_source_filename, 'U') except: return codestring = f.read() f.close() codeobject1 = compile(codestring, source_filename, 'exec') (version, timestamp, magic_int, codeobject2, is_pypy, source_size) = load_module(real_bytecode_filename) # A hack for PyPy 3.2 if magic_int == 3180 + 7: magic_int = 48 assert MAGIC == magics.int2magic(magic_int), \ ("magic_int %d vs %d in %s/%s" % (magic_int, magics.magic2int(MAGIC), os.getcwd(), real_bytecode_filename)) bytecode_filename1 = os.path.join(tempdir, "testing1.pyc") dump_compile(codeobject1, bytecode_filename1, timestamp, MAGIC) (version, timestamp, magic_int, codeobject3, is_pypy, source_size) = load_module(real_bytecode_filename, fast_load=not is_pypy) # compare_code(codeobject1, codeobject2) # compare_code(codeobject2, codeobject3) bytecode_filename2 = os.path.join(tempdir, "testing2.pyc") dump_compile(codeobject1, bytecode_filename2, timestamp, magics.int2magic(magic_int)) compare_bytecode_files(bytecode_filename1, bytecode_filename2) return
def verify_file(real_source_filename, real_bytecode_filename): """Compile *real_source_filename* using the running Python interpreter. Then write bytecode out to a new place again using Python's routines. Next load it in using two of our routines. Compare that the code objects there are equal. Next write out the bytecode (using the same Python bytecode writin routine as in step 1. Finally compare the bytecode files. """ tempdir = tempfile.gettempdir() source_filename = os.path.join(tempdir, "testing.py") if not os.path.exists(real_source_filename): return try: f = open(real_source_filename, 'U') except: return codestring = f.read() f.close() codeobject1 = compile(codestring, source_filename,'exec') (version, timestamp, magic_int, codeobject2, is_pypy, source_size) = load_module(real_bytecode_filename) # A hack for PyPy 3.2 if magic_int == 3180+7: magic_int = 48 assert MAGIC == magics.int2magic(magic_int), \ ("magic_int %d vs %d in %s/%s" % (magic_int, magics.magic2int(MAGIC), os.getcwd(), real_bytecode_filename)) bytecode_filename1 = os.path.join(tempdir, "testing1.pyc") dump_compile(codeobject1, bytecode_filename1, timestamp, MAGIC) (version, timestamp, magic_int, codeobject3, is_pypy, source_size) = load_module(real_bytecode_filename, fast_load=not is_pypy) # compare_code(codeobject1, codeobject2) # compare_code(codeobject2, codeobject3) bytecode_filename2 = os.path.join(tempdir, "testing2.pyc") dump_compile(codeobject1, bytecode_filename2, timestamp, magics.int2magic(magic_int)) compare_bytecode_files(bytecode_filename1, bytecode_filename2) return
def test_basic(self): """Basic test of magic numbers""" if hasattr(sys, 'version_info'): version = version_tuple_to_str() if IS_PYPY: version += 'pypy' self.assertTrue( version in magics.magics.keys(), "version %s is not in magic.magics.keys: %s" % (version, magics.magics.keys())) self.assertEqual(magics.MAGIC, magics.int2magic(magics.magic2int(magics.MAGIC))) lookup = str(PYTHON_VERSION) if IS_PYPY: lookup += 'pypy' self.assertTrue( lookup in magics.magics.keys(), "PYTHON VERSION %s is not in magic.magics.keys: %s" % (lookup, magics.magics.keys())) if not (3, 5, 2) <= sys.version_info < (3, 6, 0): self.assertEqual( magics.sysinfo2magic(), magics.MAGIC, "magic from imp.get_magic() for %s " "should be sysinfo2magic()" % lookup)
def test_basic(self): """Basic test of magic numbers""" current = imp.get_magic() if hasattr(sys, 'version_info'): version = '.'.join([str(v) for v in sys.version_info[0:3]]) if IS_PYPY: version += 'pypy' self.assertTrue( version in magics.magics.keys(), "version %s is not in magic.magics.keys: %s" % (version, magics.magics.keys())) self.assertEqual(current, magics.int2magic(magics.magic2int(current))) lookup = str(PYTHON_VERSION) if IS_PYPY: lookup += 'pypy' self.assertTrue( lookup in magics.magics.keys(), "PYTHON VERSION %s is not in magic.magics.keys: %s" % (lookup, magics.magics.keys())) if not (3, 5, 2) <= sys.version_info < (3, 6, 0): self.assertEqual( magics.sysinfo2magic(), current, "magic from imp.get_magic() for %s " "should be sysinfo2magic()" % lookup)
def test_basic(self): """Basic test of magic numbers""" current = imp.get_magic() if hasattr(sys, 'version_info'): version = '.'.join([str(v) for v in sys.version_info[0:3]]) self.assertTrue(version in magics.magics.keys(), "version %s is not in magic.magics.keys: %s" % (version, magics.magics.keys())) self.assertEqual(current, magics.int2magic(magics.magic2int(current))) self.assertTrue(str(PYTHON_VERSION) in magics.magics.keys(), "PYTHON VERSION %s is not in magic.magics.keys: %s" % (PYTHON_VERSION, magics.magics.keys()))
def test_basic(self): """Basic test of magic numbers""" current = imp.get_magic() if hasattr(sys, 'version_info'): version = '.'.join([str(v) for v in sys.version_info[0:3]]) if IS_PYPY: version += 'pypy' self.assertTrue(version in magics.magics.keys(), "version %s is not in magic.magics.keys: %s" % (version, magics.magics.keys())) self.assertEqual(current, magics.int2magic(magics.magic2int(current))) lookup = str(PYTHON_VERSION) if IS_PYPY: lookup += 'pypy' self.assertTrue(lookup in magics.magics.keys(), "PYTHON VERSION %s is not in magic.magics.keys: %s" % (lookup, magics.magics.keys())) if not (3, 5, 2) <= sys.version_info < (3, 6, 0): self.assertEqual(magics.sysinfo2magic(), current, "magic from imp.get_magic() for %s " "should be sysinfo2magic()" % lookup)
def load_module_from_file_object(fp, filename="<unknown>", code_objects=None, fast_load=False, get_code=True): """load a module from a file object without importing it. See :func:load_module for a list of return values. """ if code_objects is None: code_objects = {} timestamp = 0 try: magic = fp.read(4) magic_int = magic2int(magic) # For reasons I don't understand, PyPy 3.2 stores a magic # of '0'... The two values below are for Python 2.x and 3.x respectively if magic[0:1] in ["0", b"0"]: magic = int2magic(3180 + 7) try: # FIXME: use the internal routine below float_version = magic_int2float(magic_int) except KeyError: if magic_int in (2657, 22138): raise ImportError( "This smells like Pyston which is not supported.") if len(magic) >= 2: raise ImportError( "Unknown magic number %s in %s" % (ord(magic[0:1]) + 256 * ord(magic[1:2]), filename)) else: raise ImportError("Bad magic number: '%s'" % magic) if magic_int in ( 3010, 3020, 3030, 3040, 3050, 3060, 3061, 3071, 3361, 3091, 3101, 3103, 3141, 3270, 3280, 3290, 3300, 3320, 3330, 3371, 62071, 62071, 62081, 62091, 62092, 62111, ): raise ImportError("%s is interim Python %s (%d) bytecode which is " "not supported.\nFinal released versions are " "supported." % (filename, versions[magic], magic2int(magic))) elif magic_int == 62135: fp.seek(0) return fix_dropbox_pyc(fp) elif magic_int == 62215: raise ImportError( "%s is a dropbox-hacked Python %s (bytecode %d).\n" "See https://github.com/kholia/dedrop for how to " "decrypt." % (filename, versions[magic], magic2int(magic))) try: # print version my_magic_int = PYTHON_MAGIC_INT magic_int = magic2int(magic) version = magic_int2float(magic_int) timestamp = None source_size = None sip_hash = None ts = fp.read(4) if version >= 3.7: # PEP 552. https://www.python.org/dev/peps/pep-0552/ pep_bits = ts[-1] if PYTHON_VERSION <= 2.7: pep_bits = ord(pep_bits) if (pep_bits & 1) or magic_int == 3393: # 3393 is 3.7.0beta3 # SipHash sip_hash = unpack("<Q", fp.read(8))[0] else: # Uses older-style timestamp and size timestamp = unpack("<I", fp.read(4))[0] # pep552_bits source_size = unpack("<I", fp.read(4))[0] # size mod 2**32 pass else: timestamp = unpack("<I", ts)[0] # Note: a higher magic number doesn't necessarily mean a later # release. At Python 3.0 the magic number decreased # significantly. Hence the range below. Also note inclusion of # the size info, occurred within a Python major/minor # release. Hence the test on the magic value rather than # PYTHON_VERSION, although PYTHON_VERSION would probably work. if ((3200 <= magic_int < 20121) and version >= 1.5 or magic_int in IS_PYPY3): source_size = unpack("<I", fp.read(4))[0] # size mod 2**32 if get_code: if my_magic_int == magic_int: bytecode = fp.read() co = marshal.loads(bytecode) elif fast_load: co = xdis.marsh.load(fp, magicint2version[magic_int]) else: co = xdis.unmarshal.load_code(fp, magic_int, code_objects) pass else: co = None except: kind, msg = sys.exc_info()[0:2] import traceback traceback.print_exc() raise ImportError("Ill-formed bytecode file %s\n%s; %s" % (filename, kind, msg)) finally: fp.close() return ( float_version, timestamp, magic_int, co, is_pypy(magic_int), source_size, sip_hash, )
def load_module_from_file_object(fp, filename='<unknown>', code_objects=None, fast_load=False, get_code=True): """load a module from a file object without importing it. See :func:load_module for a list of return values. """ if code_objects is None: code_objects = {} timestamp = 0 try: magic = fp.read(4) magic_int = magics.magic2int(magic) # For reasons I don't understand, PyPy 3.2 stores a magic # of '0'... The two values below are for Python 2.x and 3.x respectively if magic[0:1] in ['0', b'0']: magic = magics.int2magic(3180 + 7) try: # FIXME: use the internal routine below float_version = float(magics.versions[magic][:3]) # float_version = magics.magic_int2float(magic_int) except KeyError: if magic_int in (2657, 22138): raise ImportError( "This smells like Pyston which is not supported.") if len(magic) >= 2: raise ImportError( "Unknown magic number %s in %s" % (ord(magic[0:1]) + 256 * ord(magic[1:2]), filename)) else: raise ImportError("Bad magic number: '%s'" % magic) if magic_int in (3010, 3020, 3030, 3040, 3050, 3060, 3061, 3361, 3371): raise ImportError( "%s is interim Python %s (%d) bytecode which is " "not supported.\nFinal released versions are " "supported." % (filename, magics.versions[magic], magics.magic2int(magic))) elif magic_int == 62135: fp.seek(0) return fix_dropbox_pyc(fp) elif magic_int == 62215: raise ImportError( "%s is a dropbox-hacked Python %s (bytecode %d).\n" "See https://github.com/kholia/dedrop for how to " "decrypt." % (filename, magics.versions[magic], magics.magic2int(magic))) try: # print version ts = fp.read(4) my_magic_int = magics.magic2int(imp.get_magic()) magic_int = magics.magic2int(magic) if magic_int == 3393: timestamp = 0 _ = unpack("<I", ts)[0] # hash word 1 _ = unpack("<I", fp.read(4))[0] # hash word 2 elif magic_int in (3394, 3401): timestamp = 0 _ = unpack("<I", fp.read(4))[0] # pep552_bits else: timestamp = unpack("<I", ts)[0] # Note: a higher magic number doesn't necessarily mean a later # release. At Python 3.0 the magic number decreased # significantly. Hence the range below. Also note inclusion of # the size info, occurred within a Python major/minor # release. Hence the test on the magic value rather than # PYTHON_VERSION, although PYTHON_VERSION would probably work. if (((3200 <= magic_int < 20121) and (magic_int not in (5892, 11913, 39170, 39171))) or (magic_int in magics.IS_PYPY3)): source_size = unpack("<I", fp.read(4))[0] # size mod 2**32 else: source_size = None if get_code: if my_magic_int == magic_int: bytecode = fp.read() co = marshal.loads(bytecode) elif fast_load: co = xdis.marsh.load(fp, magics.magicint2version[magic_int]) else: co = xdis.unmarshal.load_code(fp, magic_int, code_objects) pass else: co = None except: kind, msg = sys.exc_info()[0:2] import traceback traceback.print_exc() raise ImportError("Ill-formed bytecode file %s\n%s; %s" % (filename, kind, msg)) finally: fp.close() return float_version, timestamp, magic_int, co, is_pypy( magic_int), source_size
def load_module(filename, code_objects=None, fast_load=False, get_code=True): """load a module without importing it. load_module(filename: string): version, magic_int, code_object filename: name of file containing Python byte-code object (normally a .pyc) code_object: code_object from this file version: Python major/minor value e.g. 2.7. or 3.4 magic_int: more specific than version. The actual byte code version of the code object Parsing the code object takes a bit of parsing time, but sometimes all you want is the module info, time string, code size, python version, etc. For that, set get_code=False. """ if code_objects is None: code_objects = {} timestamp = 0 fp = open(filename, 'rb') try: magic = fp.read(4) magic_int = magics.magic2int(magic) # For reasons I don't understand PyPy 3.2 stores a magic # of '0'... The two values below are for Python 2.x and 3.x respectively if magic[0:1] in ['0', b'0']: magic = magics.int2magic(3180+7) try: version = float(magics.versions[magic][:3]) except KeyError: if len(magic) >= 2: raise ImportError("Unknown magic number %s in %s" % (ord(magic[0])+256*ord(magic[1]), filename)) else: raise ImportError("Bad magic number: '%s'" % magic) if magic_int in (3361,): raise ImportError("%s is interim Python %s (%d) bytecode which is " "not supported.\nFinal released versions are " "supported." % ( filename, magics.versions[magic], magics.magic2int(magic))) elif magic_int == 62135: fp.seek(0) return fix_dropbox_pyc(fp) elif magic_int == 62215: raise ImportError("%s is a dropbox-hacked Python %s (bytecode %d).\n" "See https://github.com/kholia/dedrop for how to " "decrypt." % ( filename, version, magics.magic2int(magic))) try: # print version ts = fp.read(4) timestamp = unpack("I", ts)[0] my_magic_int = magics.magic2int(imp.get_magic()) magic_int = magics.magic2int(magic) # Note: a higher magic number doesn't necessarily mean a later # release. At Python 3.0 the magic number decreased # significantly. Hence the range below. Also note inclusion of # the size info, occurred within a Python major/minor # release. Hence the test on the magic value rather than # PYTHON_VERSION, although PYTHON_VERSION would probably work. if 3200 <= magic_int < 20121: source_size = unpack("I", fp.read(4))[0] # size mod 2**32 else: source_size = None if get_code: if my_magic_int == magic_int: bytecode = fp.read() co = marshal.loads(bytecode) elif fast_load: co = xdis.marsh.load(fp, magic_int, code_objects) else: co = xdis.unmarshal.load_code(fp, magic_int, code_objects) pass else: co = None except: import traceback traceback.print_exc() raise ImportError("Ill-formed bytecode file %s" % filename) finally: fp.close() return version, timestamp, magic_int, co, is_pypy(magic_int), source_size
def load_module_from_file_object(fp, filename='<unknown>', code_objects=None, fast_load=False, get_code=True): """load a module from a file object without importing it. See :func:load_module for a list of return values. """ if code_objects is None: code_objects = {} timestamp = 0 try: magic = fp.read(4) magic_int = magics.magic2int(magic) # For reasons I don't understand, PyPy 3.2 stores a magic # of '0'... The two values below are for Python 2.x and 3.x respectively if magic[0:1] in ['0', b'0']: magic = magics.int2magic(3180+7) try: # FIXME: use the internal routine below float_version = float(magics.versions[magic][:3]) # float_version = magics.magic_int2float(magic_int) except KeyError: if magic_int in (2657, 22138): raise ImportError("This smells like Pyston which is not supported.") if len(magic) >= 2: raise ImportError("Unknown magic number %s in %s" % (ord(magic[0:1])+256*ord(magic[1:2]), filename)) else: raise ImportError("Bad magic number: '%s'" % magic) if magic_int in (3010, 3020, 3030, 3040, 3050, 3060, 3061, 3361, 3371): raise ImportError("%s is interim Python %s (%d) bytecode which is " "not supported.\nFinal released versions are " "supported." % ( filename, magics.versions[magic], magics.magic2int(magic))) elif magic_int == 62135: fp.seek(0) return fix_dropbox_pyc(fp) elif magic_int == 62215: raise ImportError("%s is a dropbox-hacked Python %s (bytecode %d).\n" "See https://github.com/kholia/dedrop for how to " "decrypt." % ( filename, magics.versions[magic], magics.magic2int(magic))) try: # print version ts = fp.read(4) my_magic_int = magics.magic2int(imp.get_magic()) magic_int = magics.magic2int(magic) if magic_int == 3393: timestamp = 0 _ = unpack("<I", ts)[0] # hash word 1 _ = unpack("<I", fp.read(4))[0] # hash word 2 elif magic_int in (3394, 3401): timestamp = 0 _ = unpack("<I", fp.read(4))[0] # pep552_bits else: timestamp = unpack("<I", ts)[0] # Note: a higher magic number doesn't necessarily mean a later # release. At Python 3.0 the magic number decreased # significantly. Hence the range below. Also note inclusion of # the size info, occurred within a Python major/minor # release. Hence the test on the magic value rather than # PYTHON_VERSION, although PYTHON_VERSION would probably work. if 3200 <= magic_int < 20121 and magic_int not in (5892, 11913, 39170, 39171): source_size = unpack("<I", fp.read(4))[0] # size mod 2**32 else: source_size = None if get_code: if my_magic_int == magic_int: bytecode = fp.read() co = marshal.loads(bytecode) elif fast_load: co = xdis.marsh.load(fp, magics.magicint2version[magic_int]) else: co = xdis.unmarshal.load_code(fp, magic_int, code_objects) pass else: co = None except: kind, msg = sys.exc_info()[0:2] import traceback traceback.print_exc() raise ImportError("Ill-formed bytecode file %s\n%s; %s" % (filename, kind, msg)) finally: fp.close() return float_version, timestamp, magic_int, co, is_pypy(magic_int), source_size