def test_source_from_cache(self): # Given the path to a PEP 3147 defined .pyc file, return the path to # its source. This tests the good path. path = os.path.join('foo', 'bar', 'baz', '__pycache__', 'qux.{}.pyc'.format(self.tag)) expect = os.path.join('foo', 'bar', 'baz', 'qux.py') self.assertEqual(imp.source_from_cache(path), expect)
def source_from_cache(path): cachepath = path path = imp.source_from_cache(cachepath) cachepre, cacheext = os.path.splitext(cachepath) if cacheext != '.pyc': pathpre, pathext = os.path.splitext(path) path = pathpre + cacheext[:-1] return path
def make_zip_script(zip_dir, zip_basename, script_name, name_in_zip=None): zip_filename = zip_basename+os.extsep+'zip' zip_name = os.path.join(zip_dir, zip_filename) zip_file = zipfile.ZipFile(zip_name, 'w') if name_in_zip is None: parts = script_name.split(os.sep) if len(parts) >= 2 and parts[-2] == '__pycache__': legacy_pyc = make_legacy_pyc(source_from_cache(script_name)) name_in_zip = os.path.basename(legacy_pyc) script_name = legacy_pyc else: name_in_zip = os.path.basename(script_name) zip_file.write(script_name, name_in_zip) zip_file.close() #if test.support.verbose: # zip_file = zipfile.ZipFile(zip_name, 'r') # print 'Contents of %r:' % zip_name # zip_file.printdir() # zip_file.close() return zip_name, os.path.join(zip_name, name_in_zip)
def make_zip_script(zip_dir, zip_basename, script_name, name_in_zip=None): zip_filename = zip_basename + os.extsep + 'zip' zip_name = os.path.join(zip_dir, zip_filename) zip_file = zipfile.ZipFile(zip_name, 'w') if name_in_zip is None: parts = script_name.split(os.sep) if len(parts) >= 2 and parts[-2] == '__pycache__': legacy_pyc = make_legacy_pyc(source_from_cache(script_name)) name_in_zip = os.path.basename(legacy_pyc) script_name = legacy_pyc else: name_in_zip = os.path.basename(script_name) zip_file.write(script_name, name_in_zip) zip_file.close() #if test.support.verbose: # zip_file = zipfile.ZipFile(zip_name, 'r') # print 'Contents of %r:' % zip_name # zip_file.printdir() # zip_file.close() return zip_name, os.path.join(zip_name, name_in_zip)
def cache_zip_file(self, zip_path): """Read a zip file and cache the modules and packages found inside it. """ zip = zipfile.ZipFile(zip_path) for archiveName in zip.namelist(): baseName, ext = os.path.splitext(archiveName) if ext not in ('.pyc', '.pyo'): continue if '__pycache__' in baseName: if not baseName.endswith(imp.get_tag()): continue baseName = \ os.path.splitext(imp.source_from_cache(archiveName))[0] nameparts = baseName.split("/") if len(nameparts) > 1 and nameparts[-1] == '__init__': # dir/__init__.pyc -> dir is a package self.record_loadable_module(nameparts[:-1], None, zip, True) self.record_loadable_module(nameparts, archiveName, zip, False)
def test_source_from_cache_no_cache_tag(self): # If sys.implementation.cache_tag is None, raise NotImplementedError. path = os.path.join('blah', '__pycache__', 'whatever.pyc') with support.swap_attr(sys.implementation, 'cache_tag', None): with self.assertRaises(NotImplementedError): imp.source_from_cache(path)
def test_source_from_cache(self): # Given the path to a PEP 3147 defined .pyc file, return the path to # its source. This tests the good path. path = os.path.join("foo", "bar", "baz", "__pycache__", "qux.{}.pyc".format(self.tag)) expect = os.path.join("foo", "bar", "baz", "qux.py") self.assertEqual(imp.source_from_cache(path), expect)
def find_module(self, name, path=None): state = self.config._assertstate state.trace("find_module called for: %s" % name) names = name.rsplit(".", 1) lastname = names[-1] pth = None if path is not None: # Starting with Python 3.3, path is a _NamespacePath(), which # causes problems if not converted to list. path = list(path) if len(path) == 1: pth = path[0] if pth is None: try: fd, fn, desc = imp.find_module(lastname, path) except ImportError: return None if fd is not None: fd.close() tp = desc[2] if tp == imp.PY_COMPILED: if hasattr(imp, "source_from_cache"): fn = imp.source_from_cache(fn) else: fn = fn[:-1] elif tp != imp.PY_SOURCE: # Don't know what this is. return None else: fn = os.path.join(pth, name.rpartition(".")[2] + ".py") fn_pypath = py.path.local(fn) if not self._should_rewrite(name, fn_pypath, state): return None # The requested module looks like a test file, so rewrite it. This is # the most magical part of the process: load the source, rewrite the # asserts, and load the rewritten source. We also cache the rewritten # module code in a special pyc. We must be aware of the possibility of # concurrent pytest processes rewriting and loading pycs. To avoid # tricky race conditions, we maintain the following invariant: The # cached pyc is always a complete, valid pyc. Operations on it must be # atomic. POSIX's atomic rename comes in handy. write = not sys.dont_write_bytecode cache_dir = os.path.join(fn_pypath.dirname, "__pycache__") if write: try: os.mkdir(cache_dir) except OSError: e = sys.exc_info()[1].errno if e == errno.EEXIST: # Either the __pycache__ directory already exists (the # common case) or it's blocked by a non-dir node. In the # latter case, we'll ignore it in _write_pyc. pass elif e in [errno.ENOENT, errno.ENOTDIR]: # One of the path components was not a directory, likely # because we're in a zip file. write = False elif e in [errno.EACCES, errno.EROFS, errno.EPERM]: state.trace("read only directory: %r" % fn_pypath.dirname) write = False else: raise cache_name = fn_pypath.basename[:-3] + PYC_TAIL pyc = os.path.join(cache_dir, cache_name) # Notice that even if we're in a read-only directory, I'm going # to check for a cached pyc. This may not be optimal... co = _read_pyc(fn_pypath, pyc, state.trace) if co is None: state.trace("rewriting %r" % (fn,)) source_stat, co = _rewrite_test(self.config, fn_pypath) if co is None: # Probably a SyntaxError in the test. return None if write: _make_rewritten_pyc(state, source_stat, pyc, co) else: state.trace("found cached rewritten pyc for %r" % (fn,)) self.modules[name] = co, pyc return self
def test_source_from_cache(self): # Given the path to a PEP 3147 defined .pyc file, return the path to # its source. This tests the good path. self.assertEqual( imp.source_from_cache('/foo/bar/baz/__pycache__/qux.{}.pyc'.format( self.tag)), '/foo/bar/baz/qux.py')
def _source_from_cache(path): if '__pycache__' in path: return source_from_cache(path) return path
def test_source_from_cache(self): # Given the path to a PEP 3147 defined .pyc file, return the path to # its source. This tests the good path. self.assertEqual(imp.source_from_cache( '/foo/bar/baz/__pycache__/qux.{}.pyc'.format(self.tag)), '/foo/bar/baz/qux.py')
def find_module(self, name, path=None): if self.session is None: return None sess = self.session state = sess.config._assertstate state.trace("find_module called for: %s" % name) names = name.rsplit(".", 1) lastname = names[-1] pth = None if path is not None and len(path) == 1: pth = path[0] if pth is None: try: fd, fn, desc = imp.find_module(lastname, path) except ImportError: return None if fd is not None: fd.close() tp = desc[2] if tp == imp.PY_COMPILED: if hasattr(imp, "source_from_cache"): fn = imp.source_from_cache(fn) else: fn = fn[:-1] elif tp != imp.PY_SOURCE: # Don't know what this is. return None else: fn = os.path.join(pth, name.rpartition(".")[2] + ".py") fn_pypath = py.path.local(fn) # Is this a test file? if not sess.isinitpath(fn): # We have to be very careful here because imports in this code can # trigger a cycle. self.session = None try: for pat in self.fnpats: if fn_pypath.fnmatch(pat): state.trace("matched test file %r" % (fn,)) break else: return None finally: self.session = sess else: state.trace("matched test file (was specified on cmdline): %r" % (fn,)) # The requested module looks like a test file, so rewrite it. This is # the most magical part of the process: load the source, rewrite the # asserts, and load the rewritten source. We also cache the rewritten # module code in a special pyc. We must be aware of the possibility of # concurrent py.test processes rewriting and loading pycs. To avoid # tricky race conditions, we maintain the following invariant: The # cached pyc is always a complete, valid pyc. Operations on it must be # atomic. POSIX's atomic rename comes in handy. write = not sys.dont_write_bytecode cache_dir = os.path.join(fn_pypath.dirname, "__pycache__") if write: try: os.mkdir(cache_dir) except OSError: e = sys.exc_info()[1].errno if e == errno.EEXIST: # Either the __pycache__ directory already exists (the # common case) or it's blocked by a non-dir node. In the # latter case, we'll ignore it in _write_pyc. pass elif e in [errno.ENOENT, errno.ENOTDIR]: # One of the path components was not a directory, likely # because we're in a zip file. write = False elif e == errno.EACCES: state.trace("read only directory: %r" % fn_pypath.dirname) write = False else: raise cache_name = fn_pypath.basename[:-3] + PYC_TAIL pyc = os.path.join(cache_dir, cache_name) # Notice that even if we're in a read-only directory, I'm going # to check for a cached pyc. This may not be optimal... co = _read_pyc(fn_pypath, pyc) if co is None: state.trace("rewriting %r" % (fn,)) co = _rewrite_test(state, fn_pypath) if co is None: # Probably a SyntaxError in the test. return None if write: _make_rewritten_pyc(state, fn_pypath, pyc, co) else: state.trace("found cached rewritten pyc for %r" % (fn,)) self.modules[name] = co, pyc return self
def test_source_from_cache(self): path = os.path.join('foo', 'bar', 'baz', '__pycache__', 'qux.{}.pyc'.format(self.tag)) expect = os.path.join('foo', 'bar', 'baz', 'qux.py') self.assertEqual(imp.source_from_cache(path), expect)
def update_event(self, inp=-1): self.set_output_val(0, imp.source_from_cache(self.input(0)))
def find_module(self, name, path=None): if self.session is None: return None sess = self.session state = sess.config._assertstate state.trace("find_module called for: %s" % name) names = name.rsplit(".", 1) lastname = names[-1] pth = None if path is not None and len(path) == 1: pth = path[0] if pth is None: try: fd, fn, desc = imp.find_module(lastname, path) except ImportError: return None if fd is not None: fd.close() tp = desc[2] if tp == imp.PY_COMPILED: if hasattr(imp, "source_from_cache"): fn = imp.source_from_cache(fn) else: fn = fn[:-1] elif tp != imp.PY_SOURCE: # Don't know what this is. return None else: fn = os.path.join(pth, name.rpartition(".")[2] + ".py") fn_pypath = py.path.local(fn) # Is this a test file? if not sess.isinitpath(fn): # We have to be very careful here because imports in this code can # trigger a cycle. self.session = None try: for pat in self.fnpats: if fn_pypath.fnmatch(pat): state.trace("matched test file %r" % (fn,)) break else: return None finally: self.session = sess else: state.trace("matched test file (was specified on cmdline): %r" % (fn,)) # The requested module looks like a test file, so rewrite it. This is # the most magical part of the process: load the source, rewrite the # asserts, and load the rewritten source. We also cache the rewritten # module code in a special pyc. We must be aware of the possibility of # concurrent py.test processes rewriting and loading pycs. To avoid # tricky race conditions, we maintain the following invariant: The # cached pyc is always a complete, valid pyc. Operations on it must be # atomic. POSIX's atomic rename comes in handy. write = not sys.dont_write_bytecode cache_dir = os.path.join(fn_pypath.dirname, "__pycache__") if write: try: os.mkdir(cache_dir) except OSError: e = sys.exc_info()[1].errno if e == errno.EEXIST: # Either the __pycache__ directory already exists (the # common case) or it's blocked by a non-dir node. In the # latter case, we'll ignore it in _write_pyc. pass elif e == PATH_COMPONENT_NOT_DIR: # One of the path components was not a directory, likely # because we're in a zip file. write = False elif e == errno.EACCES: state.trace("read only directory: %r" % (fn_pypath.dirname,)) write = False else: raise cache_name = fn_pypath.basename[:-3] + PYC_TAIL pyc = os.path.join(cache_dir, cache_name) # Notice that even if we're in a read-only directory, I'm going to check # for a cached pyc. This may not be optimal... co = _read_pyc(fn_pypath, pyc) if co is None: state.trace("rewriting %r" % (fn,)) co = _rewrite_test(state, fn_pypath) if co is None: # Probably a SyntaxError in the test. return None if write: _make_rewritten_pyc(state, fn_pypath, pyc, co) else: state.trace("found cached rewritten pyc for %r" % (fn,)) self.modules[name] = co, pyc return self
def find_module(self, name, path=None): if self._writing_pyc: return None state = self.config._assertstate if self._early_rewrite_bailout(name, state): return None state.trace("find_module called for: %s" % name) names = name.rsplit(".", 1) lastname = names[-1] pth = None if path is not None: # Starting with Python 3.3, path is a _NamespacePath(), which # causes problems if not converted to list. path = list(path) if len(path) == 1: pth = path[0] if pth is None: try: fd, fn, desc = self._imp_find_module(lastname, path) except ImportError: return None if fd is not None: fd.close() tp = desc[2] if tp == imp.PY_COMPILED: if hasattr(imp, "source_from_cache"): try: fn = imp.source_from_cache(fn) except ValueError: # Python 3 doesn't like orphaned but still-importable # .pyc files. fn = fn[:-1] else: fn = fn[:-1] elif tp != imp.PY_SOURCE: # Don't know what this is. return None else: fn = os.path.join(pth, name.rpartition(".")[2] + ".py") fn_pypath = py.path.local(fn) if not self._should_rewrite(name, fn_pypath, state): return None self._rewritten_names.add(name) # The requested module looks like a test file, so rewrite it. This is # the most magical part of the process: load the source, rewrite the # asserts, and load the rewritten source. We also cache the rewritten # module code in a special pyc. We must be aware of the possibility of # concurrent pytest processes rewriting and loading pycs. To avoid # tricky race conditions, we maintain the following invariant: The # cached pyc is always a complete, valid pyc. Operations on it must be # atomic. POSIX's atomic rename comes in handy. write = not sys.dont_write_bytecode cache_dir = os.path.join(fn_pypath.dirname, "__pycache__") if write: try: os.mkdir(cache_dir) except OSError: e = sys.exc_info()[1].errno if e == errno.EEXIST: # Either the __pycache__ directory already exists (the # common case) or it's blocked by a non-dir node. In the # latter case, we'll ignore it in _write_pyc. pass elif e in [errno.ENOENT, errno.ENOTDIR]: # One of the path components was not a directory, likely # because we're in a zip file. write = False elif e in [errno.EACCES, errno.EROFS, errno.EPERM]: state.trace("read only directory: %r" % fn_pypath.dirname) write = False else: raise cache_name = fn_pypath.basename[:-3] + PYC_TAIL pyc = os.path.join(cache_dir, cache_name) # Notice that even if we're in a read-only directory, I'm going # to check for a cached pyc. This may not be optimal... co = _read_pyc(fn_pypath, pyc, state.trace) if co is None: state.trace("rewriting %r" % (fn, )) source_stat, co = _rewrite_test(self.config, fn_pypath) if co is None: # Probably a SyntaxError in the test. return None if write: self._writing_pyc = True try: _write_pyc(state, co, source_stat, pyc) finally: self._writing_pyc = False else: state.trace("found cached rewritten pyc for %r" % (fn, )) self.modules[name] = co, pyc return self