def _expand_tripledots(pathnames, target_dirname): """ Expand pathnames of the form C{".../foo/bar"} as "../../foo/bar", "../foo/bar", "./foo/bar" etc., up to the oldest ancestor with the same st_dev. For example, suppose a partition is mounted on /u/homer; /u is a different partition. Then:: >> _expand_tripledots(["/foo", ".../tt"], "/u/homer/aa") [Filename("/foo"), Filename("/u/homer/tt"), Filename("/u/homer/aa/tt")] @type pathnames: sequence of C{str} (not C{Filename}) @type target_dirname: L{Filename} @rtype: C{list} of L{Filename} """ target_dirname = Filename(target_dirname) if not isinstance(pathnames, (tuple, list)): pathnames = [pathnames] result = [] for pathname in pathnames: if not pathname.startswith(".../"): result.append(Filename(pathname)) continue suffix = pathname[4:] expanded = [ p / suffix for p in _ancestors_on_same_partition(target_dirname) ] result.extend(expanded[::-1]) return result
def _expand_tripledots(pathnames, target_dirname): """ Expand pathnames of the form ``".../foo/bar"`` as "../../foo/bar", "../foo/bar", "./foo/bar" etc., up to the oldest ancestor with the same st_dev. For example, suppose a partition is mounted on /u/homer; /u is a different partition. Then:: >>> _expand_tripledots(["/foo", ".../tt"], "/u/homer/aa") # doctest: +SKIP [Filename("/foo"), Filename("/u/homer/tt"), Filename("/u/homer/aa/tt")] :type pathnames: sequence of ``str`` (not ``Filename``) :type target_dirname: `Filename` :rtype: ``list`` of `Filename` """ target_dirname = Filename(target_dirname) if not isinstance(pathnames, (tuple, list)): pathnames = [pathnames] result = [] for pathname in pathnames: if not pathname.startswith(".../"): result.append(Filename(pathname)) continue suffix = pathname[4:] expanded = [ p / suffix for p in _ancestors_on_same_partition(target_dirname) ] result.extend(expanded[::-1]) return result
def test_Filename_ancestors_1(): fn = Filename("/a.aa/b.bb/c.cc") result = fn.ancestors expected = (Filename("/a.aa/b.bb/c.cc"), Filename("/a.aa/b.bb"), Filename("/a.aa"), Filename("/")) assert result == expected
def _find_etc_dirs(): result = [] dirs = Filename(__file__).real.dir.ancestors[:-1] for dir in dirs: candidate = dir / "etc/pyflyby" if candidate.isdir: result.append(candidate) break global_dir = Filename("/etc/pyflyby") if global_dir.exists: result.append(global_dir) return result
def _from_filenames(cls, filenames, _mandatory_filenames_deprecated=[]): """ Load an import database from filenames. This function exists to support deprecated behavior. When we stop supporting the old behavior, we will delete this function. @type filenames: Sequence of L{Filename}s @param filenames: Filenames of files to read. @rtype: L{ImportDB} """ if not isinstance(filenames, (tuple, list)): filenames = [filenames] filenames = [Filename(f) for f in filenames] logger.debug("ImportDB: loading [%s], mandatory=[%s]", ', '.join(map(str, filenames)), ', '.join(map(str, _mandatory_filenames_deprecated))) if "SUPPORT DEPRECATED BEHAVIOR": # Before 2014-10, pyflyby read the following: # * known_imports from $PYFLYBY_PATH/known_imports/**/*.py or # $PYFLYBY_KNOWN_IMPORTS_PATH/**/*.py, # * mandatory_imports from $PYFLYBY_PATH/mandatory_imports/**/*.py or # $PYFLYBY_MANDATORY_IMPORTS_PATH/**/*.py, and # * forget_imports from $PYFLYBY_PATH/known_imports/**/__remove__.py # After 2014-10, pyflyby reads the following: # * $PYFLYBY_PATH/**/*.py # (with directives inside the file) # For backwards compatibility, for now we continue supporting the # old, deprecated behavior. blocks = [] mandatory_imports_blocks = [ Filename(f) for f in _mandatory_filenames_deprecated ] forget_imports_blocks = [] for filename in filenames: if filename.base == "__remove__.py": forget_imports_blocks.append(filename) elif "mandatory_imports" in str(filename).split("/"): mandatory_imports_blocks.append(filename) else: blocks.append(filename) return cls._from_code(blocks, mandatory_imports_blocks, forget_imports_blocks) else: return cls._from_code(filenames)
def get_executable(pid): """ Get the full path for the target process. @type pid: C{int} @rtype: L{Filename} """ uname = os.uname()[0] if uname == 'Linux': result = os.readlink('/proc/%d/exe' % (pid, )) elif uname == 'SunOS': result = os.readlink('/proc/%d/path/a.out' % (pid, )) else: # Use psutil to try to answer this. This should also work for the # above cases too, but it's simple enough to implement it directly and # avoid this dependency on those platforms. import psutil result = psutil.Process(pid).exe() result = Filename(result).real if not result.isfile: raise ValueError("Couldn't get executable for pid %s" % (pid, )) if not result.isreadable: raise ValueError("Executable %s for pid %s is not readable" % (result, pid)) return result
def filename(self): """ Return the filename, if appropriate. The module itself will not be imported, but if the module is not a top-level module/package, accessing this attribute may cause the parent package to be imported. @rtype: L{Filename} """ # Use the loader mechanism to find the filename. We do so instead of # using self.module.__file__, because the latter forces importing a # module, which may be undesirable. import pkgutil try: loader = pkgutil.get_loader(str(self.name)) except ImportError: return None if not loader: return None # Get the filename using loader.get_filename(). Note that this does # more than just loader.filename: for example, it adds /__init__.py # for packages. filename = loader.get_filename() if not filename: return None return Filename(pyc_to_py(filename))
def _find_py_commandline(): global _cached_py_commandline if _cached_py_commandline is not None: return _cached_py_commandline import pyflyby pkg_path = Filename(pyflyby.__path__[0]).real assert pkg_path.base == "pyflyby" d = pkg_path.dir if d.base == "bin": # Running from source tree bindir = d else: # Installed by setup.py while d.dir != d: d = d.dir bindir = d / "bin" if bindir.exists: break else: raise ValueError("Couldn't find 'py' script: " "couldn't find 'bin' dir from package path %s" % (pkg_path, )) candidate = bindir / "py" if not candidate.exists: raise ValueError("Couldn't find 'py' script: expected it at %s" % (candidate, )) if not candidate.isexecutable: raise ValueError("Found 'py' script at %s but it's not executable" % (candidate, )) _cached_py_commandline = candidate return candidate
def describe_xref(identifier, container): module = ModuleHandle(str(container.defining_module.canonical_name)) assert module.filename == Filename(container.defining_module.filename) linenos = get_string_linenos( module, "(L{|<)%s" % (identifier,), container.docstring) return (module, linenos, str(container.canonical_name), identifier)
def test_PythonBlock_attrs_1(): block = PythonBlock(dedent(''' foo() bar() ''').lstrip(), filename="/foo/test_PythonBlock_1.py", startpos=(101,99)) assert block.text.lines == ("foo()", "bar()", "") assert block.filename == Filename("/foo/test_PythonBlock_1.py") assert block.startpos == FilePos(101, 99)
def test_PythonBlock_FileText_1(): text = FileText(dedent(''' foo() bar() ''').lstrip(), filename="/foo/test_PythonBlock_1.py", startpos=(101,55)) block = PythonBlock(text) assert text is block.text assert text is FileText(block) assert block.filename == Filename("/foo/test_PythonBlock_1.py") assert block.startpos == FilePos(101, 55)
def _find_etc_dir(): dir = Filename(__file__).real.dir while True: candidate = dir / "etc/pyflyby" if candidate.isdir: return candidate parent = dir.dir if parent == dir: break dir = parent return None
def _get_python_path(env_var_name, default_path, target_dirname): ''' Expand an environment variable specifying pyflyby input config files. - Default to C{default_path} if the environment variable is undefined. - Process colon delimiters. - Replace "-" with C{default_path}. - Expand triple dots. - Recursively traverse directories. @rtype: C{tuple} of C{Filename}s ''' pathnames = _get_env_var(env_var_name, default_path) if pathnames == ["EMPTY"]: # The special code PYFLYBY_PATH=EMPTY means we intentionally want to # use an empty PYFLYBY_PATH (and don't fall back to the default path, # nor warn about an empty path). return () for p in pathnames: if re.match("/|[.]/|[.][.][.]/|~/", p): continue raise ValueError( "{env_var_name} components should start with / or ./ or ~/ or .../. " "Use {env_var_name}=./{p} instead of {env_var_name}={p} if you really " "want to use the current directory.".format( env_var_name=env_var_name, p=p)) pathnames = [os.path.expanduser(p) for p in pathnames] pathnames = _expand_tripledots(pathnames, target_dirname) pathnames = [Filename(fn) for fn in pathnames] pathnames = stable_unique(pathnames) pathnames = expand_py_files_from_args(pathnames) if not pathnames: logger.warning( "No import libraries found (%s=%r, default=%r)" % (env_var_name, os.environ.get(env_var_name), default_path)) return tuple(pathnames)
def test_Filename_eqne_1(): assert Filename('/foo/bar') == Filename('/foo/bar') assert not (Filename('/foo/bar') != Filename('/foo/bar')) assert Filename('/foo/bar') != Filename('/foo/BAR') assert not (Filename('/foo/bar') == Filename('/foo/BAR'))
def test_Filename_dir_1(): f = Filename('/Foo.foo.f/Bar.bar.b/Quux.quux.q') assert f.dir == Filename('/Foo.foo.f/Bar.bar.b')
def test_filename_init_1(): fn = logging.__file__ fn = Filename(re.sub("[.]pyc$", ".py", fn)).real m = ModuleHandle("logging") assert m.filename.real == fn assert m.filename.base == "__init__.py"
def _tempfile(self): from tempfile import NamedTemporaryFile f = NamedTemporaryFile() self._tmpfiles.append(f) return f, Filename(f.name)
def _get_st_dev(filename): filename = Filename(filename) try: return os.stat(str(filename)).st_dev except OSError: return None
def test_Filename_1(): f = Filename('/etc/passwd') assert str(f) == '/etc/passwd'
def test_Filename_base_1(): f = Filename('/Foo.foo.f/Bar.bar.b/Quux.quux.q') assert f.base == 'Quux.quux.q'
def test_Filename_normpath_1(): with CwdCtx("/dev"): f = Filename("../a/b/../c") assert f == Filename("/a/c")
def test_Filename_ext_1(): f = Filename('/Foo.foo.f/Bar.bar.b/Quux.quux.q') assert f.ext == '.q'
def test_FileText_attrs_from_instance_1(): text = FileText(FileText("aabb\n"), filename="/foo", startpos=(100,5)) assert text.joined == "aabb\n" assert text.filename == Filename("/foo") assert text.startpos.lineno == 100 assert text.startpos.colno == 5
def test_Filename_isdir_1(): f = Filename('/etc/passwd') assert not f.isdir assert f.dir.isdir
def remote_print_stack(pid, output=1): """ Tell a target process to print a stack trace. This currently only handles the main thread. TODO: handle multiple threads. @param pid: PID of target process. @type output: C{int}, C{file}, or C{str} @param output: Output file descriptor. """ # Interpret C{output} argument as a file-like object, file descriptor, or # filename. if hasattr(output, 'write'): # file-like object output_fh = output try: output.flush() except Exception: pass try: output_fd = output.fileno() except Exception: output_fd = None try: output_fn = Filename(output.name) except Exception: pass elif isinstance(output, int): output_fh = None output_fn = None output_fd = output elif isinstance(output, (str, Filename)): output_fh = None output_fn = Filename(output) output_fd = None else: raise TypeError( "remote_print_stack_trace(): expected file/str/int; got %s" % (type(output).__name__, )) temp_file = None remote_fn = output_fn if remote_fn is None and output_fd is not None: remote_fn = Filename("/proc/%d/fd/%d" % (os.getpid(), output_fd)) # Figure out whether the target process will be able to open output_fn for # writing. Since the target process would need to be running as the same # user as this process for us to be able to attach a debugger, we can # simply check whether we ourselves can open the file. Typically output # will be fd 1 and we will have access to write to it. However, if we're # sudoed, we won't be able to re-open it via the proc symlink, even though # we already currently have it open. Another case is C{output} is a # file-like object that isn't a real file, e.g. a StringO. In each case # we we don't have a usable filename for the remote process yet. To # address these situations, we create a temporary file for the remote # process to write to. if remote_fn is None or not remote_fn.iswritable: if not output_fh or output_fd: assert remote_fn is not None raise OSError(errno.EACCESS, "Can't write to %s" % output_fn) # We can still use the /proc/$pid/fd approach with an unnamed temp # file. If it turns out there are situations where that doesn't work, # we can switch to using a NamedTemporaryFile. from tempfile import TemporaryFile temp_file = TemporaryFile() remote_fn = Filename("/proc/%d/fd/%d" % (os.getpid(), temp_file.fileno())) assert remote_fn.iswritable # *** Do the code injection *** _remote_print_stack_to_file(pid, remote_fn) # Copy from temp file to the requested output. if temp_file is not None: data = temp_file.read() temp_file.close() if output_fh is not None: output_fh.write(data) output_fh.flush() elif output_fd is not None: with os.fdopen(output_fd, 'w') as f: f.write(data) else: raise AssertionError("unreacahable")
def test_Filename_eqne_other_1(): assert Filename("/foo") != "/foo" assert not (Filename("/foo") == "/foo") assert Filename("/foo") != object() assert not (Filename("/foo") == object())
def _from_filename(cls, filename): filename = Filename(filename) raise NotImplementedError( "TODO: look at sys.path to guess module name")
def get_default(cls, target_filename): """ Return the default import library for the given target filename. This will read various .../.pyflyby files as specified by $PYFLYBY_PATH as well as older deprecated environment variables. Memoized. @param target_filename: The target filename for which to get the import database. Note that the target filename itself is not read. Instead, the target filename is relevant because we look for .../.pyflyby based on the target filename. @rtype: L{ImportDB} """ # We're going to canonicalize target_filenames in a number of steps. # At each step, see if we've seen the input so far. We do the cache # checking incrementally since the steps involve syscalls. Since this # is going to potentially be executed inside the IPython interactive # loop, we cache as much as possible. # TODO: Consider refreshing periodically. Check if files have # been touched, and if so, return new data. Check file timestamps at # most once every 60 seconds. cache_keys = [] target_filename = Filename(target_filename or ".") if target_filename.startswith("/dev"): target_filename = Filename(".") target_dirname = target_filename # TODO: with StatCache while True: cache_keys.append((1, target_dirname, os.getenv("PYFLYBY_PATH"), os.getenv("PYFLYBY_KNOWN_IMPORTS_PATH"), os.getenv("PYFLYBY_MANDATORY_IMPORTS_PATH"))) try: return cls._default_cache[cache_keys[-1]] except KeyError: pass if target_dirname.isdir: break target_dirname = target_dirname.dir target_dirname = target_dirname.real if target_dirname != cache_keys[-1][0]: cache_keys.append((1, target_dirname, os.getenv("PYFLYBY_PATH"), os.getenv("PYFLYBY_KNOWN_IMPORTS_PATH"), os.getenv("PYFLYBY_MANDATORY_IMPORTS_PATH"))) try: return cls._default_cache[cache_keys[-1]] except KeyError: pass DEFAULT_PYFLYBY_PATH = [] etc_dir = _find_etc_dir() if etc_dir: DEFAULT_PYFLYBY_PATH.append(str(etc_dir)) DEFAULT_PYFLYBY_PATH += [ ".../.pyflyby", "~/.pyflyby", ] logger.debug("DEFAULT_PYFLYBY_PATH=%s", DEFAULT_PYFLYBY_PATH) filenames = _get_python_path("PYFLYBY_PATH", DEFAULT_PYFLYBY_PATH, target_dirname) mandatory_imports_filenames = () if "SUPPORT DEPRECATED BEHAVIOR": PYFLYBY_PATH = _get_env_var("PYFLYBY_PATH", DEFAULT_PYFLYBY_PATH) # If the old deprecated environment variables are set, then heed # them. if os.getenv("PYFLYBY_KNOWN_IMPORTS_PATH"): # Use PYFLYBY_PATH as the default for # PYFLYBY_KNOWN_IMPORTS_PATH. Note that the default is # relevant even though we only enter this code path when the # variable is set to anything, because the env var can # reference "-" to include the default. # Before pyflyby version 0.8, the default value would have # been # [d/"known_imports" for d in PYFLYBY_PATH] # Instead of using that, we just use PYFLYBY_PATH directly as # the default. This simplifies things and avoids need for a # "known_imports=>." symlink for backwards compatibility. It # means that ~/.pyflyby/**/*.py (as opposed to only # ~/.pyflyby/known_imports/**/*.py) would be included. # Although this differs slightly from the old behavior, it # matches the behavior of the newer PYFLYBY_PATH; matching the # new behavior seems higher utility than exactly matching the # old behavior. Files under ~/.pyflyby/mandatory_imports will # be included in known_imports as well, but that should not # cause any problems. default_path = PYFLYBY_PATH # Expand $PYFLYBY_KNOWN_IMPORTS_PATH. filenames = _get_python_path("PYFLYBY_KNOWN_IMPORTS_PATH", default_path, target_dirname) logger.debug( "The environment variable PYFLYBY_KNOWN_IMPORTS_PATH is deprecated. " "Use PYFLYBY_PATH.") if os.getenv("PYFLYBY_MANDATORY_IMPORTS_PATH"): # Compute the "default" path. # Note that we still calculate the erstwhile default value, # even though it's no longer the defaults, in order to still # allow the "-" in the variable. default_path = [ os.path.join(d, "mandatory_imports") for d in PYFLYBY_PATH ] # Expand $PYFLYBY_MANDATORY_IMPORTS_PATH. mandatory_imports_filenames = _get_python_path( "PYFLYBY_MANDATORY_IMPORTS_PATH", default_path, target_dirname) logger.debug( "The environment variable PYFLYBY_MANDATORY_IMPORTS_PATH is deprecated. " "Use PYFLYBY_PATH and write __mandatory_imports__=['...'] in your files." ) cache_keys.append((2, filenames, mandatory_imports_filenames)) try: return cls._default_cache[cache_keys[-1]] except KeyError: pass result = cls._from_filenames(filenames, mandatory_imports_filenames) for k in cache_keys: cls._default_cache[k] = result return result
def from_filename(cls, filename): return cls.from_text(Filename(filename))
def test_Filename_abspath_1(): with CwdCtx("/dev"): f = Filename("foo") assert f == Filename("/dev/foo")