def _expand_tripledots(pathnames, target_dirname):
"""
Expand pathnames of the form C{".../foo/bar"} as "../../foo/bar",
"../foo/bar", "./foo/bar" etc., up to the oldest ancestor with the same
st_dev.
For example, suppose a partition is mounted on /u/homer; /u is a different
partition. Then::
>> _expand_tripledots(["/foo", ".../tt"], "/u/homer/aa")
[Filename("/foo"), Filename("/u/homer/tt"), Filename("/u/homer/aa/tt")]
@type pathnames:
sequence of C{str} (not C{Filename})
@type target_dirname:
L{Filename}
@rtype:
C{list} of L{Filename}
"""
target_dirname = Filename(target_dirname)
if not isinstance(pathnames, (tuple, list)):
pathnames = [pathnames]
result = []
for pathname in pathnames:
if not pathname.startswith(".../"):
result.append(Filename(pathname))
continue
suffix = pathname[4:]
expanded = [
p / suffix for p in _ancestors_on_same_partition(target_dirname)
]
result.extend(expanded[::-1])
return result
def _expand_tripledots(pathnames, target_dirname):
"""
Expand pathnames of the form ``".../foo/bar"`` as "../../foo/bar",
"../foo/bar", "./foo/bar" etc., up to the oldest ancestor with the same
st_dev.
For example, suppose a partition is mounted on /u/homer; /u is a different
partition. Then::
>>> _expand_tripledots(["/foo", ".../tt"], "/u/homer/aa") # doctest: +SKIP
[Filename("/foo"), Filename("/u/homer/tt"), Filename("/u/homer/aa/tt")]
:type pathnames:
sequence of ``str`` (not ``Filename``)
:type target_dirname:
`Filename`
:rtype:
``list`` of `Filename`
"""
target_dirname = Filename(target_dirname)
if not isinstance(pathnames, (tuple, list)):
pathnames = [pathnames]
result = []
for pathname in pathnames:
if not pathname.startswith(".../"):
result.append(Filename(pathname))
continue
suffix = pathname[4:]
expanded = [
p / suffix for p in _ancestors_on_same_partition(target_dirname)
]
result.extend(expanded[::-1])
return result
def test_Filename_ancestors_1():
fn = Filename("/a.aa/b.bb/c.cc")
result = fn.ancestors
expected = (Filename("/a.aa/b.bb/c.cc"),
Filename("/a.aa/b.bb"),
Filename("/a.aa"),
Filename("/"))
assert result == expected
def _find_etc_dirs():
result = []
dirs = Filename(__file__).real.dir.ancestors[:-1]
for dir in dirs:
candidate = dir / "etc/pyflyby"
if candidate.isdir:
result.append(candidate)
break
global_dir = Filename("/etc/pyflyby")
if global_dir.exists:
result.append(global_dir)
return result
def _from_filenames(cls, filenames, _mandatory_filenames_deprecated=[]):
"""
Load an import database from filenames.
This function exists to support deprecated behavior.
When we stop supporting the old behavior, we will delete this function.
@type filenames:
Sequence of L{Filename}s
@param filenames:
Filenames of files to read.
@rtype:
L{ImportDB}
"""
if not isinstance(filenames, (tuple, list)):
filenames = [filenames]
filenames = [Filename(f) for f in filenames]
logger.debug("ImportDB: loading [%s], mandatory=[%s]",
', '.join(map(str, filenames)),
', '.join(map(str, _mandatory_filenames_deprecated)))
if "SUPPORT DEPRECATED BEHAVIOR":
# Before 2014-10, pyflyby read the following:
# * known_imports from $PYFLYBY_PATH/known_imports/**/*.py or
# $PYFLYBY_KNOWN_IMPORTS_PATH/**/*.py,
# * mandatory_imports from $PYFLYBY_PATH/mandatory_imports/**/*.py or
# $PYFLYBY_MANDATORY_IMPORTS_PATH/**/*.py, and
# * forget_imports from $PYFLYBY_PATH/known_imports/**/__remove__.py
# After 2014-10, pyflyby reads the following:
# * $PYFLYBY_PATH/**/*.py
# (with directives inside the file)
# For backwards compatibility, for now we continue supporting the
# old, deprecated behavior.
blocks = []
mandatory_imports_blocks = [
Filename(f) for f in _mandatory_filenames_deprecated
]
forget_imports_blocks = []
for filename in filenames:
if filename.base == "__remove__.py":
forget_imports_blocks.append(filename)
elif "mandatory_imports" in str(filename).split("/"):
mandatory_imports_blocks.append(filename)
else:
blocks.append(filename)
return cls._from_code(blocks, mandatory_imports_blocks,
forget_imports_blocks)
else:
return cls._from_code(filenames)
def get_executable(pid):
"""
Get the full path for the target process.
@type pid:
C{int}
@rtype:
L{Filename}
"""
uname = os.uname()[0]
if uname == 'Linux':
result = os.readlink('/proc/%d/exe' % (pid, ))
elif uname == 'SunOS':
result = os.readlink('/proc/%d/path/a.out' % (pid, ))
else:
# Use psutil to try to answer this. This should also work for the
# above cases too, but it's simple enough to implement it directly and
# avoid this dependency on those platforms.
import psutil
result = psutil.Process(pid).exe()
result = Filename(result).real
if not result.isfile:
raise ValueError("Couldn't get executable for pid %s" % (pid, ))
if not result.isreadable:
raise ValueError("Executable %s for pid %s is not readable" %
(result, pid))
return result
def filename(self):
"""
Return the filename, if appropriate.
The module itself will not be imported, but if the module is not a
top-level module/package, accessing this attribute may cause the
parent package to be imported.
@rtype:
L{Filename}
"""
# Use the loader mechanism to find the filename. We do so instead of
# using self.module.__file__, because the latter forces importing a
# module, which may be undesirable.
import pkgutil
try:
loader = pkgutil.get_loader(str(self.name))
except ImportError:
return None
if not loader:
return None
# Get the filename using loader.get_filename(). Note that this does
# more than just loader.filename: for example, it adds /__init__.py
# for packages.
filename = loader.get_filename()
if not filename:
return None
return Filename(pyc_to_py(filename))
def _find_py_commandline():
global _cached_py_commandline
if _cached_py_commandline is not None:
return _cached_py_commandline
import pyflyby
pkg_path = Filename(pyflyby.__path__[0]).real
assert pkg_path.base == "pyflyby"
d = pkg_path.dir
if d.base == "bin":
# Running from source tree
bindir = d
else:
# Installed by setup.py
while d.dir != d:
d = d.dir
bindir = d / "bin"
if bindir.exists:
break
else:
raise ValueError("Couldn't find 'py' script: "
"couldn't find 'bin' dir from package path %s" %
(pkg_path, ))
candidate = bindir / "py"
if not candidate.exists:
raise ValueError("Couldn't find 'py' script: expected it at %s" %
(candidate, ))
if not candidate.isexecutable:
raise ValueError("Found 'py' script at %s but it's not executable" %
(candidate, ))
_cached_py_commandline = candidate
return candidate
def describe_xref(identifier, container):
module = ModuleHandle(str(container.defining_module.canonical_name))
assert module.filename == Filename(container.defining_module.filename)
linenos = get_string_linenos(
module,
"(L{|<)%s" % (identifier,),
container.docstring)
return (module, linenos, str(container.canonical_name), identifier)
def test_PythonBlock_attrs_1():
block = PythonBlock(dedent('''
foo()
bar()
''').lstrip(), filename="/foo/test_PythonBlock_1.py", startpos=(101,99))
assert block.text.lines == ("foo()", "bar()", "")
assert block.filename == Filename("/foo/test_PythonBlock_1.py")
assert block.startpos == FilePos(101, 99)
def test_PythonBlock_FileText_1():
text = FileText(dedent('''
foo()
bar()
''').lstrip(), filename="/foo/test_PythonBlock_1.py", startpos=(101,55))
block = PythonBlock(text)
assert text is block.text
assert text is FileText(block)
assert block.filename == Filename("/foo/test_PythonBlock_1.py")
assert block.startpos == FilePos(101, 55)
def _find_etc_dir():
dir = Filename(__file__).real.dir
while True:
candidate = dir / "etc/pyflyby"
if candidate.isdir:
return candidate
parent = dir.dir
if parent == dir:
break
dir = parent
return None
def _get_python_path(env_var_name, default_path, target_dirname):
'''
Expand an environment variable specifying pyflyby input config files.
- Default to C{default_path} if the environment variable is undefined.
- Process colon delimiters.
- Replace "-" with C{default_path}.
- Expand triple dots.
- Recursively traverse directories.
@rtype:
C{tuple} of C{Filename}s
'''
pathnames = _get_env_var(env_var_name, default_path)
if pathnames == ["EMPTY"]:
# The special code PYFLYBY_PATH=EMPTY means we intentionally want to
# use an empty PYFLYBY_PATH (and don't fall back to the default path,
# nor warn about an empty path).
return ()
for p in pathnames:
if re.match("/|[.]/|[.][.][.]/|~/", p):
continue
raise ValueError(
"{env_var_name} components should start with / or ./ or ~/ or .../. "
"Use {env_var_name}=./{p} instead of {env_var_name}={p} if you really "
"want to use the current directory.".format(
env_var_name=env_var_name, p=p))
pathnames = [os.path.expanduser(p) for p in pathnames]
pathnames = _expand_tripledots(pathnames, target_dirname)
pathnames = [Filename(fn) for fn in pathnames]
pathnames = stable_unique(pathnames)
pathnames = expand_py_files_from_args(pathnames)
if not pathnames:
logger.warning(
"No import libraries found (%s=%r, default=%r)" %
(env_var_name, os.environ.get(env_var_name), default_path))
return tuple(pathnames)
def test_Filename_eqne_1():
assert Filename('/foo/bar') == Filename('/foo/bar')
assert not (Filename('/foo/bar') != Filename('/foo/bar'))
assert Filename('/foo/bar') != Filename('/foo/BAR')
assert not (Filename('/foo/bar') == Filename('/foo/BAR'))
def test_Filename_dir_1():
f = Filename('/Foo.foo.f/Bar.bar.b/Quux.quux.q')
assert f.dir == Filename('/Foo.foo.f/Bar.bar.b')
def test_filename_init_1():
fn = logging.__file__
fn = Filename(re.sub("[.]pyc$", ".py", fn)).real
m = ModuleHandle("logging")
assert m.filename.real == fn
assert m.filename.base == "__init__.py"
def _tempfile(self):
from tempfile import NamedTemporaryFile
f = NamedTemporaryFile()
self._tmpfiles.append(f)
return f, Filename(f.name)
def _get_st_dev(filename):
filename = Filename(filename)
try:
return os.stat(str(filename)).st_dev
except OSError:
return None
def test_Filename_1():
f = Filename('/etc/passwd')
assert str(f) == '/etc/passwd'
def test_Filename_base_1():
f = Filename('/Foo.foo.f/Bar.bar.b/Quux.quux.q')
assert f.base == 'Quux.quux.q'
def test_Filename_normpath_1():
with CwdCtx("/dev"):
f = Filename("../a/b/../c")
assert f == Filename("/a/c")
def test_Filename_ext_1():
f = Filename('/Foo.foo.f/Bar.bar.b/Quux.quux.q')
assert f.ext == '.q'
def test_FileText_attrs_from_instance_1():
text = FileText(FileText("aabb\n"), filename="/foo", startpos=(100,5))
assert text.joined == "aabb\n"
assert text.filename == Filename("/foo")
assert text.startpos.lineno == 100
assert text.startpos.colno == 5
def test_Filename_isdir_1():
f = Filename('/etc/passwd')
assert not f.isdir
assert f.dir.isdir
def remote_print_stack(pid, output=1):
"""
Tell a target process to print a stack trace.
This currently only handles the main thread.
TODO: handle multiple threads.
@param pid:
PID of target process.
@type output:
C{int}, C{file}, or C{str}
@param output:
Output file descriptor.
"""
# Interpret C{output} argument as a file-like object, file descriptor, or
# filename.
if hasattr(output, 'write'): # file-like object
output_fh = output
try:
output.flush()
except Exception:
pass
try:
output_fd = output.fileno()
except Exception:
output_fd = None
try:
output_fn = Filename(output.name)
except Exception:
pass
elif isinstance(output, int):
output_fh = None
output_fn = None
output_fd = output
elif isinstance(output, (str, Filename)):
output_fh = None
output_fn = Filename(output)
output_fd = None
else:
raise TypeError(
"remote_print_stack_trace(): expected file/str/int; got %s" %
(type(output).__name__, ))
temp_file = None
remote_fn = output_fn
if remote_fn is None and output_fd is not None:
remote_fn = Filename("/proc/%d/fd/%d" % (os.getpid(), output_fd))
# Figure out whether the target process will be able to open output_fn for
# writing. Since the target process would need to be running as the same
# user as this process for us to be able to attach a debugger, we can
# simply check whether we ourselves can open the file. Typically output
# will be fd 1 and we will have access to write to it. However, if we're
# sudoed, we won't be able to re-open it via the proc symlink, even though
# we already currently have it open. Another case is C{output} is a
# file-like object that isn't a real file, e.g. a StringO. In each case
# we we don't have a usable filename for the remote process yet. To
# address these situations, we create a temporary file for the remote
# process to write to.
if remote_fn is None or not remote_fn.iswritable:
if not output_fh or output_fd:
assert remote_fn is not None
raise OSError(errno.EACCESS, "Can't write to %s" % output_fn)
# We can still use the /proc/$pid/fd approach with an unnamed temp
# file. If it turns out there are situations where that doesn't work,
# we can switch to using a NamedTemporaryFile.
from tempfile import TemporaryFile
temp_file = TemporaryFile()
remote_fn = Filename("/proc/%d/fd/%d" %
(os.getpid(), temp_file.fileno()))
assert remote_fn.iswritable
# *** Do the code injection ***
_remote_print_stack_to_file(pid, remote_fn)
# Copy from temp file to the requested output.
if temp_file is not None:
data = temp_file.read()
temp_file.close()
if output_fh is not None:
output_fh.write(data)
output_fh.flush()
elif output_fd is not None:
with os.fdopen(output_fd, 'w') as f:
f.write(data)
else:
raise AssertionError("unreacahable")
def test_Filename_eqne_other_1():
assert Filename("/foo") != "/foo"
assert not (Filename("/foo") == "/foo")
assert Filename("/foo") != object()
assert not (Filename("/foo") == object())
def _from_filename(cls, filename):
filename = Filename(filename)
raise NotImplementedError(
"TODO: look at sys.path to guess module name")
def get_default(cls, target_filename):
"""
Return the default import library for the given target filename.
This will read various .../.pyflyby files as specified by
$PYFLYBY_PATH as well as older deprecated environment variables.
Memoized.
@param target_filename:
The target filename for which to get the import database. Note that
the target filename itself is not read. Instead, the target
filename is relevant because we look for .../.pyflyby based on the
target filename.
@rtype:
L{ImportDB}
"""
# We're going to canonicalize target_filenames in a number of steps.
# At each step, see if we've seen the input so far. We do the cache
# checking incrementally since the steps involve syscalls. Since this
# is going to potentially be executed inside the IPython interactive
# loop, we cache as much as possible.
# TODO: Consider refreshing periodically. Check if files have
# been touched, and if so, return new data. Check file timestamps at
# most once every 60 seconds.
cache_keys = []
target_filename = Filename(target_filename or ".")
if target_filename.startswith("/dev"):
target_filename = Filename(".")
target_dirname = target_filename
# TODO: with StatCache
while True:
cache_keys.append((1, target_dirname, os.getenv("PYFLYBY_PATH"),
os.getenv("PYFLYBY_KNOWN_IMPORTS_PATH"),
os.getenv("PYFLYBY_MANDATORY_IMPORTS_PATH")))
try:
return cls._default_cache[cache_keys[-1]]
except KeyError:
pass
if target_dirname.isdir:
break
target_dirname = target_dirname.dir
target_dirname = target_dirname.real
if target_dirname != cache_keys[-1][0]:
cache_keys.append((1, target_dirname, os.getenv("PYFLYBY_PATH"),
os.getenv("PYFLYBY_KNOWN_IMPORTS_PATH"),
os.getenv("PYFLYBY_MANDATORY_IMPORTS_PATH")))
try:
return cls._default_cache[cache_keys[-1]]
except KeyError:
pass
DEFAULT_PYFLYBY_PATH = []
etc_dir = _find_etc_dir()
if etc_dir:
DEFAULT_PYFLYBY_PATH.append(str(etc_dir))
DEFAULT_PYFLYBY_PATH += [
".../.pyflyby",
"~/.pyflyby",
]
logger.debug("DEFAULT_PYFLYBY_PATH=%s", DEFAULT_PYFLYBY_PATH)
filenames = _get_python_path("PYFLYBY_PATH", DEFAULT_PYFLYBY_PATH,
target_dirname)
mandatory_imports_filenames = ()
if "SUPPORT DEPRECATED BEHAVIOR":
PYFLYBY_PATH = _get_env_var("PYFLYBY_PATH", DEFAULT_PYFLYBY_PATH)
# If the old deprecated environment variables are set, then heed
# them.
if os.getenv("PYFLYBY_KNOWN_IMPORTS_PATH"):
# Use PYFLYBY_PATH as the default for
# PYFLYBY_KNOWN_IMPORTS_PATH. Note that the default is
# relevant even though we only enter this code path when the
# variable is set to anything, because the env var can
# reference "-" to include the default.
# Before pyflyby version 0.8, the default value would have
# been
# [d/"known_imports" for d in PYFLYBY_PATH]
# Instead of using that, we just use PYFLYBY_PATH directly as
# the default. This simplifies things and avoids need for a
# "known_imports=>." symlink for backwards compatibility. It
# means that ~/.pyflyby/**/*.py (as opposed to only
# ~/.pyflyby/known_imports/**/*.py) would be included.
# Although this differs slightly from the old behavior, it
# matches the behavior of the newer PYFLYBY_PATH; matching the
# new behavior seems higher utility than exactly matching the
# old behavior. Files under ~/.pyflyby/mandatory_imports will
# be included in known_imports as well, but that should not
# cause any problems.
default_path = PYFLYBY_PATH
# Expand $PYFLYBY_KNOWN_IMPORTS_PATH.
filenames = _get_python_path("PYFLYBY_KNOWN_IMPORTS_PATH",
default_path, target_dirname)
logger.debug(
"The environment variable PYFLYBY_KNOWN_IMPORTS_PATH is deprecated. "
"Use PYFLYBY_PATH.")
if os.getenv("PYFLYBY_MANDATORY_IMPORTS_PATH"):
# Compute the "default" path.
# Note that we still calculate the erstwhile default value,
# even though it's no longer the defaults, in order to still
# allow the "-" in the variable.
default_path = [
os.path.join(d, "mandatory_imports") for d in PYFLYBY_PATH
]
# Expand $PYFLYBY_MANDATORY_IMPORTS_PATH.
mandatory_imports_filenames = _get_python_path(
"PYFLYBY_MANDATORY_IMPORTS_PATH", default_path,
target_dirname)
logger.debug(
"The environment variable PYFLYBY_MANDATORY_IMPORTS_PATH is deprecated. "
"Use PYFLYBY_PATH and write __mandatory_imports__=['...'] in your files."
)
cache_keys.append((2, filenames, mandatory_imports_filenames))
try:
return cls._default_cache[cache_keys[-1]]
except KeyError:
pass
result = cls._from_filenames(filenames, mandatory_imports_filenames)
for k in cache_keys:
cls._default_cache[k] = result
return result
def from_filename(cls, filename):
return cls.from_text(Filename(filename))
def test_Filename_abspath_1():
with CwdCtx("/dev"):
f = Filename("foo")
assert f == Filename("/dev/foo")
