def process_python_dependencies(localpath, proto_dir, action, pkg_vars):
"""Given the path to a python file, the proto area containing that file,
the action that produced the dependency, and the variants against which
the action's package was published, produce a list of PythonDependency
objects."""
mf = modulefinder.DepthLimitedModuleFinder(proto_dir)
mf.run_script(localpath, depth=1)
deps = []
errs = []
for m in mf.modules.values():
if m.__name__ == "__main__":
# The file at localpath is returned as a loaded module
# under the name __main__.
continue
if m.__file__ is not None:
deps.append(PythonDependency(action, m.__file__,
pkg_vars, proto_dir))
else:
errs.append(PythonModuleMissingPath(m.__name__,
localpath))
return deps, errs
def process_python_dependencies(action, pkg_vars, script_path, run_paths):
"""Analyze the file delivered by the action for any python dependencies.
The 'action' parameter contain the action which delivers the file.
The 'pkg_vars' parameter contains the variants against which
the action's package was published.
The 'script_path' parameter is None of the file is not executable, or
is the path for the binary which is used to execute the file.
The 'run_paths' parameter is a list of paths that should be searched
for modules.
"""
# There are three conditions which determine whether python dependency
# analysis is performed on a file with python in its #! line.
# 1) Is the file executable. (Represented in the table below by X)
# 2) Is the file installed into a directory which provides information
# about what version of python should be used for it.
# (Represented by D)
# 3) Does the first line of the file include a specific version of
# python. (Represented by F)
#
# Conditions || Perform Analysis
# X D F || Y, if F and D disagree, display a warning in the output
# || and use D to analyze the file.
# X D !F || Y
# X !D F || Y
# X !D !F || N, and display a warning in the output.
# !X D F || Y
# !X D !F || Y
# !X !D F || N
# !X !D !F || N
local_file = action.attrs[PD_LOCAL_PATH]
deps = []
errs = []
path_version = None
dir_major = None
dir_minor = None
file_major = None
file_minor = None
cur_major = None
cur_minor = None
executable = bool(script_path)
# Version of python to use to do the analysis.
analysis_major = None
analysis_minor = None
cur_major, cur_minor = sys.version_info[0:2]
install_match = py_lib_re.match(action.attrs["path"])
if install_match:
dir_major = install_match.group("major")
dir_minor = install_match.group("minor")
script_match = None
if script_path:
script_match = py_bin_re.match(script_path)
if script_match:
file_major = script_match.group("major")
file_minor = script_match.group("minor")
if executable:
# Check whether the version of python declared in the #! line
# of the file and the version of python implied by the directory
# the file is delivered into match.
if install_match and script_match and \
(file_major != dir_major or file_minor != dir_minor):
errs.append(
PythonMismatchedVersion(
"{0}.{1}".format(dir_major, dir_minor),
"{0}.{1}".format(file_major, file_minor), local_file,
action.attrs["path"]))
if install_match:
analysis_major = dir_major
analysis_minor = dir_minor
elif script_match:
analysis_major = file_major
analysis_minor = file_minor
else:
# An example of this case is an executable file in
# /usr/bin with #!/usr/bin/python as its first line.
errs.append(
PythonUnspecifiedVersion(local_file, action.attrs["path"]))
elif install_match:
analysis_major = dir_major
analysis_minor = dir_minor
if analysis_major is None or analysis_minor is None:
return deps, errs, {}
analysis_major = int(analysis_major)
analysis_minor = int(analysis_minor)
# If the version implied by the directory hierarchy matches the version
# of python running, use the default analyzer and don't fork and exec.
if cur_major == analysis_major and cur_minor == analysis_minor:
mf = modulefinder.DepthLimitedModuleFinder(os.path.dirname(
action.attrs["path"]),
run_paths=run_paths)
try:
loaded_modules = mf.run_script(local_file)
for names, dirs in set([(tuple(m.get_file_names()), tuple(m.dirs))
for m in loaded_modules]):
# Add the directory the python file will be
# installed in to the paths used to find modules
# for import. This allows relative imports to
# work correctly.
deps.append(
PythonDependency(action, names, dirs, pkg_vars,
action.attrs[PD_PROTO_DIR]))
missing, maybe = mf.any_missing_maybe()
for name in missing:
errs.append(
PythonModuleMissingPath(name, action.attrs[PD_LOCAL_PATH]))
except SyntaxError as e:
errs.append(
PythonSyntaxError(action.attrs["path"], local_file, s_err=e))
except Exception as e:
errs.append(e)
return deps, errs, {}
# If the version implied by the directory hierarchy does not match the
# version of python running, it's necessary to fork and run the
# appropriate version of python.
root_dir = os.path.dirname(__file__)
exec_file = os.path.join(root_dir, "depthlimitedmf.py")
cmd = [
"python{0}.{1}".format(analysis_major, analysis_minor), exec_file,
os.path.dirname(action.attrs["path"]), local_file
]
newenv = os.environ.copy()
# Tell Python to not create .pyc, .pyo, etc. cache files for any Python
# modules our script imports.
newenv["PYTHONDONTWRITEBYTECODE"] = "1"
if run_paths:
cmd.extend(run_paths)
try:
sp = subprocess.Popen(cmd,
env=newenv,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding="utf-8")
except Exception as e:
return [], [PythonSubprocessError(None, " ".join(cmd),\
str(e))], {}
out, err = sp.communicate()
out = force_str(out)
if sp.returncode:
errs.append(PythonSubprocessError(sp.returncode, " ".join(cmd), err))
bad_lines = []
for l in out.splitlines():
l = l.strip()
if l.startswith("DEP "):
try:
names, dirs = eval(l[4:])
except Exception:
bad_lines.append(l)
else:
deps.append(
PythonDependency(action, names, dirs, pkg_vars,
action.attrs[PD_PROTO_DIR]))
elif l.startswith("ERR MISSING "):
errs.append(
PythonModuleMissingPath(l[len("ERR MISSING "):],
action.attrs[PD_LOCAL_PATH]))
elif l.startswith("ERR SYNTAX "):
errs.append(
PythonSyntaxError(action.attrs["path"], local_file, msg=l))
elif l.startswith("ERR "):
# Generic error which is assigned as a missing path
errs.append(
PythonModuleMissingPath(l[4:], action.attrs[PD_LOCAL_PATH]))
else:
bad_lines.append(l)
if bad_lines:
errs.append(PythonSubprocessBadLine(" ".join(cmd), bad_lines))
return deps, errs, {}
def process_python_dependencies(action, pkg_vars, script_path):
"""Analyze the file delivered by the action for any python dependencies.
The 'action' parameter contain the action which delivers the file.
The 'pkg_vars' parameter contains the variants against which
the action's package was published.
The 'script_path' parameter is None of the file is not executable, or
is the path for the binary which is used to execute the file.
"""
# There are three conditions which determine whether python dependency
# analysis is performed on a file with python in its #! line.
# 1) Is the file executable. (Represented in the table below by X)
# 2) Is the file installed into a directory which provides information
# about what version of python should be used for it.
# (Represented by D)
# 3) Does the first line of the file include a specific version of
# python. (Represented by F)
#
# Conditions || Perform Analysis
# X D F || Y, if F and D disagree, display a warning in the output
# || and use D to analyze the file.
# X D !F || Y
# X !D F || Y
# X !D !F || N, and display a warning in the output.
# !X D F || Y
# !X D !F || Y
# !X !D F || N
# !X !D !F || N
local_file = action.attrs[PD_LOCAL_PATH]
deps = []
errs = []
path_version = None
dir_major = None
dir_minor = None
file_major = None
file_minor = None
cur_major = None
cur_minor = None
executable = bool(script_path)
# Version of python to use to do the analysis.
analysis_major = None
analysis_minor = None
cur_major, cur_minor = sys.version_info[0:2]
install_match = py_lib_re.match(action.attrs["path"])
if install_match:
dir_major = install_match.group("major")
dir_minor = install_match.group("minor")
script_match = None
if script_path:
script_match = py_bin_re.match(script_path)
if script_match:
file_major = script_match.group("major")
file_minor = script_match.group("minor")
if executable:
# Check whether the version of python declared in the #! line
# of the file and the version of python implied by the directory
# the file is delivered into match.
if install_match and script_match and \
(file_major != dir_major or file_minor != dir_minor):
errs.append(
PythonMismatchedVersion("%s.%s" % (dir_major, dir_minor),
"%s.%s" % (file_major, file_minor),
local_file, action.attrs["path"]))
if install_match:
analysis_major = dir_major
analysis_minor = dir_minor
elif script_match:
analysis_major = file_major
analysis_minor = file_minor
else:
# An example of this case is an executable file in
# /usr/bin with #!/usr/bin/python as its first line.
errs.append(
PythonUnspecifiedVersion(local_file, action.attrs["path"]))
elif install_match:
analysis_major = dir_major
analysis_minor = dir_minor
if analysis_major is None or analysis_minor is None:
return deps, errs, {}
analysis_major = int(analysis_major)
analysis_minor = int(analysis_minor)
# If the version implied by the directory hierarchy matches the version
# of python running, use the default analyzer and don't fork and exec.
if cur_major == analysis_major and cur_minor == analysis_minor:
mf = modulefinder.DepthLimitedModuleFinder(action.attrs[PD_PROTO_DIR])
loaded_modules = mf.run_script(local_file)
for names, dirs in set([(tuple(m.get_file_names()), tuple(m.dirs))
for m in loaded_modules]):
deps.append(
PythonDependency(action, names, dirs, pkg_vars,
action.attrs[PD_PROTO_DIR]))
missing, maybe = mf.any_missing_maybe()
for name in missing:
errs.append(
PythonModuleMissingPath(name, action.attrs[PD_LOCAL_PATH]))
return deps, errs, {}
# If the version implied by the directory hierarchy does not match the
# version of python running, it's necessary to fork and run the
# appropriate version of python.
root_dir = os.path.dirname(__file__)
exec_file = os.path.join(
root_dir, "depthlimitedmf%s%s.py" % (analysis_major, analysis_minor))
cmd = [
"python%s.%s" % (analysis_major, analysis_minor), exec_file,
action.attrs[PD_PROTO_DIR], local_file
]
try:
sp = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except Exception, e:
return [], [PythonSubprocessError(None, " ".join(cmd),\
str(e))], {}
