def test_python_args(self):
file_path1 = _create_file("file1", "#!/usr/bin/python")
file_path2 = _create_file(
"file2",
textwrap.dedent(
"""\
#! /usr/bin/python -E
# Larger file
"""
),
)
mangling.rewrite_python_shebangs(os.path.dirname(file_path1))
self.assertThat(file_path1, FileContains("#!/usr/bin/env python"))
self.assertThat(
file_path2,
FileContains(
textwrap.dedent(
"""\
#!/bin/sh
''''exec python -E -- "$0" "$@" # '''
# Larger file
"""
)
),
)
def test_following_docstring_no_rewrite(self):
file_path = _create_file(
"file",
textwrap.dedent(
"""\
#!/usr/bin/env python3.5
'''
This is a test
=======================
"""
),
)
mangling.rewrite_python_shebangs(os.path.dirname(file_path))
self.assertThat(
file_path,
FileContains(
textwrap.dedent(
"""\
#!/usr/bin/env python3.5
'''
This is a test
=======================
"""
)
),
)
def _fix_shebangs(self, unpackdir):
"""Changes hard coded shebangs for files in _BIN_PATHS to use env."""
paths = [
p for p in _BIN_PATHS if os.path.exists(os.path.join(unpackdir, p))
]
for p in [os.path.join(unpackdir, p) for p in paths]:
mangling.rewrite_python_shebangs(p)
def build(self):
super().build()
env = os.environ.copy()
# Ensure the first provider does not attempt to validate against
# providers installed on the build host by initialising PROVIDERPATH
# to empty
env["PROVIDERPATH"] = ""
provider_stage_dir = os.path.join(self.project.stage_dir, "providers")
if os.path.exists(provider_stage_dir):
provider_dirs = [
os.path.join(provider_stage_dir, provider)
for provider in os.listdir(provider_stage_dir)
]
env["PROVIDERPATH"] = ":".join(provider_dirs)
self.run(["python3", "manage.py", "validate"], env=env)
self.run(["python3", "manage.py", "build"])
self.run(["python3", "manage.py", "i18n"])
self.run([
"python3",
"manage.py",
"install",
"--layout=relocatable",
"--prefix=/providers/{}".format(self.name),
"--root={}".format(self.installdir),
])
mangling.rewrite_python_shebangs(self.installdir)
def test_following_docstring_no_rewrite(self):
file_path = _create_file(
"file",
textwrap.dedent(
"""\
#!/usr/bin/env python3.5
'''
This is a test
=======================
"""
),
)
mangling.rewrite_python_shebangs(os.path.dirname(file_path))
self.assertThat(
file_path,
FileContains(
textwrap.dedent(
"""\
#!/usr/bin/env python3.5
'''
This is a test
=======================
"""
)
),
)
def test_python_args(self):
file_path1 = _create_file("file1", "#!/usr/bin/python")
file_path2 = _create_file(
"file2",
textwrap.dedent(
"""\
#! /usr/bin/python -E
# Larger file
"""
),
)
mangling.rewrite_python_shebangs(os.path.dirname(file_path1))
self.assertThat(file_path1, FileContains("#!/usr/bin/env python"))
self.assertThat(
file_path2,
FileContains(
textwrap.dedent(
"""\
#!/bin/sh
''''exec python -E -- "$0" "$@" # '''
# Larger file
"""
)
),
)
def build(self):
super().build()
env = os.environ.copy()
# Ensure the first provider does not attempt to validate against
# providers installed on the build host by initialising PROVIDERPATH
# to empty
env["PROVIDERPATH"] = ""
provider_stage_dir = os.path.join(self.project.stage_dir, "providers")
if os.path.exists(provider_stage_dir):
provider_dirs = [
os.path.join(provider_stage_dir, provider)
for provider in os.listdir(provider_stage_dir)
]
env["PROVIDERPATH"] = ":".join(provider_dirs)
self.run(["python3", "manage.py", "validate"], env=env)
self.run(["python3", "manage.py", "build"])
self.run(["python3", "manage.py", "i18n"])
self.run(
[
"python3",
"manage.py",
"install",
"--layout=relocatable",
"--prefix=/providers/{}".format(self.name),
"--root={}".format(self.installdir),
]
)
mangling.rewrite_python_shebangs(self.installdir)
def test_python3_args(self):
file_path1 = _create_file('file1', '#!/usr/bin/python3')
file_path2 = _create_file('file2', '#!/usr/bin/python3 -E')
mangling.rewrite_python_shebangs(os.path.dirname(file_path1))
self.assertThat(file_path1, FileContains('#!/usr/bin/env python3'))
self.assertThat(file_path2, FileContains(
textwrap.dedent("""\
#!/bin/sh
''''exec python3 -E -- "$0" "$@" # '''""")))
def _use_in_snap_python(self):
# Fix all shebangs to use the in-snap python.
mangling.rewrite_python_shebangs(self.installdir)
# Also replace all the /usr/bin/python calls in etc/catkin/profile.d/
# files with the in-snap python
profile_d_path = os.path.join(self.rosdir, "etc", "catkin",
"profile.d")
file_utils.replace_in_file(profile_d_path, re.compile(r""),
re.compile(r"/usr/bin/python"), r"python")
def _use_in_snap_python(self):
# Fix all shebangs to use the in-snap python.
mangling.rewrite_python_shebangs(self.installdir)
# Also replace all the /usr/bin/python calls in etc/catkin/profile.d/
# files with the in-snap python
profile_d_path = os.path.join(self.rosdir, "etc", "catkin", "profile.d")
file_utils.replace_in_file(
profile_d_path, re.compile(r""), re.compile(r"/usr/bin/python"), r"python"
)
def test_python_mixed_args(self):
file_path1 = _create_file('file1', '#!/usr/bin/python')
# Ensure extra spaces are chopped off
file_path2 = _create_file('file2', '#!/usr/bin/python3 -Es')
mangling.rewrite_python_shebangs(os.path.dirname(file_path1))
self.assertThat(file_path1, FileContains('#!/usr/bin/env python'))
self.assertThat(file_path2, FileContains(
textwrap.dedent("""\
#!/bin/sh
''''exec python3 -Es -- "$0" "$@" # '''""")))
def _use_in_snap_python(self):
# Fix all shebangs to use the in-snap python.
mangling.rewrite_python_shebangs(self.installdir)
# Also replace all the /usr/bin/python calls in etc/catkin/profile.d/
# files with the in-snap python
profile_d_path = os.path.join(self.rosdir, 'etc', 'catkin',
'profile.d')
file_utils.replace_in_file(profile_d_path, re.compile(r''),
re.compile(r'/usr/bin/python'), r'python')
def test_python(self):
file_path = _create_file('file', textwrap.dedent("""\
#! /usr/bin/python2.7
# Larger file
"""))
mangling.rewrite_python_shebangs(os.path.dirname(file_path))
self.assertThat(file_path, FileContains(textwrap.dedent("""\
#!/usr/bin/env python2.7
# Larger file
""")))
def _setup_tools_install(self, setup_file):
command = [
_python.get_python_command(self._python_major_version,
stage_dir=self.project.stage_dir,
install_dir=self.installdir),
os.path.basename(setup_file), '--no-user-cfg', 'install',
'--single-version-externally-managed', '--user', '--record',
'install.txt'
]
self.run(command, env=self._pip.env(), cwd=os.path.dirname(setup_file))
# Fix all shebangs to use the in-snap python. The stuff installed from
# pip has already been fixed, but anything done in this step has not.
mangling.rewrite_python_shebangs(self.installdir)
def _use_in_snap_python(self):
# Fix all shebangs to use the in-snap python.
mangling.rewrite_python_shebangs(self.installdir)
# Also replace the python usage in 10.ros.sh to use the in-snap python.
ros10_file = os.path.join(self.rosdir,
'etc/catkin/profile.d/10.ros.sh')
if os.path.isfile(ros10_file):
with open(ros10_file, 'r+') as f:
pattern = re.compile(r'/usr/bin/python')
replaced = pattern.sub(r'python', f.read())
f.seek(0)
f.truncate()
f.write(replaced)
def test_python_mixed_args(self):
file_path1 = _create_file("file1", "#!/usr/bin/python")
# Ensure extra spaces are chopped off
file_path2 = _create_file("file2", "#!/usr/bin/python3 -Es")
mangling.rewrite_python_shebangs(os.path.dirname(file_path1))
self.assertThat(file_path1, FileContains("#!/usr/bin/env python"))
self.assertThat(
file_path2,
FileContains(
textwrap.dedent(
"""\
#!/bin/sh
''''exec python3 -Es -- "$0" "$@" # '''"""
)
),
)
def _prepare_build(self):
# Fix all shebangs to use the in-snap python.
mangling.rewrite_python_shebangs(self.installdir)
# Rewrite the prefixes to point to the in-part rosdir instead of the system
self._fix_prefixes()
# Each Colcon package distributes .cmake files so they can be found via
# find_package(). However, the Ubuntu packages pulled down as
# dependencies contain .cmake files pointing to system paths (e.g.
# /usr/lib, /usr/include, etc.). They need to be rewritten to point to
# the install directory.
def _new_path(path):
if not path.startswith(self.installdir):
# Not using os.path.join here as `path` is absolute.
return self.installdir + path
return path
self._rewrite_cmake_paths(_new_path)
def _setup_tools_install(self, setup_file):
command = [
_python.get_python_command(
self._python_major_version,
stage_dir=self.project.stage_dir,
install_dir=self.installdir,
),
os.path.basename(setup_file),
"--no-user-cfg",
"install",
"--single-version-externally-managed",
"--user",
"--record",
"install.txt",
]
self.run(command, env=self._pip.env(), cwd=os.path.dirname(setup_file))
# Fix all shebangs to use the in-snap python. The stuff installed from
# pip has already been fixed, but anything done in this step has not.
mangling.rewrite_python_shebangs(self.installdir)
def build(self):
super().build()
env = os.environ.copy()
# Ensure the first provider does not attempt to validate against
# providers installed on the build host by initialising PROVIDERPATH
# to empty
env['PROVIDERPATH'] = ''
provider_stage_dir = os.path.join(self.project.stage_dir, 'providers')
if os.path.exists(provider_stage_dir):
provider_dirs = [os.path.join(provider_stage_dir, provider)
for provider in os.listdir(provider_stage_dir)]
env['PROVIDERPATH'] = ':'.join(provider_dirs)
self.run(['python3', 'manage.py', 'validate'], env=env)
self.run(['python3', 'manage.py', 'build'])
self.run(['python3', 'manage.py', 'i18n'])
self.run([
'python3', 'manage.py', 'install', '--layout=relocatable',
'--prefix=/providers/{}'.format(self.name),
'--root={}'.format(self.installdir)])
mangling.rewrite_python_shebangs(self.installdir)
def test_python(self):
file_path = _create_file(
"file",
textwrap.dedent(
"""\
#! /usr/bin/python2.7
# Larger file
"""
),
)
mangling.rewrite_python_shebangs(os.path.dirname(file_path))
self.assertThat(
file_path,
FileContains(
textwrap.dedent(
"""\
#!/usr/bin/env python2.7
# Larger file
"""
)
),
)
def _finish_build(self):
# Fix all shebangs to use the in-snap python.
mangling.rewrite_python_shebangs(self.installdir)
# We've finished the build, but we need to make sure we turn the cmake
# files back into something that doesn't include our installdir. This
# way it's usable from the staging area, and won't clash with the same
# file coming from other parts.
pattern = re.compile(r"^{}".format(self.installdir))
def _new_path(path):
return pattern.sub("$ENV{SNAPCRAFT_STAGE}", path)
self._rewrite_cmake_paths(_new_path)
# Rewrite prefixes for both the underlay and overlay.
self._fix_prefixes()
# If pip dependencies were installed, generate a sitecustomize that
# allows access to them.
if self._pip.is_setup() and self._pip.list(user=True):
_python.generate_sitecustomize("3",
stage_dir=self.project.stage_dir,
install_dir=self.installdir)
def _fix_shebangs(cls, unpackdir: str) -> None:
"""Change hard-coded shebangs in unpacked files to use env."""
mangling.rewrite_python_shebangs(unpackdir)
def install(
self,
packages,
*,
setup_py_dir=None,
constraints=None,
requirements=None,
process_dependency_links=False,
upgrade=False,
install_deps=True,
ignore_installed=False
):
"""Install packages from cache.
The packages should have already been downloaded via `download()`.
:param iterable packages: Packages to install from cache.
:param str setup_py_dir: Directory containing setup.py.
:param iterable constraints: Collection of paths to constraints files.
:param iterable requirements: Collection of paths to requirements
files.
:param boolean process_dependency_links: Enable the processing of
dependency links.
:param boolean upgrade: Recursively upgrade packages.
:param boolean install_deps: Install package dependencies.
:param boolean ignore_installed: Reinstall packages if they're already
installed
"""
args = _process_common_args(
process_dependency_links=process_dependency_links,
packages=packages,
constraints=constraints,
requirements=requirements,
)
if upgrade:
args.append("--upgrade")
if not install_deps:
args.append("--no-deps")
if ignore_installed:
args.append("--ignore-installed")
cwd = None
if setup_py_dir:
args.append(".")
cwd = setup_py_dir
if not args:
return # No operation was requested
# Using pip with a few special parameters:
#
# --user: Install packages to PYTHONUSERBASE, which we've pointed to
# the installdir.
# --no-compile: Don't compile .pyc files. FIXME: This is legacy, and
# should be removed once this refactor has been
# validated.
# --no-index: Don't hit pypi, assume the packages are already
# downloaded (i.e. by using `self.download()`)
# --find-links: Provide the directory into which the packages should
# have already been fetched
self._run(
[
"install",
"--user",
"--no-compile",
"--no-index",
"--find-links",
self._python_package_dir,
]
+ args,
cwd=cwd,
)
# Installing with --user results in a directory with 700 permissions.
# We need it a bit more open than that, so open it up.
_fix_permissions(self._install_dir)
# Fix all shebangs to use the in-snap python.
mangling.rewrite_python_shebangs(self._install_dir)
def test_python3(self):
file_path = _create_file("file", "#!/usr/bin/python3")
mangling.rewrite_python_shebangs(os.path.dirname(file_path))
self.assertThat(file_path, FileContains("#!/usr/bin/env python3"))
def _fix_shebangs(self, unpackdir):
"""Change hard-coded shebangs in unpacked files to use env."""
mangling.rewrite_python_shebangs(unpackdir)
def test_python3(self):
file_path = _create_file('file', '#!/usr/bin/python3')
mangling.rewrite_python_shebangs(os.path.dirname(file_path))
self.assertThat(file_path, FileContains('#!/usr/bin/env python3'))
def install(
self,
packages,
*,
setup_py_dir: Optional[str] = None,
constraints: Optional[Set[str]] = None,
requirements: Optional[Sequence[str]] = None,
process_dependency_links: bool = False,
upgrade: bool = False,
install_deps: bool = True,
ignore_installed: bool = False,
no_index: bool = True,
):
"""Install packages from cache.
The packages should have already been downloaded via `download()`.
:param iterable packages: Packages to install from cache.
:param str setup_py_dir: Directory containing setup.py.
:param iterable constraints: Collection of paths to constraints files.
:param iterable requirements: Collection of paths to requirements
files.
:param boolean process_dependency_links: Enable the processing of
dependency links.
:param boolean upgrade: Recursively upgrade packages.
:param boolean install_deps: Install package dependencies.
:param boolean ignore_installed: Reinstall packages if they're already
installed
:param boolean no_index: do not hit PyPI to find missing packages.
assume packages are already downloaded.
"""
package_args = _process_package_args(packages=packages,
requirements=requirements,
setup_py_dir=setup_py_dir)
if not package_args:
return # No operation was requested
args = _process_common_args(
process_dependency_links=process_dependency_links,
constraints=constraints)
if upgrade:
args.append("--upgrade")
if not install_deps:
args.append("--no-deps")
if ignore_installed:
args.append("--ignore-installed")
# --no-index: Don't hit pypi, assume the packages are already
# downloaded (i.e. by using `self.download()`)
if no_index:
args.append("--no-index")
# Using pip with a few special parameters:
#
# --user: Install packages to PYTHONUSERBASE, which we've pointed to
# the installdir.
# --no-compile: Don't compile .pyc files. FIXME: This is legacy, and
# should be removed once this refactor has been
# validated.
# --find-links: Provide the directory into which the packages should
# have already been fetched
#
# For cwd, setup_py_dir will be the actual directory we need to be in
# or None.
self._run(
[
"install",
"--user",
"--no-compile",
"--find-links",
self._python_package_dir,
] + args + package_args,
cwd=setup_py_dir,
)
# Installing with --user results in a directory with 700 permissions.
# We need it a bit more open than that, so open it up.
_fix_permissions(self._install_dir)
# Fix all shebangs to use the in-snap python.
mangling.rewrite_python_shebangs(self._install_dir)
def _fix_shebangs(self, unpackdir):
"""Change hard-coded shebangs in unpacked files to use env."""
mangling.rewrite_python_shebangs(unpackdir)
def install(self,
packages,
*,
setup_py_dir=None,
constraints=None,
requirements=None,
process_dependency_links=False,
upgrade=False,
install_deps=True,
ignore_installed=False):
"""Install packages from cache.
The packages should have already been downloaded via `download()`.
:param iterable packages: Packages to install from cache.
:param str setup_py_dir: Directory containing setup.py.
:param iterable constraints: Collection of paths to constraints files.
:param iterable requirements: Collection of paths to requirements
files.
:param boolean process_dependency_links: Enable the processing of
dependency links.
:param boolean upgrade: Recursively upgrade packages.
:param boolean install_deps: Install package dependencies.
:param boolean ignore_installed: Reinstall packages if they're already
installed
"""
args = _process_common_args(
process_dependency_links=process_dependency_links,
packages=packages,
constraints=constraints,
requirements=requirements)
if upgrade:
args.append('--upgrade')
if not install_deps:
args.append('--no-deps')
if ignore_installed:
args.append('--ignore-installed')
cwd = None
if setup_py_dir:
args.append('.')
cwd = setup_py_dir
if not args:
return # No operation was requested
# Using pip with a few special parameters:
#
# --user: Install packages to PYTHONUSERBASE, which we've pointed to
# the installdir.
# --no-compile: Don't compile .pyc files. FIXME: This is legacy, and
# should be removed once this refactor has been
# validated.
# --no-index: Don't hit pypi, assume the packages are already
# downloaded (i.e. by using `self.download()`)
# --find-links: Provide the directory into which the packages should
# have already been fetched
self._run([
'install', '--user', '--no-compile', '--no-index', '--find-links',
self._python_package_dir
] + args,
cwd=cwd)
# Installing with --user results in a directory with 700 permissions.
# We need it a bit more open than that, so open it up.
_fix_permissions(self._install_dir)
# Fix all shebangs to use the in-snap python.
mangling.rewrite_python_shebangs(self._install_dir)