def build(self): super().build() env = os.environ.copy() # Ensure the first provider does not attempt to validate against # providers installed on the build host by initialising PROVIDERPATH # to empty env['PROVIDERPATH'] = '' provider_stage_dir = os.path.join(self.project.stage_dir, 'providers') if os.path.exists(provider_stage_dir): provider_dirs = [os.path.join(provider_stage_dir, provider) for provider in os.listdir(provider_stage_dir)] env['PROVIDERPATH'] = ':'.join(provider_dirs) self.run(['python3', 'manage.py', 'validate'], env=env) self.run(['python3', 'manage.py', 'build']) self.run(['python3', 'manage.py', 'i18n']) self.run([ 'python3', 'manage.py', 'install', '--layout=relocatable', '--prefix=/providers/{}'.format(self.name), '--root={}'.format(self.installdir)]) # Fix all shebangs to use the in-snap python. file_utils.replace_in_file(self.installdir, re.compile(r''), re.compile(r'^#!.*python'), r'#!/usr/bin/env python')
def test_replace_in_file(self): os.makedirs('bin') # Place a few files with bad shebangs, and some files that shouldn't be # changed. files = [ { 'path': os.path.join('bin', '2to3'), 'contents': '#!/foo/bar/baz/python', 'expected': '#!/usr/bin/env python', }, { 'path': os.path.join('bin', 'snapcraft'), 'contents': '#!/foo/baz/python', 'expected': '#!/usr/bin/env python', }, { 'path': os.path.join('bin', 'foo'), 'contents': 'foo', 'expected': 'foo', } ] for file_info in files: with self.subTest(key=file_info['path']): with open(file_info['path'], 'w') as f: f.write(file_info['contents']) file_utils.replace_in_file('bin', re.compile(r''), re.compile(r'#!.*python'), r'#!/usr/bin/env python') with open(file_info['path'], 'r') as f: self.assertEqual(f.read(), file_info['expected'])
def _fix_shebangs(path): """Changes hard coded shebangs for files in _BIN_PATHS to use env.""" paths = [p for p in _BIN_PATHS if os.path.exists(os.path.join(path, p))] for p in [os.path.join(path, p) for p in paths]: file_utils.replace_in_file(p, re.compile(r''), re.compile(r'#!.*python\n'), r'#!/usr/bin/env python\n')
def build(self): super().build() setup_file = os.path.join(self.builddir, 'setup.py') with simple_env_bzr(os.path.join(self.installdir, 'bin')): installed_pipy_packages = self._run_pip(setup_file) # We record the requirements and constraints files only if they are # remote. If they are local, they are already tracked with the source. if self.options.requirements: self._manifest['requirements-contents'] = ( self._get_file_contents(self.options.requirements)) if self.options.constraints: self._manifest['constraints-contents'] = ( self._get_file_contents(self.options.constraints)) self._manifest['python-packages'] = [ '{}={}'.format(name, installed_pipy_packages[name]) for name in installed_pipy_packages ] self._fix_permissions() # Fix all shebangs to use the in-snap python. file_utils.replace_in_file(self.installdir, re.compile(r''), re.compile(r'^#!.*python'), r'#!/usr/bin/env python') self._setup_sitecustomize()
def test_replace_in_file(self): os.makedirs('bin') # Place a few files with bad shebangs, and some files that shouldn't be # changed. files = [{ 'path': os.path.join('bin', '2to3'), 'contents': '#!/foo/bar/baz/python', 'expected': '#!/usr/bin/env python', }, { 'path': os.path.join('bin', 'snapcraft'), 'contents': '#!/foo/baz/python', 'expected': '#!/usr/bin/env python', }, { 'path': os.path.join('bin', 'foo'), 'contents': 'foo', 'expected': 'foo', }] for file_info in files: with self.subTest(key=file_info['path']): with open(file_info['path'], 'w') as f: f.write(file_info['contents']) file_utils.replace_in_file('bin', re.compile(r''), re.compile(r'#!.*python'), r'#!/usr/bin/env python') with open(file_info['path'], 'r') as f: self.assertEqual(f.read(), file_info['expected'])
def rewrite_python_shebangs(root_dir): """Recursively change #!/usr/bin/pythonX shebangs to #!/usr/bin/env pythonX :param str root_dir: Directory that will be crawled for shebangs. """ file_pattern = re.compile(r"") argless_shebang_pattern = re.compile(r"\A#!.*(python\S*)$", re.MULTILINE) shebang_pattern_with_args = re.compile( r"\A#!.*(python\S*)[ \t\f\v]+(\S+)$", re.MULTILINE ) file_utils.replace_in_file( root_dir, file_pattern, argless_shebang_pattern, r"#!/usr/bin/env \1" ) # The above rewrite will barf if the shebang includes any args to python. # For example, if the shebang was `#!/usr/bin/python3 -Es`, just replacing # that with `#!/usr/bin/env python3 -Es` isn't going to work as `env` # doesn't support arguments like that. # # The solution is to replace the shebang with one pointing to /bin/sh, and # then exec the original shebang with included arguments. This requires # some quoting hacks to ensure the file can be interpreted by both sh as # well as python, but it's better than shipping our own `env`. file_utils.replace_in_file( root_dir, file_pattern, shebang_pattern_with_args, r"""#!/bin/sh\n''''exec \1 \2 -- "$0" "$@" # '''""", )
def build(self): super().build() setup_file = os.path.join(self.builddir, 'setup.py') with simple_env_bzr(os.path.join(self.installdir, 'bin')): installed_pipy_packages = self._run_pip(setup_file) # We record the requirements and constraints files only if they are # remote. If they are local, they are already tracked with the source. if self.options.requirements: self._manifest['requirements-contents'] = (self._get_file_contents( self.options.requirements)) if self.options.constraints: self._manifest['constraints-contents'] = (self._get_file_contents( self.options.constraints)) self._manifest['python-packages'] = [ '{}={}'.format(name, installed_pipy_packages[name]) for name in installed_pipy_packages ] self._fix_permissions() # Fix all shebangs to use the in-snap python. file_utils.replace_in_file(self.installdir, re.compile(r''), re.compile(r'^#!.*python'), r'#!/usr/bin/env python') self._setup_sitecustomize()
def rewrite_python_shebangs(root_dir): """Recursively change #!/usr/bin/pythonX shebangs to #!/usr/bin/env pythonX :param str root_dir: Directory that will be crawled for shebangs. """ file_pattern = re.compile(r'') argless_shebang_pattern = re.compile( r'\A#!.*(python\S*)$', re.MULTILINE) shebang_pattern_with_args = re.compile( r'\A#!.*(python\S*)[ \t\f\v]+(\S+)$', re.MULTILINE) file_utils.replace_in_file( root_dir, file_pattern, argless_shebang_pattern, r'#!/usr/bin/env \1') # The above rewrite will barf if the shebang includes any args to python. # For example, if the shebang was `#!/usr/bin/python3 -Es`, just replacing # that with `#!/usr/bin/env python3 -Es` isn't going to work as `env` # doesn't support arguments like that. # # The solution is to replace the shebang with one pointing to /bin/sh, and # then exec the original shebang with included arguments. This requires # some quoting hacks to ensure the file can be interpreted by both sh as # well as python, but it's better than shipping our own `env`. file_utils.replace_in_file( root_dir, file_pattern, shebang_pattern_with_args, r"""#!/bin/sh\n''''exec \1 \2 -- "$0" "$@" # '''""")
def _use_in_snap_python(self): # Fix all shebangs to use the in-snap python. mangling.rewrite_python_shebangs(self.installdir) # Also replace all the /usr/bin/python calls in etc/catkin/profile.d/ # files with the in-snap python profile_d_path = os.path.join(self.rosdir, "etc", "catkin", "profile.d") file_utils.replace_in_file(profile_d_path, re.compile(r""), re.compile(r"/usr/bin/python"), r"python")
def _use_in_snap_python(self): # Fix all shebangs to use the in-snap python. mangling.rewrite_python_shebangs(self.installdir) # Also replace all the /usr/bin/python calls in etc/catkin/profile.d/ # files with the in-snap python profile_d_path = os.path.join(self.rosdir, 'etc', 'catkin', 'profile.d') file_utils.replace_in_file(profile_d_path, re.compile(r''), re.compile(r'/usr/bin/python'), r'python')
def _use_in_snap_python(self): # Fix all shebangs to use the in-snap python. mangling.rewrite_python_shebangs(self.installdir) # Also replace all the /usr/bin/python calls in etc/catkin/profile.d/ # files with the in-snap python profile_d_path = os.path.join(self.rosdir, "etc", "catkin", "profile.d") file_utils.replace_in_file( profile_d_path, re.compile(r""), re.compile(r"/usr/bin/python"), r"python" )
def test_replace_in_file(self, tmp_work_path, file_path, contents, expected): (tmp_work_path / "bin").mkdir() with open(file_path, "w") as f: f.write(contents) file_utils.replace_in_file( "bin", re.compile(r""), re.compile(r"#!.*python"), r"#!/usr/bin/env python" ) with open(file_path, "r") as f: assert f.read() == expected
def build(self): super().build() setup_file = os.path.join(self.builddir, 'setup.py') self._run_pip(setup_file) self._fix_permissions() # Fix all shebangs to use the in-snap python. file_utils.replace_in_file(self.installdir, re.compile(r''), re.compile(r'#!.*python'), r'#!/usr/bin/env python')
def test_replace_in_file(self): os.makedirs('bin') with open(self.file_path, 'w') as f: f.write(self.contents) file_utils.replace_in_file('bin', re.compile(r''), re.compile(r'#!.*python'), r'#!/usr/bin/env python') with open(self.file_path, 'r') as f: self.assertEqual(f.read(), self.expected)
def test_replace_in_file(self): os.makedirs("bin") with open(self.file_path, "w") as f: f.write(self.contents) file_utils.replace_in_file("bin", re.compile(r""), re.compile(r"#!.*python"), r"#!/usr/bin/env python") with open(self.file_path, "r") as f: self.assertThat(f.read(), Equals(self.expected))
def test_replace_in_file(self): os.makedirs("bin") with open(self.file_path, "w") as f: f.write(self.contents) file_utils.replace_in_file( "bin", re.compile(r""), re.compile(r"#!.*python"), r"#!/usr/bin/env python" ) with open(self.file_path, "r") as f: self.assertThat(f.read(), Equals(self.expected))
def _rewrite_cmake_paths(self, new_path_callable): def _rewrite_paths(match): paths = match.group(1).strip().split(';') for i, path in enumerate(paths): # Offer the opportunity to rewrite this path if it's absolute. if os.path.isabs(path): paths[i] = new_path_callable(path) return '"' + ';'.join(paths) + '"' # Looking for any path-like string file_utils.replace_in_file(self.rosdir, re.compile(r'.*Config.cmake$'), re.compile(r'"(.*?/.*?)"'), _rewrite_paths)
def _ruby_install(self, builddir): self._ruby_tar.provision(builddir, clean_target=False, keep_tarball=True) self._run(["./configure", "--disable-install-rdoc", "--prefix=/"], cwd=builddir) self._run(["make", "-j{}".format(self.parallel_build_count)], cwd=builddir) self._run( ["make", "install", "DESTDIR={}".format(self.installdir)], cwd=builddir ) # Fix all shebangs to use the in-snap ruby file_utils.replace_in_file( self.installdir, re.compile(r""), re.compile(r"^#!.*ruby"), r"#!/usr/bin/env ruby", )
def _ruby_install(self, builddir): self._ruby_tar.provision(builddir, clean_target=False, keep_tarball=True) self._run(['./configure', '--disable-install-rdoc', '--prefix=/'], cwd=builddir) self._run(['make', '-j{}'.format(self.parallel_build_count)], cwd=builddir) self._run(['make', 'install', 'DESTDIR={}'.format(self.installdir)], cwd=builddir) # Fix all shebangs to use the in-snap ruby file_utils.replace_in_file(self.installdir, re.compile(r''), re.compile(r'^#!.*ruby'), r'#!/usr/bin/env ruby')
def _ruby_install(self, builddir): self._ruby_tar.provision( builddir, clean_target=False, keep_tarball=True) self._run(['./configure', '--disable-install-rdoc', '--prefix=/'], cwd=builddir) self._run(['make', '-j{}'.format(self.parallel_build_count)], cwd=builddir) self._run(['make', 'install', 'DESTDIR={}'.format(self.installdir)], cwd=builddir) # Fix all shebangs to use the in-snap ruby file_utils.replace_in_file( self.installdir, re.compile(r''), re.compile(r'^#!.*ruby'), r'#!/usr/bin/env ruby')
def build(self): super().build() self.run(['python3', 'manage.py', 'build']) self.run(['python3', 'manage.py', 'i18n']) self.run([ 'python3', 'manage.py', 'install', '--layout=relocatable', '--prefix=/providers/{}'.format(self.name), '--root={}'.format(self.installdir) ]) # Fix all shebangs to use the in-snap python. file_utils.replace_in_file(self.installdir, re.compile(r''), re.compile(r'^#!.*python'), r'#!/usr/bin/env python')
def _use_in_snap_python(self): # Fix all shebangs to use the in-snap python. file_utils.replace_in_file(self.rosdir, re.compile(r''), re.compile(r'^#!.*python'), r'#!/usr/bin/env python') # Also replace the python usage in 10.ros.sh to use the in-snap python. ros10_file = os.path.join(self.rosdir, 'etc/catkin/profile.d/10.ros.sh') if os.path.isfile(ros10_file): with open(ros10_file, 'r+') as f: pattern = re.compile(r'/usr/bin/python') replaced = pattern.sub(r'python', f.read()) f.seek(0) f.truncate() f.write(replaced)
def test_replace_in_file_with_permission_error(self): os.makedirs('bin') file_info = { 'path': os.path.join('bin', 'readonly'), 'contents': '#!/foo/bar/baz/python', 'expected': '#!/foo/bar/baz/python', } with open(file_info['path'], 'w') as f: f.write(file_info['contents']) # Use a mock here to force a PermissionError, even within a docker # container which always runs with elevated permissions with mock.patch('snapcraft.file_utils.open', side_effect=PermissionError('')): file_utils.replace_in_file('bin', re.compile(r''), re.compile(r'#!.*python'), r'#!/usr/bin/env python') with open(file_info['path'], 'r') as f: self.assertEqual(f.read(), file_info['expected'])
def build(self): super().build() env = os.environ.copy() provider_stage_dir = os.path.join(self.project.stage_dir, 'providers') if os.path.exists(provider_stage_dir): provider_dirs = [os.path.join(provider_stage_dir, provider) for provider in os.listdir(provider_stage_dir)] env['PROVIDERPATH'] = ':'.join(provider_dirs) self.run(['python3', 'manage.py', 'validate'], env=env) self.run(['python3', 'manage.py', 'build']) self.run(['python3', 'manage.py', 'i18n']) self.run([ 'python3', 'manage.py', 'install', '--layout=relocatable', '--prefix=/providers/{}'.format(self.name), '--root={}'.format(self.installdir)]) # Fix all shebangs to use the in-snap python. file_utils.replace_in_file(self.installdir, re.compile(r''), re.compile(r'^#!.*python'), r'#!/usr/bin/env python')
def test_replace_in_file_with_permission_error(self): os.makedirs("bin") file_info = { "path": os.path.join("bin", "readonly"), "contents": "#!/foo/bar/baz/python", "expected": "#!/foo/bar/baz/python", } with open(file_info["path"], "w") as f: f.write(file_info["contents"]) # Use a mock here to force a PermissionError, even within a docker # container which always runs with elevated permissions with mock.patch("snapcraft.file_utils.open", side_effect=PermissionError("")): file_utils.replace_in_file( "bin", re.compile(r""), re.compile(r"#!.*python"), r"#!/usr/bin/env python", ) with open(file_info["path"], "r") as f: self.assertThat(f.read(), Equals(file_info["expected"]))
def _prepare_build(self): self._use_in_snap_python() # Each Catkin package distributes .cmake files so they can be found via # find_package(). However, the Ubuntu packages pulled down as # dependencies contain .cmake files pointing to system paths (e.g. # /usr/lib, /usr/include, etc.). They need to be rewritten to point to # the install directory. def rewrite_paths(match): paths = match.group(1).strip().split(';') for i, path in enumerate(paths): # Rewrite this path if it's an absolute path and not already # within the install directory. if (os.path.isabs(path) and not path.startswith(self.installdir)): paths[i] = self.installdir + path return '"' + ';'.join(paths) + '"' # Looking for any path-like string file_utils.replace_in_file(self.rosdir, re.compile(r'.*Config.cmake$'), re.compile(r'"(.*?/.*?)"'), rewrite_paths)
def _rewrite_cmake_paths(self, new_path_callable): def _rewrite_paths(match): paths = match.group(1).strip().split(";") for i, path in enumerate(paths): # Offer the opportunity to rewrite this path if it's absolute. if os.path.isabs(path): paths[i] = new_path_callable(path) return '"' + ";".join(paths) + '"' # Looking for any path-like string file_utils.replace_in_file( self._ros_underlay, re.compile(r".*Config.cmake$"), re.compile(r'"(.*?/.*?)"'), _rewrite_paths, ) file_utils.replace_in_file( os.path.join(self.installdir, "usr", "lib", "cmake"), re.compile(r".*.cmake$"), re.compile(r'"(.*?/.*?)"'), _rewrite_paths, )
def _prepare_build(self): self._use_in_snap_python() # Each Catkin package distributes .cmake files so they can be found via # find_package(). However, the Ubuntu packages pulled down as # dependencies contain .cmake files pointing to system paths (e.g. # /usr/lib, /usr/include, etc.). They need to be rewritten to point to # the install directory. def _new_path(path): if not path.startswith(self.installdir): # Not using os.path.join here as `path` is absolute. return self.installdir + path return path self._rewrite_cmake_paths(_new_path) # Also rewrite any occurrence of $SNAPCRAFT_STAGE to be our install # directory (this may be the case if stage-snaps were used). file_utils.replace_in_file( self.rosdir, re.compile(r".*Config.cmake$"), re.compile(r"\$ENV{SNAPCRAFT_STAGE}"), self.installdir, )
def fix_shebangs(path): file_utils.replace_in_file(path, re.compile(r''), re.compile(r'^#!.*python'), r'#!/usr/bin/env python')
def _fix_prefixes(self): installdir_pattern = re.compile(r"^{}".format(self.installdir)) new_prefix = "$SNAP_COLCON_ROOT" def _rewrite_prefix(match): # Group 1 is the variable definition, group 2 is the path, which we may need # to modify. path = match.group(3).strip(" \n\t'\"") # Bail early if this isn't even a path, or if it's already been rewritten if os.path.sep not in path or new_prefix in path: return match.group() # If the path doesn't start with the installdir, then it needs to point to # the underlay given that the upstream ROS packages are expecting to be in # /opt/ros/. if not path.startswith(self.installdir): path = os.path.join(new_prefix, path.lstrip("/")) return match.expand('\\1\\2"{}"\\4'.format( installdir_pattern.sub(new_prefix, path))) # Set the AMENT_CURRENT_PREFIX throughout to the in-snap prefix file_utils.replace_in_file( self.installdir, re.compile(r""), re.compile(r"(\${)(AMENT_CURRENT_PREFIX:=)(.*)(})"), _rewrite_prefix, ) # Set the COLCON_CURRENT_PREFIX (if it's in the installdir) to the in-snap # prefix file_utils.replace_in_file( self.installdir, re.compile(r""), re.compile(r"()(COLCON_CURRENT_PREFIX=)(['\"].*{}.*)()".format( self.installdir)), _rewrite_prefix, ) # Set the _colcon_prefix_sh_COLCON_CURRENT_PREFIX throughout to the in-snap # prefix file_utils.replace_in_file( self.installdir, re.compile(r""), re.compile(r"()(_colcon_prefix_sh_COLCON_CURRENT_PREFIX=)(.*)()"), _rewrite_prefix, ) # Set the _colcon_package_sh_COLCON_CURRENT_PREFIX throughout to the in-snap # prefix file_utils.replace_in_file( self.installdir, re.compile(r""), re.compile(r"()(_colcon_package_sh_COLCON_CURRENT_PREFIX=)(.*)()"), _rewrite_prefix, ) # Set the _colcon_prefix_chain_sh_COLCON_CURRENT_PREFIX throughout to the in-snap # prefix file_utils.replace_in_file( self.installdir, re.compile(r""), re.compile( r"()(_colcon_prefix_chain_sh_COLCON_CURRENT_PREFIX=)(.*)()"), _rewrite_prefix, ) # Set the _colcon_python_executable throughout to use the in-snap python file_utils.replace_in_file( self.installdir, re.compile(r""), re.compile(r"()(_colcon_python_executable=)(.*)()"), _rewrite_prefix, )