def checkout_package(self, package_name): """Fetches and checks out the correct version of a package for the rosdistro""" pkg_info = self._rosdist.get_package_checkout_info()[package_name] repo_url = pkg_info['url'] name = os.path.basename(pkg_info['url']) repo_path = os.path.join(self.workspace, name) client = GitClient(repo_path) if client.path_exists(): tag = pkg_info['version'] if client.is_tag( pkg_info['version']) else pkg_info['full_version'] if client.get_url() == repo_url: # Sometimes debian/rules gets mussed with in the build process. # This should translate to 'git checkout debian/rules' which should reset so we can change tags client.update('debian/rules', verbose=True) updated = client.update(tag, force_fetch=True, verbose=True) if not updated: print( "WARNING: Repo at %s changed url from %s to %s or update failed. Redownloading!" % (repo_path, client.get_url(), repo_url)) shutil.rmtree(repo_path) self.fetch_with_tagcheck(client, pkg_info) else: self.fetch_with_tagcheck(client, pkg_info) return repo_path
def test_switch_branches(self): url = self.remote_path client = GitClient(self.local_path) subclient = GitClient(self.sublocal_path) subclient2 = GitClient(self.sublocal2_path) subsubclient = GitClient(self.subsublocal_path) subsubclient2 = GitClient(self.subsublocal2_path) self.assertFalse(client.path_exists()) self.assertFalse(client.detect_presence()) self.assertTrue(client.checkout(url)) self.assertTrue(client.path_exists()) self.assertTrue(subclient.path_exists()) self.assertTrue(subsubclient.path_exists()) self.assertFalse(subclient2.path_exists()) new_version = "test_branch" self.assertTrue(client.update(new_version)) # checking that update doesnt make submodule disappear (git default behavior) self.assertTrue(subclient2.path_exists()) self.assertTrue(subsubclient2.path_exists()) self.assertTrue(subclient.path_exists()) self.assertTrue(subsubclient.path_exists()) oldnew_version = "master" self.assertTrue(client.update(oldnew_version)) # checking that update doesnt make submodule2 disappear (git default behavior) self.assertTrue(subclient2.path_exists()) self.assertTrue(subsubclient2.path_exists()) self.assertTrue(subclient.path_exists()) self.assertTrue(subsubclient.path_exists())
def build_debian_package(package_fetcher, package_name, apt_cache, rd_obj, levels=0, get_dependencies=False): unstable_target_distros = { 'groovy': 'quantal', 'hydro': 'raring' } target_ubuntu_distro = unstable_target_distros[rd_obj._rosdistro] level_prefix = '--' * levels print("%s> Building package %s" % (level_prefix, package_name)) deb_package_name = rd_obj.debianize_package_name(package_name) deb_package_version = rd_obj.get_version(package_name, full_version=True) + target_ubuntu_distro print("%s--> Checking if installed (%s, %s).." % (level_prefix, deb_package_name, deb_package_version)), if deb_package_name in apt_cache: installed = apt_cache[deb_package_name].installed if installed is not None and installed.version == deb_package_version: print("OK") print("%s is installed already - remove the package if you want to re-install." % (package_name)) return True print("missing!") if get_dependencies: dependencies = package_build_order([package_name], distro_name=rd_obj._rosdistro) print("%s--> Checking Dependencies:" % (level_prefix)) for dep_pkg_name in dependencies: if dep_pkg_name != package_name: print("%s---- %s....." % (level_prefix, dep_pkg_name)), debian_pkg_name = rd_obj.debianize_package_name(dep_pkg_name) if debian_pkg_name in apt_cache and apt_cache[debian_pkg_name].installed is not None: print(" OK! (installed version %s)" % apt_cache[debian_pkg_name].installed.version) else: print(" Needs build, building...") build_debian_package(package_fetcher, dep_pkg_name, apt_cache, rd_obj, levels + 1) print("%s<<-- Dependencies OKAY." % (level_prefix)) print("%s>>> Build debian package %s from repo %s" % (level_prefix, deb_package_name, package_fetcher.url(package_name))) repo_path = package_fetcher.checkout_package(package_name) client = GitClient(repo_path) deb_package_tag = deb_package_name + '_' + rd_obj.get_version(package_name, full_version=True) + '_' + target_ubuntu_distro bloom_package_version = 'debian/' + deb_package_tag client.update(bloom_package_version) installed_builddeps = install_debian_build_dependencies(repo_path) if not installed_builddeps: raise RosGitBuildError("%s!!! Error building %s from %s: Can't install build-dependencies!" % (level_prefix, deb_package_name, package_fetcher.url(package_name))) (returncode, result, message) = run_shell_command('debuild clean', repo_path, shell=True, show_stdout=False) if returncode != 0: raise RosGitBuildError("%s!!! Error building %s from %s: %s \n %s" % (level_prefix, deb_package_name, package_fetcher.url(package_name), 'debuild clean', message)) (returncode, result, message) = run_shell_command('debuild binary', repo_path, shell=True, show_stdout=False) if returncode != 0: raise RosGitBuildError("%s!!! Error building %s from %s: %s \n %s" % (level_prefix, deb_package_name, package_fetcher.url(package_name), 'debuild binary', message)) deb_files = glob.glob(os.path.join(repo_path, '..', '%s*.deb' % (deb_package_name + '_' + rd_obj.get_version(package_name, full_version=True)))) if len(deb_files) > 0: # install the deb from apt.debfile import DebPackage deb_pkg = DebPackage(deb_files[0]) deb_pkg.check() packages_needed = ' '.join(deb_pkg.missing_deps) (returncode, result, message) = run_shell_command('sudo apt-get -y install %s' % packages_needed, shell=True, show_stdout=True) if returncode != 0: raise RosGitBuildError("%s!!! Error building %s: can't install dependent packages %s" % (level_prefix, deb_package_name, packages_needed)) (returncode, result, message) = run_shell_command('sudo dpkg -i %s' % deb_files[0], shell=True, show_stdout=True) if returncode != 0: raise RosGitBuildError("%s!!! Error building %s from %s: %s \n %s" % (level_prefix, deb_package_name, package_fetcher.url(package_name), 'debuild binary', message)) else: raise RosGitBuildError("%s!!! Can't find a built debian package for %s after the build!" % (level_prefix, deb_package_name))
def test_switch_branches_retrieve_local_subcommit(self): url = self.remote_path client = GitClient(self.local_path) subclient = GitClient(self.sublocal_path) subclient2 = GitClient(self.sublocal2_path) subsubclient = GitClient(self.subsublocal_path) subsubclient2 = GitClient(self.subsublocal2_path) self.assertFalse(client.path_exists()) self.assertFalse(client.detect_presence()) self.assertTrue(client.checkout(url)) self.assertTrue(client.path_exists()) self.assertTrue(subclient.path_exists()) self.assertTrue(subsubclient.path_exists()) self.assertFalse(subclient2.path_exists()) new_version = "test_branch" self.assertTrue(client.update(new_version)) # checking that update doesnt make submodule disappear (git default behavior) self.assertTrue(subclient2.path_exists()) self.assertTrue(subsubclient2.path_exists()) self.assertTrue(subclient.path_exists()) self.assertTrue(subsubclient.path_exists()) subprocess.check_call("touch submodif.txt", shell=True, cwd=self.sublocal2_path) subprocess.check_call("git add submodif.txt", shell=True, cwd=self.sublocal2_path) subprocess.check_call("git commit -m submodif", shell=True, cwd=self.sublocal2_path) subprocess.check_call("git add submodule2", shell=True, cwd=self.local_path) subprocess.check_call("git commit -m submodule2_modif", shell=True, cwd=self.local_path) oldnew_version = "master" self.assertTrue(client.update(oldnew_version)) # checking that update doesnt make submodule2 disappear (git default behavior) self.assertTrue(subclient2.path_exists()) self.assertTrue(subsubclient2.path_exists()) self.assertTrue(subclient.path_exists()) self.assertTrue(subsubclient.path_exists()) self.assertTrue(client.update(new_version)) # checking that update still has submodule with submodif self.assertTrue(subclient2.path_exists()) self.assertTrue(subsubclient2.path_exists()) self.assertTrue(subclient.path_exists()) self.assertTrue(subsubclient.path_exists()) self.assertTrue( os.path.exists(os.path.join(self.sublocal2_path, "submodif.txt")))
def _get_file(self, _repo_type, repo_url, version, filename): """ Fetch the file specificed by filename relative to the root of the repository""" name = simplify_repo_name(repo_url) repo_path = os.path.join(self._cache_location, name) client = GitClient(repo_path) # using git only updated = False if client.path_exists(): if client.get_url() == repo_url: if not self._skip_update: logging.disable(logging.WARNING) updated = client.update(version, force_fetch=True) logging.disable(logging.NOTSET) else: try: # catch exception which can be caused by calling internal API logging.disable(logging.WARNING) updated = client._do_update(version) logging.disable(logging.NOTSET) except GitError: updated = False if not updated: shutil.rmtree(repo_path) if not updated: logging.disable(logging.WARNING) updated = client.checkout(repo_url, version) logging.disable(logging.NOTSET) if not updated: raise VcsError("Impossible to update/checkout repo '%s' with version '%s'." % (repo_url, version)) full_filename = os.path.join(repo_path, filename) if not os.path.exists(full_filename): raise VcsError("Requested file '%s' missing from repo '%s' version '%s' (viewed at version '%s'). It was expected at: %s" % (filename, repo_url, version, client.get_version(), full_filename)) return full_filename
def _get_file(self, repo_type, repo_url, version, filename): """ Fetch the file specificed by filename relative to the root of the repository""" name = simplify_repo_name(repo_url) repo_path = os.path.join(self._cache_location, name) #client = VcsClient(repo_type, repo_path) client = GitClient(repo_path) # using git only updated = False if client.path_exists(): if client.get_url() == repo_url: if not self._skip_update: updated = client.update(version, force_fetch=True) else: updated = client._do_update(version) if not updated: shutil.rmtree(repo_path) if not updated: updated = client.checkout(repo_url, version, shallow=True) if not updated: raise VcsError("Impossible to update/checkout repo '%s' with version '%s'." % (repo_url, version)) full_filename = os.path.join(repo_path, filename) if not os.path.exists(full_filename): raise VcsError("Requested file '%s' missing from repo '%s' version '%s' (viewed at version '%s'). It was expected at: %s" % (filename, repo_url, version, client.get_version(), full_filename)) return full_filename
def test_export_hash(self): url = self.remote_path client = GitClient(self.local_path) subclient = GitClient(self.sublocal_path) subclient2 = GitClient(self.sublocal2_path) subsubclient = GitClient(self.subsublocal_path) subsubclient2 = GitClient(self.subsublocal2_path) self.assertFalse(client.path_exists()) self.assertFalse(client.detect_presence()) self.assertFalse(os.path.exists(self.export_path)) self.assertTrue(client.checkout(url, version='master')) self.assertTrue(client.path_exists()) self.assertTrue(subclient.path_exists()) self.assertTrue(subsubclient.path_exists()) self.assertFalse(subclient2.path_exists()) self.assertFalse(subsubclient2.path_exists()) # we need first to retrieve locally the hash we want to export self.assertTrue(client.update(version=self.version_test)) self.assertTrue(client.path_exists()) # git leaves old submodule around by default self.assertTrue(subclient.path_exists()) self.assertTrue(subsubclient.path_exists()) # new submodule should be there self.assertTrue(subclient2.path_exists()) self.assertTrue(subsubclient2.path_exists()) tarpath = client.export_repository(self.version_test, self.export_path) self.assertEqual(tarpath, self.export_path + '.tar.gz') os.mkdir(self.export_path) with closing(tarfile.open(tarpath, "r:gz")) as tarf: tarf.extractall(self.export_path) # Checking that we have only submodule2 in our export self.assertFalse(os.path.exists(self.subexport_path)) self.assertFalse(os.path.exists(self.subsubexport_path)) self.assertTrue(os.path.exists(self.subexport2_path)) self.assertTrue(os.path.exists(self.subsubexport2_path)) # comparing with version_test ( currently checked-out ) subsubdirdiff = filecmp.dircmp(self.subsubexport2_path, self.subsublocal_path, ignore=['.git', '.gitmodules']) self.assertEqual(subsubdirdiff.left_only, []) # same subsubfixed.txt in both subsubmodule/ self.assertEqual(subsubdirdiff.right_only, []) self.assertEqual(subsubdirdiff.diff_files, []) subdirdiff = filecmp.dircmp(self.subexport2_path, self.sublocal_path, ignore=['.git', '.gitmodules']) self.assertEqual(subdirdiff.left_only, []) self.assertEqual(subdirdiff.right_only, []) self.assertEqual(subdirdiff.diff_files, []) dirdiff = filecmp.dircmp(self.export_path, self.local_path, ignore=['.git', '.gitmodules']) self.assertEqual(dirdiff.left_only, []) # submodule is still there on local_path (git default behavior) self.assertEqual(dirdiff.right_only, ['submodule']) self.assertEqual(dirdiff.diff_files, [])
def test_switch_branches_retrieve_local_subcommit(self): url = self.remote_path client = GitClient(self.local_path) subclient = GitClient(self.sublocal_path) subclient2 = GitClient(self.sublocal2_path) subsubclient = GitClient(self.subsublocal_path) subsubclient2 = GitClient(self.subsublocal2_path) self.assertFalse(client.path_exists()) self.assertFalse(client.detect_presence()) self.assertTrue(client.checkout(url)) self.assertTrue(client.path_exists()) self.assertTrue(subclient.path_exists()) self.assertTrue(subsubclient.path_exists()) self.assertFalse(subclient2.path_exists()) new_version = "test_branch" self.assertTrue(client.update(new_version)) # checking that update doesnt make submodule disappear (git default behavior) self.assertTrue(subclient2.path_exists()) self.assertTrue(subsubclient2.path_exists()) self.assertTrue(subclient.path_exists()) self.assertTrue(subsubclient.path_exists()) subprocess.check_call("touch submodif.txt", shell=True, cwd=self.sublocal2_path) subprocess.check_call("git add submodif.txt", shell=True, cwd=self.sublocal2_path) subprocess.check_call("git commit -m submodif", shell=True, cwd=self.sublocal2_path) subprocess.check_call("git add submodule2", shell=True, cwd=self.local_path) subprocess.check_call("git commit -m submodule2_modif", shell=True, cwd=self.local_path) oldnew_version = "master" self.assertTrue(client.update(oldnew_version)) # checking that update doesnt make submodule2 disappear (git default behavior) self.assertTrue(subclient2.path_exists()) self.assertTrue(subsubclient2.path_exists()) self.assertTrue(subclient.path_exists()) self.assertTrue(subsubclient.path_exists()) self.assertTrue(client.update(new_version)) # checking that update still has submodule with submodif self.assertTrue(subclient2.path_exists()) self.assertTrue(subsubclient2.path_exists()) self.assertTrue(subclient.path_exists()) self.assertTrue(subsubclient.path_exists()) self.assertTrue(os.path.exists(os.path.join(self.sublocal2_path, "submodif.txt")))
def checkout_package(self, package_name): """Fetches and checks out the correct version of a package for the rosdistro""" pkg_info = self._rosdist.get_package_checkout_info()[package_name] repo_url = pkg_info['url'] name = os.path.basename(pkg_info['url']) repo_path = os.path.join(self.workspace, name) client = GitClient(repo_path) if client.path_exists(): tag = pkg_info['version'] if client.is_tag(pkg_info['version']) else pkg_info['full_version'] if client.get_url() == repo_url: # Sometimes debian/rules gets mussed with in the build process. # This should translate to 'git checkout debian/rules' which should reset so we can change tags client.update('debian/rules', verbose=True) updated = client.update(tag, force_fetch=True, verbose=True) if not updated: print("WARNING: Repo at %s changed url from %s to %s or update failed. Redownloading!" % (repo_path, client.get_url(), repo_url)) shutil.rmtree(repo_path) self.fetch_with_tagcheck(client, pkg_info) else: self.fetch_with_tagcheck(client, pkg_info) return repo_path
def test_switch_branches(self): url = self.remote_path client = GitClient(self.local_path) subclient = GitClient(self.sublocal_path) subclient2 = GitClient(self.sublocal2_path) subsubclient = GitClient(self.subsublocal_path) self.assertFalse(client.path_exists()) self.assertFalse(client.detect_presence()) self.assertTrue(client.checkout(url)) self.assertTrue(client.path_exists()) self.assertTrue(subclient.path_exists()) self.assertTrue(subsubclient.path_exists()) self.assertFalse(subclient2.path_exists()) new_version = "test_branch2" self.assertTrue(client.update(new_version)) self.assertTrue(subclient2.path_exists())
def _get_file(self, _repo_type, repo_url, version, filename): """ Fetch the file specificed by filename relative to the root of the repository""" name = simplify_repo_name(repo_url) repo_path = os.path.join(self._cache_location, name) client = GitClient(repo_path) # using git only updated = False if client.path_exists(): if client.get_url() == repo_url: if not self._skip_update: logging.disable(logging.WARNING) updated = client.update(version, force_fetch=True) logging.disable(logging.NOTSET) else: try: # catch exception which can be caused by calling internal API logging.disable(logging.WARNING) updated = client._do_update(version) logging.disable(logging.NOTSET) except GitError: updated = False if not updated: shutil.rmtree(repo_path) if not updated: logging.disable(logging.WARNING) updated = client.checkout(repo_url, version) logging.disable(logging.NOTSET) if not updated: raise VcsError( "Impossible to update/checkout repo '%s' with version '%s'." % (repo_url, version)) full_filename = os.path.join(repo_path, filename) if not os.path.exists(full_filename): raise VcsError( "Requested file '%s' missing from repo '%s' version '%s' (viewed at version '%s'). It was expected at: %s" % (filename, repo_url, version, client.get_version(), full_filename)) return full_filename
def test_checkout_specific_branch_and_update(self): from vcstools.git import GitClient directory = tempfile.mkdtemp() subdir = "checkout_specific_version_test" self.directories[subdir] = directory local_path = os.path.join(directory, "ros") url = self.readonly_url branch = "master" client = GitClient(local_path) self.assertFalse(client.path_exists()) self.assertFalse(client.detect_presence()) self.assertTrue(client.checkout(url, branch)) self.assertTrue(client.path_exists()) self.assertTrue(client.detect_presence()) self.assertEqual(client.get_path(), local_path) self.assertEqual(client.get_url(), url) self.assertEqual(client.get_branch_parent(), branch) new_branch = 'master' self.assertTrue(client.update(new_branch)) self.assertEqual(client.get_branch_parent(), new_branch) shutil.rmtree(directory) self.directories.pop(subdir)
def build_debian_package(package_fetcher, package_name, apt_cache, rd_obj, levels=0, get_dependencies=False): unstable_target_distros = {'groovy': 'quantal', 'hydro': 'raring'} target_ubuntu_distro = unstable_target_distros[rd_obj._rosdistro] level_prefix = '--' * levels print("%s> Building package %s" % (level_prefix, package_name)) deb_package_name = rd_obj.debianize_package_name(package_name) deb_package_version = rd_obj.get_version( package_name, full_version=True) + target_ubuntu_distro print("%s--> Checking if installed (%s, %s).." % (level_prefix, deb_package_name, deb_package_version)), if deb_package_name in apt_cache: installed = apt_cache[deb_package_name].installed if installed is not None and installed.version == deb_package_version: print("OK") print( "%s is installed already - remove the package if you want to re-install." % (package_name)) return True print("missing!") if get_dependencies: dependencies = package_build_order([package_name], distro_name=rd_obj._rosdistro) print("%s--> Checking Dependencies:" % (level_prefix)) for dep_pkg_name in dependencies: if dep_pkg_name != package_name: print("%s---- %s....." % (level_prefix, dep_pkg_name)), debian_pkg_name = rd_obj.debianize_package_name(dep_pkg_name) if debian_pkg_name in apt_cache and apt_cache[ debian_pkg_name].installed is not None: print(" OK! (installed version %s)" % apt_cache[debian_pkg_name].installed.version) else: print(" Needs build, building...") build_debian_package(package_fetcher, dep_pkg_name, apt_cache, rd_obj, levels + 1) print("%s<<-- Dependencies OKAY." % (level_prefix)) print("%s>>> Build debian package %s from repo %s" % (level_prefix, deb_package_name, package_fetcher.url(package_name))) repo_path = package_fetcher.checkout_package(package_name) client = GitClient(repo_path) deb_package_tag = deb_package_name + '_' + rd_obj.get_version( package_name, full_version=True) + '_' + target_ubuntu_distro bloom_package_version = 'debian/' + deb_package_tag client.update(bloom_package_version) installed_builddeps = install_debian_build_dependencies(repo_path) if not installed_builddeps: raise RosGitBuildError( "%s!!! Error building %s from %s: Can't install build-dependencies!" % (level_prefix, deb_package_name, package_fetcher.url(package_name))) (returncode, result, message) = run_shell_command('debuild clean', repo_path, shell=True, show_stdout=False) if returncode != 0: raise RosGitBuildError( "%s!!! Error building %s from %s: %s \n %s" % (level_prefix, deb_package_name, package_fetcher.url(package_name), 'debuild clean', message)) (returncode, result, message) = run_shell_command('debuild binary', repo_path, shell=True, show_stdout=False) if returncode != 0: raise RosGitBuildError( "%s!!! Error building %s from %s: %s \n %s" % (level_prefix, deb_package_name, package_fetcher.url(package_name), 'debuild binary', message)) deb_files = glob.glob( os.path.join( repo_path, '..', '%s*.deb' % (deb_package_name + '_' + rd_obj.get_version(package_name, full_version=True)))) if len(deb_files) > 0: # install the deb from apt.debfile import DebPackage deb_pkg = DebPackage(deb_files[0]) deb_pkg.check() packages_needed = ' '.join(deb_pkg.missing_deps) (returncode, result, message) = run_shell_command( 'sudo apt-get -y install %s' % packages_needed, shell=True, show_stdout=True) if returncode != 0: raise RosGitBuildError( "%s!!! Error building %s: can't install dependent packages %s" % (level_prefix, deb_package_name, packages_needed)) (returncode, result, message) = run_shell_command('sudo dpkg -i %s' % deb_files[0], shell=True, show_stdout=True) if returncode != 0: raise RosGitBuildError( "%s!!! Error building %s from %s: %s \n %s" % (level_prefix, deb_package_name, package_fetcher.url(package_name), 'debuild binary', message)) else: raise RosGitBuildError( "%s!!! Can't find a built debian package for %s after the build!" % (level_prefix, deb_package_name))