def prepare_upstream_tarball(repo, cp, options, tarball_dir, output_dir): """ Make sure we have an upstream tarball. This involves loooking in tarball_dir, symlinking or building it. """ options.comp_type = guess_comp_type(repo, options.comp_type, cp, options.tarball_dir) orig_files = [du.orig_file(cp, options.comp_type)] if options.subtarballs: orig_files += [du.orig_file(cp, options.comp_type, sub) for sub in options.subtarballs] # look in tarball_dir first, if found force a symlink to it if options.tarball_dir: gbp.log.debug("Looking for orig tarballs '%s' at '%s'" % (", ".join(orig_files), tarball_dir)) missing = du.DebianPkgPolicy.symlink_origs(orig_files, tarball_dir, output_dir, force=True) if missing: msg = "Tarballs '%s' not found at '%s'" % (", ".join(missing), tarball_dir) else: msg = "All Orig tarballs '%s' found at '%s'" % (", ".join(orig_files), tarball_dir) gbp.log.info(msg) if options.no_create_orig: return # Create tarball if missing or forced if not du.DebianPkgPolicy.has_origs(orig_files, output_dir) or options.force_create: if not pristine_tar_build_orig(repo, cp, output_dir, options): upstream_tree = git_archive_build_orig(repo, cp, output_dir, options) if options.pristine_tar_commit: pristine_tar_commit(repo, cp, options, output_dir, orig_files[0], upstream_tree)
def test_has_orig_multiple_true(self): for ext in ['', '-foo', '-bar']: open(self.tmpdir.join('source_1.2.orig%s.tar.gz' % ext), "w").close() orig_files = [orig_file(self.source, 'gzip')] + \ [orig_file(self.source, 'gzip', sub) for sub in ['foo', 'bar']] self.assertTrue(DebianPkgPolicy.has_origs(orig_files, str(self.tmpdir)))
def prepare_upstream_tarball(repo, cp, options, tarball_dir, output_dir): """ Make sure we have an upstream tarball. This involves loooking in tarball_dir, symlinking or building it. """ options.comp_type = guess_comp_type(repo, options.comp_type, cp, options.tarball_dir) orig_file = du.orig_file(cp, options.comp_type) # look in tarball_dir first, if found force a symlink to it if options.tarball_dir: gbp.log.debug("Looking for orig tarball '%s' at '%s'" % (orig_file, tarball_dir)) if not du.DebianPkgPolicy.symlink_orig(orig_file, tarball_dir, output_dir, force=True): gbp.log.info("Orig tarball '%s' not found at '%s'" % (orig_file, tarball_dir)) else: gbp.log.info("Orig tarball '%s' found at '%s'" % (orig_file, tarball_dir)) # build an orig unless the user forbids it, always build (and overwrite pre-existing) if user forces it if options.force_create or (not options.no_create_orig and not du.DebianPkgPolicy.has_orig(orig_file, output_dir)): if not pristine_tar_build_orig(repo, cp, output_dir, options): upstream_tree = git_archive_build_orig(repo, cp, output_dir, options) if options.pristine_tar_commit: if repo.pristine_tar.has_commit(cp.name, cp.upstream_version, options.comp_type): gbp.log.debug("%s already on pristine tar branch" % orig_file) else: archive = os.path.join(output_dir, orig_file) gbp.log.debug("Adding %s to pristine-tar branch" % archive) repo.pristine_tar.commit(archive, upstream_tree)
def git_archive_build_orig(repo, cp, output_dir, options): """ Build orig tarball using git-archive @param cp: the changelog of the package we're acting on @type cp: L{ChangeLog} @param output_dir: where to put the tarball @type output_dir: C{Str} @param options: the parsed options @type options: C{dict} of options @return: the tree we built the tarball from @rtype: C{str} """ upstream_tree = get_upstream_tree(repo, cp, options) gbp.log.info("%s does not exist, creating from '%s'" % (du.orig_file(cp, options.comp_type), upstream_tree)) gbp.log.debug("Building upstream tarball with compression '%s -%s'" % (options.comp_type, options.comp_level)) if not git_archive(repo, cp, output_dir, upstream_tree, options.comp_type, options.comp_level, options.with_submodules): raise GbpError("Cannot create upstream tarball at '%s'" % output_dir) return upstream_tree
def pristine_tar_build_orig(repo, cp, output_dir, options): """ build orig using pristine-tar @return: True: orig tarball build, False: noop """ if options.pristine_tar: if not repo.has_branch(repo.pristine_tar_branch): gbp.log.warn('Pristine-tar branch "%s" not found' % repo.pristine_tar.branch) try: repo.pristine_tar.checkout(cp.name, cp.upstream_version, options.comp_type, output_dir) pristine_tree = repo.list_tree(repo.pristine_tar.branch, recurse=True) pristine_files = [ os.path.splitext(item[3])[0] for item in pristine_tree if item[3].endswith('.id') ] for component in du.orig_components(du.orig_file(cp, options.comp_type), pristine_files).iterkeys(): repo.pristine_tar.checkout(cp.name, cp.upstream_version, options.comp_type, output_dir, component) return True except CommandExecFailed: if options.pristine_tar_commit: gbp.log.debug("pristine-tar checkout failed, " "will commit tarball due to " "'--pristine-tar-commit'") else: raise return False
def git_archive(repo, cp, output_dir, treeish, comp_type, comp_level, with_submodules): "create a compressed orig tarball in output_dir using git_archive" try: comp_opts = compressor_opts[comp_type][0] except KeyError: raise GbpError("Unsupported compression type '%s'" % comp_type) output = os.path.join(output_dir, du.orig_file(cp, comp_type)) prefix = "%s-%s" % (cp['Source'], cp['Upstream-Version']) try: if repo.has_submodules() and with_submodules: repo.update_submodules() git_archive_submodules(repo, treeish, output, prefix, comp_type, comp_level, comp_opts) else: git_archive_single(treeish, output, prefix, comp_type, comp_level, comp_opts) except (GitRepositoryError, CommandExecFailed): gbp.log.err("Error generating submodules' archives") return False except OSError as err: gbp.log.err("Error creating %s: %s" % (output, err[0])) return False except GbpError: raise except Exception as e: gbp.log.err("Error creating %s: %s" % (output, e)) return False return True
def prepare_upstream_tarball(repo, source, options, tarball_dir, output_dir): """ Make sure we have an upstream tarball. This involves loooking in tarball_dir, symlinking or building it. """ if not source.is_native() and not source.upstream_version: raise GbpError("Non-native package '%s' " "has invalid version '%s'" % (source.name, source.version)) options.comp_type = guess_comp_type(repo, options.comp_type, source, options.tarball_dir) orig_files = [du.orig_file(source, options.comp_type)] if options.components: orig_files += [ du.orig_file(source, options.comp_type, c) for c in options.components ] # look in tarball_dir first, if found force a symlink to it if options.tarball_dir: gbp.log.debug("Looking for orig tarballs '%s' at '%s'" % (", ".join(orig_files), tarball_dir)) missing = du.DebianPkgPolicy.symlink_origs(orig_files, tarball_dir, output_dir, force=True) if missing: msg = "Tarballs '%s' not found at '%s'" % (", ".join(missing), tarball_dir) else: msg = "All Orig tarballs '%s' found at '%s'" % ( ", ".join(orig_files), tarball_dir) gbp.log.info(msg) if options.no_create_orig: return # Create tarball if missing or forced if not du.DebianPkgPolicy.has_origs(orig_files, output_dir) or options.force_create: if not pristine_tar_build_orig(repo, source, output_dir, options): upstream_tree = git_archive_build_orig(repo, source, output_dir, options) if options.pristine_tar_commit: pristine_tar_commit(repo, source, options, output_dir, orig_files[0], upstream_tree) pristine_tar_verify_orig(repo, source, options, output_dir, orig_files)
def git_archive_build_orig(repo, source, output_dir, options): """ Build orig tarball(s) using git-archive @param source: the changelog of the package we're acting on @type source: L{DebianSource} @param output_dir: where to put the tarball @type output_dir: C{Str} @param options: the parsed options @type options: C{dict} of options @return: the tree we built the tarball from @rtype: C{str} """ upstream_tree = get_upstream_tree(repo, source, options) gbp.log.info("Creating %s from '%s'" % (du.orig_file(source, options.comp_type), upstream_tree)) comp_level = int(options.comp_level) if options.comp_level != '' else None gbp.log.debug("Building upstream tarball with compression '%s'%s" % (options.comp_type, "' -%s'" % comp_level if comp_level is not None else '')) main_tree = repo.tree_drop_dirs( upstream_tree, options.components) if options.components else upstream_tree if not git_archive(repo, source, output_dir, main_tree, options.comp_type, comp_level, options.with_submodules): raise GbpError("Cannot create upstream tarball at '%s'" % output_dir) for component in options.components: subtree = repo.tree_get_dir(upstream_tree, component) if not subtree: raise GbpError( "No tree for '%s' found in '%s' to create additional tarball from" % (component, upstream_tree)) gbp.log.info( "Creating additional tarball '%s' from '%s'" % (du.orig_file( source, options.comp_type, component=component), subtree)) if not git_archive(repo, source, output_dir, subtree, options.comp_type, comp_level, options.with_submodules, component=component): raise GbpError("Cannot create additional tarball %s at '%s'" % (component, output_dir)) return upstream_tree
def git_archive_build_orig(repo, cp, output_dir, options): """ Build orig tarball(s) using git-archive @param cp: the changelog of the package we're acting on @type cp: L{ChangeLog} @param output_dir: where to put the tarball @type output_dir: C{Str} @param options: the parsed options @type options: C{dict} of options @return: the tree we built the tarball from @rtype: C{str} """ upstream_tree = get_upstream_tree(repo, cp, options) gbp.log.info("Creating %s from '%s'" % (du.orig_file(cp, options.comp_type), upstream_tree)) gbp.log.debug("Building upstream tarball with compression '%s -%s'" % (options.comp_type, options.comp_level)) main_tree = repo.tree_drop_dirs(upstream_tree, options.subtarballs) if not git_archive(repo, cp, output_dir, main_tree, options.comp_type, options.comp_level, options.with_submodules): raise GbpError("Cannot create upstream tarball at '%s'" % output_dir) for subtarball in options.subtarballs: subtree = repo.tree_get_dir(upstream_tree, subtarball) if not subtree: raise GbpError( "No tree for '%s' found in '%s' to create subtarball from" % (subtarball, upstream_tree)) gbp.log.info("Creating subtarball '%s' from '%s'" % (du.orig_file( cp, options.comp_type, subtarball=subtarball), subtree)) if not git_archive(repo, cp, output_dir, subtree, options.comp_type, options.comp_level, options.with_submodules, subtarball=subtarball): raise GbpError( "Cannot create upstream component tarball %s at '%s'" % (subtarball, output_dir)) return upstream_tree
def git_archive_build_orig(repo, cp, output_dir, options): """ Build orig tarball(s) using git-archive @param cp: the changelog of the package we're acting on @type cp: L{ChangeLog} @param output_dir: where to put the tarball @type output_dir: C{Str} @param options: the parsed options @type options: C{dict} of options @return: the tree we built the tarball from @rtype: C{str} """ upstream_tree = get_upstream_tree(repo, cp, options) gbp.log.info("Creating %s from '%s'" % (du.orig_file(cp, options.comp_type), upstream_tree)) gbp.log.debug("Building upstream tarball with compression '%s -%s'" % (options.comp_type, options.comp_level)) main_tree = repo.tree_drop_dirs(upstream_tree, options.components) if not git_archive(repo, cp, output_dir, main_tree, options.comp_type, options.comp_level, options.with_submodules): raise GbpError("Cannot create upstream tarball at '%s'" % output_dir) for component in options.components: subtree = repo.tree_get_dir(upstream_tree, component) if not subtree: raise GbpError("No tree for '%s' found in '%s' to create additional tarball from" % (component, upstream_tree)) gbp.log.info("Creating additional tarball '%s' from '%s'" % (du.orig_file(cp, options.comp_type, component=component), subtree)) if not git_archive(repo, cp, output_dir, subtree, options.comp_type, options.comp_level, options.with_submodules, component=component): raise GbpError("Cannot create additional tarball %s at '%s'" % (component, output_dir)) return upstream_tree
def prepare_upstream_tarball(repo, cp, options, tarball_dir, output_dir): """ Make sure we have an upstream tarball. This involves loooking in tarball_dir, symlinking or building it. """ options.comp_type = guess_comp_type(repo, options.comp_type, cp, options.tarball_dir) orig_files = [du.orig_file(cp, options.comp_type)] if options.subtarballs: orig_files += [ du.orig_file(cp, options.comp_type, sub) for sub in options.subtarballs ] # look in tarball_dir first, if found force a symlink to it if options.tarball_dir: gbp.log.debug("Looking for orig tarballs '%s' at '%s'" % (", ".join(orig_files), tarball_dir)) missing = du.DebianPkgPolicy.symlink_origs(orig_files, tarball_dir, output_dir, force=True) if missing: msg = "Tarballs '%s' not found at '%s'" % (", ".join(missing), tarball_dir) else: msg = "All Orig tarballs '%s' found at '%s'" % ( ", ".join(orig_files), tarball_dir) gbp.log.info(msg) if options.no_create_orig: return # Create tarball if missing or forced if not du.DebianPkgPolicy.has_origs(orig_files, output_dir) or options.force_create: if not pristine_tar_build_orig(repo, cp, output_dir, options): upstream_tree = git_archive_build_orig(repo, cp, output_dir, options) if options.pristine_tar_commit: pristine_tar_commit(repo, cp, options, output_dir, orig_files[0], upstream_tree)
def prepare_upstream_tarball(repo, cp, options, tarball_dir, output_dir): """ Make sure we have an upstream tarball. This involves loooking in tarball_dir, symlinking or building it. """ gbp.log.debug('prepare_upstream_tarball()') options.comp_type = guess_comp_type(repo, options.comp_type, cp, options.tarball_dir) orig_file = du.orig_file(cp, options.comp_type) gbp.log.debug('prepare_upstream_tarball(); orig_file={!r}'.format(orig_file)) # look in tarball_dir first, if found force a symlink to it if options.tarball_dir: gbp.log.debug("Looking for orig tarball '%s' at '%s'" % (orig_file, tarball_dir)) if not du.DebianPkgPolicy.symlink_orig(orig_file, tarball_dir, output_dir, force=True): gbp.log.info("Orig tarball '%s' not found at '%s'" % (orig_file, tarball_dir)) else: gbp.log.info("Orig tarball '%s' found at '%s'" % (orig_file, tarball_dir)) # build an orig unless the user forbids it, always build (and overwrite pre-existing) if user forces it has_orig = du.DebianPkgPolicy.has_orig(orig_file, output_dir) gbp.log.info('force_create: {} no_create_orig: {} has_orig: {}'.format(options.force_create, options.no_create_orig, has_orig)) if options.force_create or (not options.no_create_orig and not has_orig): # Tarball doesn't exist, or the user has forced us to create one anyhow. gbp.log.info('Creating orig tarball...') if not pristine_tar_build_orig(repo, cp, output_dir, options): # We weren't able to get a tarball from pristine-tar. gbp.log.debug('pristine-tar was unable to provide an orig tarball.') # @KK: This function now does a bunch of nice checking for snapshot builds (e.g. that # the version number is appropriate, that the commit it references is on the # --upstream-branch, and so on). For the time being, use --force-create to get those # nice extras. (Probably always a decent idea, though.) upstream_tree = git_archive_build_orig(repo, cp, output_dir, options) if options.pristine_tar_commit: if repo.pristine_tar.has_commit(cp.name, cp.upstream_version, options.comp_type): raise AssertionError('If we already had a commit, then pristine_tar_build_orig() should have ' 'returned True. What is going on?') gbp.log.debug("%s already on pristine tar branch" % orig_file) else: archive = os.path.join(output_dir, orig_file) gbp.log.debug("Adding %s to pristine-tar branch" % archive) # import pdb; pdb.set_trace() repo.pristine_tar.commit(archive, upstream_tree)
def guess_comp_type(repo, comp_type, cp, tarball_dir): """Guess compression type""" srcpkg = cp['Source'] upstream_version = cp['Upstream-Version'] if comp_type != 'auto': comp_type = compressor_aliases.get(comp_type, comp_type) if comp_type not in compressor_opts: gbp.log.warn("Unknown compression type - guessing.") comp_type = 'auto' if comp_type == 'auto': if not repo.has_pristine_tar_branch(): if not tarball_dir: tarball_dir = '..' detected = None for comp in compressor_opts.keys(): if du.DebianPkgPolicy.has_orig(du.orig_file(cp, comp), tarball_dir): if detected is not None: raise GbpError("Multiple orig tarballs found.") detected = comp if detected is not None: comp_type = detected else: comp_type = 'gzip' else: regex = 'pristine-tar .* %s_%s\.orig.tar\.' % (srcpkg, upstream_version) commits = repo.grep_log(regex, repo.pristine_tar_branch) if commits: commit = commits[-1] gbp.log.debug("Found pristine-tar commit at '%s'" % commit) else: commit = repo.pristine_tar_branch tarball = repo.get_commit_info(commit)['subject'] (base_name, archive_fmt, comp_type) = parse_archive_filename(tarball) gbp.log.debug("Determined compression type '%s'" % comp_type) if not comp_type: comp_type = 'gzip' gbp.log.warn("Unknown compression type of %s, assuming %s" % (tarball, comp_type)) return comp_type
def guess_comp_type(repo, comp_type, cp, tarball_dir): """Guess compression type""" srcpkg = cp['Source'] upstream_version = cp['Upstream-Version'] if comp_type != 'auto': comp_type = compressor_aliases.get(comp_type, comp_type) try: dummy = compressor_opts[comp_type] except KeyError: gbp.log.warn("Unknown compression type - guessing.") comp_type = 'auto' if comp_type == 'auto': if not repo.has_pristine_tar_branch(): if not tarball_dir: tarball_dir = '..' detected = None for comp in compressor_opts.keys(): if du.DebianPkgPolicy.has_orig(du.orig_file(cp, comp), tarball_dir): if detected is not None: raise GbpError("Multiple orig tarballs found.") detected = comp if detected is not None: comp_type = detected else: comp_type = 'gzip' else: regex = 'pristine-tar .* %s_%s\.orig.tar\.' % (srcpkg, upstream_version) commits = repo.grep_log(regex, repo.pristine_tar_branch) if commits: commit = commits[-1] gbp.log.debug("Found pristine-tar commit at '%s'" % commit) else: commit = repo.pristine_tar_branch tarball = repo.get_subject(commit) comp_type = du.DebianPkgPolicy.get_compression(tarball) gbp.log.debug("Determined compression type '%s'" % comp_type) if not comp_type: comp_type = 'gzip' gbp.log.warn("Unknown compression type of %s, assuming %s" % (tarball, comp_type)) return comp_type
def export_source(repo, tree, source, options, dest_dir, tarball_dir): """ Export a version of the source tree when building in a separate directory @param repo: the git repository to export from @type repo: L{gbp.git.GitRepository} @param source: the source package @param options: options to apply @param dest_dir: where to export the source to @param tarball_dir: where to fetch the tarball from in overlay mode @returns: the temporary directory """ # Extract orig tarball if git-overlay option is selected: if options.overlay: if source.is_native(): raise GbpError("Cannot overlay Debian native package") extract_orig( os.path.join(tarball_dir, du.orig_file(source, options.comp_type)), dest_dir) gbp.log.info("Exporting '%s' to '%s'" % (options.export, dest_dir)) if not dump_tree(repo, dest_dir, tree, options.with_submodules): raise GbpError
def test_has_orig_single_false(self): self.assertFalse( DebianPkgPolicy.has_origs([orig_file(self.source, 'gzip')], str(self.tmpdir)))
def test_has_orig_multiple_false(self): orig_files = [orig_file(self.cp, 'gzip')] + \ [orig_file(self.cp, 'gzip', sub) for sub in ['foo', 'bar']] self.assertFalse(DebianPkgPolicy.has_origs(orig_files, str(self.tmpdir)))
def test_has_orig_false(self): self.assertFalse( DebianPkgPolicy.has_orig(orig_file(self.cp, 'gzip'), str(self.tmpdir)))
def test_has_orig_true(self): open(self.tmpdir.join('source_1.2.orig.tar.gz'), "w").close() self.assertTrue( DebianPkgPolicy.has_orig(orig_file(self.cp, 'gzip'), str(self.tmpdir)))