def update(self, dist): cmd = ["sudo", "-E", "ARCH=" + self.architecture, "DIST=" + dist, self.name, "--update", "--architecture", self.architecture, "--distribution", dist] Logger.command(cmd) returncode = subprocess.call(cmd) return self._update_failure(returncode, dist)
def rmadison(url, package, suite=None, arch=None): "Call rmadison and parse the result" cmd = ['rmadison', '-u', url] if suite: cmd += ['-s', suite] if arch: cmd += ['-a', arch] cmd.append(package) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) output, error_output = process.communicate() if process.wait() != 0: if error_output: Logger.error('rmadison failed with: %s', error_output) else: Logger.error('rmadison failed') sys.exit(1) # rmadison uses some shorthand if suite: suite = suite.replace('-proposed-updates', '-p-u') # pylint bug: http://www.logilab.org/ticket/46273 # pylint: disable=E1103 for line in output.strip().splitlines(): # pylint: enable=E1103 pkg, ver, dist, archs = [x.strip() for x in line.split('|')] comp = 'main' if '/' in dist: dist, comp = dist.split('/') archs = set(x.strip() for x in archs.split(',')) # rmadison returns some results outside the requested set. # It'll include backports, and when given an unknown suite, # it ignores that argument # # some versions (2.14.1ubuntu0.1) of rmadison return 'sid' when # asked about 'unstable'. Others return 'unstable'. Accept either. if (suite and dist != suite and not (suite == 'sid' and dist == 'unstable')): continue if 'source' in archs: yield { 'source': pkg, 'version': ver, 'suite': dist, 'component': comp, } archs.discard('source') if archs: yield { 'binary': pkg, 'version': ver, 'suite': dist, 'component': comp, 'architectures': archs, }
def get_patch_or_branch(bug): patch = None branch = None if not is_sync(bug): attached_patches = [a for a in bug.attachments if a.type == "Patch"] linked_branches = [b.branch for b in bug.linked_branches] if len(attached_patches) == 0 and len(linked_branches) == 0: if len(bug.attachments) == 0: Logger.error( "No attachment and no linked branch found on " "bug #%i. Add the tag sync to the bug if it is " "a sync request.", bug.id) else: Logger.error( "No attached patch and no linked branch found. " "Go to https://launchpad.net/bugs/%i and mark an " "attachment as patch.", bug.id) sys.exit(1) elif len(attached_patches) == 1 and len(linked_branches) == 0: patch = Patch(attached_patches[0]) elif len(attached_patches) == 0 and len(linked_branches) == 1: branch = linked_branches[0].bzr_identity else: patch, branch = ask_for_patch_or_branch(bug, attached_patches, linked_branches) return (patch, branch)
def ask_for_patch_or_branch(bug, attached_patches, linked_branches): patch = None branch = None if len(attached_patches) == 0: msg = "https://launchpad.net/bugs/%i has %i branches linked:" % \ (bug.id, len(linked_branches)) elif len(linked_branches) == 0: msg = "https://launchpad.net/bugs/%i has %i patches attached:" % \ (bug.id, len(attached_patches)) else: branches = "%i branch" % len(linked_branches) if len(linked_branches) > 1: branches += "es" patches = "%i patch" % len(attached_patches) if len(attached_patches) > 1: patches += "es" msg = "https://launchpad.net/bugs/%i has %s linked and %s attached:" % \ (bug.id, branches, patches) Logger.normal(msg) i = 0 for linked_branch in linked_branches: i += 1 print("%i) %s" % (i, linked_branch.display_name)) for attached_patch in attached_patches: i += 1 print("%i) %s" % (i, attached_patch.title)) selected = input_number("Which branch or patch do you want to download", 1, i, i) if selected <= len(linked_branches): branch = linked_branches[selected - 1].bzr_identity else: patch = Patch(attached_patches[selected - len(linked_branches) - 1]) return (patch, branch)
def build(self, dsc_file, dist, result_directory): _build_preparation(result_directory) cmd = [self.name, dist, self.architecture, "build", dsc_file, "--buildresult", result_directory] Logger.command(cmd) returncode = subprocess.call(cmd) return self._build_failure(returncode, dsc_file)
def _run_lintian(self): """Runs lintian on either the source or binary changes file. Returns the filename of the created lintian output file. """ # Determine whether to use the source or binary build for lintian if self._build_log: build_changes = self._package + "_" + strip_epoch(self._version) + \ "_" + self._builder.get_architecture() + ".changes" changes_for_lintian = os.path.join(self._buildresult, build_changes) else: changes_for_lintian = self._changes_file # Check lintian assert os.path.isfile(changes_for_lintian), "%s does not exist." % \ (changes_for_lintian) cmd = ["lintian", "-IE", "--pedantic", "-q", "--profile", "ubuntu", changes_for_lintian] lintian_filename = os.path.join(self._workdir, self._package + "_" + strip_epoch(self._version) + ".lintian") Logger.command(cmd + [">", lintian_filename]) process = subprocess.Popen(cmd, stdout=subprocess.PIPE) report = process.communicate()[0] # write lintian report file lintian_file = open(lintian_filename, "w") lintian_file.writelines(report) lintian_file.close() return lintian_filename
def build_source(self, keyid, upload, previous_version): """Tries to build the source package. Returns true if the source package was built successfully. Returns false if the user wants to change something. """ if self._branch: cmd = ['bzr', 'builddeb', '--builder=debuild', '-S', '--', '--no-lintian', '-nc'] else: cmd = ['debuild', '--no-lintian', '-nc', '-S'] cmd.append("-v" + previous_version.full_version) if previous_version.upstream_version == \ self._changelog.upstream_version and upload == "ubuntu": # FIXME: Add proper check that catches cases like changed # compression (.tar.gz -> tar.bz2) and multiple orig source tarballs cmd.append("-sd") else: cmd.append("-sa") if keyid is not None: cmd += ["-k" + keyid] env = os.environ if upload == 'ubuntu': env['DEB_VENDOR'] = 'Ubuntu' Logger.command(cmd) if subprocess.call(cmd, env=env) != 0: Logger.error("Failed to build source tarball.") # TODO: Add a "retry" option ask_for_manual_fixing() return False return True
def reload_changelog(self): """Reloads debian/changelog and updates the version. Returns true if the changelog was reloaded successfully. Returns false if the user wants to correct a broken changelog. """ # Check the changelog self._changelog = debian.changelog.Changelog() try: self._changelog.parse_changelog(open("debian/changelog"), max_blocks=1, strict=True) except debian.changelog.ChangelogParseError as error: Logger.error("The changelog entry doesn't validate: %s", str(error)) ask_for_manual_fixing() return False # Get new version of package try: self._version = self._changelog.get_version() except IndexError: Logger.error("Debian package version could not be determined. " "debian/changelog is probably malformed.") ask_for_manual_fixing() return False return True
def snapshot_list(self): "Return a filename -> hash dictionary from snapshot.debian.org" if self._snapshot_list is None: try: import json except ImportError: import simplejson as json except ImportError: Logger.error("Please install python-simplejson.") raise DownloadError("Unable to dowload from " "snapshot.debian.org without " "python-simplejson") try: data = self.url_opener.open( 'http://snapshot.debian.org/mr/package/%s/%s/srcfiles?fileinfo=1' % (self.source, self.version.full_version)) reader = codecs.getreader('utf-8') srcfiles = json.load(reader(data)) except HTTPError: Logger.error( 'Version %s of %s not found on ' 'snapshot.debian.org', self.version.full_version, self.source) self._snapshot_list = False return False self._snapshot_list = dict( (info[0]['name'], hash_) for hash_, info in srcfiles['fileinfo'].items()) return self._snapshot_list
def _download_and_change_into(task, dsc_file, patch, branch): """Downloads the patch and branch and changes into the source directory.""" if branch: branch_dir = download_branch(task.get_branch_link()) # change directory Logger.command(["cd", branch_dir]) os.chdir(branch_dir) else: if patch: patch.download() Logger.info("Ubuntu package: %s" % (task.package)) if task.is_merge(): Logger.info("The task is a merge request.") if task.is_sync(): Logger.info("The task is a sync request.") extract_source(dsc_file, Logger.verbose) # change directory directory = task.package + '-' + task.get_version().upstream_version Logger.command(["cd", directory]) os.chdir(directory)
def update(self, dist): cmd = ["schroot", "--list"] Logger.command(cmd) process = subprocess.Popen(cmd, stdout=subprocess.PIPE) chroots, _ = process.communicate()[0].strip().split() if process.returncode != 0: return process.returncode params = {"dist": dist, "arch": self.architecture} for chroot in ("%(dist)s-%(arch)s-sbuild-source", "%(dist)s-sbuild-source", "%(dist)s-%(arch)s-source", "%(dist)s-source"): chroot = chroot % params if chroot in chroots: break else: return 1 commands = [["sbuild-update"], ["sbuild-distupgrade"], ["sbuild-clean", "-a", "-c"]] for cmd in commands: # pylint: disable=W0631 Logger.command(cmd + [chroot]) ret = subprocess.call(cmd + [chroot]) # pylint: enable=W0631 if ret != 0: return self._update_failure(ret, dist) return 0
def extract_source(dsc_file, verbose=False): cmd = ["dpkg-source", "--no-preparation", "-x", dsc_file] if not verbose: cmd.insert(1, "-q") Logger.command(cmd) if subprocess.call(cmd) != 0: Logger.error("Extraction of %s failed." % (os.path.basename(dsc_file))) sys.exit(1)
def _update_maintainer_field(): """Update the Maintainer field in debian/control.""" Logger.command(["update-maintainer"]) try: update_maintainer("debian", Logger.verbose) except MaintainerUpdateException as e: Logger.error("update-maintainer failed: %s", str(e)) sys.exit(1)
def merge_branch(branch): edit = False cmd = ["bzr", "merge", branch] Logger.command(cmd) if subprocess.call(cmd) != 0: Logger.error("Failed to merge branch %s." % (branch)) ask_for_manual_fixing() edit = True return edit
def unpack(self, destdir=None): "Unpack in workdir" cmd = ['dpkg-source', '-x', self.dsc_name] if destdir: cmd.append(destdir) Logger.command(cmd) if subprocess.call(cmd, cwd=self.workdir): Logger.error('Source unpack failed.') sys.exit(1)
def build(self, dsc_file, dist, result_directory): _build_preparation(result_directory) cmd = ["sudo", "-E", "ARCH=" + self.architecture, "DIST=" + dist, self.name, "--build", "--architecture", self.architecture, "--distribution", dist, "--buildresult", result_directory, dsc_file] Logger.command(cmd) returncode = subprocess.call(cmd) return self._build_failure(returncode, dsc_file)
def download_branch(branch): dir_name = os.path.basename(branch) if os.path.isdir(dir_name): shutil.rmtree(dir_name) cmd = ["bzr", "branch", branch] Logger.command(cmd) if subprocess.call(cmd) != 0: Logger.error("Failed to download branch %s." % (branch)) sys.exit(1) return dir_name
def get_debian_srcpkg(name, release): debian = Distribution('debian') debian_archive = debian.getArchive() try: release = DebianDistroInfo().codename(release, None, release) except DistroDataOutdated as e: Logger.warn(e) return debian_archive.getSourcePackage(name, release)
def __init__(self, patch): self._patch = patch self._patch_file = re.sub(" |/", "_", patch.title) if not reduce(lambda r, x: r or self._patch.title.endswith(x), (".debdiff", ".diff", ".patch"), False): Logger.info("Patch %s does not have a proper file extension." % (self._patch.title)) self._patch_file += ".patch" self._full_path = os.path.realpath(self._patch_file) self._changed_files = None
def download(self): """Downloads the patch from Launchpad.""" Logger.info("Downloading %s." % (self._patch_file)) patch_f = open(self._patch_file, "w") patch_f.write(self._patch.data.open().read()) patch_f.close() cmd = ["diffstat", "-l", "-p0", self._full_path] process = subprocess.Popen(cmd, stdout=subprocess.PIPE) changed_files = process.communicate()[0] self._changed_files = [f for f in changed_files.split("\n") if f != ""]
def check_version(self, previous_version): """Check if the version of the package is greater than the given one. Return true if the version of the package is newer. Returns false if the user wants to change something. """ if self._version <= previous_version: Logger.error("The version %s is not greater than the already " "available %s.", self._version, previous_version) return ask_for_ignoring_or_fixing() return True
def edit_source(): # Spawn shell to allow modifications cmd = [get_user_shell()] Logger.command(cmd) print("""An interactive shell was launched in file://%s Edit your files. When you are done, exit the shell. If you wish to abort the process, exit the shell such that it returns an exit code other than zero. """ % (os.getcwd()), end=' ') returncode = subprocess.call(cmd) if returncode != 0: Logger.error("Shell exited with exit value %i." % (returncode)) sys.exit(1)
def _create_and_change_into(workdir): """Create (if it does not exits) and change into given working directory.""" if not os.path.isdir(workdir): try: os.makedirs(workdir) except os.error as error: Logger.error( "Failed to create the working directory %s [Errno %i]: %s." % (workdir, error.errno, error.strerror)) sys.exit(1) if workdir != os.getcwd(): Logger.command(["cd", workdir]) os.chdir(workdir)
def apply(self, task): """Applies the patch in the current directory.""" assert self._changed_files is not None, "You forgot to download the patch." edit = False if self.is_debdiff(): cmd = [ "patch", "--merge", "--force", "-p", str(self.get_strip_level()), "-i", self._full_path ] Logger.command(cmd) if subprocess.call(cmd) != 0: Logger.error("Failed to apply debdiff %s to %s %s.", self._patch_file, task.package, task.get_version()) if not edit: ask_for_manual_fixing() edit = True else: cmd = ["add-patch", self._full_path] Logger.command(cmd) if subprocess.call(cmd) != 0: Logger.error("Failed to apply diff %s to %s %s.", self._patch_file, task.package, task.get_version()) if not edit: ask_for_manual_fixing() edit = True return edit
def pull(self): "Pull into workdir" self._write_dsc() for entry in self.dsc['Files']: name = entry['name'] for url in self._source_urls(name): try: if self._download_file(url, name): break except HTTPError as e: Logger.normal('HTTP Error %i: %s', e.code, str(e)) except URLError as e: Logger.normal('URL Error: %s', e.reason) else: raise DownloadError('File %s could not be found' % name)
def check_dependencies(): "Do we have all the commands we need for full functionality?" missing = [] for cmd in ('patch', 'bzr', 'quilt', 'dput', 'lintian'): if not is_command_available(cmd): missing.append(cmd) if not is_command_available('bzr-buildpackage'): missing.append('bzr-builddeb') if not any( is_command_available(cmd, check_sbin=True) for cmd in ('pbuilder', 'sbuild', 'cowbuilder')): missing.append('pbuilder/cowbuilder/sbuild') if missing: Logger.warn( "sponsor-patch requires %s to be installed for full " "functionality", ', '.join(missing))
def restore_maintainer(debian_directory, verbose=False): """Restore the original maintainer""" try: changelog_file, control_files = _find_files(debian_directory, verbose) except MaintainerUpdateException as e: Logger.error(str(e)) raise for control_file in control_files: control = Control(control_file) orig_maintainer = control.get_original_maintainer() if not orig_maintainer: continue if verbose: print("Restoring original maintainer: %s" % orig_maintainer) control.set_maintainer(orig_maintainer) control.remove_original_maintainer() control.save()
def generate_debdiff(self, dsc_file): """Generates a debdiff between the given .dsc file and this source package.""" assert os.path.isfile(dsc_file), "%s does not exist." % (dsc_file) assert os.path.isfile(self._dsc_file), "%s does not exist." % \ (self._dsc_file) cmd = ["debdiff", dsc_file, self._dsc_file] if not Logger.verbose: cmd.insert(1, "-q") Logger.command(cmd + [">", self._debdiff_filename]) process = subprocess.Popen(cmd, stdout=subprocess.PIPE) debdiff = process.communicate()[0] # write debdiff file debdiff_file = open(self._debdiff_filename, "w") debdiff_file.writelines(debdiff) debdiff_file.close()
def get_ubuntu_delta_changelog(srcpkg): ''' Download the Ubuntu changelog and extract the entries since the last sync from Debian. ''' archive = Distribution('ubuntu').getArchive() spph = archive.getPublishedSources(source_name=srcpkg.getPackageName(), exact_match=True, pocket='Release') debian_info = DebianDistroInfo() topline = re.compile(r'^(\w%(name_chars)s*) \(([^\(\) \t]+)\)' r'((\s+%(name_chars)s+)+)\;' % {'name_chars': '[-+0-9a-z.]'}, re.IGNORECASE) delta = [] for record in spph: changes_url = record.changesFileUrl() if changes_url is None: # Native sync break try: response, body = Http().request(changes_url) except HttpLib2Error as e: Logger.error(str(e)) break if response.status != 200: Logger.error("%s: %s %s", changes_url, response.status, response.reason) break changes = Changes(Http().request(changes_url)[1]) for line in changes['Changes'].splitlines(): line = line[1:] m = topline.match(line) if m: distribution = m.group(3).split()[0].split('-')[0] if debian_info.valid(distribution): break if line.startswith(u' '): delta.append(line) else: continue break return '\n'.join(delta)
def _get_srcpkg(distro, name, release): if distro == 'debian': # Canonicalise release: debian_info = DebianDistroInfo() try: codename = debian_info.codename(release, default=release) except DistroDataOutdated as e: Logger.warn(e) lines = list(rmadison(distro, name, suite=codename, arch='source')) if not lines: lines = list(rmadison(distro, name, suite=release, arch='source')) if not lines: raise PackageNotFoundException( "'%s' doesn't appear to exist in %s '%s'" % (name, distro.capitalize(), release)) pkg = max(lines, key=lambda x: Version(x['version'])) return FakeSPPH(pkg['source'], pkg['version'], pkg['component'], distro)