def update(self, dist):
cmd = ["sudo", "-E", "ARCH=" + self.architecture, "DIST=" + dist,
self.name, "--update",
"--architecture", self.architecture, "--distribution", dist]
Logger.command(cmd)
returncode = subprocess.call(cmd)
return self._update_failure(returncode, dist)
def rmadison(url, package, suite=None, arch=None):
"Call rmadison and parse the result"
cmd = ['rmadison', '-u', url]
if suite:
cmd += ['-s', suite]
if arch:
cmd += ['-a', arch]
cmd.append(package)
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
output, error_output = process.communicate()
if process.wait() != 0:
if error_output:
Logger.error('rmadison failed with: %s', error_output)
else:
Logger.error('rmadison failed')
sys.exit(1)
# rmadison uses some shorthand
if suite:
suite = suite.replace('-proposed-updates', '-p-u')
# pylint bug: http://www.logilab.org/ticket/46273
# pylint: disable=E1103
for line in output.strip().splitlines():
# pylint: enable=E1103
pkg, ver, dist, archs = [x.strip() for x in line.split('|')]
comp = 'main'
if '/' in dist:
dist, comp = dist.split('/')
archs = set(x.strip() for x in archs.split(','))
# rmadison returns some results outside the requested set.
# It'll include backports, and when given an unknown suite,
# it ignores that argument
#
# some versions (2.14.1ubuntu0.1) of rmadison return 'sid' when
# asked about 'unstable'. Others return 'unstable'. Accept either.
if (suite and dist != suite
and not (suite == 'sid' and dist == 'unstable')):
continue
if 'source' in archs:
yield {
'source': pkg,
'version': ver,
'suite': dist,
'component': comp,
}
archs.discard('source')
if archs:
yield {
'binary': pkg,
'version': ver,
'suite': dist,
'component': comp,
'architectures': archs,
}
def get_patch_or_branch(bug):
patch = None
branch = None
if not is_sync(bug):
attached_patches = [a for a in bug.attachments if a.type == "Patch"]
linked_branches = [b.branch for b in bug.linked_branches]
if len(attached_patches) == 0 and len(linked_branches) == 0:
if len(bug.attachments) == 0:
Logger.error(
"No attachment and no linked branch found on "
"bug #%i. Add the tag sync to the bug if it is "
"a sync request.", bug.id)
else:
Logger.error(
"No attached patch and no linked branch found. "
"Go to https://launchpad.net/bugs/%i and mark an "
"attachment as patch.", bug.id)
sys.exit(1)
elif len(attached_patches) == 1 and len(linked_branches) == 0:
patch = Patch(attached_patches[0])
elif len(attached_patches) == 0 and len(linked_branches) == 1:
branch = linked_branches[0].bzr_identity
else:
patch, branch = ask_for_patch_or_branch(bug, attached_patches,
linked_branches)
return (patch, branch)
def ask_for_patch_or_branch(bug, attached_patches, linked_branches):
patch = None
branch = None
if len(attached_patches) == 0:
msg = "https://launchpad.net/bugs/%i has %i branches linked:" % \
(bug.id, len(linked_branches))
elif len(linked_branches) == 0:
msg = "https://launchpad.net/bugs/%i has %i patches attached:" % \
(bug.id, len(attached_patches))
else:
branches = "%i branch" % len(linked_branches)
if len(linked_branches) > 1:
branches += "es"
patches = "%i patch" % len(attached_patches)
if len(attached_patches) > 1:
patches += "es"
msg = "https://launchpad.net/bugs/%i has %s linked and %s attached:" % \
(bug.id, branches, patches)
Logger.normal(msg)
i = 0
for linked_branch in linked_branches:
i += 1
print("%i) %s" % (i, linked_branch.display_name))
for attached_patch in attached_patches:
i += 1
print("%i) %s" % (i, attached_patch.title))
selected = input_number("Which branch or patch do you want to download", 1,
i, i)
if selected <= len(linked_branches):
branch = linked_branches[selected - 1].bzr_identity
else:
patch = Patch(attached_patches[selected - len(linked_branches) - 1])
return (patch, branch)
def build(self, dsc_file, dist, result_directory):
_build_preparation(result_directory)
cmd = [self.name, dist, self.architecture,
"build", dsc_file, "--buildresult", result_directory]
Logger.command(cmd)
returncode = subprocess.call(cmd)
return self._build_failure(returncode, dsc_file)
def _run_lintian(self):
"""Runs lintian on either the source or binary changes file.
Returns the filename of the created lintian output file.
"""
# Determine whether to use the source or binary build for lintian
if self._build_log:
build_changes = self._package + "_" + strip_epoch(self._version) + \
"_" + self._builder.get_architecture() + ".changes"
changes_for_lintian = os.path.join(self._buildresult, build_changes)
else:
changes_for_lintian = self._changes_file
# Check lintian
assert os.path.isfile(changes_for_lintian), "%s does not exist." % \
(changes_for_lintian)
cmd = ["lintian", "-IE", "--pedantic", "-q", "--profile", "ubuntu",
changes_for_lintian]
lintian_filename = os.path.join(self._workdir,
self._package + "_" +
strip_epoch(self._version) + ".lintian")
Logger.command(cmd + [">", lintian_filename])
process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
report = process.communicate()[0]
# write lintian report file
lintian_file = open(lintian_filename, "w")
lintian_file.writelines(report)
lintian_file.close()
return lintian_filename
def build_source(self, keyid, upload, previous_version):
"""Tries to build the source package.
Returns true if the source package was built successfully. Returns false
if the user wants to change something.
"""
if self._branch:
cmd = ['bzr', 'builddeb', '--builder=debuild', '-S',
'--', '--no-lintian', '-nc']
else:
cmd = ['debuild', '--no-lintian', '-nc', '-S']
cmd.append("-v" + previous_version.full_version)
if previous_version.upstream_version == \
self._changelog.upstream_version and upload == "ubuntu":
# FIXME: Add proper check that catches cases like changed
# compression (.tar.gz -> tar.bz2) and multiple orig source tarballs
cmd.append("-sd")
else:
cmd.append("-sa")
if keyid is not None:
cmd += ["-k" + keyid]
env = os.environ
if upload == 'ubuntu':
env['DEB_VENDOR'] = 'Ubuntu'
Logger.command(cmd)
if subprocess.call(cmd, env=env) != 0:
Logger.error("Failed to build source tarball.")
# TODO: Add a "retry" option
ask_for_manual_fixing()
return False
return True
def reload_changelog(self):
"""Reloads debian/changelog and updates the version.
Returns true if the changelog was reloaded successfully. Returns false
if the user wants to correct a broken changelog.
"""
# Check the changelog
self._changelog = debian.changelog.Changelog()
try:
self._changelog.parse_changelog(open("debian/changelog"),
max_blocks=1, strict=True)
except debian.changelog.ChangelogParseError as error:
Logger.error("The changelog entry doesn't validate: %s", str(error))
ask_for_manual_fixing()
return False
# Get new version of package
try:
self._version = self._changelog.get_version()
except IndexError:
Logger.error("Debian package version could not be determined. "
"debian/changelog is probably malformed.")
ask_for_manual_fixing()
return False
return True
def snapshot_list(self):
"Return a filename -> hash dictionary from snapshot.debian.org"
if self._snapshot_list is None:
try:
import json
except ImportError:
import simplejson as json
except ImportError:
Logger.error("Please install python-simplejson.")
raise DownloadError("Unable to dowload from "
"snapshot.debian.org without "
"python-simplejson")
try:
data = self.url_opener.open(
'http://snapshot.debian.org/mr/package/%s/%s/srcfiles?fileinfo=1'
% (self.source, self.version.full_version))
reader = codecs.getreader('utf-8')
srcfiles = json.load(reader(data))
except HTTPError:
Logger.error(
'Version %s of %s not found on '
'snapshot.debian.org', self.version.full_version,
self.source)
self._snapshot_list = False
return False
self._snapshot_list = dict(
(info[0]['name'], hash_)
for hash_, info in srcfiles['fileinfo'].items())
return self._snapshot_list
def _download_and_change_into(task, dsc_file, patch, branch):
"""Downloads the patch and branch and changes into the source directory."""
if branch:
branch_dir = download_branch(task.get_branch_link())
# change directory
Logger.command(["cd", branch_dir])
os.chdir(branch_dir)
else:
if patch:
patch.download()
Logger.info("Ubuntu package: %s" % (task.package))
if task.is_merge():
Logger.info("The task is a merge request.")
if task.is_sync():
Logger.info("The task is a sync request.")
extract_source(dsc_file, Logger.verbose)
# change directory
directory = task.package + '-' + task.get_version().upstream_version
Logger.command(["cd", directory])
os.chdir(directory)
def update(self, dist):
cmd = ["schroot", "--list"]
Logger.command(cmd)
process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
chroots, _ = process.communicate()[0].strip().split()
if process.returncode != 0:
return process.returncode
params = {"dist": dist, "arch": self.architecture}
for chroot in ("%(dist)s-%(arch)s-sbuild-source",
"%(dist)s-sbuild-source",
"%(dist)s-%(arch)s-source",
"%(dist)s-source"):
chroot = chroot % params
if chroot in chroots:
break
else:
return 1
commands = [["sbuild-update"],
["sbuild-distupgrade"],
["sbuild-clean", "-a", "-c"]]
for cmd in commands:
# pylint: disable=W0631
Logger.command(cmd + [chroot])
ret = subprocess.call(cmd + [chroot])
# pylint: enable=W0631
if ret != 0:
return self._update_failure(ret, dist)
return 0
def extract_source(dsc_file, verbose=False):
cmd = ["dpkg-source", "--no-preparation", "-x", dsc_file]
if not verbose:
cmd.insert(1, "-q")
Logger.command(cmd)
if subprocess.call(cmd) != 0:
Logger.error("Extraction of %s failed." % (os.path.basename(dsc_file)))
sys.exit(1)
def _update_maintainer_field():
"""Update the Maintainer field in debian/control."""
Logger.command(["update-maintainer"])
try:
update_maintainer("debian", Logger.verbose)
except MaintainerUpdateException as e:
Logger.error("update-maintainer failed: %s", str(e))
sys.exit(1)
def merge_branch(branch):
edit = False
cmd = ["bzr", "merge", branch]
Logger.command(cmd)
if subprocess.call(cmd) != 0:
Logger.error("Failed to merge branch %s." % (branch))
ask_for_manual_fixing()
edit = True
return edit
def unpack(self, destdir=None):
"Unpack in workdir"
cmd = ['dpkg-source', '-x', self.dsc_name]
if destdir:
cmd.append(destdir)
Logger.command(cmd)
if subprocess.call(cmd, cwd=self.workdir):
Logger.error('Source unpack failed.')
sys.exit(1)
def build(self, dsc_file, dist, result_directory):
_build_preparation(result_directory)
cmd = ["sudo", "-E", "ARCH=" + self.architecture, "DIST=" + dist,
self.name, "--build",
"--architecture", self.architecture, "--distribution", dist,
"--buildresult", result_directory, dsc_file]
Logger.command(cmd)
returncode = subprocess.call(cmd)
return self._build_failure(returncode, dsc_file)
def download_branch(branch):
dir_name = os.path.basename(branch)
if os.path.isdir(dir_name):
shutil.rmtree(dir_name)
cmd = ["bzr", "branch", branch]
Logger.command(cmd)
if subprocess.call(cmd) != 0:
Logger.error("Failed to download branch %s." % (branch))
sys.exit(1)
return dir_name
def get_debian_srcpkg(name, release):
debian = Distribution('debian')
debian_archive = debian.getArchive()
try:
release = DebianDistroInfo().codename(release, None, release)
except DistroDataOutdated as e:
Logger.warn(e)
return debian_archive.getSourcePackage(name, release)
def __init__(self, patch):
self._patch = patch
self._patch_file = re.sub(" |/", "_", patch.title)
if not reduce(lambda r, x: r or self._patch.title.endswith(x),
(".debdiff", ".diff", ".patch"), False):
Logger.info("Patch %s does not have a proper file extension." %
(self._patch.title))
self._patch_file += ".patch"
self._full_path = os.path.realpath(self._patch_file)
self._changed_files = None
def download(self):
"""Downloads the patch from Launchpad."""
Logger.info("Downloading %s." % (self._patch_file))
patch_f = open(self._patch_file, "w")
patch_f.write(self._patch.data.open().read())
patch_f.close()
cmd = ["diffstat", "-l", "-p0", self._full_path]
process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
changed_files = process.communicate()[0]
self._changed_files = [f for f in changed_files.split("\n") if f != ""]
def check_version(self, previous_version):
"""Check if the version of the package is greater than the given one.
Return true if the version of the package is newer. Returns false
if the user wants to change something.
"""
if self._version <= previous_version:
Logger.error("The version %s is not greater than the already "
"available %s.", self._version, previous_version)
return ask_for_ignoring_or_fixing()
return True
def edit_source():
# Spawn shell to allow modifications
cmd = [get_user_shell()]
Logger.command(cmd)
print("""An interactive shell was launched in
file://%s
Edit your files. When you are done, exit the shell. If you wish to abort the
process, exit the shell such that it returns an exit code other than zero.
""" % (os.getcwd()),
end=' ')
returncode = subprocess.call(cmd)
if returncode != 0:
Logger.error("Shell exited with exit value %i." % (returncode))
sys.exit(1)
def _create_and_change_into(workdir):
"""Create (if it does not exits) and change into given working directory."""
if not os.path.isdir(workdir):
try:
os.makedirs(workdir)
except os.error as error:
Logger.error(
"Failed to create the working directory %s [Errno %i]: %s." %
(workdir, error.errno, error.strerror))
sys.exit(1)
if workdir != os.getcwd():
Logger.command(["cd", workdir])
os.chdir(workdir)
def apply(self, task):
"""Applies the patch in the current directory."""
assert self._changed_files is not None, "You forgot to download the patch."
edit = False
if self.is_debdiff():
cmd = [
"patch", "--merge", "--force", "-p",
str(self.get_strip_level()), "-i", self._full_path
]
Logger.command(cmd)
if subprocess.call(cmd) != 0:
Logger.error("Failed to apply debdiff %s to %s %s.",
self._patch_file, task.package,
task.get_version())
if not edit:
ask_for_manual_fixing()
edit = True
else:
cmd = ["add-patch", self._full_path]
Logger.command(cmd)
if subprocess.call(cmd) != 0:
Logger.error("Failed to apply diff %s to %s %s.",
self._patch_file, task.package,
task.get_version())
if not edit:
ask_for_manual_fixing()
edit = True
return edit
def pull(self):
"Pull into workdir"
self._write_dsc()
for entry in self.dsc['Files']:
name = entry['name']
for url in self._source_urls(name):
try:
if self._download_file(url, name):
break
except HTTPError as e:
Logger.normal('HTTP Error %i: %s', e.code, str(e))
except URLError as e:
Logger.normal('URL Error: %s', e.reason)
else:
raise DownloadError('File %s could not be found' % name)
def check_dependencies():
"Do we have all the commands we need for full functionality?"
missing = []
for cmd in ('patch', 'bzr', 'quilt', 'dput', 'lintian'):
if not is_command_available(cmd):
missing.append(cmd)
if not is_command_available('bzr-buildpackage'):
missing.append('bzr-builddeb')
if not any(
is_command_available(cmd, check_sbin=True)
for cmd in ('pbuilder', 'sbuild', 'cowbuilder')):
missing.append('pbuilder/cowbuilder/sbuild')
if missing:
Logger.warn(
"sponsor-patch requires %s to be installed for full "
"functionality", ', '.join(missing))
def restore_maintainer(debian_directory, verbose=False):
"""Restore the original maintainer"""
try:
changelog_file, control_files = _find_files(debian_directory, verbose)
except MaintainerUpdateException as e:
Logger.error(str(e))
raise
for control_file in control_files:
control = Control(control_file)
orig_maintainer = control.get_original_maintainer()
if not orig_maintainer:
continue
if verbose:
print("Restoring original maintainer: %s" % orig_maintainer)
control.set_maintainer(orig_maintainer)
control.remove_original_maintainer()
control.save()
def generate_debdiff(self, dsc_file):
"""Generates a debdiff between the given .dsc file and this source
package."""
assert os.path.isfile(dsc_file), "%s does not exist." % (dsc_file)
assert os.path.isfile(self._dsc_file), "%s does not exist." % \
(self._dsc_file)
cmd = ["debdiff", dsc_file, self._dsc_file]
if not Logger.verbose:
cmd.insert(1, "-q")
Logger.command(cmd + [">", self._debdiff_filename])
process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
debdiff = process.communicate()[0]
# write debdiff file
debdiff_file = open(self._debdiff_filename, "w")
debdiff_file.writelines(debdiff)
debdiff_file.close()
def get_ubuntu_delta_changelog(srcpkg):
'''
Download the Ubuntu changelog and extract the entries since the last sync
from Debian.
'''
archive = Distribution('ubuntu').getArchive()
spph = archive.getPublishedSources(source_name=srcpkg.getPackageName(),
exact_match=True, pocket='Release')
debian_info = DebianDistroInfo()
topline = re.compile(r'^(\w%(name_chars)s*) \(([^\(\) \t]+)\)'
r'((\s+%(name_chars)s+)+)\;'
% {'name_chars': '[-+0-9a-z.]'},
re.IGNORECASE)
delta = []
for record in spph:
changes_url = record.changesFileUrl()
if changes_url is None:
# Native sync
break
try:
response, body = Http().request(changes_url)
except HttpLib2Error as e:
Logger.error(str(e))
break
if response.status != 200:
Logger.error("%s: %s %s", changes_url, response.status,
response.reason)
break
changes = Changes(Http().request(changes_url)[1])
for line in changes['Changes'].splitlines():
line = line[1:]
m = topline.match(line)
if m:
distribution = m.group(3).split()[0].split('-')[0]
if debian_info.valid(distribution):
break
if line.startswith(u' '):
delta.append(line)
else:
continue
break
return '\n'.join(delta)
def _get_srcpkg(distro, name, release):
if distro == 'debian':
# Canonicalise release:
debian_info = DebianDistroInfo()
try:
codename = debian_info.codename(release, default=release)
except DistroDataOutdated as e:
Logger.warn(e)
lines = list(rmadison(distro, name, suite=codename, arch='source'))
if not lines:
lines = list(rmadison(distro, name, suite=release, arch='source'))
if not lines:
raise PackageNotFoundException(
"'%s' doesn't appear to exist in %s '%s'" %
(name, distro.capitalize(), release))
pkg = max(lines, key=lambda x: Version(x['version']))
return FakeSPPH(pkg['source'], pkg['version'], pkg['component'], distro)
