def remove(*package_names) -> None:
upgrade()
cache = Cache()
cache.update()
cache.open()
for name in package_names:
if name not in cache:
print('Package %s not found!' % (name, ))
continue
package = cache[name]
if not package.is_installed:
print('Package %s is not installed!' % (name, ))
continue
package.mark_delete(purge=True)
cache.commit(TextFetchProgress(), InstallProgress())
cache.close()
def install(*package_names) -> None:
upgrade()
cache = Cache()
cache.update()
cache.open()
for name in package_names:
if name not in cache:
logger.error('Package %s not found!' % (name, ))
continue
package = cache[name]
if package.is_installed:
logger.warning('Package %s already installed!' % (name, ))
continue
package.mark_install()
cache.commit(TextFetchProgress(), InstallProgress())
cache.close()
def upgrade() -> None:
cache = Cache()
cache.update()
cache.open()
cache.update()
cache.open()
cache.upgrade(dist_upgrade=True)
cache.fix_broken()
cache.commit(TextFetchProgress(), InstallProgress())
cache.close()
def do_update(mark_only):
_, progress = query_verbosity()
log.info("Getting list of eligible packages...")
cache = Cache(progress)
f_cache = FilteredCache(cache)
f_cache.set_filter(NvidiaFilter())
names = f_cache.keys()
with unhold(names, cache):
# mark_only means we just want the side-effects of exiting the
# unhold() context manager.
if mark_only:
return False
log.info("Updating package list...")
try:
cache.update()
except FetchFailedException, err:
log.warn(err)
cache.open(progress) # Refresh package list
old_versions = {name: cache[name].installed for name in names}
log.info("Updating all packages...")
for name in names:
if cache[name].is_upgradable:
cache[name].mark_upgrade()
cache.commit(None, None)
log.info("Refreshing package cache...")
cache.open(progress)
new_versions = {name: cache[name].installed for name in names}
log.info("Checking whether packages were upgraded...")
for name in old_versions:
if old_versions[name] != new_versions[name]:
log.info("Kernel module changed")
return True
return False
def do_install(self, to_install, langpacks=False):
self.nested_progress_start()
if langpacks:
self.db.progress('START', 0, 10, 'ubiquity/langpacks/title')
else:
self.db.progress('START', 0, 10, 'ubiquity/install/title')
self.db.progress('INFO', 'ubiquity/install/find_installables')
self.progress_region(0, 1)
fetchprogress = DebconfAcquireProgress(
self.db, 'ubiquity/install/title',
'ubiquity/install/apt_indices_starting',
'ubiquity/install/apt_indices')
cache = Cache()
if cache._depcache.broken_count > 0:
syslog.syslog(
'not installing additional packages, since there are broken '
'packages: %s' % ', '.join(broken_packages(cache)))
self.db.progress('STOP')
self.nested_progress_end()
return
for pkg in to_install:
mark_install(cache, pkg)
self.db.progress('SET', 1)
self.progress_region(1, 10)
if langpacks:
fetchprogress = DebconfAcquireProgress(
self.db, 'ubiquity/langpacks/title', None,
'ubiquity/langpacks/packages')
installprogress = DebconfInstallProgress(
self.db, 'ubiquity/langpacks/title',
'ubiquity/install/apt_info')
else:
fetchprogress = DebconfAcquireProgress(
self.db, 'ubiquity/install/title', None,
'ubiquity/install/fetch_remove')
installprogress = DebconfInstallProgress(
self.db, 'ubiquity/install/title',
'ubiquity/install/apt_info',
'ubiquity/install/apt_error_install')
chroot_setup(self.target)
commit_error = None
try:
try:
if not self.commit_with_verify(cache,
fetchprogress, installprogress):
fetchprogress.stop()
installprogress.finishUpdate()
self.db.progress('STOP')
self.nested_progress_end()
return
except IOError:
for line in traceback.format_exc().split('\n'):
syslog.syslog(syslog.LOG_ERR, line)
fetchprogress.stop()
installprogress.finishUpdate()
self.db.progress('STOP')
self.nested_progress_end()
return
except SystemError, e:
for line in traceback.format_exc().split('\n'):
syslog.syslog(syslog.LOG_ERR, line)
commit_error = str(e)
finally:
chroot_cleanup(self.target)
self.db.progress('SET', 10)
cache.open(None)
if commit_error or cache._depcache.broken_count > 0:
if commit_error is None:
commit_error = ''
brokenpkgs = broken_packages(cache)
self.warn_broken_packages(brokenpkgs, commit_error)
self.db.progress('STOP')
self.nested_progress_end()
class UpdateMaker():
"""A class for making updates for OreSat Linux Updater daemon"""
def __init__(self, board: str):
"""
Parameters
----------
board: str
The board to make the update for.
"""
self._board = board
self._status_file = ""
self._board = board
self._cache = Cache(rootdir=ROOT_DIR)
self._cache.update(raise_on_error=False)
self._cache.open()
self._deb_pkgs = []
self._inst_list = []
# make sure all dir exist
Path(OLU_DIR).mkdir(parents=True, exist_ok=True)
Path(ROOT_DIR).mkdir(parents=True, exist_ok=True)
Path(DOWNLOAD_DIR).mkdir(parents=True, exist_ok=True)
Path(UPDATE_CACHE_DIR).mkdir(parents=True, exist_ok=True)
Path(STATUS_CACHE_DIR).mkdir(parents=True, exist_ok=True)
# clear download dir
for i in listdir(DOWNLOAD_DIR):
if i.endswith(".deb"):
remove(DOWNLOAD_DIR + i)
status_files = []
for i in listdir(STATUS_CACHE_DIR):
status_files.append(OLMFile(load=i))
status_files.sort()
# find latest olu status tar file
for i in status_files:
if i.name == board:
self._status_file = STATUS_CACHE_DIR + i.name
break
if self._status_file == "":
msg = "No status file for {} board in cache".format(board)
raise FileNotFoundError(msg)
# update status file
dpkg_data = read_dpkg_status_file(self._status_file)
with open(DPKG_STATUS_FILE) as fptr:
fptr.write(dpkg_data)
# TODO deal with update files that are not installed yet.
def add_packages(self, packages: list):
"""Add deb packages to be installed.
Parameters
----------
packages: list
A list of deb packages to install on the board.
"""
if packages == []:
raise ValueError("Requires a list of packages to install")
inst_deb_pkgs = []
for pkg in packages:
pkg_obj = self._cache[pkg]
pkg_obj.mark_install() # this will mark all dependencies too
# find new packages (dependencies) that are marked
for deb_pkg in self._cache:
if deb_pkg.marked_install and \
deb_pkg.name not in self._deb_pkgs:
self._deb_pkgs.append(deb_pkg.name)
inst_deb_pkgs.append(deb_pkg.name)
new_inst = Instruction(InstructionType.DPKG_INSTALL, inst_deb_pkgs)
self._inst_list.append(new_inst)
def remove_packages(self, packages: list):
"""Remove deb packages on board.
Parameters
----------
packages: list
A list of deb packages to remove on the board.
"""
if packages == []:
raise ValueError("Requires a list of packages to remove")
new_inst = Instruction(InstructionType.DPKG_REMOVE, packages)
self._inst_list.append(new_inst)
def purge_packages(self, packages: list):
"""Purge deb packages on board.
Parameters
----------
packages: list
A list of deb packages to remove on the board.
"""
if packages == []:
raise ValueError("Requires a list of packages to remove")
new_inst = Instruction(InstructionType.DPKG_PURGE, packages)
self._inst_list.append(new_inst)
def add_bash_scripts(self, bash_scipts: list):
"""Run bash scripts on the board.
Parameters
----------
bash_scipts: list
A list of bash script to run on the board.
"""
if bash_scipts == []:
raise ValueError("Requires a list of bash scipts to run")
new_inst = Instruction(InstructionType.BASH_SCRIPT, bash_scipts)
self._inst_list.append(new_inst)
def add_support_files(self, support_files: list):
"""Add a support files to update archive.
Parameters
----------
support_files: list
A list of support files to add to the update.
"""
for s_file in support_files:
if isfile(s_file):
raise ValueError(" {} was not found".format(s_file))
new_inst = Instruction(InstructionType.SUPPORT_FILE, support_files)
self._inst_list.append(new_inst)
def status(self):
"""Print the contexts of instructions list"""
for i in self._inst_list:
print(i)
def make_update_archive(self):
"""Make the update archive"""
# download deb files
self._cache.fetch_archives()
# replace package name with deb filepath in instruction obj
for inst in self._inst_list:
if not inst.type == InstructionType.DPKG_INSTALL:
continue
for i in range(len(inst.items)):
found = False
for deb_file in listdir(DOWNLOAD_DIR):
if not deb_file.endswith(".deb"):
continue
if deb_file.startswith(inst.items[i]+"_"):
inst.items[i] = DOWNLOAD_DIR + deb_file
found = True
break
if found is True:
break
print("Making tar")
update_file = create_update_file(self._board, self._inst_list, "./")
print("{} was made".format(update_file))
class _AptChangelog():
def __init__(self, interactive:bool=False):
self.interactive = interactive
# constants
# apt uses MB rather than MiB, so let's stay consistent
self.MB = 1000 ** 2
# downloads larger than this require confirmation or fail
self.max_download_size_default = 1.5 * self.MB
self.max_download_size = self.max_download_size_default
max_download_size_msg_template = "\
To retrieve the full changelog, %s MB have to be downloaded.\n%s\
\n\
Proceed with the download?"
self.max_download_size_msg_lc = max_download_size_msg_template % ("%.1f",
"Otherwise we will try to retrieve just the last change.\n")
self.max_download_size_msg = max_download_size_msg_template % ("%.1f","")
self.max_download_size_msg_unknown = max_download_size_msg_template % ("an unknown amount of", "")
self.apt_cache = None
self.apt_cache_date = None
self.candidate = None
# get apt's configuration
apt_pkg.init_config()
if apt_pkg.config.exists("Acquire::Changelogs::URI::Origin"):
self.apt_origins = apt_pkg.config.subtree("Acquire::Changelogs::URI::Origin")
else:
self.apt_origins = None
if apt_pkg.config.exists("Dir::Cache::pkgcache"):
self.apt_cache_path = apt_pkg.config.find_dir("Dir::Cache")
self.pkgcache = apt_pkg.config.find_file("Dir::Cache::pkgcache")
else:
self.apt_cache = "invalid"
if (self.apt_cache or
not os.path.isdir(self.apt_cache_path) or
not os.path.isfile(self.pkgcache)
):
print("E: Invalid APT configuration found, try to run `apt update` first",
file=sys.stderr)
self.close(99)
def get_cache_date(self):
if os.path.isfile(self.pkgcache):
return os.path.getmtime(self.pkgcache)
return None
def refresh_cache(self):
cache_date = self.get_cache_date()
if not self.apt_cache:
self.apt_cache = Cache()
self.apt_cache_date = cache_date
elif cache_date != self.apt_cache_date:
self.apt_cache.open(None)
self.apt_cache_date = cache_date
def drop_cache(self):
if self.candidate:
self.candidate = None
self.apt_cache = None
def get_changelog(self, pkg_name:str, no_local:bool=False):
self.refresh_cache()
self.candidate = self.parse_package_metadata(pkg_name)
# parse the package's origin
if not self.candidate.downloadable:
origin = "local_package"
elif self.candidate.origin == "linuxmint":
origin = "linuxmint"
elif self.candidate.origin.startswith("LP-PPA-"):
origin = "LP-PPA"
elif self.apt_origins and self.candidate.origin in self.apt_origins.list():
origin = "APT"
else:
origin = "unsupported"
# Check for changelog of installed package first
has_local_changelog = False
uri = None
if not no_local and self.candidate.is_installed:
if _DEBUG: print("Package is installed...")
uri = self.get_changelog_from_filelist(
self.candidate.installed_files, local=True)
# Ubuntu kernel workarounds
if self.candidate.origin == "Ubuntu":
if self.candidate.source_name == "linux-signed":
uri = uri.replace("linux-image","linux-modules")
if self.candidate.source_name == "linux-meta":
uri = None
if uri and not os.path.isfile(uri):
uri = None
# Do nothing if local changelog exists
if uri:
has_local_changelog = True
# all origins that APT supports
elif origin == 'APT':
uri = self.get_apt_changelog_uri(
self.apt_origins.get(self.candidate.origin))
r = self.check_url(uri)
if not r:
self.exit_on_fail(2)
# Linux Mint repo
elif origin == 'linuxmint':
# Mint repos don't have .debian.tar.xz files, only full packages, so
# check the package cache first
base_uri, _ = os.path.split(self.candidate.uri)
r, uri = self.get_changelog_uri(base_uri)
if not r:
# fall back to last change info for the source package
# Mint's naming scheme seems to be using amd64 unless source
# is i386 only, we always check amd64 first
base_uri = "http://packages.linuxmint.com/dev/%s_%s_%s.changes"
uri = base_uri % (self.candidate.source_name,
self.candidate.source_version, "amd64")
r = self.check_url(uri, False)
if not r:
uri = base_uri % (self.candidate.source_name,
self.candidate.source_version, "i386")
r = self.check_url(uri, False)
if not r:
self.exit_on_fail(3)
# Launchpad PPA
elif origin == 'LP-PPA':
ppa_owner, ppa_name, _ = \
self.candidate.uri.split("ppa.launchpad.net/")[1].split("/", 2)
base_uri = "http://ppa.launchpad.net/%s/%s/ubuntu/pool/main/{self.source_prefix()}/%s" % (ppa_owner, ppa_name, self.candidate.source_name)
r, uri = self.get_changelog_uri(base_uri)
if not r:
# fall back to last change info only
uri = "https://launchpad.net/~%s/+archive/ubuntu/%s/+files/%s_%s_source.changes" % (ppa_owner, ppa_name, self.candidate.source_name, self.candidate.source_version)
r = self.check_url(uri, False)
if not r:
self.exit_on_fail(4)
# Not supported origin
elif origin == 'unsupported':
if _DEBUG: print("Unsupported Package")
base_uri, _ = os.path.split(self.candidate.uri)
r, uri = self.get_changelog_uri(base_uri)
if not r:
self.exit_on_fail(5)
# Locally installed package without local changelog or remote
# source, hope it's cached and contains a changelog
elif origin == 'local_package':
uri = self.apt_cache_path + self.candidate.filename
if not os.path.isfile(uri):
self.exit_on_fail(6)
# Changelog downloading, extracting and processing:
changelog = ""
# local changelog
if has_local_changelog and not no_local:
if _DEBUG: print("Using local changelog:",uri)
try:
filename = os.path.basename(uri)
# determine file type by name/extension
# as per debian policy 4.4 the encoding must be UTF-8
# as per policy 12.7 the name must be changelog.Debian.gz or
# changelog.gz (deprecated)
if filename.lower().endswith('.gz'):
changelog = gzip.open(uri,'r').read().decode('utf-8')
elif filename.lower().endswith('.xz'):
# just in case / future proofing
changelog = lzma.open(uri,'r').read().decode('utf-8')
elif filename.lower() == 'changelog':
changelog = open(uri, 'r').read().encode().decode('utf-8')
else:
raise ValueError('Unknown changelog format')
except Exception as e:
_generic_exception_handler(e)
self.exit_on_fail(1)
# APT-format changelog, download directly
# - unfortunately this is slow since the servers support no compression
elif origin == "APT":
if _DEBUG: print("Downloading: %s (%.2f MB)" % (uri, r.length / self.MB))
changelog = r.text
r.close()
# last change changelog, download directly
elif uri.endswith('.changes'):
if _DEBUG: print("Downloading: %s (%.2f MB)" % (uri, r.length / self.MB))
changes = r.text.split("Changes:")[1].split("Checksums")[0].split("\n")
r.close()
for change in changes:
change = change.strip()
if change:
if change == ".":
change = ""
changelog += change + "\n"
# compressed binary source, download and extract changelog
else:
source_is_cache = uri.startswith(self.apt_cache_path)
if _DEBUG: print("Using cached package:" if source_is_cache else
"Downloading: %s (%.2f MB)" % (uri, r.length / self.MB))
try:
if not source_is_cache:
# download stream to temporary file
tmpFile = tempfile.NamedTemporaryFile(prefix="apt-changelog-")
if self.interactive and r.length:
# download chunks with progress indicator
recv_length = 0
blocks = 60
for data in r.iter_content(chunk_size=16384):
recv_length += len(data)
tmpFile.write(data)
recv_pct = recv_length / r.length
recv_blocks = int(blocks * recv_pct)
print("\r[%(progress)s%(spacer)s] %(percentage).1f%%" %
{
"progress": "=" * recv_blocks,
"spacer": " " * (blocks - recv_blocks),
"percentage": recv_pct * 100
}, end="", flush=True)
# clear progress bar when done
print("\r" + " " * (blocks + 10), end="\r", flush=True)
else:
# no content-length or non-interactive, download in one go
# up to the configured max_download_size, ask only when
# exceeded
r.raw.decode_content = True
size = 0
size_exceeded = False
while True:
buf = r.raw.read(16*1024)
if not size_exceeded:
size += len(buf)
if size > self.max_download_size:
if not self.user_confirm(self.max_download_size_msg_unknown):
r.close()
tmpFile.close()
return ""
else:
size_exceeded = True
if not buf:
break
tmpFile.write(buf)
r.close()
tmpFile.seek(0)
if uri.endswith(".deb"):
# process .deb file
if source_is_cache:
f = uri
else:
f = tmpFile.name
# We could copy the downloaded .deb files to the apt
# cache here but then we'd need to run the script elevated:
# shutil.copy(f, self.apt_cache_path + os.path.basename(uri))
deb = DebPackage(f)
changelog_file = self.get_changelog_from_filelist(deb.filelist)
if changelog_file:
changelog = deb.data_content(changelog_file)
if changelog.startswith('Automatically decompressed:'):
changelog = changelog[29:]
else:
raise ValueError('Malformed Debian package')
elif uri.endswith(".diff.gz"):
# Ubuntu partner repo has .diff.gz files,
# we can extract a changelog from that
data = gzip.open(tmpFile.name, "r").read().decode('utf-8')
additions = data.split("+++")
for addition in additions:
lines = addition.split("\n")
if "/debian/changelog" in lines[0]:
for line in lines[2:]:
if line.startswith("+"):
changelog += "%s\n" % line[1:]
else:
break
if not changelog:
raise ValueError('No changelog in .diff.gz')
else:
# process .tar.xz file
with tarfile.open(fileobj=tmpFile, mode="r:xz") as tar:
changelog_file = self.get_changelog_from_filelist(
[s.name for s in tar.getmembers() if s.type in (b"0", b"2")])
if changelog_file:
changelog = tar.extractfile(changelog_file).read().decode()
else:
raise ValueError('No changelog in source package')
except Exception as e:
_generic_exception_handler(e)
self.exit_on_fail(520)
if 'tmpFile' in vars():
try:
tmpFile.close()
except Exception as e:
_generic_exception_handler(e)
# ALL DONE
return changelog
def parse_package_metadata(self, pkg_name:str):
""" Creates the self.candidate object based on package name=version/release
Wildcard matching is only used for version and release, and only the
first match is processed.
"""
# parse =version declaration
if "=" in pkg_name:
(pkg_name, pkg_version) = pkg_name.split("=", 1)
pkg_release = None
# parse /release declaration (only if no version specified)
elif "/" in pkg_name:
(pkg_name, pkg_release) = pkg_name.split("/", 1)
pkg_version = None
else:
pkg_version = None
pkg_release = None
# check if pkg_name exists
# unlike apt no pattern matching, a single exact match only
if pkg_name in self.apt_cache:
pkg = self.apt_cache[pkg_name]
else:
print("E: Unable to locate package %s" % pkg_name, file=sys.stderr)
self.close(13)
# get package data
_candidate = None
candidate = None
if pkg_release or pkg_version:
match_found = False
for _pkg in pkg.versions:
if pkg_version:
if fnmatch.fnmatch(_pkg.version, pkg_version):
match_found = True
else:
for _origin in _pkg.origins:
if fnmatch.fnmatch(_origin.archive, pkg_release):
match_found = True
if match_found:
_candidate = _pkg
break
if not match_found:
if pkg_release:
print('E: Release "%s" is unavailable for "%s"' % (pkg_release, pkg.name),
file=sys.stderr)
else:
print('E: Version "%s" is unavailable for "%s"' % (pkg_version, pkg.name),
file=sys.stderr)
self.close(14)
else:
_candidate = pkg.candidate
candidate = _Package(
version = _candidate.version,
name = _candidate.package.name,
fullname = None,
architecture = pkg.architecture,
source_name = _candidate.source_name,
source_version = _candidate.source_version,
uri = _candidate.uri,
filename = os.path.basename(_candidate.filename),
origin = _candidate.origins[0].origin,
component = _candidate.origins[0].component,
downloadable = _candidate.downloadable,
is_installed = _candidate.is_installed,
dependencies = _candidate.dependencies
)
if candidate.is_installed:
candidate.installed_files = pkg.installed_files
candidate.source_version_raw = candidate.source_version
if ":" in candidate.source_version:
candidate.source_version = candidate.source_version.split(":", 1)[1]
return candidate
def check_url(self, url:str, check_size:bool=True, stream:bool=True,
msg:str=None):
""" True if url can be downloaded and fits size requirements """
if _DEBUG: print("Checking:", url)
try:
_r = requests.get(url, stream=stream, timeout=5)
except Exception as e:
_generic_exception_handler(e)
else:
if _r:
if not _r.encoding:
_r.encoding = "utf-8"
length = _r.headers.get("Content-Length")
if length:
_r.length = int(length)
else:
_r.length = 0
if (not check_size or not
(check_size and _r.length > self.max_download_size and not
self.user_confirm(
(self.max_download_size_msg_lc if not msg else msg) %
(_r.length / self.MB))
)):
return _r
if '_r' in vars():
_r.close()
return False
@staticmethod
def close(err:int=0):
""" Exit """
sys.exit(err)
def exit_on_fail(self, err:int=404):
""" Prints error message and calls self.close() """
try:
details = "Changelog unavailable for %s=%s" % (self.candidate.source_name, self.candidate.source_version_raw)
except AttributeError:
details = ""
print("E: Failed to fetch changelog. %s" % details, file=sys.stderr)
self.close(err)
@staticmethod
def strtobool (val):
val = val.lower()
if val in ('y', 'yes'):
return True
elif val in ('n', 'no'):
return False
else:
raise ValueError("Invalid response value %s" % val)
def user_confirm(self, q:str):
""" returns bool (always False in non-interactive mode) """
if not self.interactive:
if _DEBUG: print("Maximum size exceeded, skipping in non-interactive mode")
return False
print("%s [y/n] " % q, end="")
while True:
try:
response = self.strtobool(input())
print("")
return response
except ValueError:
print("Invalid response. Try again [y/n]: ", end="")
except KeyboardInterrupt:
pass
def get_deb_or_tar(self, uri_tar:str=None):
""" Returns request and URI of the preferred source
The choice is made based on availability and size. If .deb is smaller
than comparison_trigger_size, or if check_tar is False, then .deb is
always selected.
"""
comparison_trigger_size = 50000
r_deb = self.check_url(self.candidate.uri, False)
if r_deb:
if uri_tar and r_deb.length > comparison_trigger_size:
# try for .tar.xz
r_tar = self.check_url(uri_tar, False)
# validate and compare sizes
if r_tar and r_tar.length < r_deb.length:
_r = r_tar
r_deb.close()
else:
_r = r_deb
if r_tar:
r_tar.close()
else:
_r = r_deb
if (not _r.length > self.max_download_size or
self.user_confirm(self.max_download_size_msg_lc %
(_r.length / self.MB))
):
return (_r, _r.url)
return (False, "")
def get_changelog_from_filelist(self, filelist:list, local:bool=False):
""" Returns hopefully the correct "changelog" or an empty string.
We should not need to be searching because the debian policy says it
must be at debian/changelog for source packages but not all seem to
adhere to the policy:
https://www.debian.org/doc/debian-policy/ch-source.html#debian-changelog-debian-changelog
"""
files = [s for s in filelist if "changelog" in s.lower()]
if local:
testpath = "/usr/share/doc/%s/changelog" % self.candidate.name
for item in files:
if item.lower().startswith(testpath):
return item
else:
testpath = "debian/changelog"
if testpath in files:
return testpath
testpath = "recipe/debian/changelog"
if testpath in files:
return testpath
testpath = "usr/share/doc/%s/changelog" % self.candidate.name
for item in files:
if item.lower().startswith(testpath):
return item
# no hits in the standard locations, let's try our luck in
# random locations at the risk of getting the wrong file
for item in files:
if os.path.basename(item).lower().startswith("changelog"):
return item
return None
def get_apt_changelog_uri(self, uri_template:str):
""" Returns URI based on provided apt changelog URI template.
Emulates apt's std::string pkgAcqChangelog::URI
The template must contain the @CHANGEPATH@ variable, which will
be expanded to
COMPONENT/SRC/SRCNAME/SRCNAME_SRCVER
Component is omitted for releases without one (= flat-style
repositories).
"""
source_version = self.candidate.source_version
def get_kernel_version_from_meta_package(pkg):
for dependency in pkg.dependencies:
if not dependency.target_versions or not dependency.rawtype == "Depends":
if _DEBUG: print("W: Kernel dependency not found:", dependency)
return None
deppkg = dependency.target_versions[0]
if deppkg.source_name in ("linux", "linux-signed"):
return deppkg.source_version
if deppkg.source_name.startswith("linux-meta"):
_pkg = self.parse_package_metadata(str(deppkg))
return get_kernel_version_from_meta_package(_pkg)
return None
# Ubuntu kernel meta package workaround
if self.candidate.origin == "Ubuntu" and \
self.candidate.source_name.startswith("linux-meta"):
_source_version = get_kernel_version_from_meta_package(self.candidate)
if _source_version:
source_version = _source_version
self.candidate.source_name = "linux"
# Ubuntu signed kernel workaround
if self.candidate.origin == "Ubuntu" and \
self.candidate.source_name == "linux-signed":
self.candidate.source_name = "linux"
# XXX: Debian does not seem to reliably keep changelogs for previous
# (kernel) versions, so should we always look for the latest
# version instead on Debian? apt does not do this but the
# packages.debian.org website shows the latest version in the
# selected archive
# strip epoch
if ":" in source_version:
source_version = source_version.split(":", 1)[1]
# the path is: COMPONENT/SRC/SRCNAME/SRCNAME_SRCVER, e.g.
# main/a/apt/apt_1.1 or contrib/liba/libapt/libapt_2.0
return uri_template.replace('@CHANGEPATH@',
"%(component)s%(source_prefix)s/%(source_name)s/%(source_name)s_%(source_version)s" %
{
"component": self.candidate.component + "/" if \
self.candidate.component and \
self.candidate.component != "" else "",
"source_prefix": self.source_prefix(),
"source_name": self.candidate.source_name,
"source_version": source_version
})
def source_prefix(self, source_name:str=None):
""" Return prefix used for build repository URL """
if not source_name:
source_name = self.candidate.source_name
return source_name[0] if not source_name.startswith("lib") else \
source_name[:4]
def parse_dsc(self, url:str):
""" Returns filename or None """
_r = self.check_url(url, False, False)
if _r:
target = ""
lines = _r.text.split("Files:", 1)[1].split(":", 1)[0].split("-----BEGIN", 1)[0].split("\n")
target = [s.strip() for s in lines if s.strip().lower().endswith('.debian.tar.xz')]
if not target:
target = [s.strip() for s in lines if s.strip().lower().endswith('.diff.gz')]
if not target:
target = [s.strip() for s in lines if s.strip().lower().endswith('.tar.xz')]
# don't even test for .tar.gz, it will be too big compared to the .deb
# if not target:
# target = [s.strip() for s in lines if s.strip().lower().endswith('.tar.gz')]
if target:
return target[0].split()[-1]
elif _DEBUG: print(".dsc parse error for", url)
return None
def get_changelog_uri(self, base_uri:str):
""" Tries to find a changelog in files listed in .dsc, locally cached
packages as well as the remote .deb file
Returns r and uri
"""
uri = None
# XXX: For APT sources we could just read the apt_pkg.SourceRecords()
# directly, if available, which it is not for most users, so
# probably not worth it
target_filename = self.parse_dsc("%s/%s_%s.dsc" % (base_uri, self.candidate.source_name, self.candidate.source_version))
# get .debian.tar.xz or .diff.gz as a priority as the smallest options
if (base_uri and target_filename and (
target_filename.lower().endswith('.debian.tar.xz') or
target_filename.lower().endswith('.diff.gz')
)):
uri = "%s/%s" % (base_uri, target_filename)
target_filename = None
r = self.check_url(uri, msg = self.max_download_size_msg)
else:
r = None
if not r:
# fall back to cached local package
uri = self.apt_cache_path + self.candidate.filename
if not os.path.isfile(uri):
# cache miss, download the full source package or the .deb,
# depending on size and availability
if target_filename:
uri_tar = "%s/%s" % (base_uri, target_filename)
else:
uri_tar = None
r, uri = self.get_deb_or_tar(uri_tar)
return (r, uri)
class Transition(dbus.service.Object):
def __init__(self, conn=None, object_path=None, bus_name=None):
super().__init__(conn, object_path, bus_name)
self.dbus_info = None
self.polkit = None
self.enforce_polkit = True
self.cache = Cache()
self.lock = None
self.apt_lock = None
@dbus.service.method(
'org.pop_os.transition_system.Interface',
in_signature='', out_signature='b',
sender_keyword='sender', connection_keyword='conn'
)
def obtain_lock(self, sender=None, conn=None):
""" Lock the package system. """
self._check_polkit_privilege(
sender, conn, 'org.pop_os.transition_system.removedebs'
)
print('Obtaining Package manager lock')
try:
self.lock = apt_pkg.get_lock('/var/lib/dpkg/lock-frontend', True)
self.apt_lock = apt_pkg.get_lock('/var/lib/apt/lists/lock', True)
print('Lock obtained')
return True
except apt_pkg.Error:
print('Could not obtain lock')
self.lock = None
self.apt_lock = None
return False
@dbus.service.method(
'org.pop_os.transition_system.Interface',
in_signature='', out_signature='b',
sender_keyword='sender', connection_keyword='conn'
)
def release_lock(self, sender=None, conn=None):
""" Unlock the package system. """
self._check_polkit_privilege(
sender, conn, 'org.pop_os.transition_system.removedebs'
)
print('Releasing package manager lock')
try:
os.close(self.lock)
os.close(self.apt_lock)
self.lock = None
self.apt_lock = None
print('Lock released')
return True
except:
print('Could not release lock')
return False
@dbus.service.method(
'org.pop_os.transition_system.Interface',
in_signature='', out_signature='b',
sender_keyword='sender', connection_keyword='conn'
)
def open_cache(self, sender=None, conn=None):
""" Open the package cache. """
self._check_polkit_privilege(
sender, conn, 'org.pop_os.transition_system.removedebs'
)
if self.lock and self.apt_lock:
print('Opening package cache')
self.cache.update()
self.cache.open()
print('Cache open')
return True
print('No lock, cannot open cache')
return False
@dbus.service.method(
'org.pop_os.transition_system.Interface',
in_signature='', out_signature='b',
sender_keyword='sender', connection_keyword='conn'
)
def commit_changes(self, sender=None, conn=None):
""" Commit changes to the cache. """
self._check_polkit_privilege(
sender, conn, 'org.pop_os.transition_system.removedebs'
)
if self.lock and self.apt_lock:
self.cache.commit()
print('Committed changes to cache')
return True
print('No lock, Cannot commit changes')
return False
@dbus.service.method(
'org.pop_os.transition_system.Interface',
in_signature='', out_signature='b',
sender_keyword='sender', connection_keyword='conn'
)
def close_cache(self, sender=None, conn=None):
""" Close the package cache. """
self._check_polkit_privilege(
sender, conn, 'org.pop_os.transition_system.removedebs'
)
if self.lock and self.apt_lock:
self.cache.close()
print('Package cache closed')
return True
print('No lock, cannot close cache')
return False
@dbus.service.method(
'org.pop_os.transition_system.Interface',
in_signature='s', out_signature='s',
sender_keyword='sender', connection_keyword='conn'
)
def remove_package(self, package, sender=None, conn=None):
""" Mark a package for removal."""
self._check_polkit_privilege(
sender, conn, 'org.pop_os.transition_system.removedebs'
)
if self.lock and self.apt_lock:
print(f'Marking {package} for removal')
try:
pkg = self.cache[package]
pkg.mark_delete()
return pkg.name
except:
print(f'Could not mark {package} for removal')
return ''
print('No lock, cannot mark packages')
return ''
@dbus.service.method(
'org.pop_os.transition_system.Interface',
in_signature='', out_signature='',
sender_keyword='sender', connection_keyword='conn'
)
def exit(self, sender=None, conn=None):
if self.lock and self.apt_lock:
self.close_cache()
self.release_lock()
mainloop.quit()
def _check_polkit_privilege(self, sender, conn, privilege):
'''Verify that sender has a given PolicyKit privilege.
sender is the sender's (private) D-BUS name, such as ":1:42"
(sender_keyword in @dbus.service.methods). conn is
the dbus.Connection object (connection_keyword in
@dbus.service.methods). privilege is the PolicyKit privilege string.
This method returns if the caller is privileged, and otherwise throws a
PermissionDeniedByPolicy exception.
'''
if sender is None and conn is None:
# Called locally, not through D-Bus
return
if not self.enforce_polkit:
# For testing
return
if self.dbus_info is None:
self.dbus_info = dbus.Interface(conn.get_object('org.freedesktop.DBus',
'/org/freedesktop/DBus/Bus', False), 'org.freedesktop.DBus')
pid = self.dbus_info.GetConnectionUnixProcessID(sender)
if self.polkit is None:
self.polkit = dbus.Interface(dbus.SystemBus().get_object(
'org.freedesktop.PolicyKit1',
'/org/freedesktop/PolicyKit1/Authority', False),
'org.freedesktop.PolicyKit1.Authority'
)
try:
(is_auth, _, details) = self.polkit.CheckAuthorization(
('unix-process', {'pid': dbus.UInt32(pid, variant_level=1),
'start-time': dbus.UInt64(0, variant_level=1)}),
privilege, {'': ''}, dbus.UInt32(1), '', timeout=600
)
except dbus.DBusException as e:
if e._dbus_error_name == 'org.freedesktop.DBus.Error.ServiceUnknown':
# polkitd timed out, connect again
self.polkit = None
return self._check_polkit_privilege(sender, conn, privilege)
else:
raise
if not is_auth:
raise PermissionDeniedByPolicy(privilege)
class _AptChangelog():
def __init__(self, interactive:bool=False):
self.interactive = interactive
# constants
# apt uses MB rather than MiB, so let's stay consistent
self.MB = 1000 ** 2
# downloads larger than this require confirmation or fail
self.max_download_size_default = 1.5 * self.MB
self.max_download_size = self.max_download_size_default
max_download_size_msg_template = "\
To retrieve the full changelog, %s MB have to be downloaded.\n%s\
\n\
Proceed with the download?"
self.max_download_size_msg_lc = max_download_size_msg_template % ("%.1f",
"Otherwise we will try to retrieve just the last change.\n")
self.max_download_size_msg = max_download_size_msg_template % ("%.1f","")
self.max_download_size_msg_unknown = max_download_size_msg_template % ("an unknown amount of", "")
self.apt_cache = None
self.apt_cache_date = None
self.candidate = None
# get apt's configuration
apt_pkg.init_config()
if apt_pkg.config.exists("Acquire::Changelogs::URI::Origin"):
self.apt_origins = apt_pkg.config.subtree("Acquire::Changelogs::URI::Origin")
else:
self.apt_origins = None
if apt_pkg.config.exists("Dir::Cache::pkgcache"):
self.apt_cache_path = apt_pkg.config.find_dir("Dir::Cache")
self.pkgcache = apt_pkg.config.find_file("Dir::Cache::pkgcache")
else:
self.apt_cache = "invalid"
if (self.apt_cache or
not os.path.isdir(self.apt_cache_path) or
not os.path.isfile(self.pkgcache)
):
print("E: Invalid APT configuration found, try to run `apt update` first",
file=sys.stderr)
self.close(99)
def get_cache_date(self):
if os.path.isfile(self.pkgcache):
return os.path.getmtime(self.pkgcache)
return None
def refresh_cache(self):
cache_date = self.get_cache_date()
if not self.apt_cache:
self.apt_cache = Cache()
self.apt_cache_date = cache_date
elif cache_date != self.apt_cache_date:
self.apt_cache.open(None)
self.apt_cache_date = cache_date
def drop_cache(self):
if self.candidate:
self.candidate = None
self.apt_cache = None
def get_changelog(self, pkg_name:str, no_local:bool=False):
self.refresh_cache()
self.candidate = self.parse_package_metadata(pkg_name)
# parse the package's origin
if not self.candidate.downloadable:
origin = "local_package"
elif self.candidate.origin == "linuxmint":
origin = "linuxmint"
elif self.candidate.origin.startswith("LP-PPA-"):
origin = "LP-PPA"
elif self.apt_origins and self.candidate.origin in self.apt_origins.list():
origin = "APT"
else:
origin = "unsupported"
# Check for changelog of installed package first
has_local_changelog = False
uri = None
if not no_local and self.candidate.is_installed:
if _DEBUG: print("Package is installed...")
uri = self.get_changelog_from_filelist(
self.candidate.installed_files, local=True)
# Ubuntu kernel workarounds
if self.candidate.origin == "Ubuntu":
if self.candidate.source_name == "linux-signed":
uri = uri.replace("linux-image","linux-modules")
if self.candidate.source_name == "linux-meta":
uri = None
if uri and not os.path.isfile(uri):
uri = None
# Do nothing if local changelog exists
if uri:
has_local_changelog = True
# all origins that APT supports
elif origin == 'APT':
uri = self.get_apt_changelog_uri(
self.apt_origins.get(self.candidate.origin))
r = self.check_url(uri)
if not r:
self.exit_on_fail(2)
# Linux Mint repo
elif origin == 'linuxmint':
# Mint repos don't have .debian.tar.xz files, only full packages, so
# check the package cache first
base_uri, _ = os.path.split(self.candidate.uri)
r, uri = self.get_changelog_uri(base_uri)
if not r:
# fall back to last change info for the source package
# Mint's naming scheme seems to be using amd64 unless source
# is i386 only, we always check amd64 first
base_uri = "http://packages.linuxmint.com/dev/%s_%s_%s.changes"
uri = base_uri % (self.candidate.source_name,
self.candidate.source_version, "amd64")
r = self.check_url(uri, False)
if not r:
uri = base_uri % (self.candidate.source_name,
self.candidate.source_version, "i386")
r = self.check_url(uri, False)
if not r:
self.exit_on_fail(3)
# Launchpad PPA
elif origin == 'LP-PPA':
ppa_owner, ppa_name, _ = \
self.candidate.uri.split("ppa.launchpad.net/")[1].split("/", 2)
base_uri = "http://ppa.launchpad.net/%(owner)s/%(name)s/ubuntu/pool/main/%(source_prefix)s/%(source_name)s" % \
{
"owner": ppa_owner,
"name": ppa_name,
"source_prefix": self.source_prefix(),
"source_name": self.candidate.source_name
}
r, uri = self.get_changelog_uri(base_uri)
if not r:
# fall back to last change info only
uri = "https://launchpad.net/~%(owner)s/+archive/ubuntu/%(name)s/+files/%(source_name)s_%(source_version)s_source.changes" % \
{
"owner" : ppa_owner,
"name": ppa_name,
"source_name": self.candidate.source_name,
"source_version": self.candidate.source_version
}
r = self.check_url(uri, False)
if not r:
self.exit_on_fail(4)
# Not supported origin
elif origin == 'unsupported':
if _DEBUG: print("Unsupported Package")
base_uri, _ = os.path.split(self.candidate.uri)
r, uri = self.get_changelog_uri(base_uri)
if not r:
self.exit_on_fail(5)
# Locally installed package without local changelog or remote
# source, hope it's cached and contains a changelog
elif origin == 'local_package':
uri = self.apt_cache_path + self.candidate.filename
if not os.path.isfile(uri):
self.exit_on_fail(6)
# Changelog downloading, extracting and processing:
changelog = ""
# local changelog
if has_local_changelog and not no_local:
if _DEBUG: print("Using local changelog:",uri)
try:
filename = os.path.basename(uri)
# determine file type by name/extension
# as per debian policy 4.4 the encoding must be UTF-8
# as per policy 12.7 the name must be changelog.Debian.gz or
# changelog.gz (deprecated)
if filename.lower().endswith('.gz'):
changelog = gzip.open(uri,'r').read().decode('utf-8')
elif filename.lower().endswith('.xz'):
# just in case / future proofing
changelog = lzma.open(uri,'r').read().decode('utf-8')
elif filename.lower() == 'changelog':
changelog = open(uri, 'r').read().encode().decode('utf-8')
else:
raise ValueError('Unknown changelog format')
except Exception as e:
_generic_exception_handler(e)
self.exit_on_fail(1)
# APT-format changelog, download directly
# - unfortunately this is slow since the servers support no compression
elif origin == "APT":
if _DEBUG: print("Downloading: %s (%.2f MB)" % (uri, r.length / self.MB))
changelog = r.text
r.close()
# last change changelog, download directly
elif uri.endswith('.changes'):
if _DEBUG: print("Downloading: %s (%.2f MB)" % (uri, r.length / self.MB))
changes = r.text.split("Changes:")[1].split("Checksums")[0].split("\n")
r.close()
for change in changes:
change = change.strip()
if change:
if change == ".":
change = ""
changelog += change + "\n"
# compressed binary source, download and extract changelog
else:
source_is_cache = uri.startswith(self.apt_cache_path)
if _DEBUG: print("Using cached package:" if source_is_cache else
"Downloading: %s (%.2f MB)" % (uri, r.length / self.MB))
try:
if not source_is_cache:
# download stream to temporary file
tmpFile = tempfile.NamedTemporaryFile(prefix="apt-changelog-")
if self.interactive and r.length:
# download chunks with progress indicator
recv_length = 0
blocks = 60
for data in r.iter_content(chunk_size=16384):
recv_length += len(data)
tmpFile.write(data)
recv_pct = recv_length / r.length
recv_blocks = int(blocks * recv_pct)
print("\r[%(progress)s%(spacer)s] %(percentage).1f%%" %
{
"progress": "=" * recv_blocks,
"spacer": " " * (blocks - recv_blocks),
"percentage": recv_pct * 100
}, end="", flush=True)
# clear progress bar when done
print("\r" + " " * (blocks + 10), end="\r", flush=True)
else:
# no content-length or non-interactive, download in one go
# up to the configured max_download_size, ask only when
# exceeded
r.raw.decode_content = True
size = 0
size_exceeded = False
while True:
buf = r.raw.read(16*1024)
if not size_exceeded:
size += len(buf)
if size > self.max_download_size:
if not self.user_confirm(self.max_download_size_msg_unknown):
r.close()
tmpFile.close()
return ""
else:
size_exceeded = True
if not buf:
break
tmpFile.write(buf)
r.close()
tmpFile.seek(0)
if uri.endswith(".deb"):
# process .deb file
if source_is_cache:
f = uri
else:
f = tmpFile.name
# We could copy the downloaded .deb files to the apt
# cache here but then we'd need to run the script elevated:
# shutil.copy(f, self.apt_cache_path + os.path.basename(uri))
deb = DebPackage(f)
changelog_file = self.get_changelog_from_filelist(deb.filelist)
if changelog_file:
changelog = deb.data_content(changelog_file)
if changelog.startswith('Automatically decompressed:'):
changelog = changelog[29:]
else:
raise ValueError('Malformed Debian package')
elif uri.endswith(".diff.gz"):
# Ubuntu partner repo has .diff.gz files,
# we can extract a changelog from that
data = gzip.open(tmpFile.name, "r").read().decode('utf-8')
additions = data.split("+++")
for addition in additions:
lines = addition.split("\n")
if "/debian/changelog" in lines[0]:
for line in lines[2:]:
if line.startswith("+"):
changelog += "%s\n" % line[1:]
else:
break
if not changelog:
raise ValueError('No changelog in .diff.gz')
else:
# process .tar.xz file
with tarfile.open(fileobj=tmpFile, mode="r:xz") as tar:
changelog_file = self.get_changelog_from_filelist(
[s.name for s in tar.getmembers() if s.type in (b"0", b"2")])
if changelog_file:
changelog = tar.extractfile(changelog_file).read().decode()
else:
raise ValueError('No changelog in source package')
except Exception as e:
_generic_exception_handler(e)
self.exit_on_fail(520)
if 'tmpFile' in vars():
try:
tmpFile.close()
except Exception as e:
_generic_exception_handler(e)
# ALL DONE
return changelog
def parse_package_metadata(self, pkg_name:str):
""" Creates the self.candidate object based on package name=version/release
Wildcard matching is only used for version and release, and only the
first match is processed.
"""
# parse =version declaration
if "=" in pkg_name:
(pkg_name, pkg_version) = pkg_name.split("=", 1)
pkg_release = None
# parse /release declaration (only if no version specified)
elif "/" in pkg_name:
(pkg_name, pkg_release) = pkg_name.split("/", 1)
pkg_version = None
else:
pkg_version = None
pkg_release = None
# check if pkg_name exists
# unlike apt no pattern matching, a single exact match only
if pkg_name in self.apt_cache.keys():
pkg = self.apt_cache[pkg_name]
else:
print("E: Unable to locate package %s" % pkg_name, file=sys.stderr)
self.close(13)
# get package data
_candidate = None
candidate = None
if pkg_release or pkg_version:
match_found = False
for _pkg in pkg.versions:
if pkg_version:
if fnmatch.fnmatch(_pkg.version, pkg_version):
match_found = True
else:
for _origin in _pkg.origins:
if fnmatch.fnmatch(_origin.archive, pkg_release):
match_found = True
if match_found:
_candidate = _pkg
break
if not match_found:
if pkg_release:
print('E: Release "%s" is unavailable for "%s"' %
(pkg_release, pkg.name), file=sys.stderr)
else:
print('E: Version "%s" is unavailable for "%s"' %
(pkg_version, pkg.name), file=sys.stderr)
self.close(14)
else:
_candidate = pkg.candidate
candidate = _Package(
version = _candidate.version,
name = _candidate.package.name,
fullname = None,
architecture = pkg.architecture,
source_name = _candidate.source_name,
source_version = _candidate.source_version,
uri = _candidate.uri,
filename = os.path.basename(_candidate.filename),
origin = _candidate.origins[0].origin,
component = _candidate.origins[0].component,
downloadable = _candidate.downloadable,
is_installed = _candidate.is_installed,
dependencies = _candidate.dependencies
)
if candidate.is_installed:
candidate.installed_files = pkg.installed_files
candidate.source_version_raw = candidate.source_version
if ":" in candidate.source_version:
candidate.source_version = candidate.source_version.split(":", 1)[1]
return candidate
def check_url(self, url:str, check_size:bool=True, stream:bool=True,
msg:str=None):
""" True if url can be downloaded and fits size requirements """
if _DEBUG: print("Checking:", url)
try:
_r = requests.get(url, stream=stream, timeout=5)
except Exception as e:
_generic_exception_handler(e)
else:
if _r:
length = _r.headers.get("Content-Length")
if length:
_r.length = int(length)
else:
_r.length = 0
if (not check_size or not
(check_size and _r.length > self.max_download_size and not
self.user_confirm(
(self.max_download_size_msg_lc if not msg else msg) %
(_r.length / self.MB))
)):
return _r
if '_r' in vars():
_r.close()
return False
@staticmethod
def close(err:int=0):
""" Exit """
sys.exit(err)
def exit_on_fail(self, err:int=404):
""" Prints error message and calls self.close() """
try:
details = "Changelog unavailable for %s=%s" %\
(self.candidate.source_name, self.candidate.source_version_raw)
except AttributeError:
details = ""
print("E: Failed to fetch changelog. %s" % details, file=sys.stderr)
self.close(err)
@staticmethod
def strtobool (val):
val = val.lower()
if val in ('y', 'yes'):
return True
elif val in ('n', 'no'):
return False
else:
raise ValueError("Invalid response value %r" % (val,))
def user_confirm(self, q:str):
""" returns bool (always False in non-interactive mode) """
if not self.interactive:
if _DEBUG: print("Maximum size exceeded, skipping in non-interactive mode")
return False
print('%s [y/n] ' % q, end='')
while True:
try:
response = self.strtobool(input())
print("")
return response
except ValueError:
print('Invalid response. Try again [y/n]: ', end='')
except KeyboardInterrupt:
pass
def get_deb_or_tar(self, uri_tar:str=None):
""" Returns request and URI of the preferred source
The choice is made based on availability and size. If .deb is smaller
than comparison_trigger_size, or if check_tar is False, then .deb is
always selected.
"""
comparison_trigger_size = 50000
r_deb = self.check_url(self.candidate.uri, False)
if r_deb:
if uri_tar and r_deb.length > comparison_trigger_size:
# try for .tar.xz
r_tar = self.check_url(uri_tar, False)
# validate and compare sizes
if r_tar and r_tar.length < r_deb.length:
_r = r_tar
r_deb.close()
else:
_r = r_deb
if r_tar:
r_tar.close()
else:
_r = r_deb
if (not _r.length > self.max_download_size or
self.user_confirm(self.max_download_size_msg_lc %
(_r.length / self.MB))
):
return (_r, _r.url)
return (False, "")
def get_changelog_from_filelist(self, filelist:list, local:bool=False):
""" Returns hopefully the correct "changelog" or an empty string.
We should not need to be searching because the debian policy says it
must be at debian/changelog for source packages but not all seem to
adhere to the policy:
https://www.debian.org/doc/debian-policy/ch-source.html#debian-changelog-debian-changelog
"""
files = [s for s in filelist if "changelog" in s.lower()]
if local:
testpath = '/usr/share/doc/%s/changelog' % self.candidate.name
for item in files:
if item.lower().startswith(testpath):
return item
else:
testpath = 'debian/changelog'
if testpath in files:
return testpath
testpath = 'recipe/debian/changelog'
if testpath in files:
return testpath
testpath = 'usr/share/doc/%s/changelog' % self.candidate.name
for item in files:
if item.lower().startswith(testpath):
return item
# no hits in the standard locations, let's try our luck in
# random locations at the risk of getting the wrong file
for item in files:
if os.path.basename(item).lower().startswith("changelog"):
return item
return None
def get_apt_changelog_uri(self, uri_template:str):
""" Returns URI based on provided apt changelog URI template.
Emulates apt's std::string pkgAcqChangelog::URI
The template must contain the @CHANGEPATH@ variable, which will
be expanded to
COMPONENT/SRC/SRCNAME/SRCNAME_SRCVER
Component is omitted for releases without one (= flat-style
repositories).
"""
source_version = self.candidate.source_version
def get_kernel_version_from_meta_package(pkg):
for dependency in pkg.dependencies:
if not dependency.target_versions:
if _DEBUG: print("W: Kernel dependency not found:", dependency)
return None
deppkg = dependency.target_versions[0]
if deppkg.source_name in ("linux", "linux-signed"):
return deppkg.source_version
if deppkg.source_name == "linux-meta":
_pkg = self.parse_package_metadata(str(deppkg))
return get_kernel_version_from_meta_package(_pkg)
return None
# Ubuntu kernel meta package workaround
if (self.candidate.origin == "Ubuntu" and
self.candidate.source_name == "linux-meta"
):
_source_version = get_kernel_version_from_meta_package(self.candidate)
if _source_version:
source_version = _source_version
self.candidate.source_name = "linux"
# Ubuntu signed kernel workaround
if (self.candidate.origin == "Ubuntu" and
self.candidate.source_name == "linux-signed"
):
self.candidate.source_name = "linux"
# XXX: Debian does not seem to reliably keep changelogs for previous
# (kernel) versions, so should we always look for the latest
# version instead on Debian? apt does not do this but the
# packages.debian.org website shows the latest version in the
# selected archive
# strip epoch
if ":" in source_version:
source_version = source_version.split(":", 1)[1]
# the path is: COMPONENT/SRC/SRCNAME/SRCNAME_SRCVER, e.g.
# main/a/apt/apt_1.1 or contrib/liba/libapt/libapt_2.0
return uri_template.replace('@CHANGEPATH@',
"%(component)s%(source_prefix)s/%(source_name)s/%(source_name)s_%(source_version)s" %
{
"component": self.candidate.component + "/" if \
self.candidate.component and \
self.candidate.component != "" else "",
"source_prefix": self.source_prefix(),
"source_name": self.candidate.source_name,
"source_version": source_version
})
def source_prefix(self, source_name:str=None):
""" Return prefix used for build repository URL """
if not source_name:
source_name = self.candidate.source_name
return source_name[0] if not source_name.startswith("lib") else \
source_name[:4]
def parse_dsc(self, url:str):
""" Returns filename or None """
_r = self.check_url(url, False, False)
if _r:
target = ""
lines = _r.text.split("Files:", 1)[1].split(":", 1)[0].split("-----BEGIN", 1)[0].split("\n")
target = [s.strip() for s in lines if s.strip().lower().endswith('.debian.tar.xz')]
if not target:
target = [s.strip() for s in lines if s.strip().lower().endswith('.diff.gz')]
if not target:
target = [s.strip() for s in lines if s.strip().lower().endswith('.tar.xz')]
# don't even test for .tar.gz, it will be too big compared to the .deb
# if not target:
# target = [s.strip() for s in lines if s.strip().lower().endswith('.tar.gz')]
if target:
return target[0].split()[-1]
elif _DEBUG: print(".dsc parse error for", url)
return None
def get_changelog_uri(self, base_uri:str):
""" Tries to find a changelog in files listed in .dsc, locally cached
packages as well as the remote .deb file
Returns r and uri
"""
uri = None
# XXX: For APT sources we could just read the apt_pkg.SourceRecords()
# directly, if available, which it is not for most users, so
# probably not worth it
target_filename = self.parse_dsc("%s/%s_%s.dsc" %
(base_uri, self.candidate.source_name, self.candidate.source_version))
# get .debian.tar.xz or .diff.gz as a priority as the smallest options
if (base_uri and target_filename and (
target_filename.lower().endswith('.debian.tar.xz') or
target_filename.lower().endswith('.diff.gz')
)):
uri = "%s/%s" % (base_uri, target_filename)
target_filename = None
r = self.check_url(uri, msg = self.max_download_size_msg)
else:
r = None
if not r:
# fall back to cached local package
uri = self.apt_cache_path + self.candidate.filename
if not os.path.isfile(uri):
# cache miss, download the full source package or the .deb,
# depending on size and availability
if target_filename:
uri_tar = "%s/%s" % (base_uri, target_filename)
else:
uri_tar = None
r, uri = self.get_deb_or_tar(uri_tar)
return (r, uri)
def do_install(self, to_install, langpacks=False):
self.nested_progress_start()
if langpacks:
self.db.progress('START', 0, 10, 'ubiquity/langpacks/title')
else:
self.db.progress('START', 0, 10, 'ubiquity/install/title')
self.db.progress('INFO', 'ubiquity/install/find_installables')
self.progress_region(0, 1)
fetchprogress = DebconfAcquireProgress(
self.db, 'ubiquity/install/title',
'ubiquity/install/apt_indices_starting',
'ubiquity/install/apt_indices')
cache = Cache()
if cache._depcache.broken_count > 0:
syslog.syslog(
'not installing additional packages, since there are broken '
'packages: %s' % ', '.join(broken_packages(cache)))
self.db.progress('STOP')
self.nested_progress_end()
return
with cache.actiongroup():
for pkg in to_install:
mark_install(cache, pkg)
self.db.progress('SET', 1)
self.progress_region(1, 10)
if langpacks:
fetchprogress = DebconfAcquireProgress(
self.db, 'ubiquity/langpacks/title', None,
'ubiquity/langpacks/packages')
installprogress = DebconfInstallProgress(
self.db, 'ubiquity/langpacks/title',
'ubiquity/install/apt_info')
else:
fetchprogress = DebconfAcquireProgress(
self.db, 'ubiquity/install/title', None,
'ubiquity/install/fetch_remove')
installprogress = DebconfInstallProgress(
self.db, 'ubiquity/install/title',
'ubiquity/install/apt_info',
'ubiquity/install/apt_error_install')
chroot_setup(self.target)
commit_error = None
try:
try:
if not self.commit_with_verify(cache,
fetchprogress, installprogress):
fetchprogress.stop()
installprogress.finish_update()
self.db.progress('STOP')
self.nested_progress_end()
return
except IOError:
for line in traceback.format_exc().split('\n'):
syslog.syslog(syslog.LOG_ERR, line)
fetchprogress.stop()
installprogress.finish_update()
self.db.progress('STOP')
self.nested_progress_end()
return
except SystemError as e:
for line in traceback.format_exc().split('\n'):
syslog.syslog(syslog.LOG_ERR, line)
commit_error = str(e)
finally:
chroot_cleanup(self.target)
self.db.progress('SET', 10)
cache.open(None)
if commit_error or cache._depcache.broken_count > 0:
if commit_error is None:
commit_error = ''
brokenpkgs = broken_packages(cache)
self.warn_broken_packages(brokenpkgs, commit_error)
self.db.progress('STOP')
self.nested_progress_end()
class UpdateMaker():
"""A class for making updates for OreSat Linux Updater daemon"""
def __init__(self, board: str):
"""
Parameters
----------
board: str
The board to make the update for.
"""
self._board = board
self._status_file = ""
self._board = board
self._cache = Cache(rootdir=ROOT_DIR)
self._deb_pkgs = []
self._inst_list = []
self._not_installed_yet_list = []
self._not_removed_yet_list = []
print("updating cache")
self._cache.update(raise_on_error=False)
self._cache.open()
# copying the context of the real root apt source.list file into the local one
if stat(OLU_APT_SOURCES_FILE
).st_size == 0 or not isfile(OLU_APT_SOURCES_FILE):
copyfile(SYSTEM_APT_SOURCES_FILE, OLU_APT_SOURCES_FILE)
# adding OreSat Debian apt repo
with open(OLU_APT_SOURCES_FILE, "a") as f:
f.write(
"deb [trusted=yes] https://debian.oresat.org/packages ./")
# copying the apt repo signatures
if len(listdir(OLU_SIGNATURES_DIR)) == 3:
for root, dirs, files in walk(SYSTEM_SIGNATURES_DIR):
for file in files:
if file != "lock":
copyfile(SYSTEM_SIGNATURES_DIR + file,
OLU_SIGNATURES_DIR + file)
# clear download dir
for i in listdir(DOWNLOAD_DIR):
if i.endswith(".deb"):
remove(DOWNLOAD_DIR + i)
status_files = []
for i in listdir(STATUS_CACHE_DIR):
status_files.append(OLMFile(load=i))
status_files.sort()
# find latest olu status tar file
for i in status_files:
if i.board == board:
self._status_file = STATUS_CACHE_DIR + i.name
break
if self._status_file == "":
msg = "No status file for {} board in cache".format(board)
raise FileNotFoundError(msg)
# update status file
dpkg_data = read_dpkg_status_file(self._status_file)
with open(DPKG_STATUS_FILE, "w") as fptr:
fptr.write(dpkg_data)
# dealing with update files that are not installed yet
olu_status_data = read_olu_status_file(self._status_file)
for file in literal_eval(olu_status_data):
with tarfile.open(UPDATE_CACHE_DIR + file, "r") as tar:
with tar.extractfile("instructions.txt") as instructions:
for i in json.loads(instructions.read()):
if i["type"] == "DPKG_INSTALL":
for pkg in i["items"]:
pkg_obj = self._cache[pkg.split('_')[0]]
pkg_obj.mark_install()
self._not_installed_yet_list.extend(i["items"])
elif i["type"] == "DPKG_REMOVE" or i[
"type"] == "DPKG_PURGE":
self._not_removed_yet_list.extend(i["items"])
@property
def not_installed_yet(self) -> list:
return [pkg.split('_')[0] for pkg in self._not_installed_yet_list]
@property
def not_removed_yet(self) -> list:
return [pkg.split('_')[0] for pkg in self._not_removed_yet_list]
def add_packages(self, packages: list, reinstall_not_installed: list,
reinstall_not_removed: list):
"""Add deb packages to be installed.
Parameters
----------
packages: list
A list of deb packages to install on the board.
"""
if packages == []:
raise ValueError("Requires a list of packages to install")
inst_deb_pkgs = []
for pkg in packages:
pkg_obj = self._cache[pkg]
# checking the not yet installed and removed packages
if pkg_obj.name in reinstall_not_removed:
pkg_index = self.not_removed_yet.index(pkg_obj.name)
self._not_removed_yet_list.pop(pkg_index)
pkg_obj.mark_install()
elif pkg_obj.name in reinstall_not_installed:
pkg_index = self.not_installed_yet.index(pkg_obj.name)
self._not_installed_yet_list.pop(pkg_index)
pkg_obj.mark_install()
elif pkg_obj.name not in self.not_installed_yet and pkg_obj.name not in self.not_removed_yet:
pkg_obj.mark_install()
# find new packages (dependencies) that are marked
for deb_pkg in self._cache:
if deb_pkg.marked_install and \
deb_pkg.name not in self._deb_pkgs:
self._deb_pkgs.append(deb_pkg.name)
inst_deb_pkgs.append(deb_pkg.name)
new_inst = Instruction(InstructionType.DPKG_INSTALL, inst_deb_pkgs)
self._inst_list.append(new_inst)
def remove_packages(self, packages: list):
"""Remove deb packages on board.
Parameters
----------
packages: list
A list of deb packages to remove on the board.
"""
if packages == []:
raise ValueError("Requires a list of packages to remove")
new_inst = Instruction(InstructionType.DPKG_REMOVE, packages)
self._inst_list.append(new_inst)
def purge_packages(self, packages: list):
"""Purge deb packages on board.
Parameters
----------
packages: list
A list of deb packages to remove on the board.
"""
if packages == []:
raise ValueError("Requires a list of packages to remove")
new_inst = Instruction(InstructionType.DPKG_PURGE, packages)
self._inst_list.append(new_inst)
def add_bash_scripts(self, bash_scipts: list):
"""Run bash scripts on the board.
Parameters
----------
bash_scipts: list
A list of bash script to run on the board.
"""
if bash_scipts == []:
raise ValueError("Requires a list of bash scipts to run")
new_inst = Instruction(InstructionType.BASH_SCRIPT, bash_scipts)
self._inst_list.append(new_inst)
def add_support_files(self, support_files: list):
"""Add a support files to update archive.
Parameters
----------
support_files: list
A list of support files to add to the update.
"""
for s_file in support_files:
if isfile(s_file):
raise ValueError(" {} was not found".format(s_file))
new_inst = Instruction(InstructionType.SUPPORT_FILE, support_files)
self._inst_list.append(new_inst)
def status(self):
"""Print the contexts of instructions list"""
for i in self._inst_list:
print(i)
def make_update_archive(self):
"""Make the update archive"""
# download deb files
self._cache.fetch_archives()
# replace package name with deb filepath in instruction obj
for inst in self._inst_list:
if not inst.type == InstructionType.DPKG_INSTALL:
continue
for i in range(len(inst.items)):
found = False
for deb_file in listdir(DOWNLOAD_DIR):
if not deb_file.endswith(".deb"):
continue
if deb_file.startswith(inst.items[i] + "_"):
inst.items[i] = DOWNLOAD_DIR + deb_file
found = True
break
if found is False:
break
print("Making tar")
update_file = create_update_archive(self._board, self._inst_list, "./")
print("{} was made".format(update_file))
# option to move generate updates to the update cache
command = input("-> Save copy to update cache [Y/n]: ")
if command == "Y" or command == "y" or command == "yes":
try:
copyfile(update_file, UPDATE_CACHE_DIR + basename(update_file))
except:
print("An error occurred saving the copy to update cache")
else:
print("{} was added to update cache".format(update_file))
