def _download_tagged_http_resource(self,
source_url,
target_file,
tag="",
digest=None,
connection_timeout=30):
downloader = Downloader()
if not tag:
tag = downloader.tag(source_url)
blob_file = os.path.join(os.path.dirname(target_file), tag)
try:
with open(blob_file + "$", "wb+") as f:
for chunk in downloader.get(source_url, digest=digest):
f.write(chunk)
except Exception:
if os.path.exists(blob_file + "$"):
os.unlink(blob_file + "$")
raise
#end try
# Atomically rename blob.
os.rename(blob_file + "$", blob_file)
# Create temporary symlink to new blob.
os.symlink(os.path.basename(blob_file), target_file + "$")
# Atomically rename symlink (hopefully).
os.rename(target_file + "$", target_file)
def download(self):
"""
Downloads all components of this source package to self.work_dir.
"""
downloader = Downloader(progress_bar_class=ProgressBar)
pool_dir = self.metadata.get("Directory")
orig_tarball, \
orig_components, \
deb_tarball, \
deb_patches, \
debdiff_gz = self._guess_file_components()
files_to_download = [orig_tarball, deb_tarball, debdiff_gz
] + orig_components
for filename in files_to_download:
if not filename:
continue
outfile = os.path.join(self.work_dir, filename)
url = "/".join([self.metadata.base_url, pool_dir, filename])
LOGGER.info("fetching {}".format(url))
with open(outfile, "wb+") as f:
for chunk in downloader.get(url):
f.write(chunk)
#end with
#end for
return self
def fetch_from_repo(self, repo_name, pkg_name, version, filename,
sha256sum=None):
downloader = Downloader(progress_bar_class=ProgressBar)
if pkg_name.startswith("lib"):
first_letter = pkg_name[3]
else:
first_letter = pkg_name[0]
rel_path = os.sep.join([first_letter, pkg_name, version, filename])
mirror_url = DistroInfo().pick_mirror(
release=self.release, repo_name=repo_name
)
source_url = "/".join([
mirror_url,
self.release,
repo_name,
"sources",
rel_path
])
target_url = os.sep.join([
self.cache_dir,
"bolt",
"dists",
self.release,
"sources",
repo_name,
rel_path
])
h = hashlib.sha256()
LOGGER.info("retrieving {}".format(source_url))
try:
os.makedirs(os.path.dirname(target_url), exist_ok=True)
with open(target_url, "wb+") as f:
for chunk in downloader.get(source_url, digest=h):
f.write(chunk)
except urllib.error.URLError as e:
raise NetworkError(
"failed to retrieve {}: {}".format(source_url, e.reason)
)
if sha256sum and sha256sum != h.hexdigest():
raise BoltError("file {} has invalid checksum!".format(target_url))
return target_url
def build_content_spec(self):
downloader = Downloader(progress_bar_class=ProgressBar)
filename = self.metadata["Filename"]
outfile = os.path.join(self.work_dir, os.path.basename(filename))
url = "/".join([self.metadata.base_url, filename])
LOGGER.info("fetching {}".format(url))
with open(outfile, "wb+") as f:
for chunk in downloader.get(url):
f.write(chunk)
#end with
self.contents = \
self._binary_deb_list_contents(outfile)
def _load_inrelease_file(self, component, base_url):
cache_dir = os.path.join(self._cache_dir, "dists", self.release,
component)
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
downloader = Downloader()
source = "{}/{}".format(base_url, "InRelease")
target = os.path.join(cache_dir, "InRelease")
if not os.path.islink(target):
old_tag = ""
else:
old_tag = os.path.basename(os.readlink(target))
new_tag = downloader.tag(source)
if old_tag != new_tag:
self._download_tagged_http_resource(source, target, tag=new_tag)
if old_tag:
os.unlink(os.path.join(cache_dir, old_tag))
#end if
inrelease = InReleaseFile.load(os.path.join(cache_dir, new_tag))
if self._keyring:
if not os.path.exists(self._keyring):
raise BoltError(
"keyring file '{}' not found, cannot check '{}' signature."
.format(self._keyring, target))
#end if
if not inrelease.valid_signature(keyring=self._keyring):
raise BoltError(
"unable to verify the authenticity of '{}'".format(target))
#end if
#end if
return inrelease
def _load_package_list(self,
suite,
base_url,
component,
type_,
update=False,
inrelease=None):
if not inrelease:
inrelease = self._load_inrelease_file(suite,
base_url,
update=update)
cache_dir = os.path.join(self._cache_dir, "dists", self.release, suite,
component, type_)
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
source_url = None
for ext in [".gz", ".xz"]:
if type_ == "source":
filename = "Sources" + ext
source = f"{component}/source/{filename}"
target = os.path.join(cache_dir, filename)
cache = self.source
else:
filename = "Packages" + ext
source = f"{component}/{type_}/{filename}"
target = os.path.join(cache_dir, filename)
cache = self.binary
#end if
try:
sha256sum = inrelease.hash_for_filename(source)
source_url = "{}/{}".format(base_url,
inrelease.by_hash_path(source))
except KeyError:
continue
else:
break
#end for
if not source_url:
raise DebianPackageCache.Error(
'unable to locate index file for "{}" in "{}" '
'suite'.format(type_, suite))
#end if
if not os.path.islink(target):
old_tag = ""
else:
old_tag = os.path.basename(os.readlink(target))
new_tag = None
if update:
try:
downloader = Downloader()
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
new_tag = downloader.tag(source_url)
if old_tag != new_tag:
downloader.download_named_tag(source_url,
target,
new_tag,
permissions=0o0644)
if old_tag:
try:
os.unlink(os.path.join(cache_dir, old_tag))
except OSError:
pass
#end if
#end if
except Exception as e:
raise DebianPackageCache.Error(
'failed to download "{}": {}'.format(source_url, str(e)))
#end try
#end if
try:
digest = hashlib.sha256()
with open(target, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
digest.update(chunk)
if digest.hexdigest() != sha256sum:
raise BoltError('wrong hash for "{}".'.format(target))
with ArchiveFileReader(target, raw=True) as archive:
try:
next(iter(archive))
except StopIteration:
buf = ""
else:
buf = archive.read_data().decode("utf-8")
pool_base = re.match(r"^(?P<pool_base>https?://.*?)/dists/.*$",
base_url).group("pool_base")
for chunk in re.split(r"\n\n+", buf, flags=re.MULTILINE):
chunk = chunk.strip()
if not chunk:
continue
meta_data = DebianPackageMetaData(chunk,
base_url=pool_base)
meta_data["Suite"] = suite
meta_data["Component"] = component
pkg_name = meta_data["Package"]
pkg_version = meta_data["Version"]
cache\
.setdefault(pkg_name, DebianPackageDict())\
.setdefault(pkg_version, meta_data)
#end for
#end with
except Exception as e:
files_to_delete = [
target,
os.path.join(os.path.dirname(target), new_tag or old_tag)
]
for filename in files_to_delete:
try:
os.unlink(filename)
except OSError:
pass
#end for
raise DebianPackageCache.Error('failed to load "{}": {}'.format(
target, str(e)))
def _load_inrelease_file(self, suite, base_url, update=False):
cache_dir = os.path.join(self._cache_dir, "dists", self.release, suite)
source = "{}/{}".format(base_url, "InRelease")
target = os.path.join(cache_dir, "InRelease")
if not os.path.islink(target):
old_tag = ""
else:
old_tag = os.path.basename(os.readlink(target))
new_tag = None
if update:
try:
downloader = Downloader()
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
new_tag = downloader.tag(source)
if old_tag != new_tag:
downloader.download_named_tag(source,
target,
new_tag,
permissions=0o0644)
if old_tag:
try:
os.unlink(os.path.join(cache_dir, old_tag))
except OSError:
pass
#end if
#end if
except Exception as e:
raise DebianPackageCache.Error(
'failed to download "{}": {}'.format(source, str(e)))
#end try
#end if
try:
inrelease = InReleaseFile.load(
os.path.join(cache_dir, new_tag or old_tag))
if self._keyring:
if not os.path.exists(self._keyring):
raise BoltError(
'keyring file "{}" not found, cannot check signature '
'of "{}".'.format(self._keyring, target))
#end if
if not inrelease.valid_signature(keyring=self._keyring):
raise BoltError(
'unable to verify the authenticity of "{}"'.format(
target))
#end if
#end if
except Exception as e:
files_to_delete = [
target,
os.path.join(os.path.dirname(target), new_tag or old_tag)
]
for filename in files_to_delete:
try:
os.unlink(filename)
except OSError:
pass
#end for
raise DebianPackageCache.Error('failed to load "{}": {}'.format(
target, str(e)))
#end try
return inrelease