def __init__(self,
release=None,
origin=None,
label=None,
version=None,
description=None,
codename=None,
components=None,
architectures=None,
upstream_url=None):
if release is None:
release = deb822.Release()
else:
release = deb822.Release(release)
if architectures is None:
architectures = ['amd64', 'i386']
if components is None:
components = []
self.release = release
self.release.setdefault('Architectures', ' '.join(architectures))
self.release.setdefault('Components', ' '.join(components))
codename = self.release.setdefault('Codename', codename or 'foo')
self.release.setdefault('Suite', codename)
default = codename.capitalize()
self.release.setdefault('Description', description or default)
origin = self.release.setdefault('Origin', origin or default)
self.release.setdefault('Label', label or origin)
self.release.setdefault('Version', version or REPO_VERSION)
self.set_date()
self._component_arch_binaries = []
self.upstream_url = upstream_url
def get_releaseinfo(rurl):
cache = AptListsCache()
# root URL of the repository
baseurl = '/'.join(rurl.split('/')[:-1])
# get the release file from the cache
release_file = cache.get(rurl)
# create parser instance
rp = deb822.Release(release_file)
# architectures on this dist
archs = rp['Architectures'].split()
components = rp['Components'].split()
# compile a new codename that also considers the repository label
# to distinguish between official and unofficial repos.
label = rp['Label']
origin = rp['Origin']
codename = rp['Codename']
labelcode = '_'.join([rp['Label'], rp['Codename']])
# cleanup
release_file.close()
return {
'baseurl': baseurl,
'archs': archs,
'components': components,
'codename': codename,
'label': label,
'labelcode': labelcode,
'origin': origin
}
def get_repository(self, connection, repository_data, arch, consumer):
url = utils.normalize_repository_url(repository_data['uri'])
suite = repository_data['suite']
components = repository_data.get('section')
path = repository_data.get('path')
name = repository_data.get('name')
# TODO(bgaifullin) implement support for flat repisotory format [1]
# [1] https://wiki.debian.org/RepositoryFormat#Flat_Repository_Format
if components is None:
raise ValueError("The flat format does not supported.")
for component in components:
release = self._get_url_of_metafile((url, suite, component, arch),
"Release")
try:
deb_release = deb822.Release(connection.open_stream(release))
except connection.HTTPError as e:
if e.code != 404:
raise
# some repositories does not contain release file
deb_release = {"origin": ""}
consumer(
Repository(name=name,
architecture=arch,
origin=deb_release["origin"],
url=url,
section=(suite, component),
path=path,
priority=self.get_priority(repository_data)))
def __init__(self,
publication,
codename,
components,
architectures,
label=None,
description=None):
self.publication = publication
self.release = deb822.Release()
self.release["Codename"] = codename
self.release["Architectures"] = " ".join(architectures)
if label:
self.release["Label"] = label
if description:
self.release["Description"] = description
self.release["MD5sum"] = []
self.release["SHA1"] = []
self.release["SHA256"] = []
self.release["SHA512"] = []
self.architectures = architectures
self.components = {
name: _ComponentHelper(self, name)
for name in components
}
self.signing_service = publication.signing_service
async def run(self):
"""
Parse Release content units.
Update release content with information obtained from its artifact.
"""
with ProgressReport(message="Update Release units",
code="update.release") as pb:
async for d_content in self.items():
if isinstance(d_content.content, Release):
release = d_content.content
release_artifact = d_content.d_artifacts[0].artifact
release.sha256 = release_artifact.sha256
release_dict = deb822.Release(release_artifact.file)
release.codename = release_dict["Codename"]
release.suite = release_dict["Suite"]
# TODO split of extra stuff e.g. : 'updates/main' -> 'main'
release.components = _filter_ssl(
release_dict["Components"], self.components)
release.architectures = _filter_ssl(
release_dict["Architectures"], self.architectures)
log.debug("Codename: {}".format(release.codename))
log.debug("Components: {}".format(release.components))
log.debug("Architectures: {}".format(
release.architectures))
pb.increment()
await self.put(d_content)
def test_create_repo(self):
files = []
for root, _, fl in os.walk(self.pool_dir):
for f in fl:
if f.endswith('.deb'):
files.append(os.path.join(root, f))
repo = create_repo(self.new_repo_dir,
files,
codename=self.repo_codename,
arches=self.repo_arches,
components=[self.repo_component],
desc=self.repo_description,
origin=self.repo_name)
release_file = repo.metadata.release_path(repo.base_path)
with open(release_file, 'r') as fh:
release_data = fh.read()
release_822 = deb822.Release(release_data)
self.assertEqual(release_822.get('Origin'), self.repo_name)
self.assertEqual(release_822.get('Label'), self.repo_name)
self.assertEqual(release_822.get('Description'), self.repo_description)
# Test handling of Architectures
self.assertEqual(release_822.get('Architectures'), u'i386 amd64')
self.assertEqual(repo.metadata.architectures, ['i386', 'amd64'])
# Test package paths
packagefile_paths_256 = [x['name'] for x in release_822['SHA256']]
packagefile_paths_1 = [x['name'] for x in release_822['SHA1']]
self.assertEqual(packagefile_paths_256, self.packagefile_paths)
self.assertEqual(packagefile_paths_1, self.packagefile_paths)
def create_cache(repository_root, cache_dir_path):
"""
Creates the cache and all other necessary directories to organize the
control files pulled from the repository.
:param repository_root: url of the repository from which the control files
files will be pulled.
:param cache_dir_path: path where the cache will be created.
.. versionadded:: 0.1
"""
if not os.path.isdir(cache_dir_path):
os.makedirs(cache_dir_path)
branches = (branch.split() for branch in readconfig(
os.path.join(repository_root, 'distributions')))
for name, release_path in branches:
release_path = os.path.join(repository_root, release_path)
try:
md5list = deb822.Release(
urllib.urlopen(release_path)).get('MD5sum')
except urllib2.URLError, e:
logger.warning(
'Could not read release file in %s, error code #%s' %
(release_path, e.code))
else:
for control_file_data in md5list:
if re.match('[\w]*-?[\w]*/[\w]*-[\w]*/Packages.gz$',
control_file_data['name']):
component, architecture, _ = control_file_data[
'name'].split('/')
remote_file = os.path.join(repository_root, 'dists', name,
control_file_data['name'])
local_name = '_'.join(
[name, component,
architecture.replace('binary-', '')])
f = os.path.join(cache_dir_path, local_name + '.gz')
if not os.path.isfile(f):
try:
urllib.urlretrieve(remote_file, f)
except urllib2.URLError, e:
logger.error('Could not get %s, error code #%s' %
(remote_file, e.code))
else:
if md5Checksum(f) != control_file_data['md5sum']:
os.remove(f)
try:
urllib.urlretrieve(remote_file, f)
except urllib2.URLError, e:
logger.error(
'Could not get %s, error code #%s' %
(remote_file, e.code))
def __init__(self, release=None, packages=None, meta=None, dist=None):
if release is None:
release = deb822.Release()
if meta is None:
meta = dict()
for k, v in meta.items():
release.setdefault(k.capitalize(), v)
self.release = release
self._packages = packages
self._packages_file = None
self.dist = dist
def test_metadata_not_shared(self):
# Make sure defaults are not shared between objects
rel = deb822.Release(dict(Architectures="amd64 i386 aarch64"))
md1 = AptRepoMeta(rel)
self.assertEqual(['amd64', 'i386', 'aarch64'], md1.architectures)
md1.architectures = ['amd64']
self.assertEqual(['amd64'], md1.architectures)
md2 = AptRepoMeta(rel)
self.assertEqual(['amd64', 'i386', 'aarch64'], md2.architectures)
def test_metadata(self, _strftime):
_strftime.return_value = "ABCDE"
repo_meta = AptRepoMeta(**self.defaults)
release_path = os.path.join(self.test_dir, 'dists', 'stable',
'Release')
self.assertEqual(release_path, repo_meta.release_path(self.test_dir))
repo_meta.write_release(self.test_dir)
expected = dict(self.repo_release_data, Date="ABCDE")
self.assertEqual(expected,
deb822.Release(open(release_path, "rb").read()))
comp_binary = repo_meta.get_component_arch_binary('main', 'amd64')
release_path = os.path.join(self.test_dir, 'dists', 'stable', 'main',
'binary-amd64', 'Release')
self.assertEqual(release_path, comp_binary.release_path(self.test_dir))
comp_binary.write_release(self.test_dir)
expected = self.release_data
self.assertEqual(expected,
deb822.Release(open(release_path, "rb").read()))
def sync_cache(repository_root, cache_dir_path):
"""
Synchronizes the existing control files in the cache,
comparing the the ones in the repository with the local copies.
If there are differences in the MD5sum field then the local
copies are deleted and copied again from the repository.
It is assumed that the cache directory was created previously.
:param repository_root: url of the repository from which the Packages
files will be updated.
:param cache_dir_path: path to the desired cache directory.
.. versionadded:: 0.1
"""
branches = (branch.split() for branch in readconfig(
os.path.join(repository_root, 'distributions')))
changes = []
for branch, _ in branches:
remote_branch_path = os.path.join(repository_root, 'dists', branch)
release_path = os.path.join(remote_branch_path, 'Release')
try:
md5list = deb822.Release(
urllib.urlopen(release_path)).get('MD5sum')
except urllib2.URLError, e:
logger.warning(
'Could not read release file in %s, error code #%s' %
(remote_branch_path, e.code))
else:
for package_control_file in md5list:
if re.match('[\w]*-?[\w]*/[\w]*-[\w]*/Packages.gz$',
package_control_file['name']):
component, architecture, _ = package_control_file[
'name'].split('/')
remote_package_path = os.path.join(
remote_branch_path, package_control_file['name'])
local_name = '_'.join([
branch, component,
architecture.replace('binary-', '')
])
f = os.path.join(cache_dir_path, local_name + '.gz')
if package_control_file['md5sum'] != md5Checksum(f):
if os.path.exists(f):
os.remove(f)
try:
urllib.urlretrieve(remote_package_path, f)
changes.append(os.path.basename(f))
except urllib2.URLError, e:
logger.error('Could not get %s, error code #%s' %
(remote_package_path, e.code))
else:
logger.info('There are no changes in %s' % f)
def test_AptRepo_create(self):
test_dir = os.path.join(self.test_dir, self.repo_name, '_2001')
packagefile_paths = [
u'main/binary-amd64/Packages', u'main/binary-amd64/Packages.gz',
u'main/binary-amd64/Packages.bz2'
]
files = []
for root, _, fl in os.walk(self.pool_dir):
for f in fl:
if f.endswith('.deb'):
files.append(os.path.join(root, f))
arch = 'amd64'
codename = 'stable'
component = 'main'
repometa = AptRepoMeta(codename=codename,
components=[component],
architectures=[arch])
repo = AptRepo(test_dir,
repo_name=self.repo_name,
metadata=repometa,
gpg_sign_options=None)
# Looking for bad behavior in defaults
repo.create(files,
component=component,
architecture=arch,
with_symlinks=True)
release_file = repo.metadata.release_path(repo.base_path)
with open(release_file, 'r') as fh:
release_data = fh.read()
release_822 = deb822.Release(release_data)
# Test defaults for Origin, Label, Description
expected_default_origin = codename.capitalize()
self.assertEqual(release_822.get('Origin'), expected_default_origin)
self.assertEqual(release_822.get('Label'), expected_default_origin)
self.assertEqual(release_822.get('Description'),
expected_default_origin)
# Test default for Suite
self.assertEqual(release_822.get('Suite'), codename)
# Test handling of Architectures
self.assertEqual(release_822.get('Architectures'), arch)
self.assertEqual(repo.metadata.architectures, [arch])
packagefile_paths_256 = [x['name'] for x in release_822['SHA256']]
packagefile_paths_1 = [x['name'] for x in release_822['SHA1']]
self.assertEqual(packagefile_paths_256, packagefile_paths)
self.assertEqual(packagefile_paths_1, packagefile_paths)
async def _handle_distribution(self, distribution):
log.info(
_('Downloading Release file for distribution: "{}"').format(
distribution))
# Create release_file
if distribution[-1] == "/":
release_file_dir = distribution.strip("/")
else:
release_file_dir = os.path.join("dists", distribution)
release_file_dc = DeclarativeContent(
content=ReleaseFile(distribution=distribution),
d_artifacts=[
self._to_d_artifact(os.path.join(release_file_dir, filename))
for filename in ["Release", "InRelease", "Release.gpg"]
],
)
release_file = await self._create_unit(release_file_dc)
if release_file is None:
return
# Create release object
release_unit = Release(codename=release_file.codename,
suite=release_file.suite,
distribution=distribution)
release_dc = DeclarativeContent(content=release_unit)
release = await self._create_unit(release_dc)
# Create release architectures
architectures = _filter_split_architectures(release_file.architectures,
self.remote.architectures,
distribution)
for architecture in architectures:
release_architecture_dc = DeclarativeContent(
content=ReleaseArchitecture(architecture=architecture,
release=release))
await self.put(release_architecture_dc)
# Parse release file
log.info(
_('Parsing Release file at distribution="{}"').format(
distribution))
release_artifact = await _get_main_artifact_blocking(release_file)
release_file_dict = deb822.Release(release_artifact.file)
# collect file references in new dict
file_references = defaultdict(deb822.Deb822Dict)
for digest_name in ["SHA512", "SHA256", "SHA1", "MD5sum"]:
if digest_name in release_file_dict:
for unit in release_file_dict[digest_name]:
file_references[unit["Name"]].update(unit)
await asyncio.gather(*[
self._handle_component(component, release, release_file,
file_references, architectures)
for component in _filter_split_components(
release_file.components, self.remote.components, distribution)
])
def create_cache_alt(repository_root, cache_dir_path, debug=True):
branches = ['aponwao', 'auyantepui', 'roraima', 'kerepakupai', 'kukenan']
for branch in branches:
remote_branch_path = os.path.join(repository_root, "dists", branch)
local_branch_path = os.path.join(cache_dir_path, branch)
release_data = urllib.urlopen(
os.path.join(remote_branch_path, "Release"))
release_control_file = deb822.Release(release_data)
archs = release_control_file.get('Architectures').split()
comps = release_control_file.get('Components').split()
md5sums = release_control_file.get('MD5Sum')
for comp in comps:
for arch in archs:
remote_file_path = os.path.join(remote_branch_path, comp,
'binary-' + arch)
local_file_path = os.path.join(local_branch_path, comp, arch)
if not os.path.isdir(local_file_path):
os.makedirs(local_file_path)
remote_control_file_path = os.path.join(
remote_file_path, "Packages.gz")
local_control_file_path = os.path.join(local_file_path,
"Packages.gz")
if not os.path.isfile(local_control_file_path):
try:
urllib.urlretrieve(remote_control_file_path,
local_control_file_path)
# Agregar tipo de excepcion
except:
logger.error(
'There has been an error trying to get %s' %
remote_control_file_path)
else:
for md5sum in md5sums:
if md5sum.get('name') == os.path.join(
comp, 'binary-' + arch, 'Packages.gz'):
if md5Checksum(local_control_file_path
) != md5sum.get('md5sum'):
os.remove(local_control_file_path)
try:
urllib.urlretrieve(
remote_control_file_path,
local_control_file_path)
# Agregar tipo de excepcion
except:
logger.error(
'There has been an error trying to get %s'
% remote_control_file_path)
def update_cache(repository_root, cache_dir_path):
'''
Updates the control files existent in the cache,
comparing the the ones in the repository with its local copies.
If there are differences in the MD5sum field then the local
copies are deleted and copied again from the repository.
It is assumed that the cache directory was created previously.
:param repository_root: url of the repository from which the Packages files will be updated.
:param cache_dir_path: path to the desired cache directory
.. versionadded:: 0.1
'''
local_branches = (branch.split() for branch in readconfig(
os.path.join(repository_root, "distributions")))
for branch, _ in local_branches:
remote_branch_path = os.path.join(repository_root, "dists", branch)
local_branch_path = os.path.join(cache_dir_path, branch)
release_path = os.path.join(remote_branch_path, "Release")
try:
md5list = deb822.Release(
urllib.urlopen(release_path)).get('MD5sum')
except urllib2.URLError, e:
logger.warning(
'Could not read release file in %s, error code #%s' %
(remote_branch_path, e.code))
else:
for package_control_file in md5list:
if re.match("[\w]*-?[\w]*/[\w]*-[\w]*/Packages.gz$",
package_control_file['name']):
_, architecture, _ = package_control_file['name'].split(
"/")
# BUSCAR UNA FORMA MENOS PROPENSA A ERRORES PARA HACER ESTO
architecture = architecture.split("-")[1]
remote_package_path = os.path.join(
remote_branch_path, package_control_file['name'])
local_package_path = os.path.join(
local_branch_path, package_control_file['name'])
if package_control_file['md5sum'] != md5Checksum(
local_package_path):
os.remove(local_package_path)
urllib.urlretrieve(remote_package_path,
local_package_path)
update_package_list(local_package_path, branch,
architecture)
else:
logger.info('There are no changes in %s' %
local_package_path)
def __init__(
self,
publication,
codename,
distribution,
components,
architectures,
label,
version,
description=None,
suite=None,
):
self.publication = publication
self.distribution = distribution
self.dists_subfolder = distribution.strip(
"/") if distribution != "/" else "flat-repo"
if distribution[-1] == "/":
message = "Using dists subfolder '{}' for structured publish of originally flat repo!"
log.info(_(message).format(self.dists_subfolder))
# Note: The order in which fields are added to self.release is retained in the
# published Release file. As a "nice to have" for human readers, we try to use
# the same order of fields that official Debian repositories use.
self.release = deb822.Release()
self.release["Origin"] = "Pulp 3"
self.release["Label"] = label
if suite:
self.release["Suite"] = suite
self.release["Version"] = version
if not codename:
codename = distribution.split(
"/")[0] if distribution != "/" else "flat-repo"
self.release["Codename"] = codename
self.release["Date"] = datetime.now(
tz=timezone.utc).strftime("%a, %d %b %Y %H:%M:%S %z")
self.release["Architectures"] = " ".join(architectures)
self.release["Components"] = "" # Will be set later
if description:
self.release["Description"] = description
for checksum_type, deb_field in CHECKSUM_TYPE_MAP.items():
if checksum_type in settings.ALLOWED_CONTENT_CHECKSUMS:
self.release[deb_field] = []
self.architectures = architectures
self.components = {
component: _ComponentHelper(self, component)
for component in components
}
self.signing_service = publication.signing_service
def get_packages_gz(self):
'''
'''
remote_branch_path = os.path.join(self.repo, "dists", self.branch)
release_path = os.path.join(remote_branch_path, "Release")
changes = []
try:
# TODO: Use tmpfile library here
tmp = "/tmp/Release"
download_file(release_path, tmp)
rls_file = open(tmp, "r")
md5list = deb822.Release(rls_file).get('MD5sum')
rls_file.close()
except urllib2.URLError, e:
print 'Could not read release file in %s, error code #%s' % (
remote_branch_path, e.code)
def init_sample_packages(repository_root, samples_dir):
"""
Creates a directory structure to store packages for its later use
in a test package repository.
:param repository_root: url of the repository used.
:param samples_dir: directory that will be used to store the examples for
the repository.
.. versionadded:: 0.1
"""
if not os.path.isdir(samples_dir):
os.makedirs(samples_dir)
# Puede que exista una forma mas sencilla de obtener los nombres
dist_releases = (branch.split() for branch in readconfig(
os.path.join(repository_root, "distributions")))
for release, _ in dist_releases:
release_path = os.path.join(repository_root, "dists", release,
"Release")
try:
# Riesgo poco probable de que el Release no tenga MD5sum
md5list = deb822.Release(
urllib.urlopen(release_path)).get('MD5sum')
print md5list
except urllib2.URLError, e:
logger.warning(
'Could not read release file in %s, error code #%s' %
(release_path, e.code))
else:
for l in md5list:
if re.match("[\w]*-?[\w]*/[\w]*-[\w]*/Packages.gz$",
l['name']):
list_dir = os.path.join(samples_dir, release,
os.path.dirname(l['name']))
if not os.path.isdir(list_dir):
os.makedirs(list_dir)
list_path = os.path.join(list_dir, "list")
if not os.path.isfile(list_path):
control_f_path = os.path.join(repository_root, "dists",
release, l['name'])
select_sample_packages(control_f_path, list_path,
samples_dir, False)
def _create_repository_structure(self, repository):
packages_file = utils.get_path_from_url(
self._get_url_of_metafile(repository, "Packages"))
release_file = utils.get_path_from_url(
self._get_url_of_metafile(repository, "Release"))
utils.ensure_dir_exist(os.path.dirname(release_file))
release = deb822.Release()
release["Origin"] = repository.origin
release["Label"] = repository.origin
release["Archive"] = repository.section[0]
release["Component"] = repository.section[1]
release["Architecture"] = _ARCHITECTURES[repository.architecture]
with open(release_file, "wb") as fd:
release.dump(fd)
open(packages_file, "ab").close()
gzip.open(packages_file + ".gz", "ab").close()
async def run(self):
"""
Parse ReleaseFile content units.
Update release content with information obtained from its artifact.
"""
with ProgressReport(message="Update ReleaseFile units",
code="update.release_file") as pb:
async for d_content in self.items():
if isinstance(d_content.content, ReleaseFile):
release_file = d_content.content
da_names = {
os.path.basename(da.relative_path): da
for da in d_content.d_artifacts
}
if "InRelease" in da_names:
release_file_artifact = da_names["InRelease"].artifact
release_file.relative_path = da_names[
"InRelease"].relative_path
elif "Release" in da_names:
release_file_artifact = da_names["Release"].artifact
release_file.relative_path = da_names[
"Release"].relative_path
else:
# No (proper) artifacts left -> drop it
d_content.content = None
d_content.resolve()
continue
release_file.sha256 = release_file_artifact.sha256
release_file_dict = deb822.Release(
release_file_artifact.file)
release_file.codename = release_file_dict["Codename"]
release_file.suite = release_file_dict["Suite"]
# TODO split of extra stuff e.g. : 'updates/main' -> 'main'
release_file.components = release_file_dict["Components"]
release_file.architectures = release_file_dict[
"Architectures"]
log.debug("Codename: {}".format(release_file.codename))
log.debug("Components: {}".format(release_file.components))
log.debug("Architectures: {}".format(
release_file.architectures))
pb.increment()
await self.put(d_content)
def __init__(
self,
publication,
codename,
distribution,
components,
architectures,
label,
version,
description=None,
suite=None,
):
self.publication = publication
self.distribution = distribution
# Note: The order in which fields are added to self.release is retained in the
# published Release file. As a "nice to have" for human readers, we try to use
# the same order of fields that official Debian repositories use.
self.release = deb822.Release()
self.release["Origin"] = "Pulp 3"
self.release["Label"] = label
if suite:
self.release["Suite"] = suite
self.release["Version"] = version
self.release["Codename"] = codename
self.release["Date"] = datetime.now(
tz=timezone.utc).strftime("%a, %d %b %Y %H:%M:%S %z")
self.release["Architectures"] = " ".join(architectures)
self.release["Components"] = "" # Will be set later
if description:
self.release["Description"] = description
self.release["MD5sum"] = []
self.release["SHA1"] = []
self.release["SHA256"] = []
self.release["SHA512"] = []
self.architectures = architectures
self.components = {
component: _ComponentHelper(self, component)
for component in components
}
self.signing_service = publication.signing_service
def addRelease(self, cache_path, file_path):
"""Add a Release file's info to the list of index files.
Dirty hack until python-apt supports apt-pkg/indexrecords.h
(see Bug #456141)
"""
self.indexrecords[cache_path] = {}
read_packages = False
f = file_path.open('r')
# Use python-debian routines to parse the file for hashes
rel = deb822.Release(f, fields=['MD5Sum', 'SHA1', 'SHA256'])
for hash_type in rel:
for file in rel[hash_type]:
self.indexrecords[cache_path].setdefault(
str(file['name']),
{})[hash_type.upper()] = (str(file[hash_type]),
file['size'])
f.close()
def _update_suite_index(self, repository):
"""Updates the Release file in the suite."""
path = os.path.join(utils.get_path_from_url(repository.url), "dists",
repository.section[0])
release_path = os.path.join(path, "Release")
self.logger.info("added repository suite release file: %s",
release_path)
with open(release_path, "a+b") as fd:
fcntl.flock(fd.fileno(), fcntl.LOCK_EX)
try:
fd.seek(0)
release = deb822.Release(fd)
self._add_to_release(release, repository)
for m in _CHECKSUM_METHODS:
release.setdefault(m, [])
self._add_files_to_release(release, path,
self._get_metafiles(repository))
fd.truncate(0)
release.dump(fd)
finally:
fcntl.flock(fd.fileno(), fcntl.LOCK_UN)
async def run(self):
"""
Parse ReleaseFile content units.
Update release content with information obtained from its artifact.
"""
async with ProgressReport(message="Update ReleaseFile units",
code="update.release_file") as pb:
async for d_content in self.items():
if isinstance(d_content.content, ReleaseFile):
release_file = d_content.content
da_names = {
os.path.basename(da.relative_path): da
for da in d_content.d_artifacts
}
if "Release" in da_names:
if "Release.gpg" in da_names:
if self.gpgkey:
with NamedTemporaryFile() as tmp_file:
tmp_file.write(da_names["Release"].
artifact.file.read())
tmp_file.flush()
verified = self.gpg.verify_file(
da_names["Release.gpg"].artifact.file,
tmp_file.name)
if verified.valid:
log.info(
_("Verification of Release successful."
))
release_file_artifact = da_names[
"Release"].artifact
release_file.relative_path = da_names[
"Release"].relative_path
else:
log.warning(
_("Verification of Release failed. Dropping it."
))
d_content.d_artifacts.remove(
da_names.pop("Release"))
d_content.d_artifacts.remove(
da_names.pop("Release.gpg"))
else:
release_file_artifact = da_names[
"Release"].artifact
release_file.relative_path = da_names[
"Release"].relative_path
else:
if self.gpgkey:
d_content.d_artifacts.delete(
da_names["Release"])
else:
release_file_artifact = da_names[
"Release"].artifact
release_file.relative_path = da_names[
"Release"].relative_path
else:
if "Release.gpg" in da_names:
# No need to keep the signature without "Release"
d_content.d_artifacts.remove(
da_names.pop("Release.gpg"))
if "InRelease" in da_names:
if self.gpgkey:
verified = self.gpg.verify_file(
da_names["InRelease"].artifact.file)
if verified.valid:
log.info(
_("Verification of InRelease successful."))
release_file_artifact = da_names[
"InRelease"].artifact
release_file.relative_path = da_names[
"InRelease"].relative_path
else:
log.warning(
_("Verification of InRelease failed. Dropping it."
))
d_content.d_artifacts.remove(
da_names.pop("InRelease"))
else:
release_file_artifact = da_names[
"InRelease"].artifact
release_file.relative_path = da_names[
"InRelease"].relative_path
if not d_content.d_artifacts:
# No (proper) artifacts left -> distribution not found
raise NoReleaseFile(
distribution=release_file.distribution)
release_file.sha256 = release_file_artifact.sha256
release_file_dict = deb822.Release(
release_file_artifact.file)
if "codename" in release_file_dict:
release_file.codename = release_file_dict["Codename"]
if "suite" in release_file_dict:
release_file.suite = release_file_dict["Suite"]
if "components" in release_file_dict:
release_file.components = release_file_dict[
"Components"]
elif release_file.distribution[-1] == "/":
message = (
"The Release file for distribution '{}' contains no 'Components' "
"field, but since we are dealing with a flat repo, we can continue "
"regardless.")
log.warning(
_(message).format(release_file.distribution))
# TODO: Consider not setting the field at all (requires migrations).
release_file.components = ""
else:
raise MissingReleaseFileField(
release_file.distribution, "Components")
if "architectures" in release_file_dict:
release_file.architectures = release_file_dict[
"Architectures"]
elif release_file.distribution[-1] == "/":
message = (
"The Release file for distribution '{}' contains no 'Architectures' "
"field, but since we are dealing with a flat repo, we can extract them "
"from the repos single Package index later.")
log.warning(
_(message).format(release_file.distribution))
release_file.architectures = ""
else:
raise MissingReleaseFileField(
release_file.distribution, "Architectures")
log.debug(_("Codename: {}").format(release_file.codename))
log.debug(
_("Components: {}").format(release_file.components))
log.debug(
_("Architectures: {}").format(
release_file.architectures))
await pb.aincrement()
await self.put(d_content)
async def run(self):
"""
Build and emit `DeclarativeContent` from the Release data.
"""
# TODO Merge into one list of futures
future_releases = []
future_package_indices = []
future_installer_file_indices = []
with ProgressReport(
message="Creating download requests for Release files",
code="download.release",
total=self.num_distributions,
) as pb:
for distribution in self.distributions:
log.info(
'Downloading Release file for distribution: "{}"'.format(
distribution))
release_relpath = os.path.join("dists", distribution,
"Release")
release_path = os.path.join(self.parsed_url.path,
release_relpath)
release_da = DeclarativeArtifact(
Artifact(),
urlunparse(self.parsed_url._replace(path=release_path)),
release_relpath,
self.remote,
deferred_download=False,
)
release_gpg_relpath = os.path.join("dists", distribution,
"Release.gpg")
release_gpg_path = os.path.join(self.parsed_url.path,
release_gpg_relpath)
release_gpg_da = DeclarativeFailsafeArtifact(
Artifact(),
urlunparse(
self.parsed_url._replace(path=release_gpg_path)),
release_gpg_relpath,
self.remote,
deferred_download=False,
)
inrelease_relpath = os.path.join("dists", distribution,
"InRelease")
inrelease_path = os.path.join(self.parsed_url.path,
inrelease_relpath)
inrelease_da = DeclarativeFailsafeArtifact(
Artifact(),
urlunparse(self.parsed_url._replace(path=inrelease_path)),
inrelease_relpath,
self.remote,
deferred_download=False,
)
release_unit = Release(distribution=distribution,
relative_path=release_relpath)
release_dc = DeclarativeContent(
content=release_unit,
d_artifacts=[release_da, release_gpg_da, inrelease_da],
does_batch=False,
)
future_releases.append(release_dc.get_or_create_future())
await self.put(release_dc)
pb.increment()
with ProgressReport(message="Parsing Release files",
code="parsing.release",
total=self.num_distributions) as pb:
for release_future in asyncio.as_completed(future_releases):
release = await release_future
if release is None:
continue
log.info('Parsing Release file for release: "{}"'.format(
release.codename))
release_artifact = release._artifacts.get(
sha256=release.sha256)
release_dict = deb822.Release(release_artifact.file)
async for d_content in self._read_release_file(
release, release_dict):
if isinstance(d_content.content, PackageIndex):
future_package_indices.append(
d_content.get_or_create_future())
if isinstance(d_content.content, InstallerFileIndex):
future_installer_file_indices.append(
d_content.get_or_create_future())
await self.put(d_content)
pb.increment()
with ProgressReport(message="Parsing package index files",
code="parsing.packageindex") as pb:
for package_index_future in asyncio.as_completed(
future_package_indices):
package_index = await package_index_future
if package_index is None:
continue
package_index_artifact = package_index.main_artifact
log.debug("Parsing package index for {}:{}.".format(
package_index.component, package_index.architecture))
async for package_dc in self._read_package_index(
package_index_artifact.file):
await self.put(package_dc)
pb.increment()
with ProgressReport(message="Parsing installer file index files",
code="parsing.installer") as pb:
for installer_file_index_future in asyncio.as_completed(
future_installer_file_indices):
installer_file_index = await installer_file_index_future
if installer_file_index is None:
continue
log.debug("Parsing installer file index for {}:{}.".format(
installer_file_index.component,
installer_file_index.architecture))
async for d_content in self._read_installer_file_index(
installer_file_index):
await self.put(d_content)
pb.increment()
def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = DebPublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(
_('Publishing: repository={repo}, version={ver}, publisher={pub}').
format(repo=repository_version.repository.name,
ver=repository_version.number,
pub=publisher.name))
with WorkingDirectory():
with Publication.create(repository_version,
publisher,
pass_through=False) as publication:
if publisher.simple:
repository = repository_version.repository
release = deb822.Release()
# TODO: release['Label']
release['Codename'] = 'default'
release['Components'] = 'all'
release['Architectures'] = ''
if repository.description:
release['Description'] = repository.description
release['MD5sum'] = []
release['SHA1'] = []
release['SHA256'] = []
release['SHA512'] = []
package_index_files = {}
for package in Package.objects.filter(
pk__in=repository_version.content.order_by(
'-_created')):
published_artifact = PublishedArtifact(
relative_path=package.filename(),
publication=publication,
content_artifact=package.contentartifact_set.get(),
)
published_artifact.save()
if package.architecture not in package_index_files:
package_index_path = os.path.join(
'dists',
'default',
'all',
'binary-{}'.format(package.architecture),
'Packages',
)
os.makedirs(os.path.dirname(package_index_path),
exist_ok=True)
package_index_files[package.architecture] = (open(
package_index_path, 'wb'), package_index_path)
package.to822('all').dump(
package_index_files[package.architecture][0])
package_index_files[package.architecture][0].write(b'\n')
for package_index_file, package_index_path in package_index_files.values(
):
package_index_file.close()
gz_package_index_path = _zip_file(package_index_path)
_add_to_release(release, package_index_path)
_add_to_release(release, gz_package_index_path)
package_index = PublishedMetadata(
relative_path=package_index_path,
publication=publication,
file=File(open(package_index_path, 'rb')),
)
package_index.save()
gz_package_index = PublishedMetadata(
relative_path=gz_package_index_path,
publication=publication,
file=File(open(gz_package_index_path, 'rb')),
)
gz_package_index.save()
release['Architectures'] = ', '.join(
package_index_files.keys())
release_path = os.path.join('dists', 'default', 'Release')
os.makedirs(os.path.dirname(release_path), exist_ok=True)
with open(release_path, 'wb') as release_file:
release.dump(release_file)
release_metadata = PublishedMetadata(
relative_path=release_path,
publication=publication,
file=File(open(release_path, 'rb')),
)
release_metadata.save()
if publisher.structured:
raise NotImplementedError(
"Structured publishing is not yet implemented.")
log.info(
_('Publication: {publication} created').format(
publication=publication.pk))
def publish(repository_version_pk, simple=False, structured=False):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
repository_version_pk (str): Create a publication from this repository version.
simple (bool): Create a simple publication with all packages contained in default/all.
structured (bool): Create a structured publication with releases and components.
(Not yet implemented)
"""
repo_version = RepositoryVersion.objects.get(pk=repository_version_pk)
log.info(
_("Publishing: repository={repo}, version={ver}, simple={simple}, structured={structured}"
).format( # noqa
repo=repo_version.repository.name,
ver=repo_version.number,
simple=simple,
structured=structured,
))
with WorkingDirectory():
with DebPublication.create(repo_version,
pass_through=False) as publication:
publication.simple = simple
publication.structured = structured
if simple:
repository = repo_version.repository
release = deb822.Release()
# TODO: release['Label']
release["Codename"] = "default"
release["Components"] = "all"
release["Architectures"] = ""
if repository.description:
release["Description"] = repository.description
release["MD5sum"] = []
release["SHA1"] = []
release["SHA256"] = []
release["SHA512"] = []
package_index_files = {}
for package in Package.objects.filter(
pk__in=repo_version.content.order_by("-pulp_created")):
published_artifact = PublishedArtifact(
relative_path=package.filename(),
publication=publication,
content_artifact=package.contentartifact_set.get(),
)
published_artifact.save()
if package.architecture not in package_index_files:
package_index_path = os.path.join(
"dists",
"default",
"all",
"binary-{}".format(package.architecture),
"Packages",
)
os.makedirs(os.path.dirname(package_index_path),
exist_ok=True)
package_index_files[package.architecture] = (
open(package_index_path, "wb"),
package_index_path,
)
package_serializer = Package822Serializer(
package, context={"request": None})
package_serializer.to822("all").dump(
package_index_files[package.architecture][0])
package_index_files[package.architecture][0].write(b"\n")
for (package_index_file,
package_index_path) in package_index_files.values():
package_index_file.close()
gz_package_index_path = _zip_file(package_index_path)
_add_to_release(release, package_index_path)
_add_to_release(release, gz_package_index_path)
package_index = PublishedMetadata.create_from_file(
publication=publication,
file=File(open(package_index_path, "rb")))
package_index.save()
gz_package_index = PublishedMetadata.create_from_file(
publication=publication,
file=File(open(gz_package_index_path, "rb")))
gz_package_index.save()
release["Architectures"] = ", ".join(
package_index_files.keys())
release_path = os.path.join("dists", "default", "Release")
os.makedirs(os.path.dirname(release_path), exist_ok=True)
with open(release_path, "wb") as release_file:
release.dump(release_file)
release_metadata = PublishedMetadata.create_from_file(
publication=publication,
file=File(open(release_path, "rb")))
release_metadata.save()
if structured:
raise NotImplementedError(
"Structured publishing is not yet implemented.")
log.info(
_("Publication: {publication} created").format(
publication=publication.pk))
async def run(self):
"""
Parse ReleaseFile content units.
Update release content with information obtained from its artifact.
"""
with ProgressReport(message="Update ReleaseFile units",
code="update.release_file") as pb:
async for d_content in self.items():
if isinstance(d_content.content, ReleaseFile):
release_file = d_content.content
da_names = {
os.path.basename(da.relative_path): da
for da in d_content.d_artifacts
}
if "Release" in da_names:
if "Release.gpg" in da_names:
if self.gpgkey:
with NamedTemporaryFile() as tmp_file:
tmp_file.write(da_names["Release"].
artifact.file.read())
tmp_file.flush()
verified = self.gpg.verify_file(
da_names["Release.gpg"].artifact.file,
tmp_file.name)
if verified.valid:
log.info(
"Verification of Release successful.")
release_file_artifact = da_names[
"Release"].artifact
release_file.relative_path = da_names[
"Release"].relative_path
else:
log.warn(
"Verification of Release failed. Dropping it."
)
d_content.d_artifacts.remove(
da_names.pop("Release"))
d_content.d_artifacts.remove(
da_names.pop("Release.gpg"))
else:
release_file_artifact = da_names[
"Release"].artifact
release_file.relative_path = da_names[
"Release"].relative_path
else:
if self.gpgkey:
d_content.d_artifacts.delete(
da_names["Release"])
else:
release_file_artifact = da_names[
"Release"].artifact
release_file.relative_path = da_names[
"Release"].relative_path
else:
if "Release.gpg" in da_names:
# No need to keep the signature without "Release"
d_content.d_artifacts.remove(
da_names.pop("Release.gpg"))
if "InRelease" in da_names:
if self.gpgkey:
verified = self.gpg.verify_file(
da_names["InRelease"].artifact.file)
if verified.valid:
log.info(
"Verification of InRelease successful.")
release_file_artifact = da_names[
"InRelease"].artifact
release_file.relative_path = da_names[
"InRelease"].relative_path
else:
log.warn(
"Verification of InRelease failed. Dropping it."
)
d_content.d_artifacts.remove(
da_names.pop("InRelease"))
else:
release_file_artifact = da_names[
"InRelease"].artifact
release_file.relative_path = da_names[
"InRelease"].relative_path
if not d_content.d_artifacts:
# No (proper) artifacts left -> distribution not found
raise NoReleaseFile(
distribution=release_file.distribution)
release_file.sha256 = release_file_artifact.sha256
release_file_dict = deb822.Release(
release_file_artifact.file)
release_file.codename = release_file_dict["Codename"]
if "suite" in release_file_dict:
release_file.suite = release_file_dict.get("Suite")
# TODO split of extra stuff e.g. : 'updates/main' -> 'main'
release_file.components = release_file_dict["Components"]
release_file.architectures = release_file_dict[
"Architectures"]
log.debug("Codename: {}".format(release_file.codename))
log.debug("Components: {}".format(release_file.components))
log.debug("Architectures: {}".format(
release_file.architectures))
pb.increment()
await self.put(d_content)
def write_release_file(path, meta_data, release_meta_files):
"""
Writes a 'Release' file for an apt repository.
This function assumes the file ":path:/Release" does not exist.
:param path: the path to the folder where the 'Release' file will be placed
:param meta_data: a dict containing the needed meta data
:param release_meta_files: a list of paths to files to be included
:returns: the path to the 'Release' file
"""
output_file_path = os.path.join(path, 'Release')
# Ordered list of supported non-checksum fields (tuples):
# Known to exist but currently unsupported fields include:
# "Acquire-By-Hash" (after 'date')
fields = [
('origin', 'Origin'),
('label', 'Label'),
('suite', 'Suite'),
('version', 'Version'),
('codename', 'Codename'),
('changelogs', 'Changelogs'),
('date', 'Date'),
('architectures', 'Architectures'),
('components', 'Components'),
('description', 'Description'),
]
# Ordered list of supported checksum fields (tuples):
checksum_fields = [
('md5sum', 'MD5sum'),
('sha1', 'SHA1'),
('sha256', 'SHA256'),
]
# Amend or add incomplete fields:
meta_data['architectures'] = " ".join(meta_data['architectures'])
meta_data['components'] = " ".join(meta_data['components'])
meta_data['date'] = strftime('%a, %d %b %Y %H:%M:%S +0000', gmtime())
# Initialize deb822 object:
release = deb822.Release()
# Translate meta_data to deb822 for all fields (without checksum_fields):
for field in fields:
if field[0] in meta_data:
release[field[1]] = meta_data[field[0]]
# Initialize the needed deb822 checksum fields:
release['MD5sum'] = []
release['SHA1'] = []
release['SHA256'] = []
# Add the checksum fields to the deb822 object:
for file_path in release_meta_files:
checksums = DebPackage.calculate_deb_checksums(file_path)
file_size = os.path.getsize(file_path)
relative_path = os.path.relpath(file_path, path)
for checksum_type in checksum_fields:
release[checksum_type[1]].append({checksum_type[0]: checksums[checksum_type[0]],
'size': file_size,
'name': relative_path})
# Write the deb822 object to a file:
with open(output_file_path, "wb") as release_file:
release.dump(release_file)
return output_file_path
async def _handle_distribution(self, distribution):
log.info(
_('Downloading Release file for distribution: "{}"').format(
distribution))
# Create release_file
if distribution[-1] == "/":
release_file_dir = distribution.strip("/")
else:
release_file_dir = os.path.join("dists", distribution)
release_file_dc = DeclarativeContent(
content=ReleaseFile(distribution=distribution),
d_artifacts=[
self._to_d_artifact(os.path.join(release_file_dir, filename))
for filename in ["Release", "InRelease", "Release.gpg"]
],
)
release_file = await self._create_unit(release_file_dc)
if release_file is None:
return
# Create release object
release_unit = Release(codename=release_file.codename,
suite=release_file.suite,
distribution=distribution)
release_dc = DeclarativeContent(content=release_unit)
release = await self._create_unit(release_dc)
# Create release architectures
if release_file.architectures:
architectures = _filter_split_architectures(
release_file.architectures, self.remote.architectures,
distribution)
elif distribution[-1] == "/":
message = (
"The ReleaseFile content unit architecrures are unset for the flat repo with "
"distribution '{}'. ReleaseArchitecture content creation is deferred!"
)
log.warning(_(message).format(distribution))
architectures = []
for architecture in architectures:
release_architecture_dc = DeclarativeContent(
content=ReleaseArchitecture(architecture=architecture,
release=release))
await self.put(release_architecture_dc)
# Parse release file
log.info(
_('Parsing Release file at distribution="{}"').format(
distribution))
release_artifact = await _get_main_artifact_blocking(release_file)
release_file_dict = deb822.Release(release_artifact.file)
# Retrieve and interpret any 'No-Support-for-Architecture-all' value:
# We will refer to the presence of 'No-Support-for-Architecture-all: Packages' in a Release
# file as indicating "hybrid format". For more info, see:
# https://wiki.debian.org/DebianRepository/Format#No-Support-for-Architecture-all
no_support_for_arch_all = release_file_dict.get(
"No-Support-for-Architecture-all", "")
if no_support_for_arch_all.strip() == "Packages":
hybrid_format = True
elif not no_support_for_arch_all:
hybrid_format = False
else:
raise UnknownNoSupportForArchitectureAllValue(
release_file.relative_path, no_support_for_arch_all)
# collect file references in new dict
file_references = defaultdict(deb822.Deb822Dict)
for digest_name in ["SHA512", "SHA256", "SHA1", "MD5sum"]:
if digest_name in release_file_dict:
for unit in release_file_dict[digest_name]:
file_references[unit["Name"]].update(unit)
if distribution[-1] == "/":
# Handle flat repo
sub_tasks = [
self._handle_flat_repo(file_references, release_file, release)
]
else:
# Handle components
sub_tasks = [
self._handle_component(
component,
release,
release_file,
file_references,
architectures,
hybrid_format,
) for component in _filter_split_components(
release_file.components, self.remote.components,
distribution)
]
await asyncio.gather(*sub_tasks)