def test_checksum_name_str(self):
self.assertEqual(cr.checksum_name_str(cr.MD5), "md5")
self.assertEqual(cr.checksum_name_str(cr.SHA), "sha")
self.assertEqual(cr.checksum_name_str(cr.SHA1), "sha1")
self.assertEqual(cr.checksum_name_str(cr.SHA224), "sha224")
self.assertEqual(cr.checksum_name_str(cr.SHA256), "sha256")
self.assertEqual(cr.checksum_name_str(cr.SHA384), "sha384")
self.assertEqual(cr.checksum_name_str(cr.SHA512), "sha512")
self.assertEqual(cr.checksum_name_str(65), None)
def test_checksum_name_str(self):
self.assertEqual(cr.checksum_name_str(cr.MD5), "md5")
self.assertEqual(cr.checksum_name_str(cr.SHA), "sha")
self.assertEqual(cr.checksum_name_str(cr.SHA1), "sha1")
self.assertEqual(cr.checksum_name_str(cr.SHA224), "sha224")
self.assertEqual(cr.checksum_name_str(cr.SHA256), "sha256")
self.assertEqual(cr.checksum_name_str(cr.SHA384), "sha384")
self.assertEqual(cr.checksum_name_str(cr.SHA512), "sha512")
self.assertEqual(cr.checksum_name_str(65), None)
def parse_updateinfo(path):
uinfo = cr.UpdateInfo(path)
for update in uinfo.updates:
print "From: %s" % update.fromstr
print "Status: %s" % update.status
print "Type: %s" % update.type
print "Version: %s" % update.version
print "Id: %s" % update.id
print "Title: %s" % update.title
print "Issued date: %s" % update.issued_date
print "Updated date: %s" % update.updated_date
print "Rights: %s" % update.rights
print "Release: %s" % update.release
print "Pushcount: %s" % update.pushcount
print "Severity: %s" % update.severity
print "Summary: %s" % update.summary
print "Description: %s" % update.description
print "Solution: %s" % update.solution
print "References:"
for ref in update.references:
print " Href: %s" % ref.href
print " Id: %s" % ref.id
print " Type: %s" % ref.type
print " Title: %s" % ref.title
print " ----------------------------"
print "Pkglist (collections):"
for col in update.collections:
print " Short: %s" % col.shortname
print " name: %s" % col.name
print " Packages:"
for pkg in col.packages:
print " Name: %s" % pkg.name
print " Version: %s" % pkg.version
print " Release: %s" % pkg.release
print " Epoch: %s" % pkg.epoch
print " Arch: %s" % pkg.arch
print " Src: %s" % pkg.src
print " Filename: %s" % pkg.filename
print " Sum: %s" % pkg.sum
print " Sum type: %s (%s)" % (
pkg.sum_type, cr.checksum_name_str(pkg.sum_type))
print " Reboot suggested: %s" % pkg.reboot_suggested
print " ----------------------------"
print "=============================="
def parse_updateinfo(path):
uinfo = cr.UpdateInfo(path)
for update in uinfo.updates:
print "From: %s" % update.fromstr
print "Status: %s" % update.status
print "Type: %s" % update.type
print "Version: %s" % update.version
print "Id: %s" % update.id
print "Title: %s" % update.title
print "Issued date: %s" % update.issued_date
print "Updated date: %s" % update.updated_date
print "Rights: %s" % update.rights
print "Release: %s" % update.release
print "Pushcount: %s" % update.pushcount
print "Severity: %s" % update.severity
print "Summary: %s" % update.summary
print "Description: %s" % update.description
print "Solution: %s" % update.solution
print "References:"
for ref in update.references:
print " Href: %s" % ref.href
print " Id: %s" % ref.id
print " Type: %s" % ref.type
print " Title: %s" % ref.title
print " ----------------------------"
print "Pkglist (collections):"
for col in update.collections:
print " Short: %s" % col.shortname
print " name: %s" % col.name
print " Packages:"
for pkg in col.packages:
print " Name: %s" % pkg.name
print " Version: %s" % pkg.version
print " Release: %s" % pkg.release
print " Epoch: %s" % pkg.epoch
print " Arch: %s" % pkg.arch
print " Src: %s" % pkg.src
print " Filename: %s" % pkg.filename
print " Sum: %s" % pkg.sum
print " Sum type: %s (%s)" % (pkg.sum_type, cr.checksum_name_str(pkg.sum_type))
print " Reboot suggested: %s" % pkg.reboot_suggested
print " ----------------------------"
print "=============================="
def cr_create_md(repodata_path, pkglist=None, log=sys.stdout):
if pkglist is None:
pkglist = cr_get_pkg_list(repo_base, log)
pri_xml_path = os.path.join(repodata_path, 'primary.xml.gz')
fil_xml_path = os.path.join(repodata_path, 'filelists.xml.gz')
oth_xml_path = os.path.join(repodata_path, 'other.xml.gz')
pri_db_path = os.path.join(repodata_path, 'primary.sqlite')
fil_db_path = os.path.join(repodata_path, 'filelists.sqlite')
oth_db_path = os.path.join(repodata_path, 'other.sqlite')
def __create_xml(queues, xml_path, xml_func, name):
cs = cr.ContentStat(cr.SHA256)
xml = xml_func(xml_path, contentstat=cs)
xml.set_num_of_pkgs(len(pkglist))
for pkg in pkglist:
xml.add_pkg(pkg)
xml.close()
queues['master'].put(
((name, xml_path), (cs.checksum, cs.size, cs.checksum_type)), True)
def __create_db(queues, db_path, db_func, name):
db = db_func(db_path)
for pkg in pkglist:
db.add_pkg(pkg)
db.dbinfo_update(queues[name].get(True))
db.close()
cs = cr.ContentStat(cr.SHA256)
cr.compress_file_with_stat(
db_path, db_path + cr.compression_suffix(cr.BZ2_COMPRESSION),
cr.BZ2_COMPRESSION, cs)
os.remove(db_path)
queues['master'].put(
((name + '_db',
db_path + cr.compression_suffix(cr.BZ2_COMPRESSION)),
(cs.checksum, cs.size, cs.checksum_type)), True)
queue_manager = multiprocessing.Manager()
queues = dict({
'master': queue_manager.Queue(),
'primary': queue_manager.Queue(),
'filelists': queue_manager.Queue(),
'other': queue_manager.Queue(),
})
log.write('[%s] Generating metadata in %s\n' % (stamp(), repodata_path))
th = [0] * 6
th[0] = multiprocessing.Process(target=__create_xml,
args=(queues, pri_xml_path,
cr.PrimaryXmlFile, 'primary'))
th[0].start()
th[1] = multiprocessing.Process(target=__create_xml,
args=(queues, fil_xml_path,
cr.FilelistsXmlFile, 'filelists'))
th[1].start()
th[2] = multiprocessing.Process(target=__create_xml,
args=(queues, oth_xml_path,
cr.OtherXmlFile, 'other'))
th[2].start()
th[3] = multiprocessing.Process(target=__create_db,
args=(queues, pri_db_path,
cr.PrimarySqlite, 'primary'))
th[3].start()
th[4] = multiprocessing.Process(target=__create_db,
args=(queues, fil_db_path,
cr.FilelistsSqlite, 'filelists'))
th[4].start()
th[5] = multiprocessing.Process(target=__create_db,
args=(queues, oth_db_path, cr.OtherSqlite,
'other'))
th[5].start()
repomd = cr.Repomd()
data_files = set()
for i in range(0, 6):
rf = queues['master'].get(True)
r = cr.RepomdRecord(*rf[0])
r.checksum_open_type = cr.checksum_name_str(rf[1][2])
r.checksum_open = rf[1][0]
r.size_open = rf[1][1]
r.fill(cr.SHA256)
if not rf[0][0].endswith('_db'):
queues[rf[0][0]].put(r.checksum, True)
r.rename_file()
r.location_href = os.path.join('repodata',
os.path.basename(r.location_href))
data_files.add(r.location_real)
repomd.set_record(r)
for t in th:
t.join()
repomd.sort_records()
return (repomd.xml_dump(), data_files)
def _gen_basic_delta(self, md, force_gen=False):
"""Resolve some common situation during delta generation.
There is some situation which could appear during
delta generation:
# - Metadata file has a record in repomd.xml and the file really exists
O - Metadata file has a record in repomd.xml but the file is missing
X - Metadata file doesn't have a record in repomd.xml
Old repository | New repository
---------------|---------------
# | # - Valid case
# | X - Valid case - metadata was removed
# | O - Invalid case - incomplete repo
X | # - Valid case - metadata was added
X | X - This shouldn't happen
X | O - Invalid case - incomplete repo
O | # - Invalid case - incomplete repo
O | X - Invalid case - incomplete repo
O | O - Invalid case - both repos are incomplete
By default, Deltarepo should raise an exception when a invalid
case is meet. But user could use --ignore-missing option and
in that case, the Deltarepo should handle all invalid case
like a charm.
For example:
O | # - Just copy the new metadata to the delta repo as is
O | X - Just ignore that the old metadata is missing
O | O - Just ignore this
# | O - Just ignore this
X | O - Just ignore this
Most delta plugins should be only interested to "# | #" use case.
The case where we have the both, old and new, metadata available.
Other cases are mostly not important to the delta plugins.
This is the reason why this function exits. It should solve the
cases when the sophisticated delta is not possible.
Returns (SC - Status code,
REC - Repomd record,
NOTES - Dict with persistent notes)
If RC is True, then delta plugin shouldn't continue with
processing of this metadata.
"""
if not md:
# No metadata - Nothing to do
return (True, None, None)
md.delta_rec = None
md.delta_fn_exists = False
if not md.old_rec and not md.new_rec:
# None metadata record exists.
self._debug("\"{0}\": Doesn't exist "
"in any repo".format(md.metadata_type))
return (True, None, None)
if not md.new_rec:
# This record doesn't exists in the new version of repodata
# This metadata were removed in the new version of repo
self._debug("\"{0}\": Removed in the new version of repodata"
"".format(md.metadata_type))
return (True, None, None)
if not md.new_fn_exists:
# The new metadata file is missing
assert self.globalbundle.ignore_missing
self._warning("\"{0}\": Delta cannot be generated - new metadata "
"are missing".format(md.metadata_type))
return (True, None, None)
if not md.old_rec or not md.old_fn_exists or \
(force_gen and not filecmp.cmp(md.old_fn, md.new_fn)):
# This metadata was newly added in the new version of repodata
# Or we are just missing the old version of this metadata
# Or we have both versions of metadata but the metadata are not
# same and in that case we simply want to gen a delta as a copy
if md.old_fn_exists:
self._debug(
"\"{0}\": Newly added in the new version of repodata"
"".format(md.metadata_type))
elif not md.old_fn_exists:
self._warning(
"\"{0}\": Delta cannot be generated - old metadata "
"are missing - Using copy of the new one"
"".format(md.metadata_type))
else:
self._debug(
"\"{0}\": Delta is just a copy of the new metadata")
# Suffix based detection of compression
compressed = False
for suffix in COMPRESSION_SUFFIXES:
if md.new_fn.endswith(suffix):
compressed = True
break
compression = cr.NO_COMPRESSION
if not compressed:
compression = cr.XZ
# Gen record
rec = self.gen_use_original(md, compression_type=compression)
notes = {}
notes["original"] = '1'
if compression != cr.NO_COMPRESSION:
notes["compressed"] = "1"
md.delta_rec = rec
md.delta_fn_exists = True
return (True, rec, notes)
# At this point we are sure that we have both metadata files
if filecmp.cmp(md.old_fn, md.new_fn):
# Both metadata files exists and are the same
self._debug("\"{0}\": Same in both version of repodata"
"".format(md.metadata_type))
notes = {}
if os.path.basename(md.old_fn) != os.path.basename(md.new_fn):
notes["new_name"] = os.path.basename(md.new_fn)
notes["unchanged"] = "1"
notes["checksum_name"] = cr.checksum_name_str(md.checksum_type)
return (True, None, notes)
# Both metadata files exists and are different,
# this is job for a real delta plugin :)
return (False, None, None)
def _gen_basic_delta(self, md, force_gen=False):
"""Resolve some common situation during delta generation.
There is some situation which could appear during
delta generation:
# - Metadata file has a record in repomd.xml and the file really exists
O - Metadata file has a record in repomd.xml but the file is missing
X - Metadata file doesn't have a record in repomd.xml
Old repository | New repository
---------------|---------------
# | # - Valid case
# | X - Valid case - metadata was removed
# | O - Invalid case - incomplete repo
X | # - Valid case - metadata was added
X | X - This shouldn't happen
X | O - Invalid case - incomplete repo
O | # - Invalid case - incomplete repo
O | X - Invalid case - incomplete repo
O | O - Invalid case - both repos are incomplete
By default, Deltarepo should raise an exception when a invalid
case is meet. But user could use --ignore-missing option and
in that case, the Deltarepo should handle all invalid case
like a charm.
For example:
O | # - Just copy the new metadata to the delta repo as is
O | X - Just ignore that the old metadata is missing
O | O - Just ignore this
# | O - Just ignore this
X | O - Just ignore this
Most delta plugins should be only interested to "# | #" use case.
The case where we have the both, old and new, metadata available.
Other cases are mostly not important to the delta plugins.
This is the reason why this function exits. It should solve the
cases when the sophisticated delta is not possible.
Returns (SC - Status code,
REC - Repomd record,
NOTES - Dict with persistent notes)
If RC is True, then delta plugin shouldn't continue with
processing of this metadata.
"""
if not md:
# No metadata - Nothing to do
return (True, None, None)
md.delta_rec = None
md.delta_fn_exists = False
if not md.old_rec and not md.new_rec:
# None metadata record exists.
self._debug("\"{0}\": Doesn't exist "
"in any repo".format(md.metadata_type))
return (True, None, None)
if not md.new_rec:
# This record doesn't exists in the new version of repodata
# This metadata were removed in the new version of repo
self._debug("\"{0}\": Removed in the new version of repodata"
"".format(md.metadata_type))
return (True, None, None)
if not md.new_fn_exists:
# The new metadata file is missing
assert self.globalbundle.ignore_missing
self._warning("\"{0}\": Delta cannot be generated - new metadata "
"are missing".format(md.metadata_type))
return (True, None, None)
if not md.old_rec or not md.old_fn_exists or \
(force_gen and not filecmp.cmp(md.old_fn, md.new_fn)):
# This metadata was newly added in the new version of repodata
# Or we are just missing the old version of this metadata
# Or we have both versions of metadata but the metadata are not
# same and in that case we simply want to gen a delta as a copy
if md.old_fn_exists:
self._debug("\"{0}\": Newly added in the new version of repodata"
"".format(md.metadata_type))
elif not md.old_fn_exists:
self._warning("\"{0}\": Delta cannot be generated - old metadata "
"are missing - Using copy of the new one"
"".format(md.metadata_type))
else:
self._debug("\"{0}\": Delta is just a copy of the new metadata")
# Suffix based detection of compression
compressed = False
for suffix in COMPRESSION_SUFFIXES:
if md.new_fn.endswith(suffix):
compressed = True
break
compression = cr.NO_COMPRESSION
if not compressed:
compression = cr.XZ
# Gen record
rec = self.gen_use_original(md, compression_type=compression)
notes = {}
notes["original"] = '1'
if compression != cr.NO_COMPRESSION:
notes["compressed"] = "1"
md.delta_rec = rec
md.delta_fn_exists = True
return (True, rec, notes)
# At this point we are sure that we have both metadata files
if filecmp.cmp(md.old_fn, md.new_fn):
# Both metadata files exists and are the same
self._debug("\"{0}\": Same in both version of repodata"
"".format(md.metadata_type))
notes = {}
if os.path.basename(md.old_fn) != os.path.basename(md.new_fn):
notes["new_name"] = os.path.basename(md.new_fn)
notes["unchanged"] = "1"
notes["checksum_name"] = cr.checksum_name_str(md.checksum_type)
return (True, None, notes)
# Both metadata files exists and are different,
# this is job for a real delta plugin :)
return (False, None, None)
