def gen_use_original(self, md, compression_type=cr.NO_COMPRESSION):
"""Function that takes original metadata file and
copy it to the delta repo unmodified.
Plugins could use this function when they cannot generate delta file
for some reason (eg. file is newly added, so delta is
meaningless/impossible)."""
md.delta_fn = os.path.join(md.out_dir, os.path.basename(md.new_fn))
# Compress or copy original file
stat = None
if (compression_type != cr.NO_COMPRESSION):
md.delta_fn += cr.compression_suffix(compression_type)
stat = cr.ContentStat(md.checksum_type)
cr.compress_file(md.new_fn, md.delta_fn, compression_type, stat)
else:
shutil.copy2(md.new_fn, md.delta_fn)
# Prepare repomd record of xml file
rec = cr.RepomdRecord(md.metadata_type, md.delta_fn)
if stat is not None:
rec.load_contentstat(stat)
rec.fill(md.checksum_type)
if self.globalbundle.unique_md_filenames:
rec.rename_file()
md.delta_fn = rec.location_real
return rec
def gen_use_original(self, md, compression_type=cr.NO_COMPRESSION):
"""Function that takes original metadata file and
copy it to the delta repo unmodified.
Plugins could use this function when they cannot generate delta file
for some reason (eg. file is newly added, so delta is
meaningless/impossible)."""
md.delta_fn = os.path.join(md.out_dir, os.path.basename(md.new_fn))
# Compress or copy original file
stat = None
if (compression_type != cr.NO_COMPRESSION):
md.delta_fn += cr.compression_suffix(compression_type)
stat = cr.ContentStat(md.checksum_type)
cr.compress_file(md.new_fn, md.delta_fn, compression_type, stat)
else:
shutil.copy2(md.new_fn, md.delta_fn)
# Prepare repomd record of xml file
rec = cr.RepomdRecord(md.metadata_type, md.delta_fn)
if stat is not None:
rec.load_contentstat(stat)
rec.fill(md.checksum_type)
if self.globalbundle.unique_md_filenames:
rec.rename_file()
md.delta_fn = rec.location_real
return rec
def _gen_db_from_xml(self, md):
"""Gen sqlite db from the delta metadata.
"""
mdtype = md.metadata_type
if mdtype == "primary":
dbclass = cr.PrimarySqlite
parsefunc = cr.xml_parse_primary
elif mdtype == "filelists":
dbclass = cr.FilelistsSqlite
parsefunc = cr.xml_parse_filelists
elif mdtype == "other":
dbclass = cr.OtherSqlite
parsefunc = cr.xml_parse_other
else:
raise DeltaRepoPluginError(
"Unsupported type of metadata {0}".format(mdtype))
src_fn = md.new_fn
src_rec = md.new_rec
md.db_fn = os.path.join(md.out_dir, "{0}.sqlite".format(mdtype))
db = dbclass(md.db_fn)
def pkgcb(pkg):
db.add_pkg(pkg)
parsefunc(src_fn, pkgcb=pkgcb)
db.dbinfo_update(src_rec.checksum)
db.close()
db_stat = cr.ContentStat(md.checksum_type)
db_compressed = md.db_fn + ".bz2"
cr.compress_file(md.db_fn, None, cr.BZ2, db_stat)
os.remove(md.db_fn)
# Prepare repomd record of database file
db_rec = cr.RepomdRecord("{0}_db".format(md.metadata_type),
db_compressed)
db_rec.load_contentstat(db_stat)
db_rec.fill(md.checksum_type)
if self.globalbundle.unique_md_filenames:
db_rec.rename_file()
return db_rec
def _gen_db_from_xml(self, md):
"""Gen sqlite db from the delta metadata.
"""
mdtype = md.metadata_type
if mdtype == "primary":
dbclass = cr.PrimarySqlite
parsefunc = cr.xml_parse_primary
elif mdtype == "filelists":
dbclass = cr.FilelistsSqlite
parsefunc = cr.xml_parse_filelists
elif mdtype == "other":
dbclass = cr.OtherSqlite
parsefunc = cr.xml_parse_other
else:
raise DeltaRepoPluginError("Unsupported type of metadata {0}".format(mdtype))
src_fn = md.new_fn
src_rec = md.new_rec
md.db_fn = os.path.join(md.out_dir, "{0}.sqlite".format(mdtype))
db = dbclass(md.db_fn)
def pkgcb(pkg):
db.add_pkg(pkg)
parsefunc(src_fn, pkgcb=pkgcb)
db.dbinfo_update(src_rec.checksum)
db.close()
db_stat = cr.ContentStat(md.checksum_type)
db_compressed = md.db_fn+".bz2"
cr.compress_file(md.db_fn, None, cr.BZ2, db_stat)
os.remove(md.db_fn)
# Prepare repomd record of database file
db_rec = cr.RepomdRecord("{0}_db".format(md.metadata_type),
db_compressed)
db_rec.load_contentstat(db_stat)
db_rec.fill(md.checksum_type)
if self.globalbundle.unique_md_filenames:
db_rec.rename_file()
return db_rec
def apply(self, metadata):
gen_repomd_recs = []
md_group = metadata.get("group")
md_group_gz = metadata.get("group_gz")
if md_group and (not md_group.delta_fn_exists
and not md_group.old_fn_exists):
md_group = None
if md_group_gz and (not md_group_gz.delta_fn_exists
and not md_group_gz.old_fn_exists):
md_group_gz = None
if md_group:
notes = self._metadata_notes_from_plugin_bundle(
md_group.metadata_type)
rc, rec = self._apply_basic_delta(md_group, notes)
assert rc
if rec:
gen_repomd_recs.append(rec)
if notes.get("gen_group_gz"):
# Gen group_gz metadata from the group metadata
stat = cr.ContentStat(md_group.checksum_type)
group_gz_fn = md_group.new_fn + ".gz"
cr.compress_file(md_group.new_fn, group_gz_fn, cr.GZ, stat)
rec = cr.RepomdRecord("group_gz", group_gz_fn)
rec.load_contentstat(stat)
rec.fill(md_group.checksum_type)
if self.globalbundle.unique_md_filenames:
rec.rename_file()
gen_repomd_recs.append(rec)
elif md_group_gz:
notes = self._metadata_notes_from_plugin_bundle(
md_group_gz.metadata_type)
rc, rec = self._apply_basic_delta(md_group_gz, notes)
assert rc
if rec:
gen_repomd_recs.append(rec)
return gen_repomd_recs
def test_compress_file(self):
# Non exist file
self.assertRaises(IOError, cr.compress_file, self.nofile, None, cr.BZ2)
# Compression - use the same name+suffix
cr.compress_file(self.tmpfile, None, cr.BZ2)
self.assertTrue(os.path.isfile(self.tmpfile + ".bz2"))
# Compression - new name
new_name = os.path.join(self.tmpdir, "foobar.gz")
cr.compress_file(self.tmpfile, new_name, cr.GZ)
self.assertTrue(os.path.isfile(new_name))
# Compression - with stat
stat = cr.ContentStat(cr.SHA256)
cr.compress_file(self.tmpfile, None, cr.XZ, stat)
self.assertTrue(os.path.isfile(self.tmpfile + ".xz"))
self.assertEqual(stat.checksum, "e61ebaa6241e335c779194ce7af98c590f1"\
"b26a749f219b997a0d7d5a773063b")
self.assertEqual(stat.checksum_type, cr.SHA256)
self.assertEqual(stat.size, len(self.content))
# Check directory for unexpected files
self.assertEqual(set(os.listdir(self.tmpdir)),
set(['file.bz2', 'file.xz', 'file', 'foobar.gz']))
def test_compress_file(self):
# Non exist file
self.assertRaises(IOError, cr.compress_file,
self.nofile, None, cr.BZ2)
# Compression - use the same name+suffix
cr.compress_file(self.tmpfile, None, cr.BZ2)
self.assertTrue(os.path.isfile(self.tmpfile+".bz2"))
# Compression - new name
new_name = os.path.join(self.tmpdir, "foobar.gz")
cr.compress_file(self.tmpfile, new_name, cr.GZ)
self.assertTrue(os.path.isfile(new_name))
# Compression - with stat
stat = cr.ContentStat(cr.SHA256)
cr.compress_file(self.tmpfile, None, cr.XZ, stat)
self.assertTrue(os.path.isfile(self.tmpfile+".xz"))
self.assertEqual(stat.checksum, "e61ebaa6241e335c779194ce7af98c590f1"\
"b26a749f219b997a0d7d5a773063b")
self.assertEqual(stat.checksum_type, cr.SHA256)
self.assertEqual(stat.size, len(self.content))
# Check directory for unexpected files
self.assertEqual(set(os.listdir(self.tmpdir)),
set(['file.bz2', 'file.xz', 'file', 'foobar.gz']))
def apply(self, metadata):
gen_repomd_recs = []
md_group = metadata.get("group")
md_group_gz = metadata.get("group_gz")
if md_group and (not md_group.delta_fn_exists
and not md_group.old_fn_exists):
md_group = None
if md_group_gz and (not md_group_gz.delta_fn_exists
and not md_group_gz.old_fn_exists):
md_group_gz = None
if md_group:
notes = self._metadata_notes_from_plugin_bundle(md_group.metadata_type)
rc, rec = self._apply_basic_delta(md_group, notes)
assert rc
if rec:
gen_repomd_recs.append(rec)
if notes.get("gen_group_gz"):
# Gen group_gz metadata from the group metadata
stat = cr.ContentStat(md_group.checksum_type)
group_gz_fn = md_group.new_fn+".gz"
cr.compress_file(md_group.new_fn, group_gz_fn, cr.GZ, stat)
rec = cr.RepomdRecord("group_gz", group_gz_fn)
rec.load_contentstat(stat)
rec.fill(md_group.checksum_type)
if self.globalbundle.unique_md_filenames:
rec.rename_file()
gen_repomd_recs.append(rec)
elif md_group_gz:
notes = self._metadata_notes_from_plugin_bundle(md_group_gz.metadata_type)
rc, rec = self._apply_basic_delta(md_group_gz, notes)
assert rc
if rec:
gen_repomd_recs.append(rec)
return gen_repomd_recs
def finish_metadata(md):
if md is None:
return
# Close XML file
md.new_f.close()
# Prepare repomd record of xml file
rec = cr.RepomdRecord(md.metadata_type, md.new_fn)
rec.load_contentstat(md.new_f_stat)
rec.fill(md.checksum_type)
if self.globalbundle.unique_md_filenames:
rec.rename_file()
md.new_rec = rec
md.new_fn_exists = True
gen_repomd_recs.append(rec)
# Prepare database
if hasattr(md, "db") and md.db:
self._debug("Generating database: {0}".format(md.db_fn))
md.db.dbinfo_update(rec.checksum)
md.db.close()
db_stat = cr.ContentStat(md.checksum_type)
db_compressed = md.db_fn + ".bz2"
cr.compress_file(md.db_fn, None, cr.BZ2, db_stat)
os.remove(md.db_fn)
# Prepare repomd record of database file
db_rec = cr.RepomdRecord("{0}_db".format(md.metadata_type),
db_compressed)
db_rec.load_contentstat(db_stat)
db_rec.fill(md.checksum_type)
if self.globalbundle.unique_md_filenames:
db_rec.rename_file()
gen_repomd_recs.append(db_rec)
def finish_metadata(md):
if md is None:
return
# Close XML file
md.new_f.close()
# Prepare repomd record of xml file
rec = cr.RepomdRecord(md.metadata_type, md.new_fn)
rec.load_contentstat(md.new_f_stat)
rec.fill(md.checksum_type)
if self.globalbundle.unique_md_filenames:
rec.rename_file()
md.new_rec = rec
md.new_fn_exists = True
gen_repomd_recs.append(rec)
# Prepare database
if hasattr(md, "db") and md.db:
self._debug("Generating database: {0}".format(md.db_fn))
md.db.dbinfo_update(rec.checksum)
md.db.close()
db_stat = cr.ContentStat(md.checksum_type)
db_compressed = md.db_fn+".bz2"
cr.compress_file(md.db_fn, None, cr.BZ2, db_stat)
os.remove(md.db_fn)
# Prepare repomd record of database file
db_rec = cr.RepomdRecord("{0}_db".format(md.metadata_type),
db_compressed)
db_rec.load_contentstat(db_stat)
db_rec.fill(md.checksum_type)
if self.globalbundle.unique_md_filenames:
db_rec.rename_file()
gen_repomd_recs.append(db_rec)
def applydelta(self, old_path, delta_path, out_path=None, database=False):
removedxml = RemovedXml()
hash_in_the_name = False
# Prepare variables with paths
old_repodata_path = os.path.join(old_path, "repodata/")
delta_repodata_path = os.path.join(delta_path, "repodata/")
old_repomd_path = os.path.join(old_repodata_path, "repomd.xml")
delta_repomd_path = os.path.join(delta_repodata_path, "repomd.xml")
# Prepare Repomd objects
old_repomd = cr.Repomd(old_repomd_path)
delta_repomd = cr.Repomd(delta_repomd_path)
new_repomd = cr.Repomd()
# Check if delta id correspond with used repo
if not delta_repomd.repoid or len(delta_repomd.repoid.split('-')) != 2:
raise DeltaRepoError("Bad DeltaRepoId")
self.id_type = delta_repomd.repoid_type
old_id, new_id = delta_repomd.repoid.split('-')
self._debug("Delta %s -> %s" % (old_id, new_id))
if old_repomd.repoid_type == delta_repomd.repoid_type:
if old_repomd.repoid and old_repomd.repoid != old_id:
raise DeltaRepoError("Not suitable delta for current repo " \
"(Expected: %s Real: %s)" % (old_id, old_repomd.repoid))
else:
self._debug("Different repoid types repo: %s vs delta: %s" % \
(old_repomd.repoid_type, delta_repomd.repoid_type))
# Prepare output path
new_path = os.path.join(out_path, ".repodata/")
new_repodata_path = os.path.join(new_path, "repodata/")
os.mkdir(new_path)
os.mkdir(new_repodata_path)
# Apply repomd delta
new_repomd.set_revision(delta_repomd.revision)
for tag in delta_repomd.distro_tags:
new_repomd.add_distro_tag(tag[1], tag[0])
for tag in delta_repomd.repo_tags:
new_repomd.add_repo_tag(tag)
for tag in delta_repomd.content_tags:
new_repomd.add_content_tag(tag)
old_records = dict([(record.type, record) for record in old_repomd.records ])
delta_records = dict([(record.type, record) for record in delta_repomd.records ])
old_record_types = set(old_records.keys())
delta_record_types = set(delta_records.keys())
deleted_repomd_record_types = old_record_types - delta_record_types
added_repomd_record_types = delta_record_types - old_record_types
# Prepare removedxml
if "removed" in delta_records:
removedxml_path = os.path.join(delta_path,
delta_records["removed"].location_href)
removedxml.xml_parse(removedxml_path)
else:
self._warning("\"removed\" record is missing in repomd.xml "\
"of delta repo")
# Important sanity check (repo without primary is definitely bad)
if not "primary" in old_records or not "primary" in delta_records:
raise DeltaRepoError("Missing primary metadata")
# Detect type of checksum in the delta repomd.xml
self.checksum_type = cr.checksum_type(delta_records["primary"].checksum_type)
if self.checksum_type == cr.UNKNOWN_CHECKSUM:
raise DeltaRepoError("Unknown checksum type detected: %s" % \
delta_records["primary"].checksum_type)
# Detection if use unique md filenames
if delta_records["primary"].location_href.split("primary")[0] != "":
hash_in_the_name = True
# Apply delta on primary, filelists and other
pri_old_fn = os.path.join(old_path, old_records["primary"].location_href)
pri_delta_fn = os.path.join(delta_path, delta_records["primary"].location_href)
pri_out_fn = os.path.join(new_repodata_path, "primary.xml.gz")
pri_out_f_stat = cr.ContentStat(self.checksum_type)
pri_out_f = cr.PrimaryXmlFile(pri_out_fn, cr.GZ_COMPRESSION)
pri_db_fn = None
pri_db = None
if database:
pri_db_fn = os.path.join(new_repodata_path, "primary.sqlite")
pri_db = cr.PrimarySqlite(pri_db_fn)
fil_old_fn = None
fil_delta_fn = None
fil_out_fn = None
fil_out_f_stat = None
fil_out_f = None
fil_db_fn = None
fil_db = None
if ("filelists" in delta_records):
fil_old_fn = os.path.join(old_path, old_records["filelists"].location_href)
fil_delta_fn = os.path.join(delta_path, delta_records["filelists"].location_href)
fil_out_fn = os.path.join(new_repodata_path, "filelists.xml.gz")
fil_out_f_stat = cr.ContentStat(self.checksum_type)
fil_out_f = cr.FilelistsXmlFile(fil_out_fn, cr.GZ_COMPRESSION)
if database:
fil_db_fn = os.path.join(new_repodata_path, "filelists.sqlite")
fil_db = cr.FilelistsSqlite(fil_db_fn)
oth_old_fn = None
oth_delta_fn = None
oth_out_fn = None
oth_out_f_stat = None
oth_out_f = None
oth_db_fn = None
oth_db = None
if ("other" in delta_records):
oth_old_fn = os.path.join(old_path, old_records["other"].location_href)
oth_delta_fn = os.path.join(delta_path, delta_records["other"].location_href)
oth_out_fn = os.path.join(new_repodata_path, "other.xml.gz")
oth_out_f_stat = cr.ContentStat(self.checksum_type)
oth_out_f = cr.OtherXmlFile(oth_out_fn, cr.GZ_COMPRESSION)
if database:
oth_db_fn = os.path.join(new_repodata_path, "other.sqlite")
oth_db = cr.OtherSqlite(oth_db_fn)
deltamodule = MainDeltaModule(id_type=self.id_type,
logger=self.logger)
ids = deltamodule.apply(pri_old_fn, pri_delta_fn, pri_out_f, pri_db,
fil_old_fn, fil_delta_fn, fil_out_f, fil_db,
oth_old_fn, oth_delta_fn, oth_out_f, oth_db,
removedxml)
pri_out_f.close()
fil_out_f.close()
oth_out_f.close()
# Check returned IDs
cold_id, cnew_id = ids # Calculated ids
if cold_id != old_id:
raise DeltaRepoError("Calculated old RepoId doesn't match!")
if cnew_id != new_id:
raise DeltaRepoError("Calculated new RepoId doesn't match!")
self._debug("RepoIds match")
# Prepare repomd.xml records
pri_rec = cr.RepomdRecord("primary", pri_out_fn)
pri_rec.load_contentstat(pri_out_f_stat)
pri_rec.fill(self.checksum_type)
if hash_in_the_name:
pri_rec.rename_file()
new_repomd.set_record(pri_rec)
if database:
pri_db.dbinfo_update(pri_rec.checksum)
pri_db.close()
pri_db_stat = cr.ContentStat(self.checksum_type)
pri_db_compressed = os.path.join(pri_db_fn+".bz2")
cr.compress_file(pri_db_fn, None, cr.BZ2, pri_db_stat)
os.remove(pri_db_fn)
pri_db_rec = cr.RepomdRecord("primary_db", pri_db_compressed)
pri_db_rec.load_contentstat(pri_db_stat)
pri_db_rec.fill(self.checksum_type)
if hash_in_the_name:
pri_db_rec.rename_file()
new_repomd.set_record(pri_db_rec)
if fil_out_fn:
fil_rec = cr.RepomdRecord("filelists", fil_out_fn)
fil_rec.load_contentstat(fil_out_f_stat)
fil_rec.fill(self.checksum_type)
if hash_in_the_name:
fil_rec.rename_file()
new_repomd.set_record(fil_rec)
if database:
fil_db.dbinfo_update(fil_rec.checksum)
fil_db.close()
fil_db_stat = cr.ContentStat(self.checksum_type)
fil_db_compressed = os.path.join(fil_db_fn+".bz2")
cr.compress_file(fil_db_fn, None, cr.BZ2, fil_db_stat)
os.remove(fil_db_fn)
fil_db_rec = cr.RepomdRecord("primary_db", fil_db_compressed)
fil_db_rec.load_contentstat(fil_db_stat)
fil_db_rec.fill(self.checksum_type)
if hash_in_the_name:
fil_db_rec.rename_file()
new_repomd.set_record(fil_db_rec)
if oth_out_fn:
oth_rec = cr.RepomdRecord("other", oth_out_fn)
oth_rec.load_contentstat(oth_out_f_stat)
oth_rec.fill(self.checksum_type)
if hash_in_the_name:
oth_rec.rename_file()
new_repomd.set_record(oth_rec)
if database:
oth_db.dbinfo_update(oth_rec.checksum)
oth_db.close()
oth_db_stat = cr.ContentStat(self.checksum_type)
oth_db_compressed = os.path.join(oth_db_fn+".bz2")
cr.compress_file(oth_db_fn, None, cr.BZ2, oth_db_stat)
os.remove(oth_db_fn)
oth_db_rec = cr.RepomdRecord("primary_db", oth_db_compressed)
oth_db_rec.load_contentstat(oth_db_stat)
oth_db_rec.fill(self.checksum_type)
if hash_in_the_name:
oth_db_rec.rename_file()
new_repomd.set_record(oth_db_rec)
# Write out repomd.xml
new_repomd.set_repoid(ids[1], self.id_type)
new_repomd_path = os.path.join(new_repodata_path, "repomd.xml")
new_repomd_xml = new_repomd.xml_dump()
self._debug("Writing repomd.xml")
open(new_repomd_path, "w").write(new_repomd_xml)
# Final move
final_destination = os.path.join(out_path, "repodata/")
if os.path.exists(final_destination):
self._warning("Destination dir already exists! Removing %s" % \
final_destination)
shutil.rmtree(final_destination)
self._info("Moving %s -> %s" % (new_path, final_destination))
os.rename(new_path, final_destination)
