def test_compress_file(self):
# Non exist file
self.assertRaises(IOError, cr.compress_file, self.nofile, None, cr.BZ2)
# Compression - use the same name+suffix
cr.compress_file(self.tmpfile, None, cr.BZ2)
self.assertTrue(os.path.isfile(self.tmpfile + ".bz2"))
# Compression - new name
new_name = os.path.join(self.tmpdir, "foobar.gz")
cr.compress_file(self.tmpfile, new_name, cr.GZ)
self.assertTrue(os.path.isfile(new_name))
# Compression - with stat
stat = cr.ContentStat(cr.SHA256)
cr.compress_file(self.tmpfile, None, cr.XZ, stat)
self.assertTrue(os.path.isfile(self.tmpfile + ".xz"))
self.assertEqual(stat.checksum, "e61ebaa6241e335c779194ce7af98c590f1"\
"b26a749f219b997a0d7d5a773063b")
self.assertEqual(stat.checksum_type, cr.SHA256)
self.assertEqual(stat.size, len(self.content))
# Check directory for unexpected files
self.assertEqual(set(os.listdir(self.tmpdir)),
set(['file.bz2', 'file.xz', 'file', 'foobar.gz']))
def test_decompress_file(self):
# Non exist file
self.assertRaises(IOError, cr.decompress_file, self.nofile, None,
cr.BZ2)
tmpfile_gz_comp = os.path.join(self.tmpdir, "gzipedfile.gz")
shutil.copy(FILE_TEXT_GZ, tmpfile_gz_comp)
tmpfile_gz_comp_ns = os.path.join(self.tmpdir, "gzipedfile_no_suffix")
shutil.copy(FILE_TEXT_GZ, tmpfile_gz_comp_ns)
# Decompression - use the same name without suffix
dest = os.path.join(self.tmpdir, "gzipedfile")
cr.decompress_file(tmpfile_gz_comp, None, cr.GZ)
self.assertTrue(os.path.isfile(dest))
# Decompression - use the specific name
dest = os.path.join(self.tmpdir, "decompressed.file")
cr.decompress_file(tmpfile_gz_comp, dest, cr.GZ)
self.assertTrue(os.path.isfile(dest))
# Decompression - bad suffix by default
dest = os.path.join(self.tmpdir, "gzipedfile_no_suffix.decompressed")
cr.decompress_file(tmpfile_gz_comp_ns, None, cr.GZ)
self.assertTrue(os.path.isfile(dest))
# Decompression - with stat
stat = cr.ContentStat(cr.SHA256)
dest = os.path.join(self.tmpdir, "gzipedfile")
cr.decompress_file(tmpfile_gz_comp, None, cr.AUTO_DETECT_COMPRESSION,
stat)
self.assertTrue(os.path.isfile(dest))
self.assertEqual(stat.checksum, FILE_TEXT_SHA256SUM)
self.assertEqual(stat.checksum_type, cr.SHA256)
self.assertEqual(stat.size, 910)
def prepare_paths_in_metadata(md, xmlclass, dbclass):
if md is None:
return
notes = self._metadata_notes_from_plugin_bundle(md.metadata_type)
if not notes:
# TODO: Add flag to ignore this kind of warnings (?)
self._warning("Metadata \"{0}\" doesn't have a record in "
"deltametadata.xml - Ignoring")
return
suffix = cr.compression_suffix(md.compression_type) or ""
md.new_fn = os.path.join(
md.out_dir, "{0}.xml{1}".format(md.metadata_type, suffix))
md.new_f_stat = cr.ContentStat(md.checksum_type)
md.new_f = xmlclass(md.new_fn, md.compression_type, md.new_f_stat)
if self.globalbundle.force_database or notes.get(
"database") == "1":
md.db_fn = os.path.join(md.out_dir,
"{0}.sqlite".format(md.metadata_type))
md.db = dbclass(md.db_fn)
else:
md.db_fn = None
md.db = None
def gen_use_original(self, md, compression_type=cr.NO_COMPRESSION):
"""Function that takes original metadata file and
copy it to the delta repo unmodified.
Plugins could use this function when they cannot generate delta file
for some reason (eg. file is newly added, so delta is
meaningless/impossible)."""
md.delta_fn = os.path.join(md.out_dir, os.path.basename(md.new_fn))
# Compress or copy original file
stat = None
if (compression_type != cr.NO_COMPRESSION):
md.delta_fn += cr.compression_suffix(compression_type)
stat = cr.ContentStat(md.checksum_type)
cr.compress_file(md.new_fn, md.delta_fn, compression_type, stat)
else:
shutil.copy2(md.new_fn, md.delta_fn)
# Prepare repomd record of xml file
rec = cr.RepomdRecord(md.metadata_type, md.delta_fn)
if stat is not None:
rec.load_contentstat(stat)
rec.fill(md.checksum_type)
if self.globalbundle.unique_md_filenames:
rec.rename_file()
md.delta_fn = rec.location_real
return rec
def test_contentstat(self):
pkg = cr.package_from_rpm(PKG_ARCHER_PATH)
self.assertTrue(pkg)
pkg.time_file = 1
pkg.time_build = 1
cs = cr.ContentStat(cr.SHA256)
self.assertEqual(cs.size, 0)
self.assertEqual(cs.checksum_type, cr.SHA256)
self.assertEqual(cs.checksum, None)
path = os.path.join(self.tmpdir, "primary.xml.gz")
f = cr.PrimaryXmlFile(path, cr.GZ_COMPRESSION, cs)
self.assertTrue(f)
self.assertTrue(os.path.isfile(path))
f.add_pkg(pkg)
f.close()
self.assertTrue(os.path.isfile(path))
self.assertEqual(cs.size, 2668)
self.assertEqual(cs.checksum_type, cr.SHA256)
self.assertEqual(cs.checksum, "67bc6282915fad80dc11f3d7c3210977a0bde"\
"05a762256d86083c2447d425776")
def __create_xml(queues, xml_path, xml_func, name):
cs = cr.ContentStat(cr.SHA256)
xml = xml_func(xml_path, contentstat=cs)
xml.set_num_of_pkgs(len(pkglist))
for pkg in pkglist:
xml.add_pkg(pkg)
xml.close()
queues['master'].put(
((name, xml_path), (cs.checksum, cs.size, cs.checksum_type)), True)
def _gen_db_from_xml(self, md):
"""Gen sqlite db from the delta metadata.
"""
mdtype = md.metadata_type
if mdtype == "primary":
dbclass = cr.PrimarySqlite
parsefunc = cr.xml_parse_primary
elif mdtype == "filelists":
dbclass = cr.FilelistsSqlite
parsefunc = cr.xml_parse_filelists
elif mdtype == "other":
dbclass = cr.OtherSqlite
parsefunc = cr.xml_parse_other
else:
raise DeltaRepoPluginError(
"Unsupported type of metadata {0}".format(mdtype))
src_fn = md.new_fn
src_rec = md.new_rec
md.db_fn = os.path.join(md.out_dir, "{0}.sqlite".format(mdtype))
db = dbclass(md.db_fn)
def pkgcb(pkg):
db.add_pkg(pkg)
parsefunc(src_fn, pkgcb=pkgcb)
db.dbinfo_update(src_rec.checksum)
db.close()
db_stat = cr.ContentStat(md.checksum_type)
db_compressed = md.db_fn + ".bz2"
cr.compress_file(md.db_fn, None, cr.BZ2, db_stat)
os.remove(md.db_fn)
# Prepare repomd record of database file
db_rec = cr.RepomdRecord("{0}_db".format(md.metadata_type),
db_compressed)
db_rec.load_contentstat(db_stat)
db_rec.fill(md.checksum_type)
if self.globalbundle.unique_md_filenames:
db_rec.rename_file()
return db_rec
def test_contentstat_ref_in_crfile(self):
"""Test if reference is saved properly"""
cs = cr.ContentStat(cr.SHA256)
self.assertEqual(cs.size, 0)
self.assertEqual(cs.checksum_type, cr.SHA256)
self.assertEqual(cs.checksum, None)
path = os.path.join(self.tmpdir, "foofile.gz")
f = cr.CrFile(path, cr.MODE_WRITE, cr.GZ_COMPRESSION, cs)
self.assertTrue(f)
self.assertTrue(os.path.isfile(path))
del cs
f.write("foobar")
f.close()
self.assertTrue(os.path.isfile(path))
def test_repomdrecord_load_contentstat(self):
rec = cr.RepomdRecord("primary", None)
self.assertTrue(rec)
stat = cr.ContentStat(cr.SHA256)
stat.checksum = "foobar"
stat.checksum_type = cr.SHA256
stat.size = 123
self.assertEqual(rec.checksum_open, None)
self.assertEqual(rec.checksum_open_type, None)
self.assertEqual(rec.size, 0)
rec.load_contentstat(stat);
self.assertEqual(rec.checksum_open, "foobar")
self.assertEqual(rec.checksum_open_type, "sha256")
self.assertEqual(rec.size_open, 123)
def __create_db(queues, db_path, db_func, name):
db = db_func(db_path)
for pkg in pkglist:
db.add_pkg(pkg)
db.dbinfo_update(queues[name].get(True))
db.close()
cs = cr.ContentStat(cr.SHA256)
cr.compress_file_with_stat(
db_path, db_path + cr.compression_suffix(cr.BZ2_COMPRESSION),
cr.BZ2_COMPRESSION, cs)
os.remove(db_path)
queues['master'].put(
((name + '_db',
db_path + cr.compression_suffix(cr.BZ2_COMPRESSION)),
(cs.checksum, cs.size, cs.checksum_type)), True)
def apply(self, metadata):
gen_repomd_recs = []
md_group = metadata.get("group")
md_group_gz = metadata.get("group_gz")
if md_group and (not md_group.delta_fn_exists
and not md_group.old_fn_exists):
md_group = None
if md_group_gz and (not md_group_gz.delta_fn_exists
and not md_group_gz.old_fn_exists):
md_group_gz = None
if md_group:
notes = self._metadata_notes_from_plugin_bundle(
md_group.metadata_type)
rc, rec = self._apply_basic_delta(md_group, notes)
assert rc
if rec:
gen_repomd_recs.append(rec)
if notes.get("gen_group_gz"):
# Gen group_gz metadata from the group metadata
stat = cr.ContentStat(md_group.checksum_type)
group_gz_fn = md_group.new_fn + ".gz"
cr.compress_file(md_group.new_fn, group_gz_fn, cr.GZ, stat)
rec = cr.RepomdRecord("group_gz", group_gz_fn)
rec.load_contentstat(stat)
rec.fill(md_group.checksum_type)
if self.globalbundle.unique_md_filenames:
rec.rename_file()
gen_repomd_recs.append(rec)
elif md_group_gz:
notes = self._metadata_notes_from_plugin_bundle(
md_group_gz.metadata_type)
rc, rec = self._apply_basic_delta(md_group_gz, notes)
assert rc
if rec:
gen_repomd_recs.append(rec)
return gen_repomd_recs
def prepare_paths_in_metadata(md, xmlclass):
if md is None:
return None
# Make a note about if the database should be generated
db_available = metadata.get(md.metadata_type + "_db").new_fn_exists
if db_available or self.globalbundle.force_database:
metadata_notes.setdefault(md.metadata_type,
{})["database"] = "1"
else:
metadata_notes.setdefault(md.metadata_type,
{})["database"] = "0"
suffix = cr.compression_suffix(md.compression_type) or ""
md.delta_fn = os.path.join(
md.out_dir, "{0}.xml{1}".format(md.metadata_type, suffix))
md.delta_f_stat = cr.ContentStat(md.checksum_type)
md.delta_f = xmlclass(md.delta_fn, md.compression_type,
md.delta_f_stat)
return md
def finish_metadata(md):
if md is None:
return
# Close XML file
md.new_f.close()
# Prepare repomd record of xml file
rec = cr.RepomdRecord(md.metadata_type, md.new_fn)
rec.load_contentstat(md.new_f_stat)
rec.fill(md.checksum_type)
if self.globalbundle.unique_md_filenames:
rec.rename_file()
md.new_rec = rec
md.new_fn_exists = True
gen_repomd_recs.append(rec)
# Prepare database
if hasattr(md, "db") and md.db:
self._debug("Generating database: {0}".format(md.db_fn))
md.db.dbinfo_update(rec.checksum)
md.db.close()
db_stat = cr.ContentStat(md.checksum_type)
db_compressed = md.db_fn + ".bz2"
cr.compress_file(md.db_fn, None, cr.BZ2, db_stat)
os.remove(md.db_fn)
# Prepare repomd record of database file
db_rec = cr.RepomdRecord("{0}_db".format(md.metadata_type),
db_compressed)
db_rec.load_contentstat(db_stat)
db_rec.fill(md.checksum_type)
if self.globalbundle.unique_md_filenames:
db_rec.rename_file()
gen_repomd_recs.append(db_rec)
def test_contentstat_ref_in_xmlfile(self):
"""Test if reference is saved properly"""
pkg = cr.package_from_rpm(PKG_ARCHER_PATH)
self.assertTrue(pkg)
pkg.time_file = 1
pkg.time_build = 1
cs = cr.ContentStat(cr.SHA256)
self.assertEqual(cs.size, 0)
self.assertEqual(cs.checksum_type, cr.SHA256)
self.assertEqual(cs.checksum, None)
path = os.path.join(self.tmpdir, "primary.xml.gz")
f = cr.PrimaryXmlFile(path, cr.GZ_COMPRESSION, cs)
self.assertTrue(f)
self.assertTrue(os.path.isfile(path))
del cs
f.add_pkg(pkg)
f.close()
self.assertTrue(os.path.isfile(path))
def gen(self):
# Prepare output path
os.mkdir(self.delta_repodata_path)
# Set of types of processed metadata records ("primary", "primary_db"...)
processed_metadata = set()
for plugin in PLUGINS:
# Prepare metadata for the plugin
metadata_objects = {}
for metadata_name in plugin.METADATA:
metadata_object = self._new_metadata(metadata_name)
if metadata_object is not None:
metadata_objects[metadata_name] = metadata_object
# Skip plugin if no supported metadata available
if not metadata_objects:
self._debug("Plugin {0}: Skipped - None of supported " \
"metadata {1} available".format(
plugin.NAME, plugin.METADATA))
continue
# Prepare plugin bundle
pluginbundle = PluginBundle(plugin.NAME, plugin.VERSION)
self.deltametadata.add_pluginbundle(pluginbundle)
# Use the plugin
self._debug("Plugin {0}: Active".format(plugin.NAME))
plugin_instance = plugin(pluginbundle,
self.globalbundle,
logger=self._get_logger())
repomd_records = plugin_instance.gen(metadata_objects)
# Put repomd records from processed metadatas to repomd
self._debug("Plugin {0}: Processed {1} record(s) " \
"and produced:".format(plugin.NAME, metadata_objects.keys()))
for rec in repomd_records:
self._debug(" - {0}".format(rec.type))
self.delta_repomd.set_record(rec)
# Organization stuff
for md in metadata_objects.keys():
processed_metadata.add(md)
# Process rest of the metadata files
metadata_objects = {}
for rectype, rec in self.new_records.items():
if rectype not in processed_metadata:
metadata_object = self._new_metadata(rectype)
if metadata_object is not None:
self._debug("To be processed by general delta plugin: " \
"{0}".format(rectype))
metadata_objects[rectype] = metadata_object
else:
self._debug("Not processed - even by general delta " \
"plugin: {0}".format(rectype))
if metadata_objects:
# Use the plugin
pluginbundle = PluginBundle(GENERAL_PLUGIN.NAME,
GENERAL_PLUGIN.VERSION)
self.deltametadata.add_pluginbundle(pluginbundle)
self._debug("Plugin {0}: Active".format(GENERAL_PLUGIN.NAME))
plugin_instance = GENERAL_PLUGIN(pluginbundle,
self.globalbundle,
logger=self._get_logger())
repomd_records = plugin_instance.gen(metadata_objects)
# Put repomd records from processed metadatas to repomd
self._debug("Plugin {0}: Processed {1} record(s) " \
"and produced:".format(GENERAL_PLUGIN.NAME, metadata_objects.keys()))
for rec in repomd_records:
self._debug(" - {0}".format(rec.type))
self.delta_repomd.set_record(rec)
# Check if calculated contenthashes match
# and calculate them if they don't exist
self.check_content_hashes()
# Write out deltametadata.xml
self.fill_deltametadata()
deltametadata_path = os.path.join(self.delta_repodata_path,
"deltametadata.xml")
stat = cr.ContentStat(self.checksum_type)
deltametadata_path = self.deltametadata.dump(
deltametadata_path,
compression_type=self.compression_type,
stat=stat)
deltametadata_rec = cr.RepomdRecord("deltametadata",
deltametadata_path)
deltametadata_rec.load_contentstat(stat)
deltametadata_rec.fill(self.checksum_type)
if self.unique_md_filenames:
deltametadata_rec.rename_file()
self.delta_repomd.set_record(deltametadata_rec)
# Prepare and write out the new repomd.xml
self._debug("Preparing repomd.xml ...")
deltacontenthash = "{0}-{1}".format(
self.globalbundle.calculated_old_contenthash,
self.globalbundle.calculated_new_contenthash)
self.delta_repomd.set_contenthash(deltacontenthash,
self.contenthash_type_str)
self.delta_repomd.sort_records()
delta_repomd_xml = self.delta_repomd.xml_dump()
self._debug("Writing repomd.xml ...")
open(self.delta_repomd_path, "w").write(delta_repomd_xml)
# Final move
if os.path.exists(self.final_path):
self._warning("Destination dir already exists! Removing %s" % \
self.final_path)
shutil.rmtree(self.final_path)
self._debug("Moving %s -> %s" %
(self.delta_repodata_path, self.final_path))
os.rename(self.delta_repodata_path, self.final_path)
