def test_checksum_name_str(self): self.assertEqual(cr.checksum_name_str(cr.MD5), "md5") self.assertEqual(cr.checksum_name_str(cr.SHA), "sha") self.assertEqual(cr.checksum_name_str(cr.SHA1), "sha1") self.assertEqual(cr.checksum_name_str(cr.SHA224), "sha224") self.assertEqual(cr.checksum_name_str(cr.SHA256), "sha256") self.assertEqual(cr.checksum_name_str(cr.SHA384), "sha384") self.assertEqual(cr.checksum_name_str(cr.SHA512), "sha512") self.assertEqual(cr.checksum_name_str(65), None)
def parse_updateinfo(path): uinfo = cr.UpdateInfo(path) for update in uinfo.updates: print "From: %s" % update.fromstr print "Status: %s" % update.status print "Type: %s" % update.type print "Version: %s" % update.version print "Id: %s" % update.id print "Title: %s" % update.title print "Issued date: %s" % update.issued_date print "Updated date: %s" % update.updated_date print "Rights: %s" % update.rights print "Release: %s" % update.release print "Pushcount: %s" % update.pushcount print "Severity: %s" % update.severity print "Summary: %s" % update.summary print "Description: %s" % update.description print "Solution: %s" % update.solution print "References:" for ref in update.references: print " Href: %s" % ref.href print " Id: %s" % ref.id print " Type: %s" % ref.type print " Title: %s" % ref.title print " ----------------------------" print "Pkglist (collections):" for col in update.collections: print " Short: %s" % col.shortname print " name: %s" % col.name print " Packages:" for pkg in col.packages: print " Name: %s" % pkg.name print " Version: %s" % pkg.version print " Release: %s" % pkg.release print " Epoch: %s" % pkg.epoch print " Arch: %s" % pkg.arch print " Src: %s" % pkg.src print " Filename: %s" % pkg.filename print " Sum: %s" % pkg.sum print " Sum type: %s (%s)" % ( pkg.sum_type, cr.checksum_name_str(pkg.sum_type)) print " Reboot suggested: %s" % pkg.reboot_suggested print " ----------------------------" print "=============================="
def parse_updateinfo(path): uinfo = cr.UpdateInfo(path) for update in uinfo.updates: print "From: %s" % update.fromstr print "Status: %s" % update.status print "Type: %s" % update.type print "Version: %s" % update.version print "Id: %s" % update.id print "Title: %s" % update.title print "Issued date: %s" % update.issued_date print "Updated date: %s" % update.updated_date print "Rights: %s" % update.rights print "Release: %s" % update.release print "Pushcount: %s" % update.pushcount print "Severity: %s" % update.severity print "Summary: %s" % update.summary print "Description: %s" % update.description print "Solution: %s" % update.solution print "References:" for ref in update.references: print " Href: %s" % ref.href print " Id: %s" % ref.id print " Type: %s" % ref.type print " Title: %s" % ref.title print " ----------------------------" print "Pkglist (collections):" for col in update.collections: print " Short: %s" % col.shortname print " name: %s" % col.name print " Packages:" for pkg in col.packages: print " Name: %s" % pkg.name print " Version: %s" % pkg.version print " Release: %s" % pkg.release print " Epoch: %s" % pkg.epoch print " Arch: %s" % pkg.arch print " Src: %s" % pkg.src print " Filename: %s" % pkg.filename print " Sum: %s" % pkg.sum print " Sum type: %s (%s)" % (pkg.sum_type, cr.checksum_name_str(pkg.sum_type)) print " Reboot suggested: %s" % pkg.reboot_suggested print " ----------------------------" print "=============================="
def cr_create_md(repodata_path, pkglist=None, log=sys.stdout): if pkglist is None: pkglist = cr_get_pkg_list(repo_base, log) pri_xml_path = os.path.join(repodata_path, 'primary.xml.gz') fil_xml_path = os.path.join(repodata_path, 'filelists.xml.gz') oth_xml_path = os.path.join(repodata_path, 'other.xml.gz') pri_db_path = os.path.join(repodata_path, 'primary.sqlite') fil_db_path = os.path.join(repodata_path, 'filelists.sqlite') oth_db_path = os.path.join(repodata_path, 'other.sqlite') def __create_xml(queues, xml_path, xml_func, name): cs = cr.ContentStat(cr.SHA256) xml = xml_func(xml_path, contentstat=cs) xml.set_num_of_pkgs(len(pkglist)) for pkg in pkglist: xml.add_pkg(pkg) xml.close() queues['master'].put( ((name, xml_path), (cs.checksum, cs.size, cs.checksum_type)), True) def __create_db(queues, db_path, db_func, name): db = db_func(db_path) for pkg in pkglist: db.add_pkg(pkg) db.dbinfo_update(queues[name].get(True)) db.close() cs = cr.ContentStat(cr.SHA256) cr.compress_file_with_stat( db_path, db_path + cr.compression_suffix(cr.BZ2_COMPRESSION), cr.BZ2_COMPRESSION, cs) os.remove(db_path) queues['master'].put( ((name + '_db', db_path + cr.compression_suffix(cr.BZ2_COMPRESSION)), (cs.checksum, cs.size, cs.checksum_type)), True) queue_manager = multiprocessing.Manager() queues = dict({ 'master': queue_manager.Queue(), 'primary': queue_manager.Queue(), 'filelists': queue_manager.Queue(), 'other': queue_manager.Queue(), }) log.write('[%s] Generating metadata in %s\n' % (stamp(), repodata_path)) th = [0] * 6 th[0] = multiprocessing.Process(target=__create_xml, args=(queues, pri_xml_path, cr.PrimaryXmlFile, 'primary')) th[0].start() th[1] = multiprocessing.Process(target=__create_xml, args=(queues, fil_xml_path, cr.FilelistsXmlFile, 'filelists')) th[1].start() th[2] = multiprocessing.Process(target=__create_xml, args=(queues, oth_xml_path, cr.OtherXmlFile, 'other')) th[2].start() th[3] = multiprocessing.Process(target=__create_db, args=(queues, pri_db_path, cr.PrimarySqlite, 'primary')) th[3].start() th[4] = multiprocessing.Process(target=__create_db, args=(queues, fil_db_path, cr.FilelistsSqlite, 'filelists')) th[4].start() th[5] = multiprocessing.Process(target=__create_db, args=(queues, oth_db_path, cr.OtherSqlite, 'other')) th[5].start() repomd = cr.Repomd() data_files = set() for i in range(0, 6): rf = queues['master'].get(True) r = cr.RepomdRecord(*rf[0]) r.checksum_open_type = cr.checksum_name_str(rf[1][2]) r.checksum_open = rf[1][0] r.size_open = rf[1][1] r.fill(cr.SHA256) if not rf[0][0].endswith('_db'): queues[rf[0][0]].put(r.checksum, True) r.rename_file() r.location_href = os.path.join('repodata', os.path.basename(r.location_href)) data_files.add(r.location_real) repomd.set_record(r) for t in th: t.join() repomd.sort_records() return (repomd.xml_dump(), data_files)
def _gen_basic_delta(self, md, force_gen=False): """Resolve some common situation during delta generation. There is some situation which could appear during delta generation: # - Metadata file has a record in repomd.xml and the file really exists O - Metadata file has a record in repomd.xml but the file is missing X - Metadata file doesn't have a record in repomd.xml Old repository | New repository ---------------|--------------- # | # - Valid case # | X - Valid case - metadata was removed # | O - Invalid case - incomplete repo X | # - Valid case - metadata was added X | X - This shouldn't happen X | O - Invalid case - incomplete repo O | # - Invalid case - incomplete repo O | X - Invalid case - incomplete repo O | O - Invalid case - both repos are incomplete By default, Deltarepo should raise an exception when a invalid case is meet. But user could use --ignore-missing option and in that case, the Deltarepo should handle all invalid case like a charm. For example: O | # - Just copy the new metadata to the delta repo as is O | X - Just ignore that the old metadata is missing O | O - Just ignore this # | O - Just ignore this X | O - Just ignore this Most delta plugins should be only interested to "# | #" use case. The case where we have the both, old and new, metadata available. Other cases are mostly not important to the delta plugins. This is the reason why this function exits. It should solve the cases when the sophisticated delta is not possible. Returns (SC - Status code, REC - Repomd record, NOTES - Dict with persistent notes) If RC is True, then delta plugin shouldn't continue with processing of this metadata. """ if not md: # No metadata - Nothing to do return (True, None, None) md.delta_rec = None md.delta_fn_exists = False if not md.old_rec and not md.new_rec: # None metadata record exists. self._debug("\"{0}\": Doesn't exist " "in any repo".format(md.metadata_type)) return (True, None, None) if not md.new_rec: # This record doesn't exists in the new version of repodata # This metadata were removed in the new version of repo self._debug("\"{0}\": Removed in the new version of repodata" "".format(md.metadata_type)) return (True, None, None) if not md.new_fn_exists: # The new metadata file is missing assert self.globalbundle.ignore_missing self._warning("\"{0}\": Delta cannot be generated - new metadata " "are missing".format(md.metadata_type)) return (True, None, None) if not md.old_rec or not md.old_fn_exists or \ (force_gen and not filecmp.cmp(md.old_fn, md.new_fn)): # This metadata was newly added in the new version of repodata # Or we are just missing the old version of this metadata # Or we have both versions of metadata but the metadata are not # same and in that case we simply want to gen a delta as a copy if md.old_fn_exists: self._debug( "\"{0}\": Newly added in the new version of repodata" "".format(md.metadata_type)) elif not md.old_fn_exists: self._warning( "\"{0}\": Delta cannot be generated - old metadata " "are missing - Using copy of the new one" "".format(md.metadata_type)) else: self._debug( "\"{0}\": Delta is just a copy of the new metadata") # Suffix based detection of compression compressed = False for suffix in COMPRESSION_SUFFIXES: if md.new_fn.endswith(suffix): compressed = True break compression = cr.NO_COMPRESSION if not compressed: compression = cr.XZ # Gen record rec = self.gen_use_original(md, compression_type=compression) notes = {} notes["original"] = '1' if compression != cr.NO_COMPRESSION: notes["compressed"] = "1" md.delta_rec = rec md.delta_fn_exists = True return (True, rec, notes) # At this point we are sure that we have both metadata files if filecmp.cmp(md.old_fn, md.new_fn): # Both metadata files exists and are the same self._debug("\"{0}\": Same in both version of repodata" "".format(md.metadata_type)) notes = {} if os.path.basename(md.old_fn) != os.path.basename(md.new_fn): notes["new_name"] = os.path.basename(md.new_fn) notes["unchanged"] = "1" notes["checksum_name"] = cr.checksum_name_str(md.checksum_type) return (True, None, notes) # Both metadata files exists and are different, # this is job for a real delta plugin :) return (False, None, None)
def _gen_basic_delta(self, md, force_gen=False): """Resolve some common situation during delta generation. There is some situation which could appear during delta generation: # - Metadata file has a record in repomd.xml and the file really exists O - Metadata file has a record in repomd.xml but the file is missing X - Metadata file doesn't have a record in repomd.xml Old repository | New repository ---------------|--------------- # | # - Valid case # | X - Valid case - metadata was removed # | O - Invalid case - incomplete repo X | # - Valid case - metadata was added X | X - This shouldn't happen X | O - Invalid case - incomplete repo O | # - Invalid case - incomplete repo O | X - Invalid case - incomplete repo O | O - Invalid case - both repos are incomplete By default, Deltarepo should raise an exception when a invalid case is meet. But user could use --ignore-missing option and in that case, the Deltarepo should handle all invalid case like a charm. For example: O | # - Just copy the new metadata to the delta repo as is O | X - Just ignore that the old metadata is missing O | O - Just ignore this # | O - Just ignore this X | O - Just ignore this Most delta plugins should be only interested to "# | #" use case. The case where we have the both, old and new, metadata available. Other cases are mostly not important to the delta plugins. This is the reason why this function exits. It should solve the cases when the sophisticated delta is not possible. Returns (SC - Status code, REC - Repomd record, NOTES - Dict with persistent notes) If RC is True, then delta plugin shouldn't continue with processing of this metadata. """ if not md: # No metadata - Nothing to do return (True, None, None) md.delta_rec = None md.delta_fn_exists = False if not md.old_rec and not md.new_rec: # None metadata record exists. self._debug("\"{0}\": Doesn't exist " "in any repo".format(md.metadata_type)) return (True, None, None) if not md.new_rec: # This record doesn't exists in the new version of repodata # This metadata were removed in the new version of repo self._debug("\"{0}\": Removed in the new version of repodata" "".format(md.metadata_type)) return (True, None, None) if not md.new_fn_exists: # The new metadata file is missing assert self.globalbundle.ignore_missing self._warning("\"{0}\": Delta cannot be generated - new metadata " "are missing".format(md.metadata_type)) return (True, None, None) if not md.old_rec or not md.old_fn_exists or \ (force_gen and not filecmp.cmp(md.old_fn, md.new_fn)): # This metadata was newly added in the new version of repodata # Or we are just missing the old version of this metadata # Or we have both versions of metadata but the metadata are not # same and in that case we simply want to gen a delta as a copy if md.old_fn_exists: self._debug("\"{0}\": Newly added in the new version of repodata" "".format(md.metadata_type)) elif not md.old_fn_exists: self._warning("\"{0}\": Delta cannot be generated - old metadata " "are missing - Using copy of the new one" "".format(md.metadata_type)) else: self._debug("\"{0}\": Delta is just a copy of the new metadata") # Suffix based detection of compression compressed = False for suffix in COMPRESSION_SUFFIXES: if md.new_fn.endswith(suffix): compressed = True break compression = cr.NO_COMPRESSION if not compressed: compression = cr.XZ # Gen record rec = self.gen_use_original(md, compression_type=compression) notes = {} notes["original"] = '1' if compression != cr.NO_COMPRESSION: notes["compressed"] = "1" md.delta_rec = rec md.delta_fn_exists = True return (True, rec, notes) # At this point we are sure that we have both metadata files if filecmp.cmp(md.old_fn, md.new_fn): # Both metadata files exists and are the same self._debug("\"{0}\": Same in both version of repodata" "".format(md.metadata_type)) notes = {} if os.path.basename(md.old_fn) != os.path.basename(md.new_fn): notes["new_name"] = os.path.basename(md.new_fn) notes["unchanged"] = "1" notes["checksum_name"] = cr.checksum_name_str(md.checksum_type) return (True, None, notes) # Both metadata files exists and are different, # this is job for a real delta plugin :) return (False, None, None)