def preserve_custom_metadata_on_repo_scratchpad():
"""
Lookups all the yum based repos in pulp; grabs any custom metadata
and set the the data on repo scratchpad.
"""
factory.initialize()
repos = factory.repo_query_manager().find_with_importer_type(
"yum_importer")
if not repos:
_log.debug("No repos found to perform db migrate")
return
repo_ids = [repo['id'] for repo in repos]
for repo_id in repo_ids:
_log.debug("Processing repo %s" % repo_id)
repo_scratchpad = factory.repo_manager().get_repo_scratchpad(repo_id)
if "repodata" in repo_scratchpad and repo_scratchpad["repodata"]:
# repo scratchpad already has repodata, skip migration
_log.debug(
"repo [%s] scratchpad already has repodata, skip migration" %
repo_id)
continue
repo_working_dir = importer_working_dir('yum_importer', repo_id)
importer_repodata_dir = os.path.join(repo_working_dir, repo_id,
"repodata")
repomd_xml_path = os.path.join(importer_repodata_dir, "repomd.xml")
if not os.path.exists(repomd_xml_path):
# repodata doesn't exist on filesystem cannot lookup custom data, continue to next
continue
ftypes = util.get_repomd_filetypes(repomd_xml_path)
base_ftypes = [
'primary', 'primary_db', 'filelists_db', 'filelists', 'other',
'other_db', 'group', 'group_gz', 'updateinfo', 'updateinfo_db'
]
for ftype in ftypes:
if ftype in base_ftypes:
# no need to process these again
continue
filetype_path = os.path.join(
importer_repodata_dir,
os.path.basename(
util.get_repomd_filetype_path(repomd_xml_path, ftype)))
if filetype_path.endswith('.gz'):
# if file is gzipped, decompress
data = gzip.open(filetype_path).read().decode(
"utf-8", "replace")
else:
data = open(filetype_path).read().decode("utf-8", "replace")
repo_scratchpad["repodata"].update({ftype: data})
# set the custom metadata on scratchpad
factory.repo_manager().set_repo_scratchpad(repo_id, repo_scratchpad)
_log.info("Updated repo [%s] scratchpad with new custom repodata" %
repo_id)
def merge_other_filetypes_from_backup(self):
"""
Merges any other filetypes in the backed up repodata that needs to be included
back into the repodata. This is where the presto, updateinfo and comps xmls are
looked up in old repomd.xml and merged back to the new using modifyrepo.
primary, filelists and other xmls are excluded from the process.
"""
_LOG.info("Performing merge on other file types")
try:
if not self.backup_repodata_dir:
_LOG.info("Nothing further to check; we got our fresh metadata")
return
current_repo_dir = os.path.join(self.repodir, "repodata")
# check if presto metadata exist in the backup
repodata_file = os.path.join(self.backup_repodata_dir, "repomd.xml")
ftypes = util.get_repomd_filetypes(repodata_file)
base_ftypes = ["primary", "primary_db", "filelists_db", "filelists", "other", "other_db"]
for ftype in ftypes:
if self.is_cancelled:
_LOG.warn("cancel merge other filetype metadata")
raise CancelException()
if ftype in base_ftypes:
# no need to process these again
continue
if ftype in self.skip and not self.skip[ftype]:
_LOG.info("mdtype %s part of skip metadata; skipping" % ftype)
continue
filetype_path = os.path.join(
self.backup_repodata_dir, os.path.basename(util.get_repomd_filetype_path(repodata_file, ftype))
)
# modifyrepo uses filename as mdtype, rename to type.<ext>
renamed_filetype_path = os.path.join(
os.path.dirname(filetype_path),
ftype + "." + ".".join(os.path.basename(filetype_path).split(".")[1:]),
)
os.rename(filetype_path, renamed_filetype_path)
if renamed_filetype_path.endswith(".gz"):
# if file is gzipped, decompress before passing to modifyrepo
data = gzip.open(renamed_filetype_path).read().decode("utf-8", "replace")
renamed_filetype_path = ".".join(renamed_filetype_path.split(".")[:-1])
open(renamed_filetype_path, "w").write(data.encode("UTF-8"))
if os.path.isfile(renamed_filetype_path):
_LOG.info("Modifying repo for %s metadata" % ftype)
modify_repo(current_repo_dir, renamed_filetype_path, checksum_type=self.checksum_type)
finally:
if self.backup_repodata_dir:
shutil.rmtree(self.backup_repodata_dir)
def __get_groups_xml_info(repo_dir):
groups_xml_path = None
repodata_file = os.path.join(repo_dir, "repodata", "repomd.xml")
repodata_file = encode_unicode(repodata_file)
if os.path.isfile(repodata_file):
ftypes = util.get_repomd_filetypes(repodata_file)
_LOG.debug("repodata has filetypes of %s" % (ftypes))
if "group" in ftypes:
comps_ftype = util.get_repomd_filetype_path(
repodata_file, "group")
filetype_path = os.path.join(repo_dir, comps_ftype)
# createrepo uses filename as mdtype, rename to type.<ext>
# to avoid filename too long errors
renamed_filetype_path = os.path.join(os.path.dirname(comps_ftype),
"comps" + '.' + '.'.join(os.path.basename(comps_ftype).split('.')[1:]))
groups_xml_path = os.path.join(repo_dir, renamed_filetype_path)
os.rename(filetype_path, groups_xml_path)
return groups_xml_path
def preserve_custom_metadata_on_repo_scratchpad():
"""
Lookups all the yum based repos in pulp; grabs any custom metadata
and set the the data on repo scratchpad.
"""
factory.initialize()
repos = factory.repo_query_manager().find_with_importer_type("yum_importer")
if not repos:
_log.debug("No repos found to perform db migrate")
return
repo_ids = [repo['id'] for repo in repos]
for repo_id in repo_ids:
_log.debug("Processing repo %s" % repo_id)
repo_scratchpad = factory.repo_manager().get_repo_scratchpad(repo_id)
if "repodata" in repo_scratchpad and repo_scratchpad["repodata"]:
# repo scratchpad already has repodata, skip migration
_log.debug("repo [%s] scratchpad already has repodata, skip migration" % repo_id)
continue
repo_working_dir = importer_working_dir('yum_importer', repo_id)
importer_repodata_dir = os.path.join(repo_working_dir, repo_id, "repodata")
repomd_xml_path = os.path.join(importer_repodata_dir, "repomd.xml")
if not os.path.exists(repomd_xml_path):
# repodata doesn't exist on filesystem cannot lookup custom data, continue to next
continue
ftypes = util.get_repomd_filetypes(repomd_xml_path)
base_ftypes = ['primary', 'primary_db', 'filelists_db', 'filelists', 'other', 'other_db',
'group', 'group_gz', 'updateinfo', 'updateinfo_db']
for ftype in ftypes:
if ftype in base_ftypes:
# no need to process these again
continue
filetype_path = os.path.join(importer_repodata_dir,
os.path.basename(
util.get_repomd_filetype_path(repomd_xml_path, ftype)))
if filetype_path.endswith('.gz'):
# if file is gzipped, decompress
data = gzip.open(filetype_path).read().decode("utf-8", "replace")
else:
data = open(filetype_path).read().decode("utf-8", "replace")
repo_scratchpad["repodata"].update({ftype: data})
# set the custom metadata on scratchpad
factory.repo_manager().set_repo_scratchpad(repo_id, repo_scratchpad)
_log.info("Updated repo [%s] scratchpad with new custom repodata" % repo_id)
def preserve_custom_metadata_on_scratchpad(repo, sync_conduit, config):
"""
Preserve custom metadata from repomd.xml on scratchpad for distributor to lookup and
publish. This includes prestodelta, productid or any other filetype that isn't the
standard.
@param repo: metadata describing the repository
@type repo: L{pulp.server.content.plugins.data.Repository}
@param sync_conduit
@type sync_conduit pulp.server.content.conduits.repo_sync.RepoSyncConduit
@param config: plugin configuration
@type config: L{pulp.server.content.plugins.config.PluginCallConfiguration}
"""
_LOG.debug('Determining custome filetypes to preserve for repo %s' % repo.id)
# store the importer working dir on scratchpad to lookup downloaded data
importer_repodata_dir = os.path.join(repo.working_dir, repo.id, "repodata")
repomd_xml_path = os.path.join(importer_repodata_dir, "repomd.xml")
if not os.path.exists(repomd_xml_path):
return
ftypes = util.get_repomd_filetypes(repomd_xml_path)
base_ftypes = ['primary', 'primary_db', 'filelists_db', 'filelists', 'other', 'other_db',
'group', 'group_gz', 'updateinfo', 'updateinfo_db']
existing_scratch_pad = sync_conduit.get_repo_scratchpad() or {}
skip_metadata_types = metadata.convert_content_to_metadata_type(config.get("skip") or [])
existing_scratch_pad.update({"repodata" : {}})
for ftype in ftypes:
if ftype in base_ftypes:
# no need to process these again
continue
if ftype in skip_metadata_types and not skip_metadata_types[ftype]:
_LOG.info("mdtype %s part of skip metadata; skipping" % ftype)
continue
filetype_path = os.path.join(importer_repodata_dir, os.path.basename(util.get_repomd_filetype_path(repomd_xml_path, ftype)))
if filetype_path.endswith('.gz'):
# if file is gzipped, decompress
data = gzip.open(filetype_path).read().decode("utf-8", "replace")
else:
data = open(filetype_path).read().decode("utf-8", "replace")
existing_scratch_pad["repodata"].update({ftype : data})
sync_conduit.set_repo_scratchpad(existing_scratch_pad)
def get_available_errata(repo_dir):
"""
Check and Parses the updateinfo.xml and extract errata items to sync
@param repo_dir repo directory where the metadata can be found
@type repo_dir str
@return a dict of errata items with errata id as key
@rtype {'':{}}
"""
repomd_xml = os.path.join(repo_dir, "repodata/repomd.xml")
ftypes = util.get_repomd_filetypes(repomd_xml)
errata_from_xml = {}
if "updateinfo" not in ftypes:
return errata_from_xml
updateinfo_xml_path = os.path.join(repo_dir, util.get_repomd_filetype_path(repomd_xml, "updateinfo"))
if not os.path.exists(updateinfo_xml_path):
return {}
try:
errata_from_xml = updateinfo.get_errata(updateinfo_xml_path)
except yum.Errors.YumBaseError, e:
_LOG.error("Error parsing updateinfo file [%s]; Error: %s" % (updateinfo_xml_path, e))
def get_groups_metadata_file(repo_dir, md_types=None):
"""
@param repo_dir path to a repo
@type repo_dir str
@param md_types May override the metadata type names, defaults to ['group', 'group_gz']
@type md_types str
@return path to group metadata file or None if repo doesn't have groups metadata, and the type, "group" or "group_gz"
@rtype string, str
"""
valid_md_types = ["group", "group_gz"]
if md_types:
valid_md_types = md_types
repomd_xml = os.path.join(repo_dir, "repodata/repomd.xml")
if not os.path.exists(repomd_xml):
return None
md_types = util.get_repomd_filetypes(repomd_xml)
for ret_type in valid_md_types:
if ret_type in md_types:
ret_file = util.get_repomd_filetype_path(repomd_xml, ret_type)
return ret_file, ret_type
return None, None
def create_repo(dir, groups=None, checksum_type=DEFAULT_CHECKSUM, skip_metadata_types=[]):
handle = None
# Lock the lookup and launch of a new createrepo process
# Lock is released once createrepo is launched
if not os.path.exists(dir):
_LOG.warning("create_repo invoked on a directory which doesn't exist: %s" % dir)
CREATE_REPO_PROCESS_LOOKUP_LOCK.acquire()
try:
if CREATE_REPO_PROCESS_LOOKUP.has_key(dir):
raise CreateRepoAlreadyRunningError()
current_repo_dir = os.path.join(dir, "repodata")
# Note: backup_repo_dir is used to store presto metadata and possibly other custom metadata types
# they will be copied back into new 'repodata' if needed.
backup_repo_dir = None
current_repo_dir = encode_unicode(current_repo_dir)
if os.path.exists(current_repo_dir):
_LOG.info("metadata found; taking backup.")
backup_repo_dir = os.path.join(dir, "repodata.old")
if os.path.exists(backup_repo_dir):
_LOG.debug("clean up any stale dirs")
shutil.rmtree(backup_repo_dir)
shutil.copytree(current_repo_dir, backup_repo_dir)
os.system("chmod -R u+wX %s" % (backup_repo_dir))
handle = _create_repo(dir, groups=groups, checksum_type=checksum_type)
if not handle:
raise CreateRepoError("Unable to execute createrepo on %s" % (dir))
os.system("chmod -R ug+wX %s" % (dir))
_LOG.info("Createrepo process with pid %s running on directory %s" % (handle.pid, dir))
CREATE_REPO_PROCESS_LOOKUP[dir] = handle
finally:
CREATE_REPO_PROCESS_LOOKUP_LOCK.release()
# Ensure we clean up CREATE_REPO_PROCESS_LOOKUP, surround all ops with try/finally
try:
# Block on process till complete (Note it may be async terminated)
out_msg, err_msg = handle.communicate(None)
if handle.returncode != 0:
try:
# Cleanup createrepo's temporary working directory
cleanup_dir = os.path.join(dir, ".repodata")
if os.path.exists(cleanup_dir):
shutil.rmtree(cleanup_dir)
except Exception, e:
_LOG.exception(e)
_LOG.warn("Unable to remove temporary createrepo dir: %s" % (cleanup_dir))
if handle.returncode == -9:
_LOG.warn("createrepo on %s was killed" % (dir))
raise CancelException()
else:
_LOG.error("createrepo on %s failed with returncode <%s>" % (dir, handle.returncode))
_LOG.error("createrepo stdout:\n%s" % (out_msg))
_LOG.error("createrepo stderr:\n%s" % (err_msg))
raise CreateRepoError(err_msg)
_LOG.info("createrepo on %s finished" % (dir))
if not backup_repo_dir:
_LOG.info("Nothing further to check; we got our fresh metadata")
return
#check if presto metadata exist in the backup
repodata_file = os.path.join(backup_repo_dir, "repomd.xml")
ftypes = util.get_repomd_filetypes(repodata_file)
base_ftypes = ['primary', 'primary_db', 'filelists_db', 'filelists', 'other', 'other_db', 'group', 'group_gz']
for ftype in ftypes:
if ftype in base_ftypes:
# no need to process these again
continue
if ftype in skip_metadata_types and not skip_metadata_types[ftype]:
_LOG.info("mdtype %s part of skip metadata; skipping" % ftype)
continue
filetype_path = os.path.join(backup_repo_dir, os.path.basename(util.get_repomd_filetype_path(repodata_file, ftype)))
# modifyrepo uses filename as mdtype, rename to type.<ext>
renamed_filetype_path = os.path.join(os.path.dirname(filetype_path), \
ftype + '.' + '.'.join(os.path.basename(filetype_path).split('.')[1:]))
os.rename(filetype_path, renamed_filetype_path)
if renamed_filetype_path.endswith('.gz'):
# if file is gzipped, decompress before passing to modifyrepo
data = gzip.open(renamed_filetype_path).read().decode("utf-8", "replace")
renamed_filetype_path = '.'.join(renamed_filetype_path.split('.')[:-1])
open(renamed_filetype_path, 'w').write(data.encode("UTF-8"))
if os.path.isfile(renamed_filetype_path):
_LOG.info("Modifying repo for %s metadata" % ftype)
modify_repo(current_repo_dir, renamed_filetype_path, checksum_type=checksum_type)
def test_iso_export_by_date_range(self):
feed_url = "file://%s/test_errata_local_sync/" % self.data_dir
repo = mock.Mock(spec=Repository)
repo.working_dir = self.repo_working_dir
repo.id = "test_errata_local_sync"
repo.checksumtype = 'sha'
sync_conduit = importer_mocks.get_sync_conduit(type_id=TYPE_ID_RPM, existing_units=[], pkg_dir=self.pkg_dir)
config = importer_mocks.get_basic_config(feed_url=feed_url)
importerRPM = importer_rpm.ImporterRPM()
status, summary, details = importerRPM.sync(repo, sync_conduit, config)
metadata = {'updated' : '2010-03-30 08:07:30'}
existing_units = []
unit_key_a = {'id' : '','name' :'patb', 'version' :'0.1', 'release' : '2', 'epoch':'0', 'arch' : 'x86_64', 'checksumtype' : 'sha',
'checksum': '017c12050a97cf6095892498750c2a39d2bf535e'}
rpm_unit_a = Unit(TYPE_ID_RPM, unit_key_a, metadata, '')
rpm_unit_a.storage_path = "%s/patb/0.1/2/noarch/017c12050a97cf6095892498750c2a39d2bf535e/patb-0.1-2.noarch.rpm" % self.pkg_dir
existing_units.append(rpm_unit_a)
unit_key_b = {'id' : '', 'name' :'emoticons', 'version' :'0.1', 'release' :'2', 'epoch':'0','arch' : 'x86_64', 'checksumtype' :'sha',
'checksum' : '663c89b0d29bfd5479d8736b716d50eed9495dbb'}
rpm_unit_b = Unit(TYPE_ID_RPM, unit_key_b, metadata, '')
rpm_unit_b.storage_path = "%s/emoticons/0.1/2/noarch/663c89b0d29bfd5479d8736b716d50eed9495dbb/emoticons-0.1-2.noarch.rpm" % self.pkg_dir
existing_units.append(rpm_unit_b)
sync_conduit = importer_mocks.get_sync_conduit(type_id=TYPE_ID_RPM, existing_units=existing_units, pkg_dir=self.pkg_dir)
importerErrata = errata.ImporterErrata()
status, summary, details = importerErrata.sync(repo, sync_conduit, config)
unit_key = dict()
unit_key['id'] = "RHEA-2010:9999"
mdata = { 'description' : "test",
'from_str': '*****@*****.**',
'issued': '2010-03-30 08:07:30',
'pkglist': [{'name': 'RHEL Virtualization (v. 5 for 32-bit x86)',
'packages': [{'arch': 'x86_64',
'epoch': '0',
'filename': 'patb-0.1-2.x86_64.rpm',
'name': 'patb',
'release': '2',
'src': '',
'sum': ('sha',
'017c12050a97cf6095892498750c2a39d2bf535e'),
'version': '0.1'},
{'arch': 'x86_64',
'epoch': '0',
'filename': 'emoticons-0.1-2.x86_64.rpm',
'name': 'emoticons',
'release': '2',
'src': '',
'sum': ('sha',
'663c89b0d29bfd5479d8736b716d50eed9495dbb'),
'version': '0.1'}],
'short': 'rhel-i386-server-vt-5'}],
'pushcount': 1,
'reboot_suggested': False,
'references': [],
'release': '',
'rights': '',
'status': 'final',
'summary': '',
'title': 'emoticons enhancement fix and enhancement update',
'updated': '2010-03-30 08:07:30',
'version': '1',
'type' : 'enhancement',
'severity' : 'Low',
'solution' : ''}
unit_key_2 = dict()
unit_key_2['id'] = "RHEA-2008:9999"
mdata_2 = { 'description' : "test",
'from_str': '*****@*****.**',
'issued': '2008-03-30 00:00:00',
'pkglist': [{'name': 'RHEL Virtualization (v. 5 for 32-bit x86)',
'packages': [{'arch': 'x86_64',
'epoch': '0',
'filename': 'patb-0.1-2.x86_64.rpm',
'name': 'patb',
'release': '2',
'src': '',
'sum': ('sha',
'017c12050a97cf6095892498750c2a39d2bf535e'),
'version': '0.1'},
{'arch': 'x86_64',
'epoch': '0',
'filename': 'emoticons-0.1-2.x86_64.rpm',
'name': 'emoticons',
'release': '2',
'src': '',
'sum': ('sha',
'663c89b0d29bfd5479d8736b716d50eed9495dbb'),
'version': '0.1'}],
'short': 'rhel-i386-server-vt-5'}],
'pushcount': 1,
'reboot_suggested': False,
'references': [],
'release': '',
'rights': '',
'status': 'final',
'summary': '',
'title': 'emoticons enhancement fix and enhancement update',
'updated': '2008-03-30 00:00:00',
'version': '1',
'type' : 'enhancement',
'severity' : 'Low',
'solution' : ''}
errata_unit = [Unit(TYPE_ID_ERRATA, unit_key, mdata, ''), Unit(TYPE_ID_ERRATA, unit_key_2, mdata_2, '')]
existing_units += errata_unit
print existing_units
repo.working_dir = "%s/%s" % (self.repo_working_dir, "export")
iso_distributor = ISODistributor()
publish_conduit = distributor_mocks.get_publish_conduit(existing_units=existing_units, pkg_dir=self.pkg_dir)
# test http publish
config = distributor_mocks.get_basic_config(http_publish_dir=self.http_publish_dir, https_publish_dir=self.https_publish_dir, http=True, https=False,
start_date="2009-03-30 08:07:30", end_date="2012-03-30 08:07:30", generate_metadata=True)
def cleanup(rpm_working_dir):
return
iso_util.cleanup_working_dir = mock.Mock()
iso_util.cleanup_working_dir.side_effect = cleanup
report = iso_distributor.publish_repo(repo, publish_conduit, config)
ftypes = util.get_repomd_filetypes("%s/%s" % (repo.working_dir, "repodata/repomd.xml"))
self.assertTrue("updateinfo" in ftypes)
updateinfo_path = util.get_repomd_filetype_path("%s/%s" % (repo.working_dir, "repodata/repomd.xml"), "updateinfo")
updateinfo_path = os.path.join(repo.working_dir, updateinfo_path)
self.assertTrue(os.path.exists(updateinfo_path))
elist = updateinfo.get_errata(updateinfo_path)
self.assertEquals(len(elist), 1)
self.assertTrue(unit_key_2['id'] not in elist[0])
self.assertEquals(elist[0]['id'], unit_key['id'])
self.assertEquals(elist[0]['issued'], mdata['issued'])
self.assertTrue(os.path.exists("%s/%s" % (self.http_publish_dir, repo.id)))
self.assertEquals(len(os.listdir(self.https_publish_dir)), 0)
isos_list = os.listdir("%s/%s" % (self.http_publish_dir, repo.id))
self.assertEqual(len(isos_list), 1)
def test_errata_export(self):
feed_url = "file://%s/test_errata_local_sync/" % self.data_dir
repo = mock.Mock(spec=Repository)
repo.working_dir = self.repo_working_dir
repo.id = "test_errata_local_sync"
repo.checksumtype = 'sha'
sync_conduit = importer_mocks.get_sync_conduit(type_id=TYPE_ID_RPM, existing_units=[], pkg_dir=self.pkg_dir)
config = importer_mocks.get_basic_config(feed_url=feed_url)
importerRPM = importer_rpm.ImporterRPM()
status, summary, details = importerRPM.sync(repo, sync_conduit, config)
metadata = {'updated' : '2010-03-30 08:07:30'}
unit_key_a = {'id' : '','name' :'patb', 'version' :'0.1', 'release' : '2', 'epoch':'0', 'arch' : 'x86_64', 'checksumtype' : 'md5',
'checksum': 'f3c197a29d9b66c5b65c5d62b25db5b4'}
unit_key_b = {'id' : '', 'name' :'emoticons', 'version' :'0.1', 'release' :'2', 'epoch':'0','arch' : 'x86_64', 'checksumtype' :'md5',
'checksum' : '366bb5e73a5905eacb82c96e0578f92b'}
existing_units = []
for unit in [unit_key_a, unit_key_b]:
existing_units.append(Unit(TYPE_ID_RPM, unit, metadata, ''))
sync_conduit = importer_mocks.get_sync_conduit(type_id=TYPE_ID_RPM, existing_units=existing_units, pkg_dir=self.pkg_dir)
importerErrata = errata.ImporterErrata()
status, summary, details = importerErrata.sync(repo, sync_conduit, config)
unit_key = dict()
unit_key['id'] = "RHEA-2010:9999"
mdata = { 'description' : "test",
'from_str': '*****@*****.**',
'issued': '2010-03-30 08:07:30',
'pkglist': [{'name': 'RHEL Virtualization (v. 5 for 32-bit x86)',
'packages': [{'arch': 'x86_64',
'epoch': '0',
'filename': 'patb-0.1-2.x86_64.rpm',
'name': 'patb',
'release': '2',
'src': '',
'sum': ('md5',
'f3c197a29d9b66c5b65c5d62b25db5b4'),
'version': '0.1'},
{'arch': 'x86_64',
'epoch': '0',
'filename': 'emoticons-0.1-2.x86_64.rpm',
'name': 'emoticons',
'release': '2',
'src': '',
'sum': ('md5',
'366bb5e73a5905eacb82c96e0578f92b'),
'version': '0.1'}],
'short': 'rhel-i386-server-vt-5'}],
'pushcount': 1,
'reboot_suggested': False,
'references': [],
'release': '',
'rights': '',
'status': 'final',
'summary': '',
'title': 'emoticons enhancement fix and enhancement update',
'updated': '2010-03-30 08:07:30',
'version': '1',
'type' : 'enhancement',
'severity' : 'Low',
'solution' : ''}
errata_unit = [Unit(TYPE_ID_ERRATA, unit_key, mdata, '')]
symlink_dir = "%s/%s" % (self.repo_working_dir, repo.id)
iso_distributor = ISODistributor()
publish_conduit = distributor_mocks.get_publish_conduit(existing_units=existing_units, pkg_dir=self.pkg_dir)
config = distributor_mocks.get_basic_config(https_publish_dir=self.https_publish_dir, http=False, https=True)
print symlink_dir
repo_exporter = RepoExporter(symlink_dir)
# rpm_units = iso_distributor._get_errata_rpms(errata_unit, existing_units)
rpm_units = repo_exporter.get_errata_rpms(errata_unit, existing_units)
print "RPMS in ERRATA",rpm_units
# iso_distributor._export_rpms(rpm_units, self.repo_working_dir)
repo_exporter.export_rpms(rpm_units)
status, errors = repo_exporter.export_errata(errata_unit)
# status, errors = iso_distributor._export_errata(errata_unit, symlink_dir)
self.assertTrue(os.path.exists("%s/%s" % (symlink_dir, "updateinfo.xml")))
self.assertTrue(status)
ftypes = util.get_repomd_filetypes("%s/%s" % (symlink_dir, "repodata/repomd.xml"))
print ftypes
self.assertTrue("updateinfo" in ftypes)
def test_publish_comps(self):
repo = mock.Mock(spec=Repository)
repo.id = "test_publish_comps"
repo.working_dir = self.working_dir
# Create 2 pkg groups
grp_a = self.create_dummy_pkg_group_unit(repo.id, "group_a")
grp_b = self.create_dummy_pkg_group_unit(repo.id, "group_b")
# Create 2 pkg categories
cat_a = self.create_dummy_pkg_category_unit(repo.id, "cat_a", ["group_a"])
cat_b = self.create_dummy_pkg_category_unit(repo.id, "cat_b", ["group_b"])
# Add the grps/cats to the publish_conduit
publish_conduit = distributor_mocks.get_publish_conduit(
existing_units=[grp_a, grp_b, cat_a, cat_b])
config = distributor_mocks.get_basic_config(relative_url=repo.id,
http=True, https=True, generate_metadata=True)
# Publish the repo, be sure 'generate_metadata' is True
yum_distributor = YumDistributor()
report = yum_distributor.publish_repo(repo, publish_conduit, config)
self.assertTrue(report.success_flag)
self.assertEqual(report.summary["num_package_groups_published"], 2)
self.assertEqual(report.summary["num_package_categories_published"], 2)
expected_comps_xml = os.path.join(repo.working_dir, "comps.xml")
self.assertTrue(os.path.exists(expected_comps_xml))
#
# Find the path that createrepo added the comps.xml as
#
repomd_xml = os.path.join(repo.working_dir, "repodata", "repomd.xml")
self.assertTrue(os.path.exists(repomd_xml))
md_types = util.get_repomd_filetypes(repomd_xml)
self.assertTrue('group' in md_types)
groups_path = util.get_repomd_filetype_path(repomd_xml, "group")
self.assertTrue(groups_path)
groups_path = os.path.join(repo.working_dir, groups_path)
self.assertTrue(os.path.exists(groups_path))
#
# Use yum to read the repodata and verify the info written matches
# our dummy data
#
yc = yum.comps.Comps()
yc.add(groups_path)
self.assertEqual(len(yc.groups), 2)
self.assertEqual(len(yc.categories), 2)
for g in yc.groups:
eg = None
if g.groupid == "group_a":
eg = grp_a
elif g.groupid == "group_b":
eg = grp_b
else:
# Unexpected error
self.assertTrue(False)
self.assertEqual(g.name, eg.metadata["name"])
self.assertEqual(g.description, eg.metadata["description"])
self.assertEqual(g.user_visible, eg.metadata["user_visible"])
self.assertEqual(g.display_order, eg.metadata["display_order"])
self.assertEqual(g.default, eg.metadata["default"])
self.assertEqual(g.langonly, eg.metadata["langonly"])
for pkg_name in g.mandatory_packages:
self.assertTrue(pkg_name in eg.metadata["mandatory_package_names"])
for pkg_name in g.optional_packages:
self.assertTrue(pkg_name in eg.metadata["optional_package_names"])
for pkg_name in g.default_packages:
self.assertTrue(pkg_name in eg.metadata["default_package_names"])
#
# Below is related to pymongo not liking dots in a pkg_name
# We are storing conditional_package_names as a list of tuples, (name, values)
# convert to a dictionary to make it easier to compare against yum's data
#
cond_lookup = {}
for expected_name, expected_values in eg.metadata["conditional_package_names"]:
cond_lookup[expected_name] = expected_values
for pkg_name in g.conditional_packages:
# We are converting our expected value to a str below to match the behavior
# we see from yum
self.assertEqual(g.conditional_packages[pkg_name], str(cond_lookup[pkg_name]))
for c in yc.categories:
ec = None
if c.categoryid == "cat_a":
ec = cat_a
elif c.categoryid == "cat_b":
ec = cat_b
else:
# Unexpected error
self.assertTrue(False)
self.assertEqual(c.name, ec.metadata["name"])
self.assertEqual(c.description, ec.metadata["description"])
self.assertEqual(c.display_order, ec.metadata["display_order"])
self.assertEqual(len(c._groups), len(ec.metadata["packagegroupids"]))
for grpid in c._groups:
self.assertTrue(grpid in ec.metadata["packagegroupids"])
def test_iso_export_by_date_range(self):
feed_url = "file://%s/test_errata_local_sync/" % self.data_dir
repo = mock.Mock(spec=Repository)
repo.working_dir = self.repo_working_dir
repo.id = "test_errata_local_sync"
repo.checksumtype = "sha"
sync_conduit = importer_mocks.get_sync_conduit(type_id=TYPE_ID_RPM, existing_units=[], pkg_dir=self.pkg_dir)
config = importer_mocks.get_basic_config(feed_url=feed_url)
importerRPM = importer_rpm.ImporterRPM()
status, summary, details = importerRPM.sync(repo, sync_conduit, config)
metadata = {}
existing_units = []
unit_key_a = {
"id": "",
"name": "patb",
"version": "0.1",
"release": "2",
"epoch": "0",
"arch": "x86_64",
"checksumtype": "sha",
"checksum": "017c12050a97cf6095892498750c2a39d2bf535e",
}
rpm_unit_a = Unit(TYPE_ID_RPM, unit_key_a, metadata, "")
rpm_unit_a.storage_path = (
"%s/patb/0.1/2/noarch/017c12050a97cf6095892498750c2a39d2bf535e/patb-0.1-2.noarch.rpm" % self.pkg_dir
)
existing_units.append(rpm_unit_a)
unit_key_b = {
"id": "",
"name": "emoticons",
"version": "0.1",
"release": "2",
"epoch": "0",
"arch": "x86_64",
"checksumtype": "sha",
"checksum": "663c89b0d29bfd5479d8736b716d50eed9495dbb",
}
rpm_unit_b = Unit(TYPE_ID_RPM, unit_key_b, metadata, "")
rpm_unit_b.storage_path = (
"%s/emoticons/0.1/2/noarch/663c89b0d29bfd5479d8736b716d50eed9495dbb/emoticons-0.1-2.noarch.rpm"
% self.pkg_dir
)
existing_units.append(rpm_unit_b)
sync_conduit = importer_mocks.get_sync_conduit(
type_id=TYPE_ID_RPM, existing_units=existing_units, pkg_dir=self.pkg_dir
)
importerErrata = errata.ImporterErrata()
status, summary, details = importerErrata.sync(repo, sync_conduit, config)
unit_key = dict()
unit_key["id"] = "RHEA-2010:9999"
mdata = {
"description": "test",
"from_str": "*****@*****.**",
"issued": "2010-03-30 08:07:30",
"pkglist": [
{
"name": "RHEL Virtualization (v. 5 for 32-bit x86)",
"packages": [
{
"arch": "x86_64",
"epoch": "0",
"filename": "patb-0.1-2.x86_64.rpm",
"name": "patb",
"release": "2",
"src": "",
"sum": ("sha", "017c12050a97cf6095892498750c2a39d2bf535e"),
"version": "0.1",
},
{
"arch": "x86_64",
"epoch": "0",
"filename": "emoticons-0.1-2.x86_64.rpm",
"name": "emoticons",
"release": "2",
"src": "",
"sum": ("sha", "663c89b0d29bfd5479d8736b716d50eed9495dbb"),
"version": "0.1",
},
],
"short": "rhel-i386-server-vt-5",
}
],
"pushcount": 1,
"reboot_suggested": False,
"references": [],
"release": "",
"rights": "",
"status": "final",
"summary": "",
"title": "emoticons enhancement fix and enhancement update",
"updated": "2010-03-30 08:07:30",
"version": "1",
"type": "enhancement",
"severity": "Low",
"solution": "",
}
unit_key_2 = dict()
unit_key_2["id"] = "RHEA-2008:9999"
mdata_2 = {
"description": "test",
"from_str": "*****@*****.**",
"issued": "2008-03-30 00:00:00",
"pkglist": [
{
"name": "RHEL Virtualization (v. 5 for 32-bit x86)",
"packages": [
{
"arch": "x86_64",
"epoch": "0",
"filename": "patb-0.1-2.x86_64.rpm",
"name": "patb",
"release": "2",
"src": "",
"sum": ("sha", "017c12050a97cf6095892498750c2a39d2bf535e"),
"version": "0.1",
},
{
"arch": "x86_64",
"epoch": "0",
"filename": "emoticons-0.1-2.x86_64.rpm",
"name": "emoticons",
"release": "2",
"src": "",
"sum": ("sha", "663c89b0d29bfd5479d8736b716d50eed9495dbb"),
"version": "0.1",
},
],
"short": "rhel-i386-server-vt-5",
}
],
"pushcount": 1,
"reboot_suggested": False,
"references": [],
"release": "",
"rights": "",
"status": "final",
"summary": "",
"title": "emoticons enhancement fix and enhancement update",
"updated": "2008-03-30 00:00:00",
"version": "1",
"type": "enhancement",
"severity": "Low",
"solution": "",
}
errata_unit = [Unit(TYPE_ID_ERRATA, unit_key, mdata, ""), Unit(TYPE_ID_ERRATA, unit_key_2, mdata_2, "")]
existing_units += errata_unit
print existing_units
repo.working_dir = "%s/%s" % (self.repo_working_dir, "export")
iso_distributor = ISODistributor()
publish_conduit = distributor_mocks.get_publish_conduit(existing_units=existing_units, pkg_dir=self.pkg_dir)
# test http publish
config = distributor_mocks.get_basic_config(
http_publish_dir=self.http_publish_dir,
https_publish_dir=self.https_publish_dir,
http=True,
https=False,
start_date="2009-03-30 08:07:30",
end_date="2012-03-30 08:07:30",
generate_metadata=True,
)
def cleanup(rpm_working_dir):
return
iso_util.cleanup_working_dir = mock.Mock()
iso_util.cleanup_working_dir.side_effect = cleanup
report = iso_distributor.publish_repo(repo, publish_conduit, config)
ftypes = util.get_repomd_filetypes("%s/%s" % (repo.working_dir, "repodata/repomd.xml"))
self.assertTrue("updateinfo" in ftypes)
updateinfo_path = "%s/%s" % (repo.working_dir, "updateinfo.xml")
self.assertTrue(os.path.exists(updateinfo_path))
elist = updateinfo.get_errata(updateinfo_path)
self.assertEquals(len(elist), 1)
self.assertTrue(unit_key_2["id"] not in elist[0])
self.assertEquals(elist[0]["id"], unit_key["id"])
self.assertEquals(elist[0]["issued"], mdata["issued"])
self.assertTrue(os.path.exists("%s/%s" % (self.http_publish_dir, repo.id)))
self.assertEquals(len(os.listdir(self.https_publish_dir)), 0)
isos_list = os.listdir("%s/%s" % (self.http_publish_dir, repo.id))
self.assertEqual(len(isos_list), 1)
def test_errata_export(self):
feed_url = "file://%s/test_errata_local_sync/" % self.data_dir
repo = mock.Mock(spec=Repository)
repo.working_dir = self.repo_working_dir
repo.id = "test_errata_local_sync"
repo.checksumtype = "sha"
sync_conduit = importer_mocks.get_sync_conduit(type_id=TYPE_ID_RPM, existing_units=[], pkg_dir=self.pkg_dir)
config = importer_mocks.get_basic_config(feed_url=feed_url)
importerRPM = importer_rpm.ImporterRPM()
status, summary, details = importerRPM.sync(repo, sync_conduit, config)
metadata = {}
unit_key_a = {
"id": "",
"name": "patb",
"version": "0.1",
"release": "2",
"epoch": "0",
"arch": "x86_64",
"checksumtype": "md5",
"checksum": "f3c197a29d9b66c5b65c5d62b25db5b4",
}
unit_key_b = {
"id": "",
"name": "emoticons",
"version": "0.1",
"release": "2",
"epoch": "0",
"arch": "x86_64",
"checksumtype": "md5",
"checksum": "366bb5e73a5905eacb82c96e0578f92b",
}
existing_units = []
for unit in [unit_key_a, unit_key_b]:
existing_units.append(Unit(TYPE_ID_RPM, unit, metadata, ""))
sync_conduit = importer_mocks.get_sync_conduit(
type_id=TYPE_ID_RPM, existing_units=existing_units, pkg_dir=self.pkg_dir
)
importerErrata = errata.ImporterErrata()
status, summary, details = importerErrata.sync(repo, sync_conduit, config)
unit_key = dict()
unit_key["id"] = "RHEA-2010:9999"
mdata = {
"description": "test",
"from_str": "*****@*****.**",
"issued": "2010-03-30 08:07:30",
"pkglist": [
{
"name": "RHEL Virtualization (v. 5 for 32-bit x86)",
"packages": [
{
"arch": "x86_64",
"epoch": "0",
"filename": "patb-0.1-2.x86_64.rpm",
"name": "patb",
"release": "2",
"src": "",
"sum": ("md5", "f3c197a29d9b66c5b65c5d62b25db5b4"),
"version": "0.1",
},
{
"arch": "x86_64",
"epoch": "0",
"filename": "emoticons-0.1-2.x86_64.rpm",
"name": "emoticons",
"release": "2",
"src": "",
"sum": ("md5", "366bb5e73a5905eacb82c96e0578f92b"),
"version": "0.1",
},
],
"short": "rhel-i386-server-vt-5",
}
],
"pushcount": 1,
"reboot_suggested": False,
"references": [],
"release": "",
"rights": "",
"status": "final",
"summary": "",
"title": "emoticons enhancement fix and enhancement update",
"updated": "2010-03-30 08:07:30",
"version": "1",
"type": "enhancement",
"severity": "Low",
"solution": "",
}
errata_unit = [Unit(TYPE_ID_ERRATA, unit_key, mdata, "")]
symlink_dir = "%s/%s" % (self.repo_working_dir, repo.id)
iso_distributor = ISODistributor()
publish_conduit = distributor_mocks.get_publish_conduit(existing_units=existing_units, pkg_dir=self.pkg_dir)
config = distributor_mocks.get_basic_config(https_publish_dir=self.https_publish_dir, http=False, https=True)
print symlink_dir
repo_exporter = RepoExporter(symlink_dir)
# rpm_units = iso_distributor._get_errata_rpms(errata_unit, existing_units)
rpm_units = repo_exporter.get_errata_rpms(errata_unit, existing_units)
print "RPMS in ERRATA", rpm_units
# iso_distributor._export_rpms(rpm_units, self.repo_working_dir)
repo_exporter.export_rpms(rpm_units)
status, errors = repo_exporter.export_errata(errata_unit)
# status, errors = iso_distributor._export_errata(errata_unit, symlink_dir)
self.assertTrue(os.path.exists("%s/%s" % (symlink_dir, "updateinfo.xml")))
self.assertTrue(status)
ftypes = util.get_repomd_filetypes("%s/%s" % (symlink_dir, "repodata/repomd.xml"))
print ftypes
self.assertTrue("updateinfo" in ftypes)
