Esempio n. 1
0
 def merge_other_filetypes_from_backup(self):
     """
     Merges any other filetypes in the backed up repodata that needs to be included
     back into the repodata. This is where the presto, updateinfo and comps xmls are
     looked up in old repomd.xml and merged back to the new using modifyrepo.
     primary, filelists and other xmls are excluded from the process.
     """
     _LOG.info("Performing merge on other file types")
     try:
         if not self.backup_repodata_dir:
             _LOG.info("Nothing further to check; we got our fresh metadata")
             return
         current_repo_dir = os.path.join(self.repodir, "repodata")
         # check if presto metadata exist in the backup
         repodata_file = os.path.join(self.backup_repodata_dir, "repomd.xml")
         ftypes = util.get_repomd_filetypes(repodata_file)
         base_ftypes = ["primary", "primary_db", "filelists_db", "filelists", "other", "other_db"]
         for ftype in ftypes:
             if self.is_cancelled:
                 _LOG.warn("cancel merge other filetype metadata")
                 raise CancelException()
             if ftype in base_ftypes:
                 # no need to process these again
                 continue
             if ftype in self.skip and not self.skip[ftype]:
                 _LOG.info("mdtype %s part of skip metadata; skipping" % ftype)
                 continue
             filetype_path = os.path.join(
                 self.backup_repodata_dir, os.path.basename(util.get_repomd_filetype_path(repodata_file, ftype))
             )
             # modifyrepo uses filename as mdtype, rename to type.<ext>
             renamed_filetype_path = os.path.join(
                 os.path.dirname(filetype_path),
                 ftype + "." + ".".join(os.path.basename(filetype_path).split(".")[1:]),
             )
             os.rename(filetype_path, renamed_filetype_path)
             if renamed_filetype_path.endswith(".gz"):
                 # if file is gzipped, decompress before passing to modifyrepo
                 data = gzip.open(renamed_filetype_path).read().decode("utf-8", "replace")
                 renamed_filetype_path = ".".join(renamed_filetype_path.split(".")[:-1])
                 open(renamed_filetype_path, "w").write(data.encode("UTF-8"))
             if os.path.isfile(renamed_filetype_path):
                 _LOG.info("Modifying repo for %s metadata" % ftype)
                 modify_repo(current_repo_dir, renamed_filetype_path, checksum_type=self.checksum_type)
     finally:
         if self.backup_repodata_dir:
             shutil.rmtree(self.backup_repodata_dir)
Esempio n. 2
0
def __get_groups_xml_info(repo_dir):
    groups_xml_path = None
    repodata_file = os.path.join(repo_dir, "repodata", "repomd.xml")
    repodata_file = encode_unicode(repodata_file)
    if os.path.isfile(repodata_file):
        ftypes = util.get_repomd_filetypes(repodata_file)
        _LOG.debug("repodata has filetypes of %s" % (ftypes))
        if "group" in ftypes:
            comps_ftype = util.get_repomd_filetype_path(
                    repodata_file, "group")
            filetype_path = os.path.join(repo_dir, comps_ftype)
            # createrepo uses filename as mdtype, rename to type.<ext>
            # to avoid filename too long errors
            renamed_filetype_path = os.path.join(os.path.dirname(comps_ftype),
                                     "comps" + '.' + '.'.join(os.path.basename(comps_ftype).split('.')[1:]))
            groups_xml_path = os.path.join(repo_dir, renamed_filetype_path)
            os.rename(filetype_path, groups_xml_path)
    return groups_xml_path
def preserve_custom_metadata_on_repo_scratchpad():
    """
     Lookups all the yum based repos in pulp; grabs any custom metadata
     and set the the data on repo scratchpad.
    """
    factory.initialize()
    repos = factory.repo_query_manager().find_with_importer_type("yum_importer")
    if not repos:
        _log.debug("No repos found to perform db migrate")
        return
    repo_ids = [repo['id'] for repo in repos]
    for repo_id in repo_ids:
        _log.debug("Processing repo %s" % repo_id)
        repo_scratchpad = factory.repo_manager().get_repo_scratchpad(repo_id)
        if "repodata" in repo_scratchpad and repo_scratchpad["repodata"]:
            # repo scratchpad already has repodata, skip migration
            _log.debug("repo [%s] scratchpad already has repodata, skip migration" % repo_id)
            continue
        repo_working_dir = importer_working_dir('yum_importer', repo_id)
        importer_repodata_dir = os.path.join(repo_working_dir, repo_id, "repodata")
        repomd_xml_path = os.path.join(importer_repodata_dir, "repomd.xml")
        if not os.path.exists(repomd_xml_path):
            # repodata doesn't exist on filesystem cannot lookup custom data, continue to next
            continue
        ftypes = util.get_repomd_filetypes(repomd_xml_path)
        base_ftypes = ['primary', 'primary_db', 'filelists_db', 'filelists', 'other', 'other_db',
                       'group', 'group_gz', 'updateinfo', 'updateinfo_db']
        for ftype in ftypes:
            if ftype in base_ftypes:
                # no need to process these again
                continue
            filetype_path = os.path.join(importer_repodata_dir,
                                         os.path.basename(
                                             util.get_repomd_filetype_path(repomd_xml_path, ftype)))
            if filetype_path.endswith('.gz'):
                # if file is gzipped, decompress
                data = gzip.open(filetype_path).read().decode("utf-8", "replace")
            else:
                data = open(filetype_path).read().decode("utf-8", "replace")
            repo_scratchpad["repodata"].update({ftype: data})
        # set the custom metadata on scratchpad
        factory.repo_manager().set_repo_scratchpad(repo_id, repo_scratchpad)
        _log.info("Updated repo [%s] scratchpad with new custom repodata" % repo_id)
Esempio n. 4
0
def preserve_custom_metadata_on_scratchpad(repo, sync_conduit, config):
    """
    Preserve custom metadata from repomd.xml on scratchpad for distributor to lookup and
    publish. This includes prestodelta, productid or any other filetype that isn't the
    standard.
      @param repo: metadata describing the repository
      @type  repo: L{pulp.server.content.plugins.data.Repository}

      @param sync_conduit
      @type sync_conduit pulp.server.content.conduits.repo_sync.RepoSyncConduit

      @param config: plugin configuration
      @type  config: L{pulp.server.content.plugins.config.PluginCallConfiguration}
    """
    _LOG.debug('Determining custome filetypes to preserve for repo %s' % repo.id)
    # store the importer working dir on scratchpad to lookup downloaded data
    importer_repodata_dir = os.path.join(repo.working_dir, repo.id, "repodata")
    repomd_xml_path = os.path.join(importer_repodata_dir, "repomd.xml")
    if not os.path.exists(repomd_xml_path):
        return
    ftypes = util.get_repomd_filetypes(repomd_xml_path)
    base_ftypes = ['primary', 'primary_db', 'filelists_db', 'filelists', 'other', 'other_db',
                   'group', 'group_gz', 'updateinfo', 'updateinfo_db']
    existing_scratch_pad = sync_conduit.get_repo_scratchpad() or {}
    skip_metadata_types = metadata.convert_content_to_metadata_type(config.get("skip") or [])
    existing_scratch_pad.update({"repodata" : {}})
    for ftype in ftypes:
        if ftype in base_ftypes:
            # no need to process these again
            continue
        if ftype in skip_metadata_types and not skip_metadata_types[ftype]:
            _LOG.info("mdtype %s part of skip metadata; skipping" % ftype)
            continue
        filetype_path = os.path.join(importer_repodata_dir, os.path.basename(util.get_repomd_filetype_path(repomd_xml_path, ftype)))
        if filetype_path.endswith('.gz'):
            # if file is gzipped, decompress
            data = gzip.open(filetype_path).read().decode("utf-8", "replace")
        else:
            data = open(filetype_path).read().decode("utf-8", "replace")
        existing_scratch_pad["repodata"].update({ftype : data})
    sync_conduit.set_repo_scratchpad(existing_scratch_pad)
Esempio n. 5
0
def get_available_errata(repo_dir):
    """
        Check and Parses the updateinfo.xml and extract errata items to sync

        @param repo_dir repo directory where the metadata can be found
        @type repo_dir str

        @return a dict of errata items with errata id as key
        @rtype {'':{}}
    """
    repomd_xml = os.path.join(repo_dir, "repodata/repomd.xml")
    ftypes = util.get_repomd_filetypes(repomd_xml)
    errata_from_xml = {}
    if "updateinfo" not in ftypes:
        return errata_from_xml
    updateinfo_xml_path = os.path.join(repo_dir, util.get_repomd_filetype_path(repomd_xml, "updateinfo"))
    if not os.path.exists(updateinfo_xml_path):
        return {}
    try:
        errata_from_xml = updateinfo.get_errata(updateinfo_xml_path)
    except yum.Errors.YumBaseError, e:
        _LOG.error("Error parsing updateinfo file [%s]; Error: %s" % (updateinfo_xml_path, e))
Esempio n. 6
0
File: comps.py Progetto: ehelms/pulp
def get_groups_metadata_file(repo_dir, md_types=None):
    """
    @param repo_dir path to a repo
    @type repo_dir str

    @param md_types May override the metadata type names, defaults to ['group', 'group_gz']
    @type md_types str

    @return path to group metadata file or None if repo doesn't have groups metadata, and the type, "group" or "group_gz"
    @rtype string, str
    """
    valid_md_types = ["group", "group_gz"]
    if md_types:
        valid_md_types = md_types
    repomd_xml = os.path.join(repo_dir, "repodata/repomd.xml")
    if not os.path.exists(repomd_xml):
        return None
    md_types = util.get_repomd_filetypes(repomd_xml)
    for ret_type in valid_md_types:
        if ret_type in md_types:
            ret_file = util.get_repomd_filetype_path(repomd_xml, ret_type)
            return ret_file, ret_type
    return None, None
Esempio n. 7
0
def create_repo(dir, groups=None, checksum_type=DEFAULT_CHECKSUM, skip_metadata_types=[]):
    handle = None
    # Lock the lookup and launch of a new createrepo process
    # Lock is released once createrepo is launched
    if not os.path.exists(dir):
        _LOG.warning("create_repo invoked on a directory which doesn't exist:  %s" % dir)
    CREATE_REPO_PROCESS_LOOKUP_LOCK.acquire()
    try:
        if CREATE_REPO_PROCESS_LOOKUP.has_key(dir):
            raise CreateRepoAlreadyRunningError()
        current_repo_dir = os.path.join(dir, "repodata")
        # Note: backup_repo_dir is used to store presto metadata and possibly other custom metadata types
        # they will be copied back into new 'repodata' if needed.
        backup_repo_dir = None
        current_repo_dir = encode_unicode(current_repo_dir)
        if os.path.exists(current_repo_dir):
            _LOG.info("metadata found; taking backup.")
            backup_repo_dir = os.path.join(dir, "repodata.old")
            if os.path.exists(backup_repo_dir):
                _LOG.debug("clean up any stale dirs")
                shutil.rmtree(backup_repo_dir)
            shutil.copytree(current_repo_dir, backup_repo_dir)
            os.system("chmod -R u+wX %s" % (backup_repo_dir))
        handle = _create_repo(dir, groups=groups, checksum_type=checksum_type)
        if not handle:
            raise CreateRepoError("Unable to execute createrepo on %s" % (dir))
        os.system("chmod -R ug+wX %s" % (dir))
        _LOG.info("Createrepo process with pid %s running on directory %s" % (handle.pid, dir))
        CREATE_REPO_PROCESS_LOOKUP[dir] = handle
    finally:
        CREATE_REPO_PROCESS_LOOKUP_LOCK.release()
    # Ensure we clean up CREATE_REPO_PROCESS_LOOKUP, surround all ops with try/finally
    try:
        # Block on process till complete (Note it may be async terminated)
        out_msg, err_msg = handle.communicate(None)
        if handle.returncode != 0:
            try:
                # Cleanup createrepo's temporary working directory
                cleanup_dir = os.path.join(dir, ".repodata")
                if os.path.exists(cleanup_dir):
                    shutil.rmtree(cleanup_dir)
            except Exception, e:
                _LOG.exception(e)
                _LOG.warn("Unable to remove temporary createrepo dir: %s" % (cleanup_dir))
            if handle.returncode == -9:
                _LOG.warn("createrepo on %s was killed" % (dir))
                raise CancelException()
            else:
                _LOG.error("createrepo on %s failed with returncode <%s>" % (dir, handle.returncode))
                _LOG.error("createrepo stdout:\n%s" % (out_msg))
                _LOG.error("createrepo stderr:\n%s" % (err_msg))
                raise CreateRepoError(err_msg)
        _LOG.info("createrepo on %s finished" % (dir))
        if not backup_repo_dir:
            _LOG.info("Nothing further to check; we got our fresh metadata")
            return
        #check if presto metadata exist in the backup
        repodata_file = os.path.join(backup_repo_dir, "repomd.xml")
        ftypes = util.get_repomd_filetypes(repodata_file)
        base_ftypes = ['primary', 'primary_db', 'filelists_db', 'filelists', 'other', 'other_db', 'group', 'group_gz']
        for ftype in ftypes:
            if ftype in base_ftypes:
                # no need to process these again
                continue
            if ftype in skip_metadata_types and not skip_metadata_types[ftype]:
                _LOG.info("mdtype %s part of skip metadata; skipping" % ftype)
                continue
            filetype_path = os.path.join(backup_repo_dir, os.path.basename(util.get_repomd_filetype_path(repodata_file, ftype)))
            # modifyrepo uses filename as mdtype, rename to type.<ext>
            renamed_filetype_path = os.path.join(os.path.dirname(filetype_path), \
                                         ftype + '.' + '.'.join(os.path.basename(filetype_path).split('.')[1:]))
            os.rename(filetype_path,  renamed_filetype_path)
            if renamed_filetype_path.endswith('.gz'):
                # if file is gzipped, decompress before passing to modifyrepo
                data = gzip.open(renamed_filetype_path).read().decode("utf-8", "replace")
                renamed_filetype_path = '.'.join(renamed_filetype_path.split('.')[:-1])
                open(renamed_filetype_path, 'w').write(data.encode("UTF-8"))
            if os.path.isfile(renamed_filetype_path):
                _LOG.info("Modifying repo for %s metadata" % ftype)
                modify_repo(current_repo_dir, renamed_filetype_path, checksum_type=checksum_type)
Esempio n. 8
0
    def test_iso_export_by_date_range(self):
        feed_url = "file://%s/test_errata_local_sync/" % self.data_dir
        repo = mock.Mock(spec=Repository)
        repo.working_dir = self.repo_working_dir
        repo.id = "test_errata_local_sync"
        repo.checksumtype = 'sha'
        sync_conduit = importer_mocks.get_sync_conduit(type_id=TYPE_ID_RPM, existing_units=[], pkg_dir=self.pkg_dir)
        config = importer_mocks.get_basic_config(feed_url=feed_url)
        importerRPM = importer_rpm.ImporterRPM()
        status, summary, details = importerRPM.sync(repo, sync_conduit, config)
        metadata = {'updated' : '2010-03-30 08:07:30'}
        existing_units = []
        unit_key_a = {'id' : '','name' :'patb', 'version' :'0.1', 'release' : '2', 'epoch':'0', 'arch' : 'x86_64', 'checksumtype' : 'sha',
                      'checksum': '017c12050a97cf6095892498750c2a39d2bf535e'}
        rpm_unit_a = Unit(TYPE_ID_RPM, unit_key_a, metadata, '')
        rpm_unit_a.storage_path = "%s/patb/0.1/2/noarch/017c12050a97cf6095892498750c2a39d2bf535e/patb-0.1-2.noarch.rpm" % self.pkg_dir
        existing_units.append(rpm_unit_a)
        unit_key_b = {'id' : '', 'name' :'emoticons', 'version' :'0.1', 'release' :'2', 'epoch':'0','arch' : 'x86_64', 'checksumtype' :'sha',
                      'checksum' : '663c89b0d29bfd5479d8736b716d50eed9495dbb'}
        rpm_unit_b = Unit(TYPE_ID_RPM, unit_key_b, metadata, '')
        rpm_unit_b.storage_path = "%s/emoticons/0.1/2/noarch/663c89b0d29bfd5479d8736b716d50eed9495dbb/emoticons-0.1-2.noarch.rpm" % self.pkg_dir
        existing_units.append(rpm_unit_b)
        sync_conduit = importer_mocks.get_sync_conduit(type_id=TYPE_ID_RPM, existing_units=existing_units, pkg_dir=self.pkg_dir)
        importerErrata = errata.ImporterErrata()
        status, summary, details = importerErrata.sync(repo, sync_conduit, config)
        unit_key = dict()
        unit_key['id'] = "RHEA-2010:9999"
        mdata = { 'description'  : "test",
                     'from_str': '*****@*****.**',
                    'issued': '2010-03-30 08:07:30',
                    'pkglist': [{'name': 'RHEL Virtualization (v. 5 for 32-bit x86)',
                            'packages': [{'arch': 'x86_64',
                                        'epoch': '0',
                                        'filename': 'patb-0.1-2.x86_64.rpm',
                                        'name': 'patb',
                                        'release': '2',
                                        'src': '',
                                        'sum': ('sha',
                                                '017c12050a97cf6095892498750c2a39d2bf535e'),
                                        'version': '0.1'},
                                        {'arch': 'x86_64',
                                        'epoch': '0',
                                        'filename': 'emoticons-0.1-2.x86_64.rpm',
                                        'name': 'emoticons',
                                        'release': '2',
                                        'src': '',
                                        'sum': ('sha',
                                                '663c89b0d29bfd5479d8736b716d50eed9495dbb'),
                                        'version': '0.1'}],
                            'short': 'rhel-i386-server-vt-5'}],
                    'pushcount': 1,
                    'reboot_suggested': False,
                    'references': [],
                    'release': '',
                    'rights': '',
                     'status': 'final',
                    'summary': '',
                    'title': 'emoticons enhancement fix and enhancement update',
                     'updated': '2010-03-30 08:07:30',
        'version': '1',
        'type' : 'enhancement',
        'severity' : 'Low',
        'solution' : ''}
        unit_key_2 = dict()
        unit_key_2['id'] = "RHEA-2008:9999"
        mdata_2 = { 'description'  : "test",
                     'from_str': '*****@*****.**',
                    'issued': '2008-03-30 00:00:00',
                    'pkglist': [{'name': 'RHEL Virtualization (v. 5 for 32-bit x86)',
                            'packages': [{'arch': 'x86_64',
                                        'epoch': '0',
                                        'filename': 'patb-0.1-2.x86_64.rpm',
                                        'name': 'patb',
                                        'release': '2',
                                        'src': '',
                                        'sum': ('sha',
                                                '017c12050a97cf6095892498750c2a39d2bf535e'),
                                        'version': '0.1'},
                                        {'arch': 'x86_64',
                                        'epoch': '0',
                                        'filename': 'emoticons-0.1-2.x86_64.rpm',
                                        'name': 'emoticons',
                                        'release': '2',
                                        'src': '',
                                        'sum': ('sha',
                                                '663c89b0d29bfd5479d8736b716d50eed9495dbb'),
                                        'version': '0.1'}],
                            'short': 'rhel-i386-server-vt-5'}],
                    'pushcount': 1,
                    'reboot_suggested': False,
                    'references': [],
                    'release': '',
                    'rights': '',
                     'status': 'final',
                    'summary': '',
                    'title': 'emoticons enhancement fix and enhancement update',
                     'updated': '2008-03-30 00:00:00',
        'version': '1',
        'type' : 'enhancement',
        'severity' : 'Low',
        'solution' : ''}
        errata_unit = [Unit(TYPE_ID_ERRATA, unit_key, mdata, ''), Unit(TYPE_ID_ERRATA, unit_key_2,  mdata_2, '')]
        existing_units += errata_unit
        print existing_units
        repo.working_dir = "%s/%s" % (self.repo_working_dir, "export")
        iso_distributor = ISODistributor()
        publish_conduit = distributor_mocks.get_publish_conduit(existing_units=existing_units, pkg_dir=self.pkg_dir)

        # test http publish
        config = distributor_mocks.get_basic_config(http_publish_dir=self.http_publish_dir, https_publish_dir=self.https_publish_dir, http=True, https=False,
            start_date="2009-03-30 08:07:30", end_date="2012-03-30 08:07:30", generate_metadata=True)
        def cleanup(rpm_working_dir):
            return
        iso_util.cleanup_working_dir = mock.Mock()
        iso_util.cleanup_working_dir.side_effect = cleanup
        report = iso_distributor.publish_repo(repo, publish_conduit, config)
        ftypes = util.get_repomd_filetypes("%s/%s" % (repo.working_dir, "repodata/repomd.xml"))
        self.assertTrue("updateinfo" in ftypes)
        updateinfo_path = util.get_repomd_filetype_path("%s/%s" % (repo.working_dir, "repodata/repomd.xml"), "updateinfo")
        updateinfo_path = os.path.join(repo.working_dir, updateinfo_path)
        self.assertTrue(os.path.exists(updateinfo_path))
        elist = updateinfo.get_errata(updateinfo_path)
        self.assertEquals(len(elist), 1)
        self.assertTrue(unit_key_2['id'] not in elist[0])
        self.assertEquals(elist[0]['id'], unit_key['id'])
        self.assertEquals(elist[0]['issued'], mdata['issued'])
        self.assertTrue(os.path.exists("%s/%s" % (self.http_publish_dir, repo.id)))
        self.assertEquals(len(os.listdir(self.https_publish_dir)), 0)
        isos_list = os.listdir("%s/%s" % (self.http_publish_dir, repo.id))
        self.assertEqual(len(isos_list), 1)
Esempio n. 9
0
    def test_errata_export(self):
        feed_url = "file://%s/test_errata_local_sync/" % self.data_dir
        repo = mock.Mock(spec=Repository)
        repo.working_dir = self.repo_working_dir
        repo.id = "test_errata_local_sync"
        repo.checksumtype = 'sha'
        sync_conduit = importer_mocks.get_sync_conduit(type_id=TYPE_ID_RPM, existing_units=[], pkg_dir=self.pkg_dir)
        config = importer_mocks.get_basic_config(feed_url=feed_url)
        importerRPM = importer_rpm.ImporterRPM()
        status, summary, details = importerRPM.sync(repo, sync_conduit, config)
        metadata = {'updated' : '2010-03-30 08:07:30'}
        unit_key_a = {'id' : '','name' :'patb', 'version' :'0.1', 'release' : '2', 'epoch':'0', 'arch' : 'x86_64', 'checksumtype' : 'md5',
                      'checksum': 'f3c197a29d9b66c5b65c5d62b25db5b4'}
        unit_key_b = {'id' : '', 'name' :'emoticons', 'version' :'0.1', 'release' :'2', 'epoch':'0','arch' : 'x86_64', 'checksumtype' :'md5',
                      'checksum' : '366bb5e73a5905eacb82c96e0578f92b'}

        existing_units = []
        for unit in [unit_key_a, unit_key_b]:
            existing_units.append(Unit(TYPE_ID_RPM, unit, metadata, ''))
        sync_conduit = importer_mocks.get_sync_conduit(type_id=TYPE_ID_RPM, existing_units=existing_units, pkg_dir=self.pkg_dir)
        importerErrata = errata.ImporterErrata()
        status, summary, details = importerErrata.sync(repo, sync_conduit, config)
        unit_key = dict()
        unit_key['id'] = "RHEA-2010:9999"
        mdata = { 'description'  : "test",
                     'from_str': '*****@*****.**',
                    'issued': '2010-03-30 08:07:30',
                    'pkglist': [{'name': 'RHEL Virtualization (v. 5 for 32-bit x86)',
                            'packages': [{'arch': 'x86_64',
                                        'epoch': '0',
                                        'filename': 'patb-0.1-2.x86_64.rpm',
                                        'name': 'patb',
                                        'release': '2',
                                        'src': '',
                                        'sum': ('md5',
                                                'f3c197a29d9b66c5b65c5d62b25db5b4'),
                                        'version': '0.1'},
                                        {'arch': 'x86_64',
                                        'epoch': '0',
                                        'filename': 'emoticons-0.1-2.x86_64.rpm',
                                        'name': 'emoticons',
                                        'release': '2',
                                        'src': '',
                                        'sum': ('md5',
                                                '366bb5e73a5905eacb82c96e0578f92b'),
                                        'version': '0.1'}],
                            'short': 'rhel-i386-server-vt-5'}],
                    'pushcount': 1,
                    'reboot_suggested': False,
                    'references': [],
                    'release': '',
                    'rights': '',
                     'status': 'final',
                    'summary': '',
                    'title': 'emoticons enhancement fix and enhancement update',
                     'updated': '2010-03-30 08:07:30',
        'version': '1',
        'type' : 'enhancement',
        'severity' : 'Low',
        'solution' : ''}
        errata_unit = [Unit(TYPE_ID_ERRATA, unit_key, mdata, '')]
        symlink_dir = "%s/%s" % (self.repo_working_dir, repo.id)
        iso_distributor = ISODistributor()
        publish_conduit = distributor_mocks.get_publish_conduit(existing_units=existing_units, pkg_dir=self.pkg_dir)
        config = distributor_mocks.get_basic_config(https_publish_dir=self.https_publish_dir, http=False, https=True)
        print symlink_dir
        repo_exporter = RepoExporter(symlink_dir)
#        rpm_units = iso_distributor._get_errata_rpms(errata_unit, existing_units)
        rpm_units = repo_exporter.get_errata_rpms(errata_unit, existing_units)
        print "RPMS in ERRATA",rpm_units
        #        iso_distributor._export_rpms(rpm_units, self.repo_working_dir)
        repo_exporter.export_rpms(rpm_units)
        status, errors = repo_exporter.export_errata(errata_unit)
#        status, errors = iso_distributor._export_errata(errata_unit, symlink_dir)
        self.assertTrue(os.path.exists("%s/%s" % (symlink_dir, "updateinfo.xml")))
        self.assertTrue(status)
        ftypes = util.get_repomd_filetypes("%s/%s" % (symlink_dir, "repodata/repomd.xml"))
        print ftypes
        self.assertTrue("updateinfo" in ftypes)
Esempio n. 10
0
 def test_publish_comps(self):
     repo = mock.Mock(spec=Repository)
     repo.id = "test_publish_comps"
     repo.working_dir = self.working_dir
     # Create 2 pkg groups
     grp_a = self.create_dummy_pkg_group_unit(repo.id, "group_a")
     grp_b = self.create_dummy_pkg_group_unit(repo.id, "group_b")
     # Create 2 pkg categories
     cat_a = self.create_dummy_pkg_category_unit(repo.id, "cat_a", ["group_a"])
     cat_b = self.create_dummy_pkg_category_unit(repo.id, "cat_b", ["group_b"])
     # Add the grps/cats to the publish_conduit
     publish_conduit = distributor_mocks.get_publish_conduit(
             existing_units=[grp_a, grp_b, cat_a, cat_b])
     config = distributor_mocks.get_basic_config(relative_url=repo.id, 
             http=True, https=True, generate_metadata=True)
     # Publish the repo, be sure 'generate_metadata' is True
     yum_distributor = YumDistributor()
     report = yum_distributor.publish_repo(repo, publish_conduit, config)
     self.assertTrue(report.success_flag)
     self.assertEqual(report.summary["num_package_groups_published"], 2)
     self.assertEqual(report.summary["num_package_categories_published"], 2)
     expected_comps_xml = os.path.join(repo.working_dir, "comps.xml")
     self.assertTrue(os.path.exists(expected_comps_xml))
     #
     # Find the path that createrepo added the comps.xml as
     #
     repomd_xml = os.path.join(repo.working_dir, "repodata", "repomd.xml")
     self.assertTrue(os.path.exists(repomd_xml))
     md_types = util.get_repomd_filetypes(repomd_xml)
     self.assertTrue('group' in md_types)
     groups_path = util.get_repomd_filetype_path(repomd_xml, "group")
     self.assertTrue(groups_path)
     groups_path = os.path.join(repo.working_dir, groups_path)
     self.assertTrue(os.path.exists(groups_path))
     #
     # Use yum to read the repodata and verify the info written matches
     # our dummy data
     #
     yc = yum.comps.Comps()
     yc.add(groups_path)
     self.assertEqual(len(yc.groups), 2)
     self.assertEqual(len(yc.categories), 2)
     for g in yc.groups:
         eg = None
         if g.groupid == "group_a":
             eg = grp_a
         elif g.groupid == "group_b":
             eg = grp_b
         else:
             # Unexpected error
             self.assertTrue(False)
         self.assertEqual(g.name, eg.metadata["name"])
         self.assertEqual(g.description, eg.metadata["description"])
         self.assertEqual(g.user_visible, eg.metadata["user_visible"])
         self.assertEqual(g.display_order, eg.metadata["display_order"])
         self.assertEqual(g.default, eg.metadata["default"])
         self.assertEqual(g.langonly, eg.metadata["langonly"])
         for pkg_name in g.mandatory_packages:
             self.assertTrue(pkg_name in eg.metadata["mandatory_package_names"])
         for pkg_name in g.optional_packages:
             self.assertTrue(pkg_name in eg.metadata["optional_package_names"])
         for pkg_name in g.default_packages:
             self.assertTrue(pkg_name in eg.metadata["default_package_names"])
         #
         # Below is related to pymongo not liking dots in a pkg_name
         # We are storing conditional_package_names as a list of tuples, (name, values)
         # convert to a dictionary to make it easier to compare against yum's data
         #
         cond_lookup = {}
         for expected_name, expected_values in eg.metadata["conditional_package_names"]:
             cond_lookup[expected_name] = expected_values
         for pkg_name in g.conditional_packages:
             # We are converting our expected value to a str below to match the behavior
             # we see from yum
             self.assertEqual(g.conditional_packages[pkg_name], str(cond_lookup[pkg_name]))
     for c in yc.categories:
         ec = None
         if c.categoryid == "cat_a":
             ec = cat_a
         elif c.categoryid == "cat_b":
             ec = cat_b
         else:
             # Unexpected error
             self.assertTrue(False)
         self.assertEqual(c.name, ec.metadata["name"])
         self.assertEqual(c.description, ec.metadata["description"])
         self.assertEqual(c.display_order, ec.metadata["display_order"])
         self.assertEqual(len(c._groups), len(ec.metadata["packagegroupids"]))
         for grpid in c._groups:
             self.assertTrue(grpid in ec.metadata["packagegroupids"])
Esempio n. 11
0
    def test_iso_export_by_date_range(self):
        feed_url = "file://%s/test_errata_local_sync/" % self.data_dir
        repo = mock.Mock(spec=Repository)
        repo.working_dir = self.repo_working_dir
        repo.id = "test_errata_local_sync"
        repo.checksumtype = "sha"
        sync_conduit = importer_mocks.get_sync_conduit(type_id=TYPE_ID_RPM, existing_units=[], pkg_dir=self.pkg_dir)
        config = importer_mocks.get_basic_config(feed_url=feed_url)
        importerRPM = importer_rpm.ImporterRPM()
        status, summary, details = importerRPM.sync(repo, sync_conduit, config)
        metadata = {}
        existing_units = []
        unit_key_a = {
            "id": "",
            "name": "patb",
            "version": "0.1",
            "release": "2",
            "epoch": "0",
            "arch": "x86_64",
            "checksumtype": "sha",
            "checksum": "017c12050a97cf6095892498750c2a39d2bf535e",
        }
        rpm_unit_a = Unit(TYPE_ID_RPM, unit_key_a, metadata, "")
        rpm_unit_a.storage_path = (
            "%s/patb/0.1/2/noarch/017c12050a97cf6095892498750c2a39d2bf535e/patb-0.1-2.noarch.rpm" % self.pkg_dir
        )
        existing_units.append(rpm_unit_a)
        unit_key_b = {
            "id": "",
            "name": "emoticons",
            "version": "0.1",
            "release": "2",
            "epoch": "0",
            "arch": "x86_64",
            "checksumtype": "sha",
            "checksum": "663c89b0d29bfd5479d8736b716d50eed9495dbb",
        }
        rpm_unit_b = Unit(TYPE_ID_RPM, unit_key_b, metadata, "")
        rpm_unit_b.storage_path = (
            "%s/emoticons/0.1/2/noarch/663c89b0d29bfd5479d8736b716d50eed9495dbb/emoticons-0.1-2.noarch.rpm"
            % self.pkg_dir
        )
        existing_units.append(rpm_unit_b)
        sync_conduit = importer_mocks.get_sync_conduit(
            type_id=TYPE_ID_RPM, existing_units=existing_units, pkg_dir=self.pkg_dir
        )
        importerErrata = errata.ImporterErrata()
        status, summary, details = importerErrata.sync(repo, sync_conduit, config)
        unit_key = dict()
        unit_key["id"] = "RHEA-2010:9999"
        mdata = {
            "description": "test",
            "from_str": "*****@*****.**",
            "issued": "2010-03-30 08:07:30",
            "pkglist": [
                {
                    "name": "RHEL Virtualization (v. 5 for 32-bit x86)",
                    "packages": [
                        {
                            "arch": "x86_64",
                            "epoch": "0",
                            "filename": "patb-0.1-2.x86_64.rpm",
                            "name": "patb",
                            "release": "2",
                            "src": "",
                            "sum": ("sha", "017c12050a97cf6095892498750c2a39d2bf535e"),
                            "version": "0.1",
                        },
                        {
                            "arch": "x86_64",
                            "epoch": "0",
                            "filename": "emoticons-0.1-2.x86_64.rpm",
                            "name": "emoticons",
                            "release": "2",
                            "src": "",
                            "sum": ("sha", "663c89b0d29bfd5479d8736b716d50eed9495dbb"),
                            "version": "0.1",
                        },
                    ],
                    "short": "rhel-i386-server-vt-5",
                }
            ],
            "pushcount": 1,
            "reboot_suggested": False,
            "references": [],
            "release": "",
            "rights": "",
            "status": "final",
            "summary": "",
            "title": "emoticons enhancement fix and enhancement update",
            "updated": "2010-03-30 08:07:30",
            "version": "1",
            "type": "enhancement",
            "severity": "Low",
            "solution": "",
        }
        unit_key_2 = dict()
        unit_key_2["id"] = "RHEA-2008:9999"
        mdata_2 = {
            "description": "test",
            "from_str": "*****@*****.**",
            "issued": "2008-03-30 00:00:00",
            "pkglist": [
                {
                    "name": "RHEL Virtualization (v. 5 for 32-bit x86)",
                    "packages": [
                        {
                            "arch": "x86_64",
                            "epoch": "0",
                            "filename": "patb-0.1-2.x86_64.rpm",
                            "name": "patb",
                            "release": "2",
                            "src": "",
                            "sum": ("sha", "017c12050a97cf6095892498750c2a39d2bf535e"),
                            "version": "0.1",
                        },
                        {
                            "arch": "x86_64",
                            "epoch": "0",
                            "filename": "emoticons-0.1-2.x86_64.rpm",
                            "name": "emoticons",
                            "release": "2",
                            "src": "",
                            "sum": ("sha", "663c89b0d29bfd5479d8736b716d50eed9495dbb"),
                            "version": "0.1",
                        },
                    ],
                    "short": "rhel-i386-server-vt-5",
                }
            ],
            "pushcount": 1,
            "reboot_suggested": False,
            "references": [],
            "release": "",
            "rights": "",
            "status": "final",
            "summary": "",
            "title": "emoticons enhancement fix and enhancement update",
            "updated": "2008-03-30 00:00:00",
            "version": "1",
            "type": "enhancement",
            "severity": "Low",
            "solution": "",
        }
        errata_unit = [Unit(TYPE_ID_ERRATA, unit_key, mdata, ""), Unit(TYPE_ID_ERRATA, unit_key_2, mdata_2, "")]
        existing_units += errata_unit
        print existing_units
        repo.working_dir = "%s/%s" % (self.repo_working_dir, "export")
        iso_distributor = ISODistributor()
        publish_conduit = distributor_mocks.get_publish_conduit(existing_units=existing_units, pkg_dir=self.pkg_dir)

        # test http publish
        config = distributor_mocks.get_basic_config(
            http_publish_dir=self.http_publish_dir,
            https_publish_dir=self.https_publish_dir,
            http=True,
            https=False,
            start_date="2009-03-30 08:07:30",
            end_date="2012-03-30 08:07:30",
            generate_metadata=True,
        )

        def cleanup(rpm_working_dir):
            return

        iso_util.cleanup_working_dir = mock.Mock()
        iso_util.cleanup_working_dir.side_effect = cleanup
        report = iso_distributor.publish_repo(repo, publish_conduit, config)
        ftypes = util.get_repomd_filetypes("%s/%s" % (repo.working_dir, "repodata/repomd.xml"))
        self.assertTrue("updateinfo" in ftypes)
        updateinfo_path = "%s/%s" % (repo.working_dir, "updateinfo.xml")
        self.assertTrue(os.path.exists(updateinfo_path))
        elist = updateinfo.get_errata(updateinfo_path)
        self.assertEquals(len(elist), 1)
        self.assertTrue(unit_key_2["id"] not in elist[0])
        self.assertEquals(elist[0]["id"], unit_key["id"])
        self.assertEquals(elist[0]["issued"], mdata["issued"])
        self.assertTrue(os.path.exists("%s/%s" % (self.http_publish_dir, repo.id)))
        self.assertEquals(len(os.listdir(self.https_publish_dir)), 0)
        isos_list = os.listdir("%s/%s" % (self.http_publish_dir, repo.id))
        self.assertEqual(len(isos_list), 1)
Esempio n. 12
0
    def test_errata_export(self):
        feed_url = "file://%s/test_errata_local_sync/" % self.data_dir
        repo = mock.Mock(spec=Repository)
        repo.working_dir = self.repo_working_dir
        repo.id = "test_errata_local_sync"
        repo.checksumtype = "sha"
        sync_conduit = importer_mocks.get_sync_conduit(type_id=TYPE_ID_RPM, existing_units=[], pkg_dir=self.pkg_dir)
        config = importer_mocks.get_basic_config(feed_url=feed_url)
        importerRPM = importer_rpm.ImporterRPM()
        status, summary, details = importerRPM.sync(repo, sync_conduit, config)
        metadata = {}
        unit_key_a = {
            "id": "",
            "name": "patb",
            "version": "0.1",
            "release": "2",
            "epoch": "0",
            "arch": "x86_64",
            "checksumtype": "md5",
            "checksum": "f3c197a29d9b66c5b65c5d62b25db5b4",
        }
        unit_key_b = {
            "id": "",
            "name": "emoticons",
            "version": "0.1",
            "release": "2",
            "epoch": "0",
            "arch": "x86_64",
            "checksumtype": "md5",
            "checksum": "366bb5e73a5905eacb82c96e0578f92b",
        }

        existing_units = []
        for unit in [unit_key_a, unit_key_b]:
            existing_units.append(Unit(TYPE_ID_RPM, unit, metadata, ""))
        sync_conduit = importer_mocks.get_sync_conduit(
            type_id=TYPE_ID_RPM, existing_units=existing_units, pkg_dir=self.pkg_dir
        )
        importerErrata = errata.ImporterErrata()
        status, summary, details = importerErrata.sync(repo, sync_conduit, config)
        unit_key = dict()
        unit_key["id"] = "RHEA-2010:9999"
        mdata = {
            "description": "test",
            "from_str": "*****@*****.**",
            "issued": "2010-03-30 08:07:30",
            "pkglist": [
                {
                    "name": "RHEL Virtualization (v. 5 for 32-bit x86)",
                    "packages": [
                        {
                            "arch": "x86_64",
                            "epoch": "0",
                            "filename": "patb-0.1-2.x86_64.rpm",
                            "name": "patb",
                            "release": "2",
                            "src": "",
                            "sum": ("md5", "f3c197a29d9b66c5b65c5d62b25db5b4"),
                            "version": "0.1",
                        },
                        {
                            "arch": "x86_64",
                            "epoch": "0",
                            "filename": "emoticons-0.1-2.x86_64.rpm",
                            "name": "emoticons",
                            "release": "2",
                            "src": "",
                            "sum": ("md5", "366bb5e73a5905eacb82c96e0578f92b"),
                            "version": "0.1",
                        },
                    ],
                    "short": "rhel-i386-server-vt-5",
                }
            ],
            "pushcount": 1,
            "reboot_suggested": False,
            "references": [],
            "release": "",
            "rights": "",
            "status": "final",
            "summary": "",
            "title": "emoticons enhancement fix and enhancement update",
            "updated": "2010-03-30 08:07:30",
            "version": "1",
            "type": "enhancement",
            "severity": "Low",
            "solution": "",
        }
        errata_unit = [Unit(TYPE_ID_ERRATA, unit_key, mdata, "")]
        symlink_dir = "%s/%s" % (self.repo_working_dir, repo.id)
        iso_distributor = ISODistributor()
        publish_conduit = distributor_mocks.get_publish_conduit(existing_units=existing_units, pkg_dir=self.pkg_dir)
        config = distributor_mocks.get_basic_config(https_publish_dir=self.https_publish_dir, http=False, https=True)
        print symlink_dir
        repo_exporter = RepoExporter(symlink_dir)
        #        rpm_units = iso_distributor._get_errata_rpms(errata_unit, existing_units)
        rpm_units = repo_exporter.get_errata_rpms(errata_unit, existing_units)
        print "RPMS in ERRATA", rpm_units
        #        iso_distributor._export_rpms(rpm_units, self.repo_working_dir)
        repo_exporter.export_rpms(rpm_units)
        status, errors = repo_exporter.export_errata(errata_unit)
        #        status, errors = iso_distributor._export_errata(errata_unit, symlink_dir)
        self.assertTrue(os.path.exists("%s/%s" % (symlink_dir, "updateinfo.xml")))
        self.assertTrue(status)
        ftypes = util.get_repomd_filetypes("%s/%s" % (symlink_dir, "repodata/repomd.xml"))
        print ftypes
        self.assertTrue("updateinfo" in ftypes)