Ejemplo n.º 1
0
def update_rpm_repo(repo):
    """ Update an rpm repo.
        Checks if the repo url is a mirrorlist, and extracts mirrors if so.
        If not, checks a number of common rpm repo formats to determine
        which type of repo it is, and to determine the mirror urls.
    """

    formats = ['repodata/repomd.xml.bz2', 'repodata/repomd.xml.gz', 'repodata/repomd.xml', 'suse/repodata/repomd.xml.bz2', 'suse/repodata/repomd.xml.gz', 'suse/repodata/repomd.xml', 'content']

    mirrorlists_check(repo)

    for mirror in repo.mirror_set.filter(mirrorlist=False, refresh=True):
        repo_url, res, yast = find_mirror_url(mirror.url, formats)
        mirror.last_access_ok = check_response(res)

        if mirror.last_access_ok:
            data = download_url(res, 'Downloading repo info (1/2):')
            if not yast:
                debug_message.send(sender=None, text='Found yum rpm repo - %s\n' % repo_url)
                packages = update_yum_repo(mirror, data, repo_url)
            else:
                debug_message.send(sender=None, text='Found yast rpm repo - %s\n' % repo_url)
                packages = update_yast_repo(mirror, data, repo_url)
            mirror.timestamp = datetime.now()
            mirror.last_access_ok = True
            if packages:
                update_mirror_packages(mirror, packages)
        else:
            mirror.fail()

        mirror.save()
Ejemplo n.º 2
0
def refresh_deb_repo(repo):
    """ Refresh a debian repo.
        Checks for the Packages* files to determine what the mirror urls
        are and then downloads and extracts packages from those files.
    """

    formats = ['Packages.bz2', 'Packages.gz', 'Packages']

    for mirror in repo.mirror_set.filter(refresh=True):
        repo_url, res, unused = find_mirror_url(mirror.url, formats)
        mirror.last_access_ok = check_response(res)

        if mirror.last_access_ok:
            text = 'Found deb repo - %s\n' % repo_url
            debug_message.send(sender=None, text=text)
            data = download_url(res, 'Downloading repo info:')
            if data is None:
                mirror.fail()
                return
            sha1 = get_sha1(data)
            if mirror.file_checksum == sha1:
                text = 'Mirror checksum has not changed, '
                text += 'not refreshing package metadata\n'
                info_message.send(sender=None, text=text)
            else:
                packages = set()
                extract_deb_packages(data, packages)
                mirror.last_access_ok = True
                mirror.timestamp = datetime.now()
                update_mirror_packages(mirror, packages)
                mirror.file_checksum = sha1
                packages.clear()
        else:
            mirror.fail()
        mirror.save()
Ejemplo n.º 3
0
def find_mirror_url(stored_mirror_url, formats):
    """ Find the actual URL of the mirror by trying predefined paths
    """

    for fmt in formats:
        mirror_url = stored_mirror_url
        for f in formats:
            if mirror_url.endswith(f):
                mirror_url = mirror_url[:-len(f)]
        mirror_url = mirror_url.rstrip('/') + '/' + fmt
        debug_message.send(sender=None,
                           text='Checking {0!s}'.format(mirror_url))
        res = get_url(mirror_url)
        if res is not None and res.ok:
            return res
Ejemplo n.º 4
0
def find_mirror_url(stored_mirror_url, formats):
    """ Find the actual URL of the mirror by trying predefined paths
    """

    for fmt in formats:
        mirror_url = stored_mirror_url
        for f in formats:
            if mirror_url.endswith(f):
                mirror_url = mirror_url[:-len(f)]
        mirror_url = mirror_url.rstrip('/') + '/' + fmt
        debug_message.send(sender=None,
                           text='Checking {0!s}'.format(mirror_url))
        res = get_url(mirror_url)
        if res is not None and res.ok:
            return res
Ejemplo n.º 5
0
def get_url(url):

    try:
        req = Request(url)
        res = urlopen(req)
        # don't blindly succeed with http 200 (e.g. sourceforge)
        headers = dict(res.headers.items())
        if 'content-type' in headers and not re.match('text/html', headers['content-type']):
            return res
        else:
            return -1
    except IOError, e:
        if hasattr(e, 'reason'):
            debug_message.send(sender=None, text='%s - %s\n' % (url, e.reason))
            return -1
        elif hasattr(e, 'code'):
            debug_message.send(sender=None, text='%s - %s\n' % (url, e))
            return e.code
        else:
            error_message.send(sender=None, text='Unknown error: %s - %e\n' % (url, e))
            return -1