def update_rpm_repo(repo): """ Update an rpm repo. Checks if the repo url is a mirrorlist, and extracts mirrors if so. If not, checks a number of common rpm repo formats to determine which type of repo it is, and to determine the mirror urls. """ formats = ['repodata/repomd.xml.bz2', 'repodata/repomd.xml.gz', 'repodata/repomd.xml', 'suse/repodata/repomd.xml.bz2', 'suse/repodata/repomd.xml.gz', 'suse/repodata/repomd.xml', 'content'] mirrorlists_check(repo) for mirror in repo.mirror_set.filter(mirrorlist=False, refresh=True): repo_url, res, yast = find_mirror_url(mirror.url, formats) mirror.last_access_ok = check_response(res) if mirror.last_access_ok: data = download_url(res, 'Downloading repo info (1/2):') if not yast: debug_message.send(sender=None, text='Found yum rpm repo - %s\n' % repo_url) packages = update_yum_repo(mirror, data, repo_url) else: debug_message.send(sender=None, text='Found yast rpm repo - %s\n' % repo_url) packages = update_yast_repo(mirror, data, repo_url) mirror.timestamp = datetime.now() mirror.last_access_ok = True if packages: update_mirror_packages(mirror, packages) else: mirror.fail() mirror.save()
def refresh_deb_repo(repo): """ Refresh a debian repo. Checks for the Packages* files to determine what the mirror urls are and then downloads and extracts packages from those files. """ formats = ['Packages.bz2', 'Packages.gz', 'Packages'] for mirror in repo.mirror_set.filter(refresh=True): repo_url, res, unused = find_mirror_url(mirror.url, formats) mirror.last_access_ok = check_response(res) if mirror.last_access_ok: text = 'Found deb repo - %s\n' % repo_url debug_message.send(sender=None, text=text) data = download_url(res, 'Downloading repo info:') if data is None: mirror.fail() return sha1 = get_sha1(data) if mirror.file_checksum == sha1: text = 'Mirror checksum has not changed, ' text += 'not refreshing package metadata\n' info_message.send(sender=None, text=text) else: packages = set() extract_deb_packages(data, packages) mirror.last_access_ok = True mirror.timestamp = datetime.now() update_mirror_packages(mirror, packages) mirror.file_checksum = sha1 packages.clear() else: mirror.fail() mirror.save()
def find_mirror_url(stored_mirror_url, formats): """ Find the actual URL of the mirror by trying predefined paths """ for fmt in formats: mirror_url = stored_mirror_url for f in formats: if mirror_url.endswith(f): mirror_url = mirror_url[:-len(f)] mirror_url = mirror_url.rstrip('/') + '/' + fmt debug_message.send(sender=None, text='Checking {0!s}'.format(mirror_url)) res = get_url(mirror_url) if res is not None and res.ok: return res
def get_url(url): try: req = Request(url) res = urlopen(req) # don't blindly succeed with http 200 (e.g. sourceforge) headers = dict(res.headers.items()) if 'content-type' in headers and not re.match('text/html', headers['content-type']): return res else: return -1 except IOError, e: if hasattr(e, 'reason'): debug_message.send(sender=None, text='%s - %s\n' % (url, e.reason)) return -1 elif hasattr(e, 'code'): debug_message.send(sender=None, text='%s - %s\n' % (url, e)) return e.code else: error_message.send(sender=None, text='Unknown error: %s - %e\n' % (url, e)) return -1