def update_rpm_repo(repo):
""" Update an rpm repo.
Checks if the repo url is a mirrorlist, and extracts mirrors if so.
If not, checks a number of common rpm repo formats to determine
which type of repo it is, and to determine the mirror urls.
"""
formats = ['repodata/repomd.xml.bz2', 'repodata/repomd.xml.gz', 'repodata/repomd.xml', 'suse/repodata/repomd.xml.bz2', 'suse/repodata/repomd.xml.gz', 'suse/repodata/repomd.xml', 'content']
mirrorlists_check(repo)
for mirror in repo.mirror_set.filter(mirrorlist=False, refresh=True):
repo_url, res, yast = find_mirror_url(mirror.url, formats)
mirror.last_access_ok = check_response(res)
if mirror.last_access_ok:
data = download_url(res, 'Downloading repo info (1/2):')
if not yast:
debug_message.send(sender=None, text='Found yum rpm repo - %s\n' % repo_url)
packages = update_yum_repo(mirror, data, repo_url)
else:
debug_message.send(sender=None, text='Found yast rpm repo - %s\n' % repo_url)
packages = update_yast_repo(mirror, data, repo_url)
mirror.timestamp = datetime.now()
mirror.last_access_ok = True
if packages:
update_mirror_packages(mirror, packages)
else:
mirror.fail()
mirror.save()
def refresh_deb_repo(repo):
""" Refresh a debian repo.
Checks for the Packages* files to determine what the mirror urls
are and then downloads and extracts packages from those files.
"""
formats = ['Packages.bz2', 'Packages.gz', 'Packages']
for mirror in repo.mirror_set.filter(refresh=True):
repo_url, res, unused = find_mirror_url(mirror.url, formats)
mirror.last_access_ok = check_response(res)
if mirror.last_access_ok:
text = 'Found deb repo - %s\n' % repo_url
debug_message.send(sender=None, text=text)
data = download_url(res, 'Downloading repo info:')
if data is None:
mirror.fail()
return
sha1 = get_sha1(data)
if mirror.file_checksum == sha1:
text = 'Mirror checksum has not changed, '
text += 'not refreshing package metadata\n'
info_message.send(sender=None, text=text)
else:
packages = set()
extract_deb_packages(data, packages)
mirror.last_access_ok = True
mirror.timestamp = datetime.now()
update_mirror_packages(mirror, packages)
mirror.file_checksum = sha1
packages.clear()
else:
mirror.fail()
mirror.save()
def find_mirror_url(stored_mirror_url, formats):
""" Find the actual URL of the mirror by trying predefined paths
"""
for fmt in formats:
mirror_url = stored_mirror_url
for f in formats:
if mirror_url.endswith(f):
mirror_url = mirror_url[:-len(f)]
mirror_url = mirror_url.rstrip('/') + '/' + fmt
debug_message.send(sender=None,
text='Checking {0!s}'.format(mirror_url))
res = get_url(mirror_url)
if res is not None and res.ok:
return res
def find_mirror_url(stored_mirror_url, formats):
""" Find the actual URL of the mirror by trying predefined paths
"""
for fmt in formats:
mirror_url = stored_mirror_url
for f in formats:
if mirror_url.endswith(f):
mirror_url = mirror_url[:-len(f)]
mirror_url = mirror_url.rstrip('/') + '/' + fmt
debug_message.send(sender=None,
text='Checking {0!s}'.format(mirror_url))
res = get_url(mirror_url)
if res is not None and res.ok:
return res
def get_url(url):
try:
req = Request(url)
res = urlopen(req)
# don't blindly succeed with http 200 (e.g. sourceforge)
headers = dict(res.headers.items())
if 'content-type' in headers and not re.match('text/html', headers['content-type']):
return res
else:
return -1
except IOError, e:
if hasattr(e, 'reason'):
debug_message.send(sender=None, text='%s - %s\n' % (url, e.reason))
return -1
elif hasattr(e, 'code'):
debug_message.send(sender=None, text='%s - %s\n' % (url, e))
return e.code
else:
error_message.send(sender=None, text='Unknown error: %s - %e\n' % (url, e))
return -1