def resolve(name):
resp = requests.get(CIRROS_URL,
headers={'User-Agent': util.get_user_agent()})
if resp.status_code != 200:
raise exceptions.HTTPError(
'Failed to fetch http://download.cirros-cloud.net/, '
'status code %d' % resp.status_code)
if name == 'cirros':
versions = []
dir_re = re.compile(r'.*<a href="([0-9]+\.[0-9]+\.[0-9]+)/">.*/</a>.*')
for line in resp.text.split('\n'):
m = dir_re.match(line)
if m:
versions.append(m.group(1))
LOG.info('Found cirros versions: %s' % versions)
vernum = versions[-1]
else:
try:
# Name is assumed to be in the form cirros:0.4.0
_, vernum = name.split(':')
except Exception:
raise exceptions.VersionSpecificationError(
'Cannot parse version: %s' % name)
return (config.parsed.get('DOWNLOAD_URL_CIRROS') % {'vernum': vernum})
def _open_connection(self):
resp = requests.get(self.url,
allow_redirects=True,
stream=True,
headers={'User-Agent': util.get_user_agent()})
if resp.status_code != 200:
raise exceptions.HTTPError(
'Failed to fetch HEAD of %s (status code %d)' %
(self.url, resp.status_code))
return resp
def resolve(name):
# Name is assumed to be in the form ubuntu, ubuntu:18.04, or ubuntu:bionic
resp = requests.get(config.LISTING_URL_UBUNTU,
allow_redirects=True,
headers={'User-Agent': util_general.get_user_agent()})
if resp.status_code != 200:
raise exceptions.HTTPError(
'Failed to fetch %s, status code %d' %
(config.LISTING_URL_UBUNTU, resp.status_code))
num_to_name = {}
name_to_num = {}
dir_re = re.compile(
r'.*<a href="(.*)/">.*Ubuntu Server ([0-9]+\.[0-9]+).*')
for line in resp.text.split('\n'):
m = dir_re.match(line)
if m:
num_to_name[m.group(2)] = m.group(1)
name_to_num[m.group(1)] = m.group(2)
LOG.with_field('versions', num_to_name).info('Found ubuntu versions')
vernum = None
vername = None
if name == 'ubuntu':
vernum = sorted(num_to_name.keys())[-1]
vername = num_to_name[vernum]
else:
try:
_, version = name.split(':')
if version in num_to_name:
vernum = version
vername = num_to_name[version]
else:
vername = version
vernum = name_to_num[version]
except Exception:
raise exceptions.VersionSpecificationError(
'Cannot parse version: %s' % name)
url = (config.DOWNLOAD_URL_UBUNTU % {'vernum': vernum, 'vername': vername})
checksum_url = config.CHECKSUM_URL_UBUNTU % {'vername': vername}
checksums = resolver_util.fetch_remote_checksum(checksum_url)
checksum = checksums.get('*' + os.path.basename(url))
LOG.with_fields({
'name': name,
'url': url,
'checksum': checksum
}).info('Image resolved')
if checksum:
return url, checksum, 'md5'
else:
return url, None, None
def resolve(name):
resp = requests.get(CIRROS_URL,
headers={'User-Agent': util.get_user_agent()})
if resp.status_code != 200:
raise exceptions.HTTPError(
'Failed to fetch http://download.cirros-cloud.net/, '
'status code %d' % resp.status_code)
if name == 'cirros':
versions = []
dir_re = re.compile(r'.*<a href="([0-9]+\.[0-9]+\.[0-9]+)/">.*/</a>.*')
for line in resp.text.split('\n'):
m = dir_re.match(line)
if m:
versions.append(m.group(1))
LOG.withField('versions', versions).info('Found cirros versions')
vernum = versions[-1]
else:
try:
# Name is assumed to be in the form cirros:0.4.0
_, vernum = name.split(':')
except Exception:
raise exceptions.VersionSpecificationError(
'Cannot parse version: %s' % name)
url = config.parsed.get('DOWNLOAD_URL_CIRROS') % {'vernum': vernum}
log = LOG.withField('url', url)
# Retrieve check sum file
checksum_url = CIRROS_URL + '/' + vernum + '/MD5SUMS'
resp = requests.get(checksum_url,
headers={'User-Agent': util.get_user_agent()})
log.withField('checksum_url', checksum_url).withField(
'resp', resp).debug("Checksum request response")
if resp.status_code != 200:
# Cirros does not always have a checksum file available
log.warning('Unable to retrieve MD5SUMS')
return url, None
sum_re = re.compile(r'^([0-9a-f]+) .*' + 'cirros-' + vernum +
'-x86_64-disk.img')
checksum = None
for line in resp.text.split('\n'):
m = sum_re.match(line)
if m:
checksum = m.group(1)
break
if not checksum_url:
log.warning('Did not find checksum')
log.withField('checksum', checksum).info('Checksum retrieval')
return (url, checksum)
def resolve(name):
# Name is assumed to be in the form debian or debian:10
if name.startswith('debian:'):
try:
_, vernum = name.split(':')
except Exception:
raise exceptions.VersionSpecificationError(
'Cannot parse version: %s' % name)
url = config.DOWNLOAD_URL_DEBIAN % {'vernum': vernum}
checksum_url = config.CHECKSUM_URL_DEBIAN % {'vernum': vernum}
resp = requests.head(
url,
allow_redirects=True,
headers={'User-Agent': util_general.get_user_agent()})
if resp.status_code != 200:
raise exceptions.HTTPError('Failed to fetch %s, status code %d' %
(url, resp.status_code))
else:
found_any = False
for vernum in range(9, 20):
url = config.DOWNLOAD_URL_DEBIAN % {'vernum': vernum}
resp = requests.head(
url,
allow_redirects=True,
headers={'User-Agent': util_general.get_user_agent()})
if resp.status_code != 200:
if found_any:
vernum -= 1
break
else:
found_any = True
url = config.DOWNLOAD_URL_DEBIAN % {'vernum': vernum}
checksum_url = config.CHECKSUM_URL_DEBIAN % {'vernum': vernum}
checksums = resolver_util.fetch_remote_checksum(checksum_url)
checksum = checksums.get(os.path.basename(url))
LOG.with_fields({
'name': name,
'url': url,
'checksum': checksum
}).info('Image resolved')
if checksum:
return url, checksum, 'md5'
else:
return url, None, None
def _open_connection(self, url):
proxies = {}
if config.HTTP_PROXY_SERVER:
proxies['http'] = config.HTTP_PROXY_SERVER
resp = requests.get(
url,
allow_redirects=True,
stream=True,
headers={'User-Agent': util_general.get_user_agent()},
proxies=proxies)
if resp.status_code != 200:
raise exceptions.HTTPError(
'Failed to fetch HEAD of %s (status code %d)' %
(url, resp.status_code))
return resp
def _requires_fetch(self):
resp = requests.get(self.url,
allow_redirects=True,
stream=True,
headers={'User-Agent': util.get_user_agent()})
if resp.status_code != 200:
raise exceptions.HTTPError(
'Failed to fetch HEAD of %s (status code %d)' %
(self.url, resp.status_code))
dirty_fields = {}
for field in VALIDATED_IMAGE_FIELDS:
if self.info.get(field) != resp.headers.get(field):
dirty_fields[field] = {
'before': self.info.get(field),
'after': resp.headers.get(field)
}
return dirty_fields, resp
def resolve(name):
resp = requests.get(UBUNTU_URL,
headers={'User-Agent': util.get_user_agent()})
if resp.status_code != 200:
raise exceptions.HTTPError('Failed to fetch %s, status code %d' %
(UBUNTU_URL, resp.status_code))
num_to_name = {}
name_to_num = {}
dir_re = re.compile(
r'.*<a href="(.*)/">.*Ubuntu Server ([0-9]+\.[0-9]+).*')
for line in resp.text.split('\n'):
m = dir_re.match(line)
if m:
num_to_name[m.group(2)] = m.group(1)
name_to_num[m.group(1)] = m.group(2)
logutil.info(None, 'Found ubuntu versions: %s' % num_to_name)
vernum = None
vername = None
if name == 'ubuntu':
vernum = sorted(num_to_name.keys())[-1]
vername = num_to_name[vernum]
else:
try:
# Name is assumed to be in the form ubuntu:18.04 or ubuntu:bionic
_, version = name.split(':')
if version in num_to_name:
vernum = version
vername = num_to_name[version]
else:
vername = version
vernum = name_to_num[version]
except Exception:
raise exceptions.VersionSpecificationError(
'Cannot parse version: %s' % name)
return (config.parsed.get('DOWNLOAD_URL_UBUNTU') % {
'vernum': vernum,
'vername': vername
})
def resolve(name):
# Name is assumed to be in the form cirros or cirros:0.4.0
if name == 'cirros':
resp = requests.get(
config.LISTING_URL_CIRROS,
allow_redirects=True,
headers={'User-Agent': util_general.get_user_agent()})
if resp.status_code != 200:
raise exceptions.HTTPError(
'Failed to fetch %s, status code %d' %
(config.LISTING_URL_CIRROS, resp.status_code))
versions = []
dir_re = re.compile(r'.*<a href="([0-9]+\.[0-9]+\.[0-9]+)/">.*/</a>.*')
for line in resp.text.split('\n'):
m = dir_re.match(line)
if m:
versions.append(m.group(1))
LOG.with_field('versions', versions).info('Found cirros versions')
vernum = versions[-1]
else:
try:
_, vernum = name.split(':')
except Exception:
raise exceptions.VersionSpecificationError(
'Cannot parse version: %s' % name)
url = config.DOWNLOAD_URL_CIRROS % {'vernum': vernum}
checksum_url = config.CHECKSUM_URL_CIRROS % {'vernum': vernum}
checksums = resolver_util.fetch_remote_checksum(checksum_url)
checksum = checksums.get(os.path.basename(url))
LOG.with_fields({
'name': name,
'url': url,
'checksum': checksum
}).info('Image resolved')
if checksum:
return url, checksum, 'md5'
else:
return url, None, None
def requires_fetch(image_url):
hashed_image_url = _hash_image_url(image_url)
hashed_image_path = os.path.join(_get_cache_path(), hashed_image_url)
info = _read_info(image_url, hashed_image_url, hashed_image_path)
resp = requests.get(image_url,
allow_redirects=True,
stream=True,
headers={'User-Agent': util.get_user_agent()})
if resp.status_code != 200:
raise exceptions.HTTPError(
'Failed to fetch HEAD of %s (status code %d)' %
(image_url, resp.status_code))
image_dirty = False
for field in VALIDATED_IMAGE_FIELDS:
if info.get(field) != resp.headers.get(field):
image_dirty = True
return hashed_image_path, info, image_dirty, resp
def resolve(name):
resp = requests.get(UBUNTU_URL,
headers={'User-Agent': util.get_user_agent()})
if resp.status_code != 200:
raise exceptions.HTTPError('Failed to fetch %s, status code %d' %
(UBUNTU_URL, resp.status_code))
num_to_name = {}
name_to_num = {}
dir_re = re.compile(
r'.*<a href="(.*)/">.*Ubuntu Server ([0-9]+\.[0-9]+).*')
for line in resp.text.split('\n'):
m = dir_re.match(line)
if m:
num_to_name[m.group(2)] = m.group(1)
name_to_num[m.group(1)] = m.group(2)
LOG.withField('versions', num_to_name).info('Found ubuntu versions')
vernum = None
vername = None
if name == 'ubuntu':
vernum = sorted(num_to_name.keys())[-1]
vername = num_to_name[vernum]
else:
try:
# Name is assumed to be in the form ubuntu:18.04 or ubuntu:bionic
_, version = name.split(':')
if version in num_to_name:
vernum = version
vername = num_to_name[version]
else:
vername = version
vernum = name_to_num[version]
except Exception:
raise exceptions.VersionSpecificationError(
'Cannot parse version: %s' % name)
url = (config.parsed.get('DOWNLOAD_URL_UBUNTU') % {
'vernum': vernum,
'vername': vername
})
log = LOG.withField('url', url)
# Retrieve check sum file
checksum_url = UBUNTU_URL + '/' + vername + '/current/MD5SUMS'
resp = requests.get(checksum_url,
headers={'User-Agent': util.get_user_agent()})
if resp.status_code != 200:
raise exceptions.HTTPError('Failed to fetch %s, status code %d' %
(checksum_url, resp.status_code))
sum_re = re.compile(r'^([0-9a-f]+) .*' + vername +
'-server-cloudimg-amd64.img')
checksum = None
for line in resp.text.split('\n'):
m = sum_re.match(line)
if m:
checksum = m.group(1)
break
if not checksum_url:
log.warning('Did not find checksum')
log.withField('checksum', checksum).info('Checksum check')
return (url, checksum)
