def get_instance_metadata(api_version='latest',
metadata_address='http://169.254.169.254',
ssl_details=None, timeout=5, retries=5,
leaf_decoder=None):
md_url = url_helper.combine_url(metadata_address, api_version)
# Note, 'meta-data' explicitly has trailing /.
# this is required for CloudStack (LP: #1356855)
md_url = url_helper.combine_url(md_url, 'meta-data/')
caller = functools.partial(util.read_file_or_url,
ssl_details=ssl_details, timeout=timeout,
retries=retries)
def mcaller(url):
return caller(url).contents
try:
response = caller(md_url)
materializer = MetadataMaterializer(response.contents,
md_url, mcaller,
leaf_decoder=leaf_decoder)
md = materializer.materialize()
if not isinstance(md, (dict)):
md = {}
return md
except Exception:
util.logexc(LOG, "Failed fetching metadata from url %s", md_url)
return {}
def test_metadata_fetch_no_keys(self):
base_url = "http://169.254.169.254/%s/meta-data/" % (self.VERSION)
hp.register_uri(
hp.GET,
base_url,
status=200,
body="\n".join(["hostname", "instance-id", "ami-launch-index"]),
)
hp.register_uri(
hp.GET,
uh.combine_url(base_url, "hostname"),
status=200,
body="ec2.fake.host.name.com",
)
hp.register_uri(
hp.GET,
uh.combine_url(base_url, "instance-id"),
status=200,
body="123",
)
hp.register_uri(
hp.GET,
uh.combine_url(base_url, "ami-launch-index"),
status=200,
body="1",
)
md = ec2.get_instance_metadata(self.VERSION, retries=0)
self.assertEqual(md["hostname"], "ec2.fake.host.name.com")
self.assertEqual(md["instance-id"], "123")
self.assertEqual(md["ami-launch-index"], "1")
def test_metadata_fetch_with_2_keys(self):
base_url = "http://169.254.169.254/%s/meta-data" % (self.VERSION)
hp.register_uri(hp.GET, base_url, status=200, body="\n".join(["hostname", "instance-id", "public-keys/"]))
hp.register_uri(hp.GET, uh.combine_url(base_url, "hostname"), status=200, body="ec2.fake.host.name.com")
hp.register_uri(hp.GET, uh.combine_url(base_url, "instance-id"), status=200, body="123")
hp.register_uri(
hp.GET,
uh.combine_url(base_url, "public-keys/"),
status=200,
body="\n".join(["0=my-public-key", "1=my-other-key"]),
)
hp.register_uri(
hp.GET,
uh.combine_url(base_url, "public-keys/0/openssh-key"),
status=200,
body="ssh-rsa AAAA.....wZEf my-public-key",
)
hp.register_uri(
hp.GET,
uh.combine_url(base_url, "public-keys/1/openssh-key"),
status=200,
body="ssh-rsa AAAA.....wZEf my-other-key",
)
md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
self.assertEquals(md["hostname"], "ec2.fake.host.name.com")
self.assertEquals(md["instance-id"], "123")
self.assertEquals(2, len(md["public-keys"]))
def test_metadata_fetch_bdm(self):
base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION)
hp.register_uri(hp.GET, base_url, status=200,
body="\n".join(['hostname',
'instance-id',
'block-device-mapping/']))
hp.register_uri(hp.GET, uh.combine_url(base_url, 'hostname'),
status=200, body='ec2.fake.host.name.com')
hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'),
status=200, body='123')
hp.register_uri(hp.GET,
uh.combine_url(base_url, 'block-device-mapping/'),
status=200,
body="\n".join(['ami', 'ephemeral0']))
hp.register_uri(hp.GET,
uh.combine_url(base_url, 'block-device-mapping/ami'),
status=200,
body="sdb")
hp.register_uri(hp.GET,
uh.combine_url(base_url,
'block-device-mapping/ephemeral0'),
status=200,
body="sdc")
md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
self.assertEqual(md['hostname'], 'ec2.fake.host.name.com')
self.assertEqual(md['instance-id'], '123')
bdm = md['block-device-mapping']
self.assertEqual(2, len(bdm))
self.assertEqual(bdm['ami'], 'sdb')
self.assertEqual(bdm['ephemeral0'], 'sdc')
def test_metadata_fetch_no_keys(self):
base_url = "http://169.254.169.254/%s/meta-data" % (self.VERSION)
hp.register_uri(hp.GET, base_url, status=200, body="\n".join(["hostname", "instance-id", "ami-launch-index"]))
hp.register_uri(hp.GET, uh.combine_url(base_url, "hostname"), status=200, body="ec2.fake.host.name.com")
hp.register_uri(hp.GET, uh.combine_url(base_url, "instance-id"), status=200, body="123")
hp.register_uri(hp.GET, uh.combine_url(base_url, "ami-launch-index"), status=200, body="1")
md = eu.get_instance_metadata(self.VERSION, retries=0)
self.assertEquals(md["hostname"], "ec2.fake.host.name.com")
self.assertEquals(md["instance-id"], "123")
self.assertEquals(md["ami-launch-index"], "1")
def test_metadata_fetch_no_keys(self):
base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION)
hp.register_uri(hp.GET, base_url, status=200,
body="\n".join(['hostname',
'instance-id',
'ami-launch-index']))
hp.register_uri(hp.GET, uh.combine_url(base_url, 'hostname'),
status=200, body='ec2.fake.host.name.com')
hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'),
status=200, body='123')
hp.register_uri(hp.GET, uh.combine_url(base_url, 'ami-launch-index'),
status=200, body='1')
md = eu.get_instance_metadata(self.VERSION, retries=0)
self.assertEqual(md['hostname'], 'ec2.fake.host.name.com')
self.assertEqual(md['instance-id'], '123')
self.assertEqual(md['ami-launch-index'], '1')
def wait_for_metadata_service(self):
urls = self.ds_cfg.get("metadata_urls", [DEF_MD_URL])
filtered = [x for x in urls if util.is_resolvable_url(x)]
if set(filtered) != set(urls):
LOG.debug("Removed the following from metadata urls: %s",
list((set(urls) - set(filtered))))
if len(filtered):
urls = filtered
else:
LOG.warn("Empty metadata url list! using default list")
urls = [DEF_MD_URL]
md_urls = []
url2base = {}
for url in urls:
md_url = url_helper.combine_url(url, 'openstack')
md_urls.append(md_url)
url2base[md_url] = url
(max_wait, timeout) = self._get_url_settings()
start_time = time.time()
avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait,
timeout=timeout)
if avail_url:
LOG.debug("Using metadata source: '%s'", url2base[avail_url])
else:
LOG.debug("Giving up on OpenStack md from %s after %s seconds",
md_urls, int(time.time() - start_time))
self.metadata_address = url2base.get(avail_url)
return bool(avail_url)
def test_metadata_fetch_bdm(self):
base_url = "http://169.254.169.254/%s/meta-data" % (self.VERSION)
hp.register_uri(
hp.GET, base_url, status=200, body="\n".join(["hostname", "instance-id", "block-device-mapping/"])
)
hp.register_uri(hp.GET, uh.combine_url(base_url, "hostname"), status=200, body="ec2.fake.host.name.com")
hp.register_uri(hp.GET, uh.combine_url(base_url, "instance-id"), status=200, body="123")
hp.register_uri(
hp.GET, uh.combine_url(base_url, "block-device-mapping/"), status=200, body="\n".join(["ami", "ephemeral0"])
)
hp.register_uri(hp.GET, uh.combine_url(base_url, "block-device-mapping/ami"), status=200, body="sdb")
hp.register_uri(hp.GET, uh.combine_url(base_url, "block-device-mapping/ephemeral0"), status=200, body="sdc")
md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
self.assertEquals(md["hostname"], "ec2.fake.host.name.com")
self.assertEquals(md["instance-id"], "123")
bdm = md["block-device-mapping"]
self.assertEquals(2, len(bdm))
self.assertEquals(bdm["ami"], "sdb")
self.assertEquals(bdm["ephemeral0"], "sdc")
def get_instance_metadata(api_version='latest',
metadata_address='http://169.254.169.254',
ssl_details=None, timeout=5, retries=5):
md_url = url_helper.combine_url(metadata_address, api_version)
md_url = url_helper.combine_url(md_url, 'meta-data')
caller = functools.partial(util.read_file_or_url,
ssl_details=ssl_details, timeout=timeout,
retries=retries)
try:
response = caller(md_url)
materializer = MetadataMaterializer(str(response), md_url, caller)
md = materializer.materialize()
if not isinstance(md, (dict)):
md = {}
return md
except Exception:
util.logexc(LOG, "Failed fetching metadata from url %s", md_url)
return {}
def test_metadata_fetch_key(self):
base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION)
hp.register_uri(hp.GET, base_url, status=200,
body="\n".join(['hostname',
'instance-id',
'public-keys/']))
hp.register_uri(hp.GET, uh.combine_url(base_url, 'hostname'),
status=200, body='ec2.fake.host.name.com')
hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'),
status=200, body='123')
hp.register_uri(hp.GET, uh.combine_url(base_url, 'public-keys/'),
status=200, body='0=my-public-key')
hp.register_uri(hp.GET,
uh.combine_url(base_url, 'public-keys/0/openssh-key'),
status=200, body='ssh-rsa AAAA.....wZEf my-public-key')
md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
self.assertEqual(md['hostname'], 'ec2.fake.host.name.com')
self.assertEqual(md['instance-id'], '123')
self.assertEqual(1, len(md['public-keys']))
def get_instance_metadata(api_version='latest',
metadata_address='http://169.254.169.254',
ssl_details=None, timeout=5, retries=5):
md_url = url_helper.combine_url(metadata_address, api_version)
md_url = url_helper.combine_url(md_url, 'meta-data')
caller = functools.partial(util.read_file_or_url,
ssl_details=ssl_details, timeout=timeout,
retries=retries)
try:
response = caller(md_url)
materializer = MetadataMaterializer(str(response), md_url, caller)
md = materializer.materialize()
if not isinstance(md, (dict)):
md = {}
return md
except Exception:
util.logexc(LOG, "Failed fetching metadata from url %s", md_url)
return {}
def get_instance_userdata(api_version='latest',
metadata_address='http://169.254.169.254',
ssl_details=None, timeout=5, retries=5):
ud_url = url_helper.combine_url(metadata_address, api_version)
ud_url = url_helper.combine_url(ud_url, 'user-data')
user_data = ''
try:
# It is ok for userdata to not exist (thats why we are stopping if
# NOT_FOUND occurs) and just in that case returning an empty string.
exception_cb = functools.partial(_skip_retry_on_codes,
SKIP_USERDATA_CODES)
response = url_helper.read_file_or_url(
ud_url, ssl_details=ssl_details, timeout=timeout,
retries=retries, exception_cb=exception_cb)
user_data = response.contents
except url_helper.UrlError as e:
if e.code not in SKIP_USERDATA_CODES:
util.logexc(LOG, "Failed fetching userdata from url %s", ud_url)
except Exception:
util.logexc(LOG, "Failed fetching userdata from url %s", ud_url)
return user_data
def _materialize(self, blob, base_url):
(leaves, children) = self._parse(blob)
child_contents = {}
for c in children:
child_url = url_helper.combine_url(base_url, c)
if not child_url.endswith("/"):
child_url += "/"
child_blob = str(self._caller(child_url))
child_contents[c] = self._materialize(child_blob, child_url)
leaf_contents = {}
for (field, resource) in leaves.items():
leaf_url = url_helper.combine_url(base_url, resource)
leaf_blob = str(self._caller(leaf_url))
leaf_contents[field] = self._decode_leaf_blob(field, leaf_blob)
joined = {}
joined.update(child_contents)
for field in leaf_contents.keys():
if field in joined:
LOG.warn("Duplicate key found in results from %s", base_url)
else:
joined[field] = leaf_contents[field]
return joined
def wait_for_metadata_service(self):
(max_wait, timeout) = self._get_url_settings()
urls = [uhelp.combine_url(self.metadata_address,
'latest/meta-data/instance-id')]
start_time = time.time()
url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
timeout=timeout, status_cb=LOG.warn)
if url:
LOG.debug("Using metadata source: '%s'", url)
else:
LOG.critical(("Giving up on waiting for the metadata from %s"
" after %s seconds"),
urls, int(time.time() - start_time))
return bool(url)
def test_metadata_no_security_credentials(self):
base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION)
hp.register_uri(hp.GET, base_url, status=200,
body="\n".join(['instance-id',
'iam/']))
hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'),
status=200, body='i-0123451689abcdef0')
hp.register_uri(hp.GET,
uh.combine_url(base_url, 'iam/'),
status=200,
body="\n".join(['info/', 'security-credentials/']))
hp.register_uri(hp.GET,
uh.combine_url(base_url, 'iam/info/'),
status=200,
body='LastUpdated')
hp.register_uri(hp.GET,
uh.combine_url(base_url, 'iam/info/LastUpdated'),
status=200, body='2016-10-27T17:29:39Z')
hp.register_uri(hp.GET,
uh.combine_url(base_url, 'iam/security-credentials/'),
status=200,
body='ReadOnly/')
hp.register_uri(hp.GET,
uh.combine_url(base_url,
'iam/security-credentials/ReadOnly/'),
status=200,
body="\n".join(['LastUpdated', 'Expiration']))
hp.register_uri(hp.GET,
uh.combine_url(
base_url,
'iam/security-credentials/ReadOnly/LastUpdated'),
status=200, body='2016-10-27T17:28:17Z')
hp.register_uri(hp.GET,
uh.combine_url(
base_url,
'iam/security-credentials/ReadOnly/Expiration'),
status=200, body='2016-10-28T00:00:34Z')
md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
self.assertEqual(md['instance-id'], 'i-0123451689abcdef0')
iam = md['iam']
self.assertEqual(1, len(iam))
self.assertEqual(iam['info']['LastUpdated'], '2016-10-27T17:29:39Z')
self.assertNotIn('security-credentials', iam)
def _get_instance_metadata(tree, api_version='latest',
metadata_address='http://169.254.169.254',
ssl_details=None, timeout=5, retries=5,
leaf_decoder=None):
md_url = url_helper.combine_url(metadata_address, api_version, tree)
caller = functools.partial(
url_helper.read_file_or_url, ssl_details=ssl_details,
timeout=timeout, retries=retries)
def mcaller(url):
return caller(url).contents
try:
response = caller(md_url)
materializer = MetadataMaterializer(response.contents,
md_url, mcaller,
leaf_decoder=leaf_decoder)
md = materializer.materialize()
if not isinstance(md, (dict)):
md = {}
return md
except Exception:
util.logexc(LOG, "Failed fetching %s from url %s", tree, md_url)
return {}
def _path_join(self, base, *add_ons):
return url_helper.combine_url(base, *add_ons)