def get_instance_metadata(api_version='latest', metadata_address='http://169.254.169.254', ssl_details=None, timeout=5, retries=5, leaf_decoder=None): md_url = url_helper.combine_url(metadata_address, api_version) # Note, 'meta-data' explicitly has trailing /. # this is required for CloudStack (LP: #1356855) md_url = url_helper.combine_url(md_url, 'meta-data/') caller = functools.partial(util.read_file_or_url, ssl_details=ssl_details, timeout=timeout, retries=retries) def mcaller(url): return caller(url).contents try: response = caller(md_url) materializer = MetadataMaterializer(response.contents, md_url, mcaller, leaf_decoder=leaf_decoder) md = materializer.materialize() if not isinstance(md, (dict)): md = {} return md except Exception: util.logexc(LOG, "Failed fetching metadata from url %s", md_url) return {}
def test_metadata_fetch_no_keys(self): base_url = "http://169.254.169.254/%s/meta-data/" % (self.VERSION) hp.register_uri( hp.GET, base_url, status=200, body="\n".join(["hostname", "instance-id", "ami-launch-index"]), ) hp.register_uri( hp.GET, uh.combine_url(base_url, "hostname"), status=200, body="ec2.fake.host.name.com", ) hp.register_uri( hp.GET, uh.combine_url(base_url, "instance-id"), status=200, body="123", ) hp.register_uri( hp.GET, uh.combine_url(base_url, "ami-launch-index"), status=200, body="1", ) md = ec2.get_instance_metadata(self.VERSION, retries=0) self.assertEqual(md["hostname"], "ec2.fake.host.name.com") self.assertEqual(md["instance-id"], "123") self.assertEqual(md["ami-launch-index"], "1")
def test_metadata_fetch_with_2_keys(self): base_url = "http://169.254.169.254/%s/meta-data" % (self.VERSION) hp.register_uri(hp.GET, base_url, status=200, body="\n".join(["hostname", "instance-id", "public-keys/"])) hp.register_uri(hp.GET, uh.combine_url(base_url, "hostname"), status=200, body="ec2.fake.host.name.com") hp.register_uri(hp.GET, uh.combine_url(base_url, "instance-id"), status=200, body="123") hp.register_uri( hp.GET, uh.combine_url(base_url, "public-keys/"), status=200, body="\n".join(["0=my-public-key", "1=my-other-key"]), ) hp.register_uri( hp.GET, uh.combine_url(base_url, "public-keys/0/openssh-key"), status=200, body="ssh-rsa AAAA.....wZEf my-public-key", ) hp.register_uri( hp.GET, uh.combine_url(base_url, "public-keys/1/openssh-key"), status=200, body="ssh-rsa AAAA.....wZEf my-other-key", ) md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) self.assertEquals(md["hostname"], "ec2.fake.host.name.com") self.assertEquals(md["instance-id"], "123") self.assertEquals(2, len(md["public-keys"]))
def test_metadata_fetch_bdm(self): base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION) hp.register_uri(hp.GET, base_url, status=200, body="\n".join(['hostname', 'instance-id', 'block-device-mapping/'])) hp.register_uri(hp.GET, uh.combine_url(base_url, 'hostname'), status=200, body='ec2.fake.host.name.com') hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'), status=200, body='123') hp.register_uri(hp.GET, uh.combine_url(base_url, 'block-device-mapping/'), status=200, body="\n".join(['ami', 'ephemeral0'])) hp.register_uri(hp.GET, uh.combine_url(base_url, 'block-device-mapping/ami'), status=200, body="sdb") hp.register_uri(hp.GET, uh.combine_url(base_url, 'block-device-mapping/ephemeral0'), status=200, body="sdc") md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) self.assertEqual(md['hostname'], 'ec2.fake.host.name.com') self.assertEqual(md['instance-id'], '123') bdm = md['block-device-mapping'] self.assertEqual(2, len(bdm)) self.assertEqual(bdm['ami'], 'sdb') self.assertEqual(bdm['ephemeral0'], 'sdc')
def test_metadata_fetch_no_keys(self): base_url = "http://169.254.169.254/%s/meta-data" % (self.VERSION) hp.register_uri(hp.GET, base_url, status=200, body="\n".join(["hostname", "instance-id", "ami-launch-index"])) hp.register_uri(hp.GET, uh.combine_url(base_url, "hostname"), status=200, body="ec2.fake.host.name.com") hp.register_uri(hp.GET, uh.combine_url(base_url, "instance-id"), status=200, body="123") hp.register_uri(hp.GET, uh.combine_url(base_url, "ami-launch-index"), status=200, body="1") md = eu.get_instance_metadata(self.VERSION, retries=0) self.assertEquals(md["hostname"], "ec2.fake.host.name.com") self.assertEquals(md["instance-id"], "123") self.assertEquals(md["ami-launch-index"], "1")
def test_metadata_fetch_no_keys(self): base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION) hp.register_uri(hp.GET, base_url, status=200, body="\n".join(['hostname', 'instance-id', 'ami-launch-index'])) hp.register_uri(hp.GET, uh.combine_url(base_url, 'hostname'), status=200, body='ec2.fake.host.name.com') hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'), status=200, body='123') hp.register_uri(hp.GET, uh.combine_url(base_url, 'ami-launch-index'), status=200, body='1') md = eu.get_instance_metadata(self.VERSION, retries=0) self.assertEqual(md['hostname'], 'ec2.fake.host.name.com') self.assertEqual(md['instance-id'], '123') self.assertEqual(md['ami-launch-index'], '1')
def wait_for_metadata_service(self): urls = self.ds_cfg.get("metadata_urls", [DEF_MD_URL]) filtered = [x for x in urls if util.is_resolvable_url(x)] if set(filtered) != set(urls): LOG.debug("Removed the following from metadata urls: %s", list((set(urls) - set(filtered)))) if len(filtered): urls = filtered else: LOG.warn("Empty metadata url list! using default list") urls = [DEF_MD_URL] md_urls = [] url2base = {} for url in urls: md_url = url_helper.combine_url(url, 'openstack') md_urls.append(md_url) url2base[md_url] = url (max_wait, timeout) = self._get_url_settings() start_time = time.time() avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait, timeout=timeout) if avail_url: LOG.debug("Using metadata source: '%s'", url2base[avail_url]) else: LOG.debug("Giving up on OpenStack md from %s after %s seconds", md_urls, int(time.time() - start_time)) self.metadata_address = url2base.get(avail_url) return bool(avail_url)
def test_metadata_fetch_bdm(self): base_url = "http://169.254.169.254/%s/meta-data" % (self.VERSION) hp.register_uri( hp.GET, base_url, status=200, body="\n".join(["hostname", "instance-id", "block-device-mapping/"]) ) hp.register_uri(hp.GET, uh.combine_url(base_url, "hostname"), status=200, body="ec2.fake.host.name.com") hp.register_uri(hp.GET, uh.combine_url(base_url, "instance-id"), status=200, body="123") hp.register_uri( hp.GET, uh.combine_url(base_url, "block-device-mapping/"), status=200, body="\n".join(["ami", "ephemeral0"]) ) hp.register_uri(hp.GET, uh.combine_url(base_url, "block-device-mapping/ami"), status=200, body="sdb") hp.register_uri(hp.GET, uh.combine_url(base_url, "block-device-mapping/ephemeral0"), status=200, body="sdc") md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) self.assertEquals(md["hostname"], "ec2.fake.host.name.com") self.assertEquals(md["instance-id"], "123") bdm = md["block-device-mapping"] self.assertEquals(2, len(bdm)) self.assertEquals(bdm["ami"], "sdb") self.assertEquals(bdm["ephemeral0"], "sdc")
def get_instance_metadata(api_version='latest', metadata_address='http://169.254.169.254', ssl_details=None, timeout=5, retries=5): md_url = url_helper.combine_url(metadata_address, api_version) md_url = url_helper.combine_url(md_url, 'meta-data') caller = functools.partial(util.read_file_or_url, ssl_details=ssl_details, timeout=timeout, retries=retries) try: response = caller(md_url) materializer = MetadataMaterializer(str(response), md_url, caller) md = materializer.materialize() if not isinstance(md, (dict)): md = {} return md except Exception: util.logexc(LOG, "Failed fetching metadata from url %s", md_url) return {}
def test_metadata_fetch_key(self): base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION) hp.register_uri(hp.GET, base_url, status=200, body="\n".join(['hostname', 'instance-id', 'public-keys/'])) hp.register_uri(hp.GET, uh.combine_url(base_url, 'hostname'), status=200, body='ec2.fake.host.name.com') hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'), status=200, body='123') hp.register_uri(hp.GET, uh.combine_url(base_url, 'public-keys/'), status=200, body='0=my-public-key') hp.register_uri(hp.GET, uh.combine_url(base_url, 'public-keys/0/openssh-key'), status=200, body='ssh-rsa AAAA.....wZEf my-public-key') md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) self.assertEqual(md['hostname'], 'ec2.fake.host.name.com') self.assertEqual(md['instance-id'], '123') self.assertEqual(1, len(md['public-keys']))
def get_instance_userdata(api_version='latest', metadata_address='http://169.254.169.254', ssl_details=None, timeout=5, retries=5): ud_url = url_helper.combine_url(metadata_address, api_version) ud_url = url_helper.combine_url(ud_url, 'user-data') user_data = '' try: # It is ok for userdata to not exist (thats why we are stopping if # NOT_FOUND occurs) and just in that case returning an empty string. exception_cb = functools.partial(_skip_retry_on_codes, SKIP_USERDATA_CODES) response = url_helper.read_file_or_url( ud_url, ssl_details=ssl_details, timeout=timeout, retries=retries, exception_cb=exception_cb) user_data = response.contents except url_helper.UrlError as e: if e.code not in SKIP_USERDATA_CODES: util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) except Exception: util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) return user_data
def _materialize(self, blob, base_url): (leaves, children) = self._parse(blob) child_contents = {} for c in children: child_url = url_helper.combine_url(base_url, c) if not child_url.endswith("/"): child_url += "/" child_blob = str(self._caller(child_url)) child_contents[c] = self._materialize(child_blob, child_url) leaf_contents = {} for (field, resource) in leaves.items(): leaf_url = url_helper.combine_url(base_url, resource) leaf_blob = str(self._caller(leaf_url)) leaf_contents[field] = self._decode_leaf_blob(field, leaf_blob) joined = {} joined.update(child_contents) for field in leaf_contents.keys(): if field in joined: LOG.warn("Duplicate key found in results from %s", base_url) else: joined[field] = leaf_contents[field] return joined
def wait_for_metadata_service(self): (max_wait, timeout) = self._get_url_settings() urls = [uhelp.combine_url(self.metadata_address, 'latest/meta-data/instance-id')] start_time = time.time() url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, timeout=timeout, status_cb=LOG.warn) if url: LOG.debug("Using metadata source: '%s'", url) else: LOG.critical(("Giving up on waiting for the metadata from %s" " after %s seconds"), urls, int(time.time() - start_time)) return bool(url)
def test_metadata_no_security_credentials(self): base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION) hp.register_uri(hp.GET, base_url, status=200, body="\n".join(['instance-id', 'iam/'])) hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'), status=200, body='i-0123451689abcdef0') hp.register_uri(hp.GET, uh.combine_url(base_url, 'iam/'), status=200, body="\n".join(['info/', 'security-credentials/'])) hp.register_uri(hp.GET, uh.combine_url(base_url, 'iam/info/'), status=200, body='LastUpdated') hp.register_uri(hp.GET, uh.combine_url(base_url, 'iam/info/LastUpdated'), status=200, body='2016-10-27T17:29:39Z') hp.register_uri(hp.GET, uh.combine_url(base_url, 'iam/security-credentials/'), status=200, body='ReadOnly/') hp.register_uri(hp.GET, uh.combine_url(base_url, 'iam/security-credentials/ReadOnly/'), status=200, body="\n".join(['LastUpdated', 'Expiration'])) hp.register_uri(hp.GET, uh.combine_url( base_url, 'iam/security-credentials/ReadOnly/LastUpdated'), status=200, body='2016-10-27T17:28:17Z') hp.register_uri(hp.GET, uh.combine_url( base_url, 'iam/security-credentials/ReadOnly/Expiration'), status=200, body='2016-10-28T00:00:34Z') md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) self.assertEqual(md['instance-id'], 'i-0123451689abcdef0') iam = md['iam'] self.assertEqual(1, len(iam)) self.assertEqual(iam['info']['LastUpdated'], '2016-10-27T17:29:39Z') self.assertNotIn('security-credentials', iam)
def _get_instance_metadata(tree, api_version='latest', metadata_address='http://169.254.169.254', ssl_details=None, timeout=5, retries=5, leaf_decoder=None): md_url = url_helper.combine_url(metadata_address, api_version, tree) caller = functools.partial( url_helper.read_file_or_url, ssl_details=ssl_details, timeout=timeout, retries=retries) def mcaller(url): return caller(url).contents try: response = caller(md_url) materializer = MetadataMaterializer(response.contents, md_url, mcaller, leaf_decoder=leaf_decoder) md = materializer.materialize() if not isinstance(md, (dict)): md = {} return md except Exception: util.logexc(LOG, "Failed fetching %s from url %s", tree, md_url) return {}
def _path_join(self, base, *add_ons): return url_helper.combine_url(base, *add_ons)