def wait_for_metadata_service(self, url):
mcfg = self.ds_cfg
max_wait = 120
try:
max_wait = int(mcfg.get("max_wait", max_wait))
except Exception:
util.logexc(log)
log.warn("Failed to get max wait. using %s" % max_wait)
if max_wait == 0:
return False
timeout = 50
try:
timeout = int(mcfg.get("timeout", timeout))
except Exception:
util.logexc(log)
log.warn("Failed to get timeout, using %s" % timeout)
starttime = time.time()
check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION)
url = util.wait_for_url(urls=[check_url], max_wait=max_wait,
timeout=timeout, status_cb=log.warn,
headers_cb=self.md_headers)
if url:
log.debug("Using metadata source: '%s'" % url)
else:
log.critical("giving up on md after %i seconds\n" %
int(time.time() - starttime))
return (bool(url))
def get_dhclient_d(self):
# find lease files directory
supported_dirs = ["/var/lib/dhclient", "/var/lib/dhcp"]
for d in supported_dirs:
if os.path.exists(d):
log.debug("Using %s lease directory", d)
return d
def get_data(self):
found = None
md = {}
ud = ""
defaults = {"instance-id": DEFAULT_IID, "dsmode": "pass"}
if os.path.isdir(self.seeddir):
try:
(md, ud) = read_config_drive_dir(self.seeddir)
found = self.seeddir
except nonConfigDriveDir:
pass
if not found:
dev = cfg_drive_device()
if dev:
try:
(md, ud) = util.mount_callback_umount(dev,
read_config_drive_dir)
found = dev
except (nonConfigDriveDir, util.mountFailedError):
pass
if not found:
return False
if 'dsconfig' in md:
self.cfg = md['dscfg']
md = util.mergedict(md, defaults)
# update interfaces and ifup only on the local datasource
# this way the DataSourceConfigDriveNet doesn't do it also.
if 'network-interfaces' in md and self.dsmode == "local":
if md['dsmode'] == "pass":
log.info("updating network interfaces from configdrive")
else:
log.debug("updating network interfaces from configdrive")
util.write_file("/etc/network/interfaces",
md['network-interfaces'])
try:
(out, err) = util.subp(['ifup', '--all'])
if len(out) or len(err):
log.warn("ifup --all had stderr: %s" % err)
except subprocess.CalledProcessError as exc:
log.warn("ifup --all failed: %s" % (exc.output[1]))
self.seed = found
self.metadata = md
self.userdata_raw = ud
if md['dsmode'] == self.dsmode:
return True
log.debug("%s: not claiming datasource, dsmode=%s" %
(self, md['dsmode']))
return False
def get_dhclient_d(self):
# find lease files directory
supported_dirs = ["/var/lib/dhclient", "/var/lib/dhcp"]
for d in supported_dirs:
if os.path.exists(d):
log.debug("Using %s lease directory", d)
return d
def wait_for_metadata_service(self):
mcfg = self.ds_cfg
if not hasattr(mcfg, "get"):
mcfg = {}
max_wait = 120
try:
max_wait = int(mcfg.get("max_wait", max_wait))
except Exception:
util.logexc(log)
log.warn("Failed to get max wait. using %s" % max_wait)
if max_wait == 0:
return False
timeout = 50
try:
timeout = int(mcfg.get("timeout", timeout))
except Exception:
util.logexc(log)
log.warn("Failed to get timeout, using %s" % timeout)
def_mdurls = ["http://169.254.169.254", "http://instance-data.:8773"]
mdurls = mcfg.get("metadata_urls", def_mdurls)
# Remove addresses from the list that wont resolve.
filtered = [x for x in mdurls if util.is_resolvable_url(x)]
if set(filtered) != set(mdurls):
log.debug("removed the following from metadata urls: %s" % list(
(set(mdurls) - set(filtered))))
if len(filtered):
mdurls = filtered
else:
log.warn("Empty metadata url list! using default list")
mdurls = def_mdurls
urls = []
url2base = {False: False}
for url in mdurls:
cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver)
urls.append(cur)
url2base[cur] = url
starttime = time.time()
url = util.wait_for_url(urls=urls,
max_wait=max_wait,
timeout=timeout,
status_cb=log.warn)
if url:
log.debug("Using metadata source: '%s'" % url2base[url])
else:
log.critical("giving up on md after %i seconds\n" %
int(time.time() - starttime))
self.metadata_address = url2base[url]
return (bool(url))
def get_data(self):
found = []
md = {}
ud = ""
defaults = {
"instance-id": "iid-dsovf"
}
(seedfile, contents) = get_ovf_env(base_seeddir)
if seedfile:
# found a seed dir
seed = "%s/%s" % (base_seeddir, seedfile)
(md, ud, cfg) = read_ovf_environment(contents)
self.environment = contents
found.append(seed)
else:
np = {'iso': transport_iso9660,
'vmware-guestd': transport_vmware_guestd, }
name = None
for name, transfunc in np.iteritems():
(contents, _dev, _fname) = transfunc()
if contents:
break
if contents:
(md, ud, cfg) = read_ovf_environment(contents)
self.environment = contents
found.append(name)
# There was no OVF transports found
if len(found) == 0:
return False
if 'seedfrom' in md and md['seedfrom']:
seedfrom = md['seedfrom']
seedfound = False
for proto in self.supported_seed_starts:
if seedfrom.startswith(proto):
seedfound = proto
break
if not seedfound:
log.debug("seed from %s not supported by %s" %
(seedfrom, self.__class__))
return False
(md_seed, ud) = util.read_seeded(seedfrom, timeout=None)
log.debug("using seeded cache data from %s" % seedfrom)
md = util.mergedict(md, md_seed)
found.append(seedfrom)
md = util.mergedict(md, defaults)
self.seed = ",".join(found)
self.metadata = md
self.userdata_raw = ud
self.cfg = cfg
return True
def get_data(self):
found = []
md = {}
ud = ""
defaults = {"instance-id": "iid-dsovf"}
(seedfile, contents) = get_ovf_env(base_seeddir)
if seedfile:
# found a seed dir
seed = "%s/%s" % (base_seeddir, seedfile)
(md, ud, cfg) = read_ovf_environment(contents)
self.environment = contents
found.append(seed)
else:
np = {
'iso': transport_iso9660,
'vmware-guestd': transport_vmware_guestd,
}
name = None
for name, transfunc in np.iteritems():
(contents, _dev, _fname) = transfunc()
if contents:
break
if contents:
(md, ud, cfg) = read_ovf_environment(contents)
self.environment = contents
found.append(name)
# There was no OVF transports found
if len(found) == 0:
return False
if 'seedfrom' in md and md['seedfrom']:
seedfrom = md['seedfrom']
seedfound = False
for proto in self.supported_seed_starts:
if seedfrom.startswith(proto):
seedfound = proto
break
if not seedfound:
log.debug("seed from %s not supported by %s" %
(seedfrom, self.__class__))
return False
(md_seed, ud) = util.read_seeded(seedfrom, timeout=None)
log.debug("using seeded cache data from %s" % seedfrom)
md = util.mergedict(md, md_seed)
found.append(seedfrom)
md = util.mergedict(md, defaults)
self.seed = ",".join(found)
self.metadata = md
self.userdata_raw = ud
self.cfg = cfg
return True
def wait_for_metadata_service(self):
mcfg = self.ds_cfg
if not hasattr(mcfg, "get"):
mcfg = {}
max_wait = 120
try:
max_wait = int(mcfg.get("max_wait", max_wait))
except Exception:
util.logexc(log)
log.warn("Failed to get max wait. using %s" % max_wait)
if max_wait == 0:
return False
timeout = 50
try:
timeout = int(mcfg.get("timeout", timeout))
except Exception:
util.logexc(log)
log.warn("Failed to get timeout, using %s" % timeout)
def_mdurls = ["http://169.254.169.254", "http://instance-data.:8773"]
mdurls = mcfg.get("metadata_urls", def_mdurls)
# Remove addresses from the list that wont resolve.
filtered = [x for x in mdurls if util.is_resolvable_url(x)]
if set(filtered) != set(mdurls):
log.debug("removed the following from metadata urls: %s" %
list((set(mdurls) - set(filtered))))
if len(filtered):
mdurls = filtered
else:
log.warn("Empty metadata url list! using default list")
mdurls = def_mdurls
urls = []
url2base = {False: False}
for url in mdurls:
cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver)
urls.append(cur)
url2base[cur] = url
starttime = time.time()
url = util.wait_for_url(urls=urls, max_wait=max_wait,
timeout=timeout, status_cb=log.warn)
if url:
log.debug("Using metadata source: '%s'" % url2base[url])
else:
log.critical("giving up on md after %i seconds\n" %
int(time.time() - starttime))
self.metadata_address = url2base[url]
return (bool(url))
def get_default_gateway(self):
""" Returns the default gateway ip address in the dotted format
"""
with open("/proc/net/route", "r") as f:
for line in f.readlines():
items = line.split("\t")
if items[1] == "00000000":
# found the default route, get the gateway
gw = inet_ntoa(pack("<L", int(items[2], 16)))
log.debug("found default route, gateway is %s" % gw)
return gw
def get_default_gateway(self):
""" Returns the default gateway ip address in the dotted format
"""
with open("/proc/net/route", "r") as f:
for line in f.readlines():
items = line.split("\t")
if items[1] == "00000000":
# found the default route, get the gateway
gw = inet_ntoa(pack("<L", int(items[2], 16)))
log.debug("found default route, gateway is %s" % gw)
return gw
def device_name_to_device(self, name):
# consult metadata service, that has
# ephemeral0: sdb
# and return 'sdb' for input 'ephemeral0'
if 'block-device-mapping' not in self.metadata:
return (None)
found = None
for entname, device in self.metadata['block-device-mapping'].items():
if entname == name:
found = device
break
# LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0'
if entname == "ephemeral" and name == "ephemeral0":
found = device
if found == None:
log.debug("unable to convert %s to a device" % name)
return None
# LP: #611137
# the metadata service may believe that devices are named 'sda'
# when the kernel named them 'vda' or 'xvda'
# we want to return the correct value for what will actually
# exist in this instance
mappings = {"sd": ("vd", "xvd")}
ofound = found
short = os.path.basename(found)
if not found.startswith("/"):
found = "/dev/%s" % found
if os.path.exists(found):
return (found)
for nfrom, tlist in mappings.items():
if not short.startswith(nfrom):
continue
for nto in tlist:
cand = "/dev/%s%s" % (nto, short[len(nfrom):])
if os.path.exists(cand):
log.debug("remapped device name %s => %s" % (found, cand))
return (cand)
# on t1.micro, ephemeral0 will appear in block-device-mapping from
# metadata, but it will not exist on disk (and never will)
# at this pint, we've verified that the path did not exist
# in the special case of 'ephemeral0' return None to avoid bogus
# fstab entry (LP: #744019)
if name == "ephemeral0":
return None
return ofound
def device_name_to_device(self, name):
# consult metadata service, that has
# ephemeral0: sdb
# and return 'sdb' for input 'ephemeral0'
if "block-device-mapping" not in self.metadata:
return None
found = None
for entname, device in self.metadata["block-device-mapping"].items():
if entname == name:
found = device
break
# LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0'
if entname == "ephemeral" and name == "ephemeral0":
found = device
if found == None:
log.debug("unable to convert %s to a device" % name)
return None
# LP: #611137
# the metadata service may believe that devices are named 'sda'
# when the kernel named them 'vda' or 'xvda'
# we want to return the correct value for what will actually
# exist in this instance
mappings = {"sd": ("vd", "xvd")}
ofound = found
short = os.path.basename(found)
if not found.startswith("/"):
found = "/dev/%s" % found
if os.path.exists(found):
return found
for nfrom, tlist in mappings.items():
if not short.startswith(nfrom):
continue
for nto in tlist:
cand = "/dev/%s%s" % (nto, short[len(nfrom) :])
if os.path.exists(cand):
log.debug("remapped device name %s => %s" % (found, cand))
return cand
# on t1.micro, ephemeral0 will appear in block-device-mapping from
# metadata, but it will not exist on disk (and never will)
# at this pint, we've verified that the path did not exist
# in the special case of 'ephemeral0' return None to avoid bogus
# fstab entry (LP: #744019)
if name == "ephemeral0":
return None
return ofound
def get_data(self):
seedret = {}
if util.read_optional_seed(seedret, base=self.seeddir + "/"):
self.userdata_raw = seedret["user-data"]
self.metadata = seedret["meta-data"]
log.debug("using seeded ec2 data in %s" % self.seeddir)
return True
try:
if not self.wait_for_metadata_service():
return False
start = time.time()
self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, None, self.metadata_address)
self.metadata = boto_utils.get_instance_metadata(self.api_ver, self.metadata_address)
log.debug("crawl of metadata service took %ds" % (time.time() - start))
return True
except Exception as e:
print e
return False
def get_data(self):
seedret = {}
if util.read_optional_seed(seedret, base=self.seeddir + "/"):
self.userdata_raw = seedret['user-data']
self.metadata = seedret['meta-data']
log.debug("using seeded cs data in %s" % self.seeddir)
return True
try:
start = time.time()
self.userdata_raw = boto_utils.get_instance_userdata(
self.api_ver, None, self.metadata_address)
self.metadata = boto_utils.get_instance_metadata(
self.api_ver, self.metadata_address)
log.debug("crawl of metadata service took %ds" %
(time.time() - start))
return True
except Exception as e:
log.exception(e)
return False
def get_data(self):
seedret = {}
if util.read_optional_seed(seedret, base=self.seeddir + "/"):
self.userdata_raw = seedret['user-data']
self.metadata = seedret['meta-data']
log.debug("using seeded cs data in %s" % self.seeddir)
return True
try:
start = time.time()
self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver,
None, self.metadata_address)
self.metadata = boto_utils.get_instance_metadata(self.api_ver,
self.metadata_address)
log.debug("crawl of metadata service took %ds" %
(time.time() - start))
return True
except Exception as e:
log.exception(e)
return False
def get_vr_address(self):
# Get the address of the virtual router via dhcp leases
# see http://bit.ly/T76eKC for documentation on the virtual router.
# If no virtual router is detected, fallback on default gateway.
lease_file = self.get_latest_lease()
if not lease_file:
log.debug("No lease file found, using default gateway")
return self.get_default_gateway()
latest_address = None
with open(lease_file, "r") as fd:
for line in fd:
if "dhcp-server-identifier" in line:
words = line.strip(" ;\r\n").split(" ")
if len(words) > 2:
dhcp = words[2]
log.debug("Found DHCP identifier %s", dhcp)
latest_address = dhcp
if not latest_address:
# No virtual router found, fallback on default gateway
log.debug("No DHCP found, using default gateway")
return self.get_default_gateway()
return latest_address
def get_vr_address(self):
# Get the address of the virtual router via dhcp leases
# see http://bit.ly/T76eKC for documentation on the virtual router.
# If no virtual router is detected, fallback on default gateway.
lease_file = self.get_latest_lease()
if not lease_file:
log.debug("No lease file found, using default gateway")
return self.get_default_gateway()
latest_address = None
with open(lease_file, "r") as fd:
for line in fd:
if "dhcp-server-identifier" in line:
words = line.strip(" ;\r\n").split(" ")
if len(words) > 2:
dhcp = words[2]
log.debug("Found DHCP identifier %s", dhcp)
latest_address = dhcp
if not latest_address:
# No virtual router found, fallback on default gateway
log.debug("No DHCP found, using default gateway")
return self.get_default_gateway()
return latest_address
def get_data(self):
found = None
md = {}
ud = ""
defaults = {"instance-id": DEFAULT_IID, "dsmode": "pass"}
if os.path.isdir(self.seeddir):
try:
(md, ud) = read_config_drive_dir(self.seeddir)
found = self.seeddir
except nonConfigDriveDir:
pass
if not found:
dev = cfg_drive_device()
if dev:
try:
(md,
ud) = util.mount_callback_umount(dev,
read_config_drive_dir)
found = dev
except (nonConfigDriveDir, util.mountFailedError):
pass
if not found:
return False
if 'dsconfig' in md:
self.cfg = md['dscfg']
md = util.mergedict(md, defaults)
# update interfaces and ifup only on the local datasource
# this way the DataSourceConfigDriveNet doesn't do it also.
if 'network-interfaces' in md and self.dsmode == "local":
if md['dsmode'] == "pass":
log.info("updating network interfaces from configdrive")
else:
log.debug("updating network interfaces from configdrive")
util.write_file("/etc/network/interfaces",
md['network-interfaces'])
try:
(out, err) = util.subp(['ifup', '--all'])
if len(out) or len(err):
log.warn("ifup --all had stderr: %s" % err)
except subprocess.CalledProcessError as exc:
log.warn("ifup --all failed: %s" % (exc.output[1]))
self.seed = found
self.metadata = md
self.userdata_raw = ud
if md['dsmode'] == self.dsmode:
return True
log.debug("%s: not claiming datasource, dsmode=%s" %
(self, md['dsmode']))
return False
seeded_interfaces = None
# the special argument "seedfrom" indicates we should
# attempt to seed the userdata / metadata from its value
# its primarily value is in allowing the user to type less
# on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg
if "seedfrom" in md:
seedfrom = md["seedfrom"]
seedfound = False
for proto in self.supported_seed_starts:
if seedfrom.startswith(proto):
seedfound = proto
break
if not seedfound:
log.debug("seed from %s not supported by %s" %
(seedfrom, self.__class__))
return False
if 'network-interfaces' in md:
seeded_interfaces = self.dsmode
# this could throw errors, but the user told us to do it
# so if errors are raised, let them raise
(md_seed, ud) = util.read_seeded(seedfrom, timeout=None)
log.debug("using seeded cache data from %s" % seedfrom)
# values in the command line override those from the seed
md = util.mergedict(md, md_seed)
found.append(seedfrom)
md = util.mergedict(md, defaults)
def get_data(self):
defaults = {
"instance-id": "nocloud", "dsmode": self.dsmode
}
found = []
md = {}
ud = ""
try:
# parse the kernel command line, getting data passed in
if parse_cmdline_data(self.cmdline_id, md):
found.append("cmdline")
except:
util.logexc(log)
return False
# check to see if the seeddir has data.
seedret = {}
if util.read_optional_seed(seedret, base=self.seeddir + "/"):
md = util.mergedict(md, seedret['meta-data'])
ud = seedret['user-data']
found.append(self.seeddir)
log.debug("using seeded cache data in %s" % self.seeddir)
# if the datasource config had a 'seedfrom' entry, then that takes
# precedence over a 'seedfrom' that was found in a filesystem
# but not over external medi
if 'seedfrom' in self.ds_cfg and self.ds_cfg['seedfrom']:
found.append("ds_config")
md["seedfrom"] = self.ds_cfg['seedfrom']
fslist = util.find_devs_with("TYPE=vfat")
fslist.extend(util.find_devs_with("TYPE=iso9660"))
label_list = util.find_devs_with("LABEL=cidata")
devlist = list(set(fslist) & set(label_list))
devlist.sort(reverse=True)
for dev in devlist:
try:
(newmd, newud) = util.mount_callback_umount(dev,
util.read_seeded)
md = util.mergedict(newmd, md)
ud = newud
# for seed from a device, the default mode is 'net'.
# that is more likely to be what is desired.
# If they want dsmode of local, then they must
# specify that.
if 'dsmode' not in md:
md['dsmode'] = "net"
log.debug("using data from %s" % dev)
found.append(dev)
break
except OSError, e:
if e.errno != errno.ENOENT:
raise
except util.mountFailedError:
log.warn("Failed to mount %s when looking for seed" % dev)