def test_no_required_and_optional(self): dirdata = {'f1': 'f1c', 'f2': 'f2c'} populate_dir(self.tmp, dirdata) ret = util.pathprefix2dict(self.tmp, required=None, optional=['f1', 'f2']) self.assertEqual(dirdata, ret)
def _quick_read_instance_id(cmdline_id, dirs=None): if dirs is None: dirs = [] iid_key = 'instance-id' if cmdline_id is None: fill = {} if parse_cmdline_data(cmdline_id, fill) and iid_key in fill: return fill[iid_key] for d in dirs: try: data = util.pathprefix2dict(d, required=['meta-data']) md = util.load_yaml(data['meta-data']) if iid_key in md: return md[iid_key] except ValueError: pass return None
def _quick_read_instance_id(dirs=None): if dirs is None: dirs = [] iid_key = 'instance-id' fill = {} if load_cmdline_data(fill) and iid_key in fill: return fill[iid_key] for d in dirs: if d is None: continue try: data = util.pathprefix2dict(d, required=['meta-data']) md = util.load_yaml(data['meta-data']) if iid_key in md: return md[iid_key] except ValueError: pass return None
def test_required_only(self): dirdata = {'f1': 'f1content', 'f2': 'f2content'} populate_dir(self.tmp, dirdata) ret = util.pathprefix2dict(self.tmp, required=['f1', 'f2']) self.assertEqual(dirdata, ret)
def test_required_and_optional(self): dirdata = {'f1': b'f1c', 'f2': b'f2c'} populate_dir(self.tmp, dirdata) ret = util.pathprefix2dict(self.tmp, required=['f1'], optional=['f2']) self.assertEqual(dirdata, ret)
def test_required_only(self): dirdata = {'f1': b'f1content', 'f2': b'f2content'} populate_dir(self.tmp, dirdata) ret = util.pathprefix2dict(self.tmp, required=['f1', 'f2']) self.assertEqual(dirdata, ret)
def _pp2d_callback(mp, data): return util.pathprefix2dict(mp, **data)
def get_data(self): defaults = { "instance-id": "nocloud", "dsmode": self.dsmode, } found = [] mydata = {'meta-data': {}, 'user-data': "", 'vendor-data': ""} try: # Parse the kernel command line, getting data passed in md = {} if parse_cmdline_data(self.cmdline_id, md): found.append("cmdline") mydata['meta-data'].update(md) except: util.logexc(LOG, "Unable to parse command line data") return False # Check to see if the seed dir has data. pp2d_kwargs = {'required': ['user-data', 'meta-data'], 'optional': ['vendor-data']} try: seeded = util.pathprefix2dict(self.seed_dir, **pp2d_kwargs) found.append(self.seed_dir) LOG.debug("Using seeded data from %s", self.seed_dir) except ValueError as e: pass if self.seed_dir in found: mydata = _merge_new_seed(mydata, seeded) # If the datasource config had a 'seedfrom' entry, then that takes # precedence over a 'seedfrom' that was found in a filesystem # but not over external media if self.ds_cfg.get('seedfrom'): found.append("ds_config_seedfrom") mydata['meta-data']["seedfrom"] = self.ds_cfg['seedfrom'] # fields appropriately named can also just come from the datasource # config (ie, 'user-data', 'meta-data', 'vendor-data' there) if 'user-data' in self.ds_cfg and 'meta-data' in self.ds_cfg: mydata = _merge_new_seed(mydata, self.ds_cfg) found.append("ds_config") def _pp2d_callback(mp, data): return util.pathprefix2dict(mp, **data) label = self.ds_cfg.get('fs_label', "cidata") if label is not None: # Query optical drive to get it in blkid cache for 2.6 kernels util.find_devs_with(path="/dev/sr0") util.find_devs_with(path="/dev/sr1") fslist = util.find_devs_with("TYPE=vfat") fslist.extend(util.find_devs_with("TYPE=iso9660")) label_list = util.find_devs_with("LABEL=%s" % label) devlist = list(set(fslist) & set(label_list)) devlist.sort(reverse=True) for dev in devlist: try: LOG.debug("Attempting to use data from %s", dev) try: seeded = util.mount_cb(dev, _pp2d_callback, pp2d_kwargs) except ValueError as e: if dev in label_list: LOG.warn("device %s with label=%s not a" "valid seed.", dev, label) continue mydata = _merge_new_seed(mydata, seeded) # For seed from a device, the default mode is 'net'. # that is more likely to be what is desired. If they want # dsmode of local, then they must specify that. if 'dsmode' not in mydata['meta-data']: mydata['dsmode'] = "net" LOG.debug("Using data from %s", dev) found.append(dev) break except OSError as e: if e.errno != errno.ENOENT: raise except util.MountFailedError: util.logexc(LOG, "Failed to mount %s when looking for " "data", dev) # There was no indication on kernel cmdline or data # in the seeddir suggesting this handler should be used. if len(found) == 0: return False seeded_interfaces = None # The special argument "seedfrom" indicates we should # attempt to seed the userdata / metadata from its value # its primarily value is in allowing the user to type less # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg if "seedfrom" in mydata['meta-data']: seedfrom = mydata['meta-data']["seedfrom"] seedfound = False for proto in self.supported_seed_starts: if seedfrom.startswith(proto): seedfound = proto break if not seedfound: LOG.debug("Seed from %s not supported by %s", seedfrom, self) return False if 'network-interfaces' in mydata['meta-data']: seeded_interfaces = self.dsmode # This could throw errors, but the user told us to do it # so if errors are raised, let them raise (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) LOG.debug("Using seeded cache data from %s", seedfrom) # Values in the command line override those from the seed mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], md_seed]) mydata['user-data'] = ud found.append(seedfrom) # Now that we have exhausted any other places merge in the defaults mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], defaults]) # Update the network-interfaces if metadata had 'network-interfaces' # entry and this is the local datasource, or 'seedfrom' was used # and the source of the seed was self.dsmode # ('local' for NoCloud, 'net' for NoCloudNet') if ('network-interfaces' in mydata['meta-data'] and (self.dsmode in ("local", seeded_interfaces))): LOG.debug("Updating network interfaces from %s", self) self.distro.apply_network( mydata['meta-data']['network-interfaces']) if mydata['meta-data']['dsmode'] == self.dsmode: self.seed = ",".join(found) self.metadata = mydata['meta-data'] self.userdata_raw = mydata['user-data'] self.vendordata = mydata['vendor-data'] return True LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode']) return False
def _get_data(self): defaults = { "instance-id": "nocloud", "dsmode": self.dsmode, } found = [] mydata = {'meta-data': {}, 'user-data': "", 'vendor-data': "", 'network-config': None} try: # Parse the system serial label from dmi. If not empty, try parsing # like the commandline md = {} serial = util.read_dmi_data('system-serial-number') if serial and load_cmdline_data(md, serial): found.append("dmi") mydata = _merge_new_seed(mydata, {'meta-data': md}) except Exception: util.logexc(LOG, "Unable to parse dmi data") return False try: # Parse the kernel command line, getting data passed in md = {} if load_cmdline_data(md): found.append("cmdline") mydata = _merge_new_seed(mydata, {'meta-data': md}) except Exception: util.logexc(LOG, "Unable to parse command line data") return False # Check to see if the seed dir has data. pp2d_kwargs = {'required': ['user-data', 'meta-data'], 'optional': ['vendor-data', 'network-config']} for path in self.seed_dirs: try: seeded = util.pathprefix2dict(path, **pp2d_kwargs) found.append(path) LOG.debug("Using seeded data from %s", path) mydata = _merge_new_seed(mydata, seeded) break except ValueError: pass # If the datasource config had a 'seedfrom' entry, then that takes # precedence over a 'seedfrom' that was found in a filesystem # but not over external media if self.ds_cfg.get('seedfrom'): found.append("ds_config_seedfrom") mydata['meta-data']["seedfrom"] = self.ds_cfg['seedfrom'] # fields appropriately named can also just come from the datasource # config (ie, 'user-data', 'meta-data', 'vendor-data' there) if 'user-data' in self.ds_cfg and 'meta-data' in self.ds_cfg: mydata = _merge_new_seed(mydata, self.ds_cfg) found.append("ds_config") def _pp2d_callback(mp, data): return util.pathprefix2dict(mp, **data) label = self.ds_cfg.get('fs_label', "cidata") if label is not None: for dev in self._get_devices(label): try: LOG.debug("Attempting to use data from %s", dev) try: seeded = util.mount_cb(dev, _pp2d_callback, pp2d_kwargs) except ValueError: LOG.warning("device %s with label=%s not a" "valid seed.", dev, label) continue mydata = _merge_new_seed(mydata, seeded) LOG.debug("Using data from %s", dev) found.append(dev) break except OSError as e: if e.errno != errno.ENOENT: raise except util.MountFailedError: util.logexc(LOG, "Failed to mount %s when looking for " "data", dev) # There was no indication on kernel cmdline or data # in the seeddir suggesting this handler should be used. if len(found) == 0: return False # The special argument "seedfrom" indicates we should # attempt to seed the userdata / metadata from its value # its primarily value is in allowing the user to type less # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg if "seedfrom" in mydata['meta-data']: seedfrom = mydata['meta-data']["seedfrom"] seedfound = False for proto in self.supported_seed_starts: if seedfrom.startswith(proto): seedfound = proto break if not seedfound: LOG.debug("Seed from %s not supported by %s", seedfrom, self) return False # This could throw errors, but the user told us to do it # so if errors are raised, let them raise (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) LOG.debug("Using seeded cache data from %s", seedfrom) # Values in the command line override those from the seed mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], md_seed]) mydata['user-data'] = ud found.append(seedfrom) # Now that we have exhausted any other places merge in the defaults mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], defaults]) self.dsmode = self._determine_dsmode( [mydata['meta-data'].get('dsmode')]) if self.dsmode == sources.DSMODE_DISABLED: LOG.debug("%s: not claiming datasource, dsmode=%s", self, self.dsmode) return False self.seed = ",".join(found) self.metadata = mydata['meta-data'] self.userdata_raw = mydata['user-data'] self.vendordata_raw = mydata['vendor-data'] self._network_config = mydata['network-config'] self._network_eni = mydata['meta-data'].get('network-interfaces') return True
def get_data(self): defaults = { "instance-id": "nocloud", "dsmode": self.dsmode, } found = [] mydata = {'meta-data': {}, 'user-data': "", 'vendor-data': ""} try: # Parse the kernel command line, getting data passed in md = {} if parse_cmdline_data(self.cmdline_id, md): found.append("cmdline") mydata['meta-data'].update(md) except: util.logexc(LOG, "Unable to parse command line data") return False # Check to see if the seed dir has data. pp2d_kwargs = { 'required': ['user-data', 'meta-data'], 'optional': ['vendor-data'] } try: seeded = util.pathprefix2dict(self.seed_dir, **pp2d_kwargs) found.append(self.seed_dir) LOG.debug("Using seeded data from %s", self.seed_dir) except ValueError as e: pass if self.seed_dir in found: mydata = _merge_new_seed(mydata, seeded) # If the datasource config had a 'seedfrom' entry, then that takes # precedence over a 'seedfrom' that was found in a filesystem # but not over external media if self.ds_cfg.get('seedfrom'): found.append("ds_config_seedfrom") mydata['meta-data']["seedfrom"] = self.ds_cfg['seedfrom'] # fields appropriately named can also just come from the datasource # config (ie, 'user-data', 'meta-data', 'vendor-data' there) if 'user-data' in self.ds_cfg and 'meta-data' in self.ds_cfg: mydata = _merge_new_seed(mydata, self.ds_cfg) found.append("ds_config") def _pp2d_callback(mp, data): return util.pathprefix2dict(mp, **data) label = self.ds_cfg.get('fs_label', "cidata") if label is not None: # Query optical drive to get it in blkid cache for 2.6 kernels util.find_devs_with(path="/dev/sr0") util.find_devs_with(path="/dev/sr1") fslist = util.find_devs_with("TYPE=vfat") fslist.extend(util.find_devs_with("TYPE=iso9660")) label_list = util.find_devs_with("LABEL=%s" % label) devlist = list(set(fslist) & set(label_list)) devlist.sort(reverse=True) for dev in devlist: try: LOG.debug("Attempting to use data from %s", dev) try: seeded = util.mount_cb(dev, _pp2d_callback, pp2d_kwargs) except ValueError as e: if dev in label_list: LOG.warn( "device %s with label=%s not a" "valid seed.", dev, label) continue mydata = _merge_new_seed(mydata, seeded) # For seed from a device, the default mode is 'net'. # that is more likely to be what is desired. If they want # dsmode of local, then they must specify that. if 'dsmode' not in mydata['meta-data']: mydata['dsmode'] = "net" LOG.debug("Using data from %s", dev) found.append(dev) break except OSError as e: if e.errno != errno.ENOENT: raise except util.MountFailedError: util.logexc(LOG, "Failed to mount %s when looking for " "data", dev) # There was no indication on kernel cmdline or data # in the seeddir suggesting this handler should be used. if len(found) == 0: return False seeded_interfaces = None # The special argument "seedfrom" indicates we should # attempt to seed the userdata / metadata from its value # its primarily value is in allowing the user to type less # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg if "seedfrom" in mydata['meta-data']: seedfrom = mydata['meta-data']["seedfrom"] seedfound = False for proto in self.supported_seed_starts: if seedfrom.startswith(proto): seedfound = proto break if not seedfound: LOG.debug("Seed from %s not supported by %s", seedfrom, self) return False if 'network-interfaces' in mydata['meta-data']: seeded_interfaces = self.dsmode # This could throw errors, but the user told us to do it # so if errors are raised, let them raise (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) LOG.debug("Using seeded cache data from %s", seedfrom) # Values in the command line override those from the seed mydata['meta-data'] = util.mergemanydict( [mydata['meta-data'], md_seed]) mydata['user-data'] = ud found.append(seedfrom) # Now that we have exhausted any other places merge in the defaults mydata['meta-data'] = util.mergemanydict( [mydata['meta-data'], defaults]) # Update the network-interfaces if metadata had 'network-interfaces' # entry and this is the local datasource, or 'seedfrom' was used # and the source of the seed was self.dsmode # ('local' for NoCloud, 'net' for NoCloudNet') if ('network-interfaces' in mydata['meta-data'] and (self.dsmode in ("local", seeded_interfaces))): LOG.debug("Updating network interfaces from %s", self) self.distro.apply_network( mydata['meta-data']['network-interfaces']) if mydata['meta-data']['dsmode'] == self.dsmode: self.seed = ",".join(found) self.metadata = mydata['meta-data'] self.userdata_raw = mydata['user-data'] self.vendordata_raw = mydata['vendor-data'] return True LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode']) return False
def _get_data(self): defaults = { "instance-id": "nocloud", "dsmode": self.dsmode, } found = [] mydata = {'meta-data': {}, 'user-data': "", 'vendor-data': "", 'network-config': None} try: # Parse the system serial label from dmi. If not empty, try parsing # like the commandline md = {} serial = util.read_dmi_data('system-serial-number') if serial and load_cmdline_data(md, serial): found.append("dmi") mydata = _merge_new_seed(mydata, {'meta-data': md}) except Exception: util.logexc(LOG, "Unable to parse dmi data") return False try: # Parse the kernel command line, getting data passed in md = {} if load_cmdline_data(md): found.append("cmdline") mydata = _merge_new_seed(mydata, {'meta-data': md}) except Exception: util.logexc(LOG, "Unable to parse command line data") return False # Check to see if the seed dir has data. pp2d_kwargs = {'required': ['user-data', 'meta-data'], 'optional': ['vendor-data', 'network-config']} for path in self.seed_dirs: try: seeded = util.pathprefix2dict(path, **pp2d_kwargs) found.append(path) LOG.debug("Using seeded data from %s", path) mydata = _merge_new_seed(mydata, seeded) break except ValueError: pass # If the datasource config had a 'seedfrom' entry, then that takes # precedence over a 'seedfrom' that was found in a filesystem # but not over external media if self.ds_cfg.get('seedfrom'): found.append("ds_config_seedfrom") mydata['meta-data']["seedfrom"] = self.ds_cfg['seedfrom'] # fields appropriately named can also just come from the datasource # config (ie, 'user-data', 'meta-data', 'vendor-data' there) if 'user-data' in self.ds_cfg and 'meta-data' in self.ds_cfg: mydata = _merge_new_seed(mydata, self.ds_cfg) found.append("ds_config") def _pp2d_callback(mp, data): return util.pathprefix2dict(mp, **data) label = self.ds_cfg.get('fs_label', "cidata") if label is not None: # Query optical drive to get it in blkid cache for 2.6 kernels util.find_devs_with(path="/dev/sr0") util.find_devs_with(path="/dev/sr1") fslist = util.find_devs_with("TYPE=vfat") fslist.extend(util.find_devs_with("TYPE=iso9660")) label_list = util.find_devs_with("LABEL=%s" % label.upper()) label_list.extend(util.find_devs_with("LABEL=%s" % label.lower())) devlist = list(set(fslist) & set(label_list)) devlist.sort(reverse=True) for dev in devlist: try: LOG.debug("Attempting to use data from %s", dev) try: seeded = util.mount_cb(dev, _pp2d_callback, pp2d_kwargs) except ValueError: if dev in label_list: LOG.warning("device %s with label=%s not a" "valid seed.", dev, label) continue mydata = _merge_new_seed(mydata, seeded) LOG.debug("Using data from %s", dev) found.append(dev) break except OSError as e: if e.errno != errno.ENOENT: raise except util.MountFailedError: util.logexc(LOG, "Failed to mount %s when looking for " "data", dev) # There was no indication on kernel cmdline or data # in the seeddir suggesting this handler should be used. if len(found) == 0: return False # The special argument "seedfrom" indicates we should # attempt to seed the userdata / metadata from its value # its primarily value is in allowing the user to type less # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg if "seedfrom" in mydata['meta-data']: seedfrom = mydata['meta-data']["seedfrom"] seedfound = False for proto in self.supported_seed_starts: if seedfrom.startswith(proto): seedfound = proto break if not seedfound: LOG.debug("Seed from %s not supported by %s", seedfrom, self) return False # This could throw errors, but the user told us to do it # so if errors are raised, let them raise (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) LOG.debug("Using seeded cache data from %s", seedfrom) # Values in the command line override those from the seed mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], md_seed]) mydata['user-data'] = ud found.append(seedfrom) # Now that we have exhausted any other places merge in the defaults mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], defaults]) self.dsmode = self._determine_dsmode( [mydata['meta-data'].get('dsmode')]) if self.dsmode == sources.DSMODE_DISABLED: LOG.debug("%s: not claiming datasource, dsmode=%s", self, self.dsmode) return False self.seed = ",".join(found) self.metadata = mydata['meta-data'] self.userdata_raw = mydata['user-data'] self.vendordata_raw = mydata['vendor-data'] self._network_config = mydata['network-config'] self._network_eni = mydata['meta-data'].get('network-interfaces') return True
def test_required_and_optional(self): dirdata = {"f1": b"f1c", "f2": b"f2c"} populate_dir(self.tmp, dirdata) ret = util.pathprefix2dict(self.tmp, required=["f1"], optional=["f2"]) self.assertEqual(dirdata, ret)
def test_required_only(self): dirdata = {"f1": b"f1content", "f2": b"f2content"} populate_dir(self.tmp, dirdata) ret = util.pathprefix2dict(self.tmp, required=["f1", "f2"]) self.assertEqual(dirdata, ret)