def _load_keypair(self, keypair=None): key_location = None if keypair: kp = self.ec2.get_keypair(keypair) key = self.cfg.get_key(kp.name) key_location = key.get('key_location', '') else: self.log.info("No keypair specified, picking one from config...") for kp in self.ec2.keypairs: if kp.name in self.cfg.keys: keypair = kp.name kl = self.cfg.get_key(kp.name).get('key_location', '') if os.path.exists(kl) and os.path.isfile(kl): self.log.info('Using keypair: %s' % keypair) key_location = kl break if not keypair: raise exception.ConfigError( "no keypairs in region %s defined in config" % self.ec2.region.name) if not key_location: raise exception.ConfigError( "cannot determine key_location for keypair %s" % keypair) if not os.path.exists(key_location): raise exception.ValidationError( "key_location '%s' does not exist." % key_location) elif not os.path.isfile(key_location): raise exception.ValidationError( "key_location '%s' is not a file." % key_location) return (keypair, key_location)
def _get_urlfp(self, url): log.debug("Loading url: %s" % url) try: fp = urllib.urlopen(url) if fp.getcode() == 404: raise exception.ConfigError("url %s does not exist" % url) fp.name = url return fp except IOError, e: raise exception.ConfigError( "error loading config from url %s\n%s" % (url, e))
def _load_instance_types(self, store): cluster_section = store instance_types = cluster_section.get('node_instance_type') if isinstance(instance_types, basestring): return itypes = [] cluster_section['node_instance_types'] = itypes total_num_nodes = 0 choices_string = ', '.join(static.INSTANCE_TYPES.keys()) try: default_instance_type = instance_types[-1] if default_instance_type not in static.INSTANCE_TYPES: raise exception.ConfigError( "invalid node_instance_type specified: '%s'\n" "must be one of: %s" % (default_instance_type, choices_string)) except IndexError: default_instance_type = None cluster_section['node_instance_type'] = default_instance_type for type_spec in instance_types[:-1]: type_spec = type_spec.split(':') if len(type_spec) > 3: raise exception.ConfigError( "invalid node_instance_type item specified: %s" % type_spec) itype = type_spec[0] itype_image = None itype_num = 1 if itype not in static.INSTANCE_TYPES: raise exception.ConfigError( "invalid type specified (%s) in node_instance_type " "item: '%s'\nmust be one of: %s" % (itype, type_spec, choices_string)) if len(type_spec) == 2: itype, next_var = type_spec try: itype_num = int(next_var) except (TypeError, ValueError): itype_image = next_var elif len(type_spec) == 3: itype, itype_image, itype_num = type_spec try: itype_num = int(itype_num) if itype_num < 1: raise TypeError total_num_nodes += itype_num except (ValueError, TypeError): raise exception.ConfigError( "number of instances (%s) of type '%s' must " "be an integer > 1" % (itype_num, itype)) itype_dic = AttributeDict(size=itype_num, image=itype_image, type=itype) itypes.append(itype_dic)
def _load_settings(self, section_name, settings, store, filter_settings=True): """ Load section settings into a dictionary """ section = self.config._sections.get(section_name) if not section: raise exception.ConfigSectionMissing( 'Missing section %s in config' % section_name) store.update(section) section_conf = store for setting in settings: requirements = settings[setting] func, required, default, options, callback = requirements func = self.type_validators.get(func) value = func(self.config, section_name, setting) if value is not None: if options and value not in options: raise exception.ConfigError( '"%s" setting in section "%s" must be one of: %s' % (setting, section_name, ', '.join( [str(o) for o in options]))) if callback: value = callback(value) section_conf[setting] = value if filter_settings: for key in store.keys(): if key not in settings and key != '__name__': store.pop(key)
def _get_fp(self, cfg_file): log.debug("Loading file: %s" % cfg_file) if os.path.exists(cfg_file): if not os.path.isfile(cfg_file): raise exception.ConfigError( 'config %s exists but is not a regular file' % cfg_file) else: raise exception.ConfigNotFound( "config file %s does not exist\n" % cfg_file, cfg_file) return open(cfg_file)
def _load_keypairs(self, store): cluster_section = store keyname = cluster_section.get('keyname') if not keyname: return keypair = self.keys.get(keyname) if keypair is None: raise exception.ConfigError("keypair '%s' not defined in config" % keyname) cluster_section['keyname'] = keyname cluster_section['key_location'] = keypair.get('key_location')
def get_easy_ec2(self): """ Factory for EasyEC2 class that attempts to load AWS credentials from the StarCluster config file. Returns an EasyEC2 object if successful. """ try: ec2 = awsutils.EasyEC2(**self.aws) return ec2 except TypeError: raise exception.ConfigError("no aws credentials found")
def _load_extends_settings(self, section_name, store): """ Loads all settings from other template(s) specified by a section's 'extends' setting. This method walks a dependency tree of sections from bottom up. Each step is a group of settings for a section in the form of a dictionary. A 'master' dictionary is updated with the settings at each step. This causes the next group of settings to override the previous, and so on. The 'section_name' settings are at the top of the dep tree. """ section = store[section_name] extends = section.get('extends') if extends is None: return if DEBUG_CONFIG: log.debug('%s extends %s' % (section_name, extends)) extensions = [section] while True: extends = section.get('extends', None) if not extends: break try: section = store[extends] if section in extensions: exts = ', '.join([ self._get_section_name(x['__name__']) for x in extensions ]) raise exception.ConfigError( ("Cyclical dependency between sections %s. " % exts) \ + "Check your extends settings.") extensions.insert(0, section) except KeyError: raise exception.ConfigError( "%s can't extend non-existent section %s" % \ (section_name, extends)) transform = AttributeDict() for extension in extensions: transform.update(extension) store[section_name] = transform
def _load_plugins(self, store): cluster_section = store plugins = cluster_section.get('plugins') if not plugins or isinstance(plugins[0], AttributeDict): return plugs = [] for plugin in plugins: if plugin not in self.plugins: raise exception.ConfigError( "plugin '%s' not defined in config" % plugin) plugs.append(self.plugins.get(plugin)) cluster_section['plugins'] = plugs
def _get_bool(self, config, section, option): try: opt = config.getboolean(section, option) return opt except ConfigParser.NoSectionError: pass except ConfigParser.NoOptionError: pass except ValueError: raise exception.ConfigError( "Expected True/False value for setting %s in section [%s]" % (option, section))
def _get_float(self, config, section, option): try: opt = config.getfloat(section, option) return opt except ConfigParser.NoSectionError: pass except ConfigParser.NoOptionError: pass except ValueError: raise exception.ConfigError( "Expected float value for setting %s in section [%s]" % (option, section))
def get_easy_s3(self): """ Factory for EasyEC2 class that attempts to load AWS credentials from the StarCluster config file. Returns an EasyS3 object if successful. """ aws = self.get_aws_credentials() try: s3 = awsutils.EasyS3(**aws) return s3 except TypeError: raise exception.ConfigError("no aws credentials found")
def _load_volumes(self, store): cluster_section = store volumes = cluster_section.get('volumes') if not volumes or isinstance(volumes, AttributeDict): return vols = AttributeDict() cluster_section['volumes'] = vols for volume in volumes: if not volume in self.vols: raise exception.ConfigError( "volume '%s' not defined in config" % volume) vol = self.vols.get(volume) vols[volume] = vol
def __load_config(self): """ Populates self._config with a new ConfigParser instance """ cfg = self._get_cfg_fp() try: cp = InlineCommentsIgnoredConfigParser() cp.readfp(cfg) self._config = cp try: self.globals = self._load_section('global', self.global_settings) includes = self.globals.get('include') if not includes: return cp mashup = StringIO.StringIO() cfg = self._get_cfg_fp() mashup.write(cfg.read() + '\n') for include in includes: include = os.path.expanduser(include) include = os.path.expandvars(include) try: for inc_file in glob.glob(include): contents = self._get_cfg_fp(inc_file).read() mashup.write(contents + '\n') except exception.ConfigNotFound: raise exception.ConfigError("include %s not found" % include) mashup.seek(0) cp = InlineCommentsIgnoredConfigParser() cp.readfp(mashup) self._config = cp except exception.ConfigSectionMissing: pass return cp except ConfigParser.MissingSectionHeaderError: raise exception.ConfigHasNoSections(cfg.name) except ConfigParser.ParsingError, e: raise exception.ConfigError(e)
def _load_instance_array(self, store): cluster_section = store instance_array = cluster_section.get('node_instance_array') result = [] spot = -1 # -1 unknown, 0 false, 1 true for name in instance_array: node_settings = self._load_section('node ' + name, self.node_settings) if spot == -1: if node_settings['spot_bid'] is None: spot = 0 else: spot = 1 elif spot == 0: if node_settings['spot_bid'] is not None: raise exception.ConfigError("spot_bid presence must be " "the same for all nodes") else: if node_settings['spot_bid'] is None: raise exception.ConfigError("spot_bid presence must be " "the same for all nodes") result.append(node_settings) cluster_section['node_instance_array'] = result
def _get_fp(self, cfg_file): if not os.path.isdir(static.STARCLUSTER_CFG_DIR): os.makedirs(static.STARCLUSTER_CFG_DIR) log.debug("Loading file: %s" % cfg_file) if os.path.exists(cfg_file): if not os.path.isfile(cfg_file): raise exception.ConfigError( 'config %s exists but is not a regular file' % cfg_file) else: raise exception.ConfigNotFound( ("config file %s does not exist\n") % cfg_file, cfg_file, ) return open(cfg_file)
def _load_cluster_sections(self, cluster_sections): """ Loads all cluster sections. Similar to _load_sections but also handles populating specified keypair, volume, plugins, permissions, etc. settings """ clusters = cluster_sections cluster_store = AttributeDict() for cl in clusters: name = self._get_section_name(cl) cluster_store[name] = AttributeDict() self._load_settings(cl, self.cluster_settings, cluster_store[name]) for cl in clusters: name = self._get_section_name(cl) self._load_extends_settings(name, cluster_store) self._load_defaults(self.cluster_settings, cluster_store[name]) self._load_keypairs(cluster_store[name]) self._load_volumes(cluster_store[name]) self._load_plugins(cluster_store[name]) self._load_permissions(cluster_store[name]) self._load_instance_types(cluster_store[name]) self._load_instance_array(cluster_store[name]) # checks caused by mutual exclusivity of node_instance_type/array if cluster_store[name]['node_instance_type'] is not None and \ len(cluster_store[name]['node_instance_array']) > 0: raise exception.ConfigError( "Cannot define both node_instance_type and " "node_instance_array at cluster " + name) if cluster_store[name]['node_instance_type'] is not None and \ not bool(cluster_store[name]['node_image_id']): raise exception.ConfigError( "You must define node_imagge_id at cluster " + name) self._check_required(cl, self.cluster_settings, cluster_store[name]) return cluster_store
def _load_permissions(self, store): cluster_section = store permissions = cluster_section.get('permissions') if not permissions or isinstance(permissions, AttributeDict): return perms = AttributeDict() cluster_section['permissions'] = perms for perm in permissions: if perm in self.permissions: p = self.permissions.get(perm) p['__name__'] = p['__name__'].split()[-1] perms[perm] = p else: raise exception.ConfigError( "permission '%s' not defined in config" % perm)
def _load_plugins(self, store): cluster_section = store plugins = cluster_section.get('plugins') if not plugins or isinstance(plugins[0], AttributeDict): return plugs = [] cluster_section['plugins'] = plugs for plugin in plugins: if plugin in self.plugins: p = self.plugins.get(plugin) p['__name__'] = p['__name__'].split()[-1] plugs.append(p) else: raise exception.ConfigError( "plugin '%s' not defined in config" % plugin)
def __load_config(self): """ Populates self._config with a new ConfigParser instance """ cfg = self.cfg_file if utils.is_url(cfg): cfg = self._get_urlfp(cfg) else: cfg = self._get_fp(cfg) try: cp = ConfigParser.ConfigParser() cp.readfp(cfg) return cp except ConfigParser.MissingSectionHeaderError: raise exception.ConfigHasNoSections(cfg.name) except ConfigParser.ParsingError, e: raise exception.ConfigError(e)
def _check_required(self, section_name, settings, store): """ Check that all required settings were specified in the config. Raises ConfigError otherwise. Note that if a setting specified has required=True and default is not None then this method will not raise an error because a default was given. In short, if a setting is required you must provide None as the 'default' value. """ section_conf = store for setting in settings: requirements = settings[setting] required = requirements[1] value = section_conf.get(setting) if value is None and required: raise exception.ConfigError( 'missing required option %s in section "%s"' % (setting, section_name))