def _load_sections(self,
section_prefix,
section_settings,
filter_settings=True):
"""
Loads all sections starting with section_prefix and returns a
dictionary containing the name and dictionary of settings for each
section.
keys --> section name (as returned by self._get_section_name)
values --> dictionary of settings for a given section
e.g.
$ print self._load_sections('volumes', self.plugin_settings)
{'myvol': {'__name__': 'volume myvol',
'device': None,
'mount_path': '/home',
'partition': 1,
'volume_id': 'vol-999999'},
'myvol2': {'__name__': 'volume myvol2',
'device': None,
'mount_path': '/myvol2',
'partition': 1,
'volume_id': 'vol-999999'},
"""
sections = self._get_sections(section_prefix)
sections_store = AttributeDict()
for sec in sections:
name = self._get_section_name(sec)
sections_store[name] = self._load_section(sec, section_settings,
filter_settings)
return sections_store
def _load_instance_types(self, store):
cluster_section = store
instance_types = cluster_section.get('node_instance_type')
if isinstance(instance_types, basestring):
return
itypes = []
cluster_section['node_instance_types'] = itypes
total_num_nodes = 0
choices_string = ', '.join(static.INSTANCE_TYPES.keys())
try:
default_instance_type = instance_types[-1]
if default_instance_type not in static.INSTANCE_TYPES:
raise exception.ConfigError(
"invalid node_instance_type specified: '%s'\n"
"must be one of: %s" %
(default_instance_type, choices_string))
except IndexError:
default_instance_type = None
cluster_section['node_instance_type'] = default_instance_type
for type_spec in instance_types[:-1]:
type_spec = type_spec.split(':')
if len(type_spec) > 3:
raise exception.ConfigError(
"invalid node_instance_type item specified: %s" %
type_spec)
itype = type_spec[0]
itype_image = None
itype_num = 1
if itype not in static.INSTANCE_TYPES:
raise exception.ConfigError(
"invalid type specified (%s) in node_instance_type "
"item: '%s'\nmust be one of: %s" %
(itype, type_spec, choices_string))
if len(type_spec) == 2:
itype, next_var = type_spec
try:
itype_num = int(next_var)
except (TypeError, ValueError):
itype_image = next_var
elif len(type_spec) == 3:
itype, itype_image, itype_num = type_spec
try:
itype_num = int(itype_num)
if itype_num < 1:
raise TypeError
total_num_nodes += itype_num
except (ValueError, TypeError):
raise exception.ConfigError(
"number of instances (%s) of type '%s' must "
"be an integer > 1" % (itype_num, itype))
itype_dic = AttributeDict(size=itype_num,
image=itype_image,
type=itype)
itypes.append(itype_dic)
def _load_extends_settings(self, section_name, store):
"""
Loads all settings from other template(s) specified by a section's
'extends' setting.
This method walks a dependency tree of sections from bottom up. Each
step is a group of settings for a section in the form of a dictionary.
A 'master' dictionary is updated with the settings at each step. This
causes the next group of settings to override the previous, and so on.
The 'section_name' settings are at the top of the dep tree.
"""
section = store[section_name]
extends = section.get('extends')
if extends is None:
return
if DEBUG_CONFIG:
log.debug('%s extends %s' % (section_name, extends))
extensions = [section]
while True:
extends = section.get('extends', None)
if not extends:
break
try:
section = store[extends]
if section in extensions:
exts = ', '.join([
self._get_section_name(x['__name__'])
for x in extensions
])
raise exception.ConfigError(
("Cyclical dependency between sections %s. " % exts) \
+ "Check your extends settings.")
extensions.insert(0, section)
except KeyError:
raise exception.ConfigError(
"%s can't extend non-existent section %s" % \
(section_name, extends))
transform = AttributeDict()
for extension in extensions:
transform.update(extension)
store[section_name] = transform
def _load_section(self, section_name, section_settings):
"""
Returns a dictionary containing all section_settings for a given
section_name by first loading the settings in the config, loading
the defaults for all settings not specified, and then checking
that all required options have been specified
"""
store = AttributeDict()
self._load_settings(section_name, section_settings, store)
self._load_defaults(section_settings, store)
self._check_required(section_name, section_settings, store)
return store
def _load_volumes(self, store):
cluster_section = store
volumes = cluster_section.get('volumes')
if not volumes or isinstance(volumes, AttributeDict):
return
vols = AttributeDict()
cluster_section['volumes'] = vols
for volume in volumes:
if not volume in self.vols:
raise exception.ConfigError(
"volume '%s' not defined in config" % volume)
vol = self.vols.get(volume)
vols[volume] = vol
def _load_extends_settings(self, section_name, store):
"""
Loads all settings from other template(s) specified by a section's
'extends' setting.
This method walks a dependency tree of sections from bottom up. Each
step is a group of settings for a section in the form of a dictionary.
A 'master' dictionary is updated with the settings at each step. This
causes the next group of settings to override the previous, and so on.
The 'section_name' settings are at the top of the dep tree.
"""
section = store[section_name]
extends = section.get('extends')
if extends is None:
return
if DEBUG_CONFIG:
log.debug('%s extends %s' % (section_name, extends))
extensions = [section]
while True:
extends = section.get('extends', None)
if not extends:
break
try:
section = store[extends]
if section in extensions:
exts = ', '.join([self._get_section_name(x['__name__'])
for x in extensions])
raise exception.ConfigError(
("Cyclical dependency between sections %s. " % exts) \
+ "Check your extends settings.")
extensions.insert(0, section)
except KeyError:
raise exception.ConfigError(
"%s can't extend non-existent section %s" % \
(section_name, extends))
transform = AttributeDict()
for extension in extensions:
transform.update(extension)
store[section_name] = transform
def _load_cluster_sections(self, cluster_sections):
"""
Loads all cluster sections. Similar to _load_sections but also handles
populating specified keypair,volume,plugins,permissions,etc settings
"""
clusters = cluster_sections
cluster_store = AttributeDict()
for cl in clusters:
name = self._get_section_name(cl)
cluster_store[name] = AttributeDict()
self._load_settings(cl, self.cluster_settings, cluster_store[name])
for cl in clusters:
name = self._get_section_name(cl)
self._load_extends_settings(name, cluster_store)
self._load_defaults(self.cluster_settings, cluster_store[name])
self._load_keypairs(cluster_store[name])
self._load_volumes(cluster_store[name])
self._load_plugins(cluster_store[name])
self._load_permissions(cluster_store[name])
self._load_instance_types(cluster_store[name])
self._check_required(cl, self.cluster_settings,
cluster_store[name])
return cluster_store
def _load_cluster_sections(self, cluster_sections):
"""
Loads all cluster sections. Similar to _load_sections but also handles
populating specified keypair, volume, plugins, permissions, etc.
settings
"""
clusters = cluster_sections
cluster_store = AttributeDict()
for cl in clusters:
name = self._get_section_name(cl)
cluster_store[name] = AttributeDict()
self._load_settings(cl, self.cluster_settings, cluster_store[name])
for cl in clusters:
name = self._get_section_name(cl)
self._load_extends_settings(name, cluster_store)
self._load_defaults(self.cluster_settings, cluster_store[name])
self._load_keypairs(cluster_store[name])
self._load_volumes(cluster_store[name])
self._load_plugins(cluster_store[name])
self._load_permissions(cluster_store[name])
self._load_instance_types(cluster_store[name])
self._load_instance_array(cluster_store[name])
# checks caused by mutual exclusivity of node_instance_type/array
if cluster_store[name]['node_instance_type'] is not None and \
len(cluster_store[name]['node_instance_array']) > 0:
raise exception.ConfigError(
"Cannot define both node_instance_type and "
"node_instance_array at cluster " + name)
if cluster_store[name]['node_instance_type'] is not None and \
not bool(cluster_store[name]['node_image_id']):
raise exception.ConfigError(
"You must define node_imagge_id at cluster " + name)
self._check_required(cl, self.cluster_settings,
cluster_store[name])
return cluster_store
def _load_permissions(self, store):
cluster_section = store
permissions = cluster_section.get('permissions')
if not permissions or isinstance(permissions, AttributeDict):
return
perms = AttributeDict()
cluster_section['permissions'] = perms
for perm in permissions:
if perm in self.permissions:
p = self.permissions.get(perm)
p['__name__'] = p['__name__'].split()[-1]
perms[perm] = p
else:
raise exception.ConfigError(
"permission '%s' not defined in config" % perm)
def __init__(self, config_file=None, cache=False):
self.cfg_file = config_file or static.STARCLUSTER_CFG_FILE
self.type_validators = {
int: self._get_int,
str: self._get_string,
bool: self._get_bool,
list: self._get_list,
}
self._config = None
self.globals = AttributeDict()
self.aws = AttributeDict()
self.clusters = AttributeDict()
self.keys = AttributeDict()
self.vols = AttributeDict()
self.plugins = AttributeDict()
self.permissions = AttributeDict()
self.cache = cache
def __init__(self, config_file=None, cache=False):
self.cfg_file = config_file \
or os.environ.get('STARCLUSTER_CONFIG') \
or static.STARCLUSTER_CFG_FILE
self.cfg_file = os.path.expanduser(self.cfg_file)
self.cfg_file = os.path.expandvars(self.cfg_file)
self.type_validators = {
int: self._get_int,
float: self._get_float,
str: self._get_string,
bool: self._get_bool,
list: self._get_list,
}
self._config = None
self.globals = AttributeDict()
self.aws = AttributeDict()
self.clusters = AttributeDict()
self.keys = AttributeDict()
self.vols = AttributeDict()
self.plugins = AttributeDict()
self.permissions = AttributeDict()
self.cache = cache
def __init__(self, config_file=None, cache=False):
import inspect;
#log.info("0.92.rc2 config.__init__ caller: %s, module: %s" , inspect.stack()[1][3], inspect.stack()[1][1]);
#log.info("0.92.rc2 config.__init__ config_file: %s", config_file)
self.cfg_file = config_file or static.STARCLUSTER_CFG_FILE
#log.info("0.92.rc2 config.__init__ self.cfg_file: %s", self.cfg_file)
self.type_validators = {
int: self._get_int,
float: self._get_float,
str: self._get_string,
bool: self._get_bool,
list: self._get_list,
}
self._config = None
self.globals = AttributeDict()
self.aws = AttributeDict()
self.clusters = AttributeDict()
self.keys = AttributeDict()
self.vols = AttributeDict()
self.plugins = AttributeDict()
self.permissions = AttributeDict()
self.cache = cache
try:
section = store[extends]
if section in extensions:
exts = ', '.join([self._get_section_name(x['__name__'])
for x in extensions])
raise exception.ConfigError(
("Cyclical dependency between sections %s. " %
exts) + "Check your extends settings."
)
extensions.insert(0, section)
except KeyError,e:
raise exception.ConfigError(
"%s can't extend non-existent section %s" %
(section_name,extends)
)
transform = AttributeDict()
for extension in extensions:
transform.update(extension)
store[section_name] = transform
def _load_keypairs(self, section_name, store):
cluster_section = store
keyname = cluster_section.get('keyname')
if not keyname:
return
keypair = self.keys.get(keyname)
if keypair is None:
raise exception.ConfigError("keypair '%s' not defined in config" % keyname)
cluster_section['keyname'] = keyname
cluster_section['key_location'] = keypair.get('key_location')