def _load_keypair(self, keypair=None):
key_location = None
if keypair:
kp = self.ec2.get_keypair(keypair)
key = self.cfg.get_key(kp.name)
key_location = key.get('key_location', '')
else:
self.log.info("No keypair specified, picking one from config...")
for kp in self.ec2.keypairs:
if kp.name in self.cfg.keys:
keypair = kp.name
kl = self.cfg.get_key(kp.name).get('key_location', '')
if os.path.exists(kl) and os.path.isfile(kl):
self.log.info('Using keypair: %s' % keypair)
key_location = kl
break
if not keypair:
raise exception.ConfigError(
"no keypairs in region %s defined in config" %
self.ec2.region.name)
if not key_location:
raise exception.ConfigError(
"cannot determine key_location for keypair %s" % keypair)
if not os.path.exists(key_location):
raise exception.ValidationError(
"key_location '%s' does not exist." % key_location)
elif not os.path.isfile(key_location):
raise exception.ValidationError(
"key_location '%s' is not a file." % key_location)
return (keypair, key_location)
def _get_urlfp(self, url):
log.debug("Loading url: %s" % url)
try:
fp = urllib.urlopen(url)
if fp.getcode() == 404:
raise exception.ConfigError("url %s does not exist" % url)
fp.name = url
return fp
except IOError, e:
raise exception.ConfigError(
"error loading config from url %s\n%s" % (url, e))
def _load_instance_types(self, store):
cluster_section = store
instance_types = cluster_section.get('node_instance_type')
if isinstance(instance_types, basestring):
return
itypes = []
cluster_section['node_instance_types'] = itypes
total_num_nodes = 0
choices_string = ', '.join(static.INSTANCE_TYPES.keys())
try:
default_instance_type = instance_types[-1]
if default_instance_type not in static.INSTANCE_TYPES:
raise exception.ConfigError(
"invalid node_instance_type specified: '%s'\n"
"must be one of: %s" %
(default_instance_type, choices_string))
except IndexError:
default_instance_type = None
cluster_section['node_instance_type'] = default_instance_type
for type_spec in instance_types[:-1]:
type_spec = type_spec.split(':')
if len(type_spec) > 3:
raise exception.ConfigError(
"invalid node_instance_type item specified: %s" %
type_spec)
itype = type_spec[0]
itype_image = None
itype_num = 1
if itype not in static.INSTANCE_TYPES:
raise exception.ConfigError(
"invalid type specified (%s) in node_instance_type "
"item: '%s'\nmust be one of: %s" %
(itype, type_spec, choices_string))
if len(type_spec) == 2:
itype, next_var = type_spec
try:
itype_num = int(next_var)
except (TypeError, ValueError):
itype_image = next_var
elif len(type_spec) == 3:
itype, itype_image, itype_num = type_spec
try:
itype_num = int(itype_num)
if itype_num < 1:
raise TypeError
total_num_nodes += itype_num
except (ValueError, TypeError):
raise exception.ConfigError(
"number of instances (%s) of type '%s' must "
"be an integer > 1" % (itype_num, itype))
itype_dic = AttributeDict(size=itype_num,
image=itype_image,
type=itype)
itypes.append(itype_dic)
def _load_settings(self,
section_name,
settings,
store,
filter_settings=True):
"""
Load section settings into a dictionary
"""
section = self.config._sections.get(section_name)
if not section:
raise exception.ConfigSectionMissing(
'Missing section %s in config' % section_name)
store.update(section)
section_conf = store
for setting in settings:
requirements = settings[setting]
func, required, default, options, callback = requirements
func = self.type_validators.get(func)
value = func(self.config, section_name, setting)
if value is not None:
if options and value not in options:
raise exception.ConfigError(
'"%s" setting in section "%s" must be one of: %s' %
(setting, section_name, ', '.join(
[str(o) for o in options])))
if callback:
value = callback(value)
section_conf[setting] = value
if filter_settings:
for key in store.keys():
if key not in settings and key != '__name__':
store.pop(key)
def _get_fp(self, cfg_file):
log.debug("Loading file: %s" % cfg_file)
if os.path.exists(cfg_file):
if not os.path.isfile(cfg_file):
raise exception.ConfigError(
'config %s exists but is not a regular file' % cfg_file)
else:
raise exception.ConfigNotFound(
"config file %s does not exist\n" % cfg_file, cfg_file)
return open(cfg_file)
def _load_keypairs(self, store):
cluster_section = store
keyname = cluster_section.get('keyname')
if not keyname:
return
keypair = self.keys.get(keyname)
if keypair is None:
raise exception.ConfigError("keypair '%s' not defined in config" %
keyname)
cluster_section['keyname'] = keyname
cluster_section['key_location'] = keypair.get('key_location')
def get_easy_ec2(self):
"""
Factory for EasyEC2 class that attempts to load AWS credentials from
the StarCluster config file. Returns an EasyEC2 object if
successful.
"""
try:
ec2 = awsutils.EasyEC2(**self.aws)
return ec2
except TypeError:
raise exception.ConfigError("no aws credentials found")
def _load_extends_settings(self, section_name, store):
"""
Loads all settings from other template(s) specified by a section's
'extends' setting.
This method walks a dependency tree of sections from bottom up. Each
step is a group of settings for a section in the form of a dictionary.
A 'master' dictionary is updated with the settings at each step. This
causes the next group of settings to override the previous, and so on.
The 'section_name' settings are at the top of the dep tree.
"""
section = store[section_name]
extends = section.get('extends')
if extends is None:
return
if DEBUG_CONFIG:
log.debug('%s extends %s' % (section_name, extends))
extensions = [section]
while True:
extends = section.get('extends', None)
if not extends:
break
try:
section = store[extends]
if section in extensions:
exts = ', '.join([
self._get_section_name(x['__name__'])
for x in extensions
])
raise exception.ConfigError(
("Cyclical dependency between sections %s. " % exts) \
+ "Check your extends settings.")
extensions.insert(0, section)
except KeyError:
raise exception.ConfigError(
"%s can't extend non-existent section %s" % \
(section_name, extends))
transform = AttributeDict()
for extension in extensions:
transform.update(extension)
store[section_name] = transform
def _load_plugins(self, store):
cluster_section = store
plugins = cluster_section.get('plugins')
if not plugins or isinstance(plugins[0], AttributeDict):
return
plugs = []
for plugin in plugins:
if plugin not in self.plugins:
raise exception.ConfigError(
"plugin '%s' not defined in config" % plugin)
plugs.append(self.plugins.get(plugin))
cluster_section['plugins'] = plugs
def _get_bool(self, config, section, option):
try:
opt = config.getboolean(section, option)
return opt
except ConfigParser.NoSectionError:
pass
except ConfigParser.NoOptionError:
pass
except ValueError:
raise exception.ConfigError(
"Expected True/False value for setting %s in section [%s]" %
(option, section))
def _get_float(self, config, section, option):
try:
opt = config.getfloat(section, option)
return opt
except ConfigParser.NoSectionError:
pass
except ConfigParser.NoOptionError:
pass
except ValueError:
raise exception.ConfigError(
"Expected float value for setting %s in section [%s]" %
(option, section))
def get_easy_s3(self):
"""
Factory for EasyEC2 class that attempts to load AWS credentials from
the StarCluster config file. Returns an EasyS3 object if
successful.
"""
aws = self.get_aws_credentials()
try:
s3 = awsutils.EasyS3(**aws)
return s3
except TypeError:
raise exception.ConfigError("no aws credentials found")
def _load_volumes(self, store):
cluster_section = store
volumes = cluster_section.get('volumes')
if not volumes or isinstance(volumes, AttributeDict):
return
vols = AttributeDict()
cluster_section['volumes'] = vols
for volume in volumes:
if not volume in self.vols:
raise exception.ConfigError(
"volume '%s' not defined in config" % volume)
vol = self.vols.get(volume)
vols[volume] = vol
def __load_config(self):
"""
Populates self._config with a new ConfigParser instance
"""
cfg = self._get_cfg_fp()
try:
cp = InlineCommentsIgnoredConfigParser()
cp.readfp(cfg)
self._config = cp
try:
self.globals = self._load_section('global',
self.global_settings)
includes = self.globals.get('include')
if not includes:
return cp
mashup = StringIO.StringIO()
cfg = self._get_cfg_fp()
mashup.write(cfg.read() + '\n')
for include in includes:
include = os.path.expanduser(include)
include = os.path.expandvars(include)
try:
for inc_file in glob.glob(include):
contents = self._get_cfg_fp(inc_file).read()
mashup.write(contents + '\n')
except exception.ConfigNotFound:
raise exception.ConfigError("include %s not found" %
include)
mashup.seek(0)
cp = InlineCommentsIgnoredConfigParser()
cp.readfp(mashup)
self._config = cp
except exception.ConfigSectionMissing:
pass
return cp
except ConfigParser.MissingSectionHeaderError:
raise exception.ConfigHasNoSections(cfg.name)
except ConfigParser.ParsingError, e:
raise exception.ConfigError(e)
def _load_instance_array(self, store):
cluster_section = store
instance_array = cluster_section.get('node_instance_array')
result = []
spot = -1 # -1 unknown, 0 false, 1 true
for name in instance_array:
node_settings = self._load_section('node ' + name,
self.node_settings)
if spot == -1:
if node_settings['spot_bid'] is None:
spot = 0
else:
spot = 1
elif spot == 0:
if node_settings['spot_bid'] is not None:
raise exception.ConfigError("spot_bid presence must be "
"the same for all nodes")
else:
if node_settings['spot_bid'] is None:
raise exception.ConfigError("spot_bid presence must be "
"the same for all nodes")
result.append(node_settings)
cluster_section['node_instance_array'] = result
def _get_fp(self, cfg_file):
if not os.path.isdir(static.STARCLUSTER_CFG_DIR):
os.makedirs(static.STARCLUSTER_CFG_DIR)
log.debug("Loading file: %s" % cfg_file)
if os.path.exists(cfg_file):
if not os.path.isfile(cfg_file):
raise exception.ConfigError(
'config %s exists but is not a regular file' % cfg_file)
else:
raise exception.ConfigNotFound(
("config file %s does not exist\n") %
cfg_file, cfg_file,
)
return open(cfg_file)
def _load_cluster_sections(self, cluster_sections):
"""
Loads all cluster sections. Similar to _load_sections but also handles
populating specified keypair, volume, plugins, permissions, etc.
settings
"""
clusters = cluster_sections
cluster_store = AttributeDict()
for cl in clusters:
name = self._get_section_name(cl)
cluster_store[name] = AttributeDict()
self._load_settings(cl, self.cluster_settings, cluster_store[name])
for cl in clusters:
name = self._get_section_name(cl)
self._load_extends_settings(name, cluster_store)
self._load_defaults(self.cluster_settings, cluster_store[name])
self._load_keypairs(cluster_store[name])
self._load_volumes(cluster_store[name])
self._load_plugins(cluster_store[name])
self._load_permissions(cluster_store[name])
self._load_instance_types(cluster_store[name])
self._load_instance_array(cluster_store[name])
# checks caused by mutual exclusivity of node_instance_type/array
if cluster_store[name]['node_instance_type'] is not None and \
len(cluster_store[name]['node_instance_array']) > 0:
raise exception.ConfigError(
"Cannot define both node_instance_type and "
"node_instance_array at cluster " + name)
if cluster_store[name]['node_instance_type'] is not None and \
not bool(cluster_store[name]['node_image_id']):
raise exception.ConfigError(
"You must define node_imagge_id at cluster " + name)
self._check_required(cl, self.cluster_settings,
cluster_store[name])
return cluster_store
def _load_permissions(self, store):
cluster_section = store
permissions = cluster_section.get('permissions')
if not permissions or isinstance(permissions, AttributeDict):
return
perms = AttributeDict()
cluster_section['permissions'] = perms
for perm in permissions:
if perm in self.permissions:
p = self.permissions.get(perm)
p['__name__'] = p['__name__'].split()[-1]
perms[perm] = p
else:
raise exception.ConfigError(
"permission '%s' not defined in config" % perm)
def _load_plugins(self, store):
cluster_section = store
plugins = cluster_section.get('plugins')
if not plugins or isinstance(plugins[0], AttributeDict):
return
plugs = []
cluster_section['plugins'] = plugs
for plugin in plugins:
if plugin in self.plugins:
p = self.plugins.get(plugin)
p['__name__'] = p['__name__'].split()[-1]
plugs.append(p)
else:
raise exception.ConfigError(
"plugin '%s' not defined in config" % plugin)
def __load_config(self):
"""
Populates self._config with a new ConfigParser instance
"""
cfg = self.cfg_file
if utils.is_url(cfg):
cfg = self._get_urlfp(cfg)
else:
cfg = self._get_fp(cfg)
try:
cp = ConfigParser.ConfigParser()
cp.readfp(cfg)
return cp
except ConfigParser.MissingSectionHeaderError:
raise exception.ConfigHasNoSections(cfg.name)
except ConfigParser.ParsingError, e:
raise exception.ConfigError(e)
def _check_required(self, section_name, settings, store):
"""
Check that all required settings were specified in the config.
Raises ConfigError otherwise.
Note that if a setting specified has required=True and
default is not None then this method will not raise an error
because a default was given. In short, if a setting is required
you must provide None as the 'default' value.
"""
section_conf = store
for setting in settings:
requirements = settings[setting]
required = requirements[1]
value = section_conf.get(setting)
if value is None and required:
raise exception.ConfigError(
'missing required option %s in section "%s"' %
(setting, section_name))
