def validate_conf(cls, process_filename, within):
# Init
with open(process_filename) as f:
process_dict = replace_variables(f.read())
process_dict = json.loads(process_dict)
# Ensure package required to install cluster is available
directory = cls.get_directory(process_dict, within)
for cluster in process_dict['register'][directory]:
dash_un_cluster = cluster['module'].replace('-', '_')
mods = available_recipes | loaded_modules
if cluster['module'] not in mods and dash_un_cluster not in mods:
if cluster['module'] in pip_packages:
# import
pass
elif dash_un_cluster in pip_packages:
# import
pass
else:
# `folder` is one above `offregister` directory
folder = environ.get('PKG_DIR', path.dirname(path.dirname(path.dirname(__file__))))
ls_folder = listdir(folder)
pip_install_d = partial(pip_install, options_attr={'src_dir': folder})
if cluster in ls_folder:
pip_install_d(path.join(folder, cluster))
elif dash_un_cluster in ls_folder:
pip_install_d(path.join(folder, dash_un_cluster))
else:
raise ImportError("Cannot find package for cluster: '{!s}'".format(cluster))
def test_parsing(self):
self.assertEqual(
replace_variables(
dumps(self.sample),
extra_env={"jar": "CAN"},
variables={"foo": "HAZ", "bar": "BAR", "can_haz": "CAN_HAZ"},
),
dumps(self.parsed_sample),
)
def __init__(self, process_filename, node=None, previous_clustering_results=None):
self.previous_clustering_results = previous_clustering_results
if not node:
node = fetch_node(marshall=json)
with open(process_filename) as f:
strategy = replace_variables(f.read())
self.process_dict = json.loads(strategy)
driver_cls = node.value['driver'] if '_' not in node.value['driver'] \
else node.value['driver'][:node.value['driver'].find('_')
] + node.value['driver'][node.value['driver'].rfind('_') + 1:]
self.driver_name = next(driver_name for driver_name, driver_tuple in DRIVERS.iteritems()
if driver_tuple[1] == driver_cls)
self.config_provider = next(provider for provider in self.process_dict['provider']['options']
if provider['provider']['name'] == (
self.driver_name.upper() if hasattr(Provider, self.driver_name.upper())
else next(
ifilter(lambda prov_name: getattr(Provider, prov_name) == self.driver_name,
dir(Provider)))
))
self.driver_name = self.driver_name.lower()
driver = (lambda driver: driver(
region=self.config_provider['provider']['region'],
**self.config_provider['auth']
))(get_driver(self.driver_name))
self.node_name = node.key[node.key.find('/', 1) + 1:].encode('utf8')
if self.driver_name == 'azure':
if 'create_with' not in self.config_provider or \
'ex_cloud_service_name' not in self.config_provider['create_with']:
raise KeyError('`ex_cloud_service_name` must be defined. '
'See: http://libcloud.readthedocs.org/en/latest/compute/drivers/azure.html'
'#libcloud.compute.drivers.azure.AzureNodeDriver.create_node')
nodes = driver.list_nodes(self.config_provider['create_with']['ex_cloud_service_name'])
else:
nodes = driver.list_nodes()
self.node = next(ifilter(lambda _node: _node.uuid == node.value['uuid'],
nodes), None)
if not self.node:
raise EnvironmentError('node not found. Maybe the cloud provider is still provisioning?')
if 'password' in self.node.extra:
print 'password ='******'password']
# pp(node_to_dict(self.node))
self.dns_name = self.node.extra.get('dns_name')
def destroy(config_filename, restrict_provider_to=None):
with open(config_filename, 'rt') as f:
config_contents = f.read()
config_dict = loads(replace_variables(config_contents))
del config_contents
providers = tuple(obj for obj in config_dict['provider']['options']
if obj['provider']['name'] in restrict_provider_to
or obj['provider']['name'] == restrict_provider_to) if restrict_provider_to \
else tuple(obj for obj in config_dict['provider']['options'])
client = (lambda etcd_server_location: Client(
protocol=etcd_server_location.scheme,
host=etcd_server_location.hostname,
port=etcd_server_location.port
))(urlparse(config_dict['etcd_server']))
provider2conf_and_driver = dict(
imap(lambda provider_dict: (provider_dict['provider']['name'],
namedtuple('_', 'conf driver_cls')(
provider_dict,
(lambda provider_cls: provider_cls(
region=provider_dict['provider']['region'],
**provider_dict['auth']
))(get_driver(
getattr(Provider, provider_dict['provider']['name'])
if hasattr(Provider, provider_dict['provider']['name'])
else itemgetter(1)(next(ifilter(
lambda (prov_name, value): value == provider_dict['provider'][
'name'].lower(),
imap(lambda prov_name: (prov_name, getattr(Provider, prov_name)),
dir(Provider))
)))
)))), providers)
)
# Map nodes to their provider, including ones outside etcd
provider2nodes = {
provider: tuple(
namedtuple('_', 'uuid node')(node.uuid, node) for node in
driver.driver_cls.list_nodes(*((driver.conf['create_with']['ex_cloud_service_name'],)
if driver.conf['provider']['name'] == 'AZURE'
else tuple()
))
if driver.driver_cls.NODE_STATE_MAP and node.state in (
driver.driver_cls.NODE_STATE_MAP.get(
'running',
next((node.state for k, v in driver.driver_cls.NODE_STATE_MAP.iteritems()
if 'running' in v), None)
),
driver.driver_cls.NODE_STATE_MAP.get('active')
) or not driver.driver_cls.NODE_STATE_MAP and node.state in ('running',)
)
for provider, driver in provider2conf_and_driver.iteritems()}
uuid2key = {loads(client.get(key).value)['uuid']: key
for key in flatten(etcd_ls(client))
if (lambda v: isinstance(v, basestring) and v.startswith('{'))(client.get(key).value)}
# TODO: Only call `client.get` once per `key` ^
# Filter to just ones inside etcd; then deprovision and delete from etcd
logger.info('Dropped: {}'.format(
{
provider: tuple(imap(lambda n: rm_prov_etcd(client, n.node), nodes))
for provider, nodes in provider2nodes.iteritems()
for node in nodes
if node.uuid in uuid2key
}
))
# Delete all empty etcd directories.
for i in xrange(20): # TODO: walk the tree rather than hackily rerun
it_consumes(
logger.info('rmdir {directory}'.format(directory=directory, res=client.delete(directory, dir=True)))
for directory in flatten(etcd_empty_dirs(client))
)
return client
def __init__(self, strategy_filename):
with open(strategy_filename) as f:
strategy = replace_variables(f.read())
self.strategy = loads(strategy)
self.default_pick = self.strategy['default_pick']
