def execute(self):
"""
Load the cluster and build a GC3Pie configuration snippet.
"""
log.warning("Command `elasticluster gc3pie-config` is DEPRECATED"
" and will be removed in release 1.4 of ElastiCluster")
creator = make_creator(self.params.config,
storage_path=self.params.storage)
cluster_name = self.params.cluster
try:
cluster = creator.load_cluster(cluster_name)
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Listing nodes from cluster %s: %s", cluster_name, ex)
return
from elasticluster.gc3pie_config import create_gc3pie_config_snippet
if self.params.append:
path = os.path.expanduser(self.params.append)
try:
fd = open(path, 'a')
fd.write(create_gc3pie_config_snippet(cluster))
fd.close()
except IOError as ex:
log.error("Unable to write configuration to file %s: %s", path,
ex)
else:
print(create_gc3pie_config_snippet(cluster))
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
cluster_name = self.params.cluster
try:
cluster = creator.load_cluster(cluster_name)
cluster.update()
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Setting up cluster %s: %s", cluster_name, ex)
return
# XXX: the default value of `self.params.ssh_to` should = the
# default value for `ssh_to` in `Cluster.get_ssh_to_node()`
frontend = cluster.get_ssh_to_node(self.params.ssh_to)
host = frontend.connection_ip()
if not host:
log.error("No IP address known for node %s", frontend.name)
sys.exit(1)
addr, port = parse_ip_address_and_port(host)
username = frontend.image_user
knownhostsfile = (cluster.known_hosts_file if cluster.known_hosts_file
else '/dev/null')
sftp_cmdline = [
"sftp",
"-P", "{0:d}".format(port),
"-o", "UserKnownHostsFile={0}".format(knownhostsfile),
"-o", "StrictHostKeyChecking=yes",
"-o", "IdentityFile={0}".format(frontend.user_key_private),
]
sftp_cmdline.extend(self.params.sftp_args)
sftp_cmdline.append('{0}@{1}'.format(username, addr))
os.execlp("sftp", *sftp_cmdline)
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
cluster_name = self.params.cluster
try:
cluster = creator.load_cluster(cluster_name)
cluster.update()
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Setting up cluster %s: %s\n" % (cluster_name, ex))
return
if self.params.ssh_to:
try:
nodes = dict((n.name, n) for n in cluster.get_all_nodes())
frontend = nodes[self.params.ssh_to]
except KeyError:
raise ValueError("Hostname %s not found in cluster %s" %
(self.params.ssh_to, cluster_name))
else:
frontend = cluster.get_frontend_node()
host = frontend.connection_ip()
username = frontend.image_user
knownhostsfile = cluster.known_hosts_file if cluster.known_hosts_file \
else '/dev/null'
sftp_cmdline = [
"sftp", "-o",
"UserKnownHostsFile=%s" % knownhostsfile, "-o",
"StrictHostKeyChecking=yes", "-o",
"IdentityFile=%s" % frontend.user_key_private
]
sftp_cmdline.extend(self.params.sftp_args)
sftp_cmdline.append('%s@%s' % (username, host))
os.execlp("sftp", *sftp_cmdline)
def execute(self):
"""
Load the cluster and build a GC3Pie configuration snippet.
"""
creator = make_creator(self.params.config,
storage_path=self.params.storage)
cluster_name = self.params.cluster
try:
cluster = creator.load_cluster(cluster_name)
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Listing nodes from cluster %s: %s\n" %
(cluster_name, ex))
return
from elasticluster.gc3pie_config import create_gc3pie_config_snippet
if self.params.append:
path = os.path.expanduser(self.params.append)
try:
fd = open(path, 'a')
fd.write(create_gc3pie_config_snippet(cluster))
fd.close()
except IOError as ex:
log.error("Unable to write configuration to file %s: %s", path,
ex)
else:
print(create_gc3pie_config_snippet(cluster))
def test_get_cloud_provider_openstack(tmpdir):
wd = tmpdir.strpath
ssh_key_path = os.path.join(wd, 'id_rsa.pem')
with open(ssh_key_path, 'w+') as ssh_key_file:
# don't really care about SSH key, just that the file exists
ssh_key_file.write('')
ssh_key_file.flush()
config_path = os.path.join(wd, 'config.ini')
with open(config_path, 'w+') as config_file:
config_file.write(
"""
[cloud/openstack]
provider = openstack
auth_url = http://openstack.example.com:5000/v2.0
username = ${USER}
password = XXXXXX
project_name = test
"""
+ make_config_snippet("cluster", "example_openstack")
+ make_config_snippet("login", "ubuntu", keyname='test', valid_path=ssh_key_path)
+ make_config_snippet("setup", "slurm_setup_old")
)
creator = make_creator(config_path)
cloud = creator.create_cloud_provider('example_openstack')
from elasticluster.providers.openstack import OpenStackCloudProvider
assert isinstance(cloud, OpenStackCloudProvider)
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
config = creator.cluster_conf
print("""%d cluster templates found in configuration file.""" %
len(config))
templates = config.keys()
for pattern in self.params.clusters:
templates = [t for t in templates if fnmatch(t, pattern)]
if self.params.clusters:
print("""%d cluter templates found matching pattern(s) '%s'""" %
(len(templates), str.join(", ", self.params.clusters)))
for template in templates:
try:
cluster = creator.create_cluster(template, template)
print("""
name: %s""" % template)
for nodekind in cluster.nodes:
print("%s nodes: %d" %
(nodekind, len(cluster.nodes[nodekind])))
except ConfigurationError as ex:
log.error("unable to load cluster `%s`: %s", template, ex)
def execute(self):
"""
Lists all nodes within the specified cluster with certain
information like id and ip.
"""
creator = make_creator(self.params.config,
storage_path=self.params.storage)
cluster_name = self.params.cluster
try:
cluster = creator.load_cluster(cluster_name)
if self.params.update:
cluster.update()
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Listing nodes from cluster %s: %s\n" %
(cluster_name, ex))
return
if self.params.pretty_json:
print(json.dumps(cluster, default=dict, indent=4))
elif self.params.json:
print(json.dumps(cluster, default=dict))
else:
print(cluster_summary(cluster))
for cls in cluster.nodes:
print("%s nodes:" % cls)
print("")
for node in cluster.nodes[cls]:
txt = [" " + i for i in node.pprint().splitlines()]
print(' - ' + str.join("\n", txt)[4:])
print("")
def test_get_cloud_provider_openstack(tmpdir):
wd = tmpdir.strpath
ssh_key_path = os.path.join(wd, 'id_rsa.pem')
with open(ssh_key_path, 'w+') as ssh_key_file:
# don't really care about SSH key, just that the file exists
ssh_key_file.write('')
ssh_key_file.flush()
config_path = os.path.join(wd, 'config.ini')
with open(config_path, 'w+') as config_file:
config_file.write(
"""
[cloud/openstack]
provider = openstack
auth_url = http://openstack.example.com:5000/v2.0
username = ${USER}
password = XXXXXX
project_name = test
"""
+ make_config_snippet("cluster", "example_openstack")
+ make_config_snippet("login", "ubuntu", keyname='test', valid_path=ssh_key_path)
+ make_config_snippet("setup", "slurm_setup_old")
)
creator = make_creator(config_path)
cloud = creator.create_cloud_provider('example_openstack')
from elasticluster.providers.openstack import OpenStackCloudProvider
assert isinstance(cloud, OpenStackCloudProvider)
def execute(self):
"""
Load the cluster and build a GC3Pie configuration snippet.
"""
creator = make_creator(self.params.config,
storage_path=self.params.storage)
cluster_name = self.params.cluster
try:
cluster = creator.load_cluster(cluster_name)
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Listing nodes from cluster %s: %s\n" %
(cluster_name, ex))
return
from elasticluster.gc3pie_config import create_gc3pie_config_snippet
if self.params.append:
path = os.path.expanduser(self.params.append)
try:
fd = open(path, 'a')
fd.write(create_gc3pie_config_snippet(cluster))
fd.close()
except IOError as ex:
log.error("Unable to write configuration to file %s: %s",
path, ex)
else:
print(create_gc3pie_config_snippet(cluster))
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
config = creator.cluster_conf
print("""%d cluster templates found in configuration file.""" % len(config))
templates = config.keys()
for pattern in self.params.clusters:
templates = [t for t in templates if fnmatch(t, pattern)]
if self.params.clusters:
print("""%d cluter templates found matching pattern(s) '%s'""" % (len(templates), str.join(", ", self.params.clusters)))
for template in templates:
try:
cluster = creator.create_cluster(template, template)
print("""
name: %s""" % template)
for nodekind in cluster.nodes:
print("%s nodes: %d" % (
nodekind,
len(cluster.nodes[nodekind])))
except ConfigurationError as ex:
log.error("unable to load cluster `%s`: %s", template, ex)
def test_invalid_ssh_to(tmpdir):
"""
Drop cluster definition with an invalid `ssh_to=` line.
"""
wd = tmpdir.strpath
ssh_key_path = os.path.join(wd, 'id_rsa.pem')
with open(ssh_key_path, 'w+') as ssh_key_file:
# don't really care about SSH key, just that the file exists
ssh_key_file.write('')
ssh_key_file.flush()
config_path = os.path.join(wd, 'config.ini')
with open(config_path, 'w+') as config_file:
config_file.write(
make_config_snippet("cluster", "example_openstack",
'ssh_to=non-existent') +
make_config_snippet("cloud", "openstack") +
make_config_snippet("login",
"ubuntu",
keyname='test_invalid_ssh_to',
valid_path=ssh_key_path) +
make_config_snippet("setup", "slurm_setup_old"))
config_file.flush()
creator = make_creator(config_path)
# ERROR: Cluster `example_openstack` is configured to SSH into nodes of kind `non-existent`, but no such kind is defined
with raises(ConfigurationError):
creator.create_cluster('slurm')
def execute(self):
"""
Load the cluster and build a GC3Pie configuration snippet.
"""
log.warning(
"Command `elasticluster gc3pie-config` is DEPRECATED"
" and will be removed in release 1.4 of ElastiCluster")
creator = make_creator(self.params.config,
storage_path=self.params.storage)
cluster_name = self.params.cluster
try:
cluster = creator.load_cluster(cluster_name)
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Listing nodes from cluster %s: %s", cluster_name, ex)
return
from elasticluster.gc3pie_config import create_gc3pie_config_snippet
if self.params.append:
path = os.path.expanduser(self.params.append)
try:
fd = open(path, 'a')
fd.write(create_gc3pie_config_snippet(cluster))
fd.close()
except IOError as ex:
log.error("Unable to write configuration to file %s: %s",
path, ex)
else:
print(create_gc3pie_config_snippet(cluster))
def execute(self):
"""
Lists all nodes within the specified cluster with certain
information like id and ip.
"""
creator = make_creator(self.params.config,
storage_path=self.params.storage)
cluster_name = self.params.cluster
try:
cluster = creator.load_cluster(cluster_name)
if self.params.update:
cluster.update()
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Listing nodes from cluster %s: %s\n" %
(cluster_name, ex))
return
if self.params.pretty_json:
print(json.dumps(cluster, default=dict, indent=4))
elif self.params.json:
print(json.dumps(cluster, default=dict))
else:
print(cluster_summary(cluster))
for cls in cluster.nodes:
print("%s nodes:" % cls)
print("")
for node in cluster.nodes[cls]:
txt = [" " + i for i in node.pprint().splitlines()]
print(' - ' + str.join("\n", txt)[4:])
print("")
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
cluster_name = self.params.cluster
try:
cluster = creator.load_cluster(cluster_name)
cluster.update()
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Setting up cluster %s: %s\n" %
(cluster_name, ex))
return
if self.params.ssh_to:
try:
nodes = dict((n.name,n) for n in cluster.get_all_nodes())
frontend = nodes[self.params.ssh_to]
except KeyError:
raise ValueError(
"Hostname %s not found in cluster %s" % (self.params.ssh_to, cluster_name))
else:
frontend = cluster.get_frontend_node()
host = frontend.connection_ip()
username = frontend.image_user
knownhostsfile = cluster.known_hosts_file if cluster.known_hosts_file \
else '/dev/null'
sftp_cmdline = ["sftp",
"-o", "UserKnownHostsFile=%s" % knownhostsfile,
"-o", "StrictHostKeyChecking=yes",
"-o", "IdentityFile=%s" % frontend.user_key_private]
sftp_cmdline.extend(self.params.sftp_args)
sftp_cmdline.append('%s@%s' % (username, host))
os.execlp("sftp", *sftp_cmdline)
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
cluster_name = self.params.cluster
try:
cluster = creator.load_cluster(cluster_name)
cluster.update()
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Setting up cluster %s: %s\n" %
(cluster_name, ex))
return
if self.params.ssh_to:
try:
nodes = dict((n.name,n) for n in cluster.get_all_nodes())
frontend = nodes[self.params.ssh_to]
except KeyError:
raise ValueError(
"Hostname %s not found in cluster %s" % (self.params.ssh_to, cluster_name))
else:
frontend = cluster.get_frontend_node()
try:
# ensure we can connect to the host
if not frontend.preferred_ip:
# Ensure we can connect to the node, and save the value of `preferred_ip`
ssh = frontend.connect(keyfile=cluster.known_hosts_file)
if ssh:
ssh.close()
cluster.repository.save_or_update(cluster)
except NodeNotFound as ex:
log.error("Unable to connect to the frontend node: %s" % str(ex))
sys.exit(1)
host = frontend.connection_ip()
# check for nonstandard port, either IPv4 or IPv6
addr = host
port = str(SSH_PORT)
if ':' in host:
match = IPV6_RE.match(host)
if match:
addr = match.groups()[0]
port = match.groups()[1]
else:
addr, _, port = host.partition(':')
username = frontend.image_user
knownhostsfile = cluster.known_hosts_file if cluster.known_hosts_file \
else '/dev/null'
ssh_cmdline = ["ssh",
"-i", frontend.user_key_private,
"-o", "UserKnownHostsFile=%s" % knownhostsfile,
"-o", "StrictHostKeyChecking=yes",
"-p", port,
'%s@%s' % (username, addr)]
ssh_cmdline.extend(self.params.ssh_args)
log.debug("Running command `%s`" % str.join(' ', ssh_cmdline))
os.execlp("ssh", *ssh_cmdline)
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
cluster_name = self.params.cluster
try:
cluster = creator.load_cluster(cluster_name)
cluster.update()
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Setting up cluster %s: %s\n" %
(cluster_name, ex))
return
if self.params.ssh_to:
try:
nodes = dict((n.name,n) for n in cluster.get_all_nodes())
frontend = nodes[self.params.ssh_to]
except KeyError:
raise ValueError(
"Hostname %s not found in cluster %s" % (self.params.ssh_to, cluster_name))
else:
frontend = cluster.get_frontend_node()
try:
# ensure we can connect to the host
if not frontend.preferred_ip:
# Ensure we can connect to the node, and save the value of `preferred_ip`
ssh = frontend.connect(keyfile=cluster.known_hosts_file)
if ssh:
ssh.close()
cluster.repository.save_or_update(cluster)
except NodeNotFound as ex:
log.error("Unable to connect to the frontend node: %s" % str(ex))
sys.exit(1)
host = frontend.connection_ip()
# check for nonstandard port, either IPv4 or IPv6
addr = host
port = str(SSH_PORT)
if ':' in host:
match = IPV6_RE.match(host)
if match:
addr = match.groups()[0]
port = match.groups()[1]
else:
addr, _, port = host.partition(':')
username = frontend.image_user
knownhostsfile = cluster.known_hosts_file if cluster.known_hosts_file \
else '/dev/null'
ssh_cmdline = ["ssh",
"-i", frontend.user_key_private,
"-o", "UserKnownHostsFile=%s" % knownhostsfile,
"-o", "StrictHostKeyChecking=yes",
"-p", port,
'%s@%s' % (username, addr)]
ssh_cmdline.extend(self.params.ssh_args)
log.debug("Running command `%s`" % str.join(' ', ssh_cmdline))
os.execlp("ssh", *ssh_cmdline)
def execute(self):
"""
Starts a new cluster.
"""
cluster_template = self.params.cluster
if self.params.cluster_name:
cluster_name = self.params.cluster_name
else:
cluster_name = self.params.cluster
creator = make_creator(self.params.config,
storage_path=self.params.storage)
# overwrite configuration
cluster_nodes_conf = creator.cluster_conf[cluster_template]['nodes']
for kind, num in self.params.nodes_override.iteritems():
if kind not in cluster_nodes_conf:
raise ConfigurationError(
"No node group `{kind}` defined"
" in cluster template `{template}`"
.format(kind=kind, template=cluster_template))
cluster_nodes_conf[kind]['num'] = num
# First, check if the cluster is already created.
try:
cluster = creator.load_cluster(cluster_name)
except ClusterNotFound:
try:
cluster = creator.create_cluster(
cluster_template, cluster_name)
except ConfigurationError as err:
log.error("Starting cluster %s: %s", cluster_template, err)
return
try:
print("Starting cluster `{0}` with:".format(cluster.name))
for cls in cluster.nodes:
print("* {0:d} {1} nodes.".format(len(cluster.nodes[cls]), cls))
print("(This may take a while...)")
min_nodes = dict((kind, cluster_nodes_conf[kind]['min_num'])
for kind in cluster_nodes_conf)
cluster.start(min_nodes=min_nodes)
if self.params.no_setup:
print("NOT configuring the cluster as requested.")
else:
print("Configuring the cluster.")
print("(this too may take a while...)")
ret = cluster.setup()
if ret:
print("Your cluster is ready!")
else:
print("\nWARNING: YOUR CLUSTER IS NOT READY YET!")
print(cluster_summary(cluster))
except (KeyError, ImageError, SecurityGroupError, ClusterError) as err:
log.error("Could not start cluster `%s`: %s", cluster.name, err)
raise
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
cluster_name = self.params.cluster
try:
cluster = creator.load_cluster(cluster_name)
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Setting up cluster %s: %s", cluster_name, ex)
return
# XXX: the default value of `self.params.ssh_to` should = the
# default value for `ssh_to` in `Cluster.get_ssh_to_node()`
frontend = cluster.get_ssh_to_node(self.params.ssh_to)
# ensure we can connect to the host
try:
if not frontend.connection_ip():
log.info(
"No connection address known for node `%s`,"
" updating list of IP addresses ...", frontend.name)
frontend.update_ips()
log.debug("Checking that SSH connection to node `%s` works..",
frontend.name)
# Ensure we can connect to the node, and save the value of `preferred_ip`
ssh = frontend.connect(keyfile=cluster.known_hosts_file)
if ssh:
ssh.close()
cluster.repository.save_or_update(cluster)
except Exception as err:
log.error("Unable to connect to the frontend node: %s", err)
sys.exit(os.EX_TEMPFAIL)
# now delegate real connection to `ssh`
host = frontend.connection_ip()
if not host:
log.error("No IP address known for node %s", frontend.name)
sys.exit(1)
addr, port = parse_ip_address_and_port(host)
username = frontend.image_user
knownhostsfile = cluster.known_hosts_file if cluster.known_hosts_file \
else '/dev/null'
cmdline = [
self.command, "-i", frontend.user_key_private, "-o",
"UserKnownHostsFile={0}".format(knownhostsfile), "-o",
"StrictHostKeyChecking=yes", "-o", "Port={0:d}".format(port),
'%s@%s' % (username, addr)
]
if cluster.ssh_proxy_command:
cmdline[1:1] = [
'-o',
('ProxyCommand=' + expand_ssh_proxy_command(
cluster.ssh_proxy_command, username, addr, port))
]
cmdline.extend(self.params.cmds)
log.debug("Running command `%s`", ' '.join(cmdline))
os.execlp(self.command, *cmdline)
def execute(self):
"""
Starts a new cluster.
"""
cluster_template = self.params.cluster
if self.params.cluster_name:
cluster_name = self.params.cluster_name
else:
cluster_name = self.params.cluster
creator = make_creator(self.params.config,
storage_path=self.params.storage)
# overwrite configuration
cluster_conf = creator.cluster_conf[cluster_template]
for option, value in self.params.extra_conf.iteritems():
if option in cluster_conf:
cluster_conf[option] = value
# First, check if the cluster is already created.
try:
cluster = creator.load_cluster(cluster_name)
except ClusterNotFound:
try:
cluster = creator.create_cluster(cluster_template,
cluster_name)
except ConfigurationError as err:
log.error("Starting cluster %s: %s", cluster_template, err)
return
try:
print("Starting cluster `{0}` with:".format(cluster.name))
for cls in cluster.nodes:
print("* {0:d} {1} nodes.".format(len(cluster.nodes[cls]),
cls))
print("(This may take a while...)")
min_nodes = dict((k[:-len('_nodes_min')], int(v))
for k, v in cluster_conf.iteritems()
if k.endswith('_nodes_min'))
cluster.start(min_nodes=min_nodes)
if self.params.no_setup:
print("NOT configuring the cluster as requested.")
else:
print("Configuring the cluster.")
print("(this too may take a while...)")
ret = cluster.setup()
if ret:
print("Your cluster is ready!")
else:
print("\nWARNING: YOUR CLUSTER IS NOT READY YET!")
print(cluster_summary(cluster))
except (KeyError, ImageError, SecurityGroupError, ClusterError) as err:
log.error("Could not start cluster `%s`: %s", cluster.name, err)
raise
def execute(self):
"""
Starts a new cluster.
"""
cluster_template = self.params.cluster
if self.params.cluster_name:
cluster_name = self.params.cluster_name
else:
cluster_name = self.params.cluster
creator = make_creator(self.params.config,
storage_path=self.params.storage)
# overwrite configuration
cluster_conf = creator.cluster_conf[cluster_template]
for option, value in self.params.extra_conf.iteritems():
if option in cluster_conf:
cluster_conf[option] = value
# First, check if the cluster is already created.
try:
cluster = creator.load_cluster(cluster_name)
except ClusterNotFound:
try:
cluster = creator.create_cluster(
cluster_template, cluster_name)
except ConfigurationError as err:
log.error("Starting cluster %s: %s", cluster_template, err)
return
try:
print("Starting cluster `{0}` with:".format(cluster.name))
for cls in cluster.nodes:
print("* {0:d} {1} nodes.".format(len(cluster.nodes[cls]), cls))
print("(This may take a while...)")
min_nodes = dict(
(k[:-len('_nodes_min')], int(v)) for k, v in cluster_conf.iteritems() if
k.endswith('_nodes_min'))
cluster.start(min_nodes=min_nodes)
if self.params.no_setup:
print("NOT configuring the cluster as requested.")
else:
print("Configuring the cluster.")
print("(this too may take a while...)")
ret = cluster.setup()
if ret:
print("Your cluster is ready!")
else:
print("\nWARNING: YOUR CLUSTER IS NOT READY YET!")
print(cluster_summary(cluster))
except (KeyError, ImageError, SecurityGroupError, ClusterError) as err:
log.error("Could not start cluster `%s`: %s", cluster.name, err)
raise
def execute(self):
"""Resume the cluster if it is paused."""
cluster_name = self.params.cluster
creator = make_creator(self.params.config,
storage_path=self.params.storage)
try:
cluster = creator.load_cluster(cluster_name)
except (ClusterNotFound, ConfigurationError) as e:
log.error("Cannot load cluster `%s`: %s", cluster_name, e)
return os.EX_NOINPUT
print("Resuming cluster `%s` ..." % cluster_name)
cluster.resume()
def execute(self):
"""Resume the cluster if it is paused."""
cluster_name = self.params.cluster
creator = make_creator(self.params.config,
storage_path=self.params.storage)
try:
cluster = creator.load_cluster(cluster_name)
except (ClusterNotFound, ConfigurationError) as e:
log.error("Cannot load cluster `%s`: %s", cluster_name, e)
return os.EX_NOINPUT
print("Resuming cluster `%s` ..." % cluster_name)
cluster.resume()
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
cluster_name = self.params.cluster
try:
cluster = creator.load_cluster(cluster_name)
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Setting up cluster %s: %s", cluster_name, ex)
return
# XXX: the default value of `self.params.ssh_to` should = the
# default value for `ssh_to` in `Cluster.get_ssh_to_node()`
frontend = cluster.get_ssh_to_node(self.params.ssh_to)
log.debug("Updating the ip addresses of `%s`.", frontend.name)
frontend.update_ips()
# ensure we can connect to the host
try:
if not frontend.preferred_ip:
# Ensure we can connect to the node, and save the value of `preferred_ip`
ssh = frontend.connect(keyfile=cluster.known_hosts_file)
if ssh:
ssh.close()
cluster.repository.save_or_update(cluster)
except NodeNotFound as ex:
log.error("Unable to connect to the frontend node: %s", ex)
sys.exit(1)
# now delegate real connection to `ssh`
host = frontend.connection_ip()
if not host:
log.error("No IP address known for node %s", frontend.name)
sys.exit(1)
addr, port = parse_ip_address_and_port(host)
username = frontend.image_user
knownhostsfile = cluster.known_hosts_file if cluster.known_hosts_file \
else '/dev/null'
cmdline = [self.command,
"-i", frontend.user_key_private,
"-o", "UserKnownHostsFile={0}".format(knownhostsfile),
"-o", "StrictHostKeyChecking=yes",
"-o", "Port={0:d}".format(port),
'%s@%s' % (username, addr)]
if cluster.ssh_proxy_command:
cmdline[1:1] = [
'-o', ('ProxyCommand=' +
expand_ssh_proxy_command(
cluster.ssh_proxy_command,
username, addr, port))]
cmdline.extend(self.params.cmds)
log.debug("Running command `%s`", ' '.join(cmdline))
os.execlp(self.command, *cmdline)
def execute(self):
"""Pause the cluster if it is running."""
cluster_name = self.params.cluster
creator = make_creator(self.params.config,
storage_path=self.params.storage)
try:
cluster = creator.load_cluster(cluster_name)
except (ClusterNotFound, ConfigurationError) as e:
log.error("Cannot load cluster `%s`: %s", cluster_name, e)
return os.EX_NOINPUT
if not self.params.yes:
confirm_or_abort(
"Do you want really want to pause cluster `{cluster_name}`?"
.format(cluster_name=cluster_name),
msg="Aborting upon user request.")
print("Pausing cluster `%s` ..." % cluster_name)
cluster.pause()
def execute(self):
"""Pause the cluster if it is running."""
cluster_name = self.params.cluster
creator = make_creator(self.params.config,
storage_path=self.params.storage)
try:
cluster = creator.load_cluster(cluster_name)
except (ClusterNotFound, ConfigurationError) as e:
log.error("Cannot load cluster `%s`: %s", cluster_name, e)
return os.EX_NOINPUT
if not self.params.yes:
confirm_or_abort(
"Do you want really want to pause cluster `{cluster_name}`?".
format(cluster_name=cluster_name),
msg="Aborting upon user request.")
print("Pausing cluster `%s` ..." % cluster_name)
cluster.pause()
def test_issue_376(tmpdir):
wd = tmpdir.strpath
ssh_key_path = os.path.join(wd, 'id_rsa.pem')
with open(ssh_key_path, 'w+') as ssh_key_file:
# don't really care about SSH key, just that the file exists
ssh_key_file.write('')
ssh_key_file.flush()
config_path = os.path.join(wd, 'config.ini')
with open(config_path, 'w+') as config_file:
config_file.write(
# reported by @marcbrisson in issue #376
"""
[cluster/slurm]
cloud=google
login=ubuntu
setup=slurm_setup
security_group=default
image_id=https://www.googleapis.com/compute/v1/projects/jda-labs---decision-science-01/global/images/image-python-ubuntu
flavor=n1-standard-1
master_nodes=1
worker_nodes=4
ssh_to=master
image_userdata=
boot_disk_size=20
[cluster/slurm/master]
flavor=n1-standard-2
boot_disk_size=100
""" + make_config_snippet("cloud", "google") +
make_config_snippet("login",
"ubuntu",
keyname='test_issue_376',
valid_path=ssh_key_path) +
make_config_snippet("setup", "slurm_setup"))
config_file.flush()
creator = make_creator(config_path)
cluster = creator.create_cluster('slurm')
# "master" nodes take values from their specific config section
assert cluster.nodes['master'][0].flavor == 'n1-standard-2'
assert cluster.nodes['master'][0].extra['boot_disk_size'] == '100'
# "worker" nodes take values from the cluster defaults
assert cluster.nodes['worker'][0].flavor == 'n1-standard-1'
# FIXME: Actually, does this imply that the `boot_disk_size` value
# defined at cluster level is not propagated to "worker" nodes?
assert 'boot_disk_size' not in cluster.nodes['worker'][0].extra
def test_pr_378(tmpdir):
wd = tmpdir.strpath
config_path = os.path.join(wd, 'config.ini')
with open(config_path, 'w+') as config_file:
config_file.write(
make_config_snippet("cloud", "google")
# reported by @ikhaja in PR #378
+ """
[login/google]
image_user=my_username
image_user_sudo=root
image_sudo=True
user_key_name=elasticluster
user_key_private=~/.ssh/google_compute_engine
user_key_public=~/.ssh/google_compute_engine.pub
"""
# FIXME: the std `cluster/*` snippet cannot set `login=` and `cloud=`
+ """
[cluster/slurm]
cloud=google
login=google
setup=slurm_setup
security_group=default
image_id=https://www.googleapis.com/compute/v1/projects/jda-labs---decision-science-01/global/images/image-python-ubuntu
flavor=n1-standard-1
master_nodes=1
worker_nodes=4
ssh_to=master
"""
+ make_config_snippet("setup", "slurm_setup")
)
config_file.flush()
with patch('os.path.expanduser') as expanduser:
# since `os.path.expanduser` is called from within
# `_expand_config_file_list()` we need to provide the right return
# value for it, as non-existent files will be removed from the list
expanduser.return_value = config_path
creator = make_creator(config_path)
# check that `os.expanduser` has been called on the `user_key_*` values
expanduser.assert_any_call('~/.ssh/google_compute_engine.pub')
expanduser.assert_any_call('~/.ssh/google_compute_engine')
# check that actual configured values have been expanded
cluster = creator.create_cluster("slurm")
assert os.path.isabs(cluster.user_key_public)
assert os.path.isabs(cluster.user_key_private)
def test_pr_378(tmpdir):
wd = tmpdir.strpath
config_path = os.path.join(wd, 'config.ini')
with open(config_path, 'w+') as config_file:
config_file.write(
make_config_snippet("cloud", "google")
# reported by @ikhaja in PR #378
+ """
[login/google]
image_user=my_username
image_user_sudo=root
image_sudo=True
user_key_name=elasticluster
user_key_private=~/.ssh/google_compute_engine
user_key_public=~/.ssh/google_compute_engine.pub
"""
# FIXME: the std `cluster/*` snippet cannot set `login=` and `cloud=`
+ """
[cluster/slurm]
cloud=google
login=google
setup=slurm_setup
security_group=default
image_id=https://www.googleapis.com/compute/v1/projects/jda-labs---decision-science-01/global/images/image-python-ubuntu
flavor=n1-standard-1
master_nodes=1
worker_nodes=4
ssh_to=master
"""
+ make_config_snippet("setup", "slurm_setup")
)
config_file.flush()
with patch('os.path.expanduser') as expanduser:
# since `os.path.expanduser` is called from within
# `_expand_config_file_list()` we need to provide the right return
# value for it, as non-existent files will be removed from the list
expanduser.return_value = config_path
creator = make_creator(config_path)
# check that `os.expanduser` has been called on the `user_key_*` values
expanduser.assert_any_call('~/.ssh/google_compute_engine.pub')
expanduser.assert_any_call('~/.ssh/google_compute_engine')
# check that actual configured values have been expanded
cluster = creator.create_cluster("slurm")
assert os.path.isabs(cluster.user_key_public)
assert os.path.isabs(cluster.user_key_private)
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
cluster_name = self.params.clustername
try:
cluster = creator.load_cluster(cluster_name)
cluster.update()
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Setting up cluster %s: %s", cluster_name, ex)
return
# XXX: the default value of `self.params.ssh_to` should = the
# default value for `ssh_to` in `Cluster.get_ssh_to_node()`
frontend = cluster.get_ssh_to_node(self.params.ssh_to)
# ensure we can connect to the host
try:
if not frontend.preferred_ip:
# Ensure we can connect to the node, and save the value of `preferred_ip`
ssh = frontend.connect(keyfile=cluster.known_hosts_file)
if ssh:
ssh.close()
cluster.repository.save_or_update(cluster)
except NodeNotFound as ex:
log.error("Unable to connect to the frontend node: %s", ex)
sys.exit(1)
# now delegate real connection to `ssh`
host = frontend.connection_ip()
if not host:
log.error("No IP address known for node %s", frontend.name)
sys.exit(1)
addr, port = parse_ip_address_and_port(host)
username = frontend.image_user
knownhostsfile = cluster.known_hosts_file if cluster.known_hosts_file \
else '/dev/null'
ssh_cmdline = [
"ssh", "-i", frontend.user_key_private, "-o",
"UserKnownHostsFile={0}".format(knownhostsfile), "-o",
"StrictHostKeyChecking=no", "-p", "{0:d}".format(port),
'%s@%s' % (username, addr)
]
ssh_cmdline.extend(self.params.ssh_args)
log.debug("Running command `%s`", str.join(' ', ssh_cmdline))
os.execlp("ssh", *ssh_cmdline)
def test_issue_376(tmpdir):
wd = tmpdir.strpath
ssh_key_path = os.path.join(wd, 'id_rsa.pem')
with open(ssh_key_path, 'w+') as ssh_key_file:
# don't really care about SSH key, just that the file exists
ssh_key_file.write('')
ssh_key_file.flush()
config_path = os.path.join(wd, 'config.ini')
with open(config_path, 'w+') as config_file:
config_file.write(
# reported by @marcbrisson in issue #376
"""
[cluster/slurm]
cloud=google
login=ubuntu
setup=slurm_setup
security_group=default
image_id=https://www.googleapis.com/compute/v1/projects/jda-labs---decision-science-01/global/images/image-python-ubuntu
flavor=n1-standard-1
master_nodes=1
worker_nodes=4
ssh_to=master
image_userdata=
boot_disk_size=20
[cluster/slurm/master]
flavor=n1-standard-2
boot_disk_size=100
"""
+ make_config_snippet("cloud", "google")
+ make_config_snippet("login", "ubuntu", keyname='test_issue_376', valid_path=ssh_key_path)
+ make_config_snippet("setup", "slurm_setup")
)
config_file.flush()
creator = make_creator(config_path)
cluster = creator.create_cluster('slurm')
# "master" nodes take values from their specific config section
assert cluster.nodes['master'][0].flavor == 'n1-standard-2'
assert cluster.nodes['master'][0].extra['boot_disk_size'] == '100'
# "worker" nodes take values from the cluster defaults
assert cluster.nodes['worker'][0].flavor == 'n1-standard-1'
# FIXME: Actually, does this imply that the `boot_disk_size` value
# defined at cluster level is not propagated to "worker" nodes?
assert 'boot_disk_size' not in cluster.nodes['worker'][0].extra
def test_gce_accelerator1(tmpdir):
wd = tmpdir.strpath
ssh_key_path = os.path.join(wd, 'id_rsa.pem')
with open(ssh_key_path, 'w+') as ssh_key_file:
# don't really care about SSH key, just that the file exists
ssh_key_file.write('')
ssh_key_file.flush()
config_path = os.path.join(wd, 'config.ini')
with open(config_path, 'w+') as config_file:
config_file.write(
make_config_snippet("cluster", "example_google",
'[cluster/example_google/misc]',
'accelerator_count=1')
# # ask for one GPU
# """
# [cluster/slurm]
# cloud=google
# login=ubuntu
# setup=slurm_setup
# security_group=default
# image_id=**not important**
# flavor=n1-standard-1
# master_nodes=1
# worker_nodes=4
# ssh_to=master
# [cluster/slurm/worker]
# accelerator_count=1
# """
+ make_config_snippet("cloud", "google") +
make_config_snippet("login",
"ubuntu",
keyname='test_gce_accelerator',
valid_path=ssh_key_path) +
make_config_snippet("setup", "misc_setup"))
config_file.flush()
creator = make_creator(config_path)
cluster = creator.create_cluster('example_google')
# "master" nodes take values from their specific config section
#assert cluster.nodes['master'][0].extra['accelerator_count'] == 0
# "worker" nodes take values from the cluster defaults
assert 'accelerator_count' in cluster.nodes['misc'][0].extra
assert cluster.nodes['misc'][0].extra['accelerator_count'] == 1
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
cluster_name = self.params.cluster
print("Updating cluster `%s`..." % cluster_name)
try:
cluster = creator.load_cluster(cluster_name)
cluster.update()
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Setting up cluster %s: %s\n" % (cluster_name, ex))
return
print("Configuring cluster `%s`..." % cluster_name)
ret = cluster.setup(self.params.extra)
if ret:
print("Your cluster is ready!")
else:
print("\nWARNING: YOUR CLUSTER IS NOT READY YET!")
print(cluster_summary(cluster))
def test_gce_accelerator1(tmpdir):
wd = tmpdir.strpath
ssh_key_path = os.path.join(wd, 'id_rsa.pem')
with open(ssh_key_path, 'w+') as ssh_key_file:
# don't really care about SSH key, just that the file exists
ssh_key_file.write('')
ssh_key_file.flush()
config_path = os.path.join(wd, 'config.ini')
with open(config_path, 'w+') as config_file:
config_file.write(
make_config_snippet("cluster", "example_google",
'[cluster/example_google/misc]',
'accelerator_count=1')
# # ask for one GPU
# """
# [cluster/slurm]
# cloud=google
# login=ubuntu
# setup=slurm_setup
# security_group=default
# image_id=**not important**
# flavor=n1-standard-1
# master_nodes=1
# worker_nodes=4
# ssh_to=master
# [cluster/slurm/worker]
# accelerator_count=1
# """
+ make_config_snippet("cloud", "google")
+ make_config_snippet("login", "ubuntu", keyname='test_gce_accelerator', valid_path=ssh_key_path)
+ make_config_snippet("setup", "misc_setup")
)
config_file.flush()
creator = make_creator(config_path)
cluster = creator.create_cluster('example_google')
# "master" nodes take values from their specific config section
#assert cluster.nodes['master'][0].extra['accelerator_count'] == 0
# "worker" nodes take values from the cluster defaults
assert 'accelerator_count' in cluster.nodes['misc'][0].extra
assert cluster.nodes['misc'][0].extra['accelerator_count'] == 1
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
cluster_name = self.params.cluster
print("Updating cluster `%s`..." % cluster_name)
try:
cluster = creator.load_cluster(cluster_name)
cluster.update()
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Setting up cluster %s: %s\n" %
(cluster_name, ex))
return
print("Configuring cluster `%s`..." % cluster_name)
ret = cluster.setup(self.params.extra)
if ret:
print("Your cluster is ready!")
else:
print("\nWARNING: YOUR CLUSTER IS NOT READY YET!")
print(cluster_summary(cluster))
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
cluster_name = self.params.clustername
try:
cluster = creator.load_cluster(cluster_name)
cluster.update()
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Setting up cluster %s: %s", cluster_name, ex)
return
# XXX: the default value of `self.params.ssh_to` should = the
# default value for `ssh_to` in `Cluster.get_ssh_to_node()`
frontend = cluster.get_ssh_to_node(self.params.ssh_to)
host = frontend.connection_ip()
if not host:
log.error("No IP address known for node %s", frontend.name)
sys.exit(1)
addr, port = parse_ip_address_and_port(host)
username = frontend.image_user
knownhostsfile = (cluster.known_hosts_file
if cluster.known_hosts_file else '/dev/null')
sftp_cmdline = [
"sftp",
#"-P", "{0:d}".format(port),
"-o",
"Port={0}".format(port),
"-o",
"UserKnownHostsFile={0}".format(knownhostsfile),
"-o",
"StrictHostKeyChecking=no",
"-o",
"IdentityFile={0}".format(frontend.user_key_private),
]
sftp_cmdline.extend(self.params.sftp_args)
sftp_cmdline.append('{0}@{1}'.format(username, addr))
os.execlp("sftp", *sftp_cmdline)
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
repository = creator.create_repository()
clusters = repository.get_all()
if not clusters:
print("No clusters found.")
else:
print("""
The following clusters have been started.
Please note that there's no guarantee that they are fully configured:
""")
for cluster in sorted(clusters):
print("%s " % cluster.name)
print("-" * len(cluster.name))
print(" name: %s" % cluster.name)
if cluster.name != cluster.template:
print(" template: %s" % cluster.template)
for cls in cluster.nodes:
print(" - %s nodes: %d" % (cls, len(cluster.nodes[cls])))
print("")
def test_gce_accelerator2(tmpdir):
wd = tmpdir.strpath
ssh_key_path = os.path.join(wd, 'id_rsa.pem')
with open(ssh_key_path, 'w+') as ssh_key_file:
# don't really care about SSH key, just that the file exists
ssh_key_file.write('')
ssh_key_file.flush()
config_path = os.path.join(wd, 'config.ini')
with open(config_path, 'w+') as config_file:
config_file.write(
# ask for two GPU on `worker` nodes only
"""
[cluster/test]
cloud=google
login=ubuntu
setup=slurm_setup
security_group=default
image_id=**not important**
flavor=n1-standard-1
master_nodes=1
worker_nodes=4
ssh_to=master
[cluster/test/worker]
accelerator_count=2
""" + make_config_snippet("cloud", "google") +
make_config_snippet("login",
"ubuntu",
keyname='test_gce_accelerator',
valid_path=ssh_key_path) +
make_config_snippet("setup", "slurm_setup"))
config_file.flush()
creator = make_creator(config_path)
cluster = creator.create_cluster('test')
# "master" nodes take values from their specific config section
assert cluster.nodes['master'][0].extra['accelerator_count'] == 0
# "worker" nodes take values from the cluster defaults
assert 'accelerator_count' in cluster.nodes['worker'][0].extra
assert cluster.nodes['worker'][0].extra['accelerator_count'] == 2
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
repository = creator.create_repository()
clusters = repository.get_all()
if not clusters:
print("No clusters found.")
else:
print("""
The following clusters have been started.
Please note that there's no guarantee that they are fully configured:
""")
for cluster in sorted(clusters):
print("%s " % cluster.name)
print("-" * len(cluster.name))
print(" name: %s" % cluster.name)
if cluster.name != cluster.template:
print(" template: %s" % cluster.template)
for cls in cluster.nodes:
print(" - %s nodes: %d" % (cls, len(cluster.nodes[cls])))
print("")
def test_issue_415(tmpdir):
"""
Drop cluster definition if not all node kinds are present in the `setup/*` section.
"""
wd = tmpdir.strpath
ssh_key_path = os.path.join(wd, 'id_rsa.pem')
with open(ssh_key_path, 'w+') as ssh_key_file:
# don't really care about SSH key, just that the file exists
ssh_key_file.write('')
ssh_key_file.flush()
config_path = os.path.join(wd, 'config.ini')
with open(config_path, 'w+') as config_file:
config_file.write(
# reported by @dirkpetersen in issue #415
"""
[cluster/gce-slurm]
cloud=google
#login=ubuntu
login=google
setup=slurm_setup_old
security_group=default
image_id=ubuntu-1604-xenial-v20170307
flavor=n1-standard-1
frontend_nodes=1
worker_nodes=2
image_userdata=
ssh_to=frontend
""" + make_config_snippet("cloud", "google") +
make_config_snippet("login",
"ubuntu",
keyname='test_issue_415',
valid_path=ssh_key_path) +
make_config_snippet("setup", "slurm_setup_old"))
config_file.flush()
creator = make_creator(config_path)
# ERROR: Configuration section `cluster/gce-slurm` references non-existing login section `google`. Dropping cluster definition.
with raises(ConfigurationError):
creator.create_cluster('gce-slurm')
def test_gce_accelerator2(tmpdir):
wd = tmpdir.strpath
ssh_key_path = os.path.join(wd, 'id_rsa.pem')
with open(ssh_key_path, 'w+') as ssh_key_file:
# don't really care about SSH key, just that the file exists
ssh_key_file.write('')
ssh_key_file.flush()
config_path = os.path.join(wd, 'config.ini')
with open(config_path, 'w+') as config_file:
config_file.write(
# ask for two GPU on `worker` nodes only
"""
[cluster/test]
cloud=google
login=ubuntu
setup=slurm_setup
security_group=default
image_id=**not important**
flavor=n1-standard-1
master_nodes=1
worker_nodes=4
ssh_to=master
[cluster/test/worker]
accelerator_count=2
"""
+ make_config_snippet("cloud", "google")
+ make_config_snippet("login", "ubuntu", keyname='test_gce_accelerator', valid_path=ssh_key_path)
+ make_config_snippet("setup", "slurm_setup")
)
config_file.flush()
creator = make_creator(config_path)
cluster = creator.create_cluster('test')
# "master" nodes take values from their specific config section
assert cluster.nodes['master'][0].extra['accelerator_count'] == 0
# "worker" nodes take values from the cluster defaults
assert 'accelerator_count' in cluster.nodes['worker'][0].extra
assert cluster.nodes['worker'][0].extra['accelerator_count'] == 2
def test_issue_415(tmpdir):
"""
Drop cluster definition if not all node kinds are present in the `setup/*` section.
"""
wd = tmpdir.strpath
ssh_key_path = os.path.join(wd, 'id_rsa.pem')
with open(ssh_key_path, 'w+') as ssh_key_file:
# don't really care about SSH key, just that the file exists
ssh_key_file.write('')
ssh_key_file.flush()
config_path = os.path.join(wd, 'config.ini')
with open(config_path, 'w+') as config_file:
config_file.write(
# reported by @dirkpetersen in issue #415
"""
[cluster/gce-slurm]
cloud=google
#login=ubuntu
login=google
setup=slurm_setup_old
security_group=default
image_id=ubuntu-1604-xenial-v20170307
flavor=n1-standard-1
frontend_nodes=1
worker_nodes=2
image_userdata=
ssh_to=frontend
"""
+ make_config_snippet("cloud", "google")
+ make_config_snippet("login", "ubuntu", keyname='test_issue_415', valid_path=ssh_key_path)
+ make_config_snippet("setup", "slurm_setup_old")
)
config_file.flush()
creator = make_creator(config_path)
# ERROR: Configuration section `cluster/gce-slurm` references non-existing login section `google`. Dropping cluster definition.
with raises(ConfigurationError):
creator.create_cluster('gce-slurm')
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
# Get current cluster configuration
cluster_name = self.params.cluster
try:
cluster = creator.load_cluster(cluster_name)
cluster.update()
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Error loading cluster %s: %s\n" %
(cluster_name, ex))
return
# Find the node to remove.
try:
node = cluster.get_node_by_name(self.params.node)
except NodeNotFound:
log.error("Node %s not found in cluster %s" % (
self.params.node, self.params.cluster))
sys.exit(1)
# Run
if not self.params.yes:
confirm_or_abort("Do you really want to remove node `{}`?"
.format(node.name),
msg="Aborting upon user request.")
cluster.remove_node(node, stop=True)
print("Node %s removed" % node.name)
if self.params.no_setup:
print("NOT reconfiguring the cluster as requested.")
else:
print("Reconfiguring the cluster.")
cluster.setup()
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
# Get current cluster configuration
cluster_name = self.params.cluster
try:
cluster = creator.load_cluster(cluster_name)
cluster.update()
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Error loading cluster %s: %s\n" %
(cluster_name, ex))
return
# Find the node to remove.
try:
node = cluster.get_node_by_name(self.params.node)
except NodeNotFound:
log.error("Node %s not found in cluster %s" % (
self.params.node, self.params.cluster))
sys.exit(1)
# Run
if not self.params.yes:
confirm_or_abort("Do you really want to remove node `{}`?"
.format(node.name),
msg="Aborting upon user request.")
cluster.remove_node(node, stop=True)
print("Node %s removed" % node.name)
if self.params.no_setup:
print("NOT reconfiguring the cluster as requested.")
else:
print("Reconfiguring the cluster.")
cluster.setup()
def test_default_setup_provider_is_ansible(tmpdir):
wd = tmpdir.strpath
ssh_key_path = os.path.join(wd, 'id_rsa.pem')
with open(ssh_key_path, 'w+') as ssh_key_file:
# don't really care about SSH key, just that the file exists
ssh_key_file.write('')
ssh_key_file.flush()
config_path = os.path.join(wd, 'config.ini')
with open(config_path, 'w+') as config_file:
config_file.write(
make_config_snippet("cloud", "openstack")
+ make_config_snippet("cluster", "example_openstack", 'setup=setup_no_ansible')
+ make_config_snippet("login", "ubuntu", keyname='test', valid_path=ssh_key_path)
# *note:* no `provider=` line here
+ """
[setup/setup_no_ansible]
frontend_groups = slurm_master
compute_groups = slurm_worker
"""
)
creator = make_creator(config_path)
setup = creator.create_setup_provider('example_openstack')
from elasticluster.providers.ansible_provider import AnsibleSetupProvider
assert isinstance(setup, AnsibleSetupProvider)
def test_invalid_ssh_to(tmpdir):
"""
Drop cluster definition with an invalid `ssh_to=` line.
"""
wd = tmpdir.strpath
ssh_key_path = os.path.join(wd, 'id_rsa.pem')
with open(ssh_key_path, 'w+') as ssh_key_file:
# don't really care about SSH key, just that the file exists
ssh_key_file.write('')
ssh_key_file.flush()
config_path = os.path.join(wd, 'config.ini')
with open(config_path, 'w+') as config_file:
config_file.write(
make_config_snippet("cluster", "example_openstack", 'ssh_to=non-existent')
+ make_config_snippet("cloud", "openstack")
+ make_config_snippet("login", "ubuntu",
keyname='test_invalid_ssh_to', valid_path=ssh_key_path)
+ make_config_snippet("setup", "slurm_setup_old")
)
config_file.flush()
creator = make_creator(config_path)
# ERROR: Cluster `example_openstack` is configured to SSH into nodes of kind `non-existent`, but no such kind is defined
with raises(ConfigurationError):
creator.create_cluster('slurm')
def test_default_setup_provider_is_ansible(tmpdir):
wd = tmpdir.strpath
ssh_key_path = os.path.join(wd, 'id_rsa.pem')
with open(ssh_key_path, 'w+') as ssh_key_file:
# don't really care about SSH key, just that the file exists
ssh_key_file.write('')
ssh_key_file.flush()
config_path = os.path.join(wd, 'config.ini')
with open(config_path, 'w+') as config_file:
config_file.write(
make_config_snippet("cloud", "openstack")
+ make_config_snippet("cluster", "example_openstack", 'setup=setup_no_ansible')
+ make_config_snippet("login", "ubuntu", keyname='test', valid_path=ssh_key_path)
# *note:* no `provider=` line here
+ """
[setup/setup_no_ansible]
frontend_groups = slurm_master
compute_groups = slurm_worker
"""
)
creator = make_creator(config_path)
setup = creator.create_setup_provider('example_openstack')
from elasticluster.providers.ansible_provider import AnsibleSetupProvider
assert isinstance(setup, AnsibleSetupProvider)
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
repo = creator.create_repository()
tmpdir = tempfile.mkdtemp()
log.debug("Using temporary directory %s" % tmpdir)
tmpconf = make_creator(self.params.config, storage_path=tmpdir)
tmprepo = tmpconf.create_repository()
rc = 0
# Read the zip file.
try:
with ZipFile(self.params.file, 'r') as zipfile:
# Find main cluster file
# create cluster object from it
log.debug("ZIP file %s opened" % self.params.file)
cluster = None
zipfile.extractall(tmpdir)
newclusters = tmprepo.get_all()
cluster = newclusters[0]
cur_clusternames = [c.name for c in repo.get_all()]
oldname = cluster.name
newname = self.params.rename
if self.params.rename:
cluster.name = self.params.rename
for node in cluster.get_all_nodes():
node.cluster_name = cluster.name
if cluster.name in cur_clusternames:
raise Exception(
"A cluster with name %s already exists. Use "
"option --rename to rename the cluster to be "
"imported." % cluster.name)
# Save the cluster in the new position
cluster.repository = repo
repo.save_or_update(cluster)
dest = cluster.repository.storage_path
# Copy the known hosts
srcfile = os.path.join(tmpdir, oldname + '.known_hosts')
destfile = os.path.join(dest, cluster.name + '.known_hosts')
shutil.copy(srcfile, destfile)
# Copy the ssh keys, if present
for attr in ('user_key_public', 'user_key_private'):
keyfile = getattr(cluster, attr)
keybase = os.path.basename(keyfile)
srcfile = os.path.join(tmpdir, keybase)
if os.path.isfile(srcfile):
log.info("Importing key file %s" % keybase)
destfile = os.path.join(dest, keybase)
shutil.copy(srcfile, destfile)
setattr(cluster, attr, destfile)
for node in cluster.get_all_nodes():
nodekeyfile = getattr(node, attr)
# Check if it's different from the main key
if nodekeyfile != keyfile \
and os.path.isfile(nodekeyfile):
destdir = os.path.join(dest, cluster.name,
node.kind, node.name)
nodekeybase = os.path.basename(nodekeyfile)
log.info("Importing key file %s for node %s" %
(nodekeybase, node.name))
if not os.path.isdir(destdir):
os.makedirs(destdir)
# Path to key in zip file
srcfile = os.path.join(tmpdir, oldname, node.kind,
node.name, nodekeybase)
destfile = os.path.join(destdir, nodekeybase)
shutil.copy(srcfile, destfile)
# Always save the correct destfile
setattr(node, attr, destfile)
repo.save_or_update(cluster)
if not cluster:
log.error("ZIP file %s does not contain a valid cluster." %
self.params.file)
rc = 2
# Check if a cluster already exists.
# if not, unzip the needed files, and update ssh key path if needed.
except Exception as ex:
log.error("Unable to import from zipfile %s: %s" %
(self.params.file, ex))
rc = 1
finally:
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
log.info("Cleaning up directory %s" % tmpdir)
if rc == 0:
print("Successfully imported cluster from ZIP %s to %s" %
(self.params.file, repo.storage_path))
sys.exit(rc)
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
try:
cluster = creator.load_cluster(self.params.cluster)
except ClusterNotFound:
log.error("Cluster `%s` not found in storage dir %s." %
(self.params.cluster, self.params.storage))
sys.exit(1)
if os.path.exists(self.params.zipfile) and not self.params.overwrite:
log.error("ZIP file `%s` already exists." % self.params.zipfile)
sys.exit(1)
with ZipFile(self.params.zipfile, 'w') as zipfile:
# The root of the zip file will contain:
# * the storage file
# * the known_hosts file
# * ssh public and prived keys, if --save-keys is used
#
# it will NOT contain the ansible inventory file, as this
# is automatically created when needed.
#
# Also, if --save-keys is used and there is an host with a
# different ssh private/public key than the default, they
# will be saved in:
#
# ./<cluster>/<group>/<nodename>/
#
def verbose_add(fname, basedir='', comment=None):
zipname = basedir + os.path.basename(fname)
log.info("Adding '%s' as '%s'" % (fname, zipname))
zipfile.write(fname, zipname)
if comment:
info = zipfile.getinfo(zipname)
info.comment = comment
try:
verbose_add(cluster.storage_file, comment='cluster-file')
verbose_add(cluster.known_hosts_file, comment='known_hosts')
if self.params.save_keys:
# that's sensible stuff, let's ask permission.
print("""
==========================
WARNING! WARNING! WARNING!
==========================
You are about to add your SSH *private* key to the
ZIP archive. These are sensible data: anyone with
access to the ZIP file will have access to any host
where this private key has been deployed.
""")
confirm_or_abort(
"Are you sure you still want to copy them?",
msg="Aborting upon user request.")
# Also save all the public and private keys we can find.
# Cluster keys
verbose_add(cluster.user_key_public)
verbose_add(cluster.user_key_private)
# Node keys, if found
for node in cluster.get_all_nodes():
if node.user_key_public != cluster.user_key_public:
verbose_add(
node.user_key_public, "%s/%s/%s/" %
(cluster.name, node.kind, node.name))
for node in cluster.get_all_nodes():
if node.user_key_private != cluster.user_key_private:
verbose_add(
node.user_key_private, "%s/%s/%s/" %
(cluster.name, node.kind, node.name))
except OSError as ex:
# A file is probably missing!
log.error(
"Fatal error: cannot add file %s to zip archive: %s." %
(ex.filename, ex))
sys.exit(1)
print("Cluster '%s' correctly exported into %s" %
(cluster.name, self.params.zipfile))
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
# Get current cluster configuration
cluster_name = self.params.cluster
template = self.params.template
try:
cluster = creator.load_cluster(cluster_name)
cluster.update()
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Listing nodes from cluster %s: %s\n" %
(cluster_name, ex))
return
for grp in self.params.nodes_to_add:
print("Adding %d %s node(s) to the cluster"
"" % (self.params.nodes_to_add[grp], grp))
# Currently we can't save which template was used to setup a
# cluster, therefore we imply the configuration of the new nodes
# to match already existent nodes in this group. If no node was
# added to this group yet, it will abort and ask for the
# `--template` argument.
# TODO: find a better solution for this problem, it makes things
# complicated for the user
if (not grp in cluster.nodes or not cluster.nodes[grp]) \
and not template:
print "Elasticluster can not infer which template to use for "\
"the new node(s). Please provide the template with " \
"the `-t` or `--template` option"
return
if not template:
sample_node = cluster.nodes[grp][0]
for i in range(self.params.nodes_to_add[grp]):
cluster.add_node(grp,
sample_node.image_id,
sample_node.image_user,
sample_node.flavor,
sample_node.security_group,
image_userdata=sample_node.image_userdata,
**sample_node.extra)
else:
conf = creator.cluster_conf[template]
conf_kind = conf['nodes'][grp]
image_user = conf['login']['image_user']
userdata = conf_kind.get('image_userdata', '')
extra = conf_kind.copy()
extra.pop('image_id', None)
extra.pop('flavor', None)
extra.pop('security_group', None)
extra.pop('image_userdata', None)
for i in range(self.params.nodes_to_add[grp]):
cluster.add_node(grp,
conf_kind['image_id'],
image_user,
conf_kind['flavor'],
conf_kind['security_group'],
image_userdata=userdata,
**extra)
for grp in self.params.nodes_to_remove:
n_to_rm = self.params.nodes_to_remove[grp]
print("Removing %d %s node(s) from the cluster."
"" % (n_to_rm, grp))
to_remove = cluster.nodes[grp][-n_to_rm:]
print("The following nodes will be removed from the cluster.")
print(" " + str.join("\n ", [n.name for n in to_remove]))
if not self.params.yes:
confirm_or_abort("Do you really want to remove them?",
msg="Aborting upon user request.")
for node in to_remove:
cluster.nodes[grp].remove(node)
node.stop()
cluster.start()
if self.params.no_setup:
print("NOT configuring the cluster as requested.")
else:
print("Reconfiguring the cluster.")
cluster.setup()
print(cluster_summary(cluster))
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
# Get current cluster configuration
cluster_name = self.params.cluster
template = self.params.template
try:
cluster = creator.load_cluster(cluster_name)
cluster.update()
except (ClusterNotFound, ConfigurationError) as ex:
log.error("Listing nodes from cluster %s: %s\n" %
(cluster_name, ex))
return
for grp in self.params.nodes_to_add:
print("Adding %d %s node(s) to the cluster"
"" % (self.params.nodes_to_add[grp], grp))
# Currently we can't save which template was used to setup a
# cluster, therefore we imply the configuration of the new nodes
# to match already existent nodes in this group. If no node was
# added to this group yet, it will abort and ask for the
# `--template` argument.
# TODO: find a better solution for this problem, it makes things
# complicated for the user
if (not grp in cluster.nodes or not cluster.nodes[grp]) \
and not template:
print "Elasticluster can not infer which template to use for "\
"the new node(s). Please provide the template with " \
"the `-t` or `--template` option"
return
if not template:
sample_node = cluster.nodes[grp][0]
for i in range(self.params.nodes_to_add[grp]):
cluster.add_node(grp,
sample_node.image_id,
sample_node.image_user,
sample_node.flavor,
sample_node.security_group,
image_userdata=sample_node.image_userdata,
**sample_node.extra)
else:
conf = creator.cluster_conf[template]
conf_kind = conf['nodes'][grp]
image_user = conf['login']['image_user']
userdata = conf_kind.get('image_userdata', '')
extra = conf_kind.copy()
extra.pop('image_id', None)
extra.pop('flavor', None)
extra.pop('security_group', None)
extra.pop('image_userdata', None)
for i in range(self.params.nodes_to_add[grp]):
cluster.add_node(grp,
conf_kind['image_id'],
image_user,
conf_kind['flavor'],
conf_kind['security_group'],
image_userdata=userdata,
**extra)
for grp in self.params.nodes_to_remove:
n_to_rm = self.params.nodes_to_remove[grp]
print("Removing %d %s node(s) from the cluster."
"" % (n_to_rm, grp))
to_remove = cluster.nodes[grp][-n_to_rm:]
print("The following nodes will be removed from the cluster.")
print(" " + str.join("\n ", [n.name for n in to_remove]))
if not self.params.yes:
confirm_or_abort("Do you really want to remove them?",
msg="Aborting upon user request.")
for node in to_remove:
cluster.nodes[grp].remove(node)
node.stop()
cluster.start()
if self.params.no_setup:
print("NOT configuring the cluster as requested.")
else:
print("Reconfiguring the cluster.")
cluster.setup()
print(cluster_summary(cluster))
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
repo = creator.create_repository()
tmpdir = tempfile.mkdtemp()
log.debug("Using temporary directory %s" % tmpdir)
tmpconf = make_creator(self.params.config, storage_path=tmpdir)
tmprepo = tmpconf.create_repository()
rc=0
# Read the zip file.
try:
with ZipFile(self.params.file, 'r') as zipfile:
# Find main cluster file
# create cluster object from it
log.debug("ZIP file %s opened" % self.params.file)
cluster = None
zipfile.extractall(tmpdir)
newclusters = tmprepo.get_all()
cluster = newclusters[0]
cur_clusternames = [c.name for c in repo.get_all()]
oldname = cluster.name
newname = self.params.rename
if self.params.rename:
cluster.name = self.params.rename
for node in cluster.get_all_nodes():
node.cluster_name = cluster.name
if cluster.name in cur_clusternames:
raise Exception(
"A cluster with name %s already exists. Use "
"option --rename to rename the cluster to be "
"imported." % cluster.name)
# Save the cluster in the new position
cluster.repository = repo
repo.save_or_update(cluster)
dest = cluster.repository.storage_path
# Copy the known hosts
srcfile = os.path.join(tmpdir, oldname+'.known_hosts')
destfile = os.path.join(dest, cluster.name+'.known_hosts')
shutil.copy(srcfile, destfile)
# Copy the ssh keys, if present
for attr in ('user_key_public', 'user_key_private'):
keyfile = getattr(cluster, attr)
keybase = os.path.basename(keyfile)
srcfile = os.path.join(tmpdir, keybase)
if os.path.isfile(srcfile):
log.info("Importing key file %s" % keybase)
destfile = os.path.join(dest, keybase)
shutil.copy(srcfile, destfile)
setattr(cluster, attr, destfile)
for node in cluster.get_all_nodes():
nodekeyfile = getattr(node, attr)
# Check if it's different from the main key
if nodekeyfile != keyfile \
and os.path.isfile(nodekeyfile):
destdir = os.path.join(dest,
cluster.name,
node.kind,
node.name)
nodekeybase = os.path.basename(nodekeyfile)
log.info("Importing key file %s for node %s" %
(nodekeybase, node.name))
if not os.path.isdir(destdir):
os.makedirs(destdir)
# Path to key in zip file
srcfile = os.path.join(tmpdir,
oldname,
node.kind,
node.name,
nodekeybase)
destfile = os.path.join(destdir, nodekeybase)
shutil.copy(srcfile, destfile)
# Always save the correct destfile
setattr(node, attr, destfile)
repo.save_or_update(cluster)
if not cluster:
log.error("ZIP file %s does not contain a valid cluster."
% self.params.file)
rc = 2
# Check if a cluster already exists.
# if not, unzip the needed files, and update ssh key path if needed.
except Exception as ex:
log.error("Unable to import from zipfile %s: %s"
% (self.params.file, ex))
rc=1
finally:
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
log.info("Cleaning up directory %s" % tmpdir)
if rc == 0:
print("Successfully imported cluster from ZIP %s to %s"
% (self.params.file, repo.storage_path))
sys.exit(rc)
def execute(self):
creator = make_creator(self.params.config,
storage_path=self.params.storage)
try:
cluster = creator.load_cluster(self.params.cluster)
except ClusterNotFound:
log.error("Cluster `%s` not found in storage dir %s."
% (self.params.cluster, self.params.storage))
sys.exit(1)
if os.path.exists(self.params.zipfile) and not self.params.overwrite:
log.error("ZIP file `%s` already exists." % self.params.zipfile)
sys.exit(1)
with ZipFile(self.params.zipfile, 'w') as zipfile:
# The root of the zip file will contain:
# * the storage file
# * the known_hosts file
# * ssh public and prived keys, if --save-keys is used
#
# it will NOT contain the ansible inventory file, as this
# is automatically created when needed.
#
# Also, if --save-keys is used and there is an host with a
# different ssh private/public key than the default, they
# will be saved in:
#
# ./<cluster>/<group>/<nodename>/
#
def verbose_add(fname, basedir='', comment=None):
zipname = basedir + os.path.basename(fname)
log.info("Adding '%s' as '%s'" % (fname, zipname))
zipfile.write(fname, zipname)
if comment:
info = zipfile.getinfo(zipname)
info.comment = comment
try:
verbose_add(cluster.storage_file, comment='cluster-file')
verbose_add(cluster.known_hosts_file, comment='known_hosts')
if self.params.save_keys:
# that's sensible stuff, let's ask permission.
print("""
==========================
WARNING! WARNING! WARNING!
==========================
You are about to add your SSH *private* key to the
ZIP archive. These are sensible data: anyone with
access to the ZIP file will have access to any host
where this private key has been deployed.
""")
confirm_or_abort(
"Are you sure you still want to copy them?",
msg="Aborting upon user request.")
# Also save all the public and private keys we can find.
# Cluster keys
verbose_add(cluster.user_key_public)
verbose_add(cluster.user_key_private)
# Node keys, if found
for node in cluster.get_all_nodes():
if node.user_key_public != cluster.user_key_public:
verbose_add(node.user_key_public,
"%s/%s/%s/" % (cluster.name,
node.kind,
node.name))
for node in cluster.get_all_nodes():
if node.user_key_private != cluster.user_key_private:
verbose_add(node.user_key_private,
"%s/%s/%s/" % (cluster.name,
node.kind,
node.name))
except OSError as ex:
# A file is probably missing!
log.error("Fatal error: cannot add file %s to zip archive: %s."
% (ex.filename, ex))
sys.exit(1)
print("Cluster '%s' correctly exported into %s" %
(cluster.name, self.params.zipfile))