def _run_elasticluster_cmd(argv):
"""
Run the `elasticluster` command with additional arguments.
Return STDOUT, STDERR, and the process exit status.
"""
with temporary_dir() as tmpdir:
with environment(
HOME=os.getcwd(),
PYTHONWARNINGS=(
# as support for Py2 wanes, we must silence warnings that
# Python 2.7 will no longer be supported, as they make the
# tests fail unnecessarily (functionality is OK, we just get
# some extra lines into STDERR). However,
# `cryptography.utils.CryptographyDeprecationWarning` is a
# subclass of `UserWarning` exactly because
# `DeprecationWarnings` are ignored by default, so we need to
# ignore all `UserWarnings` as well (cannot ignore a non-builtin
# exception class via an environmental variable)
'ignore::DeprecationWarning,ignore::UserWarning' if sys.version_info < (3, 6)
# display all warnings on Py3.6+
else ''),
) as env:
proc = subprocess.Popen(
['elasticluster'] + argv,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True,
shell=False)
stdout, stderr = proc.communicate()
return stdout, stderr, proc.returncode
def _run_command(argv):
"""
Run the `elasticluster` command with additional arguments.
Return STDOUT, STDERR, and the process exit status.
"""
with temporary_dir() as tmpdir:
with environment(HOME=os.getcwd()) as env:
proc = subprocess.Popen(['elasticluster'] + argv,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True,
shell=False)
stdout, stderr = proc.communicate()
return stdout, stderr, proc.returncode
def _run_command(argv):
"""
Run the `elasticluster` command with additional arguments.
Return STDOUT, STDERR, and the process exit status.
"""
with temporary_dir() as tmpdir:
with environment(HOME=os.getcwd()) as env:
proc = subprocess.Popen(
['elasticluster'] + argv,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True,
shell=False)
stdout, stderr = proc.communicate()
return stdout, stderr, proc.returncode
def _run_playbook(self, cluster, playbook, extra_args):
run_id = ('elasticluster.{name}.{date}.{pid}@{host}'.format(
name=cluster.name,
date=datetime.now().isoformat(),
pid=os.getpid(),
host=platform.node(),
))
inventory_path = self._build_inventory(cluster)
if inventory_path is None:
# no inventory file has been created: this can only happen
# if no nodes have been started nor can be reached
raise ClusterSizeError()
assert os.path.exists(inventory_path), (
"inventory file `{inventory_path}` does not exist".format(
inventory_path=inventory_path))
# build list of directories to search for roles/include files
ansible_roles_dirs = [
# include Ansible default first ...
'/etc/ansible/roles',
]
for root_path in [
# ... then ElastiCluster's built-in defaults
resource_filename('elasticluster', 'share/playbooks'),
# ... then wherever the playbook is
os.path.dirname(playbook),
]:
for path in [
root_path,
os.path.join(root_path, 'roles'),
]:
if path not in ansible_roles_dirs and os.path.exists(path):
ansible_roles_dirs.append(path)
# Use env vars to configure Ansible;
# see all values in https://github.com/ansible/ansible/blob/devel/lib/ansible/constants.py
#
# Ansible does not merge keys in configuration files: rather
# it uses the first configuration file found. However,
# environment variables can be used to selectively override
# parts of the config; according to [1]: "they are mostly
# considered to be a legacy system as compared to the config
# file, but are equally valid."
#
# [1]: http://docs.ansible.com/ansible/intro_configuration.html#environmental-configuration
#
# Provide default values for important configuration variables...
ansible_env = {
'ANSIBLE_FORKS': ('%d' % 4 * get_num_processors()),
'ANSIBLE_HOST_KEY_CHECKING': 'no',
'ANSIBLE_RETRY_FILES_ENABLED': 'no',
'ANSIBLE_ROLES_PATH': ':'.join(reversed(ansible_roles_dirs)),
'ANSIBLE_SSH_PIPELINING': 'yes',
'ANSIBLE_TIMEOUT': '120',
}
try:
import ara
ara_location = os.path.dirname(ara.__file__)
ansible_env['ANSIBLE_CALLBACK_PLUGINS'] = (
'{ara_location}/plugins/callbacks'.format(
ara_location=ara_location))
ansible_env['ANSIBLE_ACTION_PLUGINS'] = (
'{ara_location}/plugins/actions'.format(
ara_location=ara_location))
ansible_env['ANSIBLE_LIBRARY'] = (
'{ara_location}/plugins/modules'.format(
ara_location=ara_location))
ara_dir = os.getcwd()
ansible_env['ARA_DIR'] = ara_dir
ansible_env['ARA_DATABASE'] = (
'sqlite:///{ara_dir}/{run_id}.ara.sqlite'.format(
ara_dir=ara_dir, run_id=run_id))
ansible_env['ARA_LOG_CONFIG'] = ('{run_id}.ara.yml'.format(
ara_dir=ara_dir, run_id=run_id))
ansible_env['ARA_LOG_FILE'] = ('{run_id}.ara.log'.format(
ara_dir=ara_dir, run_id=run_id))
ansible_env['ARA_LOG_LEVEL'] = 'DEBUG'
ansible_env['ARA_PLAYBOOK_PER_PAGE'] = '0'
ansible_env['ARA_RESULT_PER_PAGE'] = '0'
except ImportError:
elasticluster.log.info(
"Could not import module `ara`:"
" no detailed information about the playbook will be recorded."
)
# ...override them with key/values set in the config file(s)
for k, v in self.extra_conf.items():
if k.startswith('ansible_'):
ansible_env[k.upper()] = str(v)
# ...finally allow the environment have the final word
ansible_env.update(os.environ)
# however, this is needed for correct detection of success/failure...
ansible_env['ANSIBLE_ANY_ERRORS_FATAL'] = 'yes'
# ...and this might be needed to connect (see issue #567)
if cluster.ssh_proxy_command:
ansible_env['ANSIBLE_SSH_ARGS'] = (
ansible_env.get('ANSIBLE_SSH_ARGS', '')
+ (" -o ProxyCommand='{proxy_command}'"
# NOTE: in contrast to `Node.connect()`, we must
# *not* expand %-escapes in the SSH proxy command:
# it will be done by the `ssh` client
.format(proxy_command=cluster.ssh_proxy_command)))
# report on calling environment
if __debug__:
elasticluster.log.debug(
"Calling `ansible-playbook` with the following environment:")
for var, value in sorted(ansible_env.items()):
# sanity check. Do not print password content....
if "password" in var.lower() or "secret" in var.lower():
elasticluster.log.debug("- %s=******", var)
else:
elasticluster.log.debug("- %s=%r", var, value)
elasticluster.log.debug("Using playbook file %s.", playbook)
# build `ansible-playbook` command-line
cmd = shlex.split(
self.extra_conf.get('ansible_command', 'ansible-playbook'))
cmd += [
('--private-key=' + cluster.user_key_private),
os.path.realpath(playbook),
('--inventory=' + inventory_path),
]
if self._sudo:
cmd.extend([
# force all plays to use `sudo` (even if not marked as such)
'--become',
# desired sudo-to user
('--become-user='******'s
# log level (we cannot read `ElastiCluster().params.verbose`
# here, still we can access the log configuration since it's
# global).
verbosity = int(
(logging.WARNING - elasticluster.log.getEffectiveLevel()) / 10)
if verbosity > 0:
cmd.append('-' + ('v' * verbosity)) # e.g., `-vv`
# append any additional arguments provided by users in config file
ansible_extra_args = self.extra_conf.get('ansible_extra_args', None)
if ansible_extra_args:
cmd += shlex.split(ansible_extra_args)
# finally, append any additional arguments provided on command-line
for arg in extra_args:
# XXX: since we are going to change working directory,
# make sure that anything that looks like a path to an
# existing file is made absolute before appending to
# Ansible's command line. (Yes, this is a ugly hack.)
if os.path.exists(arg):
arg = os.path.abspath(arg)
cmd.append(arg)
with temporary_dir():
# adjust execution environment, for the part that needs a
# the current directory path
cmd += ['-e', ('@' + self._write_extra_vars(cluster))]
# run it!
cmdline = ' '.join(cmd)
elasticluster.log.debug("Running Ansible command `%s` ...",
cmdline)
rc = call(cmd, env=ansible_env, bufsize=1, close_fds=True)
# check outcome
ok = False # pessimistic default
if rc != 0:
elasticluster.log.error(
"Command `%s` failed with exit code %d.", cmdline, rc)
else:
# even if Ansible exited with return code 0, the
# playbook might still have failed -- so explicitly
# check for a "done" report showing that each node run
# the playbook until the very last task
cluster_hosts = set(node.name
for node in cluster.get_all_nodes())
done_hosts = set()
for node_name in cluster_hosts:
try:
with open(node_name + '.log') as stream:
status = stream.read().strip()
if status == 'done':
done_hosts.add(node_name)
except (OSError, IOError):
# no status file for host, do not add it to
# `done_hosts`
pass
if done_hosts == cluster_hosts:
# success!
ok = True
elif len(done_hosts) == 0:
# total failure
elasticluster.log.error(
"No host reported successfully running the setup playbook!"
)
else:
# partial failure
elasticluster.log.error(
"The following nodes did not report"
" successful termination of the setup playbook:"
" %s", (', '.join(cluster_hosts - done_hosts)))
if ok:
elasticluster.log.info("Cluster correctly configured.")
return True
else:
elasticluster.log.warning(
"The cluster has likely *not* been configured correctly."
" You may need to re-run `elasticluster setup`.")
return False
def setup_cluster(self, cluster, extra_args=tuple()):
"""
Configure the cluster by running an Ansible playbook.
The ElastiCluster configuration attribute `<kind>_groups`
determines, for each node kind, what Ansible groups nodes of
that kind are assigned to.
:param cluster: cluster to configure
:type cluster: :py:class:`elasticluster.cluster.Cluster`
:param list extra_args:
List of additional command-line arguments
that are appended to each invocation of the setup program.
:return: ``True`` on success, ``False`` otherwise. Please note, if nothing
has to be configured, then ``True`` is returned.
:raises: `ConfigurationError` if the playbook can not be found
or is corrupt.
"""
inventory_path = self._build_inventory(cluster)
if inventory_path is None:
# No inventory file has been created, maybe an
# invalid class has been specified in config file? Or none?
# assume it is fine.
# chenyjie-change the default return value from True to False
elasticluster.log.info("No setup required for this cluster.")
return False
assert os.path.exists(inventory_path), (
"inventory file `{inventory_path}` does not exist".format(
inventory_path=inventory_path))
# build list of directories to search for roles/include files
ansible_roles_dirs = [
# include Ansible default first ...
'/etc/ansible/roles',
]
for root_path in [
# ... then ElastiCluster's built-in defaults
resource_filename('elasticluster', 'share/playbooks'),
# ... then wherever the playbook is
os.path.dirname(self._playbook_path),
]:
for path in [
root_path,
os.path.join(root_path, 'roles'),
]:
if path not in ansible_roles_dirs and os.path.exists(path):
ansible_roles_dirs.append(path)
# Use env vars to configure Ansible;
# see all values in https://github.com/ansible/ansible/blob/devel/lib/ansible/constants.py
#
# Ansible does not merge keys in configuration files: rather
# it uses the first configuration file found. However,
# environment variables can be used to selectively override
# parts of the config; according to [1]: "they are mostly
# considered to be a legacy system as compared to the config
# file, but are equally valid."
#
# [1]: http://docs.ansible.com/ansible/intro_configuration.html#environmental-configuration
#
# Provide default values for important configuration variables...
ansible_env = {
'ANSIBLE_FORKS': '100',
'ANSIBLE_HOST_KEY_CHECKING': 'no',
'ANSIBLE_RETRY_FILES_ENABLED': 'no',
'ANSIBLE_ROLES_PATH': ':'.join(reversed(ansible_roles_dirs)),
'ANSIBLE_SSH_PIPELINING': 'yes',
'ANSIBLE_TIMEOUT': '120',
}
# ...override them with key/values set in the config file(s)
for k, v in self.extra_conf.items():
if k.startswith('ansible_'):
ansible_env[k.upper()] = str(v)
# ...finally allow the environment have the final word
ansible_env.update(os.environ)
# however, this is needed for correct detection of success/failure
ansible_env['ANSIBLE_ANY_ERRORS_FATAL'] = 'yes'
# report on calling environment
if __debug__:
elasticluster.log.debug(
"Calling `ansible-playbook` with the following environment:")
for var, value in sorted(ansible_env.items()):
elasticluster.log.debug("- %s=%r", var, value)
elasticluster.log.debug("Using playbook file %s.", self._playbook_path)
# build `ansible-playbook` command-line
cmd = shlex.split(
self.extra_conf.get('ansible_command', 'ansible-playbook'))
cmd += [
('--private-key=' + cluster.user_key_private),
os.path.realpath(self._playbook_path),
('--inventory=' + inventory_path),
] + list(extra_args)
if self._sudo:
cmd.extend([
# force all plays to use `sudo` (even if not marked as such)
'--become',
# desired sudo-to user
('--become-user='******'s
# log level (we cannot read `ElastiCluster().params.verbose`
# here, still we can access the log configuration since it's
# global).
verbosity = (logging.WARNING -
elasticluster.log.getEffectiveLevel()) / 10
if verbosity > 0:
cmd.append('-' + ('v' * verbosity)) # e.g., `-vv`
# append any additional arguments provided by users
ansible_extra_args = self.extra_conf.get('ansible_extra_args', None)
if ansible_extra_args:
cmd += shlex.split(ansible_extra_args)
ok = False # pessimistic default
with temporary_dir():
cmd += ['-e', 'elasticluster_output_dir={0}'.format(os.getcwd())]
cmdline = ' '.join(cmd)
elasticluster.log.debug("Running Ansible command `%s` ...",
cmdline)
rc = call(cmd, env=ansible_env, bufsize=1, close_fds=True)
if rc != 0:
elasticluster.log.error(
"Command `%s` failed with exit code %d.", cmdline, rc)
else:
# even if Ansible exited with return code 0, the
# playbook might still have failed -- so explicitly
# check for a "done" report showing that each node run
# the playbook until the very last task
cluster_hosts = set(node.name
for node in cluster.get_all_nodes())
done_hosts = set()
for node_name in cluster_hosts:
try:
with open(node_name + '.log') as stream:
status = stream.read().strip()
if status == 'done':
done_hosts.add(node_name)
except (OSError, IOError):
# no status file for host, do not add it to
# `done_hosts`
pass
if done_hosts == cluster_hosts:
# success!
ok = True
elif len(done_hosts) == 0:
# total failure
elasticluster.log.error(
"No host reported successfully running the setup playbook!"
)
else:
# partial failure
elasticluster.log.error(
"The following nodes did not report"
" successful termination of the setup playbook:"
" %s", (', '.join(cluster_hosts - done_hosts)))
if ok:
elasticluster.log.info("Cluster correctly configured.")
return True
else:
elasticluster.log.warning(
"The cluster has likely *not* been configured correctly."
" You may need to re-run `elasticluster setup`.")
return False
def _run_playbook(self, cluster, playbook, extra_args):
run_id = (
'elasticluster.{name}.{date}.{pid}@{host}'
.format(
name=cluster.name,
date=datetime.now().isoformat(),
pid=os.getpid(),
host=platform.node(),
)
)
inventory_path = self._build_inventory(cluster)
if inventory_path is None:
# no inventory file has been created: this can only happen
# if no nodes have been started nor can be reached
raise ClusterSizeError()
assert os.path.exists(inventory_path), (
"inventory file `{inventory_path}` does not exist"
.format(inventory_path=inventory_path))
# build list of directories to search for roles/include files
ansible_roles_dirs = [
# include Ansible default first ...
'/etc/ansible/roles',
]
for root_path in [
# ... then ElastiCluster's built-in defaults
resource_filename('elasticluster', 'share/playbooks'),
# ... then wherever the playbook is
os.path.dirname(playbook),
]:
for path in [
root_path,
os.path.join(root_path, 'roles'),
]:
if path not in ansible_roles_dirs and os.path.exists(path):
ansible_roles_dirs.append(path)
# Use env vars to configure Ansible;
# see all values in https://github.com/ansible/ansible/blob/devel/lib/ansible/constants.py
#
# Ansible does not merge keys in configuration files: rather
# it uses the first configuration file found. However,
# environment variables can be used to selectively override
# parts of the config; according to [1]: "they are mostly
# considered to be a legacy system as compared to the config
# file, but are equally valid."
#
# [1]: http://docs.ansible.com/ansible/intro_configuration.html#environmental-configuration
#
# Provide default values for important configuration variables...
ansible_env = {
'ANSIBLE_FORKS': ('%d' % 4*get_num_processors()),
'ANSIBLE_HOST_KEY_CHECKING': 'no',
'ANSIBLE_RETRY_FILES_ENABLED': 'no',
'ANSIBLE_ROLES_PATH': ':'.join(reversed(ansible_roles_dirs)),
'ANSIBLE_SSH_PIPELINING': 'yes',
'ANSIBLE_TIMEOUT': '120',
}
try:
import ara
ara_location = os.path.dirname(ara.__file__)
ansible_env['ANSIBLE_CALLBACK_PLUGINS'] = (
'{ara_location}/plugins/callbacks'
.format(ara_location=ara_location))
ansible_env['ANSIBLE_ACTION_PLUGINS'] = (
'{ara_location}/plugins/actions'
.format(ara_location=ara_location))
ansible_env['ANSIBLE_LIBRARY'] = (
'{ara_location}/plugins/modules'
.format(ara_location=ara_location))
ara_dir = os.getcwd()
ansible_env['ARA_DIR'] = ara_dir
ansible_env['ARA_DATABASE'] = (
'sqlite:///{ara_dir}/{run_id}.ara.sqlite'
.format(ara_dir=ara_dir, run_id=run_id))
ansible_env['ARA_LOG_CONFIG'] = (
'{run_id}.ara.yml'
.format(ara_dir=ara_dir, run_id=run_id))
ansible_env['ARA_LOG_FILE'] = (
'{run_id}.ara.log'
.format(ara_dir=ara_dir, run_id=run_id))
ansible_env['ARA_LOG_LEVEL'] = 'DEBUG'
ansible_env['ARA_PLAYBOOK_PER_PAGE'] = '0'
ansible_env['ARA_RESULT_PER_PAGE'] = '0'
except ImportError:
elasticluster.log.info(
"Could not import module `ara`:"
" no detailed information about the playbook will be recorded.")
# ...override them with key/values set in the config file(s)
for k, v in self.extra_conf.items():
if k.startswith('ansible_'):
ansible_env[k.upper()] = str(v)
# ...finally allow the environment have the final word
ansible_env.update(os.environ)
# however, this is needed for correct detection of success/failure...
ansible_env['ANSIBLE_ANY_ERRORS_FATAL'] = 'yes'
# ...and this might be needed to connect (see issue #567)
if cluster.ssh_proxy_command:
ansible_env['ANSIBLE_SSH_ARGS'] = (
ansible_env.get('ANSIBLE_SSH_ARGS', '')
+ (" -o ProxyCommand='{proxy_command}'"
# NOTE: in contrast to `Node.connect()`, we must
# *not* expand %-escapes in the SSH proxy command:
# it will be done by the `ssh` client
.format(proxy_command=cluster.ssh_proxy_command)))
# report on calling environment
if __debug__:
elasticluster.log.debug(
"Calling `ansible-playbook` with the following environment:")
for var, value in sorted(ansible_env.items()):
# sanity check. Do not print password content....
if "password" in var.lower() or "secret" in var.lower():
elasticluster.log.debug("- %s=******", var)
else:
elasticluster.log.debug("- %s=%r", var, value)
elasticluster.log.debug("Using playbook file %s.", playbook)
# build `ansible-playbook` command-line
cmd = shlex.split(self.extra_conf.get('ansible_command', 'ansible-playbook'))
cmd += [
('--private-key=' + cluster.user_key_private),
os.path.realpath(playbook),
('--inventory=' + inventory_path),
]
if self._sudo:
cmd.extend([
# force all plays to use `sudo` (even if not marked as such)
'--become',
# desired sudo-to user
('--become-user='******'s
# log level (we cannot read `ElastiCluster().params.verbose`
# here, still we can access the log configuration since it's
# global).
verbosity = int((logging.WARNING - elasticluster.log.getEffectiveLevel()) / 10)
if verbosity > 0:
cmd.append('-' + ('v' * verbosity)) # e.g., `-vv`
# append any additional arguments provided by users in config file
ansible_extra_args = self.extra_conf.get('ansible_extra_args', None)
if ansible_extra_args:
cmd += shlex.split(ansible_extra_args)
# finally, append any additional arguments provided on command-line
for arg in extra_args:
# XXX: since we are going to change working directory,
# make sure that anything that looks like a path to an
# existing file is made absolute before appending to
# Ansible's command line. (Yes, this is a ugly hack.)
if os.path.exists(arg):
arg = os.path.abspath(arg)
cmd.append(arg)
with temporary_dir():
# adjust execution environment, for the part that needs a
# the current directory path
cmd += [
'-e', ('@' + self._write_extra_vars(cluster))
]
# run it!
cmdline = ' '.join(cmd)
elasticluster.log.debug(
"Running Ansible command `%s` ...", cmdline)
rc = call(cmd, env=ansible_env, bufsize=1, close_fds=True)
# check outcome
ok = False # pessimistic default
if rc != 0:
elasticluster.log.error(
"Command `%s` failed with exit code %d.", cmdline, rc)
else:
# even if Ansible exited with return code 0, the
# playbook might still have failed -- so explicitly
# check for a "done" report showing that each node run
# the playbook until the very last task
cluster_hosts = set(node.name
for node in cluster.get_all_nodes())
done_hosts = set()
for node_name in cluster_hosts:
try:
with open(node_name + '.log') as stream:
status = stream.read().strip()
if status == 'done':
done_hosts.add(node_name)
except (OSError, IOError):
# no status file for host, do not add it to
# `done_hosts`
pass
if done_hosts == cluster_hosts:
# success!
ok = True
elif len(done_hosts) == 0:
# total failure
elasticluster.log.error(
"No host reported successfully running the setup playbook!")
else:
# partial failure
elasticluster.log.error(
"The following nodes did not report"
" successful termination of the setup playbook:"
" %s", (', '.join(cluster_hosts - done_hosts)))
if ok:
elasticluster.log.info("Cluster correctly configured.")
return True
else:
elasticluster.log.warning(
"The cluster has likely *not* been configured correctly."
" You may need to re-run `elasticluster setup`.")
return False