def create_extra_vars(output_dir: str, nodes: List[NodeDescriptor],
private_path: str) -> dict:
elasticluster_vars = {
'elasticluster': {
'cloud': {},
'nodes': {},
'output_dir': output_dir
}
}
for node in nodes:
if node.configuration.provider.provider == 'local':
continue
if node.configuration.provider.provider == 'aws':
aws_access_key = open(
path_extend(private_path,
node.configuration.provider.access_keyfile),
'r').read().strip()
aws_secret_key = open(
path_extend(private_path,
node.configuration.provider.secret_access_keyfile),
'r').read().strip()
aws_region = node.configuration.provider.region
keypair_name = node.configuration.login.keypair_name
elasticluster_vars['elasticluster']['cloud']['aws_access_key_id'] = aws_access_key
elasticluster_vars['elasticluster']['cloud']['aws_secret_access_key'] = aws_secret_key
elasticluster_vars['elasticluster']['cloud']['aws_region'] = aws_region
elasticluster_vars['elasticluster']['nodes'][node.node_id] = {
'user_key_name': keypair_name,
'instance_id': node.cloud_instance_id
}
return elasticluster_vars
def __init__(self):
self.base_defaults = Defaults()
self.role_dir = path_extend(self.base_defaults.clap_path, 'roles')
self.actions_dir = path_extend(self.role_dir, 'actions.d')
self.repository_type = 'sqlite'
self.node_repository_path = path_extend(
self.base_defaults.storage_path, 'nodes.db')
def create_envvars(self, config: ProviderConfigAWS):
envvars = os.environ.copy()
acc_key = path_extend(self.private_path, config.access_keyfile)
sec_key = path_extend(self.private_path, config.secret_access_keyfile)
envvars['AWS_ACCESS_KEY'] = open(acc_key, 'r').read().strip()
envvars['AWS_SECRET_KEY'] = open(sec_key, 'r').read().strip()
envvars['AWS_REGION'] = config.region
return envvars
def __init__(self):
self.base_defaults = Defaults()
self.node_defaults = NodeDefaults()
self.role_defaults = RoleDefaults()
self.cluster_config_path = path_extend(self.base_defaults.configs_path,
'clusters')
self.repository_type = 'sqlite'
self.cluster_repository_path = path_extend(
self.base_defaults.storage_path, 'clusters.db')
def __init__(self):
self.verbosity: int = 0
self.clap_path: str = path_extend(os.environ.get('CLAP_PATH'))
self.configs_path: str = path_extend(
os.environ.get('CLAP_PATH'), 'configs')
self.private_path: str = path_extend(
os.environ.get('CLAP_PATH'), 'private')
self.storage_path: str = path_extend(
os.environ.get('CLAP_PATH'), 'storage')
def __init__(self):
self.base_defaults = Defaults()
self.repository_type = 'sqlite'
self.node_repository_path = path_extend(
self.base_defaults.storage_path, 'nodes.db')
self.providers_path = path_extend(self.base_defaults.configs_path,
'providers.yaml')
self.logins_path = path_extend(self.base_defaults.configs_path,
'logins.yaml')
self.instances_path = path_extend(self.base_defaults.configs_path,
'instances.yaml')
self.templates_path = path_extend(
os.path.dirname(os.path.abspath(__file__)), 'templates')
def connect_and_execute(self, node: NodeDescriptor) -> CommandResult:
try:
user = node.configuration.login.user
ssh_port = node.configuration.login.ssh_port
connection_ip = node.ip
key_file = path_extend(
self.private_path, node.configuration.login.keypair_private_file)
if not connection_ip:
raise ConnectionError(f"Invalid connection ip '{node.ip}' for node "
f"{node.nickname}. Check if {node.nickname} "
f"is alive first...")
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy)
client.connect(connection_ip, port=ssh_port, username=user,
key_filename=key_file, timeout=self.connection_timeout)
_, stdout, stderr = client.exec_command(
self.command, timeout=self.execution_timeout,
environment=self.environment)
stdout_lines = stdout.readlines()
stderr_lines = stderr.readlines()
return_code = stdout.channel.recv_exit_status()
client.close()
return SSHCommandExecutor.CommandResult(
ok=True, ret_code=return_code, stdout_lines=stdout_lines,
stderr_lines=stderr_lines, error=None
)
except Exception as e:
logger.error(f"Error executing command in node {node.node_id[:8]}: {e}")
return SSHCommandExecutor.CommandResult(
ok=False, ret_code=None, stdout_lines=None,
stderr_lines=None, error=str(e)
)
def __init__(self, private_dir: str, verbosity: int = 0):
self.private_path = private_dir
self.verbosity = verbosity
self.templates_path = path_extend(
os.path.dirname(os.path.abspath(__file__)), 'templates')
self.jinjaenv = jinja2.Environment(loader=jinja2.FileSystemLoader(
self.templates_path),
trim_blocks=True,
lstrip_blocks=True)
def _run_template(self, output_filename: str, rendered_template: str,
envvars: Dict[str, str] = None, quiet: bool = False) -> \
AnsiblePlaybookExecutor.PlaybookResult:
with tmpdir(suffix='.aws') as tdir:
output_filename = path_extend(tdir, output_filename)
with open(output_filename, 'w') as f:
f.write(rendered_template)
r = AnsiblePlaybookExecutor(output_filename,
self.private_path,
inventory=None,
env_vars=envvars,
verbosity=self.verbosity,
quiet=quiet)
return r.run()
def load_roles(self):
for role_file in os.listdir(self.actions_dir):
role_name: str = Path(role_file).stem
try:
role_values: dict = yaml_load(
path_extend(self.actions_dir, role_file))
role = dacite.from_dict(data_class=Role, data=role_values)
self.roles[role_name] = role
except Exception as e:
if self._discard_invalids:
logger.error(
f"Discarding role '{role_name}'. {type(e).__name__}: {e}"
)
continue
else:
raise e
def create_inventory(
hosts_node_map: Union[
List[NodeDescriptor], Dict[str, List[NodeDescriptor]]],
private_path: str,
host_vars: Dict[str, Dict[str, str]] = None,
node_vars: Dict[str, Dict[str, str]] = None) -> dict:
inventory = defaultdict(dict)
hosts = defaultdict(dict)
host_vars = host_vars or dict()
node_vars = node_vars or dict()
if type(hosts_node_map) is list:
hosts_node_map = {'all': hosts_node_map}
elif type(hosts_node_map) is not dict:
raise TypeError(f"Invalid type {type(hosts_node_map)} for "
f"hosts_node_map parameter")
for host, node_list in hosts_node_map.items():
host_dict = dict()
try:
host_dict['vars'] = host_vars[host]
except KeyError:
pass
_hosts = dict()
for node in node_list:
_host_vars = dict()
_host_vars['ansible_host'] = node.ip
_host_vars['ansible_connection'] = 'ssh'
_host_vars['ansible_user'] = node.configuration.login.user
_host_vars['ansible_ssh_private_key_file'] = path_extend(
private_path, node.configuration.login.keypair_private_file)
_host_vars['ansible_port'] = node.configuration.login.ssh_port
_host_vars.update(node_vars.get(node.node_id, dict()))
_hosts[node.node_id] = _host_vars
host_dict['hosts'] = _hosts
if host == 'all':
inventory['all'] = host_dict
else:
hosts[host] = host_dict
if hosts:
inventory['all']['children'] = defaultdict_to_dict(hosts)
return defaultdict_to_dict(inventory)
def run(self):
user = self.node.configuration.login.user
ssh_port = self.node.configuration.login.ssh_port
connection_ip = self.node.ip
key_file = path_extend(
self.private_path,
self.node.configuration.login.keypair_private_file)
ssh_verbose = "-{}".format('v' * self.verbosity) if self.verbosity > 1 \
else ""
ssh_command = f'{self.ssh_binary} -t {ssh_verbose} -o "Port={ssh_port}" ' \
f'-o StrictHostKeyChecking=no -o "User={user}" ' \
f'-i "{key_file}" {connection_ip}'
logger.debug(f"Executing ssh command: '{ssh_command}'")
try:
subprocess.check_call(ssh_command, shell=True)
logger.debug(f"SSH session to {connection_ip} finalized!")
except subprocess.CalledProcessError:
logger.error(f"Invalid connection ip: {self.node.ip}. "
f"Check if `{self.node.node_id}` is alive first...")
def cluster_playbook(cluster_id, playbook, extra, node_vars):
"""Execute an Ansible Playbook in all cluster nodes.
The CLUSTER_ID argument is the id of the cluster to execute the Ansible Playbook.
"""
cluster_manager = get_cluster_manager()
node_manager = get_node_manager()
nodes = cluster_manager.get_all_cluster_nodes(cluster_id)
nodes = node_manager.get_nodes_by_id(nodes)
if not nodes:
print("No nodes in the cluster")
return 0
extra_args = dict()
for e in extra:
if '=' not in e:
raise ValueError(f"Invalid value for extra argument: `{e}`. "
f"Did you forgot '=' character?")
extra_name, extra_value = e.split('=')[0], '='.join(e.split('=')[1:])
extra_args[extra_name] = extra_value
playbook = path_extend(playbook)
if not os.path.isfile(playbook):
raise ValueError(f"Invalid playbook file `{playbook}`")
node_variables = defaultdict(dict)
for nvar in node_vars:
if ':' not in nvar:
raise ValueError(f"Invalid value for node argument: `{nvar}`. "
f"Did you forgot ':' character?")
node_id, node_extra_args = nvar.split(':')[0], ':'.join(
nvar.split(':')[1:])
for narg in node_extra_args.split(','):
if '=' not in narg:
raise ValueError(
f"Invalid value for extra argument: '{narg}'. "
f"Did you forgot '=' character?")
extra_name, extra_value = narg.split('=')[0], '='.join(
narg.split('=')[1:])
node_variables[node_id].update({extra_name: extra_value})
node_variables = defaultdict_to_dict(node_variables)
inventory = AnsiblePlaybookExecutor.create_inventory(
nodes, cluster_defaults.base_defaults.private_path, {}, node_variables)
executor = AnsiblePlaybookExecutor(
playbook, cluster_defaults.base_defaults.private_path, inventory,
extra_args)
result = executor.run()
if not result.ok:
logger.error(f"Playbook {playbook} did not executed successfully...")
return 1
print(str_at_middle("Execution Summary", 80))
for node_id in sorted(list(result.hosts.keys())):
r = result.hosts[node_id]
print(f"{node_id}: {'ok' if r else 'not ok'}")
print(
f"Playbook at `{playbook}` were executed in {len(result.hosts)} nodes")
return 0
def perform_action(self, role_name: str, action_name: str,
hosts_node_map: Union[List[str], Dict[str, List[str]]],
host_vars: Dict[str, Dict[str, str]] = None,
node_vars: Dict[str, Dict[str, str]] = None,
extra_args: Dict[str, str] = None,
quiet: bool = False,
validate_nodes_in_role: bool = True) -> \
AnsiblePlaybookExecutor.PlaybookResult:
"""
:param role_name:
:param action_name:
"""
host_vars = host_vars or dict()
node_vars = node_vars or dict()
extra_args = extra_args or dict()
if role_name not in self.roles:
raise InvalidRoleError(role_name)
role: Role = self.roles[role_name]
if action_name not in role.actions:
raise InvalidActionError(role_name, action_name)
action = role.actions[action_name]
# Check hosts_node_map variable
if not role.hosts:
if type(hosts_node_map) is list:
_inventory = {'': hosts_node_map}
elif type(hosts_node_map) is dict:
if '' not in hosts_node_map:
raise ValueError("hosts_node_map variable must contain "
"'None' key.")
_inventory = {'': hosts_node_map['']}
else:
raise TypeError(
f"hosts_node_map variable expects a list or a dict, "
f"not a {type(hosts_node_map)}")
else:
if type(hosts_node_map) is not dict:
raise TypeError(
f"As role {role_name} defines hosts, hosts_node_map "
f"variable expects a dict, not a {type(hosts_node_map)}")
_inventory = dict()
for hname, node_list in hosts_node_map.items():
if hname not in role.hosts:
raise InvalidHostError(role_name, hname)
_inventory[hname] = node_list
# Expand node_ids to NodeDescriptors
inventory: Dict[str, List[NodeDescriptor]] = {
host_name: self.node_repository.get_nodes_by_id(list_nodes)
for host_name, list_nodes in _inventory.items()
}
if validate_nodes_in_role:
self._check_nodes_role(role_name, inventory)
if not role.hosts:
inventory = {role_name: inventory['']}
# Check if every required role's action variable is informed via extra_args
for var in action.vars:
if not var.optional:
if var.name not in extra_args:
raise MissingActionVariableError(role_name, action_name,
var.name)
# Expand playbook path
playbook_file = path_extend(self.roles_dir, action.playbook)
# Create ansible-like inventory
inventory = AnsiblePlaybookExecutor.create_inventory(
inventory, self.private_dir, host_vars, node_vars)
# Crate playbook executor and run
playbook = AnsiblePlaybookExecutor(playbook_file,
self.private_dir,
inventory,
extra_args,
quiet=quiet)
logger.info(f'Executing playbook {playbook_file} with inventory: '
f'{inventory}')
result = playbook.run()
return result
import inspect
import os
import sys
import pkgutil
import traceback
from typing import List
import click
from clap.utils import get_logger, setup_log, Singleton, path_extend
if 'CLAP_PATH' not in os.environ:
os.environ['CLAP_PATH'] = path_extend('~', '.clap')
class ArgumentError(Exception):
pass
class Defaults(metaclass=Singleton):
def __init__(self):
self.verbosity: int = 0
self.clap_path: str = path_extend(os.environ.get('CLAP_PATH'))
self.configs_path: str = path_extend(
os.environ.get('CLAP_PATH'), 'configs')
self.private_path: str = path_extend(
os.environ.get('CLAP_PATH'), 'private')
self.storage_path: str = path_extend(
os.environ.get('CLAP_PATH'), 'storage')
