def create_extra_vars(output_dir: str, nodes: List[NodeDescriptor], private_path: str) -> dict: elasticluster_vars = { 'elasticluster': { 'cloud': {}, 'nodes': {}, 'output_dir': output_dir } } for node in nodes: if node.configuration.provider.provider == 'local': continue if node.configuration.provider.provider == 'aws': aws_access_key = open( path_extend(private_path, node.configuration.provider.access_keyfile), 'r').read().strip() aws_secret_key = open( path_extend(private_path, node.configuration.provider.secret_access_keyfile), 'r').read().strip() aws_region = node.configuration.provider.region keypair_name = node.configuration.login.keypair_name elasticluster_vars['elasticluster']['cloud']['aws_access_key_id'] = aws_access_key elasticluster_vars['elasticluster']['cloud']['aws_secret_access_key'] = aws_secret_key elasticluster_vars['elasticluster']['cloud']['aws_region'] = aws_region elasticluster_vars['elasticluster']['nodes'][node.node_id] = { 'user_key_name': keypair_name, 'instance_id': node.cloud_instance_id } return elasticluster_vars
def __init__(self): self.base_defaults = Defaults() self.role_dir = path_extend(self.base_defaults.clap_path, 'roles') self.actions_dir = path_extend(self.role_dir, 'actions.d') self.repository_type = 'sqlite' self.node_repository_path = path_extend( self.base_defaults.storage_path, 'nodes.db')
def create_envvars(self, config: ProviderConfigAWS): envvars = os.environ.copy() acc_key = path_extend(self.private_path, config.access_keyfile) sec_key = path_extend(self.private_path, config.secret_access_keyfile) envvars['AWS_ACCESS_KEY'] = open(acc_key, 'r').read().strip() envvars['AWS_SECRET_KEY'] = open(sec_key, 'r').read().strip() envvars['AWS_REGION'] = config.region return envvars
def __init__(self): self.base_defaults = Defaults() self.node_defaults = NodeDefaults() self.role_defaults = RoleDefaults() self.cluster_config_path = path_extend(self.base_defaults.configs_path, 'clusters') self.repository_type = 'sqlite' self.cluster_repository_path = path_extend( self.base_defaults.storage_path, 'clusters.db')
def __init__(self): self.verbosity: int = 0 self.clap_path: str = path_extend(os.environ.get('CLAP_PATH')) self.configs_path: str = path_extend( os.environ.get('CLAP_PATH'), 'configs') self.private_path: str = path_extend( os.environ.get('CLAP_PATH'), 'private') self.storage_path: str = path_extend( os.environ.get('CLAP_PATH'), 'storage')
def __init__(self): self.base_defaults = Defaults() self.repository_type = 'sqlite' self.node_repository_path = path_extend( self.base_defaults.storage_path, 'nodes.db') self.providers_path = path_extend(self.base_defaults.configs_path, 'providers.yaml') self.logins_path = path_extend(self.base_defaults.configs_path, 'logins.yaml') self.instances_path = path_extend(self.base_defaults.configs_path, 'instances.yaml') self.templates_path = path_extend( os.path.dirname(os.path.abspath(__file__)), 'templates')
def connect_and_execute(self, node: NodeDescriptor) -> CommandResult: try: user = node.configuration.login.user ssh_port = node.configuration.login.ssh_port connection_ip = node.ip key_file = path_extend( self.private_path, node.configuration.login.keypair_private_file) if not connection_ip: raise ConnectionError(f"Invalid connection ip '{node.ip}' for node " f"{node.nickname}. Check if {node.nickname} " f"is alive first...") client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy) client.connect(connection_ip, port=ssh_port, username=user, key_filename=key_file, timeout=self.connection_timeout) _, stdout, stderr = client.exec_command( self.command, timeout=self.execution_timeout, environment=self.environment) stdout_lines = stdout.readlines() stderr_lines = stderr.readlines() return_code = stdout.channel.recv_exit_status() client.close() return SSHCommandExecutor.CommandResult( ok=True, ret_code=return_code, stdout_lines=stdout_lines, stderr_lines=stderr_lines, error=None ) except Exception as e: logger.error(f"Error executing command in node {node.node_id[:8]}: {e}") return SSHCommandExecutor.CommandResult( ok=False, ret_code=None, stdout_lines=None, stderr_lines=None, error=str(e) )
def __init__(self, private_dir: str, verbosity: int = 0): self.private_path = private_dir self.verbosity = verbosity self.templates_path = path_extend( os.path.dirname(os.path.abspath(__file__)), 'templates') self.jinjaenv = jinja2.Environment(loader=jinja2.FileSystemLoader( self.templates_path), trim_blocks=True, lstrip_blocks=True)
def _run_template(self, output_filename: str, rendered_template: str, envvars: Dict[str, str] = None, quiet: bool = False) -> \ AnsiblePlaybookExecutor.PlaybookResult: with tmpdir(suffix='.aws') as tdir: output_filename = path_extend(tdir, output_filename) with open(output_filename, 'w') as f: f.write(rendered_template) r = AnsiblePlaybookExecutor(output_filename, self.private_path, inventory=None, env_vars=envvars, verbosity=self.verbosity, quiet=quiet) return r.run()
def load_roles(self): for role_file in os.listdir(self.actions_dir): role_name: str = Path(role_file).stem try: role_values: dict = yaml_load( path_extend(self.actions_dir, role_file)) role = dacite.from_dict(data_class=Role, data=role_values) self.roles[role_name] = role except Exception as e: if self._discard_invalids: logger.error( f"Discarding role '{role_name}'. {type(e).__name__}: {e}" ) continue else: raise e
def create_inventory( hosts_node_map: Union[ List[NodeDescriptor], Dict[str, List[NodeDescriptor]]], private_path: str, host_vars: Dict[str, Dict[str, str]] = None, node_vars: Dict[str, Dict[str, str]] = None) -> dict: inventory = defaultdict(dict) hosts = defaultdict(dict) host_vars = host_vars or dict() node_vars = node_vars or dict() if type(hosts_node_map) is list: hosts_node_map = {'all': hosts_node_map} elif type(hosts_node_map) is not dict: raise TypeError(f"Invalid type {type(hosts_node_map)} for " f"hosts_node_map parameter") for host, node_list in hosts_node_map.items(): host_dict = dict() try: host_dict['vars'] = host_vars[host] except KeyError: pass _hosts = dict() for node in node_list: _host_vars = dict() _host_vars['ansible_host'] = node.ip _host_vars['ansible_connection'] = 'ssh' _host_vars['ansible_user'] = node.configuration.login.user _host_vars['ansible_ssh_private_key_file'] = path_extend( private_path, node.configuration.login.keypair_private_file) _host_vars['ansible_port'] = node.configuration.login.ssh_port _host_vars.update(node_vars.get(node.node_id, dict())) _hosts[node.node_id] = _host_vars host_dict['hosts'] = _hosts if host == 'all': inventory['all'] = host_dict else: hosts[host] = host_dict if hosts: inventory['all']['children'] = defaultdict_to_dict(hosts) return defaultdict_to_dict(inventory)
def run(self): user = self.node.configuration.login.user ssh_port = self.node.configuration.login.ssh_port connection_ip = self.node.ip key_file = path_extend( self.private_path, self.node.configuration.login.keypair_private_file) ssh_verbose = "-{}".format('v' * self.verbosity) if self.verbosity > 1 \ else "" ssh_command = f'{self.ssh_binary} -t {ssh_verbose} -o "Port={ssh_port}" ' \ f'-o StrictHostKeyChecking=no -o "User={user}" ' \ f'-i "{key_file}" {connection_ip}' logger.debug(f"Executing ssh command: '{ssh_command}'") try: subprocess.check_call(ssh_command, shell=True) logger.debug(f"SSH session to {connection_ip} finalized!") except subprocess.CalledProcessError: logger.error(f"Invalid connection ip: {self.node.ip}. " f"Check if `{self.node.node_id}` is alive first...")
def cluster_playbook(cluster_id, playbook, extra, node_vars): """Execute an Ansible Playbook in all cluster nodes. The CLUSTER_ID argument is the id of the cluster to execute the Ansible Playbook. """ cluster_manager = get_cluster_manager() node_manager = get_node_manager() nodes = cluster_manager.get_all_cluster_nodes(cluster_id) nodes = node_manager.get_nodes_by_id(nodes) if not nodes: print("No nodes in the cluster") return 0 extra_args = dict() for e in extra: if '=' not in e: raise ValueError(f"Invalid value for extra argument: `{e}`. " f"Did you forgot '=' character?") extra_name, extra_value = e.split('=')[0], '='.join(e.split('=')[1:]) extra_args[extra_name] = extra_value playbook = path_extend(playbook) if not os.path.isfile(playbook): raise ValueError(f"Invalid playbook file `{playbook}`") node_variables = defaultdict(dict) for nvar in node_vars: if ':' not in nvar: raise ValueError(f"Invalid value for node argument: `{nvar}`. " f"Did you forgot ':' character?") node_id, node_extra_args = nvar.split(':')[0], ':'.join( nvar.split(':')[1:]) for narg in node_extra_args.split(','): if '=' not in narg: raise ValueError( f"Invalid value for extra argument: '{narg}'. " f"Did you forgot '=' character?") extra_name, extra_value = narg.split('=')[0], '='.join( narg.split('=')[1:]) node_variables[node_id].update({extra_name: extra_value}) node_variables = defaultdict_to_dict(node_variables) inventory = AnsiblePlaybookExecutor.create_inventory( nodes, cluster_defaults.base_defaults.private_path, {}, node_variables) executor = AnsiblePlaybookExecutor( playbook, cluster_defaults.base_defaults.private_path, inventory, extra_args) result = executor.run() if not result.ok: logger.error(f"Playbook {playbook} did not executed successfully...") return 1 print(str_at_middle("Execution Summary", 80)) for node_id in sorted(list(result.hosts.keys())): r = result.hosts[node_id] print(f"{node_id}: {'ok' if r else 'not ok'}") print( f"Playbook at `{playbook}` were executed in {len(result.hosts)} nodes") return 0
def perform_action(self, role_name: str, action_name: str, hosts_node_map: Union[List[str], Dict[str, List[str]]], host_vars: Dict[str, Dict[str, str]] = None, node_vars: Dict[str, Dict[str, str]] = None, extra_args: Dict[str, str] = None, quiet: bool = False, validate_nodes_in_role: bool = True) -> \ AnsiblePlaybookExecutor.PlaybookResult: """ :param role_name: :param action_name: """ host_vars = host_vars or dict() node_vars = node_vars or dict() extra_args = extra_args or dict() if role_name not in self.roles: raise InvalidRoleError(role_name) role: Role = self.roles[role_name] if action_name not in role.actions: raise InvalidActionError(role_name, action_name) action = role.actions[action_name] # Check hosts_node_map variable if not role.hosts: if type(hosts_node_map) is list: _inventory = {'': hosts_node_map} elif type(hosts_node_map) is dict: if '' not in hosts_node_map: raise ValueError("hosts_node_map variable must contain " "'None' key.") _inventory = {'': hosts_node_map['']} else: raise TypeError( f"hosts_node_map variable expects a list or a dict, " f"not a {type(hosts_node_map)}") else: if type(hosts_node_map) is not dict: raise TypeError( f"As role {role_name} defines hosts, hosts_node_map " f"variable expects a dict, not a {type(hosts_node_map)}") _inventory = dict() for hname, node_list in hosts_node_map.items(): if hname not in role.hosts: raise InvalidHostError(role_name, hname) _inventory[hname] = node_list # Expand node_ids to NodeDescriptors inventory: Dict[str, List[NodeDescriptor]] = { host_name: self.node_repository.get_nodes_by_id(list_nodes) for host_name, list_nodes in _inventory.items() } if validate_nodes_in_role: self._check_nodes_role(role_name, inventory) if not role.hosts: inventory = {role_name: inventory['']} # Check if every required role's action variable is informed via extra_args for var in action.vars: if not var.optional: if var.name not in extra_args: raise MissingActionVariableError(role_name, action_name, var.name) # Expand playbook path playbook_file = path_extend(self.roles_dir, action.playbook) # Create ansible-like inventory inventory = AnsiblePlaybookExecutor.create_inventory( inventory, self.private_dir, host_vars, node_vars) # Crate playbook executor and run playbook = AnsiblePlaybookExecutor(playbook_file, self.private_dir, inventory, extra_args, quiet=quiet) logger.info(f'Executing playbook {playbook_file} with inventory: ' f'{inventory}') result = playbook.run() return result
import inspect import os import sys import pkgutil import traceback from typing import List import click from clap.utils import get_logger, setup_log, Singleton, path_extend if 'CLAP_PATH' not in os.environ: os.environ['CLAP_PATH'] = path_extend('~', '.clap') class ArgumentError(Exception): pass class Defaults(metaclass=Singleton): def __init__(self): self.verbosity: int = 0 self.clap_path: str = path_extend(os.environ.get('CLAP_PATH')) self.configs_path: str = path_extend( os.environ.get('CLAP_PATH'), 'configs') self.private_path: str = path_extend( os.environ.get('CLAP_PATH'), 'private') self.storage_path: str = path_extend( os.environ.get('CLAP_PATH'), 'storage')