def update(site: str, restart_service: bool, ssh_user: str, private_key: str, python_interpreter: str, ssh_common_args: str) -> bool: ar = AnsibleRunner( inventory=[f'{playbook_path}/inventory.d/90_os_inventory.sh']) hosts = ar.inventory_manager.get_hosts() user_hosts = [get_host_info(h) for h in hosts if is_user_host(h, site)] infra_host = get_infra_host(hosts, site) if not infra_host: log.error(f'No r_infra host found for site {site}') return False rancher_host = get_rancher_host(hosts) rancher_host.vars['ansible_ssh_user'] = ssh_user rancher_host.vars['ansible_ssh_private_key_file'] = private_key rancher_host.vars['ansible_python_interpreter'] = python_interpreter rancher_host.vars['ansible_ssh_common_args'] = ssh_common_args success = ar.play(f'{playbook_path}/update_landscaper.yml', extra_vars={ 'INFRA_HOST': infra_host.vars['ansible_ssh_host'], 'USER_HOSTS': user_hosts }) if success and restart_service: recap_stack([ 'restart', site, '--infrastructures', 'r_infra', '--services', 'landscaper' ]) return success
def install_deps(cluster_config): log_info("Installing dependencies for cluster_config: {}".format( cluster_config)) ansible_runner = AnsibleRunner(config=cluster_config) status = ansible_runner.run_ansible_playbook("os-level-modifications.yml") if status != 0: raise ProvisioningError("Failed to make os modifications") status = ansible_runner.run_ansible_playbook("install-common-tools.yml") if status != 0: raise ProvisioningError("Failed to install dependencies")
def connect_sites(playbook_path: str, inter_network: str, inter_ports: List[str], sites: List[str], site_addresses: List[str], name: str, network: str, router_ip: str) -> bool: p2p_network_name = _get_p2p_network_name(sites) p2p_subnet_name = _get_p2p_subnet_name(sites) network_name = _get_internal_network_name(name) subnet_name = _get_internal_subnet_name(name) router_name = _get_internal_router_name(name) site_routers = [_get_internal_router_name(s) for s in sites] ar = AnsibleRunner() print("network_name: " + network_name) print("subnet_name: " + subnet_name) print("network: " + network) print("router_name: " + router_name) print("router_ip: " + router_ip) return ar.play( playbook=f'{playbook_path}/connect_sites.yml', extra_vars={ 'INTER_NETWORK_NAME': p2p_network_name, 'INTER_SUBNET_NAME': p2p_subnet_name, 'INTER_NETWORK_ADDRESS': inter_network, 'SITES': [ { 'INTER_PORT': inter_ports[0], 'ROUTER_NAME': site_routers[0], 'OTHER_NETWORK_ADDRESS': site_addresses[1], 'OTHER_PORT': inter_ports[1] }, { 'INTER_PORT': inter_ports[1], 'ROUTER_NAME': site_routers[1], 'OTHER_NETWORK_ADDRESS': site_addresses[0], 'OTHER_PORT': inter_ports[0] } ], 'NETWORK_NAME': network_name, 'SUBNET_NAME': subnet_name, 'NETWORK_ADDRESS': network, 'ROUTER_NAME': router_name, 'ROUTER_IP': router_ip } )
def clean_cluster(cluster_config): log_info("Cleaning cluster: {}".format(cluster_config)) ansible_runner = AnsibleRunner(config=cluster_config) status = ansible_runner.run_ansible_playbook( "remove-previous-installs.yml") if status != 0: raise ProvisioningError("Failed to removed previous installs") # Clear firewall rules status = ansible_runner.run_ansible_playbook("flush-firewall.yml") if status != 0: raise ProvisioningError("Failed to flush firewall")
def memory_dump(victim): """ Wrapper function to create memory dump of victim vm with ansible args: victim - location of vagrantfile return: None """ runner = AnsibleRunner( './scenario_builder/forensics/playbooks/mem-dump-linux.yaml', hosts='./scenario_builder/forensics/hosts') runner.run() tmpfile = victim + 'mem-image.lime' os.rename(tmpfile, './mem-image.lime')
def disk_image(victim): """ Wrapper function to create disk image of victim vm with ansible args: victim - location of vagrantfile return: None """ tmpfile = victim + 'filesystem.image.gz' runner = AnsibleRunner( './scenario_builder/forensics/playbooks/diskimage-linux.yaml', hosts='./scenario_builder/forensics/hosts') runner.run() os.rename(tmpfile, './filesystem.image.gz')
def install_aws_credentials(cluster_config, aws_access_key_id, aws_secret_access_key): log_info("Installing aws credentials for cluster_config: {}".format(cluster_config)) ansible_runner = AnsibleRunner(config=cluster_config) status = ansible_runner.run_ansible_playbook( "install-aws-credentials.yml", extra_vars={ "aws_access_key_id": aws_access_key_id, "aws_secret_access_key": aws_secret_access_key, }, ) if status != 0: raise ProvisioningError("Failed to aws credentials")
def init_master(playbook_path: str, site: str, public_key: str, private_key: str, rancher_sites: List[str]) -> bool: ar = AnsibleRunner() vm_name = _get_rancher_master_vm_name(site) network_name = _get_internal_network_name(site) router_name = _get_internal_router_name(site) host_available = ar.play( playbook=f'{playbook_path}/init_master.yml', extra_vars={ 'VM_NAME': vm_name, 'NETWORK_NAME': network_name, 'ROUTER_NAME': router_name, 'SITE_NAME': site, 'PUBLIC_KEY_FILE': public_key, 'PRIVATE_KEY_FILE': private_key } ) if not host_available: return False host_vars = ar.inventory_manager.get_host(vm_name).get_vars() host = host_vars.get('ansible_host') python_interpreter = host_vars.get('ansible_python_interpreter') username = host_vars.get('ansible_user') print(rancher_sites) return init_rancher([ '--host', host, '--username', username, '--private-key-file', private_key, '--python-interpreter', python_interpreter, '--pip-executable', '/home/core/bin/pip', '--pip-as-non-root', '--update-config', '--rancher-username', 'recap', '--rancher-password', 'recap$123', '--rancher-env-name', 'recap', '--rancher-registry-url', 'omi-registry.e-technik.uni-ulm.de', '--rancher-registry-username', 'recap_pipeline', '--rancher-registry-password', '53qThb2ZDUaXc3L49bs8', '--rancher-sites', *rancher_sites ])
def install_nginx(cluster_config): """ Deploys nginx to nodes with the load_balancer tag 1. Get the sync_gateway endpoints from the cluster configuration 2. Use the endpoints to render the nginx config (resources/nginx_configs/nginx.conf) to distribute load across the running sync_gateways. i.e. If your 'cluster_config' has 2 sync_gateways, nginx will be setup to forward requests to both of the sync_gateways using a weighted round robin distribution. If you have 3, it will split the load between 3, etc ... 3. Deploy the config and install nginx on load_balancer nodes 4. Start the nginx service """ cluster = ClusterKeywords() # Set lb_enable to False to get the actual SG IPs for nginx.conf topology = cluster.get_cluster_topology(cluster_config, lb_enable=False) # Get sync_gateway enpoints from cluster_config # and build a string of upstream server definitions # upstream sync_gateway { # server 192.168.33.11:4984; # server 192.168.33.12:4984; # } upstream_definition = "" upstream_definition_admin = "" for sg in topology["sync_gateways"]: # string http:// to adhere to expected format for nginx.conf ip_port = sg["public"].replace("http://", "") ip_port_admin = sg["admin"].replace("http://", "") upstream_definition += "server {};\n".format(ip_port) upstream_definition_admin += "server {};\n".format(ip_port_admin) log_info("Upstream definition: {}".format(upstream_definition)) log_info("Upstream definition admin: {}".format(upstream_definition_admin)) ansible_runner = AnsibleRunner(cluster_config) status = ansible_runner.run_ansible_playbook( "install-nginx.yml", extra_vars={ "upstream_sync_gatways": upstream_definition, "upstream_sync_gatways_admin": upstream_definition_admin }) assert status == 0, "Failed to install nginx!"
def logs(logs_loc): """ Wrapper function to pull logs off of the victim vm with ansible args: logs_loc - location of the log files on the victim vm return: None """ logger = logging.getLogger('root') log_path = "log_path={}".format(logs_loc) logger.debug('Pulling logs from {}'.format(log_path)) runner = AnsibleRunner( './scenario_builder/forensics/playbooks/logs-linux.yaml', hosts='./scenario_builder/forensics/hosts', extra_var=log_path) runner.run() os.rename('./scenario_builder/forensics/playbooks/logs.zip', './logs.zip')
def add_site(playbook_path: str, name: str, network: str, router_ip: str) -> bool: ar = AnsibleRunner() network_name = _get_internal_network_name(name) subnet_name = _get_internal_subnet_name(name) router_name = _get_internal_router_name(name) return ar.play( playbook=f'{playbook_path}/add_site.yml', extra_vars={ 'NETWORK_NAME': network_name, 'SUBNET_NAME': subnet_name, 'NETWORK_ADDRESS': network, 'ROUTER_NAME': router_name, 'ROUTER_IP': router_ip } )
def main(argv: List[str]) -> bool: arg_parser = ArgumentParser( prog=recap_module_name, description='Add or Remove Hosts from RECAP Environment', formatter_class=ArgumentDefaultsHelpFormatter ) arg_parser.add_argument('action', choices=['add', 'remove']) arg_parser.add_argument('host') arg_parser.add_argument( '--infrastructure', choices=['r_user', 'r_infra', 'admin', 'global'], required=False) arg_parser.add_argument('--site', required=False) arg_parser.add_argument('--username', required=True) arg_parser.add_argument('--private-key-file', required=True) arg_parser.add_argument('--start-stack', action='store_true') arg_parser.add_argument('--python-interpreter', default='/usr/bin/python') arg_parser.add_argument('--pip-executable', default='pip') arg_parser.add_argument('--pip-as-non-root', action='store_true') arg_parser.add_argument('--proxy-host', required=False) arg_parser.add_argument('--extra-labels', required=False, type=labels_type) args = arg_parser.parse_args(argv) if args.action == 'add' and (args.infrastructure is None or args.site is None): arg_parser.error('add requires --infrastructure and --site') extra_labels = '' if args.extra_labels is not None: extra_labels = get_labels_string(args.extra_labels) if args.proxy_host is None: ssh_args = '-o StrictHostKeyChecking=no' else: ssh_args = f'-o StrictHostKeyChecking=no -o ProxyCommand="ssh -W %h:%p -q {args.username}@{args.proxy_host} -i {args.private_key_file} -o ControlMaster=auto -o ControlPersist=30m -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"' runner = AnsibleRunner() rancher_access_key_site = os.getenv('RANCHER_ACCESS_KEY_'+args.site) rancher_secret_key_site = os.getenv('RANCHER_SECRET_KEY_'+args.site) rancher_env_site = os.getenv('RANCHER_ENV_ID_'+args.site) rancher = RancherClient( rancher_env_site, rancher_access_key_site, rancher_secret_key_site) host = Host(address=args.host, username=args.username, private_key_file=args.private_key_file, variables={ 'ansible_ssh_common_args': ssh_args, 'ansible_python_interpreter': args.python_interpreter, 'type': args.infrastructure, 'site': args.site, 'extra_labels': extra_labels }) if args.action == 'add': success = add(host=host, ansible_runner=runner, rancher=rancher, pip_executable=args.pip_executable, pip_as_root=not args.pip_as_non_root) if success and args.start_stack: success = stack_cmd([ 'up', args.site, '--infrastructures', args.infrastructure ]) else: success = remove(host=host, ansible_runner=runner, rancher_client=rancher) return success
import os from ansible_runner import AnsibleRunner if __name__ == "__main__": usage = "usage: python stop_telegraf.py" try: cluster_config = os.environ["CLUSTER_CONFIG"] except KeyError as ke: print( "Make sure CLUSTER_CONFIG is defined and pointing to the configuration you would like to run against" ) raise KeyError( "CLUSTER_CONFIG not defined. Unable to stop telegraf collectors.") ansible_runner = AnsibleRunner(cluster_config) status = ansible_runner.run_ansible_playbook("stop-telegraf.yml") assert status == 0, "Failed to stop telegraf collectors"
def add_host(playbook_path: str, site: str, host_type: str, instance: int, public_key: str, private_key: str, vm_flavour: str, vm_availability_zone: str, extra_labels: Dict[str, str], is_master: bool = False) -> bool: ar = AnsibleRunner() vm_name = _get_host_vm_name(site, host_type, instance) network_name = _get_internal_network_name(site) master_host = _get_master_address() if "global" == host_type: current_playbook = f'{playbook_path}/add_host_fip.yml' else: current_playbook = f'{playbook_path}/add_host.yml' host_available = ar.play( playbook=current_playbook, extra_vars={ 'VM_NAME': vm_name, 'NETWORK_NAME': network_name, 'SITE_NAME': site, 'TYPE': host_type, 'MASTER_HOST': master_host, 'PUBLIC_KEY_FILE': public_key, 'PRIVATE_KEY_FILE': private_key, 'VM_FLAVOUR': vm_flavour, 'VM_AZ': vm_availability_zone } ) if not host_available: return False host_vars = ar.inventory_manager.get_host(vm_name).get_vars() host = host_vars.get('ansible_host') python_interpreter = host_vars.get('ansible_python_interpreter') username = host_vars.get('ansible_user') if is_master: extra_labels['master'] = 'true' if "global" == host_type: env(['set', "GLOBAL_ADDRESS", host]) agent_added = add_agent([ 'add', host, '--infrastructure', host_type, '--site', site, '--username', username, '--private-key-file', private_key, '--start-stack', '--python-interpreter', python_interpreter, '--pip-executable', '/home/core/bin/pip', '--pip-as-non-root', '--proxy-host', master_host, '--extra-labels', get_labels_string(extra_labels) or None ]) services = { 'admin': getenv('SERVICES_ADMIN').split(','), 'global': getenv('SERVICES_GLOBAL').split(','), 'r_infra': getenv('SERVICES_INFRA').split(','), 'r_user': getenv('SERVICES_USER').split(',') } # if agent_added and host_type == 'r_user': # result = update_landscaper(site=site, # restart_service=True, # ssh_user=username, # private_key=private_key, # python_interpreter=python_interpreter, # ssh_common_args='-o StrictHostKeyChecking=no') # if not result: # log.warning('There was an error updating the landscaper') return agent_added