def test_scp_recursive_async(sshd_manager, loop): with sshd_manager.run(1) as sshd_ports: workspace = str(sshd_manager.tmpdir) id = uuid.uuid4().hex pkgpanda.util.write_string(workspace + '/recursive_pilot.txt', id) runner = MultiRunner(['127.0.0.1:{}'.format(port) for port in sshd_ports], ssh_user=getpass.getuser(), ssh_key_path=sshd_manager.key_path) host_port = ['127.0.0.1:{}'.format(port) for port in sshd_ports] chain = CommandChain('test') chain.add_copy(workspace + '/recursive_pilot.txt', workspace + '/recursive_pilot.txt.copied', recursive=True) try: copy_results = loop.run_until_complete(runner.run_commands_chain_async([chain], block=True, state_json_dir=workspace)) finally: loop.close() dest_path = workspace + '/recursive_pilot.txt.copied' assert os.path.exists(dest_path) assert os.path.isfile(dest_path) assert len(copy_results) == 1 assert pkgpanda.util.load_string(dest_path) == id for host_result in copy_results: for command_result in host_result: for host, process_result in command_result.items(): assert process_result['returncode'] == 0, process_result['stderr'] assert host in host_port assert '/usr/bin/scp' in process_result['cmd'] assert '-r' in process_result['cmd'] assert workspace + '/recursive_pilot.txt' in process_result['cmd']
def test_scp_async(sshd_manager, loop): with sshd_manager.run(1) as sshd_ports: workspace = str(sshd_manager.tmpdir) id = uuid.uuid4().hex pkgpanda.util.write_string(workspace + '/pilot.txt', id) runner = MultiRunner( ['127.0.0.1:{}'.format(port) for port in sshd_ports], ssh_user=getpass.getuser(), ssh_key_path=sshd_manager.key_path) host_port = ['127.0.0.1:{}'.format(port) for port in sshd_ports] chain = CommandChain('test') chain.add_copy(workspace + '/pilot.txt', workspace + '/pilot.txt.copied') try: copy_results = loop.run_until_complete( runner.run_commands_chain_async([chain], block=True, state_json_dir=workspace)) finally: loop.close() assert len(copy_results) == 1 assert os.path.isfile(workspace + '/pilot.txt.copied') assert pkgpanda.util.load_string(workspace + '/pilot.txt.copied') == id for host_result in copy_results: for command_result in host_result: for host, process_result in command_result.items(): assert process_result['returncode'] == 0, process_result[ 'stderr'] assert host in host_port assert '/usr/bin/scp' in process_result['cmd'] assert workspace + '/pilot.txt' in process_result['cmd']
def get_local_addresses(ssh_runner, remote_dir): """Uses checked-in IP detect script to report local IP mapping Also functions as a test to verify cluster is up and accessible Args: ssh_runner: instance of ssh.ssh_runner.MultiRunner remote_dir (str): path on hosts for ip-detect to be copied and run in Returns: dict[public_IP] = local_IP """ def remote(path): return remote_dir + '/' + path ip_detect_script = pkg_resources.resource_filename('gen', 'ip-detect/aws.sh') ip_map_chain = CommandChain('ip_map') ip_map_chain.add_copy(ip_detect_script, remote('ip-detect.sh')) ip_map_chain.add_execute(['bash', remote('ip-detect.sh')]) mapping = {} result = run_loop(ssh_runner, ip_map_chain) # Check the running was successful check_results(copy.deepcopy(result)) # Gather the local IP addresses for host_result in result: host, data = host_result[-1].popitem() # Grab the last command trigging the script local_ip = data['stdout'][0].rstrip() assert local_ip != '', "Didn't get a valid IP for host {}:\n{}".format(host, data) mapping[host.split(":")[0]] = local_ip return mapping
def get_local_addresses(ssh_runner, remote_dir): """Uses checked-in IP detect script to report local IP mapping Also functions as a test to verify cluster is up and accessible Args: ssh_runner: instance of ssh.ssh_runner.MultiRunner remote_dir (str): path on hosts for ip-detect to be copied and run in Returns: dict[public_IP] = local_IP """ def remote(path): return remote_dir + '/' + path ip_detect_script = pkg_resources.resource_filename('gen', 'ip-detect/aws.sh') ip_map_chain = CommandChain('ip_map') ip_map_chain.add_copy(ip_detect_script, remote('ip-detect.sh')) ip_map_chain.add_execute(['bash', remote('ip-detect.sh')]) mapping = {} result = run_loop(ssh_runner, ip_map_chain) # Check the running was successful check_results(copy.deepcopy(result)) # Gather the local IP addresses for host_result in result: host, data = host_result[-1].popitem( ) # Grab the last command trigging the script local_ip = data['stdout'][0].rstrip() assert local_ip != '', "Didn't get a valid IP for host {}:\n{}".format( host, data) mapping[host.split(":")[0]] = local_ip return mapping
def test_command_chain(): chain = CommandChain('test') chain.add_execute(['cmd2']) chain.add_copy('/local', '/remote') chain.prepend_command(['cmd1']) chain.add_execute(['cmd3']) assert chain.get_commands() == [('execute', ['cmd1'], None, None), ('execute', ['cmd2'], None, None), ('copy', '/local', '/remote', False, False, None), ('execute', ['cmd3'], None, None)]
def test_command_chain(): chain = CommandChain('test') chain.add_execute(['cmd2']) chain.add_copy('/local', '/remote') chain.prepend_command(['cmd1']) chain.add_execute(['cmd3']) assert chain.get_commands() == [ ('execute', ['cmd1'], None, None), ('execute', ['cmd2'], None, None), ('copy', '/local', '/remote', False, False, None), ('execute', ['cmd3'], None, None) ]
def test_setup(ssh_runner, registry, remote_dir, use_zk_backend): """Transfer resources and issues commands on host to build test app, host it on a docker registry, and prepare the integration_test container Args: ssh_runner: instance of ssh.ssh_runner.MultiRunner registry (str): address of registry host that is visible to test nodes remote_dir (str): path to be used for setup and file transfer on host Returns: result from async chain that can be checked later for success """ test_server_docker = pkg_filename('docker/test_server/Dockerfile') test_server_script = pkg_filename('docker/test_server/test_server.py') pytest_docker = pkg_filename('docker/py.test/Dockerfile') test_script = pkg_filename('integration_test.py') test_setup_chain = CommandChain('test_setup') if use_zk_backend: test_setup_chain.add_execute([ 'sudo', 'docker', 'run', '-d', '-p', '2181:2181', '-p', '2888:2888', '-p', '3888:3888', 'jplock/zookeeper' ]) def remote(path): return remote_dir + '/' + path # Create test application test_setup_chain.add_execute(['mkdir', '-p', remote('test_server')]) test_setup_chain.add_copy(test_server_docker, remote('test_server/Dockerfile')) test_setup_chain.add_copy(test_server_script, remote('test_server/test_server.py')) test_setup_chain.add_execute([ 'docker', 'run', '-d', '-p', '5000:5000', '--restart=always', '--name', 'registry', 'registry:2' ]) test_setup_chain.add_execute([ 'cd', remote('test_server'), '&&', 'docker', 'build', '-t', '{}:5000/test_server'.format(registry), '.' ]) test_setup_chain.add_execute( ['docker', 'push', "{}:5000/test_server".format(registry)]) test_setup_chain.add_execute(['rm', '-rf', remote('test_server')]) # Create pytest/integration test instance on remote test_setup_chain.add_execute(['mkdir', '-p', remote('py.test')]) test_setup_chain.add_copy(pytest_docker, remote('py.test/Dockerfile')) test_setup_chain.add_copy(test_script, remote('integration_test.py')) test_setup_chain.add_execute([ 'cd', remote('py.test'), '&&', 'docker', 'build', '-t', 'py.test', '.' ]) test_setup_chain.add_execute(['rm', '-rf', remote('py.test')]) check_results(run_loop(ssh_runner, test_setup_chain))
def test_setup(ssh_runner, registry, remote_dir, use_zk_backend): """Transfer resources and issues commands on host to build test app, host it on a docker registry, and prepare the integration_test container Args: ssh_runner: instance of ssh.ssh_runner.MultiRunner registry (str): address of registry host that is visible to test nodes remote_dir (str): path to be used for setup and file transfer on host Returns: result from async chain that can be checked later for success """ test_server_docker = pkg_filename('docker/test_server/Dockerfile') test_server_script = pkg_filename('docker/test_server/test_server.py') pytest_docker = pkg_filename('docker/py.test/Dockerfile') test_script = pkg_filename('integration_test.py') test_setup_chain = CommandChain('test_setup') if use_zk_backend: test_setup_chain.add_execute([ 'sudo', 'docker', 'run', '-d', '-p', '2181:2181', '-p', '2888:2888', '-p', '3888:3888', 'jplock/zookeeper']) def remote(path): return remote_dir + '/' + path # Create test application test_setup_chain.add_execute(['mkdir', '-p', remote('test_server')]) test_setup_chain.add_copy(test_server_docker, remote('test_server/Dockerfile')) test_setup_chain.add_copy(test_server_script, remote('test_server/test_server.py')) test_setup_chain.add_execute([ 'docker', 'run', '-d', '-p', '5000:5000', '--restart=always', '--name', 'registry', 'registry:2']) test_setup_chain.add_execute([ 'cd', remote('test_server'), '&&', 'docker', 'build', '-t', '{}:5000/test_server'.format(registry), '.']) test_setup_chain.add_execute(['docker', 'push', "{}:5000/test_server".format(registry)]) test_setup_chain.add_execute(['rm', '-rf', remote('test_server')]) # Create pytest/integration test instance on remote test_setup_chain.add_execute(['mkdir', '-p', remote('py.test')]) test_setup_chain.add_copy(pytest_docker, remote('py.test/Dockerfile')) test_setup_chain.add_copy(test_script, remote('integration_test.py')) test_setup_chain.add_execute([ 'cd', remote('py.test'), '&&', 'docker', 'build', '-t', 'py.test', '.']) test_setup_chain.add_execute(['rm', '-rf', remote('py.test')]) check_results(run_loop(ssh_runner, test_setup_chain))