def run_access_test(dry_run=False): if dry_run: return CONFIG.instance_type = "t3.medium" CONFIG.run_in_background = False CONFIG.persist = False try: # Send test file to s3 results path and delete it profile = CONFIG.aws_profile region = CONFIG.region_name results_path = CONFIG.s3_results_path subprocess.check_output( "echo 'Hello World' > nimbo-access-test.txt", shell=True ) command = s3_cp_command("nimbo-access-test.txt", results_path) subprocess.check_output(command, shell=True) command = f"aws s3 ls {results_path} --profile {profile} --region {region}" subprocess.check_output(command, shell=True) command = ( f"aws s3 rm {results_path}/nimbo-access-test.txt " f"--profile {profile} --region {region}" ) subprocess.check_output(command, shell=True) print( "You have the necessary S3 read/write permissions from your computer \u2713" ) except subprocess.CalledProcessError: print("\nError.") sys.exit(1) # access.verify_nimbo_instance_profile(session) # print("Instance profile 'NimboInstanceProfile' found \u2713") # Launch instance with new volume for anaconda print("Launching test instance... ", end="", flush=True) ec2 = CONFIG.get_session().client("ec2") instance = launch_instance(ec2) instance_id = instance["InstanceId"] try: # Wait for the instance to be running wait_for_instance_running(instance_id) print(f"Instance running. Instance creation allowed \u2713") print(f"InstanceId: {instance_id}") print() print("Trying to delete this instance...") utils.delete_instance(instance_id) print("Instance deletion allowed \u2713") print("\nLaunching another instance...") instance = launch_instance(ec2) instance_id = instance["InstanceId"] print(f"Instance running. InstanceId: {instance_id}") time.sleep(5) host = utils.check_instance_host(instance_id) ssh = ( f"ssh -i {CONFIG.instance_key} -o 'StrictHostKeyChecking no' " "-o ServerAliveInterval=20" ) scp = f"scp -i {CONFIG.instance_key} -o 'StrictHostKeyChecking no'" block_until_ssh_ready(host) print("Instance key allows ssh access to remote instance \u2713") print("Security group allows ssh access to remote instance \u2713") write_nimbo_vars() subprocess.check_output( f"{scp} {CONFIG.nimbo_config_file} {NIMBO_VARS} ubuntu@{host}:/home/ubuntu/", shell=True, ) run_remote_script(ssh, scp, host, instance_id, "", "remote_s3_test.sh") print("The instance profile has the required S3 and EC2 permissions \u2713") print("\nEverything working \u2713") print("Instance has been deleted.") except BaseException as e: if type(e) != KeyboardInterrupt and type(e) != subprocess.CalledProcessError: print(e) if not CONFIG.persist: print(f"Deleting instance {instance_id} (from local)...") utils.delete_instance(instance_id) sys.exit(1)
def run_job(job_cmd, dry_run=False): if dry_run: return {"message": job_cmd + "_dry_run"} # access.verify_nimbo_instance_profile(session) # Launch instance with new volume for anaconda print("Launching instance... ", end="", flush=True) ec2 = CONFIG.get_session().client("ec2") telemetry.record_event("run") start_t = time.monotonic() instance = launch_instance(ec2) instance_id = instance["InstanceId"] try: # Wait for the instance to be running wait_for_instance_running(instance_id) end_t = time.monotonic() print(f"Instance running. ({round((end_t-start_t), 2)}s)") print(f"InstanceId: {instance_id}") print() time.sleep(5) host = utils.check_instance_host(instance_id) block_until_ssh_ready(host) if job_cmd == "_nimbo_launch": print(f"Run 'nimbo ssh {instance_id}' to log onto the instance") return {"message": job_cmd + "_success", "instance_id": instance_id} ssh = ( f"ssh -i {CONFIG.instance_key} -o 'StrictHostKeyChecking no'" " -o ServerAliveInterval=20 " ) scp = f"scp -i {CONFIG.instance_key} -o 'StrictHostKeyChecking no'" local_env = "/tmp/local_env.yml" user_conda_yml = CONFIG.conda_env # TODO: Replace this with shutil subprocess.check_output(f"cp {user_conda_yml} {local_env}", shell=True) # Send conda env yaml and setup scripts to instance print("\nSyncing conda, config, and setup files...") write_nimbo_vars() # Create project folder and send env and config files there subprocess.check_output(f"{ssh} ubuntu@{host} mkdir project", shell=True) subprocess.check_output( f"{scp} {local_env} {CONFIG.nimbo_config_file} {NIMBO_VARS}" f" ubuntu@{host}:/home/ubuntu/project/", shell=True, ) # Sync code with instance print("\nSyncing code...") sync_code(host) # Run remote_setup script on instance run_remote_script(ssh, scp, host, instance_id, job_cmd, "remote_setup.sh") return {"message": job_cmd + "_success", "instance_id": instance_id} except BaseException as e: if type(e) != KeyboardInterrupt and type(e) != subprocess.CalledProcessError: print(e) if not CONFIG.persist: print(f"Deleting instance {instance_id} (from local)...") utils.delete_instance(instance_id) return {"message": job_cmd + "_error", "instance_id": instance_id}
def delete_instance(instance_id, dry_run): """Terminates an instance by INSTANCE_ID.""" utils.delete_instance(instance_id, dry_run)
def delete_instance(instance_id, dry_run): """Terminates an instance by INSTANCE_ID.""" session, config = get_session_and_config_minimal() utils.delete_instance(session, instance_id, dry_run)
def run_access_test(session, config, dry_run=False): if dry_run: return config["instance_type"] = "t3.medium" config["run_in_background"] = False config["persist"] = False try: # Send test file to s3 results path and delete it profile = config["aws_profile"] region = config["region_name"] results_path = config["s3_results_path"] subprocess.check_output("echo 'Hellow World' > nimbo-access-test.txt", shell=True) command = f"aws s3 cp nimbo-access-test.txt {results_path} --profile {profile} --region {region}" subprocess.check_output(command, shell=True) command = f"aws s3 ls {results_path} --profile {profile} --region {region}" subprocess.check_output(command, shell=True) command = f"aws s3 rm {results_path}/nimbo-access-test.txt --profile {profile} --region {region}" subprocess.check_output(command, shell=True) print( "You have the necessary S3 read/write permissions from your computer \u2713" ) except subprocess.CalledProcessError: print("\nError.") sys.exit(1) # access.verify_nimbo_instance_profile(session) # print("Instance profile 'NimboInstanceProfile' found \u2713") # Launch instance with new volume for anaconda print("Launching test instance... ", end="", flush=True) ec2 = session.client("ec2") # print(userdata) start_t = time.time() instance = launch_instance(ec2, config) instance_id = instance["InstanceId"] try: # Wait for the instance to be running wait_for_instance_running(session, config, instance_id) end_t = time.time() print(f"Instance running. Instance creation allowed \u2713") print(f"InstanceId: {instance_id}") print() print("Trying to delete this instance...") utils.delete_instance(session, instance_id) print("Instance deletion allowed \u2713") print("\nLaunching another instance...") instance = launch_instance(ec2, config) instance_id = instance["InstanceId"] print(f"Instance running. InstanceId: {instance_id}") INSTANCE_KEY = config["instance_key"] + ".pem" time.sleep(5) host = utils.check_instance_host(session, config, instance_id) ssh = f"ssh -i {INSTANCE_KEY} -o 'StrictHostKeyChecking no' -o ServerAliveInterval=20" scp = f"scp -i {INSTANCE_KEY} -o 'StrictHostKeyChecking no'" block_until_ssh_ready(host) print("Instance key allows ssh access to remote instance \u2713") print("Security group allows ssh access to remote instance \u2713") subprocess.check_output( f"{scp} {CONFIG} " f"ubuntu@{host}:/home/ubuntu/", shell=True) run_remote_script(ssh, scp, host, instance_id, "", "remote_s3_test.sh", config) print( "The instance profile has the required S3 and EC2 permissions \u2713" ) print("\nEverything working \u2713") print("Instance has been deleted.") except Exception as e: if type(e) != KeyboardInterrupt and type( e) != subprocess.CalledProcessError: print(e) if not config["persist"]: print(f"Deleting instance {instance_id} (from local)...") utils.delete_instance(session, instance_id) sys.exit(1)
def run_job(session, config, job_cmd, dry_run=False): if dry_run: return {"message": job_cmd + "_dry_run"} print("Config:") pprint(config) print("Job command:", job_cmd) # access.verify_nimbo_instance_profile(session) # Launch instance with new volume for anaconda print("Launching instance... ", end="", flush=True) ec2 = session.client("ec2") start_t = time.time() instance = launch_instance(ec2, config) instance_id = instance["InstanceId"] try: # Wait for the instance to be running wait_for_instance_running(session, config, instance_id) end_t = time.time() print(f"Instance running. ({round((end_t-start_t), 2)}s)") print(f"InstanceId: {instance_id}") print() INSTANCE_KEY = config["instance_key"] + ".pem" time.sleep(5) host = utils.check_instance_host(session, config, instance_id) block_until_ssh_ready(host) if job_cmd == "_nimbo_launch": print(f"Run 'nimbo ssh {instance_id}' to log onto the instance") return { "message": job_cmd + "_success", "instance_id": instance_id } ssh = f"ssh -i {INSTANCE_KEY} -o 'StrictHostKeyChecking no' -o ServerAliveInterval=20 " scp = f"scp -i {INSTANCE_KEY} -o 'StrictHostKeyChecking no'" LOCAL_ENV = "local_env.yml" user_conda_yml = config["conda_env"] output = subprocess.check_output(f"cp {user_conda_yml} local_env.yml", shell=True) # Send conda env yaml and setup scripts to instance print("\nSyncing conda, config, and setup files...") # Create project folder and send env and config files there subprocess.check_output(f"{ssh} ubuntu@{host} " f"mkdir project", shell=True) subprocess.check_output( f"{scp} {LOCAL_ENV} {CONFIG} " f"ubuntu@{host}:/home/ubuntu/project/", shell=True, ) subprocess.check_output(f"rm {LOCAL_ENV}", shell=True) # Sync code with instance print("\nSyncing code...") sync_code(host, INSTANCE_KEY) # Run remote_setup script on instance run_remote_script(ssh, scp, host, instance_id, job_cmd, "remote_setup.sh", config) return {"message": job_cmd + "_success", "instance_id": instance_id} except Exception as e: if type(e) != KeyboardInterrupt and type( e) != subprocess.CalledProcessError: print(e) if not config["persist"]: print(f"Deleting instance {instance_id} (from local)...") utils.delete_instance(session, instance_id) return {"message": job_cmd + "_error", "instance_id": instance_id}