def run_access_test(dry_run=False):
if dry_run:
return
CONFIG.instance_type = "t3.medium"
CONFIG.run_in_background = False
CONFIG.persist = False
try:
# Send test file to s3 results path and delete it
profile = CONFIG.aws_profile
region = CONFIG.region_name
results_path = CONFIG.s3_results_path
subprocess.check_output(
"echo 'Hello World' > nimbo-access-test.txt", shell=True
)
command = s3_cp_command("nimbo-access-test.txt", results_path)
subprocess.check_output(command, shell=True)
command = f"aws s3 ls {results_path} --profile {profile} --region {region}"
subprocess.check_output(command, shell=True)
command = (
f"aws s3 rm {results_path}/nimbo-access-test.txt "
f"--profile {profile} --region {region}"
)
subprocess.check_output(command, shell=True)
print(
"You have the necessary S3 read/write permissions from your computer \u2713"
)
except subprocess.CalledProcessError:
print("\nError.")
sys.exit(1)
# access.verify_nimbo_instance_profile(session)
# print("Instance profile 'NimboInstanceProfile' found \u2713")
# Launch instance with new volume for anaconda
print("Launching test instance... ", end="", flush=True)
ec2 = CONFIG.get_session().client("ec2")
instance = launch_instance(ec2)
instance_id = instance["InstanceId"]
try:
# Wait for the instance to be running
wait_for_instance_running(instance_id)
print(f"Instance running. Instance creation allowed \u2713")
print(f"InstanceId: {instance_id}")
print()
print("Trying to delete this instance...")
utils.delete_instance(instance_id)
print("Instance deletion allowed \u2713")
print("\nLaunching another instance...")
instance = launch_instance(ec2)
instance_id = instance["InstanceId"]
print(f"Instance running. InstanceId: {instance_id}")
time.sleep(5)
host = utils.check_instance_host(instance_id)
ssh = (
f"ssh -i {CONFIG.instance_key} -o 'StrictHostKeyChecking no' "
"-o ServerAliveInterval=20"
)
scp = f"scp -i {CONFIG.instance_key} -o 'StrictHostKeyChecking no'"
block_until_ssh_ready(host)
print("Instance key allows ssh access to remote instance \u2713")
print("Security group allows ssh access to remote instance \u2713")
write_nimbo_vars()
subprocess.check_output(
f"{scp} {CONFIG.nimbo_config_file} {NIMBO_VARS} ubuntu@{host}:/home/ubuntu/",
shell=True,
)
run_remote_script(ssh, scp, host, instance_id, "", "remote_s3_test.sh")
print("The instance profile has the required S3 and EC2 permissions \u2713")
print("\nEverything working \u2713")
print("Instance has been deleted.")
except BaseException as e:
if type(e) != KeyboardInterrupt and type(e) != subprocess.CalledProcessError:
print(e)
if not CONFIG.persist:
print(f"Deleting instance {instance_id} (from local)...")
utils.delete_instance(instance_id)
sys.exit(1)
def run_job(job_cmd, dry_run=False):
if dry_run:
return {"message": job_cmd + "_dry_run"}
# access.verify_nimbo_instance_profile(session)
# Launch instance with new volume for anaconda
print("Launching instance... ", end="", flush=True)
ec2 = CONFIG.get_session().client("ec2")
telemetry.record_event("run")
start_t = time.monotonic()
instance = launch_instance(ec2)
instance_id = instance["InstanceId"]
try:
# Wait for the instance to be running
wait_for_instance_running(instance_id)
end_t = time.monotonic()
print(f"Instance running. ({round((end_t-start_t), 2)}s)")
print(f"InstanceId: {instance_id}")
print()
time.sleep(5)
host = utils.check_instance_host(instance_id)
block_until_ssh_ready(host)
if job_cmd == "_nimbo_launch":
print(f"Run 'nimbo ssh {instance_id}' to log onto the instance")
return {"message": job_cmd + "_success", "instance_id": instance_id}
ssh = (
f"ssh -i {CONFIG.instance_key} -o 'StrictHostKeyChecking no'"
" -o ServerAliveInterval=20 "
)
scp = f"scp -i {CONFIG.instance_key} -o 'StrictHostKeyChecking no'"
local_env = "/tmp/local_env.yml"
user_conda_yml = CONFIG.conda_env
# TODO: Replace this with shutil
subprocess.check_output(f"cp {user_conda_yml} {local_env}", shell=True)
# Send conda env yaml and setup scripts to instance
print("\nSyncing conda, config, and setup files...")
write_nimbo_vars()
# Create project folder and send env and config files there
subprocess.check_output(f"{ssh} ubuntu@{host} mkdir project", shell=True)
subprocess.check_output(
f"{scp} {local_env} {CONFIG.nimbo_config_file} {NIMBO_VARS}"
f" ubuntu@{host}:/home/ubuntu/project/",
shell=True,
)
# Sync code with instance
print("\nSyncing code...")
sync_code(host)
# Run remote_setup script on instance
run_remote_script(ssh, scp, host, instance_id, job_cmd, "remote_setup.sh")
return {"message": job_cmd + "_success", "instance_id": instance_id}
except BaseException as e:
if type(e) != KeyboardInterrupt and type(e) != subprocess.CalledProcessError:
print(e)
if not CONFIG.persist:
print(f"Deleting instance {instance_id} (from local)...")
utils.delete_instance(instance_id)
return {"message": job_cmd + "_error", "instance_id": instance_id}
def delete_instance(instance_id, dry_run):
"""Terminates an instance by INSTANCE_ID."""
utils.delete_instance(instance_id, dry_run)
def delete_instance(instance_id, dry_run):
"""Terminates an instance by INSTANCE_ID."""
session, config = get_session_and_config_minimal()
utils.delete_instance(session, instance_id, dry_run)
def run_access_test(session, config, dry_run=False):
if dry_run:
return
config["instance_type"] = "t3.medium"
config["run_in_background"] = False
config["persist"] = False
try:
# Send test file to s3 results path and delete it
profile = config["aws_profile"]
region = config["region_name"]
results_path = config["s3_results_path"]
subprocess.check_output("echo 'Hellow World' > nimbo-access-test.txt",
shell=True)
command = f"aws s3 cp nimbo-access-test.txt {results_path} --profile {profile} --region {region}"
subprocess.check_output(command, shell=True)
command = f"aws s3 ls {results_path} --profile {profile} --region {region}"
subprocess.check_output(command, shell=True)
command = f"aws s3 rm {results_path}/nimbo-access-test.txt --profile {profile} --region {region}"
subprocess.check_output(command, shell=True)
print(
"You have the necessary S3 read/write permissions from your computer \u2713"
)
except subprocess.CalledProcessError:
print("\nError.")
sys.exit(1)
# access.verify_nimbo_instance_profile(session)
# print("Instance profile 'NimboInstanceProfile' found \u2713")
# Launch instance with new volume for anaconda
print("Launching test instance... ", end="", flush=True)
ec2 = session.client("ec2")
# print(userdata)
start_t = time.time()
instance = launch_instance(ec2, config)
instance_id = instance["InstanceId"]
try:
# Wait for the instance to be running
wait_for_instance_running(session, config, instance_id)
end_t = time.time()
print(f"Instance running. Instance creation allowed \u2713")
print(f"InstanceId: {instance_id}")
print()
print("Trying to delete this instance...")
utils.delete_instance(session, instance_id)
print("Instance deletion allowed \u2713")
print("\nLaunching another instance...")
instance = launch_instance(ec2, config)
instance_id = instance["InstanceId"]
print(f"Instance running. InstanceId: {instance_id}")
INSTANCE_KEY = config["instance_key"] + ".pem"
time.sleep(5)
host = utils.check_instance_host(session, config, instance_id)
ssh = f"ssh -i {INSTANCE_KEY} -o 'StrictHostKeyChecking no' -o ServerAliveInterval=20"
scp = f"scp -i {INSTANCE_KEY} -o 'StrictHostKeyChecking no'"
block_until_ssh_ready(host)
print("Instance key allows ssh access to remote instance \u2713")
print("Security group allows ssh access to remote instance \u2713")
subprocess.check_output(
f"{scp} {CONFIG} "
f"ubuntu@{host}:/home/ubuntu/", shell=True)
run_remote_script(ssh, scp, host, instance_id, "", "remote_s3_test.sh",
config)
print(
"The instance profile has the required S3 and EC2 permissions \u2713"
)
print("\nEverything working \u2713")
print("Instance has been deleted.")
except Exception as e:
if type(e) != KeyboardInterrupt and type(
e) != subprocess.CalledProcessError:
print(e)
if not config["persist"]:
print(f"Deleting instance {instance_id} (from local)...")
utils.delete_instance(session, instance_id)
sys.exit(1)
def run_job(session, config, job_cmd, dry_run=False):
if dry_run:
return {"message": job_cmd + "_dry_run"}
print("Config:")
pprint(config)
print("Job command:", job_cmd)
# access.verify_nimbo_instance_profile(session)
# Launch instance with new volume for anaconda
print("Launching instance... ", end="", flush=True)
ec2 = session.client("ec2")
start_t = time.time()
instance = launch_instance(ec2, config)
instance_id = instance["InstanceId"]
try:
# Wait for the instance to be running
wait_for_instance_running(session, config, instance_id)
end_t = time.time()
print(f"Instance running. ({round((end_t-start_t), 2)}s)")
print(f"InstanceId: {instance_id}")
print()
INSTANCE_KEY = config["instance_key"] + ".pem"
time.sleep(5)
host = utils.check_instance_host(session, config, instance_id)
block_until_ssh_ready(host)
if job_cmd == "_nimbo_launch":
print(f"Run 'nimbo ssh {instance_id}' to log onto the instance")
return {
"message": job_cmd + "_success",
"instance_id": instance_id
}
ssh = f"ssh -i {INSTANCE_KEY} -o 'StrictHostKeyChecking no' -o ServerAliveInterval=20 "
scp = f"scp -i {INSTANCE_KEY} -o 'StrictHostKeyChecking no'"
LOCAL_ENV = "local_env.yml"
user_conda_yml = config["conda_env"]
output = subprocess.check_output(f"cp {user_conda_yml} local_env.yml",
shell=True)
# Send conda env yaml and setup scripts to instance
print("\nSyncing conda, config, and setup files...")
# Create project folder and send env and config files there
subprocess.check_output(f"{ssh} ubuntu@{host} "
f"mkdir project",
shell=True)
subprocess.check_output(
f"{scp} {LOCAL_ENV} {CONFIG} "
f"ubuntu@{host}:/home/ubuntu/project/",
shell=True,
)
subprocess.check_output(f"rm {LOCAL_ENV}", shell=True)
# Sync code with instance
print("\nSyncing code...")
sync_code(host, INSTANCE_KEY)
# Run remote_setup script on instance
run_remote_script(ssh, scp, host, instance_id, job_cmd,
"remote_setup.sh", config)
return {"message": job_cmd + "_success", "instance_id": instance_id}
except Exception as e:
if type(e) != KeyboardInterrupt and type(
e) != subprocess.CalledProcessError:
print(e)
if not config["persist"]:
print(f"Deleting instance {instance_id} (from local)...")
utils.delete_instance(session, instance_id)
return {"message": job_cmd + "_error", "instance_id": instance_id}