async def setup(
i: Instance,
git_rev: str,
) -> None:
def is_ready(i: Instance) -> bool:
return bool(i.public_ip_address and i.state
and i.state.get("Name") == "running")
done = False
async for remaining in ui.async_timeout_loop(60, 5):
say(f"Waiting for instance to become ready: {remaining}s remaining")
i.reload()
if is_ready(i):
done = True
break
if not done:
raise RuntimeError(
f"Instance {i} did not become ready in a reasonable amount of time"
)
done = False
async for remaining in ui.async_timeout_loop(180, 5):
say(f"Checking whether setup has completed: {remaining}s remaining")
try:
mssh(i, "[[ -f /opt/provision/docker-installed ]]")
done = True
break
except CalledProcessError:
continue
if not done:
raise RuntimeError(
"Instance did not finish setup in a reasonable amount of time")
mkrepo(i, git_rev)
async def run_ssm(instance_id: str,
commands: List[str],
timeout: int = 60) -> CommandResult:
id = boto3.client("ssm").send_command(
InstanceIds=[instance_id],
DocumentName="AWS-RunShellScript",
Parameters={"commands": commands},
)["Command"]["CommandId"]
async for remaining in ui.async_timeout_loop(timeout, 5):
invocation_dne = boto3.client("ssm").exceptions.InvocationDoesNotExist
SPEAKER(
f"Waiting for commands to finish running: {remaining}s remaining")
try:
result = boto3.client("ssm").get_command_invocation(
CommandId=id, InstanceId=instance_id)
except invocation_dne:
continue
if result["Status"] != "InProgress":
return CommandResult(
status=result["Status"],
stdout=result["StandardOutputContent"],
stderr=result["StandardErrorContent"],
)
raise RuntimeError(
f"Command {commands} on instance {instance_id} did not run in a reasonable amount of time"
)
async def setup(
i: Instance,
subnet_id: Optional[str],
local_pub_key: str,
identity_file: str,
git_rev: str,
) -> None:
def is_ready(i: Instance) -> bool:
return bool(i.public_ip_address and i.state
and i.state.get("Name") == "running")
done = False
async for remaining in ui.async_timeout_loop(60, 5):
SPEAKER(
f"Waiting for instance to become ready: {remaining}s remaining")
i.reload()
if is_ready(i):
done = True
break
if not done:
raise RuntimeError(
f"Instance {i} did not become ready in a reasonable amount of time"
)
done = False
invalid_instance = boto3.client("ssm").exceptions.InvalidInstanceId
commands = [
"mkdir -p ~ubuntu/.ssh",
f"echo {local_pub_key} >> ~ubuntu/.ssh/authorized_keys",
]
import pprint
print("Running commands:")
pprint.pprint(commands)
async for remaining in ui.async_timeout_loop(180, 5):
try:
await run_ssm(i.instance_id, commands, 180)
done = True
break
except invalid_instance:
pass
if not done:
raise RuntimeError(f"Failed to run SSM commands on instance {i}")
done = False
async for remaining in ui.async_timeout_loop(180, 5):
try:
ssh.runv(
["[", "-f", "/DONE", "]"],
"ubuntu",
i.public_ip_address,
identity_file=identity_file,
)
done = True
break
except CalledProcessError:
continue
if not done:
raise RuntimeError(
"Instance did not finish setup in a reasonable amount of time")
mkrepo(i, identity_file, git_rev)