async def setup( i: Instance, git_rev: str, ) -> None: def is_ready(i: Instance) -> bool: return bool(i.public_ip_address and i.state and i.state.get("Name") == "running") done = False async for remaining in ui.async_timeout_loop(60, 5): say(f"Waiting for instance to become ready: {remaining}s remaining") i.reload() if is_ready(i): done = True break if not done: raise RuntimeError( f"Instance {i} did not become ready in a reasonable amount of time" ) done = False async for remaining in ui.async_timeout_loop(180, 5): say(f"Checking whether setup has completed: {remaining}s remaining") try: mssh(i, "[[ -f /opt/provision/docker-installed ]]") done = True break except CalledProcessError: continue if not done: raise RuntimeError( "Instance did not finish setup in a reasonable amount of time") mkrepo(i, git_rev)
async def run_ssm(instance_id: str, commands: List[str], timeout: int = 60) -> CommandResult: id = boto3.client("ssm").send_command( InstanceIds=[instance_id], DocumentName="AWS-RunShellScript", Parameters={"commands": commands}, )["Command"]["CommandId"] async for remaining in ui.async_timeout_loop(timeout, 5): invocation_dne = boto3.client("ssm").exceptions.InvocationDoesNotExist SPEAKER( f"Waiting for commands to finish running: {remaining}s remaining") try: result = boto3.client("ssm").get_command_invocation( CommandId=id, InstanceId=instance_id) except invocation_dne: continue if result["Status"] != "InProgress": return CommandResult( status=result["Status"], stdout=result["StandardOutputContent"], stderr=result["StandardErrorContent"], ) raise RuntimeError( f"Command {commands} on instance {instance_id} did not run in a reasonable amount of time" )
async def setup( i: Instance, subnet_id: Optional[str], local_pub_key: str, identity_file: str, git_rev: str, ) -> None: def is_ready(i: Instance) -> bool: return bool(i.public_ip_address and i.state and i.state.get("Name") == "running") done = False async for remaining in ui.async_timeout_loop(60, 5): SPEAKER( f"Waiting for instance to become ready: {remaining}s remaining") i.reload() if is_ready(i): done = True break if not done: raise RuntimeError( f"Instance {i} did not become ready in a reasonable amount of time" ) done = False invalid_instance = boto3.client("ssm").exceptions.InvalidInstanceId commands = [ "mkdir -p ~ubuntu/.ssh", f"echo {local_pub_key} >> ~ubuntu/.ssh/authorized_keys", ] import pprint print("Running commands:") pprint.pprint(commands) async for remaining in ui.async_timeout_loop(180, 5): try: await run_ssm(i.instance_id, commands, 180) done = True break except invalid_instance: pass if not done: raise RuntimeError(f"Failed to run SSM commands on instance {i}") done = False async for remaining in ui.async_timeout_loop(180, 5): try: ssh.runv( ["[", "-f", "/DONE", "]"], "ubuntu", i.public_ip_address, identity_file=identity_file, ) done = True break except CalledProcessError: continue if not done: raise RuntimeError( "Instance did not finish setup in a reasonable amount of time") mkrepo(i, identity_file, git_rev)