def create_instance(
image_id: str = GCS_CONTAINER_ID, preemptible: str = "true", self_destruct: str = "true"
) -> Response:
status, content = 500, "Unknown error"
image_id = _get_request_param("image_id", image_id)
preemptible_param = _get_request_param("preemptible", preemptible).lower()
preemptible = BOOL_STRING_MAP.get(preemptible_param, False)
self_destruct_param = _get_request_param("self_destruct", self_destruct).lower()
self_destruct = BOOL_STRING_MAP.get(self_destruct_param, False)
try:
# Create a new preemptible instance and wait for it to come online
instance_opts = dict(
service_account=os.getenv(GCP_ENV_SERVICE_ACCOUNT),
preemptible=preemptible,
startup_script=str(GCP_SELF_DESTRUCT_SCRIPT) if self_destruct else None,
)
instance_id = start_instance_from_image(GCS_CONTAINER_ID, **instance_opts)
instance_ip = get_internal_ip(instance_id)
# Returns created instance details instead of the usual "OK"
status, content = 200, json.dumps(dict(id=instance_id, **instance_opts))
logger.log_info(f"Created worker instance {instance_id} with internal IP {instance_ip}")
except Exception as exc:
log_opts = dict(traceback=traceback.format_exc())
logger.log_error(f"Exception creating instance from image ID {image_id}", **log_opts)
status, content = 500, "Internal exception"
# Forward the response from the instance
return Response(content, status=status)
def deferred_route(url_path: str) -> Response:
status, content = 500, "Unknown error"
# Initialize the instance ID variable outside of the try so it's available at finally
instance_id = None
# Prevent chaining deferred calls
if any([token == "deferred" for token in url_path.split("/")]):
return Response(status=400)
try:
# Create a new preemptible instance and wait for it to come online
instance_opts = dict(service_account=os.getenv(ENV_SERVICE_ACCOUNT))
instance_id = start_instance_from_image(GCE_IMAGE_ID, **instance_opts)
instance_ip = get_internal_ip(instance_id)
logger.log_info(
f"Created worker instance {instance_id} with internal IP {instance_ip}"
)
# Wait 30 seconds before attempting to forward the request
time.sleep(30)
# Forward the route to the worker instance
params = dict(request.args)
headers = dict(request.headers)
url_fwd = f"http://{instance_ip}/{url_path}"
log_opts = dict(path=url_path, params=params)
logger.log_info(f"Forwarding request to {instance_id}: {instance_ip}",
**log_opts)
response = requests.get(url=url_fwd,
headers=headers,
params=params,
timeout=60 * 60 * 2)
# Retrieve the response from the worker instance
status, content = response.status_code, response.content
logger.log_info(f"Received response from {instance_id}",
status=status,
**log_opts)
except requests.exceptions.Timeout:
logger.log_error(f"Request to {instance_id} timed out", path=url_path)
status, content = 500, "Timeout"
except Exception as exc:
log_opts = dict(traceback=traceback.format_exc(), path=url_path)
logger.log_error(
f"Exception while waiting for response from {instance_id}",
**log_opts)
status, content = 500, "Internal exception"
finally:
# Shut down the worker instance now that the job is finished
if instance_id is not None:
delete_instance(instance_id)
logger.log_info(f"Deleted instance {instance_id}")
# Forward the response from the instance
return Response(content, status=status)
def deferred_route(url_path: str) -> Response:
status = 500
content = "Unknown error"
# Initialize the instance ID variable outside of the try so it's available at finally
instance_id = None
# Prevent chaining deferred calls
if any([token == "deferred" for token in url_path.split("/")]):
return Response(status=400)
try:
# Create a new preemptible instance and wait for it to come online
instance_id = start_instance(
service_account=os.getenv(ENV_SERVICE_ACCOUNT))
instance_ip = get_internal_ip(instance_id)
logger.log_info(
f"Created worker instance {instance_id} with internal IP {instance_ip}"
)
# Wait 4 minutes before attempting to forward the request
log_interval = 30
wait_seconds = 4 * 60
for _ in range(wait_seconds // log_interval):
logger.log_info("Waiting for instance to start...")
time.sleep(log_interval)
# Forward the route to the worker instance
url_fwd = f"http://{instance_ip}/{url_path}"
logger.log_info(f"Forwarding request to {url_fwd}")
params = dict(request.args)
headers = dict(request.headers)
response = requests.get(url=url_fwd,
headers=headers,
params=params,
timeout=60 * 60 * 2)
# Retrieve the response from the worker instance
status = response.status_code
content = response.content
logger.log_info(f"Received response with status code {status}")
except:
traceback.print_exc()
content = "Internal exception"
finally:
# Shut down the worker instance now that the job is finished
if instance_id is not None:
delete_instance(instance_id)
logger.log_info(f"Deleted instance {instance_id}")
# Forward the response from the instance
return Response(content, status=status)
def create_instance(image_id: str = GCE_IMAGE_ID) -> Response:
status, content = 500, "Unknown error"
image_id = _get_request_param("image_id", image_id)
try:
# Create a new preemptible instance and wait for it to come online
instance_opts = dict(service_account=os.getenv(ENV_SERVICE_ACCOUNT))
instance_id = start_instance_from_image(GCE_IMAGE_ID, **instance_opts)
instance_ip = get_internal_ip(instance_id)
status, content = 200, "OK"
logger.log_info(
f"Created worker instance {instance_id} with internal IP {instance_ip}"
)
except Exception as exc:
log_opts = dict(traceback=traceback.format_exc())
logger.log_error(
f"Exception creating instance from image ID {image_id}",
**log_opts)
status, content = 500, "Internal exception"
# Forward the response from the instance
return Response(content, status=status)
