async def start(self):
""" When user logs in, start their instance.
Must return a tuple of the ip and port for the server and Jupyterhub instance. """
self.log.debug("function start for user %s" % self.user.name)
self.user.last_activity = datetime.utcnow()
try:
instance = self.instance = await self.get_instance(
) #cannot be a thread pool...
os.environ[
'AWS_SPAWNER_WORKER_IP'] = instance.private_ip_address if type(
instance.private_ip_address) == str else "NO IP"
#comprehensive list of states: pending, running, shutting-down, terminated, stopping, stopped.
if instance.state["Name"] == "running":
ec2_run_status = await self.check_for_hanged_ec2(instance)
if ec2_run_status == "SSH_CONNECTION_FAILED":
#await self.poll()
#await self.kill_instance(instance)
#await retry(instance.start, max_retries=(LONG_RETRY_COUNT*2))
#await retry(instance.wait_until_running, max_retries=(LONG_RETRY_COUNT*2)) #this call can occasionally fail, so we wrap it in a retry.
#return instance.private_ip_address, NOTEBOOK_SERVER_PORT
return None
#start_worker_server will handle starting notebook
logger.info(
"start ip and port: %s , %s" %
(instance.private_ip_address, NOTEBOOK_SERVER_PORT))
await self.start_worker_server(instance, new_server=False)
self.ip = self.user.server.ip = instance.private_ip_address
self.port = self.user.server.port = NOTEBOOK_SERVER_PORT
elif instance.state["Name"] in [
"stopped", "stopping", "pending", "shutting-down"
]:
#For case that instance is stopped, the attributes are modified
if instance.state["Name"] == "stopped":
self.log.debug('Selected instance type %s for user %s' %
(str(self.user_options['INSTANCE_TYPE']),
self.user.name))
if self.user_options[
'INSTANCE_TYPE'] in AWS_INSTANCE_TYPES:
try:
await retry(
instance.modify_attribute,
Attribute='instanceType',
Value=self.user_options['INSTANCE_TYPE'])
except:
self.log.debug(
"Instance type for user %s could not be changed."
% self.user.name)
else:
self.log.debug(
"Instance type for user %s could not recognized." %
self.user.name)
#Server needs to be booted, do so.
self.log.info("Starting user %s instance " % self.user.name)
await retry(instance.start, max_retries=LONG_RETRY_COUNT)
#await retry(instance.start)
# blocking calls should be wrapped in a Future
await retry(
instance.wait_until_running
) #this call can occasionally fail, so we wrap it in a retry.
self.aws_instance = os.environ[
'AWS_SPAWNER_WORKER_IP'] = instance.private_ip_address
await self.start_worker_server(instance, new_server=False)
self.log.debug(
"%s , %s" %
(instance.private_ip_address, NOTEBOOK_SERVER_PORT))
# a longer sleep duration reduces the chance of a 503 or infinite redirect error (which a user can
# resolve with a page refresh). 10s seems to be a good inflection point of behavior
await asyncio.sleep(10)
self.ip = self.user.server.ip = instance.private_ip_address
self.port = self.user.server.port = NOTEBOOK_SERVER_PORT
elif instance.state["Name"] == "terminated":
# If the server is terminated ServerNotFound is raised. This leads to the try
self.log.debug(
'Instance terminated for user %s. Creating new one and try to attach old volume.'
% self.user.name)
raise ServerNotFound
else:
# if instance is in pending, shutting-down, or rebooting state
raise web.HTTPError(
503,
"Unknown server state for %s. Please try again in a few minutes"
% self.user.name)
except (ServerNotFound, Server.DoesNotExist) as e:
self.log.debug('Server not found raised for %s' % self.user.name)
try:
self.user.volume = await self.get_volume(
) #cannot be a thread pool...
except (VolumeNotFound, Server.DoesNotExist) as e:
self.user.volume = None
self.log.debug('Volume not found raised for %s' %
self.user.name)
self.log.info("\nCreate new server for user %s with volume %s\n" %
(self.user.name, self.user.volume))
instance = self.instance = await self.create_new_instance()
os.environ['AWS_SPAWNER_WORKER_IP'] = instance.private_ip_address
await self.start_worker_server(instance, new_server=True)
# self.notebook_should_be_running = False
self.log.debug("%s , %s" %
(instance.private_ip_address, NOTEBOOK_SERVER_PORT))
# to reduce chance of 503 or infinite redirect
await asyncio.sleep(10)
self.ip = self.user.server.ip
self.port = self.user.server.port = NOTEBOOK_SERVER_PORT
ec2 = boto3.client("ec2", region_name=SERVER_PARAMS["REGION"])
#update
# assign iam role to server
role = Role.get_role(self.user.name)
self.log.debug(
'Trying to attach role %s to instance %s for user %s' %
(role.role_arn, self.instance.instance_id, self.user.name))
response = await retry(ec2.associate_iam_instance_profile,
IamInstanceProfile={
'Arn': role.role_arn,
'Name': role.role_name
},
InstanceId=self.instance.instance_id,
max_retries=10)
self.log.debug(
'AWS response for tried rolle attachment for user %s: %s' %
(self.user.name, response))
return instance.private_ip_address, NOTEBOOK_SERVER_PORT
