async def start(self): """ When user logs in, start their instance. Must return a tuple of the ip and port for the server and Jupyterhub instance. """ self.log.debug("function start for user %s" % self.user.name) self.user.last_activity = datetime.utcnow() try: instance = self.instance = await self.get_instance( ) #cannot be a thread pool... os.environ[ 'AWS_SPAWNER_WORKER_IP'] = instance.private_ip_address if type( instance.private_ip_address) == str else "NO IP" #comprehensive list of states: pending, running, shutting-down, terminated, stopping, stopped. if instance.state["Name"] == "running": ec2_run_status = await self.check_for_hanged_ec2(instance) if ec2_run_status == "SSH_CONNECTION_FAILED": #await self.poll() #await self.kill_instance(instance) #await retry(instance.start, max_retries=(LONG_RETRY_COUNT*2)) #await retry(instance.wait_until_running, max_retries=(LONG_RETRY_COUNT*2)) #this call can occasionally fail, so we wrap it in a retry. #return instance.private_ip_address, NOTEBOOK_SERVER_PORT return None #start_worker_server will handle starting notebook logger.info( "start ip and port: %s , %s" % (instance.private_ip_address, NOTEBOOK_SERVER_PORT)) await self.start_worker_server(instance, new_server=False) self.ip = self.user.server.ip = instance.private_ip_address self.port = self.user.server.port = NOTEBOOK_SERVER_PORT elif instance.state["Name"] in [ "stopped", "stopping", "pending", "shutting-down" ]: #For case that instance is stopped, the attributes are modified if instance.state["Name"] == "stopped": self.log.debug('Selected instance type %s for user %s' % (str(self.user_options['INSTANCE_TYPE']), self.user.name)) if self.user_options[ 'INSTANCE_TYPE'] in AWS_INSTANCE_TYPES: try: await retry( instance.modify_attribute, Attribute='instanceType', Value=self.user_options['INSTANCE_TYPE']) except: self.log.debug( "Instance type for user %s could not be changed." % self.user.name) else: self.log.debug( "Instance type for user %s could not recognized." % self.user.name) #Server needs to be booted, do so. self.log.info("Starting user %s instance " % self.user.name) await retry(instance.start, max_retries=LONG_RETRY_COUNT) #await retry(instance.start) # blocking calls should be wrapped in a Future await retry( instance.wait_until_running ) #this call can occasionally fail, so we wrap it in a retry. self.aws_instance = os.environ[ 'AWS_SPAWNER_WORKER_IP'] = instance.private_ip_address await self.start_worker_server(instance, new_server=False) self.log.debug( "%s , %s" % (instance.private_ip_address, NOTEBOOK_SERVER_PORT)) # a longer sleep duration reduces the chance of a 503 or infinite redirect error (which a user can # resolve with a page refresh). 10s seems to be a good inflection point of behavior await asyncio.sleep(10) self.ip = self.user.server.ip = instance.private_ip_address self.port = self.user.server.port = NOTEBOOK_SERVER_PORT elif instance.state["Name"] == "terminated": # If the server is terminated ServerNotFound is raised. This leads to the try self.log.debug( 'Instance terminated for user %s. Creating new one and try to attach old volume.' % self.user.name) raise ServerNotFound else: # if instance is in pending, shutting-down, or rebooting state raise web.HTTPError( 503, "Unknown server state for %s. Please try again in a few minutes" % self.user.name) except (ServerNotFound, Server.DoesNotExist) as e: self.log.debug('Server not found raised for %s' % self.user.name) try: self.user.volume = await self.get_volume( ) #cannot be a thread pool... except (VolumeNotFound, Server.DoesNotExist) as e: self.user.volume = None self.log.debug('Volume not found raised for %s' % self.user.name) self.log.info("\nCreate new server for user %s with volume %s\n" % (self.user.name, self.user.volume)) instance = self.instance = await self.create_new_instance() os.environ['AWS_SPAWNER_WORKER_IP'] = instance.private_ip_address await self.start_worker_server(instance, new_server=True) # self.notebook_should_be_running = False self.log.debug("%s , %s" % (instance.private_ip_address, NOTEBOOK_SERVER_PORT)) # to reduce chance of 503 or infinite redirect await asyncio.sleep(10) self.ip = self.user.server.ip self.port = self.user.server.port = NOTEBOOK_SERVER_PORT ec2 = boto3.client("ec2", region_name=SERVER_PARAMS["REGION"]) #update # assign iam role to server role = Role.get_role(self.user.name) self.log.debug( 'Trying to attach role %s to instance %s for user %s' % (role.role_arn, self.instance.instance_id, self.user.name)) response = await retry(ec2.associate_iam_instance_profile, IamInstanceProfile={ 'Arn': role.role_arn, 'Name': role.role_name }, InstanceId=self.instance.instance_id, max_retries=10) self.log.debug( 'AWS response for tried rolle attachment for user %s: %s' % (self.user.name, response)) return instance.private_ip_address, NOTEBOOK_SERVER_PORT