async def get_volume(self):
""" This returns a boto volume resource for the case no volumewas found.
If boto can't find the volume or if no entry for instance in database,
it raises VolumeNotFound error and removes database entry if appropriate """
self.log.debug("function get_resource for user %s" % self.user.name)
server = Server.get_server(self.user.name)
resource = await retry(boto3.resource,
"ec2",
region_name=SERVER_PARAMS["REGION"])
try:
ret = await retry(resource.Volume, server.ebs_volume_id)
self.log.debug("return for get_volume for user %s: %s" %
(self.user.name, ret))
# boto3.Volume is lazily loaded. Force with .load()
await retry(ret.load)
if ret.meta.data is None:
Server.remove_server(server.server_id)
self.log.info("\nVolume DNE for user %s\n" % self.user.name)
raise VolumeNotFound
return ret
except ClientError as e:
self.log.error("get_instance client error: %s" % e)
if "InvalidInstanceID.NotFound" not in str(e):
self.log.error("Couldn't find volume for user '%s'" %
self.user.name)
Server.remove_server(server.server_id)
raise VolumeNotFound
raise e
def get_instance(self):
""" This returns a boto Instance resource; if boto can't find the instance or if no entry for instance in database,
it raises Server.DoesNotExist error and removes database entry if appropriate """
self.log.debug("function get_instance for user %s" % self.user.name)
server = Server.get_server(self.user.name)
resource = yield retry(boto3.resource,
"ec2",
region_name=SERVER_PARAMS["REGION"])
try:
ret = yield retry(resource.Instance, server.server_id)
self.log.debug("return for get_instance for user %s: %s" %
(self.user.name, ret))
# boto3.Instance is lazily loaded. Force with .load()
yield retry(ret.load)
if ret.meta.data is None:
Server.remove_server(server.server_id)
raise Server.DoesNotExist()
return ret
except ClientError as e:
self.log.error("get_instance client error: %s" % e)
if "InvalidInstanceID.NotFound" not in str(e):
self.log.error("Couldn't find instance for user '%s'" %
self.user.name)
Server.remove_server(server.server_id)
raise Server.DoesNotExist()
raise e
def manually_kill_server(user_name):
# Get our AWS server db's instance for the user
try:
server = Server.get_server(user_name)
app_log.debug("Checking server for %s manually..." % user_name)
except Server.DoesNotExist:
# it is not necessarily the case that a server will exist, we return early if that is the case.
app_log.warn("There is no matching, allocated server for user %s" %
user_name)
return
# get server instance information
resource = yield retry(boto3.resource,
"ec2",
region_name=SERVER_PARAMS["REGION"])
instance = yield retry(resource.Instance, server.server_id)
# instance object is lazy, run this to get full info...
yield retry(instance.load)
#stop server if state is running (possible states are stopped, stopping, pending, shutting-down, terminated, and running)
if instance.state["Name"] == "running":
retry(instance.stop)
app_log.info("manually killed server for user %s" % user_name)
else:
app_log.debug("server state for user %s is %s, no action taken" %
(user_name, instance.state["Name"]))
def get_instance(self):
#""" This returns a boto Instance resource; if boto can't find the instance or if no entry for instance in database,
# it raises Server.DoesNotExist error and removes database entry if appropriate """
""" This returns a boto Instance resource; if no entry for the instance in database,then
it raises Server.DoesNotExist error. If the instance in the database but
boto can't find the instance, it raise 500 http error """
self.log_user("get_instance()")
server = Server.get_server(self.user.name)
resource = yield retry(boto3.resource, "ec2", region_name=SERVER_PARAMS["REGION"])
try:
ret = yield retry(resource.Instance, server.server_id)
self.log_user("get_instance: returned: %s" % ret)
# boto3.Instance is lazily loaded. Force with .load()
yield retry(ret.load)
if ret.meta.data is None:
self.log_user("get_instance: could not access instance", level=logging.ERROR)
raise web.HTTPError(500, "Couldn't access instance for user '%s'. Please try again in a few minutes" % self.user.name)
#Server.remove_server(server.server_id)
#raise Server.DoesNotExist()
return ret
except ClientError as e:
self.log_user("get_instance client error: %s" % e)
if "InvalidInstanceID.NotFound" not in str(e):
self.log_user("get_instance: could not find instance for user", level=logging.ERROR)
raise web.HTTPError(500, "Couldn't access instance for user '%s'. Please try again in a few minutes" % self.user.name)
#Server.remove_server(server.server_id)
#raise Server.DoesNotExist()
raise e
def check_user_ec2(userid):
try:
server = Server.get_server(userid)
server_id = server.server_id
user_instance = ec2.Instance(server_id)
print("user %s , ec2= %s" % (userid, user_instance))
except:
print("no such user or server")
return 1
return 0
def delete_user_ec2(userid):
try:
server = Server.get_server(userid)
server_id = server.server_id
server.remove_server(server_id)
user_instance = ec2.Instance(server_id)
user_instance.terminate()
except:
print("no such user or server")
return 1
return 0
async def terminate(self, now=False, delete_volume=False):
""" Terminate instance for debugging purposes """
self.log.debug("function terminate")
self.log.info("Terminating user %s instance " % self.user.name)
try:
instance = await self.get_instance()
await retry(instance.terminate)
if delete_volume:
server = Server.get_server(self.user.name)
ec2 = boto3.client("ec2", region_name=SERVER_PARAMS["REGION"])
await retry(ec2.delete_volume,
VolumeId=server.ebs_volume_id,
max_retries=40)
return 'Terminated'
# self.notebook_should_be_running = False
except Server.DoesNotExist:
self.log.error(
"Couldn't terminate server for user '%s' as it does not exist"
% self.user.name)
# self.notebook_should_be_running = False
self.clear_state()
async def create_new_instance(self):
""" Creates and boots a new server to host the worker instance."""
self.log.debug("function create_new_instance %s" % self.user.name)
ec2 = boto3.client("ec2", region_name=SERVER_PARAMS["REGION"])
resource = boto3.resource("ec2", region_name=SERVER_PARAMS["REGION"])
BDM = []
boot_drive = {
'DeviceName': '/dev/sda1', # this is to be the boot drive
'Ebs': {
'VolumeSize':
SERVER_PARAMS["WORKER_EBS_SIZE"], # size in gigabytes
'DeleteOnTermination': True,
'VolumeType': 'gp2', # This means General Purpose SSD
# 'Iops': 1000 } # i/o speed for storage, default is 100, more is faster
}
}
BDM = [boot_drive]
# get user volume
volume_id = await self.select_volume()
# create new instance
reservation = await retry(
ec2.run_instances,
ImageId=SERVER_PARAMS["WORKER_AMI"],
MinCount=1,
MaxCount=1,
KeyName=SERVER_PARAMS["KEY_NAME"],
InstanceType=self.user_options['INSTANCE_TYPE'],
SubnetId=SERVER_PARAMS["SUBNET_ID"],
SecurityGroupIds=SERVER_PARAMS["WORKER_SECURITY_GROUPS"],
BlockDeviceMappings=BDM,
)
self.log.debug(reservation)
try:
instance_id = reservation["Instances"][0]["InstanceId"]
except TypeError as e:
raise Exception(
'AWS sends weirdly formatted JSON. Please try again...')
instance = await retry(resource.Instance, instance_id)
#if an old volume is restored from a terminated instance, the user has to be updated, e.g. delted and newly saved
# TODO: Support multiple volumes
if self.user.volume:
server = Server.get_server(self.user.name)
Server.remove_server(server.server_id)
Server.new_server(instance_id, self.user.name, volume_id)
await retry(instance.wait_until_exists)
# add server tags; tags cannot be added until server exists
await retry(instance.create_tags, Tags=WORKER_TAGS)
await retry(instance.create_tags,
Tags=[{
"Key": "User",
"Value": self.user.name
}])
await retry(instance.create_tags,
Tags=[{
"Key":
"Name",
"Value":
SERVER_PARAMS["WORKER_SERVER_NAME"] + '_' +
self.user.name
}])
# start server
# blocking calls should be wrapped in a Future
await retry(instance.wait_until_running)
# Attach persistent EBS
await retry(instance.attach_volume,
Device='/dev/sdf',
VolumeId=volume_id,
InstanceId=instance_id)
return instance
#%% Configure
class DummyUser():
def __init__(self, name,):
self.name = name
self.last_activity = ''
self.server = SimpleNamespace(**{'ip' : '',
'base_url': ''})
self.url = ''
self.settings = {}
dummyUser = DummyUser(name='developmentUser')
try:
server = Server.get_server(user_id=dummyUser.name)
except:
server = Server.new_server(server_id = '' , user_id = dummyUser.name , ebs_volume_id = '')
assert server.user_id == dummyUser.name
dummyUserOptions = {'EBS_VOL_ID' : '',
'EBS_VOL_SIZE' : 3,
'INSTANCE_TYPE': 't2.nano'}
dummyHubOptions = SimpleNamespace(**{'public_host' : '',
'api_url' : '',
'base_url' : ''})
dummyApiToken = open('api_token.txt','r').read()
