def try_launch_container(cls, user_id, max_hop=False):
sessname = unique_sessname(user_id)
cont = SessContainer.get_by_name(sessname)
cls.log_debug("have existing container for %s: %r", sessname,
None != cont)
if cont is not None:
cls.log_debug("container running: %r", cont.is_running())
if max_hop:
self_load = Compute.get_instance_stats(Compute.get_instance_id(),
'Load')
if self_load < 100:
SessContainer.invalidate_container(sessname)
JBoxAsyncJob.async_launch_by_name(sessname, user_id, True)
return True
is_leader = is_proposed_cluster_leader()
if ((cont is None) or (not cont.is_running())) and (
not Compute.should_accept_session(is_leader)):
if cont is not None:
SessContainer.invalidate_container(cont.get_name())
JBoxAsyncJob.async_backup_and_cleanup(cont.dockid)
return False
SessContainer.invalidate_container(sessname)
JBoxAsyncJob.async_launch_by_name(sessname, user_id, True)
return True
def post(self):
self.log_debug("Homework handler got POST request")
sessname = self.get_session_id()
user_id = self.get_user_id()
if (sessname is None) or (user_id is None):
self.log_info("Homework handler got invalid sessname[%r] or user_id[%r]", sessname, user_id)
self.send_error()
return
user = JBoxUserV2(user_id)
is_admin = sessname in JBoxCfg.get("admin_sessnames", []) or user.has_role(JBoxUserV2.ROLE_SUPER)
course_owner = is_admin or user.has_role(JBoxUserV2.ROLE_OFFER_COURSES)
cont = SessContainer.get_by_name(sessname)
self.log_info("user_id[%r], is_admin[%r], course_owner[%r]", user_id, is_admin, course_owner)
if cont is None:
self.log_info("user_id[%r] container not found", user_id)
self.send_error()
return
courses_offered = user.get_courses_offered()
if self.handle_if_check(user_id):
return
if self.handle_create_course(user_id):
return
if self.handle_get_metadata(is_admin, courses_offered):
return
if self.handle_if_report(user_id, is_admin, courses_offered):
return
self.log_error("no handlers found")
# only AJAX requests responded to
self.send_error()
def do_task(plugin_type, data):
if plugin_type != JBPluginTask.JBP_CMD_ASYNC:
return
mode = data['action']
user_id = data['user_id']
sessname = data['sessname']
user = JBoxUserV2(user_id)
is_allowed = user.has_resource_profile(
JBoxUserV2.RES_PROF_DISK_EBS_10G)
if not is_allowed:
JBoxEBSVolAsyncTask.log_error(
"Data volume access not allowed for user")
return
cont = SessContainer.get_by_name(sessname)
if cont is None:
return
vol = JBoxEBSVol.get_disk_from_container(sessname)
disk_state = None
try:
disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID,
region_id=CompEC2.REGION,
user_id=user_id)
except:
pass
JBoxEBSVolAsyncTask.log_debug("Data volume request %s for %s", mode,
cont.debug_str())
if mode == 'attach':
if vol is None:
vol = JBoxEBSVol.get_disk_for_user(user_id)
JBoxEBSVol.mount_host_device(vol.disk_path, cont.dockid,
JBoxVol.DATA_MOUNT_POINT)
disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID,
region_id=CompEC2.REGION,
user_id=user_id)
if disk_state.get_state() != JBoxDiskState.STATE_ATTACHED:
disk_state.set_state(JBoxDiskState.STATE_ATTACHED)
disk_state.save()
elif mode == 'detach':
if cont is not None and cont.is_running():
if vol is not None:
# unmount from container first
JBoxEBSVol.unmount_host_device(vol.disk_path, cont.dockid)
elif disk_state is not None:
# no volume attached. ensure disk state is updated
if disk_state.get_state() != JBoxDiskState.STATE_DETACHED:
disk_state.set_state(JBoxDiskState.STATE_DETACHED)
disk_state.save()
if vol is not None:
vol.release(backup=True)
JBoxEBSVolAsyncTask.log_debug(
"Data volume request %s completed for %s", mode, cont.debug_str())
def get(self):
sessname = self.get_session_id()
user_id = self.get_user_id()
if (sessname is None) or (user_id is None):
self.send_error()
return
user = JBoxUserV2(user_id)
is_admin = sessname in JBoxCfg.get("admin_sessnames", [])
manage_containers = is_admin or user.has_role(JBoxUserV2.ROLE_MANAGE_CONTAINERS)
show_report = is_admin or user.has_role(JBoxUserV2.ROLE_ACCESS_STATS)
cont = SessContainer.get_by_name(sessname)
if cont is None:
self.send_error()
return
if self.handle_if_logout(cont):
return
if self.handle_if_stats(is_admin or show_report):
return
if self.handle_if_show_cfg(is_admin):
return
if self.handle_if_instance_info(is_admin):
return
if self.handle_switch_julia_img(user):
return
if self.handle_if_open_port(sessname, user_id):
return
juliaboxver, _upgrade_available = self.get_upgrade_available(cont)
jimg_type = 0
if user.has_resource_profile(JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP):
jimg_type = JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP
expire = JBoxCfg.get('interactive.expire')
d = dict(
manage_containers=manage_containers,
show_report=show_report,
sessname=sessname,
user_id=user_id,
created=isodate.datetime_isoformat(cont.time_created()),
started=isodate.datetime_isoformat(cont.time_started()),
allowed_till=isodate.datetime_isoformat((cont.time_started() + timedelta(seconds=expire))),
mem=cont.get_memory_allocated(),
cpu=cont.get_cpu_allocated(),
disk=cont.get_disk_allocated(),
expire=expire,
juliaboxver=juliaboxver,
jimg_type=jimg_type
)
self.rendertpl("ipnbadmin.tpl", d=d)
def do_task(plugin_type, data):
if plugin_type != JBPluginTask.JBP_CMD_ASYNC:
return
mode = data["action"]
user_id = data["user_id"]
sessname = data["sessname"]
user = JBoxUserV2(user_id)
is_allowed = user.has_resource_profile(JBoxUserV2.RES_PROF_DISK_EBS_10G)
if not is_allowed:
JBoxEBSVolAsyncTask.log_error("Data volume access not allowed for user")
return
cont = SessContainer.get_by_name(sessname)
if cont is None:
return
vol = JBoxEBSVol.get_disk_from_container(sessname)
disk_state = None
try:
disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_id)
except:
pass
JBoxEBSVolAsyncTask.log_debug("Data volume request %s for %s", mode, cont.debug_str())
if mode == "attach":
if vol is None:
vol = JBoxEBSVol.get_disk_for_user(user_id)
JBoxEBSVol.mount_host_device(vol.disk_path, cont.dockid, JBoxVol.DATA_MOUNT_POINT)
disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_id)
if disk_state.get_state() != JBoxDiskState.STATE_ATTACHED:
disk_state.set_state(JBoxDiskState.STATE_ATTACHED)
disk_state.save()
elif mode == "detach":
if cont is not None and cont.is_running():
if vol is not None:
# unmount from container first
JBoxEBSVol.unmount_host_device(vol.disk_path, cont.dockid)
elif disk_state is not None:
# no volume attached. ensure disk state is updated
if disk_state.get_state() != JBoxDiskState.STATE_DETACHED:
disk_state.set_state(JBoxDiskState.STATE_DETACHED)
disk_state.save()
if vol is not None:
vol.release(backup=True)
JBoxEBSVolAsyncTask.log_debug("Data volume request %s completed for %s", mode, cont.debug_str())
def do_monitor_loading_ajax(self, user_id):
sessname = unique_sessname(user_id)
self.log_debug("AJAX monitoring loading of session [%s] user[%s]...", sessname, user_id)
cont = SessContainer.get_by_name(sessname)
if (cont is None) or (not cont.is_running()):
loading_step = int(self.get_loading_state(), 0)
if loading_step > 60:
self.log_error("Could not start instance. Session [%s] for user [%s] didn't load.", sessname, user_id)
self.write({'code': -1})
return
loading_step += 1
self.set_loading_state(loading_step)
self.write({'code': 0})
else:
self.write({'code': 1})
def do_monitor_loading_ajax(self, user_id):
sessname = unique_sessname(user_id)
self.log_debug("AJAX monitoring loading of session [%s] user[%s]...", sessname, user_id)
cont = SessContainer.get_by_name(sessname)
if (cont is None) or (not cont.is_running()):
loading_step = int(self.get_loading_state(), 0)
if loading_step > 90:
self.log_error("Could not start instance. Session [%s] for user [%s] didn't load.", sessname, user_id)
self.write({'code': -1})
return
loading_step += 1
self.set_loading_state(loading_step)
self.write({'code': 0})
else:
self.write({'code': 1})
def do_node_housekeeping():
JBoxEBSHousekeep.log_debug("starting node housekeeping")
for device, vol in JBoxEBSVol.get_mapped_volumes().iteritems():
deviceid = os.path.basename(device)
vol_id = vol.volume_id
vol = EBSVol.get_volume(vol_id)
user_id = vol.tags["Name"] if "Name" in vol.tags else None
if user_id is None:
continue
sessname = unique_sessname(user_id)
cont = SessContainer.get_by_name(sessname)
if cont is not None:
continue
JBoxEBSHousekeep.log_debug("Found orphaned volume %s for %s, %s", vol_id, user_id, sessname)
ebsvol = JBoxEBSVol(deviceid, sessname=sessname)
ebsvol.release(backup=True)
JBoxEBSHousekeep.log_debug("finished node housekeeping")
def do_monitor_loading(self, user_id):
sessname = unique_sessname(user_id)
self.log_debug("Monitoring loading of session [%s] user[%s]...",
sessname, user_id)
cont = SessContainer.get_by_name(sessname)
if (cont is None) or (not cont.is_running()):
loading_step = int(self.get_loading_state(), 0)
if loading_step > 30:
self.log_error(
"Could not start instance. Session [%s] for user [%s] didn't load.",
sessname, user_id)
self.clear_container()
self.rendertpl(
"index.tpl",
cfg=JBoxCfg.nv,
state=self.state(
error=
'Could not start your instance! Please try again.',
pending_activation=False,
user_id=user_id))
return
else:
loading_step += 1
self.set_loading_state(loading_step)
self.rendertpl("loading.tpl",
user_id=user_id,
cfg=JBoxCfg.nv,
js_includes=JBPluginHandler.PLUGIN_JAVASCRIPTS)
else:
(shellport, uplport, ipnbport) = cont.get_host_ports()
self.set_container_ports({
JBoxHandler.COOKIE_PORT_SHELL: shellport,
JBoxHandler.COOKIE_PORT_UPL: uplport,
JBoxHandler.COOKIE_PORT_IPNB: ipnbport
})
self.clear_loading()
self.rendertpl("ipnbsess.tpl",
sessname=sessname,
cfg=JBoxCfg.nv,
user_id=user_id,
plugin_features=json.dumps(
self.application.settings["plugin_features"]),
js_includes=JBPluginHandler.PLUGIN_JAVASCRIPTS)
def do_node_housekeeping():
JBoxEBSHousekeep.log_debug("starting node housekeeping")
for device, vol in JBoxEBSVol.get_mapped_volumes().iteritems():
deviceid = os.path.basename(device)
vol_id = vol.volume_id
vol = EBSVol.get_volume(vol_id)
user_id = vol.tags['Name'] if 'Name' in vol.tags else None
if user_id is None:
continue
sessname = unique_sessname(user_id)
cont = SessContainer.get_by_name(sessname)
if cont is not None:
continue
JBoxEBSHousekeep.log_debug("Found orphaned volume %s for %s, %s", vol_id, user_id, sessname)
ebsvol = JBoxEBSVol(deviceid, sessname=sessname)
ebsvol.release(backup=True)
JBoxEBSHousekeep.log_debug("finished node housekeeping")
def handle_if_open_port(self, sessname, user_id):
port = self.get_argument('open_port', None)
if port is None:
return False
portname = self.get_argument('port_name', "", strip=True)
if re.match(r"^[a-zA-Z0-9]{1,20}$", portname) is None:
response = {'code': -1, 'data': 'Port name must be alpha numeric only.'}
elif portname in ['shell', 'nb', 'file']:
response = {'code': -1, 'data': 'Port names "shell", "nb" and "file" are reserved for use by JuliaBox.'}
else:
port = int(port)
if port < 8050 or port > 8052:
response = {'code': -1, 'data': 'Only ports in the range 8050-8052 can be used.'}
else:
cont = SessContainer.get_by_name(sessname)
hostport = cont._get_host_ports([port])[0]
self.set_container_ports({
portname: hostport
})
response = {'code': 0, 'data': ''}
self.write(response)
return True
def try_launch_container(cls, user_id, max_hop=False):
sessname = unique_sessname(user_id)
cont = SessContainer.get_by_name(sessname)
cls.log_debug("have existing container for %s: %r", sessname, None != cont)
if cont is not None:
cls.log_debug("container running: %r", cont.is_running())
if max_hop:
self_load = Compute.get_instance_stats(Compute.get_instance_id(), 'Load')
if self_load < 100:
SessContainer.invalidate_container(sessname)
JBoxAsyncJob.async_launch_by_name(sessname, user_id, True)
return True
is_leader = is_proposed_cluster_leader()
if ((cont is None) or (not cont.is_running())) and (not Compute.should_accept_session(is_leader)):
if cont is not None:
SessContainer.invalidate_container(cont.get_name())
JBoxAsyncJob.async_backup_and_cleanup(cont.dockid)
return False
SessContainer.invalidate_container(sessname)
JBoxAsyncJob.async_launch_by_name(sessname, user_id, True)
return True
def do_monitor_loading(self, user_id):
sessname = unique_sessname(user_id)
self.log_debug("Monitoring loading of session [%s] user[%s]...", sessname, user_id)
cont = SessContainer.get_by_name(sessname)
if (cont is None) or (not cont.is_running()):
loading_step = int(self.get_loading_state(), 0)
if loading_step > 30:
self.log_error("Could not start instance. Session [%s] for user [%s] didn't load.", sessname, user_id)
self.clear_container()
self.rendertpl("index.tpl", cfg=JBoxCfg.nv,
state=self.state(
error='Could not start your instance! Please try again.',
pending_activation=False,
user_id=user_id))
return
else:
loading_step += 1
self.set_loading_state(loading_step)
self.rendertpl("loading.tpl",
user_id=user_id,
cfg=JBoxCfg.nv,
js_includes=JBPluginHandler.PLUGIN_JAVASCRIPTS)
else:
(shellport, uplport, ipnbport) = cont.get_host_ports()
self.set_container_ports({
JBoxHandler.COOKIE_PORT_SHELL: shellport,
JBoxHandler.COOKIE_PORT_UPL: uplport,
JBoxHandler.COOKIE_PORT_IPNB: ipnbport
})
self.clear_loading()
self.rendertpl("ipnbsess.tpl", sessname=sessname, cfg=JBoxCfg.nv, user_id=user_id,
plugin_features=json.dumps(self.application.settings["plugin_features"]),
js_includes=JBPluginHandler.PLUGIN_JAVASCRIPTS)
def post(self):
sessname = self.get_session_id()
user_id = self.get_user_id()
if (sessname is None) or (user_id is None):
self.send_error()
return
mode = self.get_argument('cluster', False)
if mode is False:
ParallelHandler.log_error("Unknown mode for parallel handler")
self.send_error()
return
user = JBoxUserV2(user_id)
is_allowed = user.has_resource_profile(JBoxUserV2.RES_PROF_CLUSTER)
if not is_allowed:
ParallelHandler.log_error("Cluster access not allowed for user")
response = {'code': -1, 'data': 'You do not have permissions to use any clusters'}
self.write(response)
return True
cont = SessContainer.get_by_name(sessname)
if cont is None:
self.send_error()
return
ParallelHandler.log_debug("Parallel request %s for %s", mode, cont.debug_str())
try:
max_cores = user.get_max_cluster_cores()
balance = user.get_balance()
uc = UserCluster(user.get_user_id())
if mode == 'status':
status = uc.status()
status['limits'] = {
'max_cores': max_cores,
'credits': balance
}
self.write_machinefile(cont, uc)
response = {'code': 0, 'data': status}
elif mode == 'terminate':
action = 'terminate' if uc.isactive() else 'delete'
uc.terminate_or_delete()
response = {'code': 0, 'data': action}
elif mode == 'create':
ninsts = int(self.get_argument('ninsts', 0))
avzone = self.get_argument('avzone', '')
spot_price = float(self.get_argument('spot_price', 0.0))
if ninsts > (max_cores / UserCluster.INSTANCE_CORES):
response = {'code': -1, 'data': 'You are allowed a maximum of ' + str(max_cores) + ' cores.'}
elif (spot_price > UserCluster.INSTANCE_COST) or (spot_price < 0):
response = {
'code': -1,
'data': 'Bid price must be between $0 - $' + str(UserCluster.INSTANCE_COST) + '.'
}
else:
uc.delete()
user_data = ParallelHandler.create_user_script(cont)
uc.create(ninsts, avzone, user_data, spot_price=spot_price)
uc.start()
response = {'code': 0, 'data': ''}
else:
response = {'code': -1, 'data': 'Unknown cluster operation ' + mode}
except Exception as ex:
ParallelHandler.log_error("exception in cluster operation")
ParallelHandler._get_logger().exception("exception in cluster operation")
response = {'code': -1, 'data': ex.message}
self.write(response)
def _wait_for_session_backup(sessname):
cont = SessContainer.get_by_name(sessname)
if (cont is not None) and JBoxd._is_scheduled(JBoxAsyncJob.CMD_BACKUP_CLEANUP, (cont.dockid,)):
JBoxd.log_debug("Waiting for backup of session %s", sessname)
return False
return True
