def try_launch_container(cls, user_id, max_hop=False): sessname = unique_sessname(user_id) cont = SessContainer.get_by_name(sessname) cls.log_debug("have existing container for %s: %r", sessname, None != cont) if cont is not None: cls.log_debug("container running: %r", cont.is_running()) if max_hop: self_load = Compute.get_instance_stats(Compute.get_instance_id(), 'Load') if self_load < 100: SessContainer.invalidate_container(sessname) JBoxAsyncJob.async_launch_by_name(sessname, user_id, True) return True is_leader = is_proposed_cluster_leader() if ((cont is None) or (not cont.is_running())) and ( not Compute.should_accept_session(is_leader)): if cont is not None: SessContainer.invalidate_container(cont.get_name()) JBoxAsyncJob.async_backup_and_cleanup(cont.dockid) return False SessContainer.invalidate_container(sessname) JBoxAsyncJob.async_launch_by_name(sessname, user_id, True) return True
def post(self): self.log_debug("Homework handler got POST request") sessname = self.get_session_id() user_id = self.get_user_id() if (sessname is None) or (user_id is None): self.log_info("Homework handler got invalid sessname[%r] or user_id[%r]", sessname, user_id) self.send_error() return user = JBoxUserV2(user_id) is_admin = sessname in JBoxCfg.get("admin_sessnames", []) or user.has_role(JBoxUserV2.ROLE_SUPER) course_owner = is_admin or user.has_role(JBoxUserV2.ROLE_OFFER_COURSES) cont = SessContainer.get_by_name(sessname) self.log_info("user_id[%r], is_admin[%r], course_owner[%r]", user_id, is_admin, course_owner) if cont is None: self.log_info("user_id[%r] container not found", user_id) self.send_error() return courses_offered = user.get_courses_offered() if self.handle_if_check(user_id): return if self.handle_create_course(user_id): return if self.handle_get_metadata(is_admin, courses_offered): return if self.handle_if_report(user_id, is_admin, courses_offered): return self.log_error("no handlers found") # only AJAX requests responded to self.send_error()
def do_task(plugin_type, data): if plugin_type != JBPluginTask.JBP_CMD_ASYNC: return mode = data['action'] user_id = data['user_id'] sessname = data['sessname'] user = JBoxUserV2(user_id) is_allowed = user.has_resource_profile( JBoxUserV2.RES_PROF_DISK_EBS_10G) if not is_allowed: JBoxEBSVolAsyncTask.log_error( "Data volume access not allowed for user") return cont = SessContainer.get_by_name(sessname) if cont is None: return vol = JBoxEBSVol.get_disk_from_container(sessname) disk_state = None try: disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_id) except: pass JBoxEBSVolAsyncTask.log_debug("Data volume request %s for %s", mode, cont.debug_str()) if mode == 'attach': if vol is None: vol = JBoxEBSVol.get_disk_for_user(user_id) JBoxEBSVol.mount_host_device(vol.disk_path, cont.dockid, JBoxVol.DATA_MOUNT_POINT) disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_id) if disk_state.get_state() != JBoxDiskState.STATE_ATTACHED: disk_state.set_state(JBoxDiskState.STATE_ATTACHED) disk_state.save() elif mode == 'detach': if cont is not None and cont.is_running(): if vol is not None: # unmount from container first JBoxEBSVol.unmount_host_device(vol.disk_path, cont.dockid) elif disk_state is not None: # no volume attached. ensure disk state is updated if disk_state.get_state() != JBoxDiskState.STATE_DETACHED: disk_state.set_state(JBoxDiskState.STATE_DETACHED) disk_state.save() if vol is not None: vol.release(backup=True) JBoxEBSVolAsyncTask.log_debug( "Data volume request %s completed for %s", mode, cont.debug_str())
def get(self): sessname = self.get_session_id() user_id = self.get_user_id() if (sessname is None) or (user_id is None): self.send_error() return user = JBoxUserV2(user_id) is_admin = sessname in JBoxCfg.get("admin_sessnames", []) manage_containers = is_admin or user.has_role(JBoxUserV2.ROLE_MANAGE_CONTAINERS) show_report = is_admin or user.has_role(JBoxUserV2.ROLE_ACCESS_STATS) cont = SessContainer.get_by_name(sessname) if cont is None: self.send_error() return if self.handle_if_logout(cont): return if self.handle_if_stats(is_admin or show_report): return if self.handle_if_show_cfg(is_admin): return if self.handle_if_instance_info(is_admin): return if self.handle_switch_julia_img(user): return if self.handle_if_open_port(sessname, user_id): return juliaboxver, _upgrade_available = self.get_upgrade_available(cont) jimg_type = 0 if user.has_resource_profile(JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP): jimg_type = JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP expire = JBoxCfg.get('interactive.expire') d = dict( manage_containers=manage_containers, show_report=show_report, sessname=sessname, user_id=user_id, created=isodate.datetime_isoformat(cont.time_created()), started=isodate.datetime_isoformat(cont.time_started()), allowed_till=isodate.datetime_isoformat((cont.time_started() + timedelta(seconds=expire))), mem=cont.get_memory_allocated(), cpu=cont.get_cpu_allocated(), disk=cont.get_disk_allocated(), expire=expire, juliaboxver=juliaboxver, jimg_type=jimg_type ) self.rendertpl("ipnbadmin.tpl", d=d)
def do_task(plugin_type, data): if plugin_type != JBPluginTask.JBP_CMD_ASYNC: return mode = data["action"] user_id = data["user_id"] sessname = data["sessname"] user = JBoxUserV2(user_id) is_allowed = user.has_resource_profile(JBoxUserV2.RES_PROF_DISK_EBS_10G) if not is_allowed: JBoxEBSVolAsyncTask.log_error("Data volume access not allowed for user") return cont = SessContainer.get_by_name(sessname) if cont is None: return vol = JBoxEBSVol.get_disk_from_container(sessname) disk_state = None try: disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_id) except: pass JBoxEBSVolAsyncTask.log_debug("Data volume request %s for %s", mode, cont.debug_str()) if mode == "attach": if vol is None: vol = JBoxEBSVol.get_disk_for_user(user_id) JBoxEBSVol.mount_host_device(vol.disk_path, cont.dockid, JBoxVol.DATA_MOUNT_POINT) disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_id) if disk_state.get_state() != JBoxDiskState.STATE_ATTACHED: disk_state.set_state(JBoxDiskState.STATE_ATTACHED) disk_state.save() elif mode == "detach": if cont is not None and cont.is_running(): if vol is not None: # unmount from container first JBoxEBSVol.unmount_host_device(vol.disk_path, cont.dockid) elif disk_state is not None: # no volume attached. ensure disk state is updated if disk_state.get_state() != JBoxDiskState.STATE_DETACHED: disk_state.set_state(JBoxDiskState.STATE_DETACHED) disk_state.save() if vol is not None: vol.release(backup=True) JBoxEBSVolAsyncTask.log_debug("Data volume request %s completed for %s", mode, cont.debug_str())
def do_monitor_loading_ajax(self, user_id): sessname = unique_sessname(user_id) self.log_debug("AJAX monitoring loading of session [%s] user[%s]...", sessname, user_id) cont = SessContainer.get_by_name(sessname) if (cont is None) or (not cont.is_running()): loading_step = int(self.get_loading_state(), 0) if loading_step > 60: self.log_error("Could not start instance. Session [%s] for user [%s] didn't load.", sessname, user_id) self.write({'code': -1}) return loading_step += 1 self.set_loading_state(loading_step) self.write({'code': 0}) else: self.write({'code': 1})
def do_monitor_loading_ajax(self, user_id): sessname = unique_sessname(user_id) self.log_debug("AJAX monitoring loading of session [%s] user[%s]...", sessname, user_id) cont = SessContainer.get_by_name(sessname) if (cont is None) or (not cont.is_running()): loading_step = int(self.get_loading_state(), 0) if loading_step > 90: self.log_error("Could not start instance. Session [%s] for user [%s] didn't load.", sessname, user_id) self.write({'code': -1}) return loading_step += 1 self.set_loading_state(loading_step) self.write({'code': 0}) else: self.write({'code': 1})
def do_node_housekeeping(): JBoxEBSHousekeep.log_debug("starting node housekeeping") for device, vol in JBoxEBSVol.get_mapped_volumes().iteritems(): deviceid = os.path.basename(device) vol_id = vol.volume_id vol = EBSVol.get_volume(vol_id) user_id = vol.tags["Name"] if "Name" in vol.tags else None if user_id is None: continue sessname = unique_sessname(user_id) cont = SessContainer.get_by_name(sessname) if cont is not None: continue JBoxEBSHousekeep.log_debug("Found orphaned volume %s for %s, %s", vol_id, user_id, sessname) ebsvol = JBoxEBSVol(deviceid, sessname=sessname) ebsvol.release(backup=True) JBoxEBSHousekeep.log_debug("finished node housekeeping")
def do_monitor_loading(self, user_id): sessname = unique_sessname(user_id) self.log_debug("Monitoring loading of session [%s] user[%s]...", sessname, user_id) cont = SessContainer.get_by_name(sessname) if (cont is None) or (not cont.is_running()): loading_step = int(self.get_loading_state(), 0) if loading_step > 30: self.log_error( "Could not start instance. Session [%s] for user [%s] didn't load.", sessname, user_id) self.clear_container() self.rendertpl( "index.tpl", cfg=JBoxCfg.nv, state=self.state( error= 'Could not start your instance! Please try again.', pending_activation=False, user_id=user_id)) return else: loading_step += 1 self.set_loading_state(loading_step) self.rendertpl("loading.tpl", user_id=user_id, cfg=JBoxCfg.nv, js_includes=JBPluginHandler.PLUGIN_JAVASCRIPTS) else: (shellport, uplport, ipnbport) = cont.get_host_ports() self.set_container_ports({ JBoxHandler.COOKIE_PORT_SHELL: shellport, JBoxHandler.COOKIE_PORT_UPL: uplport, JBoxHandler.COOKIE_PORT_IPNB: ipnbport }) self.clear_loading() self.rendertpl("ipnbsess.tpl", sessname=sessname, cfg=JBoxCfg.nv, user_id=user_id, plugin_features=json.dumps( self.application.settings["plugin_features"]), js_includes=JBPluginHandler.PLUGIN_JAVASCRIPTS)
def do_node_housekeeping(): JBoxEBSHousekeep.log_debug("starting node housekeeping") for device, vol in JBoxEBSVol.get_mapped_volumes().iteritems(): deviceid = os.path.basename(device) vol_id = vol.volume_id vol = EBSVol.get_volume(vol_id) user_id = vol.tags['Name'] if 'Name' in vol.tags else None if user_id is None: continue sessname = unique_sessname(user_id) cont = SessContainer.get_by_name(sessname) if cont is not None: continue JBoxEBSHousekeep.log_debug("Found orphaned volume %s for %s, %s", vol_id, user_id, sessname) ebsvol = JBoxEBSVol(deviceid, sessname=sessname) ebsvol.release(backup=True) JBoxEBSHousekeep.log_debug("finished node housekeeping")
def handle_if_open_port(self, sessname, user_id): port = self.get_argument('open_port', None) if port is None: return False portname = self.get_argument('port_name', "", strip=True) if re.match(r"^[a-zA-Z0-9]{1,20}$", portname) is None: response = {'code': -1, 'data': 'Port name must be alpha numeric only.'} elif portname in ['shell', 'nb', 'file']: response = {'code': -1, 'data': 'Port names "shell", "nb" and "file" are reserved for use by JuliaBox.'} else: port = int(port) if port < 8050 or port > 8052: response = {'code': -1, 'data': 'Only ports in the range 8050-8052 can be used.'} else: cont = SessContainer.get_by_name(sessname) hostport = cont._get_host_ports([port])[0] self.set_container_ports({ portname: hostport }) response = {'code': 0, 'data': ''} self.write(response) return True
def try_launch_container(cls, user_id, max_hop=False): sessname = unique_sessname(user_id) cont = SessContainer.get_by_name(sessname) cls.log_debug("have existing container for %s: %r", sessname, None != cont) if cont is not None: cls.log_debug("container running: %r", cont.is_running()) if max_hop: self_load = Compute.get_instance_stats(Compute.get_instance_id(), 'Load') if self_load < 100: SessContainer.invalidate_container(sessname) JBoxAsyncJob.async_launch_by_name(sessname, user_id, True) return True is_leader = is_proposed_cluster_leader() if ((cont is None) or (not cont.is_running())) and (not Compute.should_accept_session(is_leader)): if cont is not None: SessContainer.invalidate_container(cont.get_name()) JBoxAsyncJob.async_backup_and_cleanup(cont.dockid) return False SessContainer.invalidate_container(sessname) JBoxAsyncJob.async_launch_by_name(sessname, user_id, True) return True
def do_monitor_loading(self, user_id): sessname = unique_sessname(user_id) self.log_debug("Monitoring loading of session [%s] user[%s]...", sessname, user_id) cont = SessContainer.get_by_name(sessname) if (cont is None) or (not cont.is_running()): loading_step = int(self.get_loading_state(), 0) if loading_step > 30: self.log_error("Could not start instance. Session [%s] for user [%s] didn't load.", sessname, user_id) self.clear_container() self.rendertpl("index.tpl", cfg=JBoxCfg.nv, state=self.state( error='Could not start your instance! Please try again.', pending_activation=False, user_id=user_id)) return else: loading_step += 1 self.set_loading_state(loading_step) self.rendertpl("loading.tpl", user_id=user_id, cfg=JBoxCfg.nv, js_includes=JBPluginHandler.PLUGIN_JAVASCRIPTS) else: (shellport, uplport, ipnbport) = cont.get_host_ports() self.set_container_ports({ JBoxHandler.COOKIE_PORT_SHELL: shellport, JBoxHandler.COOKIE_PORT_UPL: uplport, JBoxHandler.COOKIE_PORT_IPNB: ipnbport }) self.clear_loading() self.rendertpl("ipnbsess.tpl", sessname=sessname, cfg=JBoxCfg.nv, user_id=user_id, plugin_features=json.dumps(self.application.settings["plugin_features"]), js_includes=JBPluginHandler.PLUGIN_JAVASCRIPTS)
def post(self): sessname = self.get_session_id() user_id = self.get_user_id() if (sessname is None) or (user_id is None): self.send_error() return mode = self.get_argument('cluster', False) if mode is False: ParallelHandler.log_error("Unknown mode for parallel handler") self.send_error() return user = JBoxUserV2(user_id) is_allowed = user.has_resource_profile(JBoxUserV2.RES_PROF_CLUSTER) if not is_allowed: ParallelHandler.log_error("Cluster access not allowed for user") response = {'code': -1, 'data': 'You do not have permissions to use any clusters'} self.write(response) return True cont = SessContainer.get_by_name(sessname) if cont is None: self.send_error() return ParallelHandler.log_debug("Parallel request %s for %s", mode, cont.debug_str()) try: max_cores = user.get_max_cluster_cores() balance = user.get_balance() uc = UserCluster(user.get_user_id()) if mode == 'status': status = uc.status() status['limits'] = { 'max_cores': max_cores, 'credits': balance } self.write_machinefile(cont, uc) response = {'code': 0, 'data': status} elif mode == 'terminate': action = 'terminate' if uc.isactive() else 'delete' uc.terminate_or_delete() response = {'code': 0, 'data': action} elif mode == 'create': ninsts = int(self.get_argument('ninsts', 0)) avzone = self.get_argument('avzone', '') spot_price = float(self.get_argument('spot_price', 0.0)) if ninsts > (max_cores / UserCluster.INSTANCE_CORES): response = {'code': -1, 'data': 'You are allowed a maximum of ' + str(max_cores) + ' cores.'} elif (spot_price > UserCluster.INSTANCE_COST) or (spot_price < 0): response = { 'code': -1, 'data': 'Bid price must be between $0 - $' + str(UserCluster.INSTANCE_COST) + '.' } else: uc.delete() user_data = ParallelHandler.create_user_script(cont) uc.create(ninsts, avzone, user_data, spot_price=spot_price) uc.start() response = {'code': 0, 'data': ''} else: response = {'code': -1, 'data': 'Unknown cluster operation ' + mode} except Exception as ex: ParallelHandler.log_error("exception in cluster operation") ParallelHandler._get_logger().exception("exception in cluster operation") response = {'code': -1, 'data': ex.message} self.write(response)
def _wait_for_session_backup(sessname): cont = SessContainer.get_by_name(sessname) if (cont is not None) and JBoxd._is_scheduled(JBoxAsyncJob.CMD_BACKUP_CLEANUP, (cont.dockid,)): JBoxd.log_debug("Waiting for backup of session %s", sessname) return False return True