def post(self): self.log_debug("API management handler got POST request") sessname = self.get_session_id() user_id = self.get_user_id() if (sessname is None) or (user_id is None): self.send_error() return user = JBoxUserV2(user_id) is_admin = sessname in JBoxCfg.get( "admin_sessnames", []) or user.has_role(JBoxUserV2.ROLE_SUPER) self.log_info("API manager. user_id[%s] is_admin[%r]", user_id, is_admin) if user.has_resource_profile(JBoxUserV2.RES_PROF_API_PUBLISHER): if self.handle_get_api_info(user_id, is_admin): return if self.handle_create_api(user_id, is_admin): return if self.handle_delete_api(user_id, is_admin): return else: if self.handle_enable_api(user_id, is_admin): return self.log_error("no handlers found") # only AJAX requests responded to self.send_error()
def get_disk_for_user(email): VolMgr.log_debug("restoring disk for %s", email) user = JBoxUserV2(email) custom_jimg = None ipython_profile = 'julia' # TODO: image path should be picked up from config if user.has_resource_profile(JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP): custom_jimg = '/opt/julia_packages/jimg/stable/sys.ji' ipython_profile = 'jboxjulia' plugin = JBoxVol.jbox_get_plugin(JBoxVol.JBP_USERHOME) if plugin is None: raise Exception("No plugin found for %s" % (JBoxVol.JBP_USERHOME, )) disk = plugin.get_disk_for_user(email) try: disk.setup_julia_image(ipython_profile, custom_jimg) disk.setup_tutorial_link() disk.gen_ssh_key() disk.gen_gitconfig() except IOError, ioe: if ioe.errno == errno.ENOSPC: # continue login on ENOSPC to allow user to delete files JBoxVol.log_exception( "No space left to configure JuliaBox for %s", email) else: raise
def post(self): self.log_debug("Homework handler got POST request") sessname = self.get_session_id() user_id = self.get_user_id() if (sessname is None) or (user_id is None): self.log_info("Homework handler got invalid sessname[%r] or user_id[%r]", sessname, user_id) self.send_error() return user = JBoxUserV2(user_id) is_admin = sessname in JBoxCfg.get("admin_sessnames", []) or user.has_role(JBoxUserV2.ROLE_SUPER) course_owner = is_admin or user.has_role(JBoxUserV2.ROLE_OFFER_COURSES) cont = SessContainer.get_by_name(sessname) self.log_info("user_id[%r], is_admin[%r], course_owner[%r]", user_id, is_admin, course_owner) if cont is None: self.log_info("user_id[%r] container not found", user_id) self.send_error() return courses_offered = user.get_courses_offered() if self.handle_if_check(user_id): return if self.handle_create_course(user_id): return if self.handle_get_metadata(is_admin, courses_offered): return if self.handle_if_report(user_id, is_admin, courses_offered): return self.log_error("no handlers found") # only AJAX requests responded to self.send_error()
def handle_get_user(self, user_id, is_admin): mode = self.get_argument('mode', None) if (mode is None) or (mode != "fetch"): return False fetch_uid = self.get_argument('user_id', '', strip=True) if fetch_uid is None or len(fetch_uid) == 0: response = {'code': -1, 'data': 'Invalid user id!'} self.write(response) return True try: fetch_user = JBoxUserV2(fetch_uid) except JBoxDBItemNotFound: response = { 'code': -1, 'data': 'No such user - %s' % (fetch_uid, ) } self.write(response) return True courses = ','.join(fetch_user.get_courses_offered()) resp = { 'user_id': fetch_user.get_user_id(), 'role': fetch_user.get_role(), 'resprof': fetch_user.get_resource_profile(), 'cores': fetch_user.get_max_cluster_cores(), 'courses': courses } response = {'code': 0, 'data': resp} self.write(response) return True
def post(self): self.log_debug("User management handler got POST request") sessname = self.get_session_id() user_id = self.get_user_id() if (sessname is None) or (user_id is None): self.send_error() return user = JBoxUserV2(user_id) is_admin = sessname in JBoxCfg.get( "admin_sessnames", []) or user.has_role(JBoxUserV2.ROLE_SUPER) self.log_info("User manager. user_id[%s] is_admin[%r]", user_id, is_admin) if not is_admin: self.send_error(status_code=403) return if self.handle_get_user(user_id, is_admin): return if self.handle_update_user(user_id, is_admin): return self.log_error("no handlers found") # only AJAX requests responded to self.send_error()
def post_auth_store_credentials(self, user_id, authtype, credtok): # TODO: make this generic for other authentication/authorization modes jbuser = JBoxUserV2(user_id, create=True) jbuser.set_gtok(base64.b64encode(credtok)) jbuser.save() self.redirect('/') return
def is_allowed(handler): user_id = UserAdminUIModule.get_user_id(handler) if user_id is None: return False user = JBoxUserV2(user_id) sessname = handler.get_session_id() is_admin = sessname in JBoxCfg.get( "admin_sessnames", []) or user.has_role(JBoxUserV2.ROLE_SUPER) return is_admin
def do_task(plugin_type, data): if plugin_type != JBPluginTask.JBP_CMD_ASYNC: return mode = data['action'] user_id = data['user_id'] sessname = data['sessname'] user = JBoxUserV2(user_id) is_allowed = user.has_resource_profile( JBoxUserV2.RES_PROF_DISK_EBS_10G) if not is_allowed: JBoxEBSVolAsyncTask.log_error( "Data volume access not allowed for user") return cont = SessContainer.get_by_name(sessname) if cont is None: return vol = JBoxEBSVol.get_disk_from_container(sessname) disk_state = None try: disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_id) except: pass JBoxEBSVolAsyncTask.log_debug("Data volume request %s for %s", mode, cont.debug_str()) if mode == 'attach': if vol is None: vol = JBoxEBSVol.get_disk_for_user(user_id) JBoxEBSVol.mount_host_device(vol.disk_path, cont.dockid, JBoxVol.DATA_MOUNT_POINT) disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_id) if disk_state.get_state() != JBoxDiskState.STATE_ATTACHED: disk_state.set_state(JBoxDiskState.STATE_ATTACHED) disk_state.save() elif mode == 'detach': if cont is not None and cont.is_running(): if vol is not None: # unmount from container first JBoxEBSVol.unmount_host_device(vol.disk_path, cont.dockid) elif disk_state is not None: # no volume attached. ensure disk state is updated if disk_state.get_state() != JBoxDiskState.STATE_DETACHED: disk_state.set_state(JBoxDiskState.STATE_DETACHED) disk_state.save() if vol is not None: vol.release(backup=True) JBoxEBSVolAsyncTask.log_debug( "Data volume request %s completed for %s", mode, cont.debug_str())
def handle_update_user(self, user_id, is_admin): mode = self.get_argument('mode', None) if (mode is None) or (mode != "update"): return False fetch_uid = self.get_argument('user_id', '', strip=True) if fetch_uid is None or len(fetch_uid) == 0: response = {'code': -1, 'data': 'Invalid user id!'} self.write(response) return True try: fetch_user = JBoxUserV2(fetch_uid) except JBoxDBItemNotFound: response = { 'code': -1, 'data': 'No such user - %s' % (fetch_uid, ) } self.write(response) return True updated = False cores = self.get_argument('cores', None, strip=True) if cores is not None and len(cores) > 0: cores = int(cores) if cores != fetch_user.get_max_cluster_cores(): fetch_user.set_max_cluster_cores(cores) updated = True courses = self.get_argument('courses', None, strip=True) if courses is not None and len(courses) > 0: courses = courses.split(',') if courses != fetch_user.get_courses_offered(): fetch_user.set_courses_offered(courses) updated = True role = self.get_argument('role', None, strip=True) if role is not None and len(role) > 0: role = int(role) if role != fetch_user.get_role(): fetch_user.set_attrib('role', role) updated = True resprof = self.get_argument('resprof', None, strip=True) if resprof is not None and len(resprof) > 0: resprof = int(resprof) if resprof != fetch_user.get_resource_profile(): fetch_user.set_attrib('resource_profile', resprof) updated = True if updated: fetch_user.save() response = {'code': 0, 'data': ''} self.write(response) return True
def handle_enable_api(self, user_id, is_admin): mode = self.get_argument('mode', None) if (mode is None) or (mode != "enable"): return False user = JBoxUserV2(user_id) user.set_resource_profile(JBoxUserV2.RES_PROF_API_PUBLISHER) user.save() response = {'code': 0, 'data': ''} self.write(response) return True
def get(self): sessname = self.get_session_id() user_id = self.get_user_id() if (sessname is None) or (user_id is None): self.send_error() return user = JBoxUserV2(user_id) is_admin = sessname in JBoxCfg.get("admin_sessnames", []) manage_containers = is_admin or user.has_role(JBoxUserV2.ROLE_MANAGE_CONTAINERS) show_report = is_admin or user.has_role(JBoxUserV2.ROLE_ACCESS_STATS) cont = SessContainer.get_by_name(sessname) if cont is None: self.send_error() return if self.handle_if_logout(cont): return if self.handle_if_stats(is_admin or show_report): return if self.handle_if_show_cfg(is_admin): return if self.handle_if_instance_info(is_admin): return if self.handle_switch_julia_img(user): return if self.handle_if_open_port(sessname, user_id): return juliaboxver, _upgrade_available = self.get_upgrade_available(cont) jimg_type = 0 if user.has_resource_profile(JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP): jimg_type = JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP expire = JBoxCfg.get('interactive.expire') d = dict( manage_containers=manage_containers, show_report=show_report, sessname=sessname, user_id=user_id, created=isodate.datetime_isoformat(cont.time_created()), started=isodate.datetime_isoformat(cont.time_started()), allowed_till=isodate.datetime_isoformat((cont.time_started() + timedelta(seconds=expire))), mem=cont.get_memory_allocated(), cpu=cont.get_cpu_allocated(), disk=cont.get_disk_allocated(), expire=expire, juliaboxver=juliaboxver, jimg_type=jimg_type ) self.rendertpl("ipnbadmin.tpl", d=d)
def calc_stats(): VolMgr.STATS = { 'date': '', 'num_users': 0, 'loopback': { 'num_files': 0, 'total_size': 0, 'sizes': [], 'min_size': 0, 'max_size': 0, 'avg_size': 0, 'sizes_hist': { 'counts': [], 'bins': [] } } } result_set = JBoxUserV2.table().scan(attributes=('user_id', )) for user in result_set: VolMgr.calc_stat(user['user_id']) sizes = VolMgr.STATS['loopback']['sizes'] VolMgr.STATS['loopback']['num_files'] = len(sizes) VolMgr.STATS['loopback']['total_size'] = sum(sizes) VolMgr.STATS['loopback']['min_size'] = min( sizes) if len(sizes) > 0 else 0 VolMgr.STATS['loopback']['max_size'] = max( sizes) if len(sizes) > 0 else 0 VolMgr.STATS['loopback']['avg_size'] = sum(sizes) / len(sizes) if len( sizes) > 0 else 0 bin_size = int((VolMgr.STATS['loopback']['max_size'] - VolMgr.STATS['loopback']['min_size']) / 10) min_size = VolMgr.STATS['loopback']['min_size'] bins = [] for idx in range(0, 10): bins.append(min_size + bin_size * idx) bins.append(VolMgr.STATS['loopback']['max_size']) counts = [0] * 10 for size in sizes: for idx in range(1, 11): if size <= bins[idx]: counts[idx - 1] += 1 break VolMgr.STATS['loopback']['sizes_hist']['counts'] = counts VolMgr.STATS['loopback']['sizes_hist']['bins'] = bins del VolMgr.STATS['loopback']['sizes'] VolMgr.STATS['date'] = datetime.datetime.now(pytz.utc).isoformat()
def test(): sprops = JBoxSessionProps(TESTCLSTR, unique_sessname('*****@*****.**')) TestDBTables.log_debug("JBoxSessionProps. user_id: %s, snapshot_id: %s, message: %s", sprops.get_user_id(), sprops.get_snapshot_id(), sprops.get_message()) JBoxDynConfig.set_cluster_leader(TESTCLSTR, 'testinstance') assert JBoxDynConfig.get_cluster_leader(TESTCLSTR) == 'testinstance' JBoxDynConfig.unset_cluster_leader(TESTCLSTR) assert JBoxDynConfig.get_cluster_leader(TESTCLSTR) is None assert JBoxDynConfig.get_allow_registration(TESTCLSTR) JBoxDynConfig.set_allow_registration(TESTCLSTR, False) assert not JBoxDynConfig.get_allow_registration(TESTCLSTR) JBoxDynConfig.set_allow_registration(TESTCLSTR, True) assert JBoxDynConfig.get_allow_registration(TESTCLSTR) assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 60 JBoxDynConfig.set_registration_hourly_rate(TESTCLSTR, 20) assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 20 JBoxDynConfig.set_registration_hourly_rate(TESTCLSTR, 60) assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 60 assert JBoxDynConfig.get_message(TESTCLSTR) is None JBoxDynConfig.set_message(TESTCLSTR, "hello world", datetime.timedelta(minutes=1)) assert JBoxDynConfig.get_message(TESTCLSTR) == "hello world" JBoxDynConfig.set_user_home_image(TESTCLSTR, "juliabox-user-home-templates", "user_home_28Nov2014.tar.gz") assert JBoxDynConfig.get_user_home_image(TESTCLSTR) == ("juliabox-user-home-templates", "user_home_28Nov2014.tar.gz") num_pending_activations = JBoxUserV2.count_pending_activations() TestDBTables.log_debug("pending activations: %d", num_pending_activations) count_created = JBoxUserV2.count_created(48) TestDBTables.log_debug("accounts created in last 1 hour: %d", count_created)
def get_updated_token(handler): user_id = handler.get_user_id() jbuser = JBoxUserV2(user_id) creds = jbuser.get_gtok() authtok = None if creds is not None: try: creds_json = json.loads(base64.b64decode(creds)) creds_json = GoogleAuthUIHandler._renew_creds(creds_json) authtok = creds_json['access_token'] except: GoogleAuthUIHandler.log_warn("stale stored creds. will renew on next use. user: " + user_id) creds = None authtok = None return {'creds': creds, 'authtok': authtok, 'user_id': user_id}
def calc_stats(): VolMgr.STATS = { 'date': '', 'num_users': 0, 'loopback': { 'num_files': 0, 'total_size': 0, 'sizes': [], 'min_size': 0, 'max_size': 0, 'avg_size': 0, 'sizes_hist': { 'counts': [], 'bins': [] } } } result_set = JBoxUserV2.table().scan(attributes=('user_id',)) for user in result_set: VolMgr.calc_stat(user['user_id']) sizes = VolMgr.STATS['loopback']['sizes'] VolMgr.STATS['loopback']['num_files'] = len(sizes) VolMgr.STATS['loopback']['total_size'] = sum(sizes) VolMgr.STATS['loopback']['min_size'] = min(sizes) if len(sizes) > 0 else 0 VolMgr.STATS['loopback']['max_size'] = max(sizes) if len(sizes) > 0 else 0 VolMgr.STATS['loopback']['avg_size'] = sum(sizes) / len(sizes) if len(sizes) > 0 else 0 bin_size = int((VolMgr.STATS['loopback']['max_size'] - VolMgr.STATS['loopback']['min_size']) / 10) min_size = VolMgr.STATS['loopback']['min_size'] bins = [] for idx in range(0, 10): bins.append(min_size + bin_size*idx) bins.append(VolMgr.STATS['loopback']['max_size']) counts = [0] * 10 for size in sizes: for idx in range(1, 11): if size <= bins[idx]: counts[idx-1] += 1 break VolMgr.STATS['loopback']['sizes_hist']['counts'] = counts VolMgr.STATS['loopback']['sizes_hist']['bins'] = bins del VolMgr.STATS['loopback']['sizes'] VolMgr.STATS['date'] = datetime.datetime.now(pytz.utc).isoformat()
def calc_stats(): VolMgr.STATS = { "date": "", "num_users": 0, "loopback": { "num_files": 0, "total_size": 0, "sizes": [], "min_size": 0, "max_size": 0, "avg_size": 0, "sizes_hist": {"counts": [], "bins": []}, }, } result_set = JBoxUserV2.table().scan(attributes=("user_id",)) for user in result_set: VolMgr.calc_stat(user["user_id"]) sizes = VolMgr.STATS["loopback"]["sizes"] VolMgr.STATS["loopback"]["num_files"] = len(sizes) VolMgr.STATS["loopback"]["total_size"] = sum(sizes) VolMgr.STATS["loopback"]["min_size"] = min(sizes) if len(sizes) > 0 else 0 VolMgr.STATS["loopback"]["max_size"] = max(sizes) if len(sizes) > 0 else 0 VolMgr.STATS["loopback"]["avg_size"] = sum(sizes) / len(sizes) if len(sizes) > 0 else 0 bin_size = int((VolMgr.STATS["loopback"]["max_size"] - VolMgr.STATS["loopback"]["min_size"]) / 10) min_size = VolMgr.STATS["loopback"]["min_size"] bins = [] for idx in range(0, 10): bins.append(min_size + bin_size * idx) bins.append(VolMgr.STATS["loopback"]["max_size"]) counts = [0] * 10 for size in sizes: for idx in range(1, 11): if size <= bins[idx]: counts[idx - 1] += 1 break VolMgr.STATS["loopback"]["sizes_hist"]["counts"] = counts VolMgr.STATS["loopback"]["sizes_hist"]["bins"] = bins del VolMgr.STATS["loopback"]["sizes"] VolMgr.STATS["date"] = datetime.datetime.now(pytz.utc).isoformat()
def process_user_id(handler, user_id): SiteRedirectHandler.configure() redirect = True if SiteRedirectHandler.REDIRECT_TYPE == SiteRedirectHandler.TYPE_NEW: try: # check if user id is already registered JBoxUserV2(user_id, create=False) redirect = False except JBoxDBItemNotFound: pass if redirect: handler.render(os.path.join(SiteRedirectHandler.TEMPLATE_PATH, "redirect.tpl"), cfg=JBoxCfg.nv, redirect_url=SiteRedirectHandler.REDIRECT_URL, redirect_msg=SiteRedirectHandler.REDIRECT_MSG) SiteRedirectHandler.log_info("Redirected user %s to %s", user_id, SiteRedirectHandler.REDIRECT_URL) return False else: return True
def post_auth_launch_container(self, user_id): jbuser = JBoxUserV2(user_id, create=True) if not JBPluginHandler.is_user_activated(jbuser): self.redirect('/?pending_activation=' + user_id) return self.set_authenticated(user_id) if jbuser.is_new: jbuser.save() if self.redirect_to_logged_in_instance(user_id): return # check if the current instance is appropriate for launching this if self.try_launch_container(user_id, max_hop=False): self.set_container_initialized(Compute.get_instance_local_ip(), user_id) else: # redirect to an appropriate instance redirect_instance = Compute.get_redirect_instance_id() if redirect_instance is not None: redirect_ip = Compute.get_instance_local_ip(redirect_instance) self.set_redirect_instance_id(redirect_ip) self.redirect('/')
def post(self): sessname = self.get_session_id() user_id = self.get_user_id() if (sessname is None) or (user_id is None): self.send_error() return mode = self.get_argument('cluster', False) if mode is False: ParallelHandler.log_error("Unknown mode for parallel handler") self.send_error() return user = JBoxUserV2(user_id) is_allowed = user.has_resource_profile(JBoxUserV2.RES_PROF_CLUSTER) if not is_allowed: ParallelHandler.log_error("Cluster access not allowed for user") response = {'code': -1, 'data': 'You do not have permissions to use any clusters'} self.write(response) return True cont = SessContainer.get_by_name(sessname) if cont is None: self.send_error() return ParallelHandler.log_debug("Parallel request %s for %s", mode, cont.debug_str()) try: max_cores = user.get_max_cluster_cores() balance = user.get_balance() uc = UserCluster(user.get_user_id()) if mode == 'status': status = uc.status() status['limits'] = { 'max_cores': max_cores, 'credits': balance } self.write_machinefile(cont, uc) response = {'code': 0, 'data': status} elif mode == 'terminate': action = 'terminate' if uc.isactive() else 'delete' uc.terminate_or_delete() response = {'code': 0, 'data': action} elif mode == 'create': ninsts = int(self.get_argument('ninsts', 0)) avzone = self.get_argument('avzone', '') spot_price = float(self.get_argument('spot_price', 0.0)) if ninsts > (max_cores / UserCluster.INSTANCE_CORES): response = {'code': -1, 'data': 'You are allowed a maximum of ' + str(max_cores) + ' cores.'} elif (spot_price > UserCluster.INSTANCE_COST) or (spot_price < 0): response = { 'code': -1, 'data': 'Bid price must be between $0 - $' + str(UserCluster.INSTANCE_COST) + '.' } else: uc.delete() user_data = ParallelHandler.create_user_script(cont) uc.create(ninsts, avzone, user_data, spot_price=spot_price) uc.start() response = {'code': 0, 'data': ''} else: response = {'code': -1, 'data': 'Unknown cluster operation ' + mode} except Exception as ex: ParallelHandler.log_error("exception in cluster operation") ParallelHandler._get_logger().exception("exception in cluster operation") response = {'code': -1, 'data': ex.message} self.write(response)
def is_allowed(handler): user_id = ParallelUIModule.get_user_id(handler) user = JBoxUserV2(user_id) return user.has_resource_profile(JBoxUserV2.RES_PROF_CLUSTER)
def is_allowed(handler): user_id = APIAdminUIModule.get_user_id(handler) user = JBoxUserV2(user_id) return user.has_resource_profile(JBoxUserV2.RES_PROF_API_PUBLISHER)
def upload_course(user_id, course): course_id = course['id'] if (user_id is not None) and (user_id not in course['admins']): course['admins'].append(user_id) existing_course = JBoxDynConfig.get_course(Compute.get_install_id(), course_id) existing_admins = existing_course['admins'] if existing_course is not None else [] existing_psets = existing_course['problemsets'] if existing_course is not None else [] question_list = {} if (existing_course is not None) and ('questions' in existing_course): question_list = existing_course['questions'] if (existing_course is not None) and (user_id is not None) and (user_id not in existing_admins): return -1 for pset in course['problemsets']: pset_id = pset['id'] if pset_id not in existing_psets: existing_psets.append(pset_id) question_ids = [q['id'] for q in pset['questions']] question_list[pset_id] = question_ids dt = datetime.datetime.now(pytz.utc) JBoxDynConfig.set_course(Compute.get_install_id(), course_id, { 'admins': course['admins'], 'problemsets': existing_psets, 'questions': question_list, 'create_time': JBoxUserV2.datetime_to_yyyymmdd(dt) }) for problemset in course['problemsets']: problemset_id = problemset['id'] questions = problemset['questions'] for question in questions: question_id = question['id'] answer = question['ans'] score = question['score'] if 'score' in question else 0 attempts = question['attempts'] if 'attempts' in question else 0 # nscore = question['nscore'] if 'nscore' in question else 0 try: ans = JBoxCourseHomework(course_id, problemset_id, question_id, JBoxCourseHomework.ANSWER_KEY, answer=answer, state=JBoxCourseHomework.STATE_CORRECT, create=True) except: ans = JBoxCourseHomework(course_id, problemset_id, question_id, JBoxCourseHomework.ANSWER_KEY) ans.set_answer(answer, JBoxCourseHomework.STATE_CORRECT) ans.set_score(score) ans.set_attempts(attempts) ans.save() for uid in course['admins']: user = JBoxUserV2(uid) courses_offered = user.get_courses_offered() if course['id'] not in courses_offered: courses_offered.append(course['id']) user.set_courses_offered(courses_offered) user.set_role(JBoxUserV2.ROLE_OFFER_COURSES) user.save() for uid in existing_admins: if uid in course['admins']: continue user = JBoxUserV2(uid) courses_offered = user.get_courses_offered() if course['id'] in courses_offered: courses_offered.remove(course['id']) user.set_courses_offered(courses_offered) user.set_role(JBoxUserV2.ROLE_OFFER_COURSES) user.save() return 0
def is_allowed(handler): user_id = JBoxEBSVolUIModule.get_user_id(handler) user = JBoxUserV2(user_id) return user.has_resource_profile(JBoxUserV2.RES_PROF_DISK_EBS_10G)