def push_backup(user_email, disk_path): sessname = unique_sessname(user_email) S3Disk.log_info("pushing %s.tar.gz from %s to %s", sessname, disk_path, JBoxVol.BACKUP_BUCKET) bkup_file = os.path.join('/tmp', sessname + ".tar.gz") bkup_tar = tarfile.open(bkup_file, 'w:gz') def set_perms(tinfo): tinfo.uid = 1000 tinfo.gid = 1000 tinfo.uname = 'ubuntu' tinfo.gname = 'ubuntu' return tinfo for f in os.listdir(disk_path): if f.startswith('.') and (f in ['.julia', '.juliabox']): continue full_path = os.path.join(disk_path, f) bkup_tar.add(full_path, os.path.join('juser', f), filter=set_perms) bkup_tar.close() os.chmod(bkup_file, 0666) # Upload to S3 if so configured. Delete from local if successful. bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \ datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET) if JBoxVol.BACKUP_BUCKET is not None: if CloudHost.push_file_to_s3( JBoxVol.BACKUP_BUCKET, bkup_file, metadata={'backup_time': bkup_file_mtime.isoformat()}) is not None: os.remove(bkup_file) S3Disk.log_info("Moved backup to S3 " + sessname)
def calc_stat(user_email): VolMgr.STATS['num_users'] += 1 sessname = unique_sessname(user_email) k = CloudHost.pull_file_from_s3(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz", metadata_only=True) if k is not None: VolMgr.STATS['loopback']['sizes'].append(k.size)
def __init__(self, disk_path, user_email=None, user_name=None, sessname=None, old_sessname=None): self.disk_path = disk_path self.user_email = user_email if user_name is not None: self.user_name = user_name elif user_email is not None: self.user_name = get_user_name(user_email) else: self.user_name = None if sessname is not None: self.sessname = sessname elif user_email is not None: self.sessname = unique_sessname(user_email) else: self.sessname = None if old_sessname is not None: self.old_sessname = old_sessname elif user_email is not None: self.old_sessname = esc_sessname(user_email) else: self.old_sessname = None self._dbg_str = str(self.sessname) + "(" + self.disk_path + ")"
def pull_backup(user_email): sessname = unique_sessname(user_email) S3Disk.log_info("pulling %s.tar.gz from %s", sessname, JBoxVol.BACKUP_BUCKET) CloudHost.pull_file_from_s3(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz", metadata_only=False)
def restore(self): sessname = unique_sessname(self.user_email) old_sessname = esc_sessname(self.user_email) src = os.path.join(JBoxVol.BACKUP_LOC, sessname + ".tar.gz") k = JBoxVol.pull_from_s3(src) # download from S3 if exists if not os.path.exists(src): if old_sessname is not None: src = os.path.join(JBoxVol.BACKUP_LOC, old_sessname + ".tar.gz") k = JBoxVol.pull_from_s3(src) # download from S3 if exists if not os.path.exists(src): return JBoxVol.log_info("Filtering out restore info from backup " + src + " to " + self.disk_path) src_tar = tarfile.open(src, 'r:gz') try: for info in src_tar.getmembers(): if not info.name.startswith('juser/'): continue if info.name.startswith('juser/.') and (info.name.split('/')[1] in ['.juliabox', '.julia', '.ipython']): continue info.name = info.name[6:] if len(info.name) == 0: continue src_tar.extract(info, self.disk_path) JBoxVol.log_info("Restored backup at " + self.disk_path) except IOError, ioe: if ioe.errno == errno.ENOSPC: # continue login on ENOSPC to allow user to delete files JBoxVol.log_exception("No space left to restore backup for %s", sessname) else: raise
def push_backup(user_email, disk_path): sessname = unique_sessname(user_email) S3Disk.log_info("pushing %s.tar.gz from %s to %s", sessname, disk_path, JBoxVol.BACKUP_BUCKET) bkup_file = os.path.join('/tmp', sessname + ".tar.gz") bkup_tar = tarfile.open(bkup_file, 'w:gz') def set_perms(tinfo): tinfo.uid = 1000 tinfo.gid = 1000 tinfo.uname = 'ubuntu' tinfo.gname = 'ubuntu' return tinfo for f in os.listdir(disk_path): if f.startswith('.') and (f in ['.julia', '.juliabox']): continue full_path = os.path.join(disk_path, f) bkup_tar.add(full_path, os.path.join('juser', f), filter=set_perms) bkup_tar.close() os.chmod(bkup_file, 0666) # Upload to S3 if so configured. Delete from local if successful. bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \ datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET) if JBoxVol.BACKUP_BUCKET is not None: if CloudHost.push_file_to_s3(JBoxVol.BACKUP_BUCKET, bkup_file, metadata={'backup_time': bkup_file_mtime.isoformat()}) is not None: os.remove(bkup_file) S3Disk.log_info("Moved backup to S3 " + sessname)
def get_disk_for_user(user_email): JBoxEBSVol.log_debug("creating EBS volume for %s", user_email) if not JBoxEBSVol.HAS_EBS: raise Exception("EBS disks not enabled") disk_id = JBoxEBSVol._reserve_disk_id() if disk_id is None: raise Exception("No free disk available") sess_id = unique_sessname(user_email) sess_props = JBoxSessionProps(sess_id, create=True, user_id=user_email) if sess_props.is_new: sess_props.save() snap_id = sess_props.get_snapshot_id() if snap_id is None: snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id, user_email) _dev_path, mnt_path = CloudHost.create_new_volume(snap_id, disk_id, JBoxEBSVol.FS_LOC, tag=user_email) ebsvol = JBoxEBSVol(mnt_path, user_email=user_email) if snap_id == JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT: JBoxEBSVol.log_debug("creating home folder on blank volume for %s", user_email) ebsvol.restore_user_home() ebsvol.restore() #else: # snap_age_days = CloudHost.get_snapshot_age(snap_id).total_seconds()/(60*60*24) # if snap_age_days > 7: # ebsvol.restore_user_home() JBoxEBSVol.log_debug("setting up instance configuration on disk for %s", user_email) ebsvol.setup_instance_config() return ebsvol
def set_loading_state(self, user_id): sessname = unique_sessname(user_id) sign = signstr(sessname + '000', self.config("sesskey")) self.set_container_cookies({ "sessname": sessname, "hostshell": 0, "hostupload": 0, "hostipnb": 0, "loading": 1, "sign": sign }) self.set_lb_tracker_cookie()
def do_monitor_loading(self, user_id): sessname = unique_sessname(user_id) self.log_debug("Monitoring loading of session [%s] user[%s]...", sessname, user_id) cont = JBoxContainer.get_by_name(sessname) if (cont is None) or (not cont.is_running()): loading_step = int(self.get_cookie("loading", 0)) if loading_step > 30: self.log_error("Could not start instance. Session [%s] for user [%s] didn't load.", sessname, user_id) self.clear_container_cookies() self.rendertpl("index.tpl", cfg=self.config(), state=self.state( error='Could not start your instance! Please try again.', pending_activation=False, user_id=user_id)) return else: loading_step += 1 self.set_cookie("loading", str(loading_step)) self.rendertpl("loading.tpl", user_id=user_id) else: if self.config("gauth"): jbuser = JBoxUserV2(user_id) creds = jbuser.get_gtok() if creds is not None: try: creds_json = json.loads(base64.b64decode(creds)) creds_json = self.renew_creds(creds_json) authtok = creds_json['access_token'] except: self.log_warn("stale stored creds. will renew on next use. user: "******"sesskey")) self.clear_cookie("loading") self.set_container_cookies({ "sessname": sessname, "hostshell": shellport, "hostupload": uplport, "hostipnb": ipnbport, "sign": sign }) self.set_lb_tracker_cookie() self.rendertpl("ipnbsess.tpl", sessname=sessname, cfg=self.config(), creds=creds, authtok=authtok, user_id=user_id)
def restore(self): sessname = unique_sessname(self.user_email) old_sessname = esc_sessname(self.user_email) src = os.path.join(JBoxVol.BACKUP_LOC, sessname + ".tar.gz") k = JBoxVol.pull_from_s3(src) # download from S3 if exists if not os.path.exists(src): if old_sessname is not None: src = os.path.join(JBoxVol.BACKUP_LOC, old_sessname + ".tar.gz") k = JBoxVol.pull_from_s3(src) # download from S3 if exists if not os.path.exists(src): return JBoxVol.log_info("Filtering out restore info from backup " + src + " to " + self.disk_path) src_tar = tarfile.open(src, 'r:gz') try: perms = {} for info in src_tar.getmembers(): if not info.name.startswith('juser/'): continue extract_name = info.name[6:] if info.name.startswith('juser/.'): folder = info.name.split('/')[1] if ( folder == '.julia' ) or JBoxVol._is_path_user_home_essential(extract_name): continue info.name = extract_name if len(info.name) == 0: continue src_tar.extract(info, self.disk_path) extracted_path = os.path.join(self.disk_path, extract_name) if os.path.isdir(extracted_path) and not os.access( extracted_path, os.W_OK): st = os.stat(extracted_path) perms[extracted_path] = st os.chmod(extracted_path, st.st_mode | stat.S_IWRITE) if len(perms) > 0: JBoxVol.log_debug("resetting permissions on %d folders", len(perms)) for extracted_path, perm in perms.iteritems(): os.chmod(extracted_path, perm) JBoxVol.log_info("Restored backup at " + self.disk_path) except IOError, ioe: if ioe.errno == errno.ENOSPC: # continue login on ENOSPC to allow user to delete files JBoxVol.log_exception("No space left to restore backup for %s", sessname) else: raise
def test(): yday = datetime.datetime.now() - datetime.timedelta(hours=24) stats = JBoxAccountingV2.get_stats(dates=(yday, )) TestDBTables.log_debug("stats for yesterday: %s", repr(stats)) stats = JBoxAccountingV2.get_stats() TestDBTables.log_debug("stats for today: %s", repr(stats)) sprops = JBoxSessionProps(unique_sessname('*****@*****.**')) TestDBTables.log_debug( "JBoxSessionProps. user_id: %s, snapshot_id: %s, message: %s", sprops.get_user_id(), sprops.get_snapshot_id(), sprops.get_message()) JBoxDynConfig.set_cluster_leader(TESTCLSTR, 'testinstance') assert JBoxDynConfig.get_cluster_leader(TESTCLSTR) == 'testinstance' JBoxDynConfig.unset_cluster_leader(TESTCLSTR) assert JBoxDynConfig.get_cluster_leader(TESTCLSTR) is None assert JBoxDynConfig.get_allow_registration(TESTCLSTR) JBoxDynConfig.set_allow_registration(TESTCLSTR, False) assert not JBoxDynConfig.get_allow_registration(TESTCLSTR) JBoxDynConfig.set_allow_registration(TESTCLSTR, True) assert JBoxDynConfig.get_allow_registration(TESTCLSTR) assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 60 JBoxDynConfig.set_registration_hourly_rate(TESTCLSTR, 20) assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 20 JBoxDynConfig.set_registration_hourly_rate(TESTCLSTR, 60) assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 60 assert JBoxDynConfig.get_message(TESTCLSTR) is None JBoxDynConfig.set_message(TESTCLSTR, "hello world", datetime.timedelta(minutes=1)) assert JBoxDynConfig.get_message(TESTCLSTR) == "hello world" JBoxDynConfig.set_user_home_image(TESTCLSTR, "juliabox-user-home-templates", "user_home_28Nov2014.tar.gz") assert JBoxDynConfig.get_user_home_image(TESTCLSTR) == ( "juliabox-user-home-templates", "user_home_28Nov2014.tar.gz") num_pending_activations = JBoxUserV2.count_pending_activations() TestDBTables.log_debug("pending activations: %d", num_pending_activations) resultset = JBoxInvite.table().scan() result_arr = [obj for obj in resultset] TestDBTables.log_debug("got array: %r", result_arr) count_created = JBoxUserV2.count_created(48) TestDBTables.log_debug("accounts created in last 1 hour: %d", count_created)
def try_launch_container(cls, user_id): sessname = unique_sessname(user_id) cont = JBoxContainer.get_by_name(sessname) cls.log_debug("have existing container for %s: %r", sessname, None != cont) if cont is not None: cls.log_debug("container running: %r", cont.is_running()) if ((cont is None) or (not cont.is_running())) and (not CloudHost.should_accept_session()): if cont is not None: cont.async_backup_and_cleanup() return False JBoxContainer.async_launch_by_name(sessname, user_id, True) return True
def do_monitor_loading_ajax(self, user_id): sessname = unique_sessname(user_id) self.log_debug("AJAX monitoring loading of session [%s] user[%s]...", sessname, user_id) cont = JBoxContainer.get_by_name(sessname) if (cont is None) or (not cont.is_running()): loading_step = int(self.get_cookie("loading", 0)) if loading_step > 30: self.log_error("Could not start instance. Session [%s] for user [%s] didn't load.", sessname, user_id) self.write({'code': -1}) return loading_step += 1 self.set_cookie("loading", str(loading_step)) self.write({'code': 0}) else: self.write({'code': 1})
def do_monitor_loading_ajax(self, user_id): sessname = unique_sessname(user_id) self.log_debug("AJAX monitoring loading of session [%s] user[%s]...", sessname, user_id) cont = JBoxContainer.get_by_name(sessname) if (cont is None) or (not cont.is_running()): loading_step = int(self.get_cookie("loading", 0)) if loading_step > 30: self.write({'code': -1}) return loading_step += 1 self.set_cookie("loading", str(loading_step)) self.write({'code': 0}) else: self.write({'code': 1})
def test(): yday = datetime.datetime.now() - datetime.timedelta(hours=24) stats = JBoxAccountingV2.get_stats(dates=(yday,)) TestDBTables.log_debug("stats for yesterday: %s", repr(stats)) stats = JBoxAccountingV2.get_stats() TestDBTables.log_debug("stats for today: %s", repr(stats)) sprops = JBoxSessionProps(unique_sessname('*****@*****.**')) TestDBTables.log_debug("JBoxSessionProps. user_id: %s, snapshot_id: %s, message: %s", sprops.get_user_id(), sprops.get_snapshot_id(), sprops.get_message()) JBoxDynConfig.set_cluster_leader(TESTCLSTR, 'testinstance') assert JBoxDynConfig.get_cluster_leader(TESTCLSTR) == 'testinstance' JBoxDynConfig.unset_cluster_leader(TESTCLSTR) assert JBoxDynConfig.get_cluster_leader(TESTCLSTR) is None assert JBoxDynConfig.get_allow_registration(TESTCLSTR) JBoxDynConfig.set_allow_registration(TESTCLSTR, False) assert not JBoxDynConfig.get_allow_registration(TESTCLSTR) JBoxDynConfig.set_allow_registration(TESTCLSTR, True) assert JBoxDynConfig.get_allow_registration(TESTCLSTR) assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 60 JBoxDynConfig.set_registration_hourly_rate(TESTCLSTR, 20) assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 20 JBoxDynConfig.set_registration_hourly_rate(TESTCLSTR, 60) assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 60 assert JBoxDynConfig.get_message(TESTCLSTR) is None JBoxDynConfig.set_message(TESTCLSTR, "hello world", datetime.timedelta(minutes=1)) assert JBoxDynConfig.get_message(TESTCLSTR) == "hello world" JBoxDynConfig.set_user_home_image(TESTCLSTR, "juliabox-user-home-templates", "user_home_28Nov2014.tar.gz") assert JBoxDynConfig.get_user_home_image(TESTCLSTR) == ("juliabox-user-home-templates", "user_home_28Nov2014.tar.gz") num_pending_activations = JBoxUserV2.count_pending_activations() TestDBTables.log_debug("pending activations: %d", num_pending_activations) resultset = JBoxInvite.table().scan() result_arr = [obj for obj in resultset] TestDBTables.log_debug("got array: %r", result_arr) count_created = JBoxUserV2.count_created(48) TestDBTables.log_debug("accounts created in last 1 hour: %d", count_created)
def restore(self): sessname = unique_sessname(self.user_email) old_sessname = esc_sessname(self.user_email) src = os.path.join(JBoxVol.BACKUP_LOC, sessname + ".tar.gz") k = JBoxVol.pull_from_s3(src) # download from S3 if exists if not os.path.exists(src): if old_sessname is not None: src = os.path.join(JBoxVol.BACKUP_LOC, old_sessname + ".tar.gz") k = JBoxVol.pull_from_s3(src) # download from S3 if exists if not os.path.exists(src): return JBoxVol.log_info("Filtering out restore info from backup " + src + " to " + self.disk_path) src_tar = tarfile.open(src, 'r:gz') try: perms = {} for info in src_tar.getmembers(): if not info.name.startswith('juser/'): continue extract_name = info.name[6:] if info.name.startswith('juser/.'): folder = info.name.split('/')[1] if (folder == '.julia') or JBoxVol._is_path_user_home_essential(extract_name): continue info.name = extract_name if len(info.name) == 0: continue src_tar.extract(info, self.disk_path) extracted_path = os.path.join(self.disk_path, extract_name) if os.path.isdir(extracted_path) and not os.access(extracted_path, os.W_OK): st = os.stat(extracted_path) perms[extracted_path] = st os.chmod(extracted_path, st.st_mode | stat.S_IWRITE) if len(perms) > 0: JBoxVol.log_debug("resetting permissions on %d folders", len(perms)) for extracted_path, perm in perms.iteritems(): os.chmod(extracted_path, perm) JBoxVol.log_info("Restored backup at " + self.disk_path) except IOError, ioe: if ioe.errno == errno.ENOSPC: # continue login on ENOSPC to allow user to delete files JBoxVol.log_exception("No space left to restore backup for %s", sessname) else: raise
def try_launch_container(cls, user_id): sessname = unique_sessname(user_id) cont = JBoxContainer.get_by_name(sessname) cls.log_debug("have existing container for %s: %r", sessname, None != cont) if cont is not None: cls.log_debug("container running: %r", cont.is_running()) is_leader = is_proposed_cluster_leader() if ((cont is None) or (not cont.is_running())) and ( not CloudHost.should_accept_session(is_leader)): if cont is not None: cont.async_backup_and_cleanup() return False JBoxContainer.async_launch_by_name(sessname, user_id, True) return True
def get_disk_for_user(user_email): JBoxEBSVol.log_debug("creating EBS volume for %s", user_email) if not JBoxEBSVol.HAS_EBS: raise Exception("EBS disks not enabled") disk_id = JBoxEBSVol._reserve_disk_id() if disk_id is None: raise Exception("No free disk available") sess_id = unique_sessname(user_email) sess_props = JBoxSessionProps(sess_id, create=True, user_id=user_email) if sess_props.is_new: sess_props.save() snap_id = sess_props.get_snapshot_id() if snap_id is None: snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id, user_email) _dev_path, mnt_path = CloudHost.create_new_volume(snap_id, disk_id, JBoxEBSVol.FS_LOC, tag=user_email) ebsvol = JBoxEBSVol(mnt_path, user_email=user_email) if snap_id == JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT: JBoxEBSVol.log_debug("creating home folder on blank volume for %s", user_email) ebsvol.restore_user_home(True) ebsvol.restore() else: JBoxEBSVol.log_debug( "updating home folder on existing volume for %s", user_email) ebsvol.restore_user_home(False) # snap_age_days = CloudHost.get_snapshot_age(snap_id).total_seconds()/(60*60*24) # if snap_age_days > 7: # ebsvol.restore_user_home() JBoxEBSVol.log_debug( "setting up instance configuration on disk for %s", user_email) ebsvol.setup_instance_config() return ebsvol
def restore(self): sessname = unique_sessname(self.user_email) old_sessname = esc_sessname(self.user_email) src = os.path.join(JBoxVol.BACKUP_LOC, sessname + ".tar.gz") k = JBoxVol.pull_from_s3(src) # download from S3 if exists if not os.path.exists(src): if old_sessname is not None: src = os.path.join(JBoxVol.BACKUP_LOC, old_sessname + ".tar.gz") k = JBoxVol.pull_from_s3(src) # download from S3 if exists if not os.path.exists(src): return JBoxVol.log_info("Filtering out restore info from backup " + src + " to " + self.disk_path) src_tar = tarfile.open(src, 'r:gz') try: for info in src_tar.getmembers(): if not info.name.startswith('juser/'): continue extract_name = info.name[6:] if info.name.startswith('juser/.'): folder = info.name.split('/')[1] if ( folder == '.julia' ) or JBoxVol._is_path_user_home_essential(extract_name): continue info.name = extract_name if len(info.name) == 0: continue src_tar.extract(info, self.disk_path) JBoxVol.log_info("Restored backup at " + self.disk_path) except IOError, ioe: if ioe.errno == errno.ENOSPC: # continue login on ENOSPC to allow user to delete files JBoxVol.log_exception("No space left to restore backup for %s", sessname) else: raise
def try_launch_container(cls, user_id, max_hop=False): sessname = unique_sessname(user_id) cont = JBoxContainer.get_by_name(sessname) cls.log_debug("have existing container for %s: %r", sessname, None != cont) if cont is not None: cls.log_debug("container running: %r", cont.is_running()) if max_hop: self_load = CloudHost.get_instance_stats(CloudHost.instance_id(), 'Load') if self_load < 100: JBoxContainer.async_launch_by_name(sessname, user_id, True) return True is_leader = is_proposed_cluster_leader() if ((cont is None) or (not cont.is_running())) and (not CloudHost.should_accept_session(is_leader)): if cont is not None: cont.async_backup_and_cleanup() return False JBoxContainer.async_launch_by_name(sessname, user_id, True) return True
def test(): sprops = JBoxSessionProps(TESTCLSTR, unique_sessname('*****@*****.**')) TestDBTables.log_debug("JBoxSessionProps. user_id: %s, snapshot_id: %s, message: %s", sprops.get_user_id(), sprops.get_snapshot_id(), sprops.get_message()) JBoxDynConfig.set_cluster_leader(TESTCLSTR, 'testinstance') assert JBoxDynConfig.get_cluster_leader(TESTCLSTR) == 'testinstance' JBoxDynConfig.unset_cluster_leader(TESTCLSTR) assert JBoxDynConfig.get_cluster_leader(TESTCLSTR) is None assert JBoxDynConfig.get_allow_registration(TESTCLSTR) JBoxDynConfig.set_allow_registration(TESTCLSTR, False) assert not JBoxDynConfig.get_allow_registration(TESTCLSTR) JBoxDynConfig.set_allow_registration(TESTCLSTR, True) assert JBoxDynConfig.get_allow_registration(TESTCLSTR) assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 60 JBoxDynConfig.set_registration_hourly_rate(TESTCLSTR, 20) assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 20 JBoxDynConfig.set_registration_hourly_rate(TESTCLSTR, 60) assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 60 assert JBoxDynConfig.get_message(TESTCLSTR) is None JBoxDynConfig.set_message(TESTCLSTR, "hello world", datetime.timedelta(minutes=1)) assert JBoxDynConfig.get_message(TESTCLSTR) == "hello world" JBoxDynConfig.set_user_home_image(TESTCLSTR, "juliabox-user-home-templates", "user_home_28Nov2014.tar.gz") assert JBoxDynConfig.get_user_home_image(TESTCLSTR) == ("juliabox-user-home-templates", "user_home_28Nov2014.tar.gz") num_pending_activations = JBoxUserV2.count_pending_activations() TestDBTables.log_debug("pending activations: %d", num_pending_activations) count_created = JBoxUserV2.count_created(48) TestDBTables.log_debug("accounts created in last 1 hour: %d", count_created)
def update_disk_states(): detached_disks = JBoxDiskState.get_detached_disks() time_now = datetime.datetime.now(pytz.utc) for disk_key in detached_disks: disk_info = JBoxDiskState(disk_key=disk_key) user_id = disk_info.get_user_id() sess_props = JBoxSessionProps(unique_sessname(user_id)) incomplete_snapshots = [] modified = False for snap_id in disk_info.get_snapshot_ids(): if not CloudHost.is_snapshot_complete(snap_id): incomplete_snapshots.append(snap_id) continue JBoxd.log_debug("updating latest snapshot of user %s to %s", user_id, snap_id) old_snap_id = sess_props.get_snapshot_id() sess_props.set_snapshot_id(snap_id) modified = True if old_snap_id is not None: CloudHost.delete_snapshot(old_snap_id) if modified: sess_props.save() disk_info.set_snapshot_ids(incomplete_snapshots) disk_info.save() if len(incomplete_snapshots) == 0: if (time_now - disk_info.get_detach_time() ).total_seconds() > 24 * 60 * 60: vol_id = disk_info.get_volume_id() JBoxd.log_debug( "volume %s for user %s unused for too long", vol_id, user_id) disk_info.delete() CloudHost.detach_volume(vol_id, delete=True) else: JBoxd.log_debug("ongoing snapshots of user %s: %r", user_id, incomplete_snapshots)
def delete(user_email): sessname = unique_sessname(user_email) CloudHost.del_file_from_s3(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz")
def rename_and_delete(user_email): sessname = unique_sessname(user_email) renamed_sessname = sessname + '_old' CloudHost.move_file_in_s3(sessname + ".tar.gz", renamed_sessname + ".tar.gz", JBoxVol.BACKUP_BUCKET)
def get(self): jbox_cookie = AuthHandler.get_session_cookie(self) if self.config("invite_only"): if self.get_argument("invite", False): self.set_cookie("is_invite", "yes") self.redirect('/hostlaunchipnb/') return if None == jbox_cookie: which_msg = int( self.get_argument("_msg", JBoxUserV2.ACTIVATION_NONE)) if self.get_argument("_msg", "") != "": self.clear_cookie("is_invite") if which_msg == JBoxUserV2.ACTIVATION_GRANTED: msg = "Your account has already been approved" elif which_msg == JBoxUserV2.ACTIVATION_REQUESTED: msg = "You have already registered for an invite" else: msg = "Thank you for your interest! We will get back to you with an invitation soon." state = self.state(success=msg) else: state = self.state() self.rendertpl("index.tpl", cfg=self.config(), state=state) else: user_id = jbox_cookie['u'] sessname = unique_sessname(user_id) if self.config("gauth"): try: jbuser = JBoxUserV2(user_id) except: # stale cookie. we don't have the user in our database anymore self.log_info( "stale cookie. we don't have the user in our database anymore. user: "******"invite_only"): code, status = jbuser.get_activation_state() if status != JBoxUserV2.ACTIVATION_GRANTED: invite_code = self.get_argument("invite_code", False) if invite_code is not False: try: invite = JBoxInvite(invite_code) except: invite = None if (invite is not None ) and invite.is_invited(user_id): jbuser.set_activation_state( invite_code, JBoxUserV2.ACTIVATION_GRANTED) invite.increment_count() invite.save() jbuser.save() self.redirect('/hostlaunchipnb/') return else: error_msg = 'You entered an invalid invitation code. Try again or request a new invitation.' else: error_msg = 'Enter an invitation code to proceed.' self.rendertpl("index.tpl", cfg=self.config(), state=self.state(error=error_msg, ask_invite_code=True, user_id=user_id)) return creds = jbuser.get_gtok() if creds is not None: try: creds_json = json.loads(base64.b64decode(creds)) creds_json = self.renew_creds(creds_json) authtok = creds_json['access_token'] except: self.log_info( "stale stored creds. will renew on next use. user: " + user_id) creds = None authtok = None else: authtok = None else: creds = None authtok = None self.chk_and_launch_docker(sessname, creds, authtok, user_id)
def get(self): jbox_cookie = AuthHandler.get_session_cookie(self) if self.config("invite_only"): if self.get_argument("invite", False): self.set_cookie("is_invite", "yes") self.redirect('/hostlaunchipnb/') return if None == jbox_cookie: which_msg = int(self.get_argument("_msg", JBoxUserV2.ACTIVATION_NONE)) if self.get_argument("_msg", "") != "": self.clear_cookie("is_invite") if which_msg == JBoxUserV2.ACTIVATION_GRANTED: msg = "Your account has already been approved" elif which_msg == JBoxUserV2.ACTIVATION_REQUESTED: msg = "You have already registered for an invite" else: msg = "Thank you for your interest! We will get back to you with an invitation soon." state = self.state(success=msg) else: state = self.state() self.rendertpl("index.tpl", cfg=self.config(), state=state) else: user_id = jbox_cookie['u'] sessname = unique_sessname(user_id) if self.config("gauth"): try: jbuser = JBoxUserV2(user_id) except: # stale cookie. we don't have the user in our database anymore self.log_info("stale cookie. we don't have the user in our database anymore. user: "******"invite_only"): code, status = jbuser.get_activation_state() if status != JBoxUserV2.ACTIVATION_GRANTED: invite_code = self.get_argument("invite_code", False) if invite_code is not False: try: invite = JBoxInvite(invite_code) except: invite = None if (invite is not None) and invite.is_invited(user_id): jbuser.set_activation_state(invite_code, JBoxUserV2.ACTIVATION_GRANTED) invite.increment_count() invite.save() jbuser.save() self.redirect('/hostlaunchipnb/') return else: error_msg = 'You entered an invalid invitation code. Try again or request a new invitation.' else: error_msg = 'Enter an invitation code to proceed.' self.rendertpl("index.tpl", cfg=self.config(), state=self.state( error=error_msg, ask_invite_code=True, user_id=user_id)) return creds = jbuser.get_gtok() if creds is not None: try: creds_json = json.loads(base64.b64decode(creds)) creds_json = self.renew_creds(creds_json) authtok = creds_json['access_token'] except: self.log_info("stale stored creds. will renew on next use. user: " + user_id) creds = None authtok = None else: authtok = None else: creds = None authtok = None self.chk_and_launch_docker(sessname, creds, authtok, user_id)
def get_disk_for_user(user_email): JBoxEBSVol.log_debug("creating EBS volume for %s", user_email) if not JBoxEBSVol.HAS_EBS: raise Exception("EBS disks not enabled") disk_id = JBoxEBSVol._reserve_disk_id() if disk_id is None: raise Exception("No free disk available") try: existing_disk = JBoxDiskState(cluster_id=CloudHost.INSTALL_ID, region_id=CloudHost.REGION, user_id=user_email) except: existing_disk = None if existing_disk is None: sess_id = unique_sessname(user_email) sess_props = JBoxSessionProps(sess_id, create=True, user_id=user_email) if sess_props.is_new: sess_props.save() snap_id = sess_props.get_snapshot_id() if snap_id is None: snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id, user_email) _dev_path, mnt_path, vol_id = CloudHost.create_new_volume( snap_id, disk_id, JBoxEBSVol.FS_LOC, tag=user_email, disk_sz_gb=JBoxEBSVol.DISK_LIMIT) existing_disk = JBoxDiskState(cluster_id=CloudHost.INSTALL_ID, region_id=CloudHost.REGION, user_id=user_email, volume_id=vol_id, attach_time=None, create=True) else: _dev_path, mnt_path = CloudHost.attach_volume( existing_disk.get_volume_id(), disk_id, JBoxEBSVol.FS_LOC) existing_disk.set_attach_time() snap_id = None existing_disk.set_state(JBoxDiskState.STATE_ATTACHED) existing_disk.save() ebsvol = JBoxEBSVol(mnt_path, user_email=user_email) if snap_id == JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT: JBoxEBSVol.log_debug("creating home folder on blank volume for %s", user_email) ebsvol.restore_user_home(True) ebsvol.restore() else: JBoxEBSVol.log_debug( "updating home folder on existing volume for %s", user_email) ebsvol.restore_user_home(False) # snap_age_days = CloudHost.get_snapshot_age(snap_id).total_seconds()/(60*60*24) # if snap_age_days > 7: # ebsvol.restore_user_home() JBoxEBSVol.log_debug( "setting up instance configuration on disk for %s", user_email) ebsvol.setup_instance_config() return ebsvol
def do_monitor_loading(self, user_id): sessname = unique_sessname(user_id) self.log_debug("Monitoring loading of session [%s] user[%s]...", sessname, user_id) cont = JBoxContainer.get_by_name(sessname) if (cont is None) or (not cont.is_running()): loading_step = int(self.get_cookie("loading", 0)) if loading_step > 30: self.log_error( "Could not start instance. Session [%s] for user [%s] didn't load.", sessname, user_id) self.clear_container_cookies() self.rendertpl( "index.tpl", cfg=self.config(), state=self.state( error= 'Could not start your instance! Please try again.', pending_activation=False, user_id=user_id)) return else: loading_step += 1 self.set_cookie("loading", str(loading_step)) self.rendertpl("loading.tpl", user_id=user_id) else: if self.config("gauth"): jbuser = JBoxUserV2(user_id) creds = jbuser.get_gtok() if creds is not None: try: creds_json = json.loads(base64.b64decode(creds)) creds_json = self.renew_creds(creds_json) authtok = creds_json['access_token'] except: self.log_warn( "stale stored creds. will renew on next use. user: "******"sesskey")) self.clear_cookie("loading") self.set_container_cookies({ "sessname": sessname, "hostshell": shellport, "hostupload": uplport, "hostipnb": ipnbport, "sign": sign }) self.set_lb_tracker_cookie() self.rendertpl("ipnbsess.tpl", sessname=sessname, cfg=self.config(), creds=creds, authtok=authtok, user_id=user_id)
class JBoxEBSVol(JBoxVol): DEVICES = [] MAX_DISKS = 0 FS_LOC = None DISK_LIMIT = None DISK_USE_STATUS = {} DISK_RESERVE_TIME = {} DISK_TEMPLATE_SNAPSHOT = None HAS_EBS = False LOCK = None @staticmethod def configure(disk_limit, fs_loc, max_disks, disk_template_snap_id): JBoxEBSVol.HAS_EBS = True JBoxEBSVol.FS_LOC = fs_loc JBoxEBSVol.DISK_LIMIT = disk_limit JBoxEBSVol.MAX_DISKS = max_disks JBoxEBSVol.DEVICES = JBoxEBSVol._get_configured_devices(JBoxEBSVol.FS_LOC) JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT = disk_template_snap_id if len(JBoxEBSVol.DEVICES) < max_disks: raise Exception("Not enough EBS mount points configured") JBoxEBSVol.LOCK = threading.Lock() JBoxEBSVol.refresh_disk_use_status() @classmethod def get_disk_allocated_size(cls): return JBoxEBSVol.DISK_LIMIT * 1000000000 @staticmethod def _id_from_device(dev_path): return dev_path.split('/')[-1] @staticmethod def _get_configured_devices(fs_loc): devices = [] with open('/etc/fstab', 'r') as fstab: for line in fstab: line = line.strip() if (len(line) == 0) or line.startswith('#'): continue comps = line.split() if (len(comps) == 6) and comps[1].startswith(fs_loc): device = comps[0] devices.append(JBoxEBSVol._id_from_device(device)) return devices @staticmethod def _get_disk_ids_used(cid): used = [] props = JBoxEBSVol.dckr().inspect_container(cid) try: vols = props['Volumes'] for _cpath, hpath in vols.iteritems(): if hpath.startswith(JBoxEBSVol.FS_LOC): used.append(hpath.split('/')[-1]) except: JBoxEBSVol.log_error("error finding disk ids used in " + cid) return [] return used @staticmethod def refresh_disk_use_status(container_id_list=None): JBoxEBSVol.LOCK.acquire() try: nfree = 0 for idx in range(0, JBoxEBSVol.MAX_DISKS): dev = JBoxEBSVol.DEVICES[idx] if JBoxEBSVol._is_reserved(dev): JBoxEBSVol.DISK_USE_STATUS[dev] = True else: JBoxEBSVol.DISK_USE_STATUS[dev] = False nfree += 1 if container_id_list is None: container_id_list = [cdesc['Id'] for cdesc in JBoxEBSVol.dckr().containers(all=True)] for cid in container_id_list: disk_ids = JBoxEBSVol._get_disk_ids_used(cid) for disk_id in disk_ids: JBoxEBSVol._mark_disk_used(disk_id) nfree -= 1 JBoxEBSVol.log_info("Disk free: " + str(nfree) + "/" + str(JBoxEBSVol.MAX_DISKS)) finally: JBoxEBSVol.LOCK.release() @staticmethod def _get_unused_disk_id(): for idx in range(0, JBoxEBSVol.MAX_DISKS): dev = JBoxEBSVol.DEVICES[idx] if not JBoxEBSVol.DISK_USE_STATUS[dev]: return dev return None @staticmethod def _is_reserved(idx): if (idx in JBoxEBSVol.DISK_RESERVE_TIME) and (JBoxEBSVol.DISK_RESERVE_TIME[idx] < time.time()): del JBoxEBSVol.DISK_RESERVE_TIME[idx] return idx in JBoxEBSVol.DISK_RESERVE_TIME @staticmethod def _mark_disk_used(idx, used=True, for_secs=0): JBoxEBSVol.DISK_USE_STATUS[idx] = used if used and (for_secs > 0): JBoxEBSVol.DISK_RESERVE_TIME[idx] = time.time() + for_secs else: if idx in JBoxEBSVol.DISK_RESERVE_TIME: del JBoxEBSVol.DISK_RESERVE_TIME[idx] @staticmethod def _reserve_disk_id(): JBoxEBSVol.LOCK.acquire() try: disk_id = JBoxEBSVol._get_unused_disk_id() JBoxEBSVol._mark_disk_used(disk_id, for_secs=120) return disk_id finally: JBoxEBSVol.LOCK.release() @staticmethod def disk_ids_used_pct(): if not JBoxEBSVol.HAS_EBS: return 0 pct = (sum(JBoxEBSVol.DISK_USE_STATUS.values()) * 100) / len(JBoxEBSVol.DISK_USE_STATUS) return min(100, max(0, pct)) @staticmethod def get_disk_for_user(user_email): JBoxEBSVol.log_debug("creating EBS volume for %s", user_email) if not JBoxEBSVol.HAS_EBS: raise Exception("EBS disks not enabled") disk_id = JBoxEBSVol._reserve_disk_id() if disk_id is None: raise Exception("No free disk available") try: existing_disk = JBoxDiskState(cluster_id=CloudHost.INSTALL_ID, region_id=CloudHost.REGION, user_id=user_email) except Exception, ex: JBoxEBSVol.log_debug("No existing disk for %s. Exception %r", user_email, ex) existing_disk = None if existing_disk is None: sess_id = unique_sessname(user_email) sess_props = JBoxSessionProps(sess_id, create=True, user_id=user_email) if sess_props.is_new: sess_props.save() snap_id = sess_props.get_snapshot_id() if snap_id is None: snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id, user_email) _dev_path, mnt_path, vol_id = CloudHost.create_new_volume(snap_id, disk_id, JBoxEBSVol.FS_LOC, tag=user_email, disk_sz_gb=JBoxEBSVol.DISK_LIMIT) existing_disk = JBoxDiskState(cluster_id=CloudHost.INSTALL_ID, region_id=CloudHost.REGION, user_id=user_email, volume_id=vol_id, attach_time=None, create=True) else: _dev_path, mnt_path = CloudHost.attach_volume(existing_disk.get_volume_id(), disk_id, JBoxEBSVol.FS_LOC) existing_disk.set_attach_time() snap_id = None existing_disk.set_state(JBoxDiskState.STATE_ATTACHED) existing_disk.save() ebsvol = JBoxEBSVol(mnt_path, user_email=user_email) if snap_id == JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT: JBoxEBSVol.log_debug("creating home folder on blank volume for %s", user_email) ebsvol.restore_user_home(True) ebsvol.restore() else: JBoxEBSVol.log_debug("updating home folder on existing volume for %s", user_email) ebsvol.restore_user_home(False) # snap_age_days = CloudHost.get_snapshot_age(snap_id).total_seconds()/(60*60*24) # if snap_age_days > 7: # ebsvol.restore_user_home() JBoxEBSVol.log_debug("setting up instance configuration on disk for %s", user_email) ebsvol.setup_instance_config() return ebsvol