def find_logged_in_instance(user_id):
sessname = unique_sessname(user_id)
try:
sess_props = JBoxSessionProps(Compute.get_install_id(), sessname)
return sess_props.get_instance_id()
except JBoxDBItemNotFound:
return None
def do_cluster_housekeeping():
JBoxEBSHousekeep.log_debug("starting cluster housekeeping")
detached_disks = JBoxDiskState.get_detached_disks()
time_now = datetime.datetime.now(pytz.utc)
for disk_key in detached_disks:
disk_info = JBoxDiskState(disk_key=disk_key)
user_id = disk_info.get_user_id()
sess_props = JBoxSessionProps(Compute.get_install_id(), unique_sessname(user_id))
incomplete_snapshots = []
modified = False
for snap_id in disk_info.get_snapshot_ids():
if not EBSVol.is_snapshot_complete(snap_id):
incomplete_snapshots.append(snap_id)
continue
JBoxEBSHousekeep.log_debug("updating latest snapshot of user %s to %s", user_id, snap_id)
old_snap_id = sess_props.get_snapshot_id()
sess_props.set_snapshot_id(snap_id)
modified = True
if old_snap_id is not None:
EBSVol.delete_snapshot(old_snap_id)
if modified:
sess_props.save()
disk_info.set_snapshot_ids(incomplete_snapshots)
disk_info.save()
if len(incomplete_snapshots) == 0:
if (time_now - disk_info.get_detach_time()).total_seconds() > 24*60*60:
vol_id = disk_info.get_volume_id()
JBoxEBSHousekeep.log_debug("volume %s for user %s unused for too long", vol_id, user_id)
disk_info.delete()
EBSVol.detach_volume(vol_id, delete=True)
else:
JBoxEBSHousekeep.log_debug("ongoing snapshots of user %s: %r", user_id, incomplete_snapshots)
JBoxEBSHousekeep.log_debug("finished cluster housekeeping")
def pull_backup(user_email):
sessname = unique_sessname(user_email)
JBoxDisk.log_info("pulling %s.tar.gz from %s", sessname,
JBoxVol.BACKUP_BUCKET)
JBoxDisk.PLUGIN.pull(JBoxVol.BACKUP_BUCKET,
sessname + ".tar.gz",
metadata_only=False)
def push_backup(user_email, disk_path):
sessname = unique_sessname(user_email)
S3Disk.log_info("pushing %s.tar.gz from %s to %s", sessname, disk_path, JBoxVol.BACKUP_BUCKET)
bkup_file = os.path.join('/tmp', sessname + ".tar.gz")
bkup_tar = tarfile.open(bkup_file, 'w:gz')
def set_perms(tinfo):
tinfo.uid = 1000
tinfo.gid = 1000
tinfo.uname = 'ubuntu'
tinfo.gname = 'ubuntu'
return tinfo
for f in os.listdir(disk_path):
if f.startswith('.') and (f in ['.julia', '.juliabox']):
continue
full_path = os.path.join(disk_path, f)
bkup_tar.add(full_path, os.path.join('juser', f), filter=set_perms)
bkup_tar.close()
os.chmod(bkup_file, 0666)
# Upload to S3 if so configured. Delete from local if successful.
bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \
datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET)
if JBoxVol.BACKUP_BUCKET is not None:
if JBoxS3.push(JBoxVol.BACKUP_BUCKET, bkup_file,
metadata={'backup_time': bkup_file_mtime.isoformat()}) is not None:
os.remove(bkup_file)
S3Disk.log_info("Moved backup to S3 " + sessname)
def do_cluster_housekeeping():
JBoxEBSHousekeep.log_debug("starting cluster housekeeping")
detached_disks = JBoxDiskState.get_detached_disks()
time_now = datetime.datetime.now(pytz.utc)
for disk_key in detached_disks:
disk_info = JBoxDiskState(disk_key=disk_key)
user_id = disk_info.get_user_id()
sess_props = JBoxSessionProps(unique_sessname(user_id))
incomplete_snapshots = []
modified = False
for snap_id in disk_info.get_snapshot_ids():
if not EBSVol.is_snapshot_complete(snap_id):
incomplete_snapshots.append(snap_id)
continue
JBoxEBSHousekeep.log_debug("updating latest snapshot of user %s to %s", user_id, snap_id)
old_snap_id = sess_props.get_snapshot_id()
sess_props.set_snapshot_id(snap_id)
modified = True
if old_snap_id is not None:
EBSVol.delete_snapshot(old_snap_id)
if modified:
sess_props.save()
disk_info.set_snapshot_ids(incomplete_snapshots)
disk_info.save()
if len(incomplete_snapshots) == 0:
if (time_now - disk_info.get_detach_time()).total_seconds() > 24 * 60 * 60:
vol_id = disk_info.get_volume_id()
JBoxEBSHousekeep.log_debug("volume %s for user %s unused for too long", vol_id, user_id)
disk_info.delete()
EBSVol.detach_volume(vol_id, delete=True)
else:
JBoxEBSHousekeep.log_debug("ongoing snapshots of user %s: %r", user_id, incomplete_snapshots)
JBoxEBSHousekeep.log_debug("finished cluster housekeeping")
def __init__(self, disk_path, user_email=None, user_name=None, sessname=None, old_sessname=None):
self.disk_path = disk_path
self.user_email = user_email
if user_name is not None:
self.user_name = user_name
elif user_email is not None:
self.user_name = get_user_name(user_email)
else:
self.user_name = None
if sessname is not None:
self.sessname = sessname
elif user_email is not None:
self.sessname = unique_sessname(user_email)
else:
self.sessname = None
if old_sessname is not None:
self.old_sessname = old_sessname
elif user_email is not None:
self.old_sessname = esc_sessname(user_email)
else:
self.old_sessname = None
self._dbg_str = str(self.sessname) + "(" + self.disk_path + ")"
def try_launch_container(cls, user_id, max_hop=False):
sessname = unique_sessname(user_id)
cont = SessContainer.get_by_name(sessname)
cls.log_debug("have existing container for %s: %r", sessname,
None != cont)
if cont is not None:
cls.log_debug("container running: %r", cont.is_running())
if max_hop:
self_load = Compute.get_instance_stats(Compute.get_instance_id(),
'Load')
if self_load < 100:
SessContainer.invalidate_container(sessname)
JBoxAsyncJob.async_launch_by_name(sessname, user_id, True)
return True
is_leader = is_proposed_cluster_leader()
if ((cont is None) or (not cont.is_running())) and (
not Compute.should_accept_session(is_leader)):
if cont is not None:
SessContainer.invalidate_container(cont.get_name())
JBoxAsyncJob.async_backup_and_cleanup(cont.dockid)
return False
SessContainer.invalidate_container(sessname)
JBoxAsyncJob.async_launch_by_name(sessname, user_id, True)
return True
def __init__(self,
disk_path,
user_email=None,
user_name=None,
sessname=None,
old_sessname=None):
self.disk_path = disk_path
self.user_email = user_email
if user_name is not None:
self.user_name = user_name
elif user_email is not None:
self.user_name = get_user_name(user_email)
else:
self.user_name = None
if sessname is not None:
self.sessname = sessname
elif user_email is not None:
self.sessname = unique_sessname(user_email)
else:
self.sessname = None
if old_sessname is not None:
self.old_sessname = old_sessname
elif user_email is not None:
self.old_sessname = esc_sessname(user_email)
else:
self.old_sessname = None
self._dbg_str = str(self.sessname) + "(" + self.disk_path + ")"
def push_backup(user_email, disk_path):
sessname = unique_sessname(user_email)
JBoxDisk.log_info("pushing %s.tar.gz from %s to %s", sessname, disk_path, JBoxVol.BACKUP_BUCKET)
bkup_file = os.path.join('/tmp', sessname + ".tar.gz")
bkup_tar = tarfile.open(bkup_file, 'w:gz')
def set_perms(tinfo):
tinfo.uid = 1000
tinfo.gid = 1000
tinfo.uname = 'ubuntu'
tinfo.gname = 'ubuntu'
return tinfo
for f in os.listdir(disk_path):
if f.startswith('.') and (f in ['.julia', '.juliabox']):
continue
full_path = os.path.join(disk_path, f)
bkup_tar.add(full_path, os.path.join('juser', f), filter=set_perms)
bkup_tar.close()
os.chmod(bkup_file, 0666)
# Upload to cloud storage if so configured. Delete from local if successful.
bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \
datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET)
if JBoxVol.BACKUP_BUCKET is not None:
if JBoxDisk.PLUGIN.push(JBoxVol.BACKUP_BUCKET, bkup_file,
metadata={'backup_time': bkup_file_mtime.isoformat()}) is not None:
os.remove(bkup_file)
JBoxDisk.log_info("Moved backup to cloud " + sessname)
def restore(self):
sessname = unique_sessname(self.user_email)
old_sessname = esc_sessname(self.user_email)
src = os.path.join(JBoxVol.BACKUP_LOC, sessname + ".tar.gz")
pull_from_bucketstore = JBoxVol.pull_from_bucketstore
mig_hndl = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_MIGRATE)
if mig_hndl and mig_hndl.should_migrate(self.user_email):
pull_from_bucketstore = mig_hndl.pull_from_bucketstore
k = pull_from_bucketstore(src) # download from S3 if exists
if not os.path.exists(src):
if old_sessname is not None:
src = os.path.join(JBoxVol.BACKUP_LOC, old_sessname + ".tar.gz")
k = pull_from_bucketstore(src) # download from S3 if exists
if not os.path.exists(src):
return
JBoxVol.log_info("Filtering out restore info from backup " + src + " to " + self.disk_path)
src_tar = tarfile.open(src, 'r:gz')
try:
perms = {}
for info in src_tar.getmembers():
if not info.name.startswith('juser/'):
continue
extract_name = info.name[6:]
if (info.type == tarfile.LNKTYPE or info.type == tarfile.SYMTYPE) and \
info.linkname.startswith('juser/'):
info.linkname = info.linkname[6:]
if info.name.startswith('juser/.'):
if JBoxVol._is_path_user_home_essential(extract_name):
continue
info.name = extract_name
if len(info.name) == 0:
continue
src_tar.extract(info, self.disk_path)
extracted_path = os.path.join(self.disk_path, extract_name)
if os.path.isdir(extracted_path) and not os.access(extracted_path, os.W_OK):
st = os.stat(extracted_path)
perms[extracted_path] = st
os.chmod(extracted_path, st.st_mode | stat.S_IWRITE)
if len(perms) > 0:
JBoxVol.log_debug("resetting permissions on %d folders", len(perms))
for extracted_path, perm in perms.iteritems():
os.chmod(extracted_path, perm)
JBoxVol.log_info("Restored backup at " + self.disk_path)
except IOError as ioe:
if ioe.errno == errno.ENOSPC:
# continue login on ENOSPC to allow user to delete files
JBoxVol.log_exception("No space left to restore backup for %s", sessname)
else:
raise
finally:
src_tar.close()
# delete local copy of backup if we have it on bucketstore
if k is not None:
os.remove(src)
def get_disk_for_user(user_email):
JBoxDefaultConfigVol.log_debug("creating configs disk for %s", user_email)
if JBoxDefaultConfigVol.FS_LOC is None:
JBoxDefaultConfigVol.configure()
disk_path = os.path.join(JBoxDefaultConfigVol.FS_LOC, unique_sessname(user_email))
cfgvol = JBoxDefaultConfigVol(disk_path, user_email=user_email)
cfgvol._unpack_config()
return cfgvol
def calc_stat(user_email):
VolMgr.STATS['num_users'] += 1
sessname = unique_sessname(user_email)
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
if plugin is not None:
k = plugin.pull(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz", metadata_only=True)
if k is not None:
VolMgr.STATS['loopback']['sizes'].append(k.size)
def calc_stat(user_email):
VolMgr.STATS["num_users"] += 1
sessname = unique_sessname(user_email)
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
if plugin is not None:
k = plugin.pull(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz", metadata_only=True)
if k is not None:
VolMgr.STATS["loopback"]["sizes"].append(k.size)
def get_disk_for_user(user_email):
JBoxPolsarDiskVol.log_debug("Mounting polsar disk for %s", user_email)
disk_id = unique_sessname(user_email)
disk_path = JBoxPolsarDiskVol.FS_LOC
if not os.path.exists(disk_path):
os.mkdir(disk_path)
polsarvol = JBoxPolsarDiskVol(disk_path, user_email=user_email)
return polsarvol
def get_disk_from_container(cid):
container_name = JBoxVol.get_cname(cid)
sessname = container_name[1:]
for dev, vol in JBoxEBSVol.get_mapped_volumes().iteritems():
vol = EBSVol.get_volume(vol.volume_id)
if 'Name' in vol.tags:
name = vol.tags['Name']
if unique_sessname(name) == sessname:
return JBoxEBSVol(dev, sessname=sessname)
return None
def get_disk_from_container(cid):
container_name = JBoxVol.get_cname(cid)
sessname = container_name[1:]
for dev, vol in JBoxEBSVol.get_mapped_volumes().iteritems():
vol = EBSVol.get_volume(vol.volume_id)
if 'Name' in vol.tags:
name = vol.tags['Name']
if unique_sessname(name) == sessname:
return JBoxEBSVol(dev, sessname=sessname)
return None
def get_disk_for_user(user_email):
JBoxDefaultConfigVol.log_debug("creating configs disk for %s",
user_email)
if JBoxDefaultConfigVol.FS_LOC is None:
JBoxDefaultConfigVol.configure()
disk_path = os.path.join(JBoxDefaultConfigVol.FS_LOC,
unique_sessname(user_email))
cfgvol = JBoxDefaultConfigVol(disk_path, user_email=user_email)
cfgvol._unpack_config()
return cfgvol
def restore(self):
sessname = unique_sessname(self.user_email)
old_sessname = esc_sessname(self.user_email)
src = os.path.join(JBoxVol.BACKUP_LOC, sessname + ".tar.gz")
k = JBoxVol.pull_from_bucketstore(src) # download from S3 if exists
if not os.path.exists(src):
if old_sessname is not None:
src = os.path.join(JBoxVol.BACKUP_LOC,
old_sessname + ".tar.gz")
k = JBoxVol.pull_from_bucketstore(
src) # download from S3 if exists
if not os.path.exists(src):
return
JBoxVol.log_info("Filtering out restore info from backup " + src +
" to " + self.disk_path)
src_tar = tarfile.open(src, 'r:gz')
try:
perms = {}
for info in src_tar.getmembers():
if not info.name.startswith('juser/'):
continue
extract_name = info.name[6:]
if (info.type == tarfile.LNKTYPE or info.type == tarfile.SYMTYPE) and \
info.linkname.startswith('juser/'):
info.linkname = info.linkname[6:]
if info.name.startswith('juser/.'):
if JBoxVol._is_path_user_home_essential(extract_name):
continue
info.name = extract_name
if len(info.name) == 0:
continue
src_tar.extract(info, self.disk_path)
extracted_path = os.path.join(self.disk_path, extract_name)
if os.path.isdir(extracted_path) and not os.access(
extracted_path, os.W_OK):
st = os.stat(extracted_path)
perms[extracted_path] = st
os.chmod(extracted_path, st.st_mode | stat.S_IWRITE)
if len(perms) > 0:
JBoxVol.log_debug("resetting permissions on %d folders",
len(perms))
for extracted_path, perm in perms.iteritems():
os.chmod(extracted_path, perm)
JBoxVol.log_info("Restored backup at " + self.disk_path)
except IOError, ioe:
if ioe.errno == errno.ENOSPC:
# continue login on ENOSPC to allow user to delete files
JBoxVol.log_exception("No space left to restore backup for %s",
sessname)
else:
raise
def get_disk_for_user(user_email):
JBoxEBSVol.log_debug("creating EBS volume for %s", user_email)
disk_id = JBoxEBSVol._reserve_disk_id()
if disk_id is None:
raise Exception("No free disk available")
try:
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID,
region_id=CompEC2.REGION,
user_id=user_email)
except Exception as ex:
JBoxEBSVol.log_debug("No existing disk for %s. Exception %r",
user_email, ex)
existing_disk = None
if existing_disk is None:
sess_id = unique_sessname(user_email)
sess_props = JBoxSessionProps(Compute.get_install_id(),
sess_id,
create=True,
user_id=user_email)
if sess_props.is_new:
sess_props.save()
snap_id = sess_props.get_snapshot_id()
if snap_id is None:
snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT
JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id,
user_email)
dev_path, vol_id = EBSVol.create_new_volume(
snap_id,
disk_id,
tag=user_email,
disk_sz_gb=JBoxEBSVol.DISK_LIMIT)
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID,
region_id=CompEC2.REGION,
user_id=user_email,
volume_id=vol_id,
attach_time=None,
create=True)
else:
dev_path = EBSVol.attach_volume(existing_disk.get_volume_id(),
disk_id)
existing_disk.set_state(JBoxDiskState.STATE_ATTACHING)
existing_disk.save()
return JBoxEBSVol(dev_path, user_email=user_email)
def find_logged_in_instance(user_id):
container_id = "/" + unique_sessname(user_id)
instances = Compute.get_all_instances()
for inst in instances:
try:
sessions = JBoxAsyncJob.sync_session_status(inst)['data']
if len(sessions) > 0:
if container_id in sessions:
return inst
except:
JBoxHandler.log_error("Error receiving sessions list from %r", inst)
pass
return None
def find_logged_in_instance(user_id):
container_id = "/" + unique_sessname(user_id)
instances = Compute.get_all_instances()
for inst in instances:
try:
sessions = JBoxAsyncJob.sync_session_status(inst)['data']
if len(sessions) > 0:
if container_id in sessions:
return inst
except:
JBoxHandler.log_error("Error receiving sessions list from %r",
inst)
pass
return None
def get_disk_for_user(user_email):
JBoxHostDiskVol.log_debug("creating host disk for %s", user_email)
disk_id = unique_sessname(user_email)
disk_path = os.path.join(JBoxHostDiskVol.FS_LOC, disk_id)
if not os.path.exists(disk_path):
os.mkdir(disk_path)
hostvol = JBoxHostDiskVol(disk_path, user_email=user_email)
hostvol.refresh_disk(mark_refreshed=False)
if JBoxVol.BACKUP_LOC is not None:
JBoxHostDiskVol.log_debug("restoring data for %s", user_email)
hostvol.restore()
return hostvol
def get_disk_for_user(user_email):
JBoxHostDiskVol.log_debug("creating host disk for %s", user_email)
disk_id = unique_sessname(user_email)
disk_path = os.path.join(JBoxHostDiskVol.FS_LOC, disk_id)
if not os.path.exists(disk_path):
os.mkdir(disk_path)
hostvol = JBoxHostDiskVol(disk_path, user_email=user_email)
hostvol.refresh_disk(mark_refreshed=False)
if JBoxVol.BACKUP_LOC is not None:
JBoxHostDiskVol.log_debug("restoring data for %s", user_email)
hostvol.restore()
return hostvol
def do_monitor_loading_ajax(self, user_id):
sessname = unique_sessname(user_id)
self.log_debug("AJAX monitoring loading of session [%s] user[%s]...", sessname, user_id)
cont = SessContainer.get_by_name(sessname)
if (cont is None) or (not cont.is_running()):
loading_step = int(self.get_loading_state(), 0)
if loading_step > 90:
self.log_error("Could not start instance. Session [%s] for user [%s] didn't load.", sessname, user_id)
self.write({'code': -1})
return
loading_step += 1
self.set_loading_state(loading_step)
self.write({'code': 0})
else:
self.write({'code': 1})
def do_monitor_loading_ajax(self, user_id):
sessname = unique_sessname(user_id)
self.log_debug("AJAX monitoring loading of session [%s] user[%s]...", sessname, user_id)
cont = SessContainer.get_by_name(sessname)
if (cont is None) or (not cont.is_running()):
loading_step = int(self.get_loading_state(), 0)
if loading_step > 60:
self.log_error("Could not start instance. Session [%s] for user [%s] didn't load.", sessname, user_id)
self.write({'code': -1})
return
loading_step += 1
self.set_loading_state(loading_step)
self.write({'code': 0})
else:
self.write({'code': 1})
def do_node_housekeeping():
JBoxEBSHousekeep.log_debug("starting node housekeeping")
for device, vol in JBoxEBSVol.get_mapped_volumes().iteritems():
deviceid = os.path.basename(device)
vol_id = vol.volume_id
vol = EBSVol.get_volume(vol_id)
user_id = vol.tags["Name"] if "Name" in vol.tags else None
if user_id is None:
continue
sessname = unique_sessname(user_id)
cont = SessContainer.get_by_name(sessname)
if cont is not None:
continue
JBoxEBSHousekeep.log_debug("Found orphaned volume %s for %s, %s", vol_id, user_id, sessname)
ebsvol = JBoxEBSVol(deviceid, sessname=sessname)
ebsvol.release(backup=True)
JBoxEBSHousekeep.log_debug("finished node housekeeping")
def do_monitor_loading(self, user_id):
sessname = unique_sessname(user_id)
self.log_debug("Monitoring loading of session [%s] user[%s]...",
sessname, user_id)
cont = SessContainer.get_by_name(sessname)
if (cont is None) or (not cont.is_running()):
loading_step = int(self.get_loading_state(), 0)
if loading_step > 30:
self.log_error(
"Could not start instance. Session [%s] for user [%s] didn't load.",
sessname, user_id)
self.clear_container()
self.rendertpl(
"index.tpl",
cfg=JBoxCfg.nv,
state=self.state(
error=
'Could not start your instance! Please try again.',
pending_activation=False,
user_id=user_id))
return
else:
loading_step += 1
self.set_loading_state(loading_step)
self.rendertpl("loading.tpl",
user_id=user_id,
cfg=JBoxCfg.nv,
js_includes=JBPluginHandler.PLUGIN_JAVASCRIPTS)
else:
(shellport, uplport, ipnbport) = cont.get_host_ports()
self.set_container_ports({
JBoxHandler.COOKIE_PORT_SHELL: shellport,
JBoxHandler.COOKIE_PORT_UPL: uplport,
JBoxHandler.COOKIE_PORT_IPNB: ipnbport
})
self.clear_loading()
self.rendertpl("ipnbsess.tpl",
sessname=sessname,
cfg=JBoxCfg.nv,
user_id=user_id,
plugin_features=json.dumps(
self.application.settings["plugin_features"]),
js_includes=JBPluginHandler.PLUGIN_JAVASCRIPTS)
def do_node_housekeeping():
JBoxEBSHousekeep.log_debug("starting node housekeeping")
for device, vol in JBoxEBSVol.get_mapped_volumes().iteritems():
deviceid = os.path.basename(device)
vol_id = vol.volume_id
vol = EBSVol.get_volume(vol_id)
user_id = vol.tags['Name'] if 'Name' in vol.tags else None
if user_id is None:
continue
sessname = unique_sessname(user_id)
cont = SessContainer.get_by_name(sessname)
if cont is not None:
continue
JBoxEBSHousekeep.log_debug("Found orphaned volume %s for %s, %s", vol_id, user_id, sessname)
ebsvol = JBoxEBSVol(deviceid, sessname=sessname)
ebsvol.release(backup=True)
JBoxEBSHousekeep.log_debug("finished node housekeeping")
def __init__(self, user_email, gname=None):
UserCluster.configure_dynamic()
self.user_email = user_email
if gname is None:
self.gname = UserCluster.NAME_PFX + unique_sessname(user_email)
else:
self.gname = gname
if user_email is not None:
self._dbg_str = ("UserCluster(%s - %s)" % (user_email, self.gname))
else:
self._dbg_str = ("UserCluster(%s)" % (self.gname,))
self.placement_group = None
self.autoscale_group = None
self.launch_config = None
self.instances = []
self.public_ips = []
self.public_hosts = []
self.private_ips = []
self.private_hosts = []
def get_disk_for_user(user_email):
JBoxEBSVol.log_debug("creating EBS volume for %s", user_email)
disk_id = JBoxEBSVol._reserve_disk_id()
if disk_id is None:
raise Exception("No free disk available")
try:
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION,
user_id=user_email)
except Exception as ex:
JBoxEBSVol.log_debug("No existing disk for %s. Exception %r", user_email, ex)
existing_disk = None
if existing_disk is None:
sess_id = unique_sessname(user_email)
sess_props = JBoxSessionProps(Compute.get_install_id(), sess_id, create=True, user_id=user_email)
if sess_props.is_new:
sess_props.save()
snap_id = sess_props.get_snapshot_id()
if snap_id is None:
snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT
JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id, user_email)
dev_path, vol_id = EBSVol.create_new_volume(snap_id, disk_id, tag=user_email,
disk_sz_gb=JBoxEBSVol.DISK_LIMIT)
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION,
user_id=user_email,
volume_id=vol_id,
attach_time=None,
create=True)
else:
dev_path = EBSVol.attach_volume(existing_disk.get_volume_id(), disk_id)
existing_disk.set_state(JBoxDiskState.STATE_ATTACHING)
existing_disk.save()
return JBoxEBSVol(dev_path, user_email=user_email)
def set_container_initialized(self, instance_id, user_id):
""" Marks a container as being allocated to a user session.
Sets a cookie named COOKIE_SESS (jb_sess). Cookie contains:
- Container id (session name / docker container name).
- Container location (instance id).
- Creation time stamp.
- Signature for validity check.
It also clears any stale port mapping cookies, and sets the loading state to 1.
:param instance_id: The instance where container is allocated, to redirect future requests to.
:param user_id: The user id for which container is allocated.
:return: None
"""
self.set_redirect_instance_id(instance_id)
t = datetime.datetime.now(pytz.utc).isoformat()
cid = unique_sessname(user_id)
sign = signstr(cid + instance_id + t, JBoxCfg.get('sesskey'))
sess_cookie = {'c': cid, 't': t, 'i': instance_id, 'x': sign}
self._set_container_cookies({
JBoxCookies.COOKIE_SESS: base64.b64encode(json.dumps(sess_cookie))
})
self._clear_container_ports()
self.set_loading_state(1)
def do_monitor_loading(self, user_id):
sessname = unique_sessname(user_id)
self.log_debug("Monitoring loading of session [%s] user[%s]...", sessname, user_id)
cont = SessContainer.get_by_name(sessname)
if (cont is None) or (not cont.is_running()):
loading_step = int(self.get_loading_state(), 0)
if loading_step > 30:
self.log_error("Could not start instance. Session [%s] for user [%s] didn't load.", sessname, user_id)
self.clear_container()
self.rendertpl("index.tpl", cfg=JBoxCfg.nv,
state=self.state(
error='Could not start your instance! Please try again.',
pending_activation=False,
user_id=user_id))
return
else:
loading_step += 1
self.set_loading_state(loading_step)
self.rendertpl("loading.tpl",
user_id=user_id,
cfg=JBoxCfg.nv,
js_includes=JBPluginHandler.PLUGIN_JAVASCRIPTS)
else:
(shellport, uplport, ipnbport) = cont.get_host_ports()
self.set_container_ports({
JBoxHandler.COOKIE_PORT_SHELL: shellport,
JBoxHandler.COOKIE_PORT_UPL: uplport,
JBoxHandler.COOKIE_PORT_IPNB: ipnbport
})
self.clear_loading()
self.rendertpl("ipnbsess.tpl", sessname=sessname, cfg=JBoxCfg.nv, user_id=user_id,
plugin_features=json.dumps(self.application.settings["plugin_features"]),
js_includes=JBPluginHandler.PLUGIN_JAVASCRIPTS)
def set_container_initialized(self, instance_id, user_id):
""" Marks a container as being allocated to a user session.
Sets a cookie named COOKIE_SESS (jb_sess). Cookie contains:
- Container id (session name / docker container name).
- Container location (instance id).
- Creation time stamp.
- Signature for validity check.
It also clears any stale port mapping cookies, and sets the loading state to 1.
:param instance_id: The instance where container is allocated, to redirect future requests to.
:param user_id: The user id for which container is allocated.
:return: None
"""
self.set_redirect_instance_id(instance_id)
t = datetime.datetime.now(pytz.utc).isoformat()
cid = unique_sessname(user_id)
sign = signstr(cid + instance_id + t, JBoxCfg.get('sesskey'))
sess_cookie = {'c': cid, 't': t, 'i': instance_id, 'x': sign}
self._set_container_cookies({
JBoxCookies.COOKIE_SESS:
base64.b64encode(json.dumps(sess_cookie))
})
self._clear_container_ports()
self.set_loading_state(1)
def try_launch_container(cls, user_id, max_hop=False):
sessname = unique_sessname(user_id)
cont = SessContainer.get_by_name(sessname)
cls.log_debug("have existing container for %s: %r", sessname, None != cont)
if cont is not None:
cls.log_debug("container running: %r", cont.is_running())
if max_hop:
self_load = Compute.get_instance_stats(Compute.get_instance_id(), 'Load')
if self_load < 100:
SessContainer.invalidate_container(sessname)
JBoxAsyncJob.async_launch_by_name(sessname, user_id, True)
return True
is_leader = is_proposed_cluster_leader()
if ((cont is None) or (not cont.is_running())) and (not Compute.should_accept_session(is_leader)):
if cont is not None:
SessContainer.invalidate_container(cont.get_name())
JBoxAsyncJob.async_backup_and_cleanup(cont.dockid)
return False
SessContainer.invalidate_container(sessname)
JBoxAsyncJob.async_launch_by_name(sessname, user_id, True)
return True
def pull_backup(user_email):
sessname = unique_sessname(user_email)
JBoxDisk.log_info("pulling %s.tar.gz from %s", sessname, JBoxVol.BACKUP_BUCKET)
JBoxDisk.PLUGIN.pull(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz", metadata_only=False)
class JBoxEBSVol(JBoxVol):
provides = [JBoxVol.JBP_DATA_EBS, JBoxVol.JBP_DATA]
DEVICES = []
MAX_DISKS = 0
DISK_LIMIT = None
DISK_USE_STATUS = {}
DISK_RESERVE_TIME = {}
DISK_TEMPLATE_SNAPSHOT = None
LOCK = None
@staticmethod
def configure():
num_disks_max = JBoxCfg.get('numdisksmax')
JBoxEBSVol.DISK_LIMIT = 10
JBoxEBSVol.MAX_DISKS = num_disks_max
JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT = JBoxCfg.get(
'cloud_host.ebs_template')
JBoxEBSVol.DEVICES = JBoxEBSVol._guess_configured_devices(
'xvd', num_disks_max)
JBoxEBSVol.log_debug(
"Assuming %d EBS volumes configured in range xvdba..xvdcz",
len(JBoxEBSVol.DEVICES))
JBoxEBSVol.LOCK = threading.Lock()
JBoxEBSVol.refresh_disk_use_status()
@classmethod
def get_disk_allocated_size(cls):
return JBoxEBSVol.DISK_LIMIT * 1000000000
@staticmethod
def _guess_configured_devices(devidpfx, num_disks):
devices = []
for pfx1 in 'bc':
for pfx2 in ascii_lowercase:
devices.append(devidpfx + pfx1 + pfx2)
if len(devices) == num_disks:
return devices
return devices
@staticmethod
def refresh_disk_use_status(container_id_list=None):
JBoxEBSVol.log_debug("Refrshing EBS disk use status")
JBoxEBSVol.LOCK.acquire()
try:
nfree = 0
for idx in range(0, JBoxEBSVol.MAX_DISKS):
dev = JBoxEBSVol.DEVICES[idx]
if JBoxEBSVol._is_reserved(dev):
JBoxEBSVol.DISK_USE_STATUS[dev] = True
else:
JBoxEBSVol.DISK_USE_STATUS[dev] = False
nfree += 1
for device, volume in JBoxEBSVol.get_mapped_volumes().iteritems():
JBoxEBSVol.DISK_USE_STATUS[os.path.basename(device)] = True
nfree -= 1
JBoxEBSVol.log_info("EBS Disk free: " + str(nfree) + "/" +
str(JBoxEBSVol.MAX_DISKS))
except:
JBoxEBSVol.log_exception("Exception refrshing EBS disk use status")
finally:
JBoxEBSVol.LOCK.release()
@staticmethod
def get_mapped_volumes():
allmaps = EBSVol.get_mapped_volumes()
JBoxEBSVol.log_debug("Devices mapped: %r", allmaps)
return dict((d, v) for d, v in allmaps.iteritems()
if os.path.basename(d) in JBoxEBSVol.DEVICES)
@staticmethod
def _get_unused_disk_id():
for idx in range(0, JBoxEBSVol.MAX_DISKS):
dev = JBoxEBSVol.DEVICES[idx]
if not JBoxEBSVol.DISK_USE_STATUS[dev]:
return dev
return None
@staticmethod
def _is_reserved(idx):
if (idx in JBoxEBSVol.DISK_RESERVE_TIME) and (
JBoxEBSVol.DISK_RESERVE_TIME[idx] < time.time()):
del JBoxEBSVol.DISK_RESERVE_TIME[idx]
return idx in JBoxEBSVol.DISK_RESERVE_TIME
@staticmethod
def _mark_disk_used(idx, used=True, for_secs=0):
JBoxEBSVol.DISK_USE_STATUS[idx] = used
if used and (for_secs > 0):
JBoxEBSVol.DISK_RESERVE_TIME[idx] = time.time() + for_secs
else:
if idx in JBoxEBSVol.DISK_RESERVE_TIME:
del JBoxEBSVol.DISK_RESERVE_TIME[idx]
@staticmethod
def _reserve_disk_id():
JBoxEBSVol.LOCK.acquire()
try:
disk_id = JBoxEBSVol._get_unused_disk_id()
JBoxEBSVol._mark_disk_used(disk_id, for_secs=120)
return disk_id
finally:
JBoxEBSVol.LOCK.release()
@staticmethod
def is_mount_path(fs_path):
# EBS volumes are not mounted on host
return False
@staticmethod
def disk_ids_used_pct():
pct = (sum(JBoxEBSVol.DISK_USE_STATUS.values()) * 100) / len(
JBoxEBSVol.DISK_USE_STATUS)
return min(100, max(0, pct))
@staticmethod
def get_disk_for_user(user_email):
JBoxEBSVol.log_debug("creating EBS volume for %s", user_email)
disk_id = JBoxEBSVol._reserve_disk_id()
if disk_id is None:
raise Exception("No free disk available")
try:
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID,
region_id=CompEC2.REGION,
user_id=user_email)
except Exception, ex:
JBoxEBSVol.log_debug("No existing disk for %s. Exception %r",
user_email, ex)
existing_disk = None
if existing_disk is None:
sess_id = unique_sessname(user_email)
sess_props = JBoxSessionProps(sess_id,
create=True,
user_id=user_email)
if sess_props.is_new:
sess_props.save()
snap_id = sess_props.get_snapshot_id()
if snap_id is None:
snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT
JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id,
user_email)
dev_path, vol_id = EBSVol.create_new_volume(
snap_id,
disk_id,
tag=user_email,
disk_sz_gb=JBoxEBSVol.DISK_LIMIT)
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID,
region_id=CompEC2.REGION,
user_id=user_email,
volume_id=vol_id,
attach_time=None,
create=True)
else:
dev_path = EBSVol.attach_volume(existing_disk.get_volume_id(),
disk_id)
existing_disk.set_state(JBoxDiskState.STATE_ATTACHING)
existing_disk.save()
return JBoxEBSVol(dev_path, user_email=user_email)
def rename_and_delete(user_email):
sessname = unique_sessname(user_email)
renamed_sessname = sessname + '_old'
JBoxDisk.PLUGIN.move(sessname + ".tar.gz", renamed_sessname + ".tar.gz", JBoxVol.BACKUP_BUCKET)
def rename_and_delete(user_email):
sessname = unique_sessname(user_email)
renamed_sessname = sessname + '_old'
JBoxS3.move(sessname + ".tar.gz", renamed_sessname + ".tar.gz", JBoxVol.BACKUP_BUCKET)
def delete(user_email):
sessname = unique_sessname(user_email)
JBoxS3.delete(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz")
def delete(user_email):
sessname = unique_sessname(user_email)
JBoxDisk.PLUGIN.delete(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz")