def configure(dckr, cfg):
cloud_cfg = cfg['cloud_host']
backup_location = os.path.expanduser(cfg['backup_location'])
user_home_img = os.path.expanduser(cfg['user_home_image'])
mnt_location = os.path.expanduser(cfg['mnt_location'])
backup_bucket = cloud_cfg['backup_bucket']
num_disks_max = cfg["numdisksmax"]
make_sure_path_exists(backup_location)
JBoxVol.configure_base(dckr, user_home_img, backup_location, backup_bucket=backup_bucket)
JBoxLoopbackVol.configure(cfg['disk_limit'], mnt_location, num_disks_max)
if cloud_cfg['ebs']:
VolMgr.HAS_EBS = True
ebs_mnt_location = os.path.expanduser(cloud_cfg['ebs_mnt_location'])
JBoxEBSVol.configure(1000000000, ebs_mnt_location, num_disks_max, cloud_cfg['ebs_template'])
def get_disk_for_user(email):
VolMgr.log_debug("restoring disk for %s", email)
user = JBoxUserV2(email)
ebs = False
if VolMgr.HAS_EBS:
ebs = user.has_resource_profile(JBoxUserV2.RES_PROF_DISK_EBS_1G)
custom_jimg = None
ipython_profile = 'julia'
# TODO: image path should be picked up from config
if user.has_resource_profile(JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP):
custom_jimg = '/home/juser/.juliabox/jimg/sys.ji'
ipython_profile = 'jboxjulia'
if ebs:
disk = JBoxEBSVol.get_disk_for_user(email)
else:
disk = JBoxLoopbackVol.get_disk_for_user(email)
try:
disk.setup_julia_image(ipython_profile, custom_jimg)
disk.setup_tutorial_link()
disk.gen_ssh_key()
disk.gen_gitconfig()
except IOError, ioe:
if ioe.errno == errno.ENOSPC:
# continue login on ENOSPC to allow user to delete files
JBoxVol.log_exception("No space left to configure JuliaBox for %s", email)
else:
raise
def get_disk_for_user(email):
VolMgr.log_debug("restoring disk for %s", email)
user = JBoxUserV2(email)
ebs = False
if VolMgr.HAS_EBS:
ebs = user.has_resource_profile(JBoxUserV2.RES_PROF_DISK_EBS_1G)
custom_jimg = None
ipython_profile = 'julia'
# TODO: image path should be picked up from config
if user.has_resource_profile(JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP):
custom_jimg = '/home/juser/.juliabox/jimg/sys.ji'
ipython_profile = 'jboxjulia'
if ebs:
disk = JBoxEBSVol.get_disk_for_user(email)
else:
disk = JBoxLoopbackVol.get_disk_for_user(email)
try:
disk.setup_julia_image(ipython_profile, custom_jimg)
disk.setup_tutorial_link()
disk.gen_ssh_key()
disk.gen_gitconfig()
except IOError, ioe:
if ioe.errno == errno.ENOSPC:
# continue login on ENOSPC to allow user to delete files
JBoxVol.log_exception(
"No space left to configure JuliaBox for %s", email)
else:
raise
def do_node_housekeeping():
JBoxEBSHousekeep.log_debug("starting node housekeeping")
for device, vol in JBoxEBSVol.get_mapped_volumes().iteritems():
deviceid = os.path.basename(device)
vol_id = vol.volume_id
vol = EBSVol.get_volume(vol_id)
user_id = vol.tags["Name"] if "Name" in vol.tags else None
if user_id is None:
continue
sessname = unique_sessname(user_id)
cont = SessContainer.get_by_name(sessname)
if cont is not None:
continue
JBoxEBSHousekeep.log_debug("Found orphaned volume %s for %s, %s", vol_id, user_id, sessname)
ebsvol = JBoxEBSVol(deviceid, sessname=sessname)
ebsvol.release(backup=True)
JBoxEBSHousekeep.log_debug("finished node housekeeping")
def do_node_housekeeping():
JBoxEBSHousekeep.log_debug("starting node housekeeping")
for device, vol in JBoxEBSVol.get_mapped_volumes().iteritems():
deviceid = os.path.basename(device)
vol_id = vol.volume_id
vol = EBSVol.get_volume(vol_id)
user_id = vol.tags['Name'] if 'Name' in vol.tags else None
if user_id is None:
continue
sessname = unique_sessname(user_id)
cont = SessContainer.get_by_name(sessname)
if cont is not None:
continue
JBoxEBSHousekeep.log_debug("Found orphaned volume %s for %s, %s", vol_id, user_id, sessname)
ebsvol = JBoxEBSVol(deviceid, sessname=sessname)
ebsvol.release(backup=True)
JBoxEBSHousekeep.log_debug("finished node housekeeping")
def get_disk_from_container(cid):
props = JBoxVol.dckr().inspect_container(cid)
vols = props['Volumes']
for _cpath, hpath in vols.iteritems():
if hpath.startswith(JBoxLoopbackVol.FS_LOC):
return JBoxLoopbackVol.get_disk_from_container(cid)
elif VolMgr.HAS_EBS and hpath.startswith(JBoxEBSVol.FS_LOC):
return JBoxEBSVol.get_disk_from_container(cid)
return None
def get_disk_from_container(cid):
props = JBoxVol.dckr().inspect_container(cid)
vols = props['Volumes']
for _cpath, hpath in vols.iteritems():
if hpath.startswith(JBoxLoopbackVol.FS_LOC):
return JBoxLoopbackVol.get_disk_from_container(cid)
elif VolMgr.HAS_EBS and hpath.startswith(JBoxEBSVol.FS_LOC):
return JBoxEBSVol.get_disk_from_container(cid)
return None
def get_disk_for_user(email):
VolMgr.log_debug("restoring disk for %s", email)
ebs = False
if VolMgr.HAS_EBS:
user = JBoxUserV2(email)
ebs = user.has_resource_profile(JBoxUserV2.RESOURCE_PROFILE_DISK_EBS_1G)
if ebs:
return JBoxEBSVol.get_disk_for_user(email)
else:
return JBoxLoopbackVol.get_disk_for_user(email)
def get_disk_from_container(cid):
props = JBoxVol.dckr().inspect_container(cid)
try:
vols = props['Volumes']
for _cpath, hpath in vols.iteritems():
if hpath.startswith(JBoxLoopbackVol.FS_LOC):
return JBoxLoopbackVol.get_disk_from_container(cid)
elif VolMgr.HAS_EBS and hpath.startswith(JBoxEBSVol.FS_LOC):
return JBoxEBSVol.get_disk_from_container(cid)
except:
VolMgr.log_error("error finding disk ids used in " + cid)
return None
def get_disk_for_user(email):
VolMgr.log_debug("restoring disk for %s", email)
ebs = False
if VolMgr.HAS_EBS:
user = JBoxUserV2(email)
ebs = user.has_resource_profile(
JBoxUserV2.RESOURCE_PROFILE_DISK_EBS_1G)
if ebs:
return JBoxEBSVol.get_disk_for_user(email)
else:
return JBoxLoopbackVol.get_disk_for_user(email)
def get_disk_from_container(cid):
props = JBoxVol.dckr().inspect_container(cid)
try:
vols = props['Volumes']
for _cpath, hpath in vols.iteritems():
if hpath.startswith(JBoxLoopbackVol.FS_LOC):
return JBoxLoopbackVol.get_disk_from_container(cid)
elif VolMgr.HAS_EBS and hpath.startswith(JBoxEBSVol.FS_LOC):
return JBoxEBSVol.get_disk_from_container(cid)
except:
VolMgr.log_error("error finding disk ids used in " + cid)
return None
def do_task(plugin_type, data):
if plugin_type != JBPluginTask.JBP_CMD_ASYNC:
return
mode = data['action']
user_id = data['user_id']
sessname = data['sessname']
user = JBoxUserV2(user_id)
is_allowed = user.has_resource_profile(
JBoxUserV2.RES_PROF_DISK_EBS_10G)
if not is_allowed:
JBoxEBSVolAsyncTask.log_error(
"Data volume access not allowed for user")
return
cont = SessContainer.get_by_name(sessname)
if cont is None:
return
vol = JBoxEBSVol.get_disk_from_container(sessname)
disk_state = None
try:
disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID,
region_id=CompEC2.REGION,
user_id=user_id)
except:
pass
JBoxEBSVolAsyncTask.log_debug("Data volume request %s for %s", mode,
cont.debug_str())
if mode == 'attach':
if vol is None:
vol = JBoxEBSVol.get_disk_for_user(user_id)
JBoxEBSVol.mount_host_device(vol.disk_path, cont.dockid,
JBoxVol.DATA_MOUNT_POINT)
disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID,
region_id=CompEC2.REGION,
user_id=user_id)
if disk_state.get_state() != JBoxDiskState.STATE_ATTACHED:
disk_state.set_state(JBoxDiskState.STATE_ATTACHED)
disk_state.save()
elif mode == 'detach':
if cont is not None and cont.is_running():
if vol is not None:
# unmount from container first
JBoxEBSVol.unmount_host_device(vol.disk_path, cont.dockid)
elif disk_state is not None:
# no volume attached. ensure disk state is updated
if disk_state.get_state() != JBoxDiskState.STATE_DETACHED:
disk_state.set_state(JBoxDiskState.STATE_DETACHED)
disk_state.save()
if vol is not None:
vol.release(backup=True)
JBoxEBSVolAsyncTask.log_debug(
"Data volume request %s completed for %s", mode, cont.debug_str())
def configure(dckr, cfg):
cloud_cfg = cfg['cloud_host']
backup_location = os.path.expanduser(cfg['backup_location'])
user_home_img = os.path.expanduser(cfg['user_home_image'])
wsock_proto = cfg['websocket_protocol']
mnt_location = os.path.expanduser(cfg['mnt_location'])
backup_bucket = cloud_cfg['backup_bucket']
num_disks_max = cfg["numdisksmax"]
make_sure_path_exists(backup_location)
JBoxVol.configure_base(dckr,
wsock_proto,
user_home_img,
backup_location,
backup_bucket=backup_bucket)
JBoxLoopbackVol.configure(cfg['disk_limit'], mnt_location,
num_disks_max)
if cloud_cfg['ebs']:
VolMgr.HAS_EBS = True
ebs_mnt_location = os.path.expanduser(
cloud_cfg['ebs_mnt_location'])
JBoxEBSVol.configure(1, ebs_mnt_location, num_disks_max,
cloud_cfg['ebs_template'])
def _get_state(self, sessname, user_id):
vol = JBoxEBSVol.get_disk_from_container(sessname)
state_code = JBoxDiskState.STATE_DETACHED
try:
disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_id)
state_code = disk_state.get_state()
except:
pass
if ((state_code == JBoxDiskState.STATE_ATTACHED) and (vol is None)) or (
(state_code == JBoxDiskState.STATE_DETACHED) and (vol is not None)
):
state_code = -1
self.log_debug("EBS disk state: %r", state_code)
return {"disk_size": "10 GB", "state": state_code}
def _get_state(self, sessname, user_id):
vol = JBoxEBSVol.get_disk_from_container(sessname)
state_code = JBoxDiskState.STATE_DETACHED
try:
disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID,
region_id=CompEC2.REGION,
user_id=user_id)
state_code = disk_state.get_state()
except:
pass
if ((state_code == JBoxDiskState.STATE_ATTACHED) and (vol is None)) or \
((state_code == JBoxDiskState.STATE_DETACHED) and (vol is not None)):
state_code = -1
self.log_debug("EBS disk state: %r", state_code)
return {'disk_size': '10 GB', 'state': state_code}
def do_task(plugin_type, data):
if plugin_type != JBPluginTask.JBP_CMD_ASYNC:
return
mode = data["action"]
user_id = data["user_id"]
sessname = data["sessname"]
user = JBoxUserV2(user_id)
is_allowed = user.has_resource_profile(JBoxUserV2.RES_PROF_DISK_EBS_10G)
if not is_allowed:
JBoxEBSVolAsyncTask.log_error("Data volume access not allowed for user")
return
cont = SessContainer.get_by_name(sessname)
if cont is None:
return
vol = JBoxEBSVol.get_disk_from_container(sessname)
disk_state = None
try:
disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_id)
except:
pass
JBoxEBSVolAsyncTask.log_debug("Data volume request %s for %s", mode, cont.debug_str())
if mode == "attach":
if vol is None:
vol = JBoxEBSVol.get_disk_for_user(user_id)
JBoxEBSVol.mount_host_device(vol.disk_path, cont.dockid, JBoxVol.DATA_MOUNT_POINT)
disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_id)
if disk_state.get_state() != JBoxDiskState.STATE_ATTACHED:
disk_state.set_state(JBoxDiskState.STATE_ATTACHED)
disk_state.save()
elif mode == "detach":
if cont is not None and cont.is_running():
if vol is not None:
# unmount from container first
JBoxEBSVol.unmount_host_device(vol.disk_path, cont.dockid)
elif disk_state is not None:
# no volume attached. ensure disk state is updated
if disk_state.get_state() != JBoxDiskState.STATE_DETACHED:
disk_state.set_state(JBoxDiskState.STATE_DETACHED)
disk_state.save()
if vol is not None:
vol.release(backup=True)
JBoxEBSVolAsyncTask.log_debug("Data volume request %s completed for %s", mode, cont.debug_str())
def get_disk_for_user(email):
VolMgr.log_debug("restoring disk for %s", email)
user = JBoxUserV2(email)
ebs = False
if VolMgr.HAS_EBS:
ebs = user.has_resource_profile(JBoxUserV2.RES_PROF_DISK_EBS_1G)
custom_jimg = None
ipython_profile = 'julia'
# TODO: image path should be picked up from config
if user.has_resource_profile(JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP):
custom_jimg = '/home/juser/.juliabox/jimg/sys.ji'
ipython_profile = 'jboxjulia'
if ebs:
disk = JBoxEBSVol.get_disk_for_user(email)
else:
disk = JBoxLoopbackVol.get_disk_for_user(email)
disk.setup_julia_image(ipython_profile, custom_jimg)
disk.setup_tutorial_link()
return disk
def refresh_disk_use_status(container_id_list=None):
JBoxLoopbackVol.refresh_disk_use_status(container_id_list=container_id_list)
if VolMgr.HAS_EBS:
JBoxEBSVol.refresh_disk_use_status(container_id_list=container_id_list)
def refresh_disk_use_status(container_id_list=None):
JBoxLoopbackVol.refresh_disk_use_status(
container_id_list=container_id_list)
if VolMgr.HAS_EBS:
JBoxEBSVol.refresh_disk_use_status(
container_id_list=container_id_list)
def used_pct():
pct = JBoxLoopbackVol.disk_ids_used_pct()
if VolMgr.HAS_EBS:
pct += JBoxEBSVol.disk_ids_used_pct()
return min(100, max(0, pct))
def used_pct():
pct = JBoxLoopbackVol.disk_ids_used_pct()
if VolMgr.HAS_EBS:
pct += JBoxEBSVol.disk_ids_used_pct()
return min(100, max(0, pct))
