def configure(dckr, cfg): cloud_cfg = cfg['cloud_host'] backup_location = os.path.expanduser(cfg['backup_location']) user_home_img = os.path.expanduser(cfg['user_home_image']) mnt_location = os.path.expanduser(cfg['mnt_location']) backup_bucket = cloud_cfg['backup_bucket'] num_disks_max = cfg["numdisksmax"] make_sure_path_exists(backup_location) JBoxVol.configure_base(dckr, user_home_img, backup_location, backup_bucket=backup_bucket) JBoxLoopbackVol.configure(cfg['disk_limit'], mnt_location, num_disks_max) if cloud_cfg['ebs']: VolMgr.HAS_EBS = True ebs_mnt_location = os.path.expanduser(cloud_cfg['ebs_mnt_location']) JBoxEBSVol.configure(1000000000, ebs_mnt_location, num_disks_max, cloud_cfg['ebs_template'])
def get_disk_for_user(email): VolMgr.log_debug("restoring disk for %s", email) user = JBoxUserV2(email) ebs = False if VolMgr.HAS_EBS: ebs = user.has_resource_profile(JBoxUserV2.RES_PROF_DISK_EBS_1G) custom_jimg = None ipython_profile = 'julia' # TODO: image path should be picked up from config if user.has_resource_profile(JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP): custom_jimg = '/home/juser/.juliabox/jimg/sys.ji' ipython_profile = 'jboxjulia' if ebs: disk = JBoxEBSVol.get_disk_for_user(email) else: disk = JBoxLoopbackVol.get_disk_for_user(email) try: disk.setup_julia_image(ipython_profile, custom_jimg) disk.setup_tutorial_link() disk.gen_ssh_key() disk.gen_gitconfig() except IOError, ioe: if ioe.errno == errno.ENOSPC: # continue login on ENOSPC to allow user to delete files JBoxVol.log_exception("No space left to configure JuliaBox for %s", email) else: raise
def get_disk_for_user(email): VolMgr.log_debug("restoring disk for %s", email) user = JBoxUserV2(email) ebs = False if VolMgr.HAS_EBS: ebs = user.has_resource_profile(JBoxUserV2.RES_PROF_DISK_EBS_1G) custom_jimg = None ipython_profile = 'julia' # TODO: image path should be picked up from config if user.has_resource_profile(JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP): custom_jimg = '/home/juser/.juliabox/jimg/sys.ji' ipython_profile = 'jboxjulia' if ebs: disk = JBoxEBSVol.get_disk_for_user(email) else: disk = JBoxLoopbackVol.get_disk_for_user(email) try: disk.setup_julia_image(ipython_profile, custom_jimg) disk.setup_tutorial_link() disk.gen_ssh_key() disk.gen_gitconfig() except IOError, ioe: if ioe.errno == errno.ENOSPC: # continue login on ENOSPC to allow user to delete files JBoxVol.log_exception( "No space left to configure JuliaBox for %s", email) else: raise
def do_node_housekeeping(): JBoxEBSHousekeep.log_debug("starting node housekeeping") for device, vol in JBoxEBSVol.get_mapped_volumes().iteritems(): deviceid = os.path.basename(device) vol_id = vol.volume_id vol = EBSVol.get_volume(vol_id) user_id = vol.tags["Name"] if "Name" in vol.tags else None if user_id is None: continue sessname = unique_sessname(user_id) cont = SessContainer.get_by_name(sessname) if cont is not None: continue JBoxEBSHousekeep.log_debug("Found orphaned volume %s for %s, %s", vol_id, user_id, sessname) ebsvol = JBoxEBSVol(deviceid, sessname=sessname) ebsvol.release(backup=True) JBoxEBSHousekeep.log_debug("finished node housekeeping")
def do_node_housekeeping(): JBoxEBSHousekeep.log_debug("starting node housekeeping") for device, vol in JBoxEBSVol.get_mapped_volumes().iteritems(): deviceid = os.path.basename(device) vol_id = vol.volume_id vol = EBSVol.get_volume(vol_id) user_id = vol.tags['Name'] if 'Name' in vol.tags else None if user_id is None: continue sessname = unique_sessname(user_id) cont = SessContainer.get_by_name(sessname) if cont is not None: continue JBoxEBSHousekeep.log_debug("Found orphaned volume %s for %s, %s", vol_id, user_id, sessname) ebsvol = JBoxEBSVol(deviceid, sessname=sessname) ebsvol.release(backup=True) JBoxEBSHousekeep.log_debug("finished node housekeeping")
def get_disk_from_container(cid): props = JBoxVol.dckr().inspect_container(cid) vols = props['Volumes'] for _cpath, hpath in vols.iteritems(): if hpath.startswith(JBoxLoopbackVol.FS_LOC): return JBoxLoopbackVol.get_disk_from_container(cid) elif VolMgr.HAS_EBS and hpath.startswith(JBoxEBSVol.FS_LOC): return JBoxEBSVol.get_disk_from_container(cid) return None
def get_disk_for_user(email): VolMgr.log_debug("restoring disk for %s", email) ebs = False if VolMgr.HAS_EBS: user = JBoxUserV2(email) ebs = user.has_resource_profile(JBoxUserV2.RESOURCE_PROFILE_DISK_EBS_1G) if ebs: return JBoxEBSVol.get_disk_for_user(email) else: return JBoxLoopbackVol.get_disk_for_user(email)
def get_disk_from_container(cid): props = JBoxVol.dckr().inspect_container(cid) try: vols = props['Volumes'] for _cpath, hpath in vols.iteritems(): if hpath.startswith(JBoxLoopbackVol.FS_LOC): return JBoxLoopbackVol.get_disk_from_container(cid) elif VolMgr.HAS_EBS and hpath.startswith(JBoxEBSVol.FS_LOC): return JBoxEBSVol.get_disk_from_container(cid) except: VolMgr.log_error("error finding disk ids used in " + cid) return None
def get_disk_for_user(email): VolMgr.log_debug("restoring disk for %s", email) ebs = False if VolMgr.HAS_EBS: user = JBoxUserV2(email) ebs = user.has_resource_profile( JBoxUserV2.RESOURCE_PROFILE_DISK_EBS_1G) if ebs: return JBoxEBSVol.get_disk_for_user(email) else: return JBoxLoopbackVol.get_disk_for_user(email)
def do_task(plugin_type, data): if plugin_type != JBPluginTask.JBP_CMD_ASYNC: return mode = data['action'] user_id = data['user_id'] sessname = data['sessname'] user = JBoxUserV2(user_id) is_allowed = user.has_resource_profile( JBoxUserV2.RES_PROF_DISK_EBS_10G) if not is_allowed: JBoxEBSVolAsyncTask.log_error( "Data volume access not allowed for user") return cont = SessContainer.get_by_name(sessname) if cont is None: return vol = JBoxEBSVol.get_disk_from_container(sessname) disk_state = None try: disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_id) except: pass JBoxEBSVolAsyncTask.log_debug("Data volume request %s for %s", mode, cont.debug_str()) if mode == 'attach': if vol is None: vol = JBoxEBSVol.get_disk_for_user(user_id) JBoxEBSVol.mount_host_device(vol.disk_path, cont.dockid, JBoxVol.DATA_MOUNT_POINT) disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_id) if disk_state.get_state() != JBoxDiskState.STATE_ATTACHED: disk_state.set_state(JBoxDiskState.STATE_ATTACHED) disk_state.save() elif mode == 'detach': if cont is not None and cont.is_running(): if vol is not None: # unmount from container first JBoxEBSVol.unmount_host_device(vol.disk_path, cont.dockid) elif disk_state is not None: # no volume attached. ensure disk state is updated if disk_state.get_state() != JBoxDiskState.STATE_DETACHED: disk_state.set_state(JBoxDiskState.STATE_DETACHED) disk_state.save() if vol is not None: vol.release(backup=True) JBoxEBSVolAsyncTask.log_debug( "Data volume request %s completed for %s", mode, cont.debug_str())
def configure(dckr, cfg): cloud_cfg = cfg['cloud_host'] backup_location = os.path.expanduser(cfg['backup_location']) user_home_img = os.path.expanduser(cfg['user_home_image']) wsock_proto = cfg['websocket_protocol'] mnt_location = os.path.expanduser(cfg['mnt_location']) backup_bucket = cloud_cfg['backup_bucket'] num_disks_max = cfg["numdisksmax"] make_sure_path_exists(backup_location) JBoxVol.configure_base(dckr, wsock_proto, user_home_img, backup_location, backup_bucket=backup_bucket) JBoxLoopbackVol.configure(cfg['disk_limit'], mnt_location, num_disks_max) if cloud_cfg['ebs']: VolMgr.HAS_EBS = True ebs_mnt_location = os.path.expanduser( cloud_cfg['ebs_mnt_location']) JBoxEBSVol.configure(1, ebs_mnt_location, num_disks_max, cloud_cfg['ebs_template'])
def _get_state(self, sessname, user_id): vol = JBoxEBSVol.get_disk_from_container(sessname) state_code = JBoxDiskState.STATE_DETACHED try: disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_id) state_code = disk_state.get_state() except: pass if ((state_code == JBoxDiskState.STATE_ATTACHED) and (vol is None)) or ( (state_code == JBoxDiskState.STATE_DETACHED) and (vol is not None) ): state_code = -1 self.log_debug("EBS disk state: %r", state_code) return {"disk_size": "10 GB", "state": state_code}
def _get_state(self, sessname, user_id): vol = JBoxEBSVol.get_disk_from_container(sessname) state_code = JBoxDiskState.STATE_DETACHED try: disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_id) state_code = disk_state.get_state() except: pass if ((state_code == JBoxDiskState.STATE_ATTACHED) and (vol is None)) or \ ((state_code == JBoxDiskState.STATE_DETACHED) and (vol is not None)): state_code = -1 self.log_debug("EBS disk state: %r", state_code) return {'disk_size': '10 GB', 'state': state_code}
def do_task(plugin_type, data): if plugin_type != JBPluginTask.JBP_CMD_ASYNC: return mode = data["action"] user_id = data["user_id"] sessname = data["sessname"] user = JBoxUserV2(user_id) is_allowed = user.has_resource_profile(JBoxUserV2.RES_PROF_DISK_EBS_10G) if not is_allowed: JBoxEBSVolAsyncTask.log_error("Data volume access not allowed for user") return cont = SessContainer.get_by_name(sessname) if cont is None: return vol = JBoxEBSVol.get_disk_from_container(sessname) disk_state = None try: disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_id) except: pass JBoxEBSVolAsyncTask.log_debug("Data volume request %s for %s", mode, cont.debug_str()) if mode == "attach": if vol is None: vol = JBoxEBSVol.get_disk_for_user(user_id) JBoxEBSVol.mount_host_device(vol.disk_path, cont.dockid, JBoxVol.DATA_MOUNT_POINT) disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_id) if disk_state.get_state() != JBoxDiskState.STATE_ATTACHED: disk_state.set_state(JBoxDiskState.STATE_ATTACHED) disk_state.save() elif mode == "detach": if cont is not None and cont.is_running(): if vol is not None: # unmount from container first JBoxEBSVol.unmount_host_device(vol.disk_path, cont.dockid) elif disk_state is not None: # no volume attached. ensure disk state is updated if disk_state.get_state() != JBoxDiskState.STATE_DETACHED: disk_state.set_state(JBoxDiskState.STATE_DETACHED) disk_state.save() if vol is not None: vol.release(backup=True) JBoxEBSVolAsyncTask.log_debug("Data volume request %s completed for %s", mode, cont.debug_str())
def get_disk_for_user(email): VolMgr.log_debug("restoring disk for %s", email) user = JBoxUserV2(email) ebs = False if VolMgr.HAS_EBS: ebs = user.has_resource_profile(JBoxUserV2.RES_PROF_DISK_EBS_1G) custom_jimg = None ipython_profile = 'julia' # TODO: image path should be picked up from config if user.has_resource_profile(JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP): custom_jimg = '/home/juser/.juliabox/jimg/sys.ji' ipython_profile = 'jboxjulia' if ebs: disk = JBoxEBSVol.get_disk_for_user(email) else: disk = JBoxLoopbackVol.get_disk_for_user(email) disk.setup_julia_image(ipython_profile, custom_jimg) disk.setup_tutorial_link() return disk
def refresh_disk_use_status(container_id_list=None): JBoxLoopbackVol.refresh_disk_use_status(container_id_list=container_id_list) if VolMgr.HAS_EBS: JBoxEBSVol.refresh_disk_use_status(container_id_list=container_id_list)
def refresh_disk_use_status(container_id_list=None): JBoxLoopbackVol.refresh_disk_use_status( container_id_list=container_id_list) if VolMgr.HAS_EBS: JBoxEBSVol.refresh_disk_use_status( container_id_list=container_id_list)
def used_pct(): pct = JBoxLoopbackVol.disk_ids_used_pct() if VolMgr.HAS_EBS: pct += JBoxEBSVol.disk_ids_used_pct() return min(100, max(0, pct))