def find_logged_in_instance(user_id): sessname = unique_sessname(user_id) try: sess_props = JBoxSessionProps(Compute.get_install_id(), sessname) return sess_props.get_instance_id() except JBoxDBItemNotFound: return None
def _backup(self, clear_volume=False): sess_props = JBoxSessionProps(Compute.get_install_id(), self.sessname) desc = sess_props.get_user_id() + " JuliaBox Backup" disk_id = self.disk_path.split('/')[-1] snap_id = EBSVol.snapshot_volume(dev_id=disk_id, tag=self.sessname, description=desc, wait_till_complete=False) return snap_id
def get_disk_for_user(user_email): JBoxEBSVol.log_debug("creating EBS volume for %s", user_email) disk_id = JBoxEBSVol._reserve_disk_id() if disk_id is None: raise Exception("No free disk available") try: existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_email) except Exception as ex: JBoxEBSVol.log_debug("No existing disk for %s. Exception %r", user_email, ex) existing_disk = None if existing_disk is None: sess_id = unique_sessname(user_email) sess_props = JBoxSessionProps(Compute.get_install_id(), sess_id, create=True, user_id=user_email) if sess_props.is_new: sess_props.save() snap_id = sess_props.get_snapshot_id() if snap_id is None: snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id, user_email) dev_path, vol_id = EBSVol.create_new_volume( snap_id, disk_id, tag=user_email, disk_sz_gb=JBoxEBSVol.DISK_LIMIT) existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_email, volume_id=vol_id, attach_time=None, create=True) else: dev_path = EBSVol.attach_volume(existing_disk.get_volume_id(), disk_id) existing_disk.set_state(JBoxDiskState.STATE_ATTACHING) existing_disk.save() return JBoxEBSVol(dev_path, user_email=user_email)
def do_periodic_task(_mode): active_clusters = UserCluster.list_all_groupids() ParallelHousekeep.log_info("%d active clusters", len(active_clusters)) if len(active_clusters) == 0: return for cluster_id in active_clusters: sessname = UserCluster.sessname_for_cluster(cluster_id) try: sess_props = JBoxSessionProps(Compute.get_install_id(), sessname) if not sess_props.get_instance_id(): ParallelHousekeep.log_info( "Session (%s) corresponding to cluster (%s) not found. Terminating cluster.", sessname, cluster_id) ParallelHousekeep.terminate_or_delete_cluster(cluster_id) except JBoxDBItemNotFound: pass
def do_cluster_housekeeping(): JBoxEBSHousekeep.log_debug("starting cluster housekeeping") detached_disks = JBoxDiskState.get_detached_disks() time_now = datetime.datetime.now(pytz.utc) for disk_key in detached_disks: disk_info = JBoxDiskState(disk_key=disk_key) user_id = disk_info.get_user_id() sess_props = JBoxSessionProps(Compute.get_install_id(), unique_sessname(user_id)) incomplete_snapshots = [] modified = False for snap_id in disk_info.get_snapshot_ids(): if not EBSVol.is_snapshot_complete(snap_id): incomplete_snapshots.append(snap_id) continue JBoxEBSHousekeep.log_debug("updating latest snapshot of user %s to %s", user_id, snap_id) old_snap_id = sess_props.get_snapshot_id() sess_props.set_snapshot_id(snap_id) modified = True if old_snap_id is not None: EBSVol.delete_snapshot(old_snap_id) if modified: sess_props.save() disk_info.set_snapshot_ids(incomplete_snapshots) disk_info.save() if len(incomplete_snapshots) == 0: if (time_now - disk_info.get_detach_time()).total_seconds() > 24*60*60: vol_id = disk_info.get_volume_id() JBoxEBSHousekeep.log_debug("volume %s for user %s unused for too long", vol_id, user_id) disk_info.delete() EBSVol.detach_volume(vol_id, delete=True) else: JBoxEBSHousekeep.log_debug("ongoing snapshots of user %s: %r", user_id, incomplete_snapshots) JBoxEBSHousekeep.log_debug("finished cluster housekeeping")
def release(self, backup=False): sess_props = JBoxSessionProps(Compute.get_install_id(), self.sessname) existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=sess_props.get_user_id()) existing_disk.set_state(JBoxDiskState.STATE_DETACHING) existing_disk.save() disk_id = self.disk_path.split('/')[-1] if backup: snap_id = self._backup() else: snap_id = None vol_id = EBSVol.get_volume_id_from_device(disk_id) EBSVol.detach_volume(vol_id, delete=False) if snap_id is not None: existing_disk.add_snapshot_id(snap_id) existing_disk.set_state(JBoxDiskState.STATE_DETACHED) existing_disk.save()
def do_cluster_housekeeping(): JBoxEBSHousekeep.log_debug("starting cluster housekeeping") detached_disks = JBoxDiskState.get_detached_disks() time_now = datetime.datetime.now(pytz.utc) for disk_key in detached_disks: disk_info = JBoxDiskState(disk_key=disk_key) user_id = disk_info.get_user_id() sess_props = JBoxSessionProps(unique_sessname(user_id)) incomplete_snapshots = [] modified = False for snap_id in disk_info.get_snapshot_ids(): if not EBSVol.is_snapshot_complete(snap_id): incomplete_snapshots.append(snap_id) continue JBoxEBSHousekeep.log_debug("updating latest snapshot of user %s to %s", user_id, snap_id) old_snap_id = sess_props.get_snapshot_id() sess_props.set_snapshot_id(snap_id) modified = True if old_snap_id is not None: EBSVol.delete_snapshot(old_snap_id) if modified: sess_props.save() disk_info.set_snapshot_ids(incomplete_snapshots) disk_info.save() if len(incomplete_snapshots) == 0: if (time_now - disk_info.get_detach_time()).total_seconds() > 24 * 60 * 60: vol_id = disk_info.get_volume_id() JBoxEBSHousekeep.log_debug("volume %s for user %s unused for too long", vol_id, user_id) disk_info.delete() EBSVol.detach_volume(vol_id, delete=True) else: JBoxEBSHousekeep.log_debug("ongoing snapshots of user %s: %r", user_id, incomplete_snapshots) JBoxEBSHousekeep.log_debug("finished cluster housekeeping")
def get_disk_for_user(user_email): JBoxEBSVol.log_debug("creating EBS volume for %s", user_email) disk_id = JBoxEBSVol._reserve_disk_id() if disk_id is None: raise Exception("No free disk available") try: existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_email) except Exception as ex: JBoxEBSVol.log_debug("No existing disk for %s. Exception %r", user_email, ex) existing_disk = None if existing_disk is None: sess_id = unique_sessname(user_email) sess_props = JBoxSessionProps(Compute.get_install_id(), sess_id, create=True, user_id=user_email) if sess_props.is_new: sess_props.save() snap_id = sess_props.get_snapshot_id() if snap_id is None: snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id, user_email) dev_path, vol_id = EBSVol.create_new_volume(snap_id, disk_id, tag=user_email, disk_sz_gb=JBoxEBSVol.DISK_LIMIT) existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_email, volume_id=vol_id, attach_time=None, create=True) else: dev_path = EBSVol.attach_volume(existing_disk.get_volume_id(), disk_id) existing_disk.set_state(JBoxDiskState.STATE_ATTACHING) existing_disk.save() return JBoxEBSVol(dev_path, user_email=user_email)
def test(): sprops = JBoxSessionProps(TESTCLSTR, unique_sessname('*****@*****.**')) TestDBTables.log_debug("JBoxSessionProps. user_id: %s, snapshot_id: %s, message: %s", sprops.get_user_id(), sprops.get_snapshot_id(), sprops.get_message()) JBoxDynConfig.set_cluster_leader(TESTCLSTR, 'testinstance') assert JBoxDynConfig.get_cluster_leader(TESTCLSTR) == 'testinstance' JBoxDynConfig.unset_cluster_leader(TESTCLSTR) assert JBoxDynConfig.get_cluster_leader(TESTCLSTR) is None assert JBoxDynConfig.get_allow_registration(TESTCLSTR) JBoxDynConfig.set_allow_registration(TESTCLSTR, False) assert not JBoxDynConfig.get_allow_registration(TESTCLSTR) JBoxDynConfig.set_allow_registration(TESTCLSTR, True) assert JBoxDynConfig.get_allow_registration(TESTCLSTR) assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 60 JBoxDynConfig.set_registration_hourly_rate(TESTCLSTR, 20) assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 20 JBoxDynConfig.set_registration_hourly_rate(TESTCLSTR, 60) assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 60 assert JBoxDynConfig.get_message(TESTCLSTR) is None JBoxDynConfig.set_message(TESTCLSTR, "hello world", datetime.timedelta(minutes=1)) assert JBoxDynConfig.get_message(TESTCLSTR) == "hello world" JBoxDynConfig.set_user_home_image(TESTCLSTR, "juliabox-user-home-templates", "user_home_28Nov2014.tar.gz") assert JBoxDynConfig.get_user_home_image(TESTCLSTR) == ("juliabox-user-home-templates", "user_home_28Nov2014.tar.gz") num_pending_activations = JBoxUserV2.count_pending_activations() TestDBTables.log_debug("pending activations: %d", num_pending_activations) count_created = JBoxUserV2.count_created(48) TestDBTables.log_debug("accounts created in last 1 hour: %d", count_created)
def handle_if_instance_info(self, is_allowed): stats = self.get_argument('instance_info', None) if stats is None: return False if not is_allowed: AdminHandler.log_error("Show instance info not allowed for user") response = {'code': -1, 'data': 'You do not have permissions to view these stats'} else: try: if stats == 'load': result = {} # get cluster loads average_load = Compute.get_cluster_average_stats('Load') if average_load is not None: result['Average Load'] = average_load machine_loads = Compute.get_cluster_stats('Load') if machine_loads is not None: for n, v in machine_loads.iteritems(): result['Instance ' + n] = v elif stats == 'sessions': result = JBoxSessionProps.get_active_sessions(Compute.get_install_id()) elif stats == 'apis': result = JBoxInstanceProps.get_instance_status(Compute.get_install_id()) else: raise Exception("unknown command %s" % (stats,)) response = {'code': 0, 'data': result} except: AdminHandler.log_error("exception while getting stats") AdminHandler._get_logger().exception("exception while getting stats") response = {'code': -1, 'data': 'error getting stats'} self.write(response) return True
class JBoxEBSVol(JBoxVol): provides = [JBoxVol.JBP_DATA_EBS, JBoxVol.JBP_DATA] DEVICES = [] MAX_DISKS = 0 DISK_LIMIT = None DISK_USE_STATUS = {} DISK_RESERVE_TIME = {} DISK_TEMPLATE_SNAPSHOT = None LOCK = None @staticmethod def configure(): num_disks_max = JBoxCfg.get('numdisksmax') JBoxEBSVol.DISK_LIMIT = 10 JBoxEBSVol.MAX_DISKS = num_disks_max JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT = JBoxCfg.get( 'cloud_host.ebs_template') JBoxEBSVol.DEVICES = JBoxEBSVol._guess_configured_devices( 'xvd', num_disks_max) JBoxEBSVol.log_debug( "Assuming %d EBS volumes configured in range xvdba..xvdcz", len(JBoxEBSVol.DEVICES)) JBoxEBSVol.LOCK = threading.Lock() JBoxEBSVol.refresh_disk_use_status() @classmethod def get_disk_allocated_size(cls): return JBoxEBSVol.DISK_LIMIT * 1000000000 @staticmethod def _guess_configured_devices(devidpfx, num_disks): devices = [] for pfx1 in 'bc': for pfx2 in ascii_lowercase: devices.append(devidpfx + pfx1 + pfx2) if len(devices) == num_disks: return devices return devices @staticmethod def refresh_disk_use_status(container_id_list=None): JBoxEBSVol.log_debug("Refrshing EBS disk use status") JBoxEBSVol.LOCK.acquire() try: nfree = 0 for idx in range(0, JBoxEBSVol.MAX_DISKS): dev = JBoxEBSVol.DEVICES[idx] if JBoxEBSVol._is_reserved(dev): JBoxEBSVol.DISK_USE_STATUS[dev] = True else: JBoxEBSVol.DISK_USE_STATUS[dev] = False nfree += 1 for device, volume in JBoxEBSVol.get_mapped_volumes().iteritems(): JBoxEBSVol.DISK_USE_STATUS[os.path.basename(device)] = True nfree -= 1 JBoxEBSVol.log_info("EBS Disk free: " + str(nfree) + "/" + str(JBoxEBSVol.MAX_DISKS)) except: JBoxEBSVol.log_exception("Exception refrshing EBS disk use status") finally: JBoxEBSVol.LOCK.release() @staticmethod def get_mapped_volumes(): allmaps = EBSVol.get_mapped_volumes() JBoxEBSVol.log_debug("Devices mapped: %r", allmaps) return dict((d, v) for d, v in allmaps.iteritems() if os.path.basename(d) in JBoxEBSVol.DEVICES) @staticmethod def _get_unused_disk_id(): for idx in range(0, JBoxEBSVol.MAX_DISKS): dev = JBoxEBSVol.DEVICES[idx] if not JBoxEBSVol.DISK_USE_STATUS[dev]: return dev return None @staticmethod def _is_reserved(idx): if (idx in JBoxEBSVol.DISK_RESERVE_TIME) and ( JBoxEBSVol.DISK_RESERVE_TIME[idx] < time.time()): del JBoxEBSVol.DISK_RESERVE_TIME[idx] return idx in JBoxEBSVol.DISK_RESERVE_TIME @staticmethod def _mark_disk_used(idx, used=True, for_secs=0): JBoxEBSVol.DISK_USE_STATUS[idx] = used if used and (for_secs > 0): JBoxEBSVol.DISK_RESERVE_TIME[idx] = time.time() + for_secs else: if idx in JBoxEBSVol.DISK_RESERVE_TIME: del JBoxEBSVol.DISK_RESERVE_TIME[idx] @staticmethod def _reserve_disk_id(): JBoxEBSVol.LOCK.acquire() try: disk_id = JBoxEBSVol._get_unused_disk_id() JBoxEBSVol._mark_disk_used(disk_id, for_secs=120) return disk_id finally: JBoxEBSVol.LOCK.release() @staticmethod def is_mount_path(fs_path): # EBS volumes are not mounted on host return False @staticmethod def disk_ids_used_pct(): pct = (sum(JBoxEBSVol.DISK_USE_STATUS.values()) * 100) / len( JBoxEBSVol.DISK_USE_STATUS) return min(100, max(0, pct)) @staticmethod def get_disk_for_user(user_email): JBoxEBSVol.log_debug("creating EBS volume for %s", user_email) disk_id = JBoxEBSVol._reserve_disk_id() if disk_id is None: raise Exception("No free disk available") try: existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_email) except Exception, ex: JBoxEBSVol.log_debug("No existing disk for %s. Exception %r", user_email, ex) existing_disk = None if existing_disk is None: sess_id = unique_sessname(user_email) sess_props = JBoxSessionProps(sess_id, create=True, user_id=user_email) if sess_props.is_new: sess_props.save() snap_id = sess_props.get_snapshot_id() if snap_id is None: snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id, user_email) dev_path, vol_id = EBSVol.create_new_volume( snap_id, disk_id, tag=user_email, disk_sz_gb=JBoxEBSVol.DISK_LIMIT) existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_email, volume_id=vol_id, attach_time=None, create=True) else: dev_path = EBSVol.attach_volume(existing_disk.get_volume_id(), disk_id) existing_disk.set_state(JBoxDiskState.STATE_ATTACHING) existing_disk.save() return JBoxEBSVol(dev_path, user_email=user_email)