def find_logged_in_instance(user_id):
sessname = unique_sessname(user_id)
try:
sess_props = JBoxSessionProps(Compute.get_install_id(), sessname)
return sess_props.get_instance_id()
except JBoxDBItemNotFound:
return None
def _backup(self, clear_volume=False):
sess_props = JBoxSessionProps(Compute.get_install_id(), self.sessname)
desc = sess_props.get_user_id() + " JuliaBox Backup"
disk_id = self.disk_path.split('/')[-1]
snap_id = EBSVol.snapshot_volume(dev_id=disk_id,
tag=self.sessname,
description=desc,
wait_till_complete=False)
return snap_id
def get_disk_for_user(user_email):
JBoxEBSVol.log_debug("creating EBS volume for %s", user_email)
disk_id = JBoxEBSVol._reserve_disk_id()
if disk_id is None:
raise Exception("No free disk available")
try:
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID,
region_id=CompEC2.REGION,
user_id=user_email)
except Exception as ex:
JBoxEBSVol.log_debug("No existing disk for %s. Exception %r",
user_email, ex)
existing_disk = None
if existing_disk is None:
sess_id = unique_sessname(user_email)
sess_props = JBoxSessionProps(Compute.get_install_id(),
sess_id,
create=True,
user_id=user_email)
if sess_props.is_new:
sess_props.save()
snap_id = sess_props.get_snapshot_id()
if snap_id is None:
snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT
JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id,
user_email)
dev_path, vol_id = EBSVol.create_new_volume(
snap_id,
disk_id,
tag=user_email,
disk_sz_gb=JBoxEBSVol.DISK_LIMIT)
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID,
region_id=CompEC2.REGION,
user_id=user_email,
volume_id=vol_id,
attach_time=None,
create=True)
else:
dev_path = EBSVol.attach_volume(existing_disk.get_volume_id(),
disk_id)
existing_disk.set_state(JBoxDiskState.STATE_ATTACHING)
existing_disk.save()
return JBoxEBSVol(dev_path, user_email=user_email)
def do_periodic_task(_mode):
active_clusters = UserCluster.list_all_groupids()
ParallelHousekeep.log_info("%d active clusters", len(active_clusters))
if len(active_clusters) == 0:
return
for cluster_id in active_clusters:
sessname = UserCluster.sessname_for_cluster(cluster_id)
try:
sess_props = JBoxSessionProps(Compute.get_install_id(), sessname)
if not sess_props.get_instance_id():
ParallelHousekeep.log_info(
"Session (%s) corresponding to cluster (%s) not found. Terminating cluster.",
sessname, cluster_id)
ParallelHousekeep.terminate_or_delete_cluster(cluster_id)
except JBoxDBItemNotFound:
pass
def do_periodic_task(_mode):
active_clusters = UserCluster.list_all_groupids()
ParallelHousekeep.log_info("%d active clusters", len(active_clusters))
if len(active_clusters) == 0:
return
for cluster_id in active_clusters:
sessname = UserCluster.sessname_for_cluster(cluster_id)
try:
sess_props = JBoxSessionProps(Compute.get_install_id(),
sessname)
if not sess_props.get_instance_id():
ParallelHousekeep.log_info(
"Session (%s) corresponding to cluster (%s) not found. Terminating cluster.",
sessname, cluster_id)
ParallelHousekeep.terminate_or_delete_cluster(cluster_id)
except JBoxDBItemNotFound:
pass
def do_cluster_housekeeping():
JBoxEBSHousekeep.log_debug("starting cluster housekeeping")
detached_disks = JBoxDiskState.get_detached_disks()
time_now = datetime.datetime.now(pytz.utc)
for disk_key in detached_disks:
disk_info = JBoxDiskState(disk_key=disk_key)
user_id = disk_info.get_user_id()
sess_props = JBoxSessionProps(Compute.get_install_id(), unique_sessname(user_id))
incomplete_snapshots = []
modified = False
for snap_id in disk_info.get_snapshot_ids():
if not EBSVol.is_snapshot_complete(snap_id):
incomplete_snapshots.append(snap_id)
continue
JBoxEBSHousekeep.log_debug("updating latest snapshot of user %s to %s", user_id, snap_id)
old_snap_id = sess_props.get_snapshot_id()
sess_props.set_snapshot_id(snap_id)
modified = True
if old_snap_id is not None:
EBSVol.delete_snapshot(old_snap_id)
if modified:
sess_props.save()
disk_info.set_snapshot_ids(incomplete_snapshots)
disk_info.save()
if len(incomplete_snapshots) == 0:
if (time_now - disk_info.get_detach_time()).total_seconds() > 24*60*60:
vol_id = disk_info.get_volume_id()
JBoxEBSHousekeep.log_debug("volume %s for user %s unused for too long", vol_id, user_id)
disk_info.delete()
EBSVol.detach_volume(vol_id, delete=True)
else:
JBoxEBSHousekeep.log_debug("ongoing snapshots of user %s: %r", user_id, incomplete_snapshots)
JBoxEBSHousekeep.log_debug("finished cluster housekeeping")
def release(self, backup=False):
sess_props = JBoxSessionProps(Compute.get_install_id(), self.sessname)
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION,
user_id=sess_props.get_user_id())
existing_disk.set_state(JBoxDiskState.STATE_DETACHING)
existing_disk.save()
disk_id = self.disk_path.split('/')[-1]
if backup:
snap_id = self._backup()
else:
snap_id = None
vol_id = EBSVol.get_volume_id_from_device(disk_id)
EBSVol.detach_volume(vol_id, delete=False)
if snap_id is not None:
existing_disk.add_snapshot_id(snap_id)
existing_disk.set_state(JBoxDiskState.STATE_DETACHED)
existing_disk.save()
def release(self, backup=False):
sess_props = JBoxSessionProps(Compute.get_install_id(), self.sessname)
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID,
region_id=CompEC2.REGION,
user_id=sess_props.get_user_id())
existing_disk.set_state(JBoxDiskState.STATE_DETACHING)
existing_disk.save()
disk_id = self.disk_path.split('/')[-1]
if backup:
snap_id = self._backup()
else:
snap_id = None
vol_id = EBSVol.get_volume_id_from_device(disk_id)
EBSVol.detach_volume(vol_id, delete=False)
if snap_id is not None:
existing_disk.add_snapshot_id(snap_id)
existing_disk.set_state(JBoxDiskState.STATE_DETACHED)
existing_disk.save()
def do_cluster_housekeeping():
JBoxEBSHousekeep.log_debug("starting cluster housekeeping")
detached_disks = JBoxDiskState.get_detached_disks()
time_now = datetime.datetime.now(pytz.utc)
for disk_key in detached_disks:
disk_info = JBoxDiskState(disk_key=disk_key)
user_id = disk_info.get_user_id()
sess_props = JBoxSessionProps(unique_sessname(user_id))
incomplete_snapshots = []
modified = False
for snap_id in disk_info.get_snapshot_ids():
if not EBSVol.is_snapshot_complete(snap_id):
incomplete_snapshots.append(snap_id)
continue
JBoxEBSHousekeep.log_debug("updating latest snapshot of user %s to %s", user_id, snap_id)
old_snap_id = sess_props.get_snapshot_id()
sess_props.set_snapshot_id(snap_id)
modified = True
if old_snap_id is not None:
EBSVol.delete_snapshot(old_snap_id)
if modified:
sess_props.save()
disk_info.set_snapshot_ids(incomplete_snapshots)
disk_info.save()
if len(incomplete_snapshots) == 0:
if (time_now - disk_info.get_detach_time()).total_seconds() > 24 * 60 * 60:
vol_id = disk_info.get_volume_id()
JBoxEBSHousekeep.log_debug("volume %s for user %s unused for too long", vol_id, user_id)
disk_info.delete()
EBSVol.detach_volume(vol_id, delete=True)
else:
JBoxEBSHousekeep.log_debug("ongoing snapshots of user %s: %r", user_id, incomplete_snapshots)
JBoxEBSHousekeep.log_debug("finished cluster housekeeping")
def get_disk_for_user(user_email):
JBoxEBSVol.log_debug("creating EBS volume for %s", user_email)
disk_id = JBoxEBSVol._reserve_disk_id()
if disk_id is None:
raise Exception("No free disk available")
try:
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION,
user_id=user_email)
except Exception as ex:
JBoxEBSVol.log_debug("No existing disk for %s. Exception %r", user_email, ex)
existing_disk = None
if existing_disk is None:
sess_id = unique_sessname(user_email)
sess_props = JBoxSessionProps(Compute.get_install_id(), sess_id, create=True, user_id=user_email)
if sess_props.is_new:
sess_props.save()
snap_id = sess_props.get_snapshot_id()
if snap_id is None:
snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT
JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id, user_email)
dev_path, vol_id = EBSVol.create_new_volume(snap_id, disk_id, tag=user_email,
disk_sz_gb=JBoxEBSVol.DISK_LIMIT)
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION,
user_id=user_email,
volume_id=vol_id,
attach_time=None,
create=True)
else:
dev_path = EBSVol.attach_volume(existing_disk.get_volume_id(), disk_id)
existing_disk.set_state(JBoxDiskState.STATE_ATTACHING)
existing_disk.save()
return JBoxEBSVol(dev_path, user_email=user_email)
def test():
sprops = JBoxSessionProps(TESTCLSTR, unique_sessname('*****@*****.**'))
TestDBTables.log_debug("JBoxSessionProps. user_id: %s, snapshot_id: %s, message: %s",
sprops.get_user_id(),
sprops.get_snapshot_id(),
sprops.get_message())
JBoxDynConfig.set_cluster_leader(TESTCLSTR, 'testinstance')
assert JBoxDynConfig.get_cluster_leader(TESTCLSTR) == 'testinstance'
JBoxDynConfig.unset_cluster_leader(TESTCLSTR)
assert JBoxDynConfig.get_cluster_leader(TESTCLSTR) is None
assert JBoxDynConfig.get_allow_registration(TESTCLSTR)
JBoxDynConfig.set_allow_registration(TESTCLSTR, False)
assert not JBoxDynConfig.get_allow_registration(TESTCLSTR)
JBoxDynConfig.set_allow_registration(TESTCLSTR, True)
assert JBoxDynConfig.get_allow_registration(TESTCLSTR)
assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 60
JBoxDynConfig.set_registration_hourly_rate(TESTCLSTR, 20)
assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 20
JBoxDynConfig.set_registration_hourly_rate(TESTCLSTR, 60)
assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 60
assert JBoxDynConfig.get_message(TESTCLSTR) is None
JBoxDynConfig.set_message(TESTCLSTR, "hello world", datetime.timedelta(minutes=1))
assert JBoxDynConfig.get_message(TESTCLSTR) == "hello world"
JBoxDynConfig.set_user_home_image(TESTCLSTR, "juliabox-user-home-templates", "user_home_28Nov2014.tar.gz")
assert JBoxDynConfig.get_user_home_image(TESTCLSTR) == ("juliabox-user-home-templates",
"user_home_28Nov2014.tar.gz")
num_pending_activations = JBoxUserV2.count_pending_activations()
TestDBTables.log_debug("pending activations: %d", num_pending_activations)
count_created = JBoxUserV2.count_created(48)
TestDBTables.log_debug("accounts created in last 1 hour: %d", count_created)
def handle_if_instance_info(self, is_allowed):
stats = self.get_argument('instance_info', None)
if stats is None:
return False
if not is_allowed:
AdminHandler.log_error("Show instance info not allowed for user")
response = {'code': -1, 'data': 'You do not have permissions to view these stats'}
else:
try:
if stats == 'load':
result = {}
# get cluster loads
average_load = Compute.get_cluster_average_stats('Load')
if average_load is not None:
result['Average Load'] = average_load
machine_loads = Compute.get_cluster_stats('Load')
if machine_loads is not None:
for n, v in machine_loads.iteritems():
result['Instance ' + n] = v
elif stats == 'sessions':
result = JBoxSessionProps.get_active_sessions(Compute.get_install_id())
elif stats == 'apis':
result = JBoxInstanceProps.get_instance_status(Compute.get_install_id())
else:
raise Exception("unknown command %s" % (stats,))
response = {'code': 0, 'data': result}
except:
AdminHandler.log_error("exception while getting stats")
AdminHandler._get_logger().exception("exception while getting stats")
response = {'code': -1, 'data': 'error getting stats'}
self.write(response)
return True
def _backup(self, clear_volume=False):
sess_props = JBoxSessionProps(Compute.get_install_id(), self.sessname)
desc = sess_props.get_user_id() + " JuliaBox Backup"
disk_id = self.disk_path.split('/')[-1]
snap_id = EBSVol.snapshot_volume(dev_id=disk_id, tag=self.sessname, description=desc, wait_till_complete=False)
return snap_id
class JBoxEBSVol(JBoxVol):
provides = [JBoxVol.JBP_DATA_EBS, JBoxVol.JBP_DATA]
DEVICES = []
MAX_DISKS = 0
DISK_LIMIT = None
DISK_USE_STATUS = {}
DISK_RESERVE_TIME = {}
DISK_TEMPLATE_SNAPSHOT = None
LOCK = None
@staticmethod
def configure():
num_disks_max = JBoxCfg.get('numdisksmax')
JBoxEBSVol.DISK_LIMIT = 10
JBoxEBSVol.MAX_DISKS = num_disks_max
JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT = JBoxCfg.get(
'cloud_host.ebs_template')
JBoxEBSVol.DEVICES = JBoxEBSVol._guess_configured_devices(
'xvd', num_disks_max)
JBoxEBSVol.log_debug(
"Assuming %d EBS volumes configured in range xvdba..xvdcz",
len(JBoxEBSVol.DEVICES))
JBoxEBSVol.LOCK = threading.Lock()
JBoxEBSVol.refresh_disk_use_status()
@classmethod
def get_disk_allocated_size(cls):
return JBoxEBSVol.DISK_LIMIT * 1000000000
@staticmethod
def _guess_configured_devices(devidpfx, num_disks):
devices = []
for pfx1 in 'bc':
for pfx2 in ascii_lowercase:
devices.append(devidpfx + pfx1 + pfx2)
if len(devices) == num_disks:
return devices
return devices
@staticmethod
def refresh_disk_use_status(container_id_list=None):
JBoxEBSVol.log_debug("Refrshing EBS disk use status")
JBoxEBSVol.LOCK.acquire()
try:
nfree = 0
for idx in range(0, JBoxEBSVol.MAX_DISKS):
dev = JBoxEBSVol.DEVICES[idx]
if JBoxEBSVol._is_reserved(dev):
JBoxEBSVol.DISK_USE_STATUS[dev] = True
else:
JBoxEBSVol.DISK_USE_STATUS[dev] = False
nfree += 1
for device, volume in JBoxEBSVol.get_mapped_volumes().iteritems():
JBoxEBSVol.DISK_USE_STATUS[os.path.basename(device)] = True
nfree -= 1
JBoxEBSVol.log_info("EBS Disk free: " + str(nfree) + "/" +
str(JBoxEBSVol.MAX_DISKS))
except:
JBoxEBSVol.log_exception("Exception refrshing EBS disk use status")
finally:
JBoxEBSVol.LOCK.release()
@staticmethod
def get_mapped_volumes():
allmaps = EBSVol.get_mapped_volumes()
JBoxEBSVol.log_debug("Devices mapped: %r", allmaps)
return dict((d, v) for d, v in allmaps.iteritems()
if os.path.basename(d) in JBoxEBSVol.DEVICES)
@staticmethod
def _get_unused_disk_id():
for idx in range(0, JBoxEBSVol.MAX_DISKS):
dev = JBoxEBSVol.DEVICES[idx]
if not JBoxEBSVol.DISK_USE_STATUS[dev]:
return dev
return None
@staticmethod
def _is_reserved(idx):
if (idx in JBoxEBSVol.DISK_RESERVE_TIME) and (
JBoxEBSVol.DISK_RESERVE_TIME[idx] < time.time()):
del JBoxEBSVol.DISK_RESERVE_TIME[idx]
return idx in JBoxEBSVol.DISK_RESERVE_TIME
@staticmethod
def _mark_disk_used(idx, used=True, for_secs=0):
JBoxEBSVol.DISK_USE_STATUS[idx] = used
if used and (for_secs > 0):
JBoxEBSVol.DISK_RESERVE_TIME[idx] = time.time() + for_secs
else:
if idx in JBoxEBSVol.DISK_RESERVE_TIME:
del JBoxEBSVol.DISK_RESERVE_TIME[idx]
@staticmethod
def _reserve_disk_id():
JBoxEBSVol.LOCK.acquire()
try:
disk_id = JBoxEBSVol._get_unused_disk_id()
JBoxEBSVol._mark_disk_used(disk_id, for_secs=120)
return disk_id
finally:
JBoxEBSVol.LOCK.release()
@staticmethod
def is_mount_path(fs_path):
# EBS volumes are not mounted on host
return False
@staticmethod
def disk_ids_used_pct():
pct = (sum(JBoxEBSVol.DISK_USE_STATUS.values()) * 100) / len(
JBoxEBSVol.DISK_USE_STATUS)
return min(100, max(0, pct))
@staticmethod
def get_disk_for_user(user_email):
JBoxEBSVol.log_debug("creating EBS volume for %s", user_email)
disk_id = JBoxEBSVol._reserve_disk_id()
if disk_id is None:
raise Exception("No free disk available")
try:
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID,
region_id=CompEC2.REGION,
user_id=user_email)
except Exception, ex:
JBoxEBSVol.log_debug("No existing disk for %s. Exception %r",
user_email, ex)
existing_disk = None
if existing_disk is None:
sess_id = unique_sessname(user_email)
sess_props = JBoxSessionProps(sess_id,
create=True,
user_id=user_email)
if sess_props.is_new:
sess_props.save()
snap_id = sess_props.get_snapshot_id()
if snap_id is None:
snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT
JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id,
user_email)
dev_path, vol_id = EBSVol.create_new_volume(
snap_id,
disk_id,
tag=user_email,
disk_sz_gb=JBoxEBSVol.DISK_LIMIT)
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID,
region_id=CompEC2.REGION,
user_id=user_email,
volume_id=vol_id,
attach_time=None,
create=True)
else:
dev_path = EBSVol.attach_volume(existing_disk.get_volume_id(),
disk_id)
existing_disk.set_state(JBoxDiskState.STATE_ATTACHING)
existing_disk.save()
return JBoxEBSVol(dev_path, user_email=user_email)