def get_disk_for_user(user_email):
JBoxEBSVol.log_debug("creating EBS volume for %s", user_email)
if not JBoxEBSVol.HAS_EBS:
raise Exception("EBS disks not enabled")
disk_id = JBoxEBSVol._reserve_disk_id()
if disk_id is None:
raise Exception("No free disk available")
try:
existing_disk = JBoxDiskState(cluster_id=CloudHost.INSTALL_ID,
region_id=CloudHost.REGION,
user_id=user_email)
except:
existing_disk = None
if existing_disk is None:
sess_id = unique_sessname(user_email)
sess_props = JBoxSessionProps(sess_id,
create=True,
user_id=user_email)
if sess_props.is_new:
sess_props.save()
snap_id = sess_props.get_snapshot_id()
if snap_id is None:
snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT
JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id,
user_email)
_dev_path, mnt_path, vol_id = CloudHost.create_new_volume(
snap_id,
disk_id,
JBoxEBSVol.FS_LOC,
tag=user_email,
disk_sz_gb=JBoxEBSVol.DISK_LIMIT)
existing_disk = JBoxDiskState(cluster_id=CloudHost.INSTALL_ID,
region_id=CloudHost.REGION,
user_id=user_email,
volume_id=vol_id,
attach_time=None,
create=True)
else:
_dev_path, mnt_path = CloudHost.attach_volume(
existing_disk.get_volume_id(), disk_id, JBoxEBSVol.FS_LOC)
existing_disk.set_attach_time()
snap_id = None
existing_disk.set_state(JBoxDiskState.STATE_ATTACHED)
existing_disk.save()
ebsvol = JBoxEBSVol(mnt_path, user_email=user_email)
if snap_id == JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT:
JBoxEBSVol.log_debug("creating home folder on blank volume for %s",
user_email)
ebsvol.restore_user_home(True)
ebsvol.restore()
else:
JBoxEBSVol.log_debug(
"updating home folder on existing volume for %s", user_email)
ebsvol.restore_user_home(False)
# snap_age_days = CloudHost.get_snapshot_age(snap_id).total_seconds()/(60*60*24)
# if snap_age_days > 7:
# ebsvol.restore_user_home()
JBoxEBSVol.log_debug(
"setting up instance configuration on disk for %s", user_email)
ebsvol.setup_instance_config()
return ebsvol
class JBoxEBSVol(JBoxVol):
DEVICES = []
MAX_DISKS = 0
FS_LOC = None
DISK_LIMIT = None
DISK_USE_STATUS = {}
DISK_RESERVE_TIME = {}
DISK_TEMPLATE_SNAPSHOT = None
HAS_EBS = False
LOCK = None
@staticmethod
def configure(disk_limit, fs_loc, max_disks, disk_template_snap_id):
JBoxEBSVol.HAS_EBS = True
JBoxEBSVol.FS_LOC = fs_loc
JBoxEBSVol.DISK_LIMIT = disk_limit
JBoxEBSVol.MAX_DISKS = max_disks
JBoxEBSVol.DEVICES = JBoxEBSVol._get_configured_devices(JBoxEBSVol.FS_LOC)
JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT = disk_template_snap_id
if len(JBoxEBSVol.DEVICES) < max_disks:
raise Exception("Not enough EBS mount points configured")
JBoxEBSVol.LOCK = threading.Lock()
JBoxEBSVol.refresh_disk_use_status()
@classmethod
def get_disk_allocated_size(cls):
return JBoxEBSVol.DISK_LIMIT * 1000000000
@staticmethod
def _id_from_device(dev_path):
return dev_path.split('/')[-1]
@staticmethod
def _get_configured_devices(fs_loc):
devices = []
with open('/etc/fstab', 'r') as fstab:
for line in fstab:
line = line.strip()
if (len(line) == 0) or line.startswith('#'):
continue
comps = line.split()
if (len(comps) == 6) and comps[1].startswith(fs_loc):
device = comps[0]
devices.append(JBoxEBSVol._id_from_device(device))
return devices
@staticmethod
def _get_disk_ids_used(cid):
used = []
props = JBoxEBSVol.dckr().inspect_container(cid)
try:
vols = props['Volumes']
for _cpath, hpath in vols.iteritems():
if hpath.startswith(JBoxEBSVol.FS_LOC):
used.append(hpath.split('/')[-1])
except:
JBoxEBSVol.log_error("error finding disk ids used in " + cid)
return []
return used
@staticmethod
def refresh_disk_use_status(container_id_list=None):
JBoxEBSVol.LOCK.acquire()
try:
nfree = 0
for idx in range(0, JBoxEBSVol.MAX_DISKS):
dev = JBoxEBSVol.DEVICES[idx]
if JBoxEBSVol._is_reserved(dev):
JBoxEBSVol.DISK_USE_STATUS[dev] = True
else:
JBoxEBSVol.DISK_USE_STATUS[dev] = False
nfree += 1
if container_id_list is None:
container_id_list = [cdesc['Id'] for cdesc in JBoxEBSVol.dckr().containers(all=True)]
for cid in container_id_list:
disk_ids = JBoxEBSVol._get_disk_ids_used(cid)
for disk_id in disk_ids:
JBoxEBSVol._mark_disk_used(disk_id)
nfree -= 1
JBoxEBSVol.log_info("Disk free: " + str(nfree) + "/" + str(JBoxEBSVol.MAX_DISKS))
finally:
JBoxEBSVol.LOCK.release()
@staticmethod
def _get_unused_disk_id():
for idx in range(0, JBoxEBSVol.MAX_DISKS):
dev = JBoxEBSVol.DEVICES[idx]
if not JBoxEBSVol.DISK_USE_STATUS[dev]:
return dev
return None
@staticmethod
def _is_reserved(idx):
if (idx in JBoxEBSVol.DISK_RESERVE_TIME) and (JBoxEBSVol.DISK_RESERVE_TIME[idx] < time.time()):
del JBoxEBSVol.DISK_RESERVE_TIME[idx]
return idx in JBoxEBSVol.DISK_RESERVE_TIME
@staticmethod
def _mark_disk_used(idx, used=True, for_secs=0):
JBoxEBSVol.DISK_USE_STATUS[idx] = used
if used and (for_secs > 0):
JBoxEBSVol.DISK_RESERVE_TIME[idx] = time.time() + for_secs
else:
if idx in JBoxEBSVol.DISK_RESERVE_TIME:
del JBoxEBSVol.DISK_RESERVE_TIME[idx]
@staticmethod
def _reserve_disk_id():
JBoxEBSVol.LOCK.acquire()
try:
disk_id = JBoxEBSVol._get_unused_disk_id()
JBoxEBSVol._mark_disk_used(disk_id, for_secs=120)
return disk_id
finally:
JBoxEBSVol.LOCK.release()
@staticmethod
def disk_ids_used_pct():
if not JBoxEBSVol.HAS_EBS:
return 0
pct = (sum(JBoxEBSVol.DISK_USE_STATUS.values()) * 100) / len(JBoxEBSVol.DISK_USE_STATUS)
return min(100, max(0, pct))
@staticmethod
def get_disk_for_user(user_email):
JBoxEBSVol.log_debug("creating EBS volume for %s", user_email)
if not JBoxEBSVol.HAS_EBS:
raise Exception("EBS disks not enabled")
disk_id = JBoxEBSVol._reserve_disk_id()
if disk_id is None:
raise Exception("No free disk available")
try:
existing_disk = JBoxDiskState(cluster_id=CloudHost.INSTALL_ID, region_id=CloudHost.REGION,
user_id=user_email)
except Exception, ex:
JBoxEBSVol.log_debug("No existing disk for %s. Exception %r", user_email, ex)
existing_disk = None
if existing_disk is None:
sess_id = unique_sessname(user_email)
sess_props = JBoxSessionProps(sess_id, create=True, user_id=user_email)
if sess_props.is_new:
sess_props.save()
snap_id = sess_props.get_snapshot_id()
if snap_id is None:
snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT
JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id, user_email)
_dev_path, mnt_path, vol_id = CloudHost.create_new_volume(snap_id, disk_id,
JBoxEBSVol.FS_LOC,
tag=user_email,
disk_sz_gb=JBoxEBSVol.DISK_LIMIT)
existing_disk = JBoxDiskState(cluster_id=CloudHost.INSTALL_ID, region_id=CloudHost.REGION,
user_id=user_email,
volume_id=vol_id,
attach_time=None,
create=True)
else:
_dev_path, mnt_path = CloudHost.attach_volume(existing_disk.get_volume_id(), disk_id, JBoxEBSVol.FS_LOC)
existing_disk.set_attach_time()
snap_id = None
existing_disk.set_state(JBoxDiskState.STATE_ATTACHED)
existing_disk.save()
ebsvol = JBoxEBSVol(mnt_path, user_email=user_email)
if snap_id == JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT:
JBoxEBSVol.log_debug("creating home folder on blank volume for %s", user_email)
ebsvol.restore_user_home(True)
ebsvol.restore()
else:
JBoxEBSVol.log_debug("updating home folder on existing volume for %s", user_email)
ebsvol.restore_user_home(False)
# snap_age_days = CloudHost.get_snapshot_age(snap_id).total_seconds()/(60*60*24)
# if snap_age_days > 7:
# ebsvol.restore_user_home()
JBoxEBSVol.log_debug("setting up instance configuration on disk for %s", user_email)
ebsvol.setup_instance_config()
return ebsvol
