def push_backup(user_email, disk_path):
sessname = unique_sessname(user_email)
S3Disk.log_info("pushing %s.tar.gz from %s to %s", sessname, disk_path,
JBoxVol.BACKUP_BUCKET)
bkup_file = os.path.join('/tmp', sessname + ".tar.gz")
bkup_tar = tarfile.open(bkup_file, 'w:gz')
def set_perms(tinfo):
tinfo.uid = 1000
tinfo.gid = 1000
tinfo.uname = 'ubuntu'
tinfo.gname = 'ubuntu'
return tinfo
for f in os.listdir(disk_path):
if f.startswith('.') and (f in ['.julia', '.juliabox']):
continue
full_path = os.path.join(disk_path, f)
bkup_tar.add(full_path, os.path.join('juser', f), filter=set_perms)
bkup_tar.close()
os.chmod(bkup_file, 0666)
# Upload to S3 if so configured. Delete from local if successful.
bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \
datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET)
if JBoxVol.BACKUP_BUCKET is not None:
if CloudHost.push_file_to_s3(
JBoxVol.BACKUP_BUCKET,
bkup_file,
metadata={'backup_time':
bkup_file_mtime.isoformat()}) is not None:
os.remove(bkup_file)
S3Disk.log_info("Moved backup to S3 " + sessname)
def calc_stat(user_email):
VolMgr.STATS['num_users'] += 1
sessname = unique_sessname(user_email)
k = CloudHost.pull_file_from_s3(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz", metadata_only=True)
if k is not None:
VolMgr.STATS['loopback']['sizes'].append(k.size)
def __init__(self,
disk_path,
user_email=None,
user_name=None,
sessname=None,
old_sessname=None):
self.disk_path = disk_path
self.user_email = user_email
if user_name is not None:
self.user_name = user_name
elif user_email is not None:
self.user_name = get_user_name(user_email)
else:
self.user_name = None
if sessname is not None:
self.sessname = sessname
elif user_email is not None:
self.sessname = unique_sessname(user_email)
else:
self.sessname = None
if old_sessname is not None:
self.old_sessname = old_sessname
elif user_email is not None:
self.old_sessname = esc_sessname(user_email)
else:
self.old_sessname = None
self._dbg_str = str(self.sessname) + "(" + self.disk_path + ")"
def pull_backup(user_email):
sessname = unique_sessname(user_email)
S3Disk.log_info("pulling %s.tar.gz from %s", sessname,
JBoxVol.BACKUP_BUCKET)
CloudHost.pull_file_from_s3(JBoxVol.BACKUP_BUCKET,
sessname + ".tar.gz",
metadata_only=False)
def restore(self):
sessname = unique_sessname(self.user_email)
old_sessname = esc_sessname(self.user_email)
src = os.path.join(JBoxVol.BACKUP_LOC, sessname + ".tar.gz")
k = JBoxVol.pull_from_s3(src) # download from S3 if exists
if not os.path.exists(src):
if old_sessname is not None:
src = os.path.join(JBoxVol.BACKUP_LOC, old_sessname + ".tar.gz")
k = JBoxVol.pull_from_s3(src) # download from S3 if exists
if not os.path.exists(src):
return
JBoxVol.log_info("Filtering out restore info from backup " + src + " to " + self.disk_path)
src_tar = tarfile.open(src, 'r:gz')
try:
for info in src_tar.getmembers():
if not info.name.startswith('juser/'):
continue
if info.name.startswith('juser/.') and (info.name.split('/')[1] in ['.juliabox', '.julia', '.ipython']):
continue
info.name = info.name[6:]
if len(info.name) == 0:
continue
src_tar.extract(info, self.disk_path)
JBoxVol.log_info("Restored backup at " + self.disk_path)
except IOError, ioe:
if ioe.errno == errno.ENOSPC:
# continue login on ENOSPC to allow user to delete files
JBoxVol.log_exception("No space left to restore backup for %s", sessname)
else:
raise
def __init__(self, disk_path, user_email=None, user_name=None, sessname=None, old_sessname=None):
self.disk_path = disk_path
self.user_email = user_email
if user_name is not None:
self.user_name = user_name
elif user_email is not None:
self.user_name = get_user_name(user_email)
else:
self.user_name = None
if sessname is not None:
self.sessname = sessname
elif user_email is not None:
self.sessname = unique_sessname(user_email)
else:
self.sessname = None
if old_sessname is not None:
self.old_sessname = old_sessname
elif user_email is not None:
self.old_sessname = esc_sessname(user_email)
else:
self.old_sessname = None
self._dbg_str = str(self.sessname) + "(" + self.disk_path + ")"
def push_backup(user_email, disk_path):
sessname = unique_sessname(user_email)
S3Disk.log_info("pushing %s.tar.gz from %s to %s", sessname, disk_path, JBoxVol.BACKUP_BUCKET)
bkup_file = os.path.join('/tmp', sessname + ".tar.gz")
bkup_tar = tarfile.open(bkup_file, 'w:gz')
def set_perms(tinfo):
tinfo.uid = 1000
tinfo.gid = 1000
tinfo.uname = 'ubuntu'
tinfo.gname = 'ubuntu'
return tinfo
for f in os.listdir(disk_path):
if f.startswith('.') and (f in ['.julia', '.juliabox']):
continue
full_path = os.path.join(disk_path, f)
bkup_tar.add(full_path, os.path.join('juser', f), filter=set_perms)
bkup_tar.close()
os.chmod(bkup_file, 0666)
# Upload to S3 if so configured. Delete from local if successful.
bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \
datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET)
if JBoxVol.BACKUP_BUCKET is not None:
if CloudHost.push_file_to_s3(JBoxVol.BACKUP_BUCKET, bkup_file,
metadata={'backup_time': bkup_file_mtime.isoformat()}) is not None:
os.remove(bkup_file)
S3Disk.log_info("Moved backup to S3 " + sessname)
def get_disk_for_user(user_email):
JBoxEBSVol.log_debug("creating EBS volume for %s", user_email)
if not JBoxEBSVol.HAS_EBS:
raise Exception("EBS disks not enabled")
disk_id = JBoxEBSVol._reserve_disk_id()
if disk_id is None:
raise Exception("No free disk available")
sess_id = unique_sessname(user_email)
sess_props = JBoxSessionProps(sess_id, create=True, user_id=user_email)
if sess_props.is_new:
sess_props.save()
snap_id = sess_props.get_snapshot_id()
if snap_id is None:
snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT
JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id, user_email)
_dev_path, mnt_path = CloudHost.create_new_volume(snap_id, disk_id, JBoxEBSVol.FS_LOC, tag=user_email)
ebsvol = JBoxEBSVol(mnt_path, user_email=user_email)
if snap_id == JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT:
JBoxEBSVol.log_debug("creating home folder on blank volume for %s", user_email)
ebsvol.restore_user_home()
ebsvol.restore()
#else:
# snap_age_days = CloudHost.get_snapshot_age(snap_id).total_seconds()/(60*60*24)
# if snap_age_days > 7:
# ebsvol.restore_user_home()
JBoxEBSVol.log_debug("setting up instance configuration on disk for %s", user_email)
ebsvol.setup_instance_config()
return ebsvol
def calc_stat(user_email):
VolMgr.STATS['num_users'] += 1
sessname = unique_sessname(user_email)
k = CloudHost.pull_file_from_s3(JBoxVol.BACKUP_BUCKET,
sessname + ".tar.gz",
metadata_only=True)
if k is not None:
VolMgr.STATS['loopback']['sizes'].append(k.size)
def set_loading_state(self, user_id):
sessname = unique_sessname(user_id)
sign = signstr(sessname + '000', self.config("sesskey"))
self.set_container_cookies({
"sessname": sessname,
"hostshell": 0,
"hostupload": 0,
"hostipnb": 0,
"loading": 1,
"sign": sign
})
self.set_lb_tracker_cookie()
def do_monitor_loading(self, user_id):
sessname = unique_sessname(user_id)
self.log_debug("Monitoring loading of session [%s] user[%s]...", sessname, user_id)
cont = JBoxContainer.get_by_name(sessname)
if (cont is None) or (not cont.is_running()):
loading_step = int(self.get_cookie("loading", 0))
if loading_step > 30:
self.log_error("Could not start instance. Session [%s] for user [%s] didn't load.", sessname, user_id)
self.clear_container_cookies()
self.rendertpl("index.tpl", cfg=self.config(),
state=self.state(
error='Could not start your instance! Please try again.',
pending_activation=False,
user_id=user_id))
return
else:
loading_step += 1
self.set_cookie("loading", str(loading_step))
self.rendertpl("loading.tpl", user_id=user_id)
else:
if self.config("gauth"):
jbuser = JBoxUserV2(user_id)
creds = jbuser.get_gtok()
if creds is not None:
try:
creds_json = json.loads(base64.b64decode(creds))
creds_json = self.renew_creds(creds_json)
authtok = creds_json['access_token']
except:
self.log_warn("stale stored creds. will renew on next use. user: "******"sesskey"))
self.clear_cookie("loading")
self.set_container_cookies({
"sessname": sessname,
"hostshell": shellport,
"hostupload": uplport,
"hostipnb": ipnbport,
"sign": sign
})
self.set_lb_tracker_cookie()
self.rendertpl("ipnbsess.tpl", sessname=sessname, cfg=self.config(), creds=creds, authtok=authtok,
user_id=user_id)
def set_loading_state(self, user_id):
sessname = unique_sessname(user_id)
sign = signstr(sessname + '000', self.config("sesskey"))
self.set_container_cookies({
"sessname": sessname,
"hostshell": 0,
"hostupload": 0,
"hostipnb": 0,
"loading": 1,
"sign": sign
})
self.set_lb_tracker_cookie()
def restore(self):
sessname = unique_sessname(self.user_email)
old_sessname = esc_sessname(self.user_email)
src = os.path.join(JBoxVol.BACKUP_LOC, sessname + ".tar.gz")
k = JBoxVol.pull_from_s3(src) # download from S3 if exists
if not os.path.exists(src):
if old_sessname is not None:
src = os.path.join(JBoxVol.BACKUP_LOC,
old_sessname + ".tar.gz")
k = JBoxVol.pull_from_s3(src) # download from S3 if exists
if not os.path.exists(src):
return
JBoxVol.log_info("Filtering out restore info from backup " + src +
" to " + self.disk_path)
src_tar = tarfile.open(src, 'r:gz')
try:
perms = {}
for info in src_tar.getmembers():
if not info.name.startswith('juser/'):
continue
extract_name = info.name[6:]
if info.name.startswith('juser/.'):
folder = info.name.split('/')[1]
if (
folder == '.julia'
) or JBoxVol._is_path_user_home_essential(extract_name):
continue
info.name = extract_name
if len(info.name) == 0:
continue
src_tar.extract(info, self.disk_path)
extracted_path = os.path.join(self.disk_path, extract_name)
if os.path.isdir(extracted_path) and not os.access(
extracted_path, os.W_OK):
st = os.stat(extracted_path)
perms[extracted_path] = st
os.chmod(extracted_path, st.st_mode | stat.S_IWRITE)
if len(perms) > 0:
JBoxVol.log_debug("resetting permissions on %d folders",
len(perms))
for extracted_path, perm in perms.iteritems():
os.chmod(extracted_path, perm)
JBoxVol.log_info("Restored backup at " + self.disk_path)
except IOError, ioe:
if ioe.errno == errno.ENOSPC:
# continue login on ENOSPC to allow user to delete files
JBoxVol.log_exception("No space left to restore backup for %s",
sessname)
else:
raise
def test():
yday = datetime.datetime.now() - datetime.timedelta(hours=24)
stats = JBoxAccountingV2.get_stats(dates=(yday, ))
TestDBTables.log_debug("stats for yesterday: %s", repr(stats))
stats = JBoxAccountingV2.get_stats()
TestDBTables.log_debug("stats for today: %s", repr(stats))
sprops = JBoxSessionProps(unique_sessname('*****@*****.**'))
TestDBTables.log_debug(
"JBoxSessionProps. user_id: %s, snapshot_id: %s, message: %s",
sprops.get_user_id(), sprops.get_snapshot_id(),
sprops.get_message())
JBoxDynConfig.set_cluster_leader(TESTCLSTR, 'testinstance')
assert JBoxDynConfig.get_cluster_leader(TESTCLSTR) == 'testinstance'
JBoxDynConfig.unset_cluster_leader(TESTCLSTR)
assert JBoxDynConfig.get_cluster_leader(TESTCLSTR) is None
assert JBoxDynConfig.get_allow_registration(TESTCLSTR)
JBoxDynConfig.set_allow_registration(TESTCLSTR, False)
assert not JBoxDynConfig.get_allow_registration(TESTCLSTR)
JBoxDynConfig.set_allow_registration(TESTCLSTR, True)
assert JBoxDynConfig.get_allow_registration(TESTCLSTR)
assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 60
JBoxDynConfig.set_registration_hourly_rate(TESTCLSTR, 20)
assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 20
JBoxDynConfig.set_registration_hourly_rate(TESTCLSTR, 60)
assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 60
assert JBoxDynConfig.get_message(TESTCLSTR) is None
JBoxDynConfig.set_message(TESTCLSTR, "hello world",
datetime.timedelta(minutes=1))
assert JBoxDynConfig.get_message(TESTCLSTR) == "hello world"
JBoxDynConfig.set_user_home_image(TESTCLSTR,
"juliabox-user-home-templates",
"user_home_28Nov2014.tar.gz")
assert JBoxDynConfig.get_user_home_image(TESTCLSTR) == (
"juliabox-user-home-templates", "user_home_28Nov2014.tar.gz")
num_pending_activations = JBoxUserV2.count_pending_activations()
TestDBTables.log_debug("pending activations: %d",
num_pending_activations)
resultset = JBoxInvite.table().scan()
result_arr = [obj for obj in resultset]
TestDBTables.log_debug("got array: %r", result_arr)
count_created = JBoxUserV2.count_created(48)
TestDBTables.log_debug("accounts created in last 1 hour: %d",
count_created)
def try_launch_container(cls, user_id):
sessname = unique_sessname(user_id)
cont = JBoxContainer.get_by_name(sessname)
cls.log_debug("have existing container for %s: %r", sessname, None != cont)
if cont is not None:
cls.log_debug("container running: %r", cont.is_running())
if ((cont is None) or (not cont.is_running())) and (not CloudHost.should_accept_session()):
if cont is not None:
cont.async_backup_and_cleanup()
return False
JBoxContainer.async_launch_by_name(sessname, user_id, True)
return True
def do_monitor_loading_ajax(self, user_id):
sessname = unique_sessname(user_id)
self.log_debug("AJAX monitoring loading of session [%s] user[%s]...", sessname, user_id)
cont = JBoxContainer.get_by_name(sessname)
if (cont is None) or (not cont.is_running()):
loading_step = int(self.get_cookie("loading", 0))
if loading_step > 30:
self.log_error("Could not start instance. Session [%s] for user [%s] didn't load.", sessname, user_id)
self.write({'code': -1})
return
loading_step += 1
self.set_cookie("loading", str(loading_step))
self.write({'code': 0})
else:
self.write({'code': 1})
def do_monitor_loading_ajax(self, user_id):
sessname = unique_sessname(user_id)
self.log_debug("AJAX monitoring loading of session [%s] user[%s]...",
sessname, user_id)
cont = JBoxContainer.get_by_name(sessname)
if (cont is None) or (not cont.is_running()):
loading_step = int(self.get_cookie("loading", 0))
if loading_step > 30:
self.write({'code': -1})
return
loading_step += 1
self.set_cookie("loading", str(loading_step))
self.write({'code': 0})
else:
self.write({'code': 1})
def test():
yday = datetime.datetime.now() - datetime.timedelta(hours=24)
stats = JBoxAccountingV2.get_stats(dates=(yday,))
TestDBTables.log_debug("stats for yesterday: %s", repr(stats))
stats = JBoxAccountingV2.get_stats()
TestDBTables.log_debug("stats for today: %s", repr(stats))
sprops = JBoxSessionProps(unique_sessname('*****@*****.**'))
TestDBTables.log_debug("JBoxSessionProps. user_id: %s, snapshot_id: %s, message: %s",
sprops.get_user_id(),
sprops.get_snapshot_id(),
sprops.get_message())
JBoxDynConfig.set_cluster_leader(TESTCLSTR, 'testinstance')
assert JBoxDynConfig.get_cluster_leader(TESTCLSTR) == 'testinstance'
JBoxDynConfig.unset_cluster_leader(TESTCLSTR)
assert JBoxDynConfig.get_cluster_leader(TESTCLSTR) is None
assert JBoxDynConfig.get_allow_registration(TESTCLSTR)
JBoxDynConfig.set_allow_registration(TESTCLSTR, False)
assert not JBoxDynConfig.get_allow_registration(TESTCLSTR)
JBoxDynConfig.set_allow_registration(TESTCLSTR, True)
assert JBoxDynConfig.get_allow_registration(TESTCLSTR)
assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 60
JBoxDynConfig.set_registration_hourly_rate(TESTCLSTR, 20)
assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 20
JBoxDynConfig.set_registration_hourly_rate(TESTCLSTR, 60)
assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 60
assert JBoxDynConfig.get_message(TESTCLSTR) is None
JBoxDynConfig.set_message(TESTCLSTR, "hello world", datetime.timedelta(minutes=1))
assert JBoxDynConfig.get_message(TESTCLSTR) == "hello world"
JBoxDynConfig.set_user_home_image(TESTCLSTR, "juliabox-user-home-templates", "user_home_28Nov2014.tar.gz")
assert JBoxDynConfig.get_user_home_image(TESTCLSTR) == ("juliabox-user-home-templates",
"user_home_28Nov2014.tar.gz")
num_pending_activations = JBoxUserV2.count_pending_activations()
TestDBTables.log_debug("pending activations: %d", num_pending_activations)
resultset = JBoxInvite.table().scan()
result_arr = [obj for obj in resultset]
TestDBTables.log_debug("got array: %r", result_arr)
count_created = JBoxUserV2.count_created(48)
TestDBTables.log_debug("accounts created in last 1 hour: %d", count_created)
def restore(self):
sessname = unique_sessname(self.user_email)
old_sessname = esc_sessname(self.user_email)
src = os.path.join(JBoxVol.BACKUP_LOC, sessname + ".tar.gz")
k = JBoxVol.pull_from_s3(src) # download from S3 if exists
if not os.path.exists(src):
if old_sessname is not None:
src = os.path.join(JBoxVol.BACKUP_LOC, old_sessname + ".tar.gz")
k = JBoxVol.pull_from_s3(src) # download from S3 if exists
if not os.path.exists(src):
return
JBoxVol.log_info("Filtering out restore info from backup " + src + " to " + self.disk_path)
src_tar = tarfile.open(src, 'r:gz')
try:
perms = {}
for info in src_tar.getmembers():
if not info.name.startswith('juser/'):
continue
extract_name = info.name[6:]
if info.name.startswith('juser/.'):
folder = info.name.split('/')[1]
if (folder == '.julia') or JBoxVol._is_path_user_home_essential(extract_name):
continue
info.name = extract_name
if len(info.name) == 0:
continue
src_tar.extract(info, self.disk_path)
extracted_path = os.path.join(self.disk_path, extract_name)
if os.path.isdir(extracted_path) and not os.access(extracted_path, os.W_OK):
st = os.stat(extracted_path)
perms[extracted_path] = st
os.chmod(extracted_path, st.st_mode | stat.S_IWRITE)
if len(perms) > 0:
JBoxVol.log_debug("resetting permissions on %d folders", len(perms))
for extracted_path, perm in perms.iteritems():
os.chmod(extracted_path, perm)
JBoxVol.log_info("Restored backup at " + self.disk_path)
except IOError, ioe:
if ioe.errno == errno.ENOSPC:
# continue login on ENOSPC to allow user to delete files
JBoxVol.log_exception("No space left to restore backup for %s", sessname)
else:
raise
def try_launch_container(cls, user_id):
sessname = unique_sessname(user_id)
cont = JBoxContainer.get_by_name(sessname)
cls.log_debug("have existing container for %s: %r", sessname,
None != cont)
if cont is not None:
cls.log_debug("container running: %r", cont.is_running())
is_leader = is_proposed_cluster_leader()
if ((cont is None) or (not cont.is_running())) and (
not CloudHost.should_accept_session(is_leader)):
if cont is not None:
cont.async_backup_and_cleanup()
return False
JBoxContainer.async_launch_by_name(sessname, user_id, True)
return True
def get_disk_for_user(user_email):
JBoxEBSVol.log_debug("creating EBS volume for %s", user_email)
if not JBoxEBSVol.HAS_EBS:
raise Exception("EBS disks not enabled")
disk_id = JBoxEBSVol._reserve_disk_id()
if disk_id is None:
raise Exception("No free disk available")
sess_id = unique_sessname(user_email)
sess_props = JBoxSessionProps(sess_id, create=True, user_id=user_email)
if sess_props.is_new:
sess_props.save()
snap_id = sess_props.get_snapshot_id()
if snap_id is None:
snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT
JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id,
user_email)
_dev_path, mnt_path = CloudHost.create_new_volume(snap_id,
disk_id,
JBoxEBSVol.FS_LOC,
tag=user_email)
ebsvol = JBoxEBSVol(mnt_path, user_email=user_email)
if snap_id == JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT:
JBoxEBSVol.log_debug("creating home folder on blank volume for %s",
user_email)
ebsvol.restore_user_home(True)
ebsvol.restore()
else:
JBoxEBSVol.log_debug(
"updating home folder on existing volume for %s", user_email)
ebsvol.restore_user_home(False)
# snap_age_days = CloudHost.get_snapshot_age(snap_id).total_seconds()/(60*60*24)
# if snap_age_days > 7:
# ebsvol.restore_user_home()
JBoxEBSVol.log_debug(
"setting up instance configuration on disk for %s", user_email)
ebsvol.setup_instance_config()
return ebsvol
def restore(self):
sessname = unique_sessname(self.user_email)
old_sessname = esc_sessname(self.user_email)
src = os.path.join(JBoxVol.BACKUP_LOC, sessname + ".tar.gz")
k = JBoxVol.pull_from_s3(src) # download from S3 if exists
if not os.path.exists(src):
if old_sessname is not None:
src = os.path.join(JBoxVol.BACKUP_LOC,
old_sessname + ".tar.gz")
k = JBoxVol.pull_from_s3(src) # download from S3 if exists
if not os.path.exists(src):
return
JBoxVol.log_info("Filtering out restore info from backup " + src +
" to " + self.disk_path)
src_tar = tarfile.open(src, 'r:gz')
try:
for info in src_tar.getmembers():
if not info.name.startswith('juser/'):
continue
extract_name = info.name[6:]
if info.name.startswith('juser/.'):
folder = info.name.split('/')[1]
if (
folder == '.julia'
) or JBoxVol._is_path_user_home_essential(extract_name):
continue
info.name = extract_name
if len(info.name) == 0:
continue
src_tar.extract(info, self.disk_path)
JBoxVol.log_info("Restored backup at " + self.disk_path)
except IOError, ioe:
if ioe.errno == errno.ENOSPC:
# continue login on ENOSPC to allow user to delete files
JBoxVol.log_exception("No space left to restore backup for %s",
sessname)
else:
raise
def try_launch_container(cls, user_id, max_hop=False):
sessname = unique_sessname(user_id)
cont = JBoxContainer.get_by_name(sessname)
cls.log_debug("have existing container for %s: %r", sessname, None != cont)
if cont is not None:
cls.log_debug("container running: %r", cont.is_running())
if max_hop:
self_load = CloudHost.get_instance_stats(CloudHost.instance_id(), 'Load')
if self_load < 100:
JBoxContainer.async_launch_by_name(sessname, user_id, True)
return True
is_leader = is_proposed_cluster_leader()
if ((cont is None) or (not cont.is_running())) and (not CloudHost.should_accept_session(is_leader)):
if cont is not None:
cont.async_backup_and_cleanup()
return False
JBoxContainer.async_launch_by_name(sessname, user_id, True)
return True
def test():
sprops = JBoxSessionProps(TESTCLSTR, unique_sessname('*****@*****.**'))
TestDBTables.log_debug("JBoxSessionProps. user_id: %s, snapshot_id: %s, message: %s",
sprops.get_user_id(),
sprops.get_snapshot_id(),
sprops.get_message())
JBoxDynConfig.set_cluster_leader(TESTCLSTR, 'testinstance')
assert JBoxDynConfig.get_cluster_leader(TESTCLSTR) == 'testinstance'
JBoxDynConfig.unset_cluster_leader(TESTCLSTR)
assert JBoxDynConfig.get_cluster_leader(TESTCLSTR) is None
assert JBoxDynConfig.get_allow_registration(TESTCLSTR)
JBoxDynConfig.set_allow_registration(TESTCLSTR, False)
assert not JBoxDynConfig.get_allow_registration(TESTCLSTR)
JBoxDynConfig.set_allow_registration(TESTCLSTR, True)
assert JBoxDynConfig.get_allow_registration(TESTCLSTR)
assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 60
JBoxDynConfig.set_registration_hourly_rate(TESTCLSTR, 20)
assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 20
JBoxDynConfig.set_registration_hourly_rate(TESTCLSTR, 60)
assert JBoxDynConfig.get_registration_hourly_rate(TESTCLSTR) == 60
assert JBoxDynConfig.get_message(TESTCLSTR) is None
JBoxDynConfig.set_message(TESTCLSTR, "hello world", datetime.timedelta(minutes=1))
assert JBoxDynConfig.get_message(TESTCLSTR) == "hello world"
JBoxDynConfig.set_user_home_image(TESTCLSTR, "juliabox-user-home-templates", "user_home_28Nov2014.tar.gz")
assert JBoxDynConfig.get_user_home_image(TESTCLSTR) == ("juliabox-user-home-templates",
"user_home_28Nov2014.tar.gz")
num_pending_activations = JBoxUserV2.count_pending_activations()
TestDBTables.log_debug("pending activations: %d", num_pending_activations)
count_created = JBoxUserV2.count_created(48)
TestDBTables.log_debug("accounts created in last 1 hour: %d", count_created)
def update_disk_states():
detached_disks = JBoxDiskState.get_detached_disks()
time_now = datetime.datetime.now(pytz.utc)
for disk_key in detached_disks:
disk_info = JBoxDiskState(disk_key=disk_key)
user_id = disk_info.get_user_id()
sess_props = JBoxSessionProps(unique_sessname(user_id))
incomplete_snapshots = []
modified = False
for snap_id in disk_info.get_snapshot_ids():
if not CloudHost.is_snapshot_complete(snap_id):
incomplete_snapshots.append(snap_id)
continue
JBoxd.log_debug("updating latest snapshot of user %s to %s",
user_id, snap_id)
old_snap_id = sess_props.get_snapshot_id()
sess_props.set_snapshot_id(snap_id)
modified = True
if old_snap_id is not None:
CloudHost.delete_snapshot(old_snap_id)
if modified:
sess_props.save()
disk_info.set_snapshot_ids(incomplete_snapshots)
disk_info.save()
if len(incomplete_snapshots) == 0:
if (time_now - disk_info.get_detach_time()
).total_seconds() > 24 * 60 * 60:
vol_id = disk_info.get_volume_id()
JBoxd.log_debug(
"volume %s for user %s unused for too long", vol_id,
user_id)
disk_info.delete()
CloudHost.detach_volume(vol_id, delete=True)
else:
JBoxd.log_debug("ongoing snapshots of user %s: %r", user_id,
incomplete_snapshots)
def pull_backup(user_email):
sessname = unique_sessname(user_email)
S3Disk.log_info("pulling %s.tar.gz from %s", sessname, JBoxVol.BACKUP_BUCKET)
CloudHost.pull_file_from_s3(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz", metadata_only=False)
def delete(user_email):
sessname = unique_sessname(user_email)
CloudHost.del_file_from_s3(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz")
def rename_and_delete(user_email):
sessname = unique_sessname(user_email)
renamed_sessname = sessname + '_old'
CloudHost.move_file_in_s3(sessname + ".tar.gz", renamed_sessname + ".tar.gz", JBoxVol.BACKUP_BUCKET)
def get(self):
jbox_cookie = AuthHandler.get_session_cookie(self)
if self.config("invite_only"):
if self.get_argument("invite", False):
self.set_cookie("is_invite", "yes")
self.redirect('/hostlaunchipnb/')
return
if None == jbox_cookie:
which_msg = int(
self.get_argument("_msg", JBoxUserV2.ACTIVATION_NONE))
if self.get_argument("_msg", "") != "":
self.clear_cookie("is_invite")
if which_msg == JBoxUserV2.ACTIVATION_GRANTED:
msg = "Your account has already been approved"
elif which_msg == JBoxUserV2.ACTIVATION_REQUESTED:
msg = "You have already registered for an invite"
else:
msg = "Thank you for your interest! We will get back to you with an invitation soon."
state = self.state(success=msg)
else:
state = self.state()
self.rendertpl("index.tpl", cfg=self.config(), state=state)
else:
user_id = jbox_cookie['u']
sessname = unique_sessname(user_id)
if self.config("gauth"):
try:
jbuser = JBoxUserV2(user_id)
except:
# stale cookie. we don't have the user in our database anymore
self.log_info(
"stale cookie. we don't have the user in our database anymore. user: "******"invite_only"):
code, status = jbuser.get_activation_state()
if status != JBoxUserV2.ACTIVATION_GRANTED:
invite_code = self.get_argument("invite_code", False)
if invite_code is not False:
try:
invite = JBoxInvite(invite_code)
except:
invite = None
if (invite is not None
) and invite.is_invited(user_id):
jbuser.set_activation_state(
invite_code, JBoxUserV2.ACTIVATION_GRANTED)
invite.increment_count()
invite.save()
jbuser.save()
self.redirect('/hostlaunchipnb/')
return
else:
error_msg = 'You entered an invalid invitation code. Try again or request a new invitation.'
else:
error_msg = 'Enter an invitation code to proceed.'
self.rendertpl("index.tpl",
cfg=self.config(),
state=self.state(error=error_msg,
ask_invite_code=True,
user_id=user_id))
return
creds = jbuser.get_gtok()
if creds is not None:
try:
creds_json = json.loads(base64.b64decode(creds))
creds_json = self.renew_creds(creds_json)
authtok = creds_json['access_token']
except:
self.log_info(
"stale stored creds. will renew on next use. user: "
+ user_id)
creds = None
authtok = None
else:
authtok = None
else:
creds = None
authtok = None
self.chk_and_launch_docker(sessname, creds, authtok, user_id)
def get(self):
jbox_cookie = AuthHandler.get_session_cookie(self)
if self.config("invite_only"):
if self.get_argument("invite", False):
self.set_cookie("is_invite", "yes")
self.redirect('/hostlaunchipnb/')
return
if None == jbox_cookie:
which_msg = int(self.get_argument("_msg", JBoxUserV2.ACTIVATION_NONE))
if self.get_argument("_msg", "") != "":
self.clear_cookie("is_invite")
if which_msg == JBoxUserV2.ACTIVATION_GRANTED:
msg = "Your account has already been approved"
elif which_msg == JBoxUserV2.ACTIVATION_REQUESTED:
msg = "You have already registered for an invite"
else:
msg = "Thank you for your interest! We will get back to you with an invitation soon."
state = self.state(success=msg)
else:
state = self.state()
self.rendertpl("index.tpl", cfg=self.config(), state=state)
else:
user_id = jbox_cookie['u']
sessname = unique_sessname(user_id)
if self.config("gauth"):
try:
jbuser = JBoxUserV2(user_id)
except:
# stale cookie. we don't have the user in our database anymore
self.log_info("stale cookie. we don't have the user in our database anymore. user: "******"invite_only"):
code, status = jbuser.get_activation_state()
if status != JBoxUserV2.ACTIVATION_GRANTED:
invite_code = self.get_argument("invite_code", False)
if invite_code is not False:
try:
invite = JBoxInvite(invite_code)
except:
invite = None
if (invite is not None) and invite.is_invited(user_id):
jbuser.set_activation_state(invite_code, JBoxUserV2.ACTIVATION_GRANTED)
invite.increment_count()
invite.save()
jbuser.save()
self.redirect('/hostlaunchipnb/')
return
else:
error_msg = 'You entered an invalid invitation code. Try again or request a new invitation.'
else:
error_msg = 'Enter an invitation code to proceed.'
self.rendertpl("index.tpl", cfg=self.config(), state=self.state(
error=error_msg,
ask_invite_code=True, user_id=user_id))
return
creds = jbuser.get_gtok()
if creds is not None:
try:
creds_json = json.loads(base64.b64decode(creds))
creds_json = self.renew_creds(creds_json)
authtok = creds_json['access_token']
except:
self.log_info("stale stored creds. will renew on next use. user: " + user_id)
creds = None
authtok = None
else:
authtok = None
else:
creds = None
authtok = None
self.chk_and_launch_docker(sessname, creds, authtok, user_id)
def delete(user_email):
sessname = unique_sessname(user_email)
CloudHost.del_file_from_s3(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz")
def get_disk_for_user(user_email):
JBoxEBSVol.log_debug("creating EBS volume for %s", user_email)
if not JBoxEBSVol.HAS_EBS:
raise Exception("EBS disks not enabled")
disk_id = JBoxEBSVol._reserve_disk_id()
if disk_id is None:
raise Exception("No free disk available")
try:
existing_disk = JBoxDiskState(cluster_id=CloudHost.INSTALL_ID,
region_id=CloudHost.REGION,
user_id=user_email)
except:
existing_disk = None
if existing_disk is None:
sess_id = unique_sessname(user_email)
sess_props = JBoxSessionProps(sess_id,
create=True,
user_id=user_email)
if sess_props.is_new:
sess_props.save()
snap_id = sess_props.get_snapshot_id()
if snap_id is None:
snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT
JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id,
user_email)
_dev_path, mnt_path, vol_id = CloudHost.create_new_volume(
snap_id,
disk_id,
JBoxEBSVol.FS_LOC,
tag=user_email,
disk_sz_gb=JBoxEBSVol.DISK_LIMIT)
existing_disk = JBoxDiskState(cluster_id=CloudHost.INSTALL_ID,
region_id=CloudHost.REGION,
user_id=user_email,
volume_id=vol_id,
attach_time=None,
create=True)
else:
_dev_path, mnt_path = CloudHost.attach_volume(
existing_disk.get_volume_id(), disk_id, JBoxEBSVol.FS_LOC)
existing_disk.set_attach_time()
snap_id = None
existing_disk.set_state(JBoxDiskState.STATE_ATTACHED)
existing_disk.save()
ebsvol = JBoxEBSVol(mnt_path, user_email=user_email)
if snap_id == JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT:
JBoxEBSVol.log_debug("creating home folder on blank volume for %s",
user_email)
ebsvol.restore_user_home(True)
ebsvol.restore()
else:
JBoxEBSVol.log_debug(
"updating home folder on existing volume for %s", user_email)
ebsvol.restore_user_home(False)
# snap_age_days = CloudHost.get_snapshot_age(snap_id).total_seconds()/(60*60*24)
# if snap_age_days > 7:
# ebsvol.restore_user_home()
JBoxEBSVol.log_debug(
"setting up instance configuration on disk for %s", user_email)
ebsvol.setup_instance_config()
return ebsvol
def rename_and_delete(user_email):
sessname = unique_sessname(user_email)
renamed_sessname = sessname + '_old'
CloudHost.move_file_in_s3(sessname + ".tar.gz",
renamed_sessname + ".tar.gz",
JBoxVol.BACKUP_BUCKET)
def do_monitor_loading(self, user_id):
sessname = unique_sessname(user_id)
self.log_debug("Monitoring loading of session [%s] user[%s]...",
sessname, user_id)
cont = JBoxContainer.get_by_name(sessname)
if (cont is None) or (not cont.is_running()):
loading_step = int(self.get_cookie("loading", 0))
if loading_step > 30:
self.log_error(
"Could not start instance. Session [%s] for user [%s] didn't load.",
sessname, user_id)
self.clear_container_cookies()
self.rendertpl(
"index.tpl",
cfg=self.config(),
state=self.state(
error=
'Could not start your instance! Please try again.',
pending_activation=False,
user_id=user_id))
return
else:
loading_step += 1
self.set_cookie("loading", str(loading_step))
self.rendertpl("loading.tpl", user_id=user_id)
else:
if self.config("gauth"):
jbuser = JBoxUserV2(user_id)
creds = jbuser.get_gtok()
if creds is not None:
try:
creds_json = json.loads(base64.b64decode(creds))
creds_json = self.renew_creds(creds_json)
authtok = creds_json['access_token']
except:
self.log_warn(
"stale stored creds. will renew on next use. user: "******"sesskey"))
self.clear_cookie("loading")
self.set_container_cookies({
"sessname": sessname,
"hostshell": shellport,
"hostupload": uplport,
"hostipnb": ipnbport,
"sign": sign
})
self.set_lb_tracker_cookie()
self.rendertpl("ipnbsess.tpl",
sessname=sessname,
cfg=self.config(),
creds=creds,
authtok=authtok,
user_id=user_id)
class JBoxEBSVol(JBoxVol):
DEVICES = []
MAX_DISKS = 0
FS_LOC = None
DISK_LIMIT = None
DISK_USE_STATUS = {}
DISK_RESERVE_TIME = {}
DISK_TEMPLATE_SNAPSHOT = None
HAS_EBS = False
LOCK = None
@staticmethod
def configure(disk_limit, fs_loc, max_disks, disk_template_snap_id):
JBoxEBSVol.HAS_EBS = True
JBoxEBSVol.FS_LOC = fs_loc
JBoxEBSVol.DISK_LIMIT = disk_limit
JBoxEBSVol.MAX_DISKS = max_disks
JBoxEBSVol.DEVICES = JBoxEBSVol._get_configured_devices(JBoxEBSVol.FS_LOC)
JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT = disk_template_snap_id
if len(JBoxEBSVol.DEVICES) < max_disks:
raise Exception("Not enough EBS mount points configured")
JBoxEBSVol.LOCK = threading.Lock()
JBoxEBSVol.refresh_disk_use_status()
@classmethod
def get_disk_allocated_size(cls):
return JBoxEBSVol.DISK_LIMIT * 1000000000
@staticmethod
def _id_from_device(dev_path):
return dev_path.split('/')[-1]
@staticmethod
def _get_configured_devices(fs_loc):
devices = []
with open('/etc/fstab', 'r') as fstab:
for line in fstab:
line = line.strip()
if (len(line) == 0) or line.startswith('#'):
continue
comps = line.split()
if (len(comps) == 6) and comps[1].startswith(fs_loc):
device = comps[0]
devices.append(JBoxEBSVol._id_from_device(device))
return devices
@staticmethod
def _get_disk_ids_used(cid):
used = []
props = JBoxEBSVol.dckr().inspect_container(cid)
try:
vols = props['Volumes']
for _cpath, hpath in vols.iteritems():
if hpath.startswith(JBoxEBSVol.FS_LOC):
used.append(hpath.split('/')[-1])
except:
JBoxEBSVol.log_error("error finding disk ids used in " + cid)
return []
return used
@staticmethod
def refresh_disk_use_status(container_id_list=None):
JBoxEBSVol.LOCK.acquire()
try:
nfree = 0
for idx in range(0, JBoxEBSVol.MAX_DISKS):
dev = JBoxEBSVol.DEVICES[idx]
if JBoxEBSVol._is_reserved(dev):
JBoxEBSVol.DISK_USE_STATUS[dev] = True
else:
JBoxEBSVol.DISK_USE_STATUS[dev] = False
nfree += 1
if container_id_list is None:
container_id_list = [cdesc['Id'] for cdesc in JBoxEBSVol.dckr().containers(all=True)]
for cid in container_id_list:
disk_ids = JBoxEBSVol._get_disk_ids_used(cid)
for disk_id in disk_ids:
JBoxEBSVol._mark_disk_used(disk_id)
nfree -= 1
JBoxEBSVol.log_info("Disk free: " + str(nfree) + "/" + str(JBoxEBSVol.MAX_DISKS))
finally:
JBoxEBSVol.LOCK.release()
@staticmethod
def _get_unused_disk_id():
for idx in range(0, JBoxEBSVol.MAX_DISKS):
dev = JBoxEBSVol.DEVICES[idx]
if not JBoxEBSVol.DISK_USE_STATUS[dev]:
return dev
return None
@staticmethod
def _is_reserved(idx):
if (idx in JBoxEBSVol.DISK_RESERVE_TIME) and (JBoxEBSVol.DISK_RESERVE_TIME[idx] < time.time()):
del JBoxEBSVol.DISK_RESERVE_TIME[idx]
return idx in JBoxEBSVol.DISK_RESERVE_TIME
@staticmethod
def _mark_disk_used(idx, used=True, for_secs=0):
JBoxEBSVol.DISK_USE_STATUS[idx] = used
if used and (for_secs > 0):
JBoxEBSVol.DISK_RESERVE_TIME[idx] = time.time() + for_secs
else:
if idx in JBoxEBSVol.DISK_RESERVE_TIME:
del JBoxEBSVol.DISK_RESERVE_TIME[idx]
@staticmethod
def _reserve_disk_id():
JBoxEBSVol.LOCK.acquire()
try:
disk_id = JBoxEBSVol._get_unused_disk_id()
JBoxEBSVol._mark_disk_used(disk_id, for_secs=120)
return disk_id
finally:
JBoxEBSVol.LOCK.release()
@staticmethod
def disk_ids_used_pct():
if not JBoxEBSVol.HAS_EBS:
return 0
pct = (sum(JBoxEBSVol.DISK_USE_STATUS.values()) * 100) / len(JBoxEBSVol.DISK_USE_STATUS)
return min(100, max(0, pct))
@staticmethod
def get_disk_for_user(user_email):
JBoxEBSVol.log_debug("creating EBS volume for %s", user_email)
if not JBoxEBSVol.HAS_EBS:
raise Exception("EBS disks not enabled")
disk_id = JBoxEBSVol._reserve_disk_id()
if disk_id is None:
raise Exception("No free disk available")
try:
existing_disk = JBoxDiskState(cluster_id=CloudHost.INSTALL_ID, region_id=CloudHost.REGION,
user_id=user_email)
except Exception, ex:
JBoxEBSVol.log_debug("No existing disk for %s. Exception %r", user_email, ex)
existing_disk = None
if existing_disk is None:
sess_id = unique_sessname(user_email)
sess_props = JBoxSessionProps(sess_id, create=True, user_id=user_email)
if sess_props.is_new:
sess_props.save()
snap_id = sess_props.get_snapshot_id()
if snap_id is None:
snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT
JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id, user_email)
_dev_path, mnt_path, vol_id = CloudHost.create_new_volume(snap_id, disk_id,
JBoxEBSVol.FS_LOC,
tag=user_email,
disk_sz_gb=JBoxEBSVol.DISK_LIMIT)
existing_disk = JBoxDiskState(cluster_id=CloudHost.INSTALL_ID, region_id=CloudHost.REGION,
user_id=user_email,
volume_id=vol_id,
attach_time=None,
create=True)
else:
_dev_path, mnt_path = CloudHost.attach_volume(existing_disk.get_volume_id(), disk_id, JBoxEBSVol.FS_LOC)
existing_disk.set_attach_time()
snap_id = None
existing_disk.set_state(JBoxDiskState.STATE_ATTACHED)
existing_disk.save()
ebsvol = JBoxEBSVol(mnt_path, user_email=user_email)
if snap_id == JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT:
JBoxEBSVol.log_debug("creating home folder on blank volume for %s", user_email)
ebsvol.restore_user_home(True)
ebsvol.restore()
else:
JBoxEBSVol.log_debug("updating home folder on existing volume for %s", user_email)
ebsvol.restore_user_home(False)
# snap_age_days = CloudHost.get_snapshot_age(snap_id).total_seconds()/(60*60*24)
# if snap_age_days > 7:
# ebsvol.restore_user_home()
JBoxEBSVol.log_debug("setting up instance configuration on disk for %s", user_email)
ebsvol.setup_instance_config()
return ebsvol
