def try_launch_container(cls, user_id, max_hop=False):
sessname = unique_sessname(user_id)
cont = SessContainer.get_by_name(sessname)
cls.log_debug("have existing container for %s: %r", sessname,
None != cont)
if cont is not None:
cls.log_debug("container running: %r", cont.is_running())
if max_hop:
self_load = Compute.get_instance_stats(Compute.get_instance_id(),
'Load')
if self_load < 100:
SessContainer.invalidate_container(sessname)
JBoxAsyncJob.async_launch_by_name(sessname, user_id, True)
return True
is_leader = is_proposed_cluster_leader()
if ((cont is None) or (not cont.is_running())) and (
not Compute.should_accept_session(is_leader)):
if cont is not None:
SessContainer.invalidate_container(cont.get_name())
JBoxAsyncJob.async_backup_and_cleanup(cont.dockid)
return False
SessContainer.invalidate_container(sessname)
JBoxAsyncJob.async_launch_by_name(sessname, user_id, True)
return True
def is_cluster_leader():
self_id = Compute.get_instance_id()
cluster = Compute.get_install_id()
instances = Compute.get_all_instances()
leader = JBoxDynConfig.get_cluster_leader(cluster)
img_recentness = Compute.get_image_recentness()
JBoxDB.log_debug(
"cluster: %s. instances: %s. leader: %s. image recentness: %d",
cluster, repr(instances), repr(leader), img_recentness)
# if none set, or set instance is dead elect self as leader, but wait till next cycle to prevent conflicts
if (leader is None) or (leader not in instances) and (img_recentness >= 0):
JBoxDB.log_info("setting self (%s) as cluster leader", self_id)
try:
JBoxDynConfig.set_cluster_leader(cluster, self_id)
except:
JBoxDB.log_info(
"error setting self (%s) as cluster leader, will retry",
self_id)
return False
is_leader = (leader == self_id)
# if running an older ami, step down from cluster leader
if (img_recentness < 0) and is_leader:
JBoxDB.log_info("unmarking self (%s) as cluster leader", self_id)
JBoxDynConfig.unset_cluster_leader(cluster)
return False
return is_leader
def is_cluster_leader():
self_id = Compute.get_instance_id()
cluster = Compute.get_install_id()
instances = Compute.get_all_instances()
leader = JBoxDynConfig.get_cluster_leader(cluster)
img_recentness = Compute.get_image_recentness()
JBoxDB.log_debug("cluster: %s. instances: %s. leader: %s. image recentness: %d",
cluster, repr(instances), repr(leader), img_recentness)
# if none set, or set instance is dead elect self as leader, but wait till next cycle to prevent conflicts
if (leader is None) or (leader not in instances) and (img_recentness >= 0):
JBoxDB.log_info("setting self (%s) as cluster leader", self_id)
try:
JBoxDynConfig.set_cluster_leader(cluster, self_id)
except:
JBoxDB.log_info("error setting self (%s) as cluster leader, will retry", self_id)
return False
is_leader = (leader == self_id)
# if running an older ami, step down from cluster leader
if (img_recentness < 0) and is_leader:
JBoxDB.log_info("unmarking self (%s) as cluster leader", self_id)
JBoxDynConfig.unset_cluster_leader(cluster)
return False
return is_leader
def process_commands(argv):
with open(argv[2]) as f:
uplcourse = eval(f.read())
conf_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../../host/tornado/conf'))
conf_file = os.path.join(conf_dir, 'tornado.conf')
user_conf_file = os.path.join('/jboxengine/conf', 'jbox.user')
JBoxCfg.read(conf_file, user_conf_file)
LoggerMixin.configure()
db.configure()
Compute.configure()
cmd = argv[1]
if cmd == "upload":
HomeworkHandler.upload_course(None, uplcourse)
elif cmd == "report":
as_csv = (argv[3] == "csv") if len(argv) > 3 else False
get_report(uplcourse, as_csv)
elif cmd == "answers":
get_answers(uplcourse)
else:
print("Unknown option %s" % (cmd, ))
print("DONE!")
def chk_and_launch_docker(self, user_id):
if self.redirect_to_logged_in_instance(user_id):
return
nhops = int(self.get_argument('h', 0))
numhopmax = JBoxCfg.get('numhopmax', 0)
max_hop = nhops > numhopmax
launched = self.try_launch_container(user_id, max_hop=max_hop)
if launched:
self.set_container_initialized(Compute.get_instance_local_ip(), user_id)
self.rendertpl("loading.tpl",
user_id=user_id,
cfg=JBoxCfg.nv,
js_includes=JBPluginHandler.PLUGIN_JAVASCRIPTS)
return
self.unset_affinity()
self.log_debug("at hop %d for user %s", nhops, user_id)
if max_hop:
if JBoxCfg.get('cloud_host.scale_down'):
msg = "JuliaBox is experiencing a sudden surge. Please try in a few minutes while we increase our capacity."
else:
msg = "JuliaBox servers are fully loaded. Please try after sometime."
self.log_error("Server maxed out. Can't launch container at hop %d for user %s", nhops, user_id)
self.rendertpl("index.tpl", cfg=JBoxCfg.nv, state=self.state(error=msg, success=''))
else:
redirect_instance = Compute.get_redirect_instance_id()
if redirect_instance is not None:
redirect_ip = Compute.get_instance_local_ip(redirect_instance)
self.set_redirect_instance_id(redirect_ip)
self.redirect('/?h=' + str(nhops + 1))
def chk_and_launch_docker(self, user_id):
if self.redirect_to_logged_in_instance(user_id):
return
nhops = int(self.get_argument('h', 0))
numhopmax = JBoxCfg.get('numhopmax', 0)
max_hop = nhops > numhopmax
launched = self.try_launch_container(user_id, max_hop=max_hop)
if launched:
self.set_container_initialized(Compute.get_instance_local_ip(), user_id)
self.rendertpl("loading.tpl",
user_id=user_id,
cfg=JBoxCfg.nv,
js_includes=JBPluginHandler.PLUGIN_JAVASCRIPTS)
return
self.unset_affinity()
self.log_debug("at hop %d for user %s", nhops, user_id)
if max_hop:
if JBoxCfg.get('cloud_host.scale_down'):
msg = "JuliaBox is experiencing a sudden surge. Please try in a few minutes while we increase our capacity."
else:
msg = "JuliaBox servers are fully loaded. Please try after sometime."
self.log_error("Server maxed out. Can't launch container at hop %d for user %s", nhops, user_id)
self.rendertpl("index.tpl", cfg=JBoxCfg.nv, state=self.state(error=msg, success=''))
else:
redirect_instance = Compute.get_redirect_instance_id()
if redirect_instance is not None:
redirect_ip = Compute.get_instance_local_ip(redirect_instance)
self.set_redirect_instance_id(redirect_ip)
self.redirect('/?h=' + str(nhops + 1))
def post_auth_launch_container(self, user_id):
for plugin in JBPluginHandler.jbox_get_plugins(JBPluginHandler.JBP_HANDLER_POST_AUTH):
self.log_info("Passing user %r to post auth plugin %r", user_id, plugin)
pass_allowed = plugin.process_user_id(self, user_id)
if not pass_allowed:
self.log_info('Login restricted for user %r by plugin %r', user_id, plugin)
return
jbuser = JBoxUserV2(user_id, create=True)
if not JBPluginHandler.is_user_activated(jbuser):
self.redirect('/?pending_activation=' + user_id)
return
self.set_authenticated(user_id)
if jbuser.is_new:
jbuser.save()
if self.redirect_to_logged_in_instance(user_id):
return
# check if the current instance is appropriate for launching this
if self.try_launch_container(user_id, max_hop=False):
self.set_container_initialized(Compute.get_instance_local_ip(), user_id)
else:
# redirect to an appropriate instance
redirect_instance = Compute.get_redirect_instance_id()
if redirect_instance is not None:
redirect_ip = Compute.get_instance_local_ip(redirect_instance)
self.set_redirect_instance_id(redirect_ip)
self.redirect('/')
def redirect_to_logged_in_instance(self, user_id):
loggedin_instance = self.find_logged_in_instance(user_id)
if loggedin_instance is not None \
and loggedin_instance != Compute.get_instance_id() \
and loggedin_instance != 'localhost':
# redirect to the instance that has the user's session
self.log_info("Already logged in to %s. Redirecting", loggedin_instance)
redirect_ip = Compute.get_instance_local_ip(loggedin_instance)
self.set_redirect_instance_id(redirect_ip)
self.redirect('/')
return True
self.log_info("Logged in %s", "nowhere" if loggedin_instance is None else "here already")
return False
def handle_get_metadata(self, is_admin, courses_offered):
mode = self.get_argument('mode', None)
if (mode is None) or (mode != "metadata"):
return False
self.log_debug("handling answers")
params = self.get_argument('params', None)
params = json.loads(params)
course_id = params['course']
problemset_id = params['problemset']
question_ids = params['questions'] if 'questions' in params else None
send_answers = True
if (not is_admin) and (course_id not in courses_offered):
send_answers = False
err = None
course = JBoxDynConfig.get_course(Compute.get_install_id(), course_id)
self.log_debug("got course %r", course)
if problemset_id not in course['problemsets']:
err = "Problem set %s not found!" % (problemset_id,)
if question_ids is None:
question_ids = course['questions'][problemset_id]
if err is None:
report = JBoxCourseHomework.get_problemset_metadata(course_id, problemset_id, question_ids, send_answers)
code = 0
else:
report = err
code = -1
response = {'code': code, 'data': report}
self.write(response)
return True
def find_logged_in_instance(user_id):
sessname = unique_sessname(user_id)
try:
sess_props = JBoxSessionProps(Compute.get_install_id(), sessname)
return sess_props.get_instance_id()
except JBoxDBItemNotFound:
return None
def get(self):
self.log_debug("APIInfo handler got GET request")
key = self.get_argument("key", None)
sign = self.get_argument("sign", None)
if key is None or sign is None:
self.send_error()
return
sign2 = signstr(key, JBoxCfg.get('sesskey'))
if sign != sign2:
self.log_info("signature mismatch. key:%r sign:%r expected:%r", key, sign, sign2)
self.send_error()
return
api_status = JBoxInstanceProps.get_instance_status(Compute.get_install_id())
self.log_info("cluster api status: %r", api_status)
# filter out instances that should not accept more load
filtered_api_status = {k: v for (k, v) in api_status.iteritems() if v['accept']}
preferred_instances = filtered_api_status.keys()
# flip the dict
per_api_instances = dict()
for (inst, status) in filtered_api_status.iteritems():
api_names = status['api_status'].keys()
for api_name in api_names:
v = per_api_instances.get(api_name, [])
v.append(inst)
per_api_instances[" preferred "] = preferred_instances
self.log_info("per api instances: %r", per_api_instances)
self.write(per_api_instances)
return
def update_user_home_image(fetch=True):
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
if plugin is None:
VolMgr.log_info("No plugin provided for bucketstore. Can not update packages and user home images")
return
home_img_dir, curr_home_img = os.path.split(JBoxVol.USER_HOME_IMG)
pkg_img_dir, curr_pkg_img = os.path.split(JBoxVol.PKG_IMG)
bucket, new_pkg_img, new_home_img = JBoxDynConfig.get_user_home_image(Compute.get_install_id())
new_home_img_path = os.path.join(home_img_dir, new_home_img)
new_pkg_img_path = os.path.join(pkg_img_dir, new_pkg_img)
updated = False
for img_path in (new_home_img_path, new_pkg_img_path):
if not os.path.exists(img_path):
if fetch:
VolMgr.log_debug("fetching new image to %s", img_path)
k = plugin.pull(bucket, img_path)
if k is not None:
VolMgr.log_debug("fetched new image")
if os.path.exists(new_home_img_path):
VolMgr.log_debug("set new home image to %s", new_home_img_path)
JBoxVol.USER_HOME_IMG = new_home_img_path
updated = True
if os.path.exists(new_pkg_img_path):
VolMgr.log_debug("set new pkg image to %s", new_pkg_img_path)
JBoxVol.PKG_IMG = new_pkg_img_path
updated = True
return updated
def do_cluster_housekeeping():
JBoxEBSHousekeep.log_debug("starting cluster housekeeping")
detached_disks = JBoxDiskState.get_detached_disks()
time_now = datetime.datetime.now(pytz.utc)
for disk_key in detached_disks:
disk_info = JBoxDiskState(disk_key=disk_key)
user_id = disk_info.get_user_id()
sess_props = JBoxSessionProps(Compute.get_install_id(), unique_sessname(user_id))
incomplete_snapshots = []
modified = False
for snap_id in disk_info.get_snapshot_ids():
if not EBSVol.is_snapshot_complete(snap_id):
incomplete_snapshots.append(snap_id)
continue
JBoxEBSHousekeep.log_debug("updating latest snapshot of user %s to %s", user_id, snap_id)
old_snap_id = sess_props.get_snapshot_id()
sess_props.set_snapshot_id(snap_id)
modified = True
if old_snap_id is not None:
EBSVol.delete_snapshot(old_snap_id)
if modified:
sess_props.save()
disk_info.set_snapshot_ids(incomplete_snapshots)
disk_info.save()
if len(incomplete_snapshots) == 0:
if (time_now - disk_info.get_detach_time()).total_seconds() > 24*60*60:
vol_id = disk_info.get_volume_id()
JBoxEBSHousekeep.log_debug("volume %s for user %s unused for too long", vol_id, user_id)
disk_info.delete()
EBSVol.detach_volume(vol_id, delete=True)
else:
JBoxEBSHousekeep.log_debug("ongoing snapshots of user %s: %r", user_id, incomplete_snapshots)
JBoxEBSHousekeep.log_debug("finished cluster housekeeping")
def do_cluster_housekeeping():
JBoxEBSHousekeep.log_debug("starting cluster housekeeping")
detached_disks = JBoxDiskState.get_detached_disks()
time_now = datetime.datetime.now(pytz.utc)
for disk_key in detached_disks:
disk_info = JBoxDiskState(disk_key=disk_key)
user_id = disk_info.get_user_id()
sess_props = JBoxSessionProps(Compute.get_install_id(), unique_sessname(user_id))
incomplete_snapshots = []
modified = False
for snap_id in disk_info.get_snapshot_ids():
if not EBSVol.is_snapshot_complete(snap_id):
incomplete_snapshots.append(snap_id)
continue
JBoxEBSHousekeep.log_debug("updating latest snapshot of user %s to %s", user_id, snap_id)
old_snap_id = sess_props.get_snapshot_id()
sess_props.set_snapshot_id(snap_id)
modified = True
if old_snap_id is not None:
EBSVol.delete_snapshot(old_snap_id)
if modified:
sess_props.save()
disk_info.set_snapshot_ids(incomplete_snapshots)
disk_info.save()
if len(incomplete_snapshots) == 0:
if (time_now - disk_info.get_detach_time()).total_seconds() > 24 * 60 * 60:
vol_id = disk_info.get_volume_id()
JBoxEBSHousekeep.log_debug("volume %s for user %s unused for too long", vol_id, user_id)
disk_info.delete()
EBSVol.detach_volume(vol_id, delete=True)
else:
JBoxEBSHousekeep.log_debug("ongoing snapshots of user %s: %r", user_id, incomplete_snapshots)
JBoxEBSHousekeep.log_debug("finished cluster housekeeping")
def update_user_home_image(fetch=True):
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
if plugin is None:
VolMgr.log_info("No plugin provided for bucketstore. Can not update packages and user home images")
return
home_img_dir, curr_home_img = os.path.split(JBoxVol.USER_HOME_IMG)
pkg_img_dir, curr_pkg_img = os.path.split(JBoxVol.PKG_IMG)
bucket, new_pkg_img, new_home_img = JBoxDynConfig.get_user_home_image(Compute.get_install_id())
new_home_img_path = os.path.join(home_img_dir, new_home_img)
new_pkg_img_path = os.path.join(pkg_img_dir, new_pkg_img)
updated = False
for img_path in (new_home_img_path, new_pkg_img_path):
if not os.path.exists(img_path):
if fetch:
VolMgr.log_debug("fetching new image to %s", img_path)
k = plugin.pull(bucket, img_path)
if k is not None:
VolMgr.log_debug("fetched new image")
if os.path.exists(new_home_img_path):
VolMgr.log_debug("set new home image to %s", new_home_img_path)
JBoxVol.USER_HOME_IMG = new_home_img_path
updated = True
if os.path.exists(new_pkg_img_path):
VolMgr.log_debug("set new pkg image to %s", new_pkg_img_path)
JBoxVol.PKG_IMG = new_pkg_img_path
updated = True
return updated
def redirect_to_logged_in_instance(self, user_id):
loggedin_instance = self.find_logged_in_instance(user_id)
if loggedin_instance is not None \
and loggedin_instance != Compute.get_instance_id() \
and loggedin_instance != 'localhost':
# redirect to the instance that has the user's session
self.log_info("Already logged in to %s. Redirecting",
loggedin_instance)
redirect_ip = Compute.get_instance_local_ip(loggedin_instance)
self.set_redirect_instance_id(redirect_ip)
self.redirect('/')
return True
self.log_info(
"Logged in %s",
"nowhere" if loggedin_instance is None else "here already")
return False
def has_update_for_user_home_image():
home_img_dir, curr_home_img = os.path.split(JBoxVol.USER_HOME_IMG)
pkg_img_dir, curr_pkg_img = os.path.split(JBoxVol.PKG_IMG)
# VolMgr.log_debug("checking for updates to user home image %s/%s", img_dir, curr_img)
bucket, new_pkg_img, new_home_img = JBoxDynConfig.get_user_home_image(
Compute.get_install_id())
if bucket is None:
VolMgr.log_info("Home: none configured. current: %s/%s",
home_img_dir, curr_home_img)
return False
if new_home_img == curr_home_img and new_pkg_img == curr_pkg_img:
VolMgr.log_info("Home: no updates. current: %s/%s", home_img_dir,
curr_home_img)
VolMgr.log_info("Packages: no updates. current: %s/%s",
pkg_img_dir, curr_pkg_img)
return False
else:
VolMgr.log_info("Home: update: %s/%s. current: %s/%s", bucket,
new_home_img, home_img_dir, curr_home_img)
VolMgr.log_info("Packages: update: %s/%s. current: %s/%s", bucket,
new_pkg_img, home_img_dir, curr_pkg_img)
return True
def is_user_activated(jbuser):
reg_allowed = JBoxDynConfig.get_allow_registration(
Compute.get_install_id())
if jbuser.is_new:
if not reg_allowed:
activation_state = JBoxUserV2.ACTIVATION_REQUESTED
else:
activation_state = JBoxUserV2.ACTIVATION_GRANTED
jbuser.set_activation_state(JBoxUserV2.ACTIVATION_CODE_AUTO,
activation_state)
jbuser.save()
else:
activation_code, activation_state = jbuser.get_activation_state()
if reg_allowed and (activation_state !=
JBoxUserV2.ACTIVATION_GRANTED):
activation_state = JBoxUserV2.ACTIVATION_GRANTED
jbuser.set_activation_state(JBoxUserV2.ACTIVATION_CODE_AUTO,
activation_state)
jbuser.save()
elif activation_state != JBoxUserV2.ACTIVATION_GRANTED:
if not ((activation_state == JBoxUserV2.ACTIVATION_REQUESTED)
and
(activation_code == JBoxUserV2.ACTIVATION_CODE_AUTO)):
activation_state = JBoxUserV2.ACTIVATION_REQUESTED
jbuser.set_activation_state(
JBoxUserV2.ACTIVATION_CODE_AUTO, activation_state)
jbuser.save()
return activation_state == JBoxUserV2.ACTIVATION_GRANTED
def _backup(self, clear_volume=False):
sess_props = JBoxSessionProps(Compute.get_install_id(), self.sessname)
desc = sess_props.get_user_id() + " JuliaBox Backup"
disk_id = self.disk_path.split('/')[-1]
snap_id = EBSVol.snapshot_volume(dev_id=disk_id,
tag=self.sessname,
description=desc,
wait_till_complete=False)
return snap_id
def handle_if_instance_info(self, is_allowed):
stats = self.get_argument('instance_info', None)
if stats is None:
return False
if not is_allowed:
AdminHandler.log_error("Show instance info not allowed for user")
response = {'code': -1, 'data': 'You do not have permissions to view these stats'}
else:
try:
if stats == 'load':
result = {}
# get cluster loads
average_load = Compute.get_cluster_average_stats('Load')
if None != average_load:
result['Average Load'] = average_load
machine_loads = Compute.get_cluster_stats('Load')
if None != machine_loads:
for n, v in machine_loads.iteritems():
result['Instance ' + n] = v
elif stats == 'sessions':
result = dict()
instances = Compute.get_all_instances()
for idx in range(0, len(instances)):
try:
inst = instances[idx]
result[inst] = JBoxAsyncJob.sync_session_status(inst)['data']
except:
JBoxHandler.log_error("Error receiving sessions list from %r", inst)
elif stats == 'apis':
result = APIContainer.get_cluster_api_status()
else:
raise Exception("unknown command %s" % (stats,))
response = {'code': 0, 'data': result}
except:
AdminHandler.log_error("exception while getting stats")
AdminHandler._get_logger().exception("exception while getting stats")
response = {'code': -1, 'data': 'error getting stats'}
self.write(response)
return True
def setup_instance_config(self, profiles=('default',)):
for profile in profiles:
profile_path = '.ipython/profile_' + profile
profile_path = os.path.join(self.disk_path, profile_path)
if not os.path.exists(profile_path):
continue
nbconfig = os.path.join(profile_path, 'ipython_notebook_config.py')
wsock_cfg = "c.NotebookApp.websocket_url = '" + JBoxVol.NOTEBOOK_WEBSOCK_PROTO + \
Compute.get_alias_hostname() + "'\n"
JBoxVol.replace_in_file(nbconfig, "c.NotebookApp.websocket_url", wsock_cfg)
def setup_instance_config(self, profiles=('default', )):
for profile in profiles:
profile_path = '.ipython/profile_' + profile
profile_path = os.path.join(self.disk_path, profile_path)
if not os.path.exists(profile_path):
continue
nbconfig = os.path.join(profile_path, 'ipython_notebook_config.py')
wsock_cfg = "c.NotebookApp.websocket_url = '" + JBoxVol.NOTEBOOK_WEBSOCK_PROTO + \
Compute.get_alias_hostname() + "'\n"
JBoxVol.replace_in_file(nbconfig, "c.NotebookApp.websocket_url",
wsock_cfg)
def get(self):
args = self.get_argument('m', default=None)
if args is not None:
self.log_debug("setting cookies")
self.unpack(args)
self.set_status(status_code=204)
self.finish()
else:
args = self.pack()
url = "//" + Compute.get_alias_hostname() + "/jboxcors/?m=" + args
self.log_debug("redirecting to " + url)
self.redirect(url)
def get_cluster_api_status():
result = dict()
for inst in Compute.get_all_instances():
try:
api_status = JBoxAsyncJob.sync_api_status(inst)
if api_status['code'] == 0:
result[inst] = api_status['data']
else:
APIContainer.log_error("error fetching api status from %s", inst)
except:
APIContainer.log_error("exception fetching api status from %s", inst)
APIContainer.log_debug("api status: %r", result)
return result
def get_active_sessions():
instances = Compute.get_all_instances()
active_sessions = set()
for inst in instances:
try:
sessions = JBoxAsyncJob.sync_session_status(inst)['data']
if len(sessions) > 0:
for sess_id in sessions.keys():
active_sessions.add(sess_id)
except:
SessContainer.log_error("Error receiving sessions list from %r", inst)
return active_sessions
def allocate_random_endpoints():
ctx = zmq.Context.instance()
binder = ctx.socket(zmq.REQ)
bind_pfx = "tcp://" + Compute.get_docker_bridge_ip()
port_in = binder.bind_to_random_port(bind_pfx)
port_out = binder.bind_to_random_port(bind_pfx)
binder.close()
time.sleep(0.25)
endpoint_in = bind_pfx + str(':') + str(port_in)
endpoint_out = bind_pfx + str(':') + str(port_out)
return endpoint_in, endpoint_out
def find_logged_in_instance(user_id):
container_id = "/" + unique_sessname(user_id)
instances = Compute.get_all_instances()
for inst in instances:
try:
sessions = JBoxAsyncJob.sync_session_status(inst)['data']
if len(sessions) > 0:
if container_id in sessions:
return inst
except:
JBoxHandler.log_error("Error receiving sessions list from %r", inst)
pass
return None
def get_disk_for_user(user_email):
JBoxEBSVol.log_debug("creating EBS volume for %s", user_email)
disk_id = JBoxEBSVol._reserve_disk_id()
if disk_id is None:
raise Exception("No free disk available")
try:
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID,
region_id=CompEC2.REGION,
user_id=user_email)
except Exception as ex:
JBoxEBSVol.log_debug("No existing disk for %s. Exception %r",
user_email, ex)
existing_disk = None
if existing_disk is None:
sess_id = unique_sessname(user_email)
sess_props = JBoxSessionProps(Compute.get_install_id(),
sess_id,
create=True,
user_id=user_email)
if sess_props.is_new:
sess_props.save()
snap_id = sess_props.get_snapshot_id()
if snap_id is None:
snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT
JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id,
user_email)
dev_path, vol_id = EBSVol.create_new_volume(
snap_id,
disk_id,
tag=user_email,
disk_sz_gb=JBoxEBSVol.DISK_LIMIT)
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID,
region_id=CompEC2.REGION,
user_id=user_email,
volume_id=vol_id,
attach_time=None,
create=True)
else:
dev_path = EBSVol.attach_volume(existing_disk.get_volume_id(),
disk_id)
existing_disk.set_state(JBoxDiskState.STATE_ATTACHING)
existing_disk.save()
return JBoxEBSVol(dev_path, user_email=user_email)
def post_auth_launch_container(self, user_id):
jbuser = JBoxUserV2(user_id, create=True)
if not JBPluginHandler.is_user_activated(jbuser):
self.redirect('/?pending_activation=' + user_id)
return
self.set_authenticated(user_id)
if jbuser.is_new:
jbuser.save()
if self.redirect_to_logged_in_instance(user_id):
return
# check if the current instance is appropriate for launching this
if self.try_launch_container(user_id, max_hop=False):
self.set_container_initialized(Compute.get_instance_local_ip(), user_id)
else:
# redirect to an appropriate instance
redirect_instance = Compute.get_redirect_instance_id()
if redirect_instance is not None:
redirect_ip = Compute.get_instance_local_ip(redirect_instance)
self.set_redirect_instance_id(redirect_ip)
self.redirect('/')
def allocate_random_endpoints():
ctx = zmq.Context.instance()
binder = ctx.socket(zmq.REQ)
bind_pfx = "tcp://" + Compute.get_docker_bridge_ip()
port_in = binder.bind_to_random_port(bind_pfx)
port_out = binder.bind_to_random_port(bind_pfx)
binder.close()
time.sleep(0.25)
endpoint_in = bind_pfx + str(':') + str(port_in)
endpoint_out = bind_pfx + str(':') + str(port_out)
return endpoint_in, endpoint_out
def get_cluster_api_status():
result = dict()
for inst in Compute.get_all_instances():
try:
api_status = JBoxAsyncJob.sync_api_status(inst)
if api_status['code'] == 0:
result[inst] = api_status['data']
else:
APIContainer.log_error("error fetching api status from %s",
inst)
except:
APIContainer.log_error("exception fetching api status from %s",
inst)
APIContainer.log_debug("api status: %r", result)
return result
def try_launch_container(cls, user_id, max_hop=False):
sessname = unique_sessname(user_id)
cont = SessContainer.get_by_name(sessname)
cls.log_debug("have existing container for %s: %r", sessname, None != cont)
if cont is not None:
cls.log_debug("container running: %r", cont.is_running())
if max_hop:
self_load = Compute.get_instance_stats(Compute.get_instance_id(), 'Load')
if self_load < 100:
SessContainer.invalidate_container(sessname)
JBoxAsyncJob.async_launch_by_name(sessname, user_id, True)
return True
is_leader = is_proposed_cluster_leader()
if ((cont is None) or (not cont.is_running())) and (not Compute.should_accept_session(is_leader)):
if cont is not None:
SessContainer.invalidate_container(cont.get_name())
JBoxAsyncJob.async_backup_and_cleanup(cont.dockid)
return False
SessContainer.invalidate_container(sessname)
JBoxAsyncJob.async_launch_by_name(sessname, user_id, True)
return True
def find_logged_in_instance(user_id):
container_id = "/" + unique_sessname(user_id)
instances = Compute.get_all_instances()
for inst in instances:
try:
sessions = JBoxAsyncJob.sync_session_status(inst)['data']
if len(sessions) > 0:
if container_id in sessions:
return inst
except:
JBoxHandler.log_error("Error receiving sessions list from %r",
inst)
pass
return None
def get_active_sessions():
instances = Compute.get_all_instances()
active_sessions = set()
for inst in instances:
try:
sessions = JBoxAsyncJob.sync_session_status(inst)['data']
if len(sessions) > 0:
for sess_id in sessions.keys():
active_sessions.add(sess_id)
except:
SessContainer.log_error(
"Error receiving sessions list from %r", inst)
return active_sessions
def post_auth_launch_container(self, user_id):
jbuser = JBoxUserV2(user_id, create=True)
if not JBPluginHandler.is_user_activated(jbuser):
self.redirect('/?pending_activation=' + user_id)
return
self.set_authenticated(user_id)
if jbuser.is_new:
jbuser.save()
if self.redirect_to_logged_in_instance(user_id):
return
# check if the current instance is appropriate for launching this
if self.try_launch_container(user_id, max_hop=False):
self.set_container_initialized(Compute.get_instance_local_ip(),
user_id)
else:
# redirect to an appropriate instance
redirect_instance = Compute.get_redirect_instance_id()
if redirect_instance is not None:
redirect_ip = Compute.get_instance_local_ip(redirect_instance)
self.set_redirect_instance_id(redirect_ip)
self.redirect('/')
def handle_if_instance_info(self, is_allowed):
stats = self.get_argument('instance_info', None)
if stats is None:
return False
if not is_allowed:
AdminHandler.log_error("Show instance info not allowed for user")
response = {'code': -1, 'data': 'You do not have permissions to view these stats'}
else:
try:
if stats == 'load':
result = {}
# get cluster loads
average_load = Compute.get_cluster_average_stats('Load')
if average_load is not None:
result['Average Load'] = average_load
machine_loads = Compute.get_cluster_stats('Load')
if machine_loads is not None:
for n, v in machine_loads.iteritems():
result['Instance ' + n] = v
elif stats == 'sessions':
result = JBoxSessionProps.get_active_sessions(Compute.get_install_id())
elif stats == 'apis':
result = JBoxInstanceProps.get_instance_status(Compute.get_install_id())
else:
raise Exception("unknown command %s" % (stats,))
response = {'code': 0, 'data': result}
except:
AdminHandler.log_error("exception while getting stats")
AdminHandler._get_logger().exception("exception while getting stats")
response = {'code': -1, 'data': 'error getting stats'}
self.write(response)
return True
def do_periodic_task(_mode):
active_clusters = UserCluster.list_all_groupids()
ParallelHousekeep.log_info("%d active clusters", len(active_clusters))
if len(active_clusters) == 0:
return
for cluster_id in active_clusters:
sessname = UserCluster.sessname_for_cluster(cluster_id)
try:
sess_props = JBoxSessionProps(Compute.get_install_id(), sessname)
if not sess_props.get_instance_id():
ParallelHousekeep.log_info(
"Session (%s) corresponding to cluster (%s) not found. Terminating cluster.",
sessname, cluster_id)
ParallelHousekeep.terminate_or_delete_cluster(cluster_id)
except JBoxDBItemNotFound:
pass
def do_periodic_task(_mode):
active_clusters = UserCluster.list_all_groupids()
ParallelHousekeep.log_info("%d active clusters", len(active_clusters))
if len(active_clusters) == 0:
return
for cluster_id in active_clusters:
sessname = UserCluster.sessname_for_cluster(cluster_id)
try:
sess_props = JBoxSessionProps(Compute.get_install_id(),
sessname)
if not sess_props.get_instance_id():
ParallelHousekeep.log_info(
"Session (%s) corresponding to cluster (%s) not found. Terminating cluster.",
sessname, cluster_id)
ParallelHousekeep.terminate_or_delete_cluster(cluster_id)
except JBoxDBItemNotFound:
pass
def handle_if_report(self, user_id, is_admin, courses_offered):
mode = self.get_argument('mode', None)
if (mode is None) or ((mode != "report") and (mode != "myreport")):
return False
self.log_debug("handling report")
params = self.get_argument('params', None)
params = json.loads(params)
course_id = params['course']
problemset_id = params['problemset']
question_ids = params['questions'] if 'questions' in params else None
if mode == "myreport":
student_id = user_id
else:
student_id = params['student'] if 'student' in params else None
err = None
if (not is_admin) and (course_id not in courses_offered):
if student_id is None:
student_id = user_id
elif student_id != user_id:
err = "Course %s not found!" % (course_id, )
if err is None:
course = JBoxDynConfig.get_course(Compute.get_install_id(),
course_id)
if problemset_id not in course['problemsets']:
err = "Problem set %s not found!" % (problemset_id, )
elif question_ids is None:
question_ids = course['questions'][problemset_id]
if err is None:
report = JBoxCourseHomework.get_report(course_id,
problemset_id,
question_ids,
student_id=student_id)
code = 0
else:
report = err
code = -1
response = {'code': code, 'data': report}
self.write(response)
return True
def release(self, backup=False):
sess_props = JBoxSessionProps(Compute.get_install_id(), self.sessname)
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION,
user_id=sess_props.get_user_id())
existing_disk.set_state(JBoxDiskState.STATE_DETACHING)
existing_disk.save()
disk_id = self.disk_path.split('/')[-1]
if backup:
snap_id = self._backup()
else:
snap_id = None
vol_id = EBSVol.get_volume_id_from_device(disk_id)
EBSVol.detach_volume(vol_id, delete=False)
if snap_id is not None:
existing_disk.add_snapshot_id(snap_id)
existing_disk.set_state(JBoxDiskState.STATE_DETACHED)
existing_disk.save()
def has_update_for_user_home_image():
home_img_dir, curr_home_img = os.path.split(JBoxVol.USER_HOME_IMG)
pkg_img_dir, curr_pkg_img = os.path.split(JBoxVol.PKG_IMG)
# VolMgr.log_debug("checking for updates to user home image %s/%s", img_dir, curr_img)
bucket, new_pkg_img, new_home_img = JBoxDynConfig.get_user_home_image(Compute.get_install_id())
if bucket is None:
VolMgr.log_info("Home: none configured. current: %s/%s", home_img_dir, curr_home_img)
return False
if new_home_img == curr_home_img and new_pkg_img == curr_pkg_img:
VolMgr.log_info("Home: no updates. current: %s/%s", home_img_dir, curr_home_img)
VolMgr.log_info("Packages: no updates. current: %s/%s", pkg_img_dir, curr_pkg_img)
return False
else:
VolMgr.log_info("Home: update: %s/%s. current: %s/%s", bucket, new_home_img, home_img_dir, curr_home_img)
VolMgr.log_info("Packages: update: %s/%s. current: %s/%s", bucket, new_pkg_img, home_img_dir, curr_pkg_img)
return True
def handle_if_report(self, user_id, is_admin, courses_offered):
mode = self.get_argument('mode', None)
if (mode is None) or ((mode != "report") and (mode != "myreport")):
return False
self.log_debug("handling report")
params = self.get_argument('params', None)
params = json.loads(params)
course_id = params['course']
problemset_id = params['problemset']
question_ids = params['questions'] if 'questions' in params else None
if mode == "myreport":
student_id = user_id
else:
student_id = params['student'] if 'student' in params else None
err = None
if (not is_admin) and (course_id not in courses_offered):
if student_id is None:
student_id = user_id
elif student_id != user_id:
err = "Course %s not found!" % (course_id,)
if err is None:
course = JBoxDynConfig.get_course(Compute.get_install_id(), course_id)
if problemset_id not in course['problemsets']:
err = "Problem set %s not found!" % (problemset_id,)
elif question_ids is None:
question_ids = course['questions'][problemset_id]
if err is None:
report = JBoxCourseHomework.get_report(course_id, problemset_id, question_ids, student_id=student_id)
code = 0
else:
report = err
code = -1
response = {'code': code, 'data': report}
self.write(response)
return True
def release(self, backup=False):
sess_props = JBoxSessionProps(Compute.get_install_id(), self.sessname)
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID,
region_id=CompEC2.REGION,
user_id=sess_props.get_user_id())
existing_disk.set_state(JBoxDiskState.STATE_DETACHING)
existing_disk.save()
disk_id = self.disk_path.split('/')[-1]
if backup:
snap_id = self._backup()
else:
snap_id = None
vol_id = EBSVol.get_volume_id_from_device(disk_id)
EBSVol.detach_volume(vol_id, delete=False)
if snap_id is not None:
existing_disk.add_snapshot_id(snap_id)
existing_disk.set_state(JBoxDiskState.STATE_DETACHED)
existing_disk.save()
def get(self):
self.log_debug("APIInfo handler got GET request")
key = self.get_argument("key", None)
sign = self.get_argument("sign", None)
if key is None or sign is None:
self.send_error()
return
sign2 = signstr(key, JBoxCfg.get('sesskey'))
if sign != sign2:
self.log_info("signature mismatch. key:%r sign:%r expected:%r",
key, sign, sign2)
self.send_error()
return
api_status = JBoxInstanceProps.get_instance_status(
Compute.get_install_id())
self.log_info("cluster api status: %r", api_status)
# filter out instances that should not accept more load
filtered_api_status = {
k: v
for (k, v) in api_status.iteritems() if v['accept']
}
preferred_instances = filtered_api_status.keys()
# flip the dict
per_api_instances = dict()
for (inst, status) in filtered_api_status.iteritems():
api_names = status['api_status'].keys()
for api_name in api_names:
v = per_api_instances.get(api_name, [])
v.append(inst)
per_api_instances[" preferred "] = preferred_instances
self.log_info("per api instances: %r", per_api_instances)
self.write(per_api_instances)
return
def handle_if_stats(self, is_allowed):
stats = self.get_argument('stats', None)
if stats is None:
return False
if not is_allowed:
AdminHandler.log_error("Show stats not allowed for user")
response = {'code': -1, 'data': 'You do not have permissions to view these stats'}
else:
try:
if stats == 'stat_sessions':
stats = self.get_session_stats()
else:
stats = JBoxDynConfig.get_stat(Compute.get_install_id(), stats)
response = {'code': 0, 'data': stats} if stats is not None else {'code': 1, 'data': {}}
except:
AdminHandler.log_error("exception while getting stats")
AdminHandler._get_logger().exception("exception while getting stats")
response = {'code': -1, 'data': 'error getting stats'}
self.write(response)
return True
def get_disk_for_user(user_email):
JBoxEBSVol.log_debug("creating EBS volume for %s", user_email)
disk_id = JBoxEBSVol._reserve_disk_id()
if disk_id is None:
raise Exception("No free disk available")
try:
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION,
user_id=user_email)
except Exception as ex:
JBoxEBSVol.log_debug("No existing disk for %s. Exception %r", user_email, ex)
existing_disk = None
if existing_disk is None:
sess_id = unique_sessname(user_email)
sess_props = JBoxSessionProps(Compute.get_install_id(), sess_id, create=True, user_id=user_email)
if sess_props.is_new:
sess_props.save()
snap_id = sess_props.get_snapshot_id()
if snap_id is None:
snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT
JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id, user_email)
dev_path, vol_id = EBSVol.create_new_volume(snap_id, disk_id, tag=user_email,
disk_sz_gb=JBoxEBSVol.DISK_LIMIT)
existing_disk = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION,
user_id=user_email,
volume_id=vol_id,
attach_time=None,
create=True)
else:
dev_path = EBSVol.attach_volume(existing_disk.get_volume_id(), disk_id)
existing_disk.set_state(JBoxDiskState.STATE_ATTACHING)
existing_disk.save()
return JBoxEBSVol(dev_path, user_email=user_email)
def is_user_activated(jbuser):
reg_allowed = JBoxDynConfig.get_allow_registration(Compute.get_install_id())
if jbuser.is_new:
if not reg_allowed:
activation_state = JBoxUserV2.ACTIVATION_REQUESTED
else:
activation_state = JBoxUserV2.ACTIVATION_GRANTED
jbuser.set_activation_state(JBoxUserV2.ACTIVATION_CODE_AUTO, activation_state)
jbuser.save()
else:
activation_code, activation_state = jbuser.get_activation_state()
if reg_allowed and (activation_state != JBoxUserV2.ACTIVATION_GRANTED):
activation_state = JBoxUserV2.ACTIVATION_GRANTED
jbuser.set_activation_state(JBoxUserV2.ACTIVATION_CODE_AUTO, activation_state)
jbuser.save()
elif activation_state != JBoxUserV2.ACTIVATION_GRANTED:
if not ((activation_state == JBoxUserV2.ACTIVATION_REQUESTED) and
(activation_code == JBoxUserV2.ACTIVATION_CODE_AUTO)):
activation_state = JBoxUserV2.ACTIVATION_REQUESTED
jbuser.set_activation_state(JBoxUserV2.ACTIVATION_CODE_AUTO, activation_state)
jbuser.save()
return activation_state == JBoxUserV2.ACTIVATION_GRANTED
def publish_stats():
VolMgr.calc_stats()
VolMgr.log_debug("stats: %r", VolMgr.STATS)
JBoxDynConfig.set_stat(Compute.get_install_id(), VolMgr.STAT_NAME, VolMgr.STATS)
def upload_course(user_id, course):
course_id = course['id']
if (user_id is not None) and (user_id not in course['admins']):
course['admins'].append(user_id)
existing_course = JBoxDynConfig.get_course(Compute.get_install_id(), course_id)
existing_admins = existing_course['admins'] if existing_course is not None else []
existing_psets = existing_course['problemsets'] if existing_course is not None else []
question_list = {}
if (existing_course is not None) and ('questions' in existing_course):
question_list = existing_course['questions']
if (existing_course is not None) and (user_id is not None) and (user_id not in existing_admins):
return -1
for pset in course['problemsets']:
pset_id = pset['id']
if pset_id not in existing_psets:
existing_psets.append(pset_id)
question_ids = [q['id'] for q in pset['questions']]
question_list[pset_id] = question_ids
dt = datetime.datetime.now(pytz.utc)
JBoxDynConfig.set_course(Compute.get_install_id(), course_id, {
'admins': course['admins'],
'problemsets': existing_psets,
'questions': question_list,
'create_time': JBoxUserV2.datetime_to_yyyymmdd(dt)
})
for problemset in course['problemsets']:
problemset_id = problemset['id']
questions = problemset['questions']
for question in questions:
question_id = question['id']
answer = question['ans']
score = question['score'] if 'score' in question else 0
attempts = question['attempts'] if 'attempts' in question else 0
# nscore = question['nscore'] if 'nscore' in question else 0
try:
ans = JBoxCourseHomework(course_id, problemset_id, question_id, JBoxCourseHomework.ANSWER_KEY,
answer=answer, state=JBoxCourseHomework.STATE_CORRECT, create=True)
except:
ans = JBoxCourseHomework(course_id, problemset_id, question_id, JBoxCourseHomework.ANSWER_KEY)
ans.set_answer(answer, JBoxCourseHomework.STATE_CORRECT)
ans.set_score(score)
ans.set_attempts(attempts)
ans.save()
for uid in course['admins']:
user = JBoxUserV2(uid)
courses_offered = user.get_courses_offered()
if course['id'] not in courses_offered:
courses_offered.append(course['id'])
user.set_courses_offered(courses_offered)
user.set_role(JBoxUserV2.ROLE_OFFER_COURSES)
user.save()
for uid in existing_admins:
if uid in course['admins']:
continue
user = JBoxUserV2(uid)
courses_offered = user.get_courses_offered()
if course['id'] in courses_offered:
courses_offered.remove(course['id'])
user.set_courses_offered(courses_offered)
user.set_role(JBoxUserV2.ROLE_OFFER_COURSES)
user.save()
return 0
from jbox_util import JBoxCfg, LoggerMixin, unique_sessname
from juliabox import db
from juliabox.db import JBoxDynConfig, JBoxSessionProps, JBoxUserV2
from juliabox.cloud import JBPluginCloud
from juliabox.cloud import Compute
conf_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../conf'))
conf_file = os.path.join(conf_dir, 'tornado.conf')
user_conf_file = os.path.join(conf_dir, 'jbox.user')
JBoxCfg.read(conf_file, user_conf_file)
JBoxCfg.dckr = docker.Client()
LoggerMixin.configure()
db.configure()
Compute.configure()
TESTCLSTR = 'testcluster'
class TestDBTables(LoggerMixin):
@staticmethod
def test():
sprops = JBoxSessionProps(TESTCLSTR, unique_sessname('*****@*****.**'))
TestDBTables.log_debug("JBoxSessionProps. user_id: %s, snapshot_id: %s, message: %s",
sprops.get_user_id(),
sprops.get_snapshot_id(),
sprops.get_message())
JBoxDynConfig.set_cluster_leader(TESTCLSTR, 'testinstance')
assert JBoxDynConfig.get_cluster_leader(TESTCLSTR) == 'testinstance'
def _backup(self, clear_volume=False):
sess_props = JBoxSessionProps(Compute.get_install_id(), self.sessname)
desc = sess_props.get_user_id() + " JuliaBox Backup"
disk_id = self.disk_path.split('/')[-1]
snap_id = EBSVol.snapshot_volume(dev_id=disk_id, tag=self.sessname, description=desc, wait_till_complete=False)
return snap_id
def publish_stats():
VolMgr.calc_stats()
VolMgr.log_debug("stats: %r", VolMgr.STATS)
JBoxDynConfig.set_stat(Compute.get_install_id(), VolMgr.STAT_NAME, VolMgr.STATS)
def publish_stats():
JBoxUserV2.calc_stats()
JBoxDynConfig.set_stat(Compute.get_install_id(), JBoxUserV2.STAT_NAME,
JBoxUserV2.STATS)
