def init():
dckr = docker.Client()
cfg = read_config()
cloud_cfg = cfg['cloud_host']
cloud_cfg['backup_bucket'] = "juliabox_userbackup"
LoggerMixin.setup_logger(level=cfg['root_log_level'])
LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level']
CloudHost.configure(
has_s3=True, #cloud_cfg['s3'],
has_dynamodb=cloud_cfg['dynamodb'],
has_cloudwatch=cloud_cfg['cloudwatch'],
has_autoscale=cloud_cfg['autoscale'],
has_route53=cloud_cfg['route53'],
has_ebs=cloud_cfg['ebs'],
has_ses=cloud_cfg['ses'],
scale_up_at_load=cloud_cfg['scale_up_at_load'],
scale_up_policy=cloud_cfg['scale_up_policy'],
autoscale_group=cloud_cfg['autoscale_group'],
route53_domain=cloud_cfg['route53_domain'],
region=cloud_cfg['region'],
install_id=cloud_cfg['install_id'])
VolMgr.configure(dckr, cfg)
def pull_backup(user_email):
sessname = unique_sessname(user_email)
S3Disk.log_info("pulling %s.tar.gz from %s", sessname,
JBoxVol.BACKUP_BUCKET)
CloudHost.pull_file_from_s3(JBoxVol.BACKUP_BUCKET,
sessname + ".tar.gz",
metadata_only=False)
def auto_activate():
try:
num_mails_24h, rate = CloudHost.get_email_rates()
rate_per_second = min(JBoxd.MAX_ACTIVATIONS_PER_SEC, rate)
num_mails = min(JBoxd.MAX_AUTO_ACTIVATIONS_PER_RUN, num_mails_24h)
JBoxd.log_info(
"Will activate max %d users at %d per second. AWS limits: %d mails at %d per second",
num_mails, rate_per_second, num_mails_24h, rate)
user_ids = JBoxUserV2.get_pending_activations(num_mails)
JBoxd.log_info("Got %d user_ids to be activated", len(user_ids))
for user_id in user_ids:
JBoxd.log_info("Activating %s", user_id)
# send email by SES
CloudHost.send_email(user_id, JBoxd.ACTIVATION_SENDER,
JBoxd.ACTIVATION_SUBJECT,
JBoxd.ACTIVATION_BODY)
# set user as activated
user = JBoxUserV2(user_id)
user.set_activation_state(JBoxUserV2.ACTIVATION_CODE_AUTO,
JBoxUserV2.ACTIVATION_GRANTED)
user.save()
rate_per_second -= 1
if rate_per_second <= 0:
time.sleep(1)
rate_per_second = min(JBoxd.MAX_ACTIVATIONS_PER_SEC, rate)
finally:
JBoxd.finish_thread()
def __init__(self):
dckr = docker.Client()
cfg = read_config()
cloud_cfg = cfg['cloud_host']
user_activation_cfg = cfg['user_activation']
LoggerMixin.setup_logger(level=cfg['root_log_level'])
LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level']
db.configure_db(cfg)
CloudHost.configure(has_s3=cloud_cfg['s3'],
has_dynamodb=cloud_cfg['dynamodb'],
has_cloudwatch=cloud_cfg['cloudwatch'],
has_autoscale=cloud_cfg['autoscale'],
has_route53=cloud_cfg['route53'],
has_ebs=cloud_cfg['ebs'],
has_ses=cloud_cfg['ses'],
scale_up_at_load=cloud_cfg['scale_up_at_load'],
scale_up_policy=cloud_cfg['scale_up_policy'],
autoscale_group=cloud_cfg['autoscale_group'],
route53_domain=cloud_cfg['route53_domain'],
region=cloud_cfg['region'],
install_id=cloud_cfg['install_id'])
VolMgr.configure(dckr, cfg)
JBoxContainer.configure(dckr, cfg['docker_image'], cfg['mem_limit'], cfg['cpu_limit'],
cfg['numlocalmax'], cfg['async_job_port'], async_mode=JBoxAsyncJob.MODE_SUB)
self.log_debug("Backup daemon listening on port: " + str(cfg['async_job_port']))
self.queue = JBoxContainer.ASYNC_JOB
JBoxd.MAX_ACTIVATIONS_PER_SEC = user_activation_cfg['max_activations_per_sec']
JBoxd.MAX_AUTO_ACTIVATIONS_PER_RUN = user_activation_cfg['max_activations_per_run']
JBoxd.ACTIVATION_SUBJECT = user_activation_cfg['mail_subject']
JBoxd.ACTIVATION_BODY = user_activation_cfg['mail_body']
JBoxd.ACTIVATION_SENDER = user_activation_cfg['sender']
def handle_if_instance_info(self, is_allowed):
stats = self.get_argument('instance_info', None)
if stats is None:
return False
if not is_allowed:
AdminHandler.log_error("Show instance info not allowed for user")
response = {'code': -1, 'data': 'You do not have permissions to view these stats'}
else:
try:
if stats == 'load':
stats = {}
# get cluster loads
average_load = CloudHost.get_cluster_average_stats('Load')
if None != average_load:
stats['Average Load'] = average_load;
machine_loads = CloudHost.get_cluster_stats('Load')
if None != machine_loads:
for n, v in machine_loads.iteritems():
stats['Instance ' + n] = v
response = {'code': 0, 'data': stats} if stats is not None else {'code': 1, 'data': {}}
except:
AdminHandler.log_error("exception while getting stats")
response = {'code': -1, 'data': 'error getting stats'}
self.write(response)
return True
def auto_activate():
try:
num_mails_24h, rate = CloudHost.get_email_rates()
rate_per_second = min(JBoxd.MAX_ACTIVATIONS_PER_SEC, rate)
num_mails = min(JBoxd.MAX_AUTO_ACTIVATIONS_PER_RUN, num_mails_24h)
JBoxd.log_info("Will activate max %d users at %d per second. AWS limits: %d mails at %d per second",
num_mails, rate_per_second,
num_mails_24h, rate)
user_ids = JBoxUserV2.get_pending_activations(num_mails)
JBoxd.log_info("Got %d user_ids to be activated", len(user_ids))
for user_id in user_ids:
JBoxd.log_info("Activating %s", user_id)
# send email by SES
CloudHost.send_email(user_id, JBoxd.ACTIVATION_SENDER, JBoxd.ACTIVATION_SUBJECT, JBoxd.ACTIVATION_BODY)
# set user as activated
user = JBoxUserV2(user_id)
user.set_activation_state(JBoxUserV2.ACTIVATION_CODE_AUTO, JBoxUserV2.ACTIVATION_GRANTED)
user.save()
rate_per_second -= 1
if rate_per_second <= 0:
time.sleep(1)
rate_per_second = min(JBoxd.MAX_ACTIVATIONS_PER_SEC, rate)
finally:
JBoxd.finish_thread()
def release(self, backup=False):
if not JBoxEBSVol.HAS_EBS:
raise Exception("EBS disks not enabled")
disk_id = self.disk_path.split('/')[-1]
CloudHost.unmount_device(disk_id, JBoxEBSVol.FS_LOC)
if backup:
self._backup()
vol_id = CloudHost.get_volume_id_from_device(disk_id)
CloudHost.detach_volume(vol_id, delete=True)
def release(self, backup=False):
if not JBoxEBSVol.HAS_EBS:
raise Exception("EBS disks not enabled")
disk_id = self.disk_path.split('/')[-1]
CloudHost.unmount_device(disk_id, JBoxEBSVol.FS_LOC)
if backup:
self._backup()
vol_id = CloudHost.get_volume_id_from_device(disk_id)
CloudHost.detach_volume(vol_id, delete=True)
def test():
CloudHost.send_email('*****@*****.**', '*****@*****.**', "test SES",
"""hello world
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt
ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation
ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in
reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit
anim id est laborum.""")
def _backup(self, clear_volume=False):
sess_props = JBoxSessionProps(self.sessname)
desc = sess_props.get_user_id() + " JuliaBox Backup"
disk_id = self.disk_path.split('/')[-1]
snap_id = CloudHost.snapshot_volume(dev_id=disk_id, tag=self.sessname, description=desc)
old_snap_id = sess_props.get_snapshot_id()
sess_props.set_snapshot_id(snap_id)
sess_props.save()
if old_snap_id is not None:
CloudHost.delete_snapshot(old_snap_id)
def test():
CloudHost.send_email(
'*****@*****.**', '*****@*****.**', "test SES",
"""hello world
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt
ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation
ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in
reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit
anim id est laborum.""")
def _backup(self, clear_volume=False):
sess_props = JBoxSessionProps(self.sessname)
desc = sess_props.get_user_id() + " JuliaBox Backup"
disk_id = self.disk_path.split('/')[-1]
snap_id = CloudHost.snapshot_volume(dev_id=disk_id,
tag=self.sessname,
description=desc)
old_snap_id = sess_props.get_snapshot_id()
sess_props.set_snapshot_id(snap_id)
sess_props.save()
if old_snap_id is not None:
CloudHost.delete_snapshot(old_snap_id)
def __init__(self):
dckr = docker.Client()
cfg = JBox.cfg = read_config()
cloud_cfg = cfg['cloud_host']
LoggerMixin.setup_logger(level=cfg['root_log_level'])
LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level']
JBoxHandler.configure(cfg)
db.configure_db(cfg)
CloudHost.configure(has_s3=cloud_cfg['s3'],
has_dynamodb=cloud_cfg['dynamodb'],
has_cloudwatch=cloud_cfg['cloudwatch'],
has_autoscale=cloud_cfg['autoscale'],
has_route53=cloud_cfg['route53'],
has_ebs=cloud_cfg['ebs'],
has_ses=cloud_cfg['ses'],
scale_up_at_load=cloud_cfg['scale_up_at_load'],
scale_up_policy=cloud_cfg['scale_up_policy'],
autoscale_group=cloud_cfg['autoscale_group'],
route53_domain=cloud_cfg['route53_domain'],
region=cloud_cfg['region'],
install_id=cloud_cfg['install_id'])
VolMgr.configure(dckr, cfg)
JBoxAsyncJob.configure(cfg)
JBoxContainer.configure(dckr, cfg['docker_image'], cfg['mem_limit'],
cfg['cpu_limit'], cfg['numlocalmax'],
cfg['async_job_ports'])
self.application = tornado.web.Application([
(r"/", MainHandler), (r"/hostlaunchipnb/", AuthHandler),
(r"/hostadmin/", AdminHandler), (r"/ping/", PingHandler),
(r"/cors/", CorsHandler)
])
cookie_secret = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for x in xrange(32))
self.application.settings["cookie_secret"] = cookie_secret
self.application.settings["google_oauth"] = cfg["google_oauth"]
self.application.listen(cfg["port"])
self.ioloop = tornado.ioloop.IOLoop.instance()
# run container maintainence every 5 minutes
run_interval = 5 * 60 * 1000
self.log_info("Container maintenance every " +
str(run_interval / (60 * 1000)) + " minutes")
self.ct = tornado.ioloop.PeriodicCallback(JBox.do_housekeeping,
run_interval, self.ioloop)
def __init__(self):
dckr = docker.Client()
cfg = JBox.cfg = read_config()
cloud_cfg = cfg['cloud_host']
LoggerMixin.setup_logger(level=cfg['root_log_level'])
LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level']
JBoxHandler.configure(cfg)
db.configure_db(cfg)
CloudHost.configure(has_s3=cloud_cfg['s3'],
has_dynamodb=cloud_cfg['dynamodb'],
has_cloudwatch=cloud_cfg['cloudwatch'],
has_autoscale=cloud_cfg['autoscale'],
has_route53=cloud_cfg['route53'],
has_ebs=cloud_cfg['ebs'],
has_ses=cloud_cfg['ses'],
scale_up_at_load=cloud_cfg['scale_up_at_load'],
scale_up_policy=cloud_cfg['scale_up_policy'],
autoscale_group=cloud_cfg['autoscale_group'],
route53_domain=cloud_cfg['route53_domain'],
region=cloud_cfg['region'],
install_id=cloud_cfg['install_id'])
VolMgr.configure(dckr, cfg)
JBoxAsyncJob.configure(cfg)
JBoxContainer.configure(dckr, cfg['docker_image'], cfg['mem_limit'], cfg['cpu_limit'],
cfg['numlocalmax'], cfg['async_job_ports'])
self.application = tornado.web.Application([
(r"/", MainHandler),
(r"/hostlaunchipnb/", AuthHandler),
(r"/hostadmin/", AdminHandler),
(r"/ping/", PingHandler),
(r"/cors/", CorsHandler),
(r"/hw/", HomeworkHandler)
])
cookie_secret = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in xrange(32))
self.application.settings["cookie_secret"] = cookie_secret
self.application.settings["google_oauth"] = cfg["google_oauth"]
self.application.listen(cfg["port"])
self.ioloop = tornado.ioloop.IOLoop.instance()
# run container maintainence every 5 minutes
run_interval = 5 * 60 * 1000
self.log_info("Container maintenance every " + str(run_interval / (60 * 1000)) + " minutes")
self.ct = tornado.ioloop.PeriodicCallback(JBox.do_housekeeping, run_interval, self.ioloop)
def __init__(self, ports, mode):
self._mode = mode
self._ctx = zmq.Context()
ppmode = zmq.PUSH if (mode == JBoxAsyncJob.MODE_PUB) else zmq.PULL
self._push_pull_sock = self._ctx.socket(ppmode)
rrmode = zmq.REQ if (mode == JBoxAsyncJob.MODE_PUB) else zmq.REP
local_ip = CloudHost.instance_local_ip()
JBoxAsyncJob.log_debug("local hostname [%s]", local_ip)
ppbindaddr = 'tcp://%s:%d' % (local_ip, ports[0],)
ppconnaddr = 'tcp://%s:%d' % (local_ip, ports[0],)
rraddr = 'tcp://%s:%d' % (local_ip, ports[1],)
self._rrport = ports[1]
self._poller = zmq.Poller()
if mode == JBoxAsyncJob.MODE_PUB:
self._push_pull_sock.bind(ppbindaddr)
else:
self._push_pull_sock.connect(ppconnaddr)
self._poller.register(self._push_pull_sock, zmq.POLLIN)
self._req_rep_sock = self._ctx.socket(rrmode)
self._req_rep_sock.bind(rraddr)
def push_backup(user_email, disk_path):
sessname = unique_sessname(user_email)
S3Disk.log_info("pushing %s.tar.gz from %s to %s", sessname, disk_path,
JBoxVol.BACKUP_BUCKET)
bkup_file = os.path.join('/tmp', sessname + ".tar.gz")
bkup_tar = tarfile.open(bkup_file, 'w:gz')
def set_perms(tinfo):
tinfo.uid = 1000
tinfo.gid = 1000
tinfo.uname = 'ubuntu'
tinfo.gname = 'ubuntu'
return tinfo
for f in os.listdir(disk_path):
if f.startswith('.') and (f in ['.julia', '.juliabox']):
continue
full_path = os.path.join(disk_path, f)
bkup_tar.add(full_path, os.path.join('juser', f), filter=set_perms)
bkup_tar.close()
os.chmod(bkup_file, 0666)
# Upload to S3 if so configured. Delete from local if successful.
bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \
datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET)
if JBoxVol.BACKUP_BUCKET is not None:
if CloudHost.push_file_to_s3(
JBoxVol.BACKUP_BUCKET,
bkup_file,
metadata={'backup_time':
bkup_file_mtime.isoformat()}) is not None:
os.remove(bkup_file)
S3Disk.log_info("Moved backup to S3 " + sessname)
def sendrecv(self, cmd, data, dest=None, port=None):
if (dest is None) or (dest == 'localhost'):
dest = '127.0.0.1'
else:
dest = CloudHost.instance_local_ip(dest)
if port is None:
port = self._rrport
rraddr = 'tcp://%s:%d' % (dest, port)
JBoxAsyncJob.log_debug("sendrecv to %s. connecting...", rraddr)
sock = self._ctx.socket(zmq.REQ)
sock.setsockopt(zmq.LINGER, 5 * 1000)
sock.connect(rraddr)
poller = zmq.Poller()
poller.register(sock, zmq.POLLOUT)
if poller.poll(10 * 1000):
sock.send_json(self._make_msg(cmd, data))
else:
sock.close()
raise IOError("could not connect to %s", rraddr)
poller.modify(sock, zmq.POLLIN)
if poller.poll(10 * 1000):
msg = sock.recv_json()
else:
sock.close()
raise IOError("did not receive anything from %s", rraddr)
JBoxAsyncJob.log_debug("sendrecv to %s. received.", rraddr)
sock.close()
return msg
def sendrecv(self, cmd, data, dest=None, port=None):
if (dest is None) or (dest == 'localhost'):
dest = '127.0.0.1'
else:
dest = CloudHost.instance_local_ip(dest)
if port is None:
port = self._rrport
rraddr = 'tcp://%s:%d' % (dest, port)
JBoxAsyncJob.log_debug("sendrecv to %s. connecting...", rraddr)
sock = self._ctx.socket(zmq.REQ)
sock.setsockopt(zmq.LINGER, 5*1000)
sock.connect(rraddr)
poller = zmq.Poller()
poller.register(sock, zmq.POLLOUT)
if poller.poll(10*1000):
sock.send_json(self._make_msg(cmd, data))
else:
sock.close()
raise IOError("could not connect to %s", rraddr)
poller.modify(sock, zmq.POLLIN)
if poller.poll(10*1000):
msg = sock.recv_json()
else:
sock.close()
raise IOError("did not receive anything from %s", rraddr)
JBoxAsyncJob.log_debug("sendrecv to %s. received.", rraddr)
sock.close()
return msg
def _backup(self, clear_volume=False):
JBoxVol.log_info("Backing up " + self.sessname + " at " + str(JBoxVol.BACKUP_LOC))
bkup_file = os.path.join(JBoxVol.BACKUP_LOC, self.sessname + ".tar.gz")
bkup_tar = tarfile.open(bkup_file, 'w:gz')
for f in os.listdir(self.disk_path):
if f.startswith('.') and (f in ['.julia', '.ipython']):
continue
full_path = os.path.join(self.disk_path, f)
bkup_tar.add(full_path, os.path.join('juser', f))
bkup_tar.close()
os.chmod(bkup_file, 0666)
if clear_volume:
ensure_delete(self.disk_path)
# Upload to S3 if so configured. Delete from local if successful.
bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \
datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET)
if JBoxVol.BACKUP_BUCKET is not None:
if CloudHost.push_file_to_s3(JBoxVol.BACKUP_BUCKET, bkup_file,
metadata={'backup_time': bkup_file_mtime.isoformat()}) is not None:
os.remove(bkup_file)
JBoxVol.log_info("Moved backup to S3 " + self.sessname)
def calc_stat(user_email):
VolMgr.STATS['num_users'] += 1
sessname = unique_sessname(user_email)
k = CloudHost.pull_file_from_s3(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz", metadata_only=True)
if k is not None:
VolMgr.STATS['loopback']['sizes'].append(k.size)
def push_backup(user_email, disk_path):
sessname = unique_sessname(user_email)
S3Disk.log_info("pushing %s.tar.gz from %s to %s", sessname, disk_path, JBoxVol.BACKUP_BUCKET)
bkup_file = os.path.join('/tmp', sessname + ".tar.gz")
bkup_tar = tarfile.open(bkup_file, 'w:gz')
def set_perms(tinfo):
tinfo.uid = 1000
tinfo.gid = 1000
tinfo.uname = 'ubuntu'
tinfo.gname = 'ubuntu'
return tinfo
for f in os.listdir(disk_path):
if f.startswith('.') and (f in ['.julia', '.juliabox']):
continue
full_path = os.path.join(disk_path, f)
bkup_tar.add(full_path, os.path.join('juser', f), filter=set_perms)
bkup_tar.close()
os.chmod(bkup_file, 0666)
# Upload to S3 if so configured. Delete from local if successful.
bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \
datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET)
if JBoxVol.BACKUP_BUCKET is not None:
if CloudHost.push_file_to_s3(JBoxVol.BACKUP_BUCKET, bkup_file,
metadata={'backup_time': bkup_file_mtime.isoformat()}) is not None:
os.remove(bkup_file)
S3Disk.log_info("Moved backup to S3 " + sessname)
def __init__(self, ports, mode):
self._mode = mode
self._ctx = zmq.Context()
ppmode = zmq.PUSH if (mode == JBoxAsyncJob.MODE_PUB) else zmq.PULL
self._push_pull_sock = self._ctx.socket(ppmode)
rrmode = zmq.REQ if (mode == JBoxAsyncJob.MODE_PUB) else zmq.REP
local_ip = CloudHost.instance_local_ip()
JBoxAsyncJob.log_debug("local hostname [%s]", local_ip)
ppbindaddr = 'tcp://%s:%d' % (
local_ip,
ports[0],
)
ppconnaddr = 'tcp://%s:%d' % (
local_ip,
ports[0],
)
rraddr = 'tcp://%s:%d' % (
local_ip,
ports[1],
)
self._rrport = ports[1]
self._poller = zmq.Poller()
if mode == JBoxAsyncJob.MODE_PUB:
self._push_pull_sock.bind(ppbindaddr)
else:
self._push_pull_sock.connect(ppconnaddr)
self._poller.register(self._push_pull_sock, zmq.POLLIN)
self._req_rep_sock = self._ctx.socket(rrmode)
self._req_rep_sock.bind(rraddr)
def _backup(self, clear_volume=False):
JBoxVol.log_info("Backing up " + self.sessname + " at " +
str(JBoxVol.BACKUP_LOC))
bkup_file = os.path.join(JBoxVol.BACKUP_LOC, self.sessname + ".tar.gz")
bkup_tar = tarfile.open(bkup_file, 'w:gz')
for f in os.listdir(self.disk_path):
if f.startswith('.') and (f in ['.julia', '.juliabox']):
continue
full_path = os.path.join(self.disk_path, f)
bkup_tar.add(full_path, os.path.join('juser', f))
bkup_tar.close()
os.chmod(bkup_file, 0666)
if clear_volume:
ensure_delete(self.disk_path)
# Upload to S3 if so configured. Delete from local if successful.
bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \
datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET)
if JBoxVol.BACKUP_BUCKET is not None:
if CloudHost.push_file_to_s3(
JBoxVol.BACKUP_BUCKET,
bkup_file,
metadata={'backup_time':
bkup_file_mtime.isoformat()}) is not None:
os.remove(bkup_file)
JBoxVol.log_info("Moved backup to S3 " + self.sessname)
def is_proposed_cluster_leader():
if not CloudHost.ENABLED['cloudwatch']:
return False
cluster = CloudHost.INSTALL_ID
leader = JBoxDynConfig.get_cluster_leader(cluster)
return leader == CloudHost.instance_id()
def get(self):
args = self.get_argument('m', default=None)
if args is not None:
args = json.loads(
decrypt(base64.b64decode(args), self.config(key='sesskey')))
if args is not None:
self.log_debug("setting cookies")
for cname in [
'sessname', 'hostshell', 'hostupload', 'hostipnb', 'sign',
'juliabox'
]:
self.set_cookie(cname, args[cname])
self.set_status(status_code=204)
self.finish()
else:
args = dict()
for cname in [
'sessname', 'hostshell', 'hostupload', 'hostipnb', 'sign',
'juliabox'
]:
args[cname] = self.get_cookie(cname)
args = tornado.escape.url_escape(
base64.b64encode(
encrypt(json.dumps(args), self.config(key='sesskey'))))
url = "//" + CloudHost.notebook_websocket_hostname(
) + "/cors/?m=" + args
self.log_debug("redirecting to " + url)
self.redirect(url)
def get_disk_for_user(user_email):
JBoxEBSVol.log_debug("creating EBS volume for %s", user_email)
if not JBoxEBSVol.HAS_EBS:
raise Exception("EBS disks not enabled")
disk_id = JBoxEBSVol._reserve_disk_id()
if disk_id is None:
raise Exception("No free disk available")
sess_id = unique_sessname(user_email)
sess_props = JBoxSessionProps(sess_id, create=True, user_id=user_email)
if sess_props.is_new:
sess_props.save()
snap_id = sess_props.get_snapshot_id()
if snap_id is None:
snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT
JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id, user_email)
_dev_path, mnt_path = CloudHost.create_new_volume(snap_id, disk_id, JBoxEBSVol.FS_LOC, tag=user_email)
ebsvol = JBoxEBSVol(mnt_path, user_email=user_email)
if snap_id == JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT:
JBoxEBSVol.log_debug("creating home folder on blank volume for %s", user_email)
ebsvol.restore_user_home()
ebsvol.restore()
#else:
# snap_age_days = CloudHost.get_snapshot_age(snap_id).total_seconds()/(60*60*24)
# if snap_age_days > 7:
# ebsvol.restore_user_home()
JBoxEBSVol.log_debug("setting up instance configuration on disk for %s", user_email)
ebsvol.setup_instance_config()
return ebsvol
def is_proposed_cluster_leader():
if not CloudHost.ENABLED['cloudwatch']:
return False
cluster = CloudHost.INSTALL_ID
leader = JBoxDynConfig.get_cluster_leader(cluster)
return leader == CloudHost.instance_id()
def is_ready_to_terminate():
if not JBox.cfg['cloud_host']['scale_down']:
return False
num_containers = JBoxContainer.num_active(
) + JBoxContainer.num_stopped()
return (num_containers == 0) and CloudHost.can_terminate(
is_proposed_cluster_leader())
def calc_stat(user_email):
VolMgr.STATS['num_users'] += 1
sessname = unique_sessname(user_email)
k = CloudHost.pull_file_from_s3(JBoxVol.BACKUP_BUCKET,
sessname + ".tar.gz",
metadata_only=True)
if k is not None:
VolMgr.STATS['loopback']['sizes'].append(k.size)
def __init__(self):
dckr = docker.Client()
cfg = read_config()
cloud_cfg = cfg['cloud_host']
user_activation_cfg = cfg['user_activation']
LoggerMixin.setup_logger(level=cfg['root_log_level'])
LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level']
db.configure_db(cfg)
CloudHost.configure(has_s3=cloud_cfg['s3'],
has_dynamodb=cloud_cfg['dynamodb'],
has_cloudwatch=cloud_cfg['cloudwatch'],
has_autoscale=cloud_cfg['autoscale'],
has_route53=cloud_cfg['route53'],
has_ebs=cloud_cfg['ebs'],
has_ses=cloud_cfg['ses'],
scale_up_at_load=cloud_cfg['scale_up_at_load'],
scale_up_policy=cloud_cfg['scale_up_policy'],
autoscale_group=cloud_cfg['autoscale_group'],
route53_domain=cloud_cfg['route53_domain'],
region=cloud_cfg['region'],
install_id=cloud_cfg['install_id'])
VolMgr.configure(dckr, cfg)
JBoxAsyncJob.configure(cfg)
JBoxContainer.configure(dckr,
cfg['docker_image'],
cfg['mem_limit'],
cfg['cpu_limit'],
cfg['numlocalmax'],
cfg['async_job_ports'],
async_mode=JBoxAsyncJob.MODE_SUB)
self.log_debug("Backup daemon listening on ports: %s",
repr(cfg['async_job_ports']))
JBoxd.QUEUE = JBoxContainer.ASYNC_JOB
JBoxd.MAX_ACTIVATIONS_PER_SEC = user_activation_cfg[
'max_activations_per_sec']
JBoxd.MAX_AUTO_ACTIVATIONS_PER_RUN = user_activation_cfg[
'max_activations_per_run']
JBoxd.ACTIVATION_SUBJECT = user_activation_cfg['mail_subject']
JBoxd.ACTIVATION_BODY = user_activation_cfg['mail_body']
JBoxd.ACTIVATION_SENDER = user_activation_cfg['sender']
def setup_instance_config(self, profiles=('julia', 'jboxjulia')):
for profile in profiles:
profile_path = '.ipython/profile_' + profile
profile_path = os.path.join(self.disk_path, profile_path)
if not os.path.exists(profile_path):
continue
nbconfig = os.path.join(profile_path, 'ipython_notebook_config.py')
wsock_cfg = "c.NotebookApp.websocket_url = '" + JBoxVol.NOTEBOOK_WEBSOCK_PROTO + \
CloudHost.notebook_websocket_hostname() + "'\n"
JBoxVol.replace_in_file(nbconfig, "c.NotebookApp.websocket_url", wsock_cfg)
def setup_instance_config(self, profiles=('julia', 'jboxjulia')):
for profile in profiles:
profile_path = '.ipython/profile_' + profile
profile_path = os.path.join(self.disk_path, profile_path)
if not os.path.exists(profile_path):
continue
nbconfig = os.path.join(profile_path, 'ipython_notebook_config.py')
wsock_cfg = "c.NotebookApp.websocket_url = '" + JBoxVol.NOTEBOOK_WEBSOCK_PROTO + \
CloudHost.notebook_websocket_hostname() + "'\n"
JBoxVol.replace_in_file(nbconfig, "c.NotebookApp.websocket_url", wsock_cfg)
def handle_if_instance_info(self, is_allowed):
stats = self.get_argument('instance_info', None)
if stats is None:
return False
if not is_allowed:
AdminHandler.log_error("Show instance info not allowed for user")
response = {'code': -1, 'data': 'You do not have permissions to view these stats'}
else:
try:
if stats == 'load':
result = {}
# get cluster loads
average_load = CloudHost.get_cluster_average_stats('Load')
if None != average_load:
result['Average Load'] = average_load;
machine_loads = CloudHost.get_cluster_stats('Load')
if None != machine_loads:
for n, v in machine_loads.iteritems():
result['Instance ' + n] = v
elif stats == 'sessions':
result = {}
if CloudHost.ENABLED['autoscale']:
instances = CloudHost.get_autoscaled_instances()
else:
instances = ['localhost']
for idx in range(0, len(instances)):
inst = instances[idx]
result[inst] = JBoxContainer.sync_session_status(inst)['data']
else:
raise Exception("unknown command %s" % (stats,))
response = {'code': 0, 'data': result}
except:
AdminHandler.log_error("exception while getting stats")
AdminHandler._get_logger().exception("exception while getting stats")
response = {'code': -1, 'data': 'error getting stats'}
self.write(response)
return True
def run(self):
if CloudHost.ENABLED['route53']:
try:
CloudHost.deregister_instance_dns()
CloudHost.log_warn("Prior dns registration was found for the instance")
except:
CloudHost.log_debug("No prior dns registration found for the instance")
CloudHost.register_instance_dns()
JBoxContainer.publish_container_stats()
JBox.do_update_user_home_image()
JBoxContainer.async_refresh_disks()
self.ct.start()
self.ioloop.start()
def release(self, backup=False):
if not JBoxEBSVol.HAS_EBS:
raise Exception("EBS disks not enabled")
disk_id = self.disk_path.split('/')[-1]
CloudHost.unmount_device(disk_id, JBoxEBSVol.FS_LOC)
if backup:
snap_id = self._backup()
else:
snap_id = None
vol_id = CloudHost.get_volume_id_from_device(disk_id)
CloudHost.detach_volume(vol_id, delete=False)
sess_props = JBoxSessionProps(self.sessname)
existing_disk = JBoxDiskState(cluster_id=CloudHost.INSTALL_ID, region_id=CloudHost.REGION,
user_id=sess_props.get_user_id())
if snap_id is not None:
existing_disk.add_snapshot_id(snap_id)
existing_disk.set_detach_time()
existing_disk.set_state(JBoxDiskState.STATE_DETACHED)
existing_disk.save()
def try_launch_container(cls, user_id, max_hop=False):
sessname = unique_sessname(user_id)
cont = JBoxContainer.get_by_name(sessname)
cls.log_debug("have existing container for %s: %r", sessname, None != cont)
if cont is not None:
cls.log_debug("container running: %r", cont.is_running())
if max_hop:
self_load = CloudHost.get_instance_stats(CloudHost.instance_id(), 'Load')
if self_load < 100:
JBoxContainer.async_launch_by_name(sessname, user_id, True)
return True
is_leader = is_proposed_cluster_leader()
if ((cont is None) or (not cont.is_running())) and (not CloudHost.should_accept_session(is_leader)):
if cont is not None:
cont.async_backup_and_cleanup()
return False
JBoxContainer.async_launch_by_name(sessname, user_id, True)
return True
def is_cluster_leader():
if not CloudHost.ENABLED['cloudwatch']:
return False
cluster = CloudHost.INSTALL_ID
instances = CloudHost.get_autoscaled_instances()
leader = JBoxDynConfig.get_cluster_leader(cluster)
ami_recentness = CloudHost.get_ami_recentness()
JBoxDB.log_debug("cluster: %s. instances: %s. leader: %s. ami_recentness: %d",
cluster, repr(instances), repr(leader), ami_recentness)
# if none set, or set instance is dead elect self as leader, but wait till next cycle to prevent conflicts
if (leader is None) or (leader not in instances) and (ami_recentness >= 0):
JBoxDB.log_info("setting self (%s) as cluster leader", CloudHost.instance_id())
JBoxDynConfig.set_cluster_leader(cluster, CloudHost.instance_id())
return False
is_leader = (leader == CloudHost.instance_id())
# if running an older ami, step down from cluster leader
if (ami_recentness < 0) and is_leader:
JBoxDB.log_info("unmarking self (%s) as cluster leader", CloudHost.instance_id())
JBoxDynConfig.unset_cluster_leader(cluster)
return False
return is_leader
def monitor_registrations():
max_rate = JBoxDynConfig.get_registration_hourly_rate(
CloudHost.INSTALL_ID)
rate = JBoxUserV2.count_created(1)
reg_allowed = JBoxDynConfig.get_allow_registration(
CloudHost.INSTALL_ID)
CloudHost.log_debug(
"registration allowed: %r, rate: %d, max allowed: %d", reg_allowed,
rate, max_rate)
if (reg_allowed and
(rate > max_rate * 1.1)) or ((not reg_allowed) and
(rate < max_rate * 0.9)):
reg_allowed = not reg_allowed
CloudHost.log_info("Changing registration allowed to %r",
reg_allowed)
JBoxDynConfig.set_allow_registration(CloudHost.INSTALL_ID,
reg_allowed)
if reg_allowed:
num_pending_activations = JBoxUserV2.count_pending_activations()
if num_pending_activations > 0:
CloudHost.log_info(
"scheduling activations for %d pending activations",
num_pending_activations)
JBoxContainer.async_schedule_activations()
def run(self):
try:
CloudHost.deregister_instance_dns()
CloudHost.log_warn(
"Prior dns registration was found for the instance")
except:
CloudHost.log_debug(
"No prior dns registration found for the instance")
CloudHost.register_instance_dns()
JBoxContainer.publish_container_stats()
JBox.do_update_user_home_image()
JBoxContainer.async_refresh_disks()
self.ct.start()
self.ioloop.start()
def is_cluster_leader():
if not CloudHost.ENABLED['cloudwatch']:
return False
cluster = CloudHost.INSTALL_ID
instances = CloudHost.get_autoscaled_instances()
leader = JBoxDynConfig.get_cluster_leader(cluster)
ami_recentness = CloudHost.get_ami_recentness()
JBoxDB.log_debug(
"cluster: %s. instances: %s. leader: %s. ami_recentness: %d", cluster,
repr(instances), repr(leader), ami_recentness)
# if none set, or set instance is dead elect self as leader, but wait till next cycle to prevent conflicts
if (leader is None) or (leader not in instances) and (ami_recentness >= 0):
JBoxDB.log_info("setting self (%s) as cluster leader",
CloudHost.instance_id())
JBoxDynConfig.set_cluster_leader(cluster, CloudHost.instance_id())
return False
is_leader = (leader == CloudHost.instance_id())
# if running an older ami, step down from cluster leader
if (ami_recentness < 0) and is_leader:
JBoxDB.log_info("unmarking self (%s) as cluster leader",
CloudHost.instance_id())
JBoxDynConfig.unset_cluster_leader(cluster)
return False
return is_leader
def do_housekeeping():
JBox.do_update_user_home_image()
server_delete_timeout = JBox.cfg['expire']
JBoxContainer.maintain(max_timeout=server_delete_timeout, inactive_timeout=JBox.cfg['inactivity_timeout'],
protected_names=JBox.cfg['protected_docknames'])
if JBox.cfg['cloud_host']['scale_down'] and (JBoxContainer.num_active() == 0) and \
(JBoxContainer.num_stopped() == 0) and CloudHost.should_terminate():
JBox.log_info("terminating to scale down")
try:
CloudHost.deregister_instance_dns()
except:
CloudHost.log_error("Error deregistering instance dns")
CloudHost.terminate_instance()
elif is_cluster_leader():
CloudHost.log_info("I am the cluster leader")
JBox.monitor_registrations()
if not JBoxDynConfig.is_stat_collected_within(CloudHost.INSTALL_ID, 7):
JBoxContainer.async_collect_stats()
def handle_if_instance_info(self, is_allowed):
stats = self.get_argument('instance_info', None)
if stats is None:
return False
if not is_allowed:
AdminHandler.log_error("Show instance info not allowed for user")
response = {
'code': -1,
'data': 'You do not have permissions to view these stats'
}
else:
try:
if stats == 'load':
stats = {}
# get cluster loads
average_load = CloudHost.get_cluster_average_stats('Load')
if None != average_load:
stats['Average Load'] = average_load
machine_loads = CloudHost.get_cluster_stats('Load')
if None != machine_loads:
for n, v in machine_loads.iteritems():
stats['Instance ' + n] = v
response = {
'code': 0,
'data': stats
} if stats is not None else {
'code': 1,
'data': {}
}
except:
AdminHandler.log_error("exception while getting stats")
response = {'code': -1, 'data': 'error getting stats'}
self.write(response)
return True
def process_commands(argv):
with open(argv[2]) as f:
uplcourse = eval(f.read())
cfg = read_config()
cloud_cfg = cfg['cloud_host']
LoggerMixin.setup_logger(level=cfg['root_log_level'])
LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level']
db.configure_db(cfg)
CloudHost.configure(has_s3=cloud_cfg['s3'],
has_dynamodb=cloud_cfg['dynamodb'],
has_cloudwatch=cloud_cfg['cloudwatch'],
has_autoscale=cloud_cfg['autoscale'],
has_route53=cloud_cfg['route53'],
has_ebs=cloud_cfg['ebs'],
has_ses=cloud_cfg['ses'],
scale_up_at_load=cloud_cfg['scale_up_at_load'],
scale_up_policy=cloud_cfg['scale_up_policy'],
autoscale_group=cloud_cfg['autoscale_group'],
route53_domain=cloud_cfg['route53_domain'],
region=cloud_cfg['region'],
install_id=cloud_cfg['install_id'])
cmd = argv[1]
if cmd == "upload":
HomeworkHandler.upload_course(None, uplcourse)
elif cmd == "report":
as_csv = (argv[3] == "csv") if len(argv) > 3 else False
get_report(uplcourse, as_csv)
elif cmd == "answers":
get_answers(uplcourse)
else:
print("Unknown option %s" % (cmd, ))
print("DONE!")
def process_commands(argv):
with open(argv[2]) as f:
uplcourse = eval(f.read())
cfg = read_config()
cloud_cfg = cfg['cloud_host']
LoggerMixin.setup_logger(level=cfg['root_log_level'])
LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level']
db.configure_db(cfg)
CloudHost.configure(has_s3=cloud_cfg['s3'],
has_dynamodb=cloud_cfg['dynamodb'],
has_cloudwatch=cloud_cfg['cloudwatch'],
has_autoscale=cloud_cfg['autoscale'],
has_route53=cloud_cfg['route53'],
has_ebs=cloud_cfg['ebs'],
has_ses=cloud_cfg['ses'],
scale_up_at_load=cloud_cfg['scale_up_at_load'],
scale_up_policy=cloud_cfg['scale_up_policy'],
autoscale_group=cloud_cfg['autoscale_group'],
route53_domain=cloud_cfg['route53_domain'],
region=cloud_cfg['region'],
install_id=cloud_cfg['install_id'])
cmd = argv[1]
if cmd == "upload":
HomeworkHandler.upload_course(None, uplcourse)
elif cmd == "report":
as_csv = (argv[3] == "csv") if len(argv) > 3 else False
get_report(uplcourse, as_csv)
elif cmd == "answers":
get_answers(uplcourse)
else:
print("Unknown option %s" % (cmd,))
print("DONE!")
def try_launch_container(cls, user_id):
sessname = unique_sessname(user_id)
cont = JBoxContainer.get_by_name(sessname)
cls.log_debug("have existing container for %s: %r", sessname, None != cont)
if cont is not None:
cls.log_debug("container running: %r", cont.is_running())
if ((cont is None) or (not cont.is_running())) and (not CloudHost.should_accept_session()):
if cont is not None:
cont.async_backup_and_cleanup()
return False
JBoxContainer.async_launch_by_name(sessname, user_id, True)
return True
def init():
dckr = docker.Client()
cfg = read_config()
cloud_cfg = cfg['cloud_host']
cloud_cfg['backup_bucket'] = "juliabox_userbackup"
LoggerMixin.setup_logger(level=cfg['root_log_level'])
LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level']
CloudHost.configure(has_s3=True, #cloud_cfg['s3'],
has_dynamodb=cloud_cfg['dynamodb'],
has_cloudwatch=cloud_cfg['cloudwatch'],
has_autoscale=cloud_cfg['autoscale'],
has_route53=cloud_cfg['route53'],
has_ebs=cloud_cfg['ebs'],
has_ses=cloud_cfg['ses'],
scale_up_at_load=cloud_cfg['scale_up_at_load'],
scale_up_policy=cloud_cfg['scale_up_policy'],
autoscale_group=cloud_cfg['autoscale_group'],
route53_domain=cloud_cfg['route53_domain'],
region=cloud_cfg['region'],
install_id=cloud_cfg['install_id'])
VolMgr.configure(dckr, cfg)
def try_launch_container(cls, user_id, max_hop=False):
sessname = unique_sessname(user_id)
cont = JBoxContainer.get_by_name(sessname)
cls.log_debug("have existing container for %s: %r", sessname,
None != cont)
if cont is not None:
cls.log_debug("container running: %r", cont.is_running())
if max_hop:
self_load = CloudHost.get_instance_stats(CloudHost.instance_id(),
'Load')
if self_load < 100:
JBoxContainer.async_launch_by_name(sessname, user_id, True)
return True
is_leader = is_proposed_cluster_leader()
if ((cont is None) or (not cont.is_running())) and (
not CloudHost.should_accept_session(is_leader)):
if cont is not None:
cont.async_backup_and_cleanup()
return False
JBoxContainer.async_launch_by_name(sessname, user_id, True)
return True
def update_disk_states():
detached_disks = JBoxDiskState.get_detached_disks()
time_now = datetime.datetime.now(pytz.utc)
for disk_key in detached_disks:
disk_info = JBoxDiskState(disk_key=disk_key)
user_id = disk_info.get_user_id()
sess_props = JBoxSessionProps(unique_sessname(user_id))
incomplete_snapshots = []
modified = False
for snap_id in disk_info.get_snapshot_ids():
if not CloudHost.is_snapshot_complete(snap_id):
incomplete_snapshots.append(snap_id)
continue
JBoxd.log_debug("updating latest snapshot of user %s to %s",
user_id, snap_id)
old_snap_id = sess_props.get_snapshot_id()
sess_props.set_snapshot_id(snap_id)
modified = True
if old_snap_id is not None:
CloudHost.delete_snapshot(old_snap_id)
if modified:
sess_props.save()
disk_info.set_snapshot_ids(incomplete_snapshots)
disk_info.save()
if len(incomplete_snapshots) == 0:
if (time_now - disk_info.get_detach_time()
).total_seconds() > 24 * 60 * 60:
vol_id = disk_info.get_volume_id()
JBoxd.log_debug(
"volume %s for user %s unused for too long", vol_id,
user_id)
disk_info.delete()
CloudHost.detach_volume(vol_id, delete=True)
else:
JBoxd.log_debug("ongoing snapshots of user %s: %r", user_id,
incomplete_snapshots)
def update_user_home_image(fetch=True):
img_dir, curr_img = os.path.split(JBoxVol.USER_HOME_IMG)
bucket, new_img = JBoxDynConfig.get_user_home_image(CloudHost.INSTALL_ID)
new_img_path = os.path.join(img_dir, new_img)
if fetch and (not os.path.exists(new_img_path)):
VolMgr.log_debug("fetching new image to %s", new_img_path)
k = CloudHost.pull_file_from_s3(bucket, new_img_path)
if k is not None:
VolMgr.log_debug("fetched new user home image")
if os.path.exists(new_img_path):
VolMgr.log_debug("set new image to %s", new_img_path)
JBoxVol.USER_HOME_IMG = new_img_path
return True
return False
def run(self):
try:
CloudHost.deregister_instance_dns()
except:
CloudHost.log_info("No prior dns registration found for the instance")
CloudHost.register_instance_dns()
JBoxContainer.publish_container_stats()
JBox.do_update_user_home_image()
JBoxContainer.async_refresh_disks()
self.ct.start()
self.ioloop.start()
def update_user_home_image(fetch=True):
img_dir, curr_img = os.path.split(JBoxVol.USER_HOME_IMG)
bucket, new_img = JBoxDynConfig.get_user_home_image(
CloudHost.INSTALL_ID)
new_img_path = os.path.join(img_dir, new_img)
if fetch and (not os.path.exists(new_img_path)):
VolMgr.log_debug("fetching new image to %s", new_img_path)
k = CloudHost.pull_file_from_s3(bucket, new_img_path)
if k is not None:
VolMgr.log_debug("fetched new user home image")
if os.path.exists(new_img_path):
VolMgr.log_debug("set new image to %s", new_img_path)
JBoxVol.USER_HOME_IMG = new_img_path
return True
return False
def setup_instance_config(self):
nbconfig = os.path.join(self.disk_path, '.ipython/profile_julia/ipython_notebook_config.py')
nbconfig_temp = os.path.join(self.disk_path, '.ipython/profile_julia/ipython_notebook_config.py.temp')
if os.path.exists(nbconfig_temp):
os.remove(nbconfig_temp)
os.rename(nbconfig, nbconfig_temp)
wsock_cfg = "c.NotebookApp.websocket_url = 'wss://" + CloudHost.notebook_websocket_hostname() + "'\n"
replaced = False
with open(nbconfig_temp) as fin, open(nbconfig, 'w') as fout:
for line in fin:
if line.startswith("c.NotebookApp.websocket_url"):
line = wsock_cfg
replaced = True
fout.write(line)
if not replaced:
fout.write(wsock_cfg)
def get_disk_for_user(user_email):
JBoxEBSVol.log_debug("creating EBS volume for %s", user_email)
if not JBoxEBSVol.HAS_EBS:
raise Exception("EBS disks not enabled")
disk_id = JBoxEBSVol._reserve_disk_id()
if disk_id is None:
raise Exception("No free disk available")
sess_id = unique_sessname(user_email)
sess_props = JBoxSessionProps(sess_id, create=True, user_id=user_email)
if sess_props.is_new:
sess_props.save()
snap_id = sess_props.get_snapshot_id()
if snap_id is None:
snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT
JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id,
user_email)
_dev_path, mnt_path = CloudHost.create_new_volume(snap_id,
disk_id,
JBoxEBSVol.FS_LOC,
tag=user_email)
ebsvol = JBoxEBSVol(mnt_path, user_email=user_email)
if snap_id == JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT:
JBoxEBSVol.log_debug("creating home folder on blank volume for %s",
user_email)
ebsvol.restore_user_home(True)
ebsvol.restore()
else:
JBoxEBSVol.log_debug(
"updating home folder on existing volume for %s", user_email)
ebsvol.restore_user_home(False)
# snap_age_days = CloudHost.get_snapshot_age(snap_id).total_seconds()/(60*60*24)
# if snap_age_days > 7:
# ebsvol.restore_user_home()
JBoxEBSVol.log_debug(
"setting up instance configuration on disk for %s", user_email)
ebsvol.setup_instance_config()
return ebsvol
def get(self):
args = self.get_argument('m', default=None)
if args is not None:
args = json.loads(decrypt(base64.b64decode(args), self.config(key='sesskey')))
if args is not None:
self.log_debug("setting cookies")
for cname in ['sessname', 'hostshell', 'hostupload', 'hostipnb', 'sign', 'juliabox']:
self.set_cookie(cname, args[cname])
self.set_status(status_code=204)
self.finish()
else:
args = dict()
for cname in ['sessname', 'hostshell', 'hostupload', 'hostipnb', 'sign', 'juliabox']:
args[cname] = self.get_cookie(cname)
args = tornado.escape.url_escape(base64.b64encode(encrypt(json.dumps(args), self.config(key='sesskey'))))
url = "//" + CloudHost.notebook_websocket_hostname() + "/cors/?m=" + args
self.log_debug("redirecting to " + url)
self.redirect(url)
def do_housekeeping():
terminating = False
server_delete_timeout = JBox.cfg['expire']
JBoxContainer.maintain(max_timeout=server_delete_timeout, inactive_timeout=JBox.cfg['inactivity_timeout'],
protected_names=JBox.cfg['protected_docknames'])
if is_cluster_leader():
CloudHost.log_info("I am the cluster leader")
JBox.monitor_registrations()
if not JBoxDynConfig.is_stat_collected_within(CloudHost.INSTALL_ID, 1):
JBoxContainer.async_collect_stats()
JBoxContainer.async_update_disk_state()
elif JBox.is_ready_to_terminate():
terminating = True
JBox.log_warn("terminating to scale down")
try:
CloudHost.deregister_instance_dns()
except:
CloudHost.log_error("Error deregistering instance dns")
CloudHost.terminate_instance()
if not terminating:
JBox.do_update_user_home_image()
def do_housekeeping():
terminating = False
server_delete_timeout = JBox.cfg['expire']
JBoxContainer.maintain(max_timeout=server_delete_timeout,
inactive_timeout=JBox.cfg['inactivity_timeout'],
protected_names=JBox.cfg['protected_docknames'])
if is_cluster_leader():
CloudHost.log_info("I am the cluster leader")
JBox.monitor_registrations()
if not JBoxDynConfig.is_stat_collected_within(
CloudHost.INSTALL_ID, 7):
JBoxContainer.async_collect_stats()
elif JBox.is_ready_to_terminate():
terminating = True
JBox.log_info("terminating to scale down")
try:
CloudHost.deregister_instance_dns()
except:
CloudHost.log_error("Error deregistering instance dns")
CloudHost.terminate_instance()
if not terminating:
JBox.do_update_user_home_image()
def monitor_registrations():
max_rate = JBoxDynConfig.get_registration_hourly_rate(CloudHost.INSTALL_ID)
rate = JBoxUserV2.count_created(1)
reg_allowed = JBoxDynConfig.get_allow_registration(CloudHost.INSTALL_ID)
CloudHost.log_debug("registration allowed: %r, rate: %d, max allowed: %d", reg_allowed, rate, max_rate)
if (reg_allowed and (rate > max_rate*1.1)) or ((not reg_allowed) and (rate < max_rate*0.9)):
reg_allowed = not reg_allowed
CloudHost.log_warn("Changing registration allowed to %r", reg_allowed)
JBoxDynConfig.set_allow_registration(CloudHost.INSTALL_ID, reg_allowed)
if reg_allowed:
num_pending_activations = JBoxUserV2.count_pending_activations()
if num_pending_activations > 0:
CloudHost.log_info("scheduling activations for %d pending activations", num_pending_activations)
JBoxContainer.async_schedule_activations()
def setup_instance_config(self):
nbconfig = os.path.join(
self.disk_path,
'.ipython/profile_julia/ipython_notebook_config.py')
nbconfig_temp = os.path.join(
self.disk_path,
'.ipython/profile_julia/ipython_notebook_config.py.temp')
if os.path.exists(nbconfig_temp):
os.remove(nbconfig_temp)
os.rename(nbconfig, nbconfig_temp)
wsock_cfg = "c.NotebookApp.websocket_url = '" + JBoxVol.NOTEBOOK_WEBSOCK_PROTO + \
CloudHost.notebook_websocket_hostname() + "'\n"
replaced = False
with open(nbconfig_temp) as fin, open(nbconfig, 'w') as fout:
for line in fin:
if line.startswith("c.NotebookApp.websocket_url"):
line = wsock_cfg
replaced = True
fout.write(line)
if not replaced:
fout.write(wsock_cfg)
def pull_backup(user_email):
sessname = unique_sessname(user_email)
S3Disk.log_info("pulling %s.tar.gz from %s", sessname, JBoxVol.BACKUP_BUCKET)
CloudHost.pull_file_from_s3(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz", metadata_only=False)
