def publish_container_stats():
""" Publish custom cloudwatch statistics. Used for status monitoring and auto scaling. """
nactive = JBoxContainer.num_active()
CloudHost.publish_stats("NumActiveContainers", "Count", nactive)
curr_cpu_used_pct = psutil.cpu_percent()
last_cpu_used_pct = curr_cpu_used_pct if JBoxContainer.LAST_CPU_PCT is None else JBoxContainer.LAST_CPU_PCT
JBoxContainer.LAST_CPU_PCT = curr_cpu_used_pct
cpu_used_pct = int((curr_cpu_used_pct + last_cpu_used_pct)/2)
mem_used_pct = psutil.virtual_memory().percent
CloudHost.publish_stats("MemUsed", "Percent", mem_used_pct)
disk_used_pct = 0
for x in psutil.disk_partitions():
if not VolMgr.is_mount_path(x.mountpoint):
try:
disk_used_pct = max(psutil.disk_usage(x.mountpoint).percent, disk_used_pct)
except:
pass
if JBoxContainer.INITIAL_DISK_USED_PCT is None:
JBoxContainer.INITIAL_DISK_USED_PCT = disk_used_pct
disk_used_pct = max(0, (disk_used_pct - JBoxContainer.INITIAL_DISK_USED_PCT))
CloudHost.publish_stats("DiskUsed", "Percent", disk_used_pct)
cont_load_pct = min(100, max(0, nactive * 100 / JBoxContainer.MAX_CONTAINERS))
CloudHost.publish_stats("ContainersUsed", "Percent", cont_load_pct)
CloudHost.publish_stats("DiskIdsUsed", "Percent", VolMgr.used_pct())
overall_load_pct = max(cont_load_pct, disk_used_pct, mem_used_pct, cpu_used_pct, VolMgr.used_pct())
CloudHost.publish_stats("Load", "Percent", overall_load_pct)
def collect_stats():
try:
VolMgr.publish_stats()
db.publish_stats()
JBoxDynConfig.set_stat_collected_date(CloudHost.INSTALL_ID)
finally:
JBoxd.finish_thread()
def collect_stats():
try:
VolMgr.publish_stats()
db.publish_stats()
JBoxDynConfig.set_stat_collected_date(CloudHost.INSTALL_ID)
finally:
JBoxd.finish_thread()
def init():
dckr = docker.Client()
cfg = read_config()
cloud_cfg = cfg['cloud_host']
cloud_cfg['backup_bucket'] = "juliabox_userbackup"
LoggerMixin.setup_logger(level=cfg['root_log_level'])
LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level']
CloudHost.configure(
has_s3=True, #cloud_cfg['s3'],
has_dynamodb=cloud_cfg['dynamodb'],
has_cloudwatch=cloud_cfg['cloudwatch'],
has_autoscale=cloud_cfg['autoscale'],
has_route53=cloud_cfg['route53'],
has_ebs=cloud_cfg['ebs'],
has_ses=cloud_cfg['ses'],
scale_up_at_load=cloud_cfg['scale_up_at_load'],
scale_up_policy=cloud_cfg['scale_up_policy'],
autoscale_group=cloud_cfg['autoscale_group'],
route53_domain=cloud_cfg['route53_domain'],
region=cloud_cfg['region'],
install_id=cloud_cfg['install_id'])
VolMgr.configure(dckr, cfg)
def __init__(self):
LoggerMixin.configure()
db.configure()
Compute.configure()
SessContainer.configure()
VolMgr.configure()
JBoxAsyncJob.configure()
JBoxAsyncJob.init(JBoxAsyncJob.MODE_PUB)
self.application = tornado.web.Application(handlers=[
(r"/", MainHandler),
(r"/jboxadmin/", AdminHandler),
(r"/jboxping/", PingHandler),
(r"/jboxcors/", CorsHandler)
])
JBPluginHandler.add_plugin_handlers(self.application)
JBPluginUI.create_include_files()
# cookie_secret = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in xrange(32))
# use sesskey as cookie secret to be able to span multiple tornado servers
self.application.settings["cookie_secret"] = JBoxCfg.get('sesskey')
self.application.settings["plugin_features"] = JBox.get_pluggedin_features()
self.application.listen(JBoxCfg.get('interactive.manager_port'), address=socket.gethostname())
self.application.listen(JBoxCfg.get('interactive.manager_port'), address='localhost')
self.ioloop = tornado.ioloop.IOLoop.instance()
# run container maintainence every 5 minutes
run_interval = 5 * 60 * 1000
self.log_info("Container maintenance every " + str(run_interval / (60 * 1000)) + " minutes")
self.ct = tornado.ioloop.PeriodicCallback(JBox.do_housekeeping, run_interval, self.ioloop)
self.sigct = tornado.ioloop.PeriodicCallback(JBox.do_signals, 1000, self.ioloop)
def run(self):
if VolMgr.has_update_for_user_home_image():
VolMgr.update_user_home_image(fetch=False)
while True:
self.log_debug("JBox daemon waiting for commands...")
cmd, data = self.queue.recv()
if cmd == JBoxAsyncJob.CMD_BACKUP_CLEANUP:
args = (data,)
fn = JBoxd.backup_and_cleanup
elif cmd == JBoxAsyncJob.CMD_LAUNCH_SESSION:
args = (data[0], data[1], data[2])
fn = JBoxd.launch_session
elif cmd == JBoxAsyncJob.CMD_AUTO_ACTIVATE:
args = ()
fn = JBoxd.auto_activate
elif cmd == JBoxAsyncJob.CMD_UPDATE_USER_HOME_IMAGE:
args = ()
fn = JBoxd.update_user_home_image
elif cmd == JBoxAsyncJob.CMD_REFRESH_DISKS:
args = ()
fn = JBoxd.refresh_disks
elif cmd == JBoxAsyncJob.CMD_COLLECT_STATS:
args = ()
fn = JBoxd.collect_stats
else:
self.log_error("Unknown command " + str(cmd))
continue
JBoxd.schedule_thread(cmd, fn, args)
def run(self):
if VolMgr.has_update_for_user_home_image():
VolMgr.update_user_home_image(fetch=False)
while True:
self.log_debug("JBox daemon waiting for commands...")
cmd, data = self.queue.recv()
if cmd == JBoxAsyncJob.CMD_BACKUP_CLEANUP:
args = (data, )
fn = JBoxd.backup_and_cleanup
elif cmd == JBoxAsyncJob.CMD_LAUNCH_SESSION:
args = (data[0], data[1], data[2])
fn = JBoxd.launch_session
elif cmd == JBoxAsyncJob.CMD_AUTO_ACTIVATE:
args = ()
fn = JBoxd.auto_activate
elif cmd == JBoxAsyncJob.CMD_UPDATE_USER_HOME_IMAGE:
args = ()
fn = JBoxd.update_user_home_image
elif cmd == JBoxAsyncJob.CMD_REFRESH_DISKS:
args = ()
fn = JBoxd.refresh_disks
elif cmd == JBoxAsyncJob.CMD_COLLECT_STATS:
args = ()
fn = JBoxd.collect_stats
else:
self.log_error("Unknown command " + str(cmd))
continue
JBoxd.schedule_thread(cmd, fn, args)
def __init__(self):
dckr = docker.Client()
cfg = read_config()
cloud_cfg = cfg['cloud_host']
user_activation_cfg = cfg['user_activation']
LoggerMixin.setup_logger(level=cfg['root_log_level'])
LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level']
db.configure_db(cfg)
CloudHost.configure(has_s3=cloud_cfg['s3'],
has_dynamodb=cloud_cfg['dynamodb'],
has_cloudwatch=cloud_cfg['cloudwatch'],
has_autoscale=cloud_cfg['autoscale'],
has_route53=cloud_cfg['route53'],
has_ebs=cloud_cfg['ebs'],
has_ses=cloud_cfg['ses'],
scale_up_at_load=cloud_cfg['scale_up_at_load'],
scale_up_policy=cloud_cfg['scale_up_policy'],
autoscale_group=cloud_cfg['autoscale_group'],
route53_domain=cloud_cfg['route53_domain'],
region=cloud_cfg['region'],
install_id=cloud_cfg['install_id'])
VolMgr.configure(dckr, cfg)
JBoxContainer.configure(dckr, cfg['docker_image'], cfg['mem_limit'], cfg['cpu_limit'],
cfg['numlocalmax'], cfg['async_job_port'], async_mode=JBoxAsyncJob.MODE_SUB)
self.log_debug("Backup daemon listening on port: " + str(cfg['async_job_port']))
self.queue = JBoxContainer.ASYNC_JOB
JBoxd.MAX_ACTIVATIONS_PER_SEC = user_activation_cfg['max_activations_per_sec']
JBoxd.MAX_AUTO_ACTIVATIONS_PER_RUN = user_activation_cfg['max_activations_per_run']
JBoxd.ACTIVATION_SUBJECT = user_activation_cfg['mail_subject']
JBoxd.ACTIVATION_BODY = user_activation_cfg['mail_body']
JBoxd.ACTIVATION_SENDER = user_activation_cfg['sender']
def launch_session(name, email, reuse=True):
try:
JBoxd._wait_for_session_backup(name)
VolMgr.refresh_disk_use_status()
JBoxContainer.launch_by_name(name, email, reuse=reuse)
finally:
JBoxd.finish_thread()
def launch_session(name, email, reuse=True):
try:
JBoxd._wait_for_session_backup(name)
VolMgr.refresh_disk_use_status()
JBoxContainer.launch_by_name(name, email, reuse=reuse)
finally:
JBoxd.finish_thread()
def run(self):
Compute.deregister_instance_dns()
Compute.register_instance_dns()
JBoxd.publish_perf_counters()
JBoxd.log_debug("Setting up signal handlers")
signal.signal(signal.SIGINT, JBoxd.signal_handler)
signal.signal(signal.SIGTERM, JBoxd.signal_handler)
if VolMgr.has_update_for_user_home_image():
VolMgr.update_user_home_image(fetch=False)
while True:
self.log_debug("JBox daemon waiting for commands...")
try:
offline, reply_req = JBoxd.QUEUE.poll(self._is_scheduled(JBoxAsyncJob.CMD_REQ_RESP, ()))
except ValueError:
self.log_exception("Exception reading command. Will retry after 10 seconds")
time.sleep(10)
continue
if offline:
try:
self.process_offline()
except:
self.log_exception("Exception scheduling request")
if reply_req:
JBoxd.schedule_thread(JBoxAsyncJob.CMD_REQ_RESP, JBoxd.process_and_respond, ())
def maintain(max_timeout=0, inactive_timeout=0, protected_names=()):
JBoxContainer.log_info("Starting container maintenance...")
tnow = datetime.datetime.now(pytz.utc)
tmin = datetime.datetime(datetime.MINYEAR, 1, 1, tzinfo=pytz.utc)
stop_before = (tnow - datetime.timedelta(seconds=max_timeout)) if (max_timeout > 0) else tmin
stop_inacive_before = (tnow - datetime.timedelta(seconds=inactive_timeout)) if (inactive_timeout > 0) else tmin
all_containers = JBoxContainer.DCKR.containers(all=True)
all_cnames = {}
container_id_list = []
for cdesc in all_containers:
cid = cdesc['Id']
cont = JBoxContainer(cid)
container_id_list.append(cid)
cname = cont.get_name()
all_cnames[cname] = cid
if (cname is None) or (cname in protected_names):
JBoxContainer.log_debug("Ignoring %s", cont.debug_str())
continue
c_is_active = cont.is_running() or cont.is_restarting()
last_ping = JBoxContainer._get_last_ping(cname)
# if we don't have a ping record, create one (we must have restarted)
if (last_ping is None) and c_is_active:
JBoxContainer.log_info("Discovered new container %s", cont.debug_str())
JBoxContainer.record_ping(cname)
start_time = cont.time_started()
# check that start time is not absurdly small (indicates a continer that's starting up)
start_time_not_zero = (tnow-start_time).total_seconds() < (365*24*60*60)
if (start_time < stop_before) and start_time_not_zero:
# don't allow running beyond the limit for long running sessions
# JBoxContainer.log_info("time_started " + str(cont.time_started()) +
# " delete_before: " + str(delete_before) +
# " cond: " + str(cont.time_started() < delete_before))
JBoxContainer.log_info("Running beyond allowed time %s", cont.debug_str())
cont.async_backup_and_cleanup()
elif (last_ping is not None) and c_is_active and (last_ping < stop_inacive_before):
# if inactive for too long, stop it
# JBoxContainer.log_info("last_ping " + str(last_ping) + " stop_before: " + str(stop_before) +
# " cond: " + str(last_ping < stop_before))
JBoxContainer.log_info("Inactive beyond allowed time %s", cont.debug_str())
cont.async_backup_and_cleanup()
# delete ping entries for non exixtent containers
for cname in JBoxContainer.PINGS.keys():
if cname not in all_cnames:
del JBoxContainer.PINGS[cname]
JBoxContainer.VALID_CONTAINERS = all_cnames
JBoxContainer.publish_container_stats()
VolMgr.refresh_disk_use_status(container_id_list=container_id_list)
JBoxContainer.log_info("Finished container maintenance.")
def __init__(self):
dckr = docker.Client()
cfg = JBox.cfg = read_config()
cloud_cfg = cfg['cloud_host']
LoggerMixin.setup_logger(level=cfg['root_log_level'])
LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level']
JBoxHandler.configure(cfg)
db.configure_db(cfg)
CloudHost.configure(has_s3=cloud_cfg['s3'],
has_dynamodb=cloud_cfg['dynamodb'],
has_cloudwatch=cloud_cfg['cloudwatch'],
has_autoscale=cloud_cfg['autoscale'],
has_route53=cloud_cfg['route53'],
has_ebs=cloud_cfg['ebs'],
has_ses=cloud_cfg['ses'],
scale_up_at_load=cloud_cfg['scale_up_at_load'],
scale_up_policy=cloud_cfg['scale_up_policy'],
autoscale_group=cloud_cfg['autoscale_group'],
route53_domain=cloud_cfg['route53_domain'],
region=cloud_cfg['region'],
install_id=cloud_cfg['install_id'])
VolMgr.configure(dckr, cfg)
JBoxAsyncJob.configure(cfg)
JBoxContainer.configure(dckr, cfg['docker_image'], cfg['mem_limit'],
cfg['cpu_limit'], cfg['numlocalmax'],
cfg['async_job_ports'])
self.application = tornado.web.Application([
(r"/", MainHandler), (r"/hostlaunchipnb/", AuthHandler),
(r"/hostadmin/", AdminHandler), (r"/ping/", PingHandler),
(r"/cors/", CorsHandler)
])
cookie_secret = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for x in xrange(32))
self.application.settings["cookie_secret"] = cookie_secret
self.application.settings["google_oauth"] = cfg["google_oauth"]
self.application.listen(cfg["port"])
self.ioloop = tornado.ioloop.IOLoop.instance()
# run container maintainence every 5 minutes
run_interval = 5 * 60 * 1000
self.log_info("Container maintenance every " +
str(run_interval / (60 * 1000)) + " minutes")
self.ct = tornado.ioloop.PeriodicCallback(JBox.do_housekeeping,
run_interval, self.ioloop)
def __init__(self):
dckr = docker.Client()
cfg = JBox.cfg = read_config()
cloud_cfg = cfg['cloud_host']
LoggerMixin.setup_logger(level=cfg['root_log_level'])
LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level']
JBoxHandler.configure(cfg)
db.configure_db(cfg)
CloudHost.configure(has_s3=cloud_cfg['s3'],
has_dynamodb=cloud_cfg['dynamodb'],
has_cloudwatch=cloud_cfg['cloudwatch'],
has_autoscale=cloud_cfg['autoscale'],
has_route53=cloud_cfg['route53'],
has_ebs=cloud_cfg['ebs'],
has_ses=cloud_cfg['ses'],
scale_up_at_load=cloud_cfg['scale_up_at_load'],
scale_up_policy=cloud_cfg['scale_up_policy'],
autoscale_group=cloud_cfg['autoscale_group'],
route53_domain=cloud_cfg['route53_domain'],
region=cloud_cfg['region'],
install_id=cloud_cfg['install_id'])
VolMgr.configure(dckr, cfg)
JBoxAsyncJob.configure(cfg)
JBoxContainer.configure(dckr, cfg['docker_image'], cfg['mem_limit'], cfg['cpu_limit'],
cfg['numlocalmax'], cfg['async_job_ports'])
self.application = tornado.web.Application([
(r"/", MainHandler),
(r"/hostlaunchipnb/", AuthHandler),
(r"/hostadmin/", AdminHandler),
(r"/ping/", PingHandler),
(r"/cors/", CorsHandler),
(r"/hw/", HomeworkHandler)
])
cookie_secret = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in xrange(32))
self.application.settings["cookie_secret"] = cookie_secret
self.application.settings["google_oauth"] = cfg["google_oauth"]
self.application.listen(cfg["port"])
self.ioloop = tornado.ioloop.IOLoop.instance()
# run container maintainence every 5 minutes
run_interval = 5 * 60 * 1000
self.log_info("Container maintenance every " + str(run_interval / (60 * 1000)) + " minutes")
self.ct = tornado.ioloop.PeriodicCallback(JBox.do_housekeeping, run_interval, self.ioloop)
def start(self, email):
self.refresh()
JBoxContainer.log_info("Starting %s", self.debug_str())
if self.is_running() or self.is_restarting():
JBoxContainer.log_warn(
"Already started %s. Browser connectivity issues?",
self.debug_str())
return
disk = VolMgr.get_disk_for_user(email)
vols = {
disk.disk_path: {
'bind': JBoxContainer.VOLUMES[0],
'ro': False
}
}
JBoxContainer.DCKR.start(
self.dockid,
port_bindings=JBoxContainer.CONTAINER_PORT_BINDINGS,
binds=vols)
self.refresh()
JBoxContainer.log_info("Started %s", self.debug_str())
cname = self.get_name()
if cname is not None:
JBoxContainer.record_ping(cname)
def publish_perf_counters():
""" Publish performance counters. Used for status monitoring and auto scaling. """
VolMgr.refresh_disk_use_status()
nactive = BaseContainer.num_active(BaseContainer.SFX_INT)
stats = []
stats.append(("NumActiveContainers", "Count", nactive))
nactive_api = BaseContainer.num_active(BaseContainer.SFX_API)
stats.append(("NumActiveAPIContainers", "Count", nactive_api))
curr_cpu_used_pct = psutil.cpu_percent()
last_cpu_used_pct = curr_cpu_used_pct if BaseContainer.LAST_CPU_PCT is None else BaseContainer.LAST_CPU_PCT
BaseContainer.LAST_CPU_PCT = curr_cpu_used_pct
cpu_used_pct = int((curr_cpu_used_pct + last_cpu_used_pct) / 2)
stats.append(("CPUUsed", "Percent", cpu_used_pct))
mem_used_pct = psutil.virtual_memory().percent
stats.append(("MemUsed", "Percent", mem_used_pct))
disk_used_pct = 0
for x in psutil.disk_partitions():
if not VolMgr.is_mount_path(x.mountpoint):
try:
disk_used_pct = max(psutil.disk_usage(x.mountpoint).percent, disk_used_pct)
except:
pass
if BaseContainer.INITIAL_DISK_USED_PCT is None:
BaseContainer.INITIAL_DISK_USED_PCT = disk_used_pct
disk_used_pct = max(0, (disk_used_pct - BaseContainer.INITIAL_DISK_USED_PCT))
stats.append(("DiskUsed", "Percent", disk_used_pct))
cont_load_pct = min(100, max(0, nactive * 100 / SessContainer.MAX_CONTAINERS))
stats.append(("ContainersUsed", "Percent", cont_load_pct))
api_cont_load_pct = min(100, max(0, nactive_api * 100 / APIContainer.MAX_CONTAINERS))
stats.append(("APIContainersUsed", "Percent", api_cont_load_pct))
stats.append(("DiskIdsUsed", "Percent", VolMgr.used_pct()))
overall_load_pct = max(
cont_load_pct, api_cont_load_pct, disk_used_pct, mem_used_pct, cpu_used_pct, VolMgr.used_pct()
)
stats.append(("Load", "Percent", overall_load_pct))
Compute.publish_stats_multi(stats)
def __init__(self):
dckr = docker.Client()
cfg = read_config()
cloud_cfg = cfg['cloud_host']
user_activation_cfg = cfg['user_activation']
LoggerMixin.setup_logger(level=cfg['root_log_level'])
LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level']
db.configure_db(cfg)
CloudHost.configure(has_s3=cloud_cfg['s3'],
has_dynamodb=cloud_cfg['dynamodb'],
has_cloudwatch=cloud_cfg['cloudwatch'],
has_autoscale=cloud_cfg['autoscale'],
has_route53=cloud_cfg['route53'],
has_ebs=cloud_cfg['ebs'],
has_ses=cloud_cfg['ses'],
scale_up_at_load=cloud_cfg['scale_up_at_load'],
scale_up_policy=cloud_cfg['scale_up_policy'],
autoscale_group=cloud_cfg['autoscale_group'],
route53_domain=cloud_cfg['route53_domain'],
region=cloud_cfg['region'],
install_id=cloud_cfg['install_id'])
VolMgr.configure(dckr, cfg)
JBoxAsyncJob.configure(cfg)
JBoxContainer.configure(dckr,
cfg['docker_image'],
cfg['mem_limit'],
cfg['cpu_limit'],
cfg['numlocalmax'],
cfg['async_job_ports'],
async_mode=JBoxAsyncJob.MODE_SUB)
self.log_debug("Backup daemon listening on ports: %s",
repr(cfg['async_job_ports']))
JBoxd.QUEUE = JBoxContainer.ASYNC_JOB
JBoxd.MAX_ACTIVATIONS_PER_SEC = user_activation_cfg[
'max_activations_per_sec']
JBoxd.MAX_AUTO_ACTIVATIONS_PER_RUN = user_activation_cfg[
'max_activations_per_run']
JBoxd.ACTIVATION_SUBJECT = user_activation_cfg['mail_subject']
JBoxd.ACTIVATION_BODY = user_activation_cfg['mail_body']
JBoxd.ACTIVATION_SENDER = user_activation_cfg['sender']
def publish_perf_counters():
""" Publish performance counters. Used for status monitoring and auto scaling. """
VolMgr.refresh_disk_use_status()
nactive = BaseContainer.num_active(BaseContainer.SFX_INT)
stats = []
stats.append(("NumActiveContainers", "Count", nactive))
nactive_api = BaseContainer.num_active(BaseContainer.SFX_API)
stats.append(("NumActiveAPIContainers", "Count", nactive_api))
curr_cpu_used_pct = psutil.cpu_percent()
last_cpu_used_pct = curr_cpu_used_pct if BaseContainer.LAST_CPU_PCT is None else BaseContainer.LAST_CPU_PCT
BaseContainer.LAST_CPU_PCT = curr_cpu_used_pct
cpu_used_pct = int((curr_cpu_used_pct + last_cpu_used_pct)/2)
stats.append(("CPUUsed", "Percent", cpu_used_pct))
mem_used_pct = psutil.virtual_memory().percent
stats.append(("MemUsed", "Percent", mem_used_pct))
disk_used_pct = 0
for x in psutil.disk_partitions():
if not VolMgr.is_mount_path(x.mountpoint):
try:
disk_used_pct = max(psutil.disk_usage(x.mountpoint).percent, disk_used_pct)
except:
pass
if BaseContainer.INITIAL_DISK_USED_PCT is None:
BaseContainer.INITIAL_DISK_USED_PCT = disk_used_pct
disk_used_pct = max(0, (disk_used_pct - BaseContainer.INITIAL_DISK_USED_PCT))
stats.append(("DiskUsed", "Percent", disk_used_pct))
cont_load_pct = min(100, max(0, nactive * 100 / SessContainer.MAX_CONTAINERS))
stats.append(("ContainersUsed", "Percent", cont_load_pct))
api_cont_load_pct = min(100, max(0, nactive_api * 100 / APIContainer.MAX_CONTAINERS))
stats.append(("APIContainersUsed", "Percent", api_cont_load_pct))
stats.append(("DiskIdsUsed", "Percent", VolMgr.used_pct()))
overall_load_pct = max(cont_load_pct, api_cont_load_pct, disk_used_pct, mem_used_pct, cpu_used_pct, VolMgr.used_pct())
stats.append(("Load", "Percent", overall_load_pct))
Compute.publish_stats_multi(stats)
def __init__(self):
LoggerMixin.configure()
db.configure()
Compute.configure()
SessContainer.configure()
APIContainer.configure()
VolMgr.configure()
JBoxAsyncJob.configure()
JBoxAsyncJob.init(JBoxAsyncJob.MODE_SUB)
self.log_debug("Container manager listening on ports: %s", repr(JBoxCfg.get('container_manager_ports')))
JBoxd.QUEUE = JBoxAsyncJob.get()
JBoxd.MAX_ACTIVATIONS_PER_SEC = JBoxCfg.get('user_activation.max_activations_per_sec')
JBoxd.MAX_AUTO_ACTIVATIONS_PER_RUN = JBoxCfg.get('user_activation.max_activations_per_run')
JBoxd.ACTIVATION_SUBJECT = JBoxCfg.get('user_activation.mail_subject')
JBoxd.ACTIVATION_BODY = JBoxCfg.get('user_activation.mail_body')
JBoxd.ACTIVATION_SENDER = JBoxCfg.get('user_activation.sender')
def __init__(self):
LoggerMixin.configure()
db.configure()
Compute.configure()
SessContainer.configure()
VolMgr.configure()
JBoxAsyncJob.configure()
JBoxAsyncJob.init(JBoxAsyncJob.MODE_PUB)
self.application = tornado.web.Application(
handlers=[(r"/", MainHandler), (
r"/jboxadmin/",
AdminHandler), (r"/jboxping/",
PingHandler), (r"/jboxcors/", CorsHandler)])
JBPluginHandler.add_plugin_handlers(self.application)
JBPluginUI.create_include_files()
# cookie_secret = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in xrange(32))
# use sesskey as cookie secret to be able to span multiple tornado servers
self.application.settings["cookie_secret"] = JBoxCfg.get('sesskey')
self.application.settings[
"plugin_features"] = JBox.get_pluggedin_features()
self.application.listen(JBoxCfg.get('interactive.manager_port'),
address=socket.gethostname())
self.application.listen(JBoxCfg.get('interactive.manager_port'),
address='localhost')
self.ioloop = tornado.ioloop.IOLoop.instance()
# run container maintainence every 5 minutes
run_interval = 5 * 60 * 1000
self.log_info("Container maintenance every " +
str(run_interval / (60 * 1000)) + " minutes")
self.ct = tornado.ioloop.PeriodicCallback(JBox.do_housekeeping,
run_interval, self.ioloop)
self.sigct = tornado.ioloop.PeriodicCallback(JBox.do_signals, 1000,
self.ioloop)
# or configure cacerts
AsyncHTTPClient.configure(None, defaults=dict(validate_cert=None))
def run(self):
if VolMgr.has_update_for_user_home_image():
VolMgr.update_user_home_image(fetch=False)
while True:
self.log_debug("JBox daemon waiting for commands...")
try:
offline, reply_req = JBoxd.QUEUE.poll(
self._is_scheduled(JBoxAsyncJob.CMD_REQ_RESP, ()))
except ValueError:
self.log_exception(
"Exception reading command. Will retry after 10 seconds")
time.sleep(10)
continue
if offline:
try:
self.process_offline()
except:
self.log_exception("Exception scheduling request")
if reply_req:
JBoxd.schedule_thread(JBoxAsyncJob.CMD_REQ_RESP,
JBoxd.process_and_respond, ())
def delete(self, backup=False):
JBoxContainer.log_info("Deleting %s", self.debug_str())
self.refresh()
cname = self.get_name()
if self.is_running() or self.is_restarting():
self.kill()
disk = VolMgr.get_disk_from_container(self.dockid)
if disk is not None:
disk.release(backup=backup)
if cname is not None:
JBoxContainer.PINGS.pop(cname, None)
JBoxContainer.DCKR.remove_container(self.dockid)
JBoxContainer.log_info("Deleted %s", self.debug_str())
def init():
dckr = docker.Client()
cfg = read_config()
cloud_cfg = cfg['cloud_host']
cloud_cfg['backup_bucket'] = "juliabox_userbackup"
LoggerMixin.setup_logger(level=cfg['root_log_level'])
LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level']
CloudHost.configure(has_s3=True, #cloud_cfg['s3'],
has_dynamodb=cloud_cfg['dynamodb'],
has_cloudwatch=cloud_cfg['cloudwatch'],
has_autoscale=cloud_cfg['autoscale'],
has_route53=cloud_cfg['route53'],
has_ebs=cloud_cfg['ebs'],
has_ses=cloud_cfg['ses'],
scale_up_at_load=cloud_cfg['scale_up_at_load'],
scale_up_policy=cloud_cfg['scale_up_policy'],
autoscale_group=cloud_cfg['autoscale_group'],
route53_domain=cloud_cfg['route53_domain'],
region=cloud_cfg['region'],
install_id=cloud_cfg['install_id'])
VolMgr.configure(dckr, cfg)
def publish_container_stats():
""" Publish custom cloudwatch statistics. Used for status monitoring and auto scaling. """
nactive = JBoxContainer.num_active()
CloudHost.publish_stats("NumActiveContainers", "Count", nactive)
curr_cpu_used_pct = psutil.cpu_percent()
last_cpu_used_pct = curr_cpu_used_pct if JBoxContainer.LAST_CPU_PCT is None else JBoxContainer.LAST_CPU_PCT
JBoxContainer.LAST_CPU_PCT = curr_cpu_used_pct
cpu_used_pct = int((curr_cpu_used_pct + last_cpu_used_pct) / 2)
mem_used_pct = psutil.virtual_memory().percent
CloudHost.publish_stats("MemUsed", "Percent", mem_used_pct)
disk_used_pct = 0
for x in psutil.disk_partitions():
if not VolMgr.is_mount_path(x.mountpoint):
try:
disk_used_pct = max(
psutil.disk_usage(x.mountpoint).percent, disk_used_pct)
except:
pass
if JBoxContainer.INITIAL_DISK_USED_PCT is None:
JBoxContainer.INITIAL_DISK_USED_PCT = disk_used_pct
disk_used_pct = max(
0, (disk_used_pct - JBoxContainer.INITIAL_DISK_USED_PCT))
CloudHost.publish_stats("DiskUsed", "Percent", disk_used_pct)
cont_load_pct = min(
100, max(0, nactive * 100 / JBoxContainer.MAX_CONTAINERS))
CloudHost.publish_stats("ContainersUsed", "Percent", cont_load_pct)
CloudHost.publish_stats("DiskIdsUsed", "Percent", VolMgr.used_pct())
overall_load_pct = max(cont_load_pct, disk_used_pct, mem_used_pct,
cpu_used_pct, VolMgr.used_pct())
CloudHost.publish_stats("Load", "Percent", overall_load_pct)
def delete(self, backup=False):
JBoxContainer.log_info("Deleting %s", self.debug_str())
self.refresh()
cname = self.get_name()
if self.is_running() or self.is_restarting():
self.kill()
disk = VolMgr.get_disk_from_container(self.dockid)
if disk is not None:
disk.release(backup=backup)
if cname is not None:
JBoxContainer.PINGS.pop(cname, None)
JBoxContainer.DCKR.remove_container(self.dockid)
JBoxContainer.log_info("Deleted %s", self.debug_str())
def start(self, email):
self.refresh()
JBoxContainer.log_info("Starting %s", self.debug_str())
if self.is_running() or self.is_restarting():
JBoxContainer.log_info("Already started %s", self.debug_str())
return
disk = VolMgr.get_disk_for_user(email)
vols = {
disk.disk_path: {
'bind': JBoxContainer.VOLUMES[0],
'ro': False
}
}
JBoxContainer.DCKR.start(self.dockid, port_bindings=JBoxContainer.CONTAINER_PORT_BINDINGS, binds=vols)
self.refresh()
JBoxContainer.log_info("Started %s", self.debug_str())
cname = self.get_name()
if cname is not None:
JBoxContainer.record_ping(cname)
def get_disk_allocated(self):
disk = VolMgr.get_disk_from_container(self.dockid)
if disk is not None:
return disk.get_disk_allocated_size()
return 0
def refresh_disks():
if JBoxd._is_scheduled(JBoxAsyncJob.CMD_UPDATE_USER_HOME_IMAGE, ()):
return
VolMgr.refresh_user_home_image()
def publish_container_stats():
VolMgr.publish_stats()
db.publish_stats()
JBoxDynConfig.set_stat_collected_date(Compute.get_install_id())
def launch_session(name, email, reuse=True):
JBoxd._wait_for_session_backup(name)
VolMgr.refresh_disk_use_status()
SessContainer.launch_by_name(name, email, reuse=reuse)
JBoxd.publish_perf_counters()
def update_user_home_image():
VolMgr.update_user_home_image(fetch=True)
VolMgr.refresh_user_home_image()
def launch_session(name, email, reuse=True):
JBoxd.publish_anticipated_load(name)
JBoxd._wait_for_session_backup(name)
VolMgr.refresh_disk_use_status()
JBoxd._launch_session(name, email, reuse)
def get_disk_allocated(self):
disk = VolMgr.get_disk_from_container(self.dockid)
if disk is not None:
return disk.get_disk_allocated_size()
return 0
def update_user_home_image():
try:
VolMgr.update_user_home_image(fetch=True)
JBoxLoopbackVol.refresh_all_disks()
finally:
JBoxd.finish_thread()
def launch_session(name, email, reuse=True):
JBoxd.publish_anticipated_load(name)
JBoxd._wait_for_session_backup(name)
VolMgr.refresh_disk_use_status()
JBoxd._launch_session(name, email, reuse)
def maintain(max_timeout=0, inactive_timeout=0, protected_names=()):
JBoxContainer.log_info("Starting container maintenance...")
tnow = datetime.datetime.now(pytz.utc)
tmin = datetime.datetime(datetime.MINYEAR, 1, 1, tzinfo=pytz.utc)
stop_before = (tnow - datetime.timedelta(seconds=max_timeout)) if (
max_timeout > 0) else tmin
stop_inacive_before = (tnow - datetime.timedelta(
seconds=inactive_timeout)) if (inactive_timeout > 0) else tmin
all_containers = JBoxContainer.DCKR.containers(all=True)
all_cnames = {}
container_id_list = []
for cdesc in all_containers:
cid = cdesc['Id']
cont = JBoxContainer(cid)
container_id_list.append(cid)
cname = cont.get_name()
all_cnames[cname] = cid
if (cname is None) or (cname in protected_names):
JBoxContainer.log_debug("Ignoring %s", cont.debug_str())
continue
c_is_active = cont.is_running() or cont.is_restarting()
last_ping = JBoxContainer._get_last_ping(cname)
# if we don't have a ping record, create one (we must have restarted)
if (last_ping is None) and c_is_active:
JBoxContainer.log_info("Discovered new container %s",
cont.debug_str())
JBoxContainer.record_ping(cname)
start_time = cont.time_started()
# check that start time is not absurdly small (indicates a continer that's starting up)
start_time_not_zero = (tnow - start_time).total_seconds() < (
365 * 24 * 60 * 60)
if (start_time < stop_before) and start_time_not_zero:
# don't allow running beyond the limit for long running sessions
# JBoxContainer.log_info("time_started " + str(cont.time_started()) +
# " delete_before: " + str(delete_before) +
# " cond: " + str(cont.time_started() < delete_before))
JBoxContainer.log_warn("Running beyond allowed time %s",
cont.debug_str())
cont.async_backup_and_cleanup()
elif (last_ping is not None) and c_is_active and (
last_ping < stop_inacive_before):
# if inactive for too long, stop it
# JBoxContainer.log_info("last_ping " + str(last_ping) + " stop_before: " + str(stop_before) +
# " cond: " + str(last_ping < stop_before))
JBoxContainer.log_warn("Inactive beyond allowed time %s",
cont.debug_str())
cont.async_backup_and_cleanup()
# delete ping entries for non exixtent containers
for cname in JBoxContainer.PINGS.keys():
if cname not in all_cnames:
del JBoxContainer.PINGS[cname]
JBoxContainer.VALID_CONTAINERS = all_cnames
JBoxContainer.publish_container_stats()
VolMgr.refresh_disk_use_status(container_id_list=container_id_list)
JBoxContainer.log_info("Finished container maintenance.")
def update_user_home_image():
try:
VolMgr.update_user_home_image(fetch=True)
JBoxLoopbackVol.refresh_all_disks()
finally:
JBoxd.finish_thread()
def collect_stats():
VolMgr.publish_stats()
db.publish_stats()
JBoxDynConfig.set_stat_collected_date(CloudHost.INSTALL_ID)
def do_update_user_home_image():
if VolMgr.has_update_for_user_home_image():
if not VolMgr.update_user_home_image(fetch=False):
JBoxContainer.async_update_user_home_image()
def update_user_home_image():
VolMgr.update_user_home_image(fetch=True)
JBoxLoopbackVol.refresh_all_disks()
def do_update_user_home_image():
if VolMgr.has_update_for_user_home_image():
if not VolMgr.update_user_home_image(fetch=False):
JBoxContainer.async_update_user_home_image()
CloudHost.configure(has_s3=cloud_cfg['s3'],
has_dynamodb=cloud_cfg['dynamodb'],
has_cloudwatch=cloud_cfg['cloudwatch'],
has_autoscale=cloud_cfg['autoscale'],
has_route53=cloud_cfg['route53'],
has_ebs=cloud_cfg['ebs'],
has_ses=cloud_cfg['ses'],
scale_up_at_load=cloud_cfg['scale_up_at_load'],
scale_up_policy=cloud_cfg['scale_up_policy'],
autoscale_group=cloud_cfg['autoscale_group'],
route53_domain=cloud_cfg['route53_domain'],
region=cloud_cfg['region'],
install_id=cloud_cfg['install_id'])
VolMgr.configure(dckr, cfg)
ts = JBoxVol._get_user_home_timestamp()
VolMgr.log_debug("user_home_timestamp: %s", ts.strftime("%Y%m%d_%H%M"))
img_dir, img_file = os.path.split(JBoxVol.USER_HOME_IMG)
new_img_file_name = 'user_home_' + ts.strftime("%Y%m%d_%H%M") + '.tar.gz'
new_img_file = os.path.join(img_dir, new_img_file_name)
shutil.copyfile(JBoxVol.USER_HOME_IMG, new_img_file)
VolMgr.log_debug("new image file is at : %s", new_img_file)
bucket = 'juliabox-user-home-templates'
VolMgr.log_debug("pushing new image file to s3 at: %s", bucket)
CloudHost.push_file_to_s3(bucket, new_img_file)