def update_juliabox_status():
in_error = len(
JBoxInstanceProps.get_stale_instances(Compute.get_install_id()))
instance_status = JBoxInstanceProps.get_instance_status(
Compute.get_install_id())
HTML = "<html><body><center><pre>\nJuliaBox is Up.\n\nLast updated: " + datetime.datetime.now(
).isoformat() + " UTC\n\nLoads: "
for iid in instance_status:
HTML += (str(instance_status[iid]['load']) + '% ')
HTML += (
"\n\nErrors: " + str(in_error) +
"\n\nAWS Status: <a href='http://status.aws.amazon.com/'>status.aws.amazon.com</a></pre></center></body></html>"
)
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
bkt = JBoxCfg.get("cloud_host.status_bucket")
if plugin is not None and bkt is not None:
try:
f = open("/tmp/index.html", "w")
f.write(HTML)
f.close()
plugin.push(bkt, "/tmp/index.html")
finally:
os.remove("/tmp/index.html")
else:
JBox.log_debug("Status: %s", HTML)
return None
def publish_anticipated_load(session_name=None):
iid = Compute.get_instance_id()
if session_name is None:
nactive = BaseContainer.num_active(BaseContainer.SFX_INT)
else:
JBoxSessionProps.attach_instance(Compute.get_install_id(), session_name, iid, "Preparing")
nactive = BaseContainer.num_active(BaseContainer.SFX_INT) + 1
cont_load_pct = min(100, max(0, nactive * 100 / SessContainer.MAX_CONTAINERS))
self_load = max(Compute.get_instance_stats(iid, 'Load'), cont_load_pct)
Compute.publish_stats("Load", "Percent", self_load)
accept = Compute.should_accept_session(is_proposed_cluster_leader())
JBoxInstanceProps.set_props(Compute.get_install_id(), iid, load=self_load, accept=accept)
def publish_anticipated_load(session_name=None):
iid = Compute.get_instance_id()
if session_name is None:
nactive = BaseContainer.num_active(BaseContainer.SFX_INT)
else:
JBoxSessionProps.attach_instance(Compute.get_install_id(), session_name, iid, "Preparing")
nactive = BaseContainer.num_active(BaseContainer.SFX_INT) + 1
cont_load_pct = min(100, max(0, nactive * 100 / SessContainer.MAX_CONTAINERS))
self_load = max(Compute.get_instance_stats(iid, 'Load'), cont_load_pct)
Compute.publish_stats("Load", "Percent", self_load)
accept = Compute.should_accept_session(is_proposed_cluster_leader())
JBoxInstanceProps.set_props(Compute.get_install_id(), iid, load=self_load, accept=accept)
def backup_and_cleanup(dockid):
cont = SessContainer(dockid)
cont.stop()
cont.delete(backup=True)
JBoxSessionProps.detach_instance(Compute.get_install_id(), cont.get_name(), Compute.get_instance_id())
JBoxd.publish_perf_counters()
JBoxd.publish_anticipated_load()
def publish_sessions():
iid = Compute.get_instance_id()
for c in SessContainer.session_containers(allcontainers=True):
if ("Names" in c) and (c["Names"] is not None):
sessname = SessContainer(c["Id"]).get_name()
if sessname:
JBoxSessionProps.attach_instance(Compute.get_install_id(), sessname, iid, c["Status"])
def backup_and_cleanup(dockid):
cont = SessContainer(dockid)
cont.stop()
cont.delete(backup=True)
JBoxSessionProps.detach_instance(Compute.get_install_id(), cont.get_name(), Compute.get_instance_id())
JBoxd.publish_perf_counters()
JBoxd.publish_anticipated_load()
def do_housekeeping():
terminating = False
server_delete_timeout = JBoxCfg.get('interactive.expire')
inactive_timeout = JBoxCfg.get('interactive.inactivity_timeout')
SessContainer.maintain(max_timeout=server_delete_timeout, inactive_timeout=inactive_timeout)
is_leader = is_cluster_leader()
if is_leader:
terminating = False
else:
try:
terminating = JBoxAsyncJob.sync_is_terminating()
if terminating['code'] == 0:
terminating = terminating['data']
else:
JBox.log_error("Error checking if instance is terminating. Assuming False.")
terminating = False
except:
JBox.log_error("Exception checking if instance is terminating. Assuming False.")
terminating = False
if is_leader:
JBox.log_info("I am the cluster leader")
JBox.update_juliabox_status()
JBox.monitor_registrations()
if not JBoxDynConfig.is_stat_collected_within(Compute.get_install_id(), 1):
JBoxAsyncJob.async_collect_stats()
if terminating:
JBox.log_warn("terminating to scale down")
else:
JBox.do_update_user_home_image()
JBoxAsyncJob.async_plugin_maintenance(is_leader)
def monitor_registrations():
max_rate = JBoxDynConfig.get_registration_hourly_rate(Compute.get_install_id())
rate = JBoxUserV2.count_created(1)
reg_allowed = JBoxDynConfig.get_allow_registration(Compute.get_install_id())
JBox.log_debug("registration allowed: %r, rate: %d, max allowed: %d", reg_allowed, rate, max_rate)
if (reg_allowed and (rate > max_rate*1.1)) or ((not reg_allowed) and (rate < max_rate*0.9)):
reg_allowed = not reg_allowed
JBox.log_warn("Changing registration allowed to %r", reg_allowed)
JBoxDynConfig.set_allow_registration(Compute.get_install_id(), reg_allowed)
if reg_allowed:
num_pending_activations = JBoxUserV2.count_pending_activations()
if num_pending_activations > 0:
JBox.log_info("scheduling activations for %d pending activations", num_pending_activations)
JBoxAsyncJob.async_schedule_activations()
def publish_sessions():
iid = Compute.get_instance_id()
for c in SessContainer.session_containers(allcontainers=True):
if ('Names' in c) and (c['Names'] is not None):
sessname = SessContainer(c['Id']).get_name()
if sessname:
JBoxSessionProps.attach_instance(Compute.get_install_id(),
sessname, iid, c["Status"])
def schedule_housekeeping(cmd, is_leader):
JBoxd.publish_perf_counters()
JBoxd.publish_sessions()
JBoxd.publish_instance_state()
features = [JBPluginTask.JBP_NODE]
if is_leader is True:
JBoxInstanceProps.purge_stale_instances(Compute.get_install_id())
features.append(JBPluginTask.JBP_CLUSTER)
for feature in features:
for plugin in JBPluginTask.jbox_get_plugins(feature):
JBoxd.schedule_thread(cmd, plugin.do_periodic_task, (feature,))
def schedule_housekeeping(cmd, is_leader):
JBoxd.publish_perf_counters()
JBoxd.publish_sessions()
JBoxd.publish_instance_state()
features = [JBPluginTask.JBP_NODE]
if is_leader is True:
JBoxInstanceProps.purge_stale_instances(Compute.get_install_id())
features.append(JBPluginTask.JBP_CLUSTER)
for feature in features:
for plugin in JBPluginTask.jbox_get_plugins(feature):
JBoxd.schedule_thread(cmd, plugin.do_periodic_task, (feature,))
def publish_instance_state():
iid = Compute.get_instance_id()
api_status = dict()
for c in BaseContainer.api_containers(allcontainers=True):
name = c["Names"][0] if (("Names" in c) and (c["Names"] is not None)) else c["Id"][0:12]
api_name = APIContainer.get_api_name_from_container_name(name)
if api_name is None:
continue
cnt = api_status.get(api_name, 0)
api_status[api_name] = cnt + 1
self_load = Compute.get_instance_stats(iid, 'Load')
accept = Compute.should_accept_session(is_proposed_cluster_leader())
JBoxInstanceProps.set_props(Compute.get_install_id(), iid, load=self_load, accept=accept, api_status=api_status)
def publish_instance_state():
iid = Compute.get_instance_id()
api_status = dict()
for c in BaseContainer.api_containers(allcontainers=True):
name = c["Names"][0] if (("Names" in c) and (c["Names"] is not None)) else c["Id"][0:12]
api_name = APIContainer.get_api_name_from_container_name(name)
if api_name is None:
continue
cnt = api_status.get(api_name, 0)
api_status[api_name] = cnt + 1
self_load = Compute.get_instance_stats(iid, 'Load')
accept = Compute.should_accept_session(is_proposed_cluster_leader())
JBoxInstanceProps.set_props(Compute.get_install_id(), iid, load=self_load, accept=accept, api_status=api_status)
def update_juliabox_status():
in_error = len(JBoxInstanceProps.get_stale_instances(Compute.get_install_id()))
instance_status = JBoxInstanceProps.get_instance_status(Compute.get_install_id())
HTML = "<html><body><center><pre>\nJuliaBox is Up.\n\nLast updated: " + datetime.datetime.now().isoformat() + " UTC\n\nLoads: "
for iid in instance_status:
HTML += (str(instance_status[iid]['load']) + '% ')
HTML += ("\n\nErrors: " + str(in_error) + "\n\nAWS Status: <a href='http://status.aws.amazon.com/'>status.aws.amazon.com</a></pre></center></body></html>")
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
bkt = JBoxCfg.get("cloud_host.status_bucket")
if plugin is not None and bkt is not None:
try:
f = open("/tmp/index.html", "w")
f.write(HTML)
f.close()
plugin.push(bkt, "/tmp/index.html")
finally:
os.remove("/tmp/index.html")
else:
JBox.log_debug("Status: %s", HTML)
return None
def monitor_registrations():
max_rate = JBoxDynConfig.get_registration_hourly_rate(
Compute.get_install_id())
rate = JBoxUserV2.count_created(1)
reg_allowed = JBoxDynConfig.get_allow_registration(
Compute.get_install_id())
JBox.log_debug("registration allowed: %r, rate: %d, max allowed: %d",
reg_allowed, rate, max_rate)
if (reg_allowed and
(rate > max_rate * 1.1)) or ((not reg_allowed) and
(rate < max_rate * 0.9)):
reg_allowed = not reg_allowed
JBox.log_warn("Changing registration allowed to %r", reg_allowed)
JBoxDynConfig.set_allow_registration(Compute.get_install_id(),
reg_allowed)
if reg_allowed:
num_pending_activations = JBoxUserV2.count_pending_activations()
if num_pending_activations > 0:
JBox.log_info(
"scheduling activations for %d pending activations",
num_pending_activations)
JBoxAsyncJob.async_schedule_activations()
def do_housekeeping():
terminating = False
server_delete_timeout = JBoxCfg.get('interactive.expire')
inactive_timeout = JBoxCfg.get('interactive.inactivity_timeout')
SessContainer.maintain(max_timeout=server_delete_timeout,
inactive_timeout=inactive_timeout)
is_leader = is_cluster_leader()
if is_leader:
terminating = False
else:
try:
terminating = JBoxAsyncJob.sync_is_terminating()
if terminating['code'] == 0:
terminating = terminating['data']
else:
JBox.log_error(
"Error checking if instance is terminating. Assuming False."
)
terminating = False
except:
JBox.log_error(
"Exception checking if instance is terminating. Assuming False."
)
terminating = False
if is_leader:
JBox.log_info("I am the cluster leader")
JBox.update_juliabox_status()
JBox.monitor_registrations()
if not JBoxDynConfig.is_stat_collected_within(
Compute.get_install_id(), 1):
JBoxAsyncJob.async_collect_stats()
if terminating:
JBox.log_warn("terminating to scale down")
else:
JBox.do_update_user_home_image()
JBoxAsyncJob.async_plugin_maintenance(is_leader)
def publish_container_stats():
VolMgr.publish_stats()
db.publish_stats()
JBoxDynConfig.set_stat_collected_date(Compute.get_install_id())
def publish_sessions():
iid = Compute.get_instance_id()
for c in SessContainer.session_containers(allcontainers=True):
if ('Names' in c) and (c['Names'] is not None):
JBoxSessionProps.attach_instance(Compute.get_install_id(), SessContainer(c['Id']).get_name(), iid,
c["Status"])
