def update_juliabox_status(): in_error = len( JBoxInstanceProps.get_stale_instances(Compute.get_install_id())) instance_status = JBoxInstanceProps.get_instance_status( Compute.get_install_id()) HTML = "<html><body><center><pre>\nJuliaBox is Up.\n\nLast updated: " + datetime.datetime.now( ).isoformat() + " UTC\n\nLoads: " for iid in instance_status: HTML += (str(instance_status[iid]['load']) + '% ') HTML += ( "\n\nErrors: " + str(in_error) + "\n\nAWS Status: <a href='http://status.aws.amazon.com/'>status.aws.amazon.com</a></pre></center></body></html>" ) plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE) bkt = JBoxCfg.get("cloud_host.status_bucket") if plugin is not None and bkt is not None: try: f = open("/tmp/index.html", "w") f.write(HTML) f.close() plugin.push(bkt, "/tmp/index.html") finally: os.remove("/tmp/index.html") else: JBox.log_debug("Status: %s", HTML) return None
def publish_anticipated_load(session_name=None): iid = Compute.get_instance_id() if session_name is None: nactive = BaseContainer.num_active(BaseContainer.SFX_INT) else: JBoxSessionProps.attach_instance(Compute.get_install_id(), session_name, iid, "Preparing") nactive = BaseContainer.num_active(BaseContainer.SFX_INT) + 1 cont_load_pct = min(100, max(0, nactive * 100 / SessContainer.MAX_CONTAINERS)) self_load = max(Compute.get_instance_stats(iid, 'Load'), cont_load_pct) Compute.publish_stats("Load", "Percent", self_load) accept = Compute.should_accept_session(is_proposed_cluster_leader()) JBoxInstanceProps.set_props(Compute.get_install_id(), iid, load=self_load, accept=accept)
def backup_and_cleanup(dockid): cont = SessContainer(dockid) cont.stop() cont.delete(backup=True) JBoxSessionProps.detach_instance(Compute.get_install_id(), cont.get_name(), Compute.get_instance_id()) JBoxd.publish_perf_counters() JBoxd.publish_anticipated_load()
def publish_sessions(): iid = Compute.get_instance_id() for c in SessContainer.session_containers(allcontainers=True): if ("Names" in c) and (c["Names"] is not None): sessname = SessContainer(c["Id"]).get_name() if sessname: JBoxSessionProps.attach_instance(Compute.get_install_id(), sessname, iid, c["Status"])
def do_housekeeping(): terminating = False server_delete_timeout = JBoxCfg.get('interactive.expire') inactive_timeout = JBoxCfg.get('interactive.inactivity_timeout') SessContainer.maintain(max_timeout=server_delete_timeout, inactive_timeout=inactive_timeout) is_leader = is_cluster_leader() if is_leader: terminating = False else: try: terminating = JBoxAsyncJob.sync_is_terminating() if terminating['code'] == 0: terminating = terminating['data'] else: JBox.log_error("Error checking if instance is terminating. Assuming False.") terminating = False except: JBox.log_error("Exception checking if instance is terminating. Assuming False.") terminating = False if is_leader: JBox.log_info("I am the cluster leader") JBox.update_juliabox_status() JBox.monitor_registrations() if not JBoxDynConfig.is_stat_collected_within(Compute.get_install_id(), 1): JBoxAsyncJob.async_collect_stats() if terminating: JBox.log_warn("terminating to scale down") else: JBox.do_update_user_home_image() JBoxAsyncJob.async_plugin_maintenance(is_leader)
def monitor_registrations(): max_rate = JBoxDynConfig.get_registration_hourly_rate(Compute.get_install_id()) rate = JBoxUserV2.count_created(1) reg_allowed = JBoxDynConfig.get_allow_registration(Compute.get_install_id()) JBox.log_debug("registration allowed: %r, rate: %d, max allowed: %d", reg_allowed, rate, max_rate) if (reg_allowed and (rate > max_rate*1.1)) or ((not reg_allowed) and (rate < max_rate*0.9)): reg_allowed = not reg_allowed JBox.log_warn("Changing registration allowed to %r", reg_allowed) JBoxDynConfig.set_allow_registration(Compute.get_install_id(), reg_allowed) if reg_allowed: num_pending_activations = JBoxUserV2.count_pending_activations() if num_pending_activations > 0: JBox.log_info("scheduling activations for %d pending activations", num_pending_activations) JBoxAsyncJob.async_schedule_activations()
def publish_sessions(): iid = Compute.get_instance_id() for c in SessContainer.session_containers(allcontainers=True): if ('Names' in c) and (c['Names'] is not None): sessname = SessContainer(c['Id']).get_name() if sessname: JBoxSessionProps.attach_instance(Compute.get_install_id(), sessname, iid, c["Status"])
def schedule_housekeeping(cmd, is_leader): JBoxd.publish_perf_counters() JBoxd.publish_sessions() JBoxd.publish_instance_state() features = [JBPluginTask.JBP_NODE] if is_leader is True: JBoxInstanceProps.purge_stale_instances(Compute.get_install_id()) features.append(JBPluginTask.JBP_CLUSTER) for feature in features: for plugin in JBPluginTask.jbox_get_plugins(feature): JBoxd.schedule_thread(cmd, plugin.do_periodic_task, (feature,))
def publish_instance_state(): iid = Compute.get_instance_id() api_status = dict() for c in BaseContainer.api_containers(allcontainers=True): name = c["Names"][0] if (("Names" in c) and (c["Names"] is not None)) else c["Id"][0:12] api_name = APIContainer.get_api_name_from_container_name(name) if api_name is None: continue cnt = api_status.get(api_name, 0) api_status[api_name] = cnt + 1 self_load = Compute.get_instance_stats(iid, 'Load') accept = Compute.should_accept_session(is_proposed_cluster_leader()) JBoxInstanceProps.set_props(Compute.get_install_id(), iid, load=self_load, accept=accept, api_status=api_status)
def update_juliabox_status(): in_error = len(JBoxInstanceProps.get_stale_instances(Compute.get_install_id())) instance_status = JBoxInstanceProps.get_instance_status(Compute.get_install_id()) HTML = "<html><body><center><pre>\nJuliaBox is Up.\n\nLast updated: " + datetime.datetime.now().isoformat() + " UTC\n\nLoads: " for iid in instance_status: HTML += (str(instance_status[iid]['load']) + '% ') HTML += ("\n\nErrors: " + str(in_error) + "\n\nAWS Status: <a href='http://status.aws.amazon.com/'>status.aws.amazon.com</a></pre></center></body></html>") plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE) bkt = JBoxCfg.get("cloud_host.status_bucket") if plugin is not None and bkt is not None: try: f = open("/tmp/index.html", "w") f.write(HTML) f.close() plugin.push(bkt, "/tmp/index.html") finally: os.remove("/tmp/index.html") else: JBox.log_debug("Status: %s", HTML) return None
def monitor_registrations(): max_rate = JBoxDynConfig.get_registration_hourly_rate( Compute.get_install_id()) rate = JBoxUserV2.count_created(1) reg_allowed = JBoxDynConfig.get_allow_registration( Compute.get_install_id()) JBox.log_debug("registration allowed: %r, rate: %d, max allowed: %d", reg_allowed, rate, max_rate) if (reg_allowed and (rate > max_rate * 1.1)) or ((not reg_allowed) and (rate < max_rate * 0.9)): reg_allowed = not reg_allowed JBox.log_warn("Changing registration allowed to %r", reg_allowed) JBoxDynConfig.set_allow_registration(Compute.get_install_id(), reg_allowed) if reg_allowed: num_pending_activations = JBoxUserV2.count_pending_activations() if num_pending_activations > 0: JBox.log_info( "scheduling activations for %d pending activations", num_pending_activations) JBoxAsyncJob.async_schedule_activations()
def do_housekeeping(): terminating = False server_delete_timeout = JBoxCfg.get('interactive.expire') inactive_timeout = JBoxCfg.get('interactive.inactivity_timeout') SessContainer.maintain(max_timeout=server_delete_timeout, inactive_timeout=inactive_timeout) is_leader = is_cluster_leader() if is_leader: terminating = False else: try: terminating = JBoxAsyncJob.sync_is_terminating() if terminating['code'] == 0: terminating = terminating['data'] else: JBox.log_error( "Error checking if instance is terminating. Assuming False." ) terminating = False except: JBox.log_error( "Exception checking if instance is terminating. Assuming False." ) terminating = False if is_leader: JBox.log_info("I am the cluster leader") JBox.update_juliabox_status() JBox.monitor_registrations() if not JBoxDynConfig.is_stat_collected_within( Compute.get_install_id(), 1): JBoxAsyncJob.async_collect_stats() if terminating: JBox.log_warn("terminating to scale down") else: JBox.do_update_user_home_image() JBoxAsyncJob.async_plugin_maintenance(is_leader)
def publish_container_stats(): VolMgr.publish_stats() db.publish_stats() JBoxDynConfig.set_stat_collected_date(Compute.get_install_id())
def publish_sessions(): iid = Compute.get_instance_id() for c in SessContainer.session_containers(allcontainers=True): if ('Names' in c) and (c['Names'] is not None): JBoxSessionProps.attach_instance(Compute.get_install_id(), SessContainer(c['Id']).get_name(), iid, c["Status"])