def launch_session(name, email, reuse=True): try: JBoxd._wait_for_session_backup(name) VolMgr.refresh_disk_use_status() JBoxContainer.launch_by_name(name, email, reuse=reuse) finally: JBoxd.finish_thread()
def maintain(max_timeout=0, inactive_timeout=0, protected_names=()): JBoxContainer.log_info("Starting container maintenance...") tnow = datetime.datetime.now(pytz.utc) tmin = datetime.datetime(datetime.MINYEAR, 1, 1, tzinfo=pytz.utc) stop_before = (tnow - datetime.timedelta(seconds=max_timeout)) if (max_timeout > 0) else tmin stop_inacive_before = (tnow - datetime.timedelta(seconds=inactive_timeout)) if (inactive_timeout > 0) else tmin all_containers = JBoxContainer.DCKR.containers(all=True) all_cnames = {} container_id_list = [] for cdesc in all_containers: cid = cdesc['Id'] cont = JBoxContainer(cid) container_id_list.append(cid) cname = cont.get_name() all_cnames[cname] = cid if (cname is None) or (cname in protected_names): JBoxContainer.log_debug("Ignoring %s", cont.debug_str()) continue c_is_active = cont.is_running() or cont.is_restarting() last_ping = JBoxContainer._get_last_ping(cname) # if we don't have a ping record, create one (we must have restarted) if (last_ping is None) and c_is_active: JBoxContainer.log_info("Discovered new container %s", cont.debug_str()) JBoxContainer.record_ping(cname) start_time = cont.time_started() # check that start time is not absurdly small (indicates a continer that's starting up) start_time_not_zero = (tnow-start_time).total_seconds() < (365*24*60*60) if (start_time < stop_before) and start_time_not_zero: # don't allow running beyond the limit for long running sessions # JBoxContainer.log_info("time_started " + str(cont.time_started()) + # " delete_before: " + str(delete_before) + # " cond: " + str(cont.time_started() < delete_before)) JBoxContainer.log_info("Running beyond allowed time %s", cont.debug_str()) cont.async_backup_and_cleanup() elif (last_ping is not None) and c_is_active and (last_ping < stop_inacive_before): # if inactive for too long, stop it # JBoxContainer.log_info("last_ping " + str(last_ping) + " stop_before: " + str(stop_before) + # " cond: " + str(last_ping < stop_before)) JBoxContainer.log_info("Inactive beyond allowed time %s", cont.debug_str()) cont.async_backup_and_cleanup() # delete ping entries for non exixtent containers for cname in JBoxContainer.PINGS.keys(): if cname not in all_cnames: del JBoxContainer.PINGS[cname] JBoxContainer.VALID_CONTAINERS = all_cnames JBoxContainer.publish_container_stats() VolMgr.refresh_disk_use_status(container_id_list=container_id_list) JBoxContainer.log_info("Finished container maintenance.")
def publish_perf_counters(): """ Publish performance counters. Used for status monitoring and auto scaling. """ VolMgr.refresh_disk_use_status() nactive = BaseContainer.num_active(BaseContainer.SFX_INT) stats = [] stats.append(("NumActiveContainers", "Count", nactive)) nactive_api = BaseContainer.num_active(BaseContainer.SFX_API) stats.append(("NumActiveAPIContainers", "Count", nactive_api)) curr_cpu_used_pct = psutil.cpu_percent() last_cpu_used_pct = curr_cpu_used_pct if BaseContainer.LAST_CPU_PCT is None else BaseContainer.LAST_CPU_PCT BaseContainer.LAST_CPU_PCT = curr_cpu_used_pct cpu_used_pct = int((curr_cpu_used_pct + last_cpu_used_pct) / 2) stats.append(("CPUUsed", "Percent", cpu_used_pct)) mem_used_pct = psutil.virtual_memory().percent stats.append(("MemUsed", "Percent", mem_used_pct)) disk_used_pct = 0 for x in psutil.disk_partitions(): if not VolMgr.is_mount_path(x.mountpoint): try: disk_used_pct = max(psutil.disk_usage(x.mountpoint).percent, disk_used_pct) except: pass if BaseContainer.INITIAL_DISK_USED_PCT is None: BaseContainer.INITIAL_DISK_USED_PCT = disk_used_pct disk_used_pct = max(0, (disk_used_pct - BaseContainer.INITIAL_DISK_USED_PCT)) stats.append(("DiskUsed", "Percent", disk_used_pct)) cont_load_pct = min(100, max(0, nactive * 100 / SessContainer.MAX_CONTAINERS)) stats.append(("ContainersUsed", "Percent", cont_load_pct)) api_cont_load_pct = min(100, max(0, nactive_api * 100 / APIContainer.MAX_CONTAINERS)) stats.append(("APIContainersUsed", "Percent", api_cont_load_pct)) stats.append(("DiskIdsUsed", "Percent", VolMgr.used_pct())) overall_load_pct = max( cont_load_pct, api_cont_load_pct, disk_used_pct, mem_used_pct, cpu_used_pct, VolMgr.used_pct() ) stats.append(("Load", "Percent", overall_load_pct)) Compute.publish_stats_multi(stats)
def publish_perf_counters(): """ Publish performance counters. Used for status monitoring and auto scaling. """ VolMgr.refresh_disk_use_status() nactive = BaseContainer.num_active(BaseContainer.SFX_INT) stats = [] stats.append(("NumActiveContainers", "Count", nactive)) nactive_api = BaseContainer.num_active(BaseContainer.SFX_API) stats.append(("NumActiveAPIContainers", "Count", nactive_api)) curr_cpu_used_pct = psutil.cpu_percent() last_cpu_used_pct = curr_cpu_used_pct if BaseContainer.LAST_CPU_PCT is None else BaseContainer.LAST_CPU_PCT BaseContainer.LAST_CPU_PCT = curr_cpu_used_pct cpu_used_pct = int((curr_cpu_used_pct + last_cpu_used_pct)/2) stats.append(("CPUUsed", "Percent", cpu_used_pct)) mem_used_pct = psutil.virtual_memory().percent stats.append(("MemUsed", "Percent", mem_used_pct)) disk_used_pct = 0 for x in psutil.disk_partitions(): if not VolMgr.is_mount_path(x.mountpoint): try: disk_used_pct = max(psutil.disk_usage(x.mountpoint).percent, disk_used_pct) except: pass if BaseContainer.INITIAL_DISK_USED_PCT is None: BaseContainer.INITIAL_DISK_USED_PCT = disk_used_pct disk_used_pct = max(0, (disk_used_pct - BaseContainer.INITIAL_DISK_USED_PCT)) stats.append(("DiskUsed", "Percent", disk_used_pct)) cont_load_pct = min(100, max(0, nactive * 100 / SessContainer.MAX_CONTAINERS)) stats.append(("ContainersUsed", "Percent", cont_load_pct)) api_cont_load_pct = min(100, max(0, nactive_api * 100 / APIContainer.MAX_CONTAINERS)) stats.append(("APIContainersUsed", "Percent", api_cont_load_pct)) stats.append(("DiskIdsUsed", "Percent", VolMgr.used_pct())) overall_load_pct = max(cont_load_pct, api_cont_load_pct, disk_used_pct, mem_used_pct, cpu_used_pct, VolMgr.used_pct()) stats.append(("Load", "Percent", overall_load_pct)) Compute.publish_stats_multi(stats)
def launch_session(name, email, reuse=True): JBoxd._wait_for_session_backup(name) VolMgr.refresh_disk_use_status() SessContainer.launch_by_name(name, email, reuse=reuse) JBoxd.publish_perf_counters()
def launch_session(name, email, reuse=True): JBoxd.publish_anticipated_load(name) JBoxd._wait_for_session_backup(name) VolMgr.refresh_disk_use_status() JBoxd._launch_session(name, email, reuse)
def maintain(max_timeout=0, inactive_timeout=0, protected_names=()): JBoxContainer.log_info("Starting container maintenance...") tnow = datetime.datetime.now(pytz.utc) tmin = datetime.datetime(datetime.MINYEAR, 1, 1, tzinfo=pytz.utc) stop_before = (tnow - datetime.timedelta(seconds=max_timeout)) if ( max_timeout > 0) else tmin stop_inacive_before = (tnow - datetime.timedelta( seconds=inactive_timeout)) if (inactive_timeout > 0) else tmin all_containers = JBoxContainer.DCKR.containers(all=True) all_cnames = {} container_id_list = [] for cdesc in all_containers: cid = cdesc['Id'] cont = JBoxContainer(cid) container_id_list.append(cid) cname = cont.get_name() all_cnames[cname] = cid if (cname is None) or (cname in protected_names): JBoxContainer.log_debug("Ignoring %s", cont.debug_str()) continue c_is_active = cont.is_running() or cont.is_restarting() last_ping = JBoxContainer._get_last_ping(cname) # if we don't have a ping record, create one (we must have restarted) if (last_ping is None) and c_is_active: JBoxContainer.log_info("Discovered new container %s", cont.debug_str()) JBoxContainer.record_ping(cname) start_time = cont.time_started() # check that start time is not absurdly small (indicates a continer that's starting up) start_time_not_zero = (tnow - start_time).total_seconds() < ( 365 * 24 * 60 * 60) if (start_time < stop_before) and start_time_not_zero: # don't allow running beyond the limit for long running sessions # JBoxContainer.log_info("time_started " + str(cont.time_started()) + # " delete_before: " + str(delete_before) + # " cond: " + str(cont.time_started() < delete_before)) JBoxContainer.log_warn("Running beyond allowed time %s", cont.debug_str()) cont.async_backup_and_cleanup() elif (last_ping is not None) and c_is_active and ( last_ping < stop_inacive_before): # if inactive for too long, stop it # JBoxContainer.log_info("last_ping " + str(last_ping) + " stop_before: " + str(stop_before) + # " cond: " + str(last_ping < stop_before)) JBoxContainer.log_warn("Inactive beyond allowed time %s", cont.debug_str()) cont.async_backup_and_cleanup() # delete ping entries for non exixtent containers for cname in JBoxContainer.PINGS.keys(): if cname not in all_cnames: del JBoxContainer.PINGS[cname] JBoxContainer.VALID_CONTAINERS = all_cnames JBoxContainer.publish_container_stats() VolMgr.refresh_disk_use_status(container_id_list=container_id_list) JBoxContainer.log_info("Finished container maintenance.")