def launch_session(name, email, reuse=True):
try:
JBoxd._wait_for_session_backup(name)
VolMgr.refresh_disk_use_status()
JBoxContainer.launch_by_name(name, email, reuse=reuse)
finally:
JBoxd.finish_thread()
def launch_session(name, email, reuse=True):
try:
JBoxd._wait_for_session_backup(name)
VolMgr.refresh_disk_use_status()
JBoxContainer.launch_by_name(name, email, reuse=reuse)
finally:
JBoxd.finish_thread()
def maintain(max_timeout=0, inactive_timeout=0, protected_names=()):
JBoxContainer.log_info("Starting container maintenance...")
tnow = datetime.datetime.now(pytz.utc)
tmin = datetime.datetime(datetime.MINYEAR, 1, 1, tzinfo=pytz.utc)
stop_before = (tnow - datetime.timedelta(seconds=max_timeout)) if (max_timeout > 0) else tmin
stop_inacive_before = (tnow - datetime.timedelta(seconds=inactive_timeout)) if (inactive_timeout > 0) else tmin
all_containers = JBoxContainer.DCKR.containers(all=True)
all_cnames = {}
container_id_list = []
for cdesc in all_containers:
cid = cdesc['Id']
cont = JBoxContainer(cid)
container_id_list.append(cid)
cname = cont.get_name()
all_cnames[cname] = cid
if (cname is None) or (cname in protected_names):
JBoxContainer.log_debug("Ignoring %s", cont.debug_str())
continue
c_is_active = cont.is_running() or cont.is_restarting()
last_ping = JBoxContainer._get_last_ping(cname)
# if we don't have a ping record, create one (we must have restarted)
if (last_ping is None) and c_is_active:
JBoxContainer.log_info("Discovered new container %s", cont.debug_str())
JBoxContainer.record_ping(cname)
start_time = cont.time_started()
# check that start time is not absurdly small (indicates a continer that's starting up)
start_time_not_zero = (tnow-start_time).total_seconds() < (365*24*60*60)
if (start_time < stop_before) and start_time_not_zero:
# don't allow running beyond the limit for long running sessions
# JBoxContainer.log_info("time_started " + str(cont.time_started()) +
# " delete_before: " + str(delete_before) +
# " cond: " + str(cont.time_started() < delete_before))
JBoxContainer.log_info("Running beyond allowed time %s", cont.debug_str())
cont.async_backup_and_cleanup()
elif (last_ping is not None) and c_is_active and (last_ping < stop_inacive_before):
# if inactive for too long, stop it
# JBoxContainer.log_info("last_ping " + str(last_ping) + " stop_before: " + str(stop_before) +
# " cond: " + str(last_ping < stop_before))
JBoxContainer.log_info("Inactive beyond allowed time %s", cont.debug_str())
cont.async_backup_and_cleanup()
# delete ping entries for non exixtent containers
for cname in JBoxContainer.PINGS.keys():
if cname not in all_cnames:
del JBoxContainer.PINGS[cname]
JBoxContainer.VALID_CONTAINERS = all_cnames
JBoxContainer.publish_container_stats()
VolMgr.refresh_disk_use_status(container_id_list=container_id_list)
JBoxContainer.log_info("Finished container maintenance.")
def publish_perf_counters():
""" Publish performance counters. Used for status monitoring and auto scaling. """
VolMgr.refresh_disk_use_status()
nactive = BaseContainer.num_active(BaseContainer.SFX_INT)
stats = []
stats.append(("NumActiveContainers", "Count", nactive))
nactive_api = BaseContainer.num_active(BaseContainer.SFX_API)
stats.append(("NumActiveAPIContainers", "Count", nactive_api))
curr_cpu_used_pct = psutil.cpu_percent()
last_cpu_used_pct = curr_cpu_used_pct if BaseContainer.LAST_CPU_PCT is None else BaseContainer.LAST_CPU_PCT
BaseContainer.LAST_CPU_PCT = curr_cpu_used_pct
cpu_used_pct = int((curr_cpu_used_pct + last_cpu_used_pct) / 2)
stats.append(("CPUUsed", "Percent", cpu_used_pct))
mem_used_pct = psutil.virtual_memory().percent
stats.append(("MemUsed", "Percent", mem_used_pct))
disk_used_pct = 0
for x in psutil.disk_partitions():
if not VolMgr.is_mount_path(x.mountpoint):
try:
disk_used_pct = max(psutil.disk_usage(x.mountpoint).percent, disk_used_pct)
except:
pass
if BaseContainer.INITIAL_DISK_USED_PCT is None:
BaseContainer.INITIAL_DISK_USED_PCT = disk_used_pct
disk_used_pct = max(0, (disk_used_pct - BaseContainer.INITIAL_DISK_USED_PCT))
stats.append(("DiskUsed", "Percent", disk_used_pct))
cont_load_pct = min(100, max(0, nactive * 100 / SessContainer.MAX_CONTAINERS))
stats.append(("ContainersUsed", "Percent", cont_load_pct))
api_cont_load_pct = min(100, max(0, nactive_api * 100 / APIContainer.MAX_CONTAINERS))
stats.append(("APIContainersUsed", "Percent", api_cont_load_pct))
stats.append(("DiskIdsUsed", "Percent", VolMgr.used_pct()))
overall_load_pct = max(
cont_load_pct, api_cont_load_pct, disk_used_pct, mem_used_pct, cpu_used_pct, VolMgr.used_pct()
)
stats.append(("Load", "Percent", overall_load_pct))
Compute.publish_stats_multi(stats)
def publish_perf_counters():
""" Publish performance counters. Used for status monitoring and auto scaling. """
VolMgr.refresh_disk_use_status()
nactive = BaseContainer.num_active(BaseContainer.SFX_INT)
stats = []
stats.append(("NumActiveContainers", "Count", nactive))
nactive_api = BaseContainer.num_active(BaseContainer.SFX_API)
stats.append(("NumActiveAPIContainers", "Count", nactive_api))
curr_cpu_used_pct = psutil.cpu_percent()
last_cpu_used_pct = curr_cpu_used_pct if BaseContainer.LAST_CPU_PCT is None else BaseContainer.LAST_CPU_PCT
BaseContainer.LAST_CPU_PCT = curr_cpu_used_pct
cpu_used_pct = int((curr_cpu_used_pct + last_cpu_used_pct)/2)
stats.append(("CPUUsed", "Percent", cpu_used_pct))
mem_used_pct = psutil.virtual_memory().percent
stats.append(("MemUsed", "Percent", mem_used_pct))
disk_used_pct = 0
for x in psutil.disk_partitions():
if not VolMgr.is_mount_path(x.mountpoint):
try:
disk_used_pct = max(psutil.disk_usage(x.mountpoint).percent, disk_used_pct)
except:
pass
if BaseContainer.INITIAL_DISK_USED_PCT is None:
BaseContainer.INITIAL_DISK_USED_PCT = disk_used_pct
disk_used_pct = max(0, (disk_used_pct - BaseContainer.INITIAL_DISK_USED_PCT))
stats.append(("DiskUsed", "Percent", disk_used_pct))
cont_load_pct = min(100, max(0, nactive * 100 / SessContainer.MAX_CONTAINERS))
stats.append(("ContainersUsed", "Percent", cont_load_pct))
api_cont_load_pct = min(100, max(0, nactive_api * 100 / APIContainer.MAX_CONTAINERS))
stats.append(("APIContainersUsed", "Percent", api_cont_load_pct))
stats.append(("DiskIdsUsed", "Percent", VolMgr.used_pct()))
overall_load_pct = max(cont_load_pct, api_cont_load_pct, disk_used_pct, mem_used_pct, cpu_used_pct, VolMgr.used_pct())
stats.append(("Load", "Percent", overall_load_pct))
Compute.publish_stats_multi(stats)
def launch_session(name, email, reuse=True):
JBoxd._wait_for_session_backup(name)
VolMgr.refresh_disk_use_status()
SessContainer.launch_by_name(name, email, reuse=reuse)
JBoxd.publish_perf_counters()
def launch_session(name, email, reuse=True):
JBoxd.publish_anticipated_load(name)
JBoxd._wait_for_session_backup(name)
VolMgr.refresh_disk_use_status()
JBoxd._launch_session(name, email, reuse)
def maintain(max_timeout=0, inactive_timeout=0, protected_names=()):
JBoxContainer.log_info("Starting container maintenance...")
tnow = datetime.datetime.now(pytz.utc)
tmin = datetime.datetime(datetime.MINYEAR, 1, 1, tzinfo=pytz.utc)
stop_before = (tnow - datetime.timedelta(seconds=max_timeout)) if (
max_timeout > 0) else tmin
stop_inacive_before = (tnow - datetime.timedelta(
seconds=inactive_timeout)) if (inactive_timeout > 0) else tmin
all_containers = JBoxContainer.DCKR.containers(all=True)
all_cnames = {}
container_id_list = []
for cdesc in all_containers:
cid = cdesc['Id']
cont = JBoxContainer(cid)
container_id_list.append(cid)
cname = cont.get_name()
all_cnames[cname] = cid
if (cname is None) or (cname in protected_names):
JBoxContainer.log_debug("Ignoring %s", cont.debug_str())
continue
c_is_active = cont.is_running() or cont.is_restarting()
last_ping = JBoxContainer._get_last_ping(cname)
# if we don't have a ping record, create one (we must have restarted)
if (last_ping is None) and c_is_active:
JBoxContainer.log_info("Discovered new container %s",
cont.debug_str())
JBoxContainer.record_ping(cname)
start_time = cont.time_started()
# check that start time is not absurdly small (indicates a continer that's starting up)
start_time_not_zero = (tnow - start_time).total_seconds() < (
365 * 24 * 60 * 60)
if (start_time < stop_before) and start_time_not_zero:
# don't allow running beyond the limit for long running sessions
# JBoxContainer.log_info("time_started " + str(cont.time_started()) +
# " delete_before: " + str(delete_before) +
# " cond: " + str(cont.time_started() < delete_before))
JBoxContainer.log_warn("Running beyond allowed time %s",
cont.debug_str())
cont.async_backup_and_cleanup()
elif (last_ping is not None) and c_is_active and (
last_ping < stop_inacive_before):
# if inactive for too long, stop it
# JBoxContainer.log_info("last_ping " + str(last_ping) + " stop_before: " + str(stop_before) +
# " cond: " + str(last_ping < stop_before))
JBoxContainer.log_warn("Inactive beyond allowed time %s",
cont.debug_str())
cont.async_backup_and_cleanup()
# delete ping entries for non exixtent containers
for cname in JBoxContainer.PINGS.keys():
if cname not in all_cnames:
del JBoxContainer.PINGS[cname]
JBoxContainer.VALID_CONTAINERS = all_cnames
JBoxContainer.publish_container_stats()
VolMgr.refresh_disk_use_status(container_id_list=container_id_list)
JBoxContainer.log_info("Finished container maintenance.")
def launch_session(name, email, reuse=True):
JBoxd.publish_anticipated_load(name)
JBoxd._wait_for_session_backup(name)
VolMgr.refresh_disk_use_status()
JBoxd._launch_session(name, email, reuse)
