def try_launch_container(cls, user_id, max_hop=False): sessname = unique_sessname(user_id) cont = SessContainer.get_by_name(sessname) cls.log_debug("have existing container for %s: %r", sessname, None != cont) if cont is not None: cls.log_debug("container running: %r", cont.is_running()) if max_hop: self_load = Compute.get_instance_stats(Compute.get_instance_id(), 'Load') if self_load < 100: SessContainer.invalidate_container(sessname) JBoxAsyncJob.async_launch_by_name(sessname, user_id, True) return True is_leader = is_proposed_cluster_leader() if ((cont is None) or (not cont.is_running())) and ( not Compute.should_accept_session(is_leader)): if cont is not None: SessContainer.invalidate_container(cont.get_name()) JBoxAsyncJob.async_backup_and_cleanup(cont.dockid) return False SessContainer.invalidate_container(sessname) JBoxAsyncJob.async_launch_by_name(sessname, user_id, True) return True
def maintain(max_timeout=0, inactive_timeout=0): SessContainer.log_info("Starting container maintenance...") tnow = datetime.datetime.now(pytz.utc) tmin = datetime.datetime(datetime.MINYEAR, 1, 1, tzinfo=pytz.utc) stop_before = (tnow - datetime.timedelta(seconds=max_timeout)) if (max_timeout > 0) else tmin stop_inacive_before = (tnow - datetime.timedelta(seconds=inactive_timeout)) if (inactive_timeout > 0) else tmin all_containers = BaseContainer.session_containers(allcontainers=True) all_cnames = {} container_id_list = [] for cdesc in all_containers: cid = cdesc['Id'] cont = SessContainer(cid) container_id_list.append(cid) cname = cont.get_name() if cname is None: SessContainer.log_debug("Ignoring %s", cont.debug_str()) continue all_cnames[cname] = cid c_is_active = cont.is_running() or cont.is_restarting() last_ping = SessContainer._get_last_ping(cname) # if we don't have a ping record, create one (we must have restarted) if (last_ping is None) and c_is_active: SessContainer.log_info("Discovered new container %s", cont.debug_str()) SessContainer.record_ping(cname) start_time = cont.time_started() # check that start time is not absurdly small (indicates a continer that's starting up) start_time_not_zero = (tnow-start_time).total_seconds() < (365*24*60*60) if (start_time < stop_before) and start_time_not_zero: # don't allow running beyond the limit for long running sessions # SessContainer.log_info("time_started " + str(cont.time_started()) + # " delete_before: " + str(delete_before) + # " cond: " + str(cont.time_started() < delete_before)) SessContainer.log_warn("Running beyond allowed time %s. Scheduling cleanup.", cont.debug_str()) SessContainer.invalidate_container(cont.get_name()) JBoxAsyncJob.async_backup_and_cleanup(cont.dockid) elif (last_ping is not None) and c_is_active and (last_ping < stop_inacive_before): # if inactive for too long, stop it # SessContainer.log_info("last_ping " + str(last_ping) + " stop_before: " + str(stop_before) + # " cond: " + str(last_ping < stop_before)) SessContainer.log_warn("Inactive beyond allowed time %s. Scheduling cleanup.", cont.debug_str()) SessContainer.invalidate_container(cont.get_name()) JBoxAsyncJob.async_backup_and_cleanup(cont.dockid) # delete ping entries for non exixtent containers for cname in SessContainer.PINGS.keys(): if cname not in all_cnames: del SessContainer.PINGS[cname] SessContainer.VALID_CONTAINERS = all_cnames VolMgr.refresh_disk_use_status(container_id_list=container_id_list) SessContainer.log_info("Finished container maintenance.")
def handle_if_logout(self, cont): logout = self.get_argument('logout', False) if logout == 'me': SessContainer.invalidate_container(cont.get_name()) JBoxAsyncJob.async_backup_and_cleanup(cont.dockid) response = {'code': 0, 'data': ''} self.write(response) return True return False
def try_launch_container(cls, user_id, max_hop=False): sessname = unique_sessname(user_id) cont = SessContainer.get_by_name(sessname) cls.log_debug("have existing container for %s: %r", sessname, None != cont) if cont is not None: cls.log_debug("container running: %r", cont.is_running()) if max_hop: self_load = Compute.get_instance_stats(Compute.get_instance_id(), 'Load') if self_load < 100: SessContainer.invalidate_container(sessname) JBoxAsyncJob.async_launch_by_name(sessname, user_id, True) return True is_leader = is_proposed_cluster_leader() if ((cont is None) or (not cont.is_running())) and (not Compute.should_accept_session(is_leader)): if cont is not None: SessContainer.invalidate_container(cont.get_name()) JBoxAsyncJob.async_backup_and_cleanup(cont.dockid) return False SessContainer.invalidate_container(sessname) JBoxAsyncJob.async_launch_by_name(sessname, user_id, True) return True
def maintain(max_timeout=0, inactive_timeout=0): SessContainer.log_info("Starting container maintenance...") tnow = datetime.datetime.now(pytz.utc) tmin = datetime.datetime(datetime.MINYEAR, 1, 1, tzinfo=pytz.utc) stop_before = (tnow - datetime.timedelta(seconds=max_timeout)) if ( max_timeout > 0) else tmin stop_inacive_before = (tnow - datetime.timedelta( seconds=inactive_timeout)) if (inactive_timeout > 0) else tmin all_containers = BaseContainer.session_containers(allcontainers=True) all_cnames = {} container_id_list = [] for cdesc in all_containers: cid = cdesc['Id'] cont = SessContainer(cid) container_id_list.append(cid) cname = cont.get_name() if cname is None: SessContainer.log_debug("Ignoring %s", cont.debug_str()) continue all_cnames[cname] = cid c_is_active = cont.is_running() or cont.is_restarting() last_ping = SessContainer._get_last_ping(cname) # if we don't have a ping record, create one (we must have restarted) if (last_ping is None) and c_is_active: SessContainer.log_info("Discovered new container %s", cont.debug_str()) SessContainer.record_ping(cname) start_time = cont.time_started() # check that start time is not absurdly small (indicates a continer that's starting up) start_time_not_zero = (tnow - start_time).total_seconds() < ( 365 * 24 * 60 * 60) if (start_time < stop_before) and start_time_not_zero: # don't allow running beyond the limit for long running sessions # SessContainer.log_info("time_started " + str(cont.time_started()) + # " delete_before: " + str(delete_before) + # " cond: " + str(cont.time_started() < delete_before)) SessContainer.log_warn( "Running beyond allowed time %s. Scheduling cleanup.", cont.debug_str()) SessContainer.invalidate_container(cont.get_name()) JBoxAsyncJob.async_backup_and_cleanup(cont.dockid) elif (last_ping is not None) and c_is_active and ( last_ping < stop_inacive_before): # if inactive for too long, stop it # SessContainer.log_info("last_ping " + str(last_ping) + " stop_before: " + str(stop_before) + # " cond: " + str(last_ping < stop_before)) SessContainer.log_warn( "Inactive beyond allowed time %s. Scheduling cleanup.", cont.debug_str()) SessContainer.invalidate_container(cont.get_name()) JBoxAsyncJob.async_backup_and_cleanup(cont.dockid) elif not c_is_active and ( (tnow - cont.time_finished()).total_seconds() > (10 * 60)): SessContainer.log_warn("Dead container %s. Deleting.", cont.debug_str()) cont.delete(backup=False) del all_cnames[cname] container_id_list.remove(cid) # delete ping entries for non exixtent containers for cname in SessContainer.PINGS.keys(): if cname not in all_cnames: del SessContainer.PINGS[cname] SessContainer.VALID_CONTAINERS = all_cnames VolMgr.refresh_disk_use_status(container_id_list=container_id_list) SessContainer.log_info("Finished container maintenance.")