def _quit(self):
# Stop resource agents.
agent_pid = self._redis_connection.hget(LocalParams.RESOURCE_INFO,
"agent_pid")
close_by_pid(pid=int(agent_pid), recursive=True)
logger.info("Resource agents exited!")
# Close Resource Redis.
redis_pid = get_redis_pid_by_port(LocalParams.RESOURCE_REDIS_PORT)
close_by_pid(redis_pid, recursive=False)
logger.info("Resource Redis exited!")
def _check_killed_tickets(self):
# Check pending job ticket
killed_job_names = self.redis_connection.lrange(ProcessRedisName.KILLED_JOB_TICKETS, 0, -1)
for job_name in killed_job_names:
job_detail = json.loads(self.redis_connection.hget(ProcessRedisName.JOB_DETAILS, job_name))
if job_detail["status"] == JobStatus.RUNNING:
close_by_pid(pid=job_detail["pid_list"], recursive=False)
del job_detail["pid_list"]
elif job_detail["status"] == JobStatus.PENDING:
self.redis_connection.lrem(ProcessRedisName.PENDING_JOB_TICKETS, 0, job_name)
elif job_detail["status"] == JobStatus.FINISH:
continue
job_detail["status"] = JobStatus.KILLED
self.redis_connection.hset(ProcessRedisName.JOB_DETAILS, job_name, json.dumps(job_detail))
self.redis_connection.lrem(ProcessRedisName.KILLED_JOB_TICKETS, 0, job_name)
def _close_agents(self):
if (not self.redis_connection.hlen(ProcessRedisName.RUNNING_JOB)
and not self.redis_connection.llen(
ProcessRedisName.PENDING_JOB_TICKETS)):
self._shutdown_count += 1
else:
self._shutdown_count = 0
if self._shutdown_count >= self._countdown:
agent_pid = int(
self.redis_connection.hget(ProcessRedisName.SETTING,
"agent_pid"))
# close agent
close_by_pid(pid=agent_pid, recursive=True)
# Set agent status to 0
self.redis_connection.hset(ProcessRedisName.SETTING,
"agent_status", 0)
def _close_agents(self):
if (
not len(
JobTrackingAgent.get_running_jobs(self.redis_connection.hgetall(ProcessRedisName.JOB_DETAILS))
) and
not self.redis_connection.llen(ProcessRedisName.PENDING_JOB_TICKETS)
):
self._shutdown_count += 1
else:
self._shutdown_count = 0
if self._shutdown_count >= self._countdown:
agent_pid = int(self.redis_connection.hget(ProcessRedisName.SETTING, "agent_pid"))
# close agent
close_by_pid(pid=agent_pid, recursive=True)
# Set agent status to 0
self.redis_connection.hset(ProcessRedisName.SETTING, "agent_status", 0)
def _check_kill_ticket(self):
# Check pending job ticket
killed_job_names = self.redis_connection.lrange(
ProcessRedisName.KILLED_JOB_TICKETS, 0, -1)
for job_name in killed_job_names:
if self.redis_connection.hexists(ProcessRedisName.RUNNING_JOB,
job_name):
pid_list = json.loads(
self.redis_connection.hget(ProcessRedisName.RUNNING_JOB,
job_name))
close_by_pid(pid=pid_list, recursive=False)
self.redis_connection.hdel(ProcessRedisName.RUNNING_JOB,
job_name)
else:
self.redis_connection.lrem(
ProcessRedisName.PENDING_JOB_TICKETS, 0, job_name)
self.redis_connection.lrem(ProcessRedisName.KILLED_JOB_TICKETS, 0,
job_name)
def delete(**kwargs):
setting_info = load_setting_info()
# Build connection
redis_connection = redis.Redis(host=setting_info["redis_info"]["host"], port=setting_info["redis_info"]["port"])
# Stop running jobs
running_jobs = redis_connection.hgetall(ProcessRedisName.RUNNING_JOB)
if running_jobs:
for job_name, pid_list in running_jobs.items():
pid_list = json.loads(pid_list)
close_by_pid(pid=pid_list, recursive=False)
logger.info(f"Stop running job {job_name.decode()}.")
# Stop Agents
agent_status = int(redis_connection.hget(ProcessRedisName.SETTING, "agent_status"))
if agent_status:
agent_pid = int(redis_connection.hget(ProcessRedisName.SETTING, "agent_pid"))
close_by_pid(pid=agent_pid, recursive=True)
redis_connection.hset(ProcessRedisName.SETTING, "agent_status", 0)
logger.info("Close agents.")
else:
logger.info("Agents' status is already closed.")
# close Redis
redis_mode = redis_connection.hget(ProcessRedisName.SETTING, "redis_mode").decode()
if redis_mode == "MARO":
get_redis_pid_command = f"pidof 'redis-server *:{setting_info['redis_info']['port']}'"
get_redis_pid_process = subprocess.Popen(get_redis_pid_command, shell=True, stdout=subprocess.PIPE)
redis_pid = int(get_redis_pid_process.stdout.read())
get_redis_pid_process.wait()
close_by_pid(pid=redis_pid, recursive=False)
logger.info(f"Close Redis server with port {setting_info['redis_info']['port']}")
else:
logger.info(f"MARO does not close Redis server with mode {redis_mode}.")
# Rm process environment setting
os.remove(os.path.expanduser(LocalPaths.MARO_PROCESS_SETTING))
def _agents_stop(self):
try:
agent_pid = int(self._redis_connection.hget(f"{self.cluster_name}:runtime_detail", "agent_id"))
close_by_pid(agent_pid, recursive=True)
except Exception as e:
logger.warning(f"Failure to close {self.cluster_name}'s agents, due to {e}")