def remove_progs():
log.info('Removing programs')
log.debug('Removing programs: {}'.format(dashboard.prog_to_remove))
stop_programs(None, dashboard.prog_to_remove)
for prog_name in dashboard.prog_to_remove:
prog = dashboard.programs.pop(prog_name, None)
for process in prog.processes:
dashboard.name_procs.pop(process.name)
def launch_manager():
"""
Launches all programs in the list contained in the dashboard
:return:
"""
log.info('starting launch_manager')
# kill old programs in reload or create reload_manager
log.info('debug: fds_buff: {0}'.format(dashboard.fds_buff.keys()))
for program in dashboard.programs.values():
launch_program(program)
def serve_relaod(cs, query_list):
log.info('serving: reload: {0}'.format(query_list))
if len(query_list) < 2:
utils.socket_send(cs, 'err: config file is required')
return
config_server = tm_config.ConfigServer(query_list[1])
if not config_server.valid:
log.debug('invalid config has occurred')
utils.socket_send(cs, 'warn: the config is invalid')
return
new_programs = mod_dash.load_programs(config_server.data)
utils.thread_start(reload_launch_manager, (new_programs, ))
def serve_status(cs, query_list):
log.info('serving: status: {0}'.format(query_list))
if len(query_list) > 1:
prog_names = query_list[1:]
else:
prog_names = list(dashboard.programs.keys())
log.debug('something good')
for prog_name in prog_names:
utils.socket_send(cs, 'program status: {0}'.format(prog_name))
program = dashboard.programs.get(prog_name)
for process in program.processes:
utils.socket_send(
cs, '\t{0} status {1}'.format(process.name, process.state))
def serve_client(cs, addr, configServer):
while True:
# query = cs.recv(1024).decode('utf-8')
query = utils.socket_recv(cs)
if query == '':
continue
query_list = query.split()
log.info('query: {0}'.format(query_list))
# utils.socket_send(cs, 'something from server')
if query_list[0] in services.keys():
log.info('found service')
service = services.get(query_list[0])
service(cs, query_list)
utils.socket_send(cs, '\r')
def reload_launch_manager(new_programs: {str: Program}):
log.info('starting reload launch manager')
old_programs = dashboard.programs
nprog_names = list(new_programs.keys())
oprog_names = list(old_programs.keys())
for oprog_name, oprog in old_programs.items():
if oprog_name not in nprog_names:
dashboard.prog_to_remove.append(oprog_name)
else:
nprog = new_programs.get(oprog_name)
if nprog.cmd is not oprog.cmd \
or nprog.numprocs is not oprog.numprocs\
or nprog.argv is not oprog.argv\
or nprog.wdir is not oprog.wdir:
dashboard.prog_to_remove.append(oprog_name)
remove_progs()
dashboard.programs[oprog_name] = nprog
launch_program(nprog)
def authenticate(self) -> bool:
if not self.config.username:
self.config.username = input("Enter username: "******"Enter password: "******"Enter password: ")
query = 'auth\r\n{0}\r\n{1}\r\n'\
.format(self.config.username, self.config.password)
self.csocket.send(query.encode('utf-8'))
response = self.csocket.recv(1024).decode('utf-8')
log.debug('received {0} from server'.format(response))
response = response.rsplit('\r\n')
log.debug('response list {0}'.format(response))
if response[0] != 'OK':
log.info('failed to authenticate with response: {0}'.format(
response[0]))
self.config.password = None
return False
return True
def launch_program(program: Program, force_start=False):
"""
Launches a program by launching all the processes created by this program
if the start is forced by client or autostart is enabled then
Otherwise, all processes will be moved to STOPPED state.
:param program: Program object to be launched.
:param force_start: True if client requesting a start.
:return:
"""
if not program or not isinstance(program, Program):
return
log.info('launching program {0}, umask={1}'.format(program.name,
program.umask))
log.info('launching {0}'.format(program.cmd.split(' ')))
for process in program.processes:
log.debug('Starting new process !')
# dashboard.name_procs[process.name] = process # move to higher level
log.debug('AUTO_START -------- {} === {}'.format(
program.autostart, configmap.Start.NO))
if program.autostart is False:
process.state = ProcessState.STOPPED
else:
launch_process(program, process)
def state_manager():
# os.mkdir('/tmp/statedir')
log.info('starting state_manager')
while True:
log.info('checking processes state')
for pid_exit in list(dashboard.pid_wexit):
log.info('updating state of pid: {0}'.format(pid_exit[0]))
process = dashboard.pid_procs.get(pid_exit[0])
program = dashboard.programs.get(process.program_name)
try:
process.control_starting_state(program)
if process.to_remove:
clean_proccess(process, ProcessState.REMOVED)
program.processes.pop(process.name)
if process.state == ProcessState.RUNNING:
log.info('process {0} exited with code {1}'.format(
pid_exit[0], pid_exit[1]))
clean_proccess(process, ProcessState.EXITED)
log.info('process {0} has been cleaned'.format(
pid_exit[0]))
if pid_exit[1] in program.exitcodes:
if program.autorestart == 'true':
launch_process(program, process)
elif program.autorestart == 'unexpected':
launch_process(program, process)
elif process.state == ProcessState.STOPPING:
clean_proccess(process, ProcessState.STOPPED)
elif process.state == ProcessState.BACKOFF:
clean_proccess(process, ProcessState.FATAL)
launch_process(program, process, retry=True)
except OSError as err:
process.state = ProcessState.UNKNOWN
dashboard.pid_wexit.remove(pid_exit)
time.sleep(1)
def serve_restart(cs, query_list):
log.info('serving: restart: {0}'.format(query_list))
prog_names = query_list[1:]
for prog_name in prog_names:
if prog_name not in dashboard.programs.keys():
utils.socket_send(cs, 'program {0} not found'.format(prog_name))
else:
utils.socket_send(cs, 'restarting {0}'.format(prog_name))
program = dashboard.programs.get(prog_name)
for process in program.process:
utils.socket_send(
cs, 'process {0} is in {1} state'.format(
process.name, process.state))
if process.state == ProcessState.STARTING \
or process.state == ProcessState.BACKOFF \
or process.state == ProcessState.RUNNING:
utils.socket_send(
cs, 'stopping process {0}'.format(process.name))
#kill process
utils.thread_start(kill_process,
(process, program.stopsig))
utils.socket_send(
cs, 'starting process: {0}'.format(process.name))
launch_process(program, process)
def serve_start(cs, query_list):
log.info('serving: start: {0}'.format(query_list))
# utils.socket_send(cs, 'something there from start')
# utils.socket_send(cs, '\r')
prog_names = query_list[1:]
log.debug('prog_names: {0}'.format(prog_names))
for prog_name in prog_names:
if prog_name not in dashboard.programs.keys():
utils.socket_send(cs, 'program {0} not found'.format(prog_name))
else:
utils.socket_send(cs, 'starting {0}'.format(prog_name))
program = dashboard.programs.get(prog_name)
changed = 0
for process in program.processes:
utils.socket_send(
cs, 'process {0} is in {1} state'.format(
process.name, process.state))
if process.state != ProcessState.STARTING \
and process.state != ProcessState.BACKOFF \
and process.state != ProcessState.RUNNING:
# see if it needs to reset retries
utils.socket_send(
cs, 'starting process: {0}'.format(process.name))
launch_process(program, process)
def buff_manager():
"""\
Buffer Manager is responsible for dispatching
standard out/err flow of running processes by
by selecting open fds which are ready to read from
and transport the data to the appropriate files.
"""
log.info('starting buff_manager')
while 1:
time.sleep(TIME_SLEEP)
log.debug('checking fds')
fds = list(dashboard.fds_buff.keys())
if not len(fds):
log.debug('empty fd list')
continue
rfds = []
try: # select open fds which are ready to read from
rfds, wfds, xfds = select(fds, [], [])
except OSError:
log.error('error occurred upon select fds')
continue
log.info('found some fds: {0}'.format(rfds))
for fd in rfds: # transporting data from each open fd
log.info('fd={0} ready to write in {1}'.format(
fd, dashboard.fds_buff.get(fd)))
data = None
try:
data = os.read(fd, BUFF_SIZE)
except OSError as err:
log.error('Failed to read from fd={0}'.format(fd))
if not data:
if fd in dashboard.fds_zombie: # close and remove the fd if it's in zombie list
dashboard.fds_zombie.remove(fd)
os.close(fd)
dashboard.fds_buff.pop(fd)
continue
file = dashboard.fds_buff.get(
fd) # get the linked file from the dashboard
if isinstance(file, pathlib.Path):
if not file.exists():
file.touch(exist_ok=True)
with file.open('a') as f:
f.write(data.decode(DECODE_FORMAT))
def launch_process(program: Program, process: Process, retry: bool = False):
"""
Launches a process and update the dashboard by the pid of the launched process
and the opened file descriptors.
:param program: Parent program of the process.
:param process: The process to be launched.
:param retry: True if the process is being restarted from a retry
:return:
"""
log.info('launching process: {0}'.format(process.name))
if retry:
process.retries = process.retries - 1
if process.retries < 1:
return
while process.state == ProcessState.STOPPING: # recheck
time.sleep(1)
pid, fds = process.exec(program)
dashboard.pid_procs[pid] = process
dashboard.fds_buff.update(fds)
log.info('process {0} has been executed with pid={1}'.format(
process.name, pid))
for fd, file in fds.items():
log.info('fd={0} file={1}'.format(fd, file))
dashboard.name_procs[process.name] = process # move to higher level
def serve_stop(cs, query_list):
log.info('serving: stop: {0}'.format(query_list))
prog_names = query_list[1:]
stop_programs(cs, prog_names)
def serve_shutdown(cs, query_list):
log.info("Shutting down !")
stop_programs(cs, dashboard.programs.keys())
utils.socket_send(cs, 'Shutting down all programs!')
def service_manager(cs: socket.socket, addr, config):
username, auth = authenticate_client(cs, addr, config)
if not auth:
log.info('client {0} failed to authenticate'.format(username))
return
serve_client(cs, addr, config)
def run(self):
log.info('running the server daemon')
log.info('binding the server socket')
self.socket_bound = socket_bind(self.socket,
(self.config.host, self.config.port))
log.info('starting thread: state_handler')
thread_start(state_manager, ()) # not none
# thread -> buff_handler
log.info('starting thread: buff_manager')
thread_start(buff_manager, ()) # not none
# thread -> launch_handler
log.info('starting thread: launch_manager')
thread_start(launch_manager, ()) # not none
log.info('starting thread: clients_manager')
thread_start(clients_manager, (self, ))
time.sleep(20)
log.info('Server Daemon run ends !')