class Master(dict):
"""
task management and scheduling
"""
def __init__(self, params_obj: ParamsParser):
super(Master, self).__init__()
self._params_obj = params_obj
self.logger = TaskLogger(self._params_obj.out_dir, 'MAIN-APP').get_logger('Master')
self._work_dir = self._params_obj.out_dir
self.server = None
self.endpoint = 'tcp://{}:{:.0f}'.format(SERVER_HOST, SERVER_PORT)
self.main_process = os.getpid()
@property
def work_dir(self):
return self._work_dir
def parser_params(self):
tmp_dic = defaultdict(list)
with open(self._params_obj.cmd_conf) as in_handler:
for name, cpu, mem, relys, cmd in csv.reader(in_handler, delimiter='\t'):
if name in self:
self.logger.error('the cmd name must not be repeated [%s is repeated]' % name)
raise KeyError('the cmd name must not be repeated [%s is repeated]' % name)
self[name] = Agent(worker_id=name, cmd=cmd, master=self, cpu=cpu, mem=mem)
relys = [i.strip() for i in relys.strip().split(',') if i]
tmp_dic[name].extend(relys)
for k, vs in tmp_dic.items():
if len(vs) == 0:
continue
# async_result = event.AsyncResult()
k_agent = self[k]
vs_agents = [self[i] for i in vs]
# k_agent.add_relys(async_result)
k_agent.add_relys(*vs_agents)
# for v_agent in vs_agents:
# v_agent.add_relied(async_result)
def stop(self):
self.server.close()
def run(self):
self.logger.info('Main APP start: [PID %s]' % self.main_process)
self.server = Server(bind_obj=ServerWorker(self))
self.logger.info('Server start')
self.server.start()
self.parser_params()
agents = self.values()
# start all agent
self.logger.info('Worker Agent start')
_ = [agent.start() for agent in agents]
self.logger.info('Agent Monitor start')
monitor = AgentMonitor(self)
joinall(agents)
for agent in agents:
agent.get_end_signal()
monitor.is_end = True
self.logger.info('all workers were completed')
self.logger.info('Main APP stop: [PID %s]' % self.main_process)
class Feluda:
def __init__(self, configPath):
self.config = config.load(configPath)
if self.config.operators:
self.operators = Operator(self.config.operators)
if self.config.store:
self.store = store.get_store(self.config.store)
if self.config.queue:
# print("---> 1", self.config.queue)
self.queue = Queue.make(self.config.queue)
if self.config.server:
self.server = Server(self.config.server)
def set_endpoints(self, endpoints):
if self.server:
for endpoint in endpoints:
self.server.add_endpoint(endpoint(self))
self.server.enable_endpoints()
else:
raise Exception("Server is not Configured")
def start(self):
if self.store:
self.store.connect()
self.store.optionally_create_index()
if self.queue:
self.queue.connect()
if self.server:
self.server.start()
def start_component(self, component_type: ComponentType):
if component_type == ComponentType.SERVER and self.server:
self.server.start()
elif component_type == ComponentType.STORE and self.store:
self.store.connect()
self.store.optionally_create_index()
elif component_type == ComponentType.QUEUE and self.queue:
self.queue.connect()
self.queue.initialize()
else:
raise Exception("Unsupported Component Type")
def get_state(self):
pass
import webbrowser
from core.server import Server
HOST = "localhost"
PORT = 8081
server = Server(HOST, PORT)
webbrowser.open("http://{}:{}".format(HOST, PORT))
server.start()
def start_server(self): #self.__load_ssl__() server = Server(); server.start();
