import Queue from Util import logger import Task appmgr_log = logger.getLogger('AppMgr') class IAppManager: def __init__(self, apps): self.applist = {} # A list of applications id:app self.app_status = {} # appid: true/false if app finalized #self.task_list = {} # tid: task self.app_task_list = {} # appid: {tid:taskobj} self.tid = 0 self.runflag = False index = 0 for app in apps: if not app.checkApp(): appmgr_log.warning('[AppMgr] APP %s is incompatible, skip' % app.name) continue self.applist[index] = app self.app_status[index] = False self.app_task_list[index] = [] app.set_id(index) index += 1 appmgr_log.info('[AppMgr] APP %s is loaded' % app.name) appmgr_log.debug('[AppMgr] Load apps, the number of app = %s' % self.applist) if len(self.applist) > 0: self.current_app = self.applist[0]
def __init__(self, name=None, capacity=1): import uuid as uuid_mod self.uuid = str(uuid_mod.uuid4()) if name is None: name = self.uuid #print "my name = %s"%self.uuid global wlog wlog = logger.getLogger('Worker_%s' % name) self.worker_class = None self.recv_buff = IM.IRecv_buffer() self.__should_stop = False Config.Config() self.cfg = Config.Config import socket wlog.info('[Agent] Running on host: %s' % socket.gethostname()) wlog.debug('[Agent] Start to connect to service <%s>' % self.cfg.getCFGattr('svc_name')) self.client = Client(self.recv_buff, self.cfg.getCFGattr('svc_name'), self.uuid) self.client.set_portfile( Config.Config.getCFGattr('rundir') + '/port.txt') ret = self.client.initial() #----test---- #self.client=None #ret = 0 #----test---- if ret != 0: #client initial error, add handler self.client.stop(True) wlog.error('[Agent] Client initialize error, errcode = %d' % ret) exit() if self.cfg.getCFGattr('delay_rec') == 'True': self.recoder = Recoder.BaseRecoder( Config.Config.getCFGattr('rundir') + '/Agent-' + self.uuid) self.wid = None self.appid = None self.capacity = capacity self.task_queue = Queue.Queue(maxsize=self.capacity + 1) #store task obj self.removed_tasks = [] self.task_completed_queue = Queue.Queue() # store task obj self.ignoreTask = [] self.initExecutor = None #init task obj self.tmpLock = threading.RLock() self.finExecutor = None self.fin_flag = False self.initial_flag = False self.app_fin_flag = False self.halt_flag = False self.task_acquire = False self.heartcond = threading.Condition() self.heartbeat = HeartbeatThread(self.client, self, self.heartcond) self.list_lock = threading.RLock() self.worker_list = {} self.worker_status = {} self.cond_list = {} self.status = WorkerStatus.NEW self.loop_time = 0.1 #code for read_data self.mult_file_path = [] self.res_data = None self.read_flag = False
import threading from Util import logger base_thread_log = logger.getLogger("BaseThread") class BaseThread(threading.Thread): """ Base Thread class for this framework """ def __init__(self, name=""): name = '.'.join(['DistJET.BaseThread',name]) print("[BaseThread]: create new thread : %s" % name) super(BaseThread, self).__init__(name=name) self.setDaemon(1) self.lock = threading.RLock() self.__should_stop = False base_thread_log.debug('BaseThread object created:%s', self.__class__.__name__) def get_stop_flag(self): return self.__should_stop def stop(self): self.lock.acquire() if not self.__should_stop: base_thread_log.debug('Stopping: %s', self.__class__.__name__) self.__should_stop = True self.lock.release()
import Queue import WorkerRegistry import Task import IAppManager from Util import logger from Util.Config import Config scheduler_log = logger.getLogger('AppMgr') class IScheduler: def __init__(self, master, appmgr, worker_registry=None): self.master = master self.appid = None self.worker_registry = worker_registry self.appmgr = appmgr self.task_todo_queue = Queue.Queue() # task queue, store task obj scheduler_log.info('[Scheduler] Load tasks created by AppMgr') self.task_list = self.appmgr.get_app_task_list() for tid, task in self.task_list.items(): if not isinstance(task, Task.ChainTask): self.task_todo_queue.put(tid) elif task.father_len() == 0: self.task_todo_queue.put(tid) scheduler_log.info('[Scheduler] Load %d tasks, add %d task to queue' % (len(self.task_list), self.task_todo_queue.qsize())) self.scheduled_task_list = {} # wid: tid_list self.completed_queue = Queue.Queue()
import Queue import time import os import traceback import threading from Util import logger control_log = logger.getLogger('WatchDog_Log') handler_log = logger.getLogger('Handler') master_log = logger.getLogger('Master') health_log = logger.getLogger('Health') import IR_Buffer_Module as IM import IScheduler import WorkerRegistry import python.Util.Package as Pack WorkerRegistry.wRegistery_log = master_log import MPI_Wrapper MPI_Wrapper.MPI_log = master_log from BaseThread import BaseThread from IAppManager import SimpleAppManager from Task import TaskStatus from python.Process.Process import status from python.Util.Config import Config, set_inipath from python.Util.Recoder import BaseElement, BaseRecoder from python.WorkerRegistry import WorkerStatus