import Queue
from Util import logger
import Task
appmgr_log = logger.getLogger('AppMgr')
class IAppManager:
def __init__(self, apps):
self.applist = {} # A list of applications id:app
self.app_status = {} # appid: true/false if app finalized
#self.task_list = {} # tid: task
self.app_task_list = {} # appid: {tid:taskobj}
self.tid = 0
self.runflag = False
index = 0
for app in apps:
if not app.checkApp():
appmgr_log.warning('[AppMgr] APP %s is incompatible, skip' %
app.name)
continue
self.applist[index] = app
self.app_status[index] = False
self.app_task_list[index] = []
app.set_id(index)
index += 1
appmgr_log.info('[AppMgr] APP %s is loaded' % app.name)
appmgr_log.debug('[AppMgr] Load apps, the number of app = %s' %
self.applist)
if len(self.applist) > 0:
self.current_app = self.applist[0]
def __init__(self, name=None, capacity=1):
import uuid as uuid_mod
self.uuid = str(uuid_mod.uuid4())
if name is None:
name = self.uuid
#print "my name = %s"%self.uuid
global wlog
wlog = logger.getLogger('Worker_%s' % name)
self.worker_class = None
self.recv_buff = IM.IRecv_buffer()
self.__should_stop = False
Config.Config()
self.cfg = Config.Config
import socket
wlog.info('[Agent] Running on host: %s' % socket.gethostname())
wlog.debug('[Agent] Start to connect to service <%s>' %
self.cfg.getCFGattr('svc_name'))
self.client = Client(self.recv_buff, self.cfg.getCFGattr('svc_name'),
self.uuid)
self.client.set_portfile(
Config.Config.getCFGattr('rundir') + '/port.txt')
ret = self.client.initial()
#----test----
#self.client=None
#ret = 0
#----test----
if ret != 0:
#client initial error, add handler
self.client.stop(True)
wlog.error('[Agent] Client initialize error, errcode = %d' % ret)
exit()
if self.cfg.getCFGattr('delay_rec') == 'True':
self.recoder = Recoder.BaseRecoder(
Config.Config.getCFGattr('rundir') + '/Agent-' + self.uuid)
self.wid = None
self.appid = None
self.capacity = capacity
self.task_queue = Queue.Queue(maxsize=self.capacity +
1) #store task obj
self.removed_tasks = []
self.task_completed_queue = Queue.Queue() # store task obj
self.ignoreTask = []
self.initExecutor = None #init task obj
self.tmpLock = threading.RLock()
self.finExecutor = None
self.fin_flag = False
self.initial_flag = False
self.app_fin_flag = False
self.halt_flag = False
self.task_acquire = False
self.heartcond = threading.Condition()
self.heartbeat = HeartbeatThread(self.client, self, self.heartcond)
self.list_lock = threading.RLock()
self.worker_list = {}
self.worker_status = {}
self.cond_list = {}
self.status = WorkerStatus.NEW
self.loop_time = 0.1
#code for read_data
self.mult_file_path = []
self.res_data = None
self.read_flag = False
import threading
from Util import logger
base_thread_log = logger.getLogger("BaseThread")
class BaseThread(threading.Thread):
"""
Base Thread class for this framework
"""
def __init__(self, name=""):
name = '.'.join(['DistJET.BaseThread',name])
print("[BaseThread]: create new thread : %s" % name)
super(BaseThread, self).__init__(name=name)
self.setDaemon(1)
self.lock = threading.RLock()
self.__should_stop = False
base_thread_log.debug('BaseThread object created:%s', self.__class__.__name__)
def get_stop_flag(self):
return self.__should_stop
def stop(self):
self.lock.acquire()
if not self.__should_stop:
base_thread_log.debug('Stopping: %s', self.__class__.__name__)
self.__should_stop = True
self.lock.release()
import Queue
import WorkerRegistry
import Task
import IAppManager
from Util import logger
from Util.Config import Config
scheduler_log = logger.getLogger('AppMgr')
class IScheduler:
def __init__(self, master, appmgr, worker_registry=None):
self.master = master
self.appid = None
self.worker_registry = worker_registry
self.appmgr = appmgr
self.task_todo_queue = Queue.Queue() # task queue, store task obj
scheduler_log.info('[Scheduler] Load tasks created by AppMgr')
self.task_list = self.appmgr.get_app_task_list()
for tid, task in self.task_list.items():
if not isinstance(task, Task.ChainTask):
self.task_todo_queue.put(tid)
elif task.father_len() == 0:
self.task_todo_queue.put(tid)
scheduler_log.info('[Scheduler] Load %d tasks, add %d task to queue' %
(len(self.task_list), self.task_todo_queue.qsize()))
self.scheduled_task_list = {} # wid: tid_list
self.completed_queue = Queue.Queue()
import Queue
import time
import os
import traceback
import threading
from Util import logger
control_log = logger.getLogger('WatchDog_Log')
handler_log = logger.getLogger('Handler')
master_log = logger.getLogger('Master')
health_log = logger.getLogger('Health')
import IR_Buffer_Module as IM
import IScheduler
import WorkerRegistry
import python.Util.Package as Pack
WorkerRegistry.wRegistery_log = master_log
import MPI_Wrapper
MPI_Wrapper.MPI_log = master_log
from BaseThread import BaseThread
from IAppManager import SimpleAppManager
from Task import TaskStatus
from python.Process.Process import status
from python.Util.Config import Config, set_inipath
from python.Util.Recoder import BaseElement, BaseRecoder
from python.WorkerRegistry import WorkerStatus