def get_worker(self, name):
try:
return self._worker_cache[name]
except KeyError:
msg = 'Worker[{}] already disconnected'.format(name)
logging.warning(msg)
raise Exception(msg)
def del_worker(self, name):
try:
request = self._worker_cache.pop(name)
except KeyError:
msg = 'Worker[{}] already disconnected'.format(name)
logging.warning(msg)
else:
request.close()
self._condition.notifyAll()
def unregister_callback(self, event_name, func):
if event_name not in self.callback_dict:
logging.warning(u'未找到该事件名称')
return -1
else:
callback_list = self.callback_dict.get(event_name)
if func not in callback_list:
logging.warning(u'未注册该回调函数')
return -1
else:
callback_list.remove(func)
if not callback_list:
self.callback_dict.pop(event_name)
def send_task(self, name, task):
while True:
try:
worker = self.get_worker(name)
worker.sendall(
TmProtocol(src='MASTER',
dest=name,
payload=task,
msg_type=MessageType.Private).serial())
except Exception as err:
logging.warning(err)
self.wait_for_worker()
name = self.worker_arb()
else:
break
def event_relay(self, event):
"""
任务分发器，由外部callback触发
"""
if event.Name == EventName.TaskDispatch:
worker_name = self.worker_arb()
task = event.Data
self.send_task(worker_name, task)
logging.info('Task[{}] assigned to worker[{}]'.format(
event.Data.task_uuid, worker_name))
self.send_result(
TaskResult(task.queue_uuid, task.task_uuid,
TaskStatus.Dispatched,
MSG_DICT[TaskStatus.Dispatched], task.session))
else:
logging.warning('Invalid event[{}] routed'.format(event.Name))
def _deserial(self, cache):
current_time = time.time()
directory = os.path.join(os.path.dirname(__file__), 'dump')
for each_file in os.listdir(directory):
if each_file.endswith('.yaml'):
try:
dict_data = yaml.load(
open(os.path.join(directory, each_file), mode='r'))
except Exception as err:
logging.warning(
'Invalid yaml dump file[{}]'.format(each_file))
else:
destroy_time = time.mktime(
time.strptime(dict_data['destroy_time'],
'%Y-%m-%d %H:%M:%S'))
if current_time < destroy_time:
TaskQueue.from_dict(cache, dict_data)
def make_todo_task_queue(self, force=False):
"""
初始化待做任务队列
"""
if self.queue_status not in (QueueStatus.Normal,
QueueStatus.Initiating) and not force:
logging.warning('Invalid queue status.')
return False
self.todo_task_queue = Queue()
for task in self.task_list[len(self.task_result_list):]:
self.todo_task_queue.put(task)
if not force and self.todo_task_queue.empty():
logging.warning('Queue[{}] todo queue empty in status[{}]'.format(
self.queue_uuid, self.queue_status.name))
return False
elif not force:
self.queue_status = QueueStatus.Normal
return True
def _notify_outside(result):
result_dict = {
'controller_queue_uuid':
result.queue_uuid,
'task_uuid':
result.task_uuid,
'task_status': (result.status_code.value, result.status_msg),
'session':
result.session,
'task_result':
result.task_result.to_dict() if result.task_result else None
}
try:
url = "http://{ip}:{port}/api/operation/uuid/{id}/callback".format(
ip=FLASK_HOST, port=FLASK_PORT, id=result.task_uuid)
requests.post(url, json=result_dict)
except Exception as err:
logging.warning('Notify outside failed.')
def handle(self):
while True:
try:
buff = self.request.recv(8192)
except socket.error:
logging.warning('Client disconnect: {0}'.format(
self.client_address))
break
if buff:
try:
data = TmProtocol.deserial(buff)
except (DeserialError, ValueError) as err:
logging.warning('Protocol format error: {0}'.format(
err.message))
else:
self._process(data)
else:
break
def event_relay(self, event):
logging.info('Event received: {} {}'.format(event.Name,
event.Data.to_dict()))
if event.Name == EventName.TaskResult:
result = event.Data
if self.queue_exist(result.queue_uuid):
queue = self.get_queue(result.queue_uuid)
logging.info('Result for task[{}] dispatched.'.format(
result.task_uuid))
if queue.update_status_by_result(result):
TaskQueueManager._notify_outside(result)
if queue.Status == QueueStatus.Normal and queue.IsRunAll:
task = queue.get()
task.session = result.session
self.send_event(EventName.TaskDispatch, task)
else:
logging.warning('Queue[{}] not exist.'.format(result.queue_id))
else:
logging.warning('Can not handle event[{}]'.format(event.Name))
def wrapper(self, *args, **kwargs):
if isinstance(self, TaskQueue):
queue_uuid = args[1]
file_name = os.path.join(os.path.dirname(__file__),
'dump/{}.yaml'.format(queue_uuid))
if os.path.exists(file_name):
try:
dict_data = yaml.load(open(file_name, mode='r'))
except Exception as err:
logging.warning(
'Invalid yaml dump file[{}]'.format(file_name))
else:
current_time = time.time()
destroy_time = time.mktime(
time.strptime(dict_data['destroy_time']))
if current_time < destroy_time:
return TaskQueue.from_dict(dict_data)
return func(self, *args, **kwargs)
else:
return func(self, *args, **kwargs)
def skip_task(self, session=None):
"""
跳过下一项任务
"""
if self.todo_task_queue.empty():
return False
next_task = self.get()
if isinstance(next_task, Task):
result = TaskResult(next_task.queue_uuid, next_task.task_uuid,
TaskStatus.Skipped,
MSG_DICT[TaskStatus.Skipped], session)
self.task_result_list.append(result)
# TaskQueueManager._notify_outside(result)
if self.todo_task_queue.empty():
self.queue_status = QueueStatus.Done
else:
self.queue_status = QueueStatus.Normal
return True
else:
logging.warning(next_task)
raise Exception(u'任务定义不正确')
def update_status_by_result(self, task_result):
"""
根据任务结果更新队列状态
"""
current_task = self.task_list[len(self.task_list) -
self.todo_task_queue.qsize() - 1]
if task_result.task_uuid != current_task.task_uuid:
logging.warning(u'任务结果与当前执行任务不匹配！')
logging.warning(u'当前任务：{}'.format(
json.dumps(current_task.to_dict())))
logging.warning(u'任务结果：{}'.format(json.dumps(
task_result.to_dict())))
return False
switch = {
TaskStatus.UnKnown:
lambda: QueueStatus.NotRecoverable,
TaskStatus.Dispatched:
lambda: QueueStatus.JobDispatched,
TaskStatus.InitFailed:
lambda: QueueStatus.NotRecoverable,
TaskStatus.Runnable:
lambda: QueueStatus.JobRunning,
TaskStatus.TriggerTimeWaiting:
lambda: QueueStatus.JobWaiting,
TaskStatus.WorkerWaiting:
lambda: QueueStatus.JobWaiting,
TaskStatus.TimeRangeExcept:
lambda: QueueStatus.JobFailed,
TaskStatus.Running:
lambda: QueueStatus.JobRunning
if self.sync_group else QueueStatus.Normal,
TaskStatus.Success:
lambda: QueueStatus.Done
if self.todo_task_queue.empty() else QueueStatus.Normal,
TaskStatus.Failed:
lambda: QueueStatus.JobFailed,
TaskStatus.Timeout:
lambda: QueueStatus.JobFailed,
TaskStatus.Terminated:
lambda: QueueStatus.JobFailed,
TaskStatus.Skipped:
lambda: QueueStatus.Done
if self.todo_task_queue.empty() else QueueStatus.Normal
}
try:
self.queue_status = switch[task_result.status_code]()
except KeyError:
logging.warning('Invalid status[{}] in result: {}'.format(
task_result.task_status, task_result.to_dict()))
return False
else:
if self.queue_status == QueueStatus.JobFailed:
self.run_all = False
if self.task_result_list:
last_result = self.task_result_list[-1]
if last_result.task_uuid == task_result.task_uuid:
self.task_result_list.pop()
self.task_result_list.append(task_result)
logging.info('Queue status[{}] updated by result: {}'.format(
self.queue_status, task_result.to_dict()))
if not self.queue_status.Blocking:
self._condition.notifyAll()
return True