def process(data):
try:
data = TmProtocol.deserial(data)
logging.info('External Receive: {0}'.format(
json.dumps(data.to_dict(), ensure_ascii=False)))
except DeserialError, e:
logging.warning('Task Deserialize Error: {0}'.format(e))
def run_task(task, queue_uuid, task_uuid, conf, exe):
# 开始正式执行
ret_code, ret_msg = get_time.compare_timestamps(
task.trigger_time, task.task_earliest, task.task_latest
)
if ret_code == 2:
logging.info('Start Sleeping {0} Seconds'.format(ret_msg))
time.sleep(ret_msg)
status_code = TaskStatus.Running
status_msg = MSG_DICT.get(status_code)
result = TaskResult(queue_uuid=queue_uuid, task_uuid=task_uuid, status_code=status_code,
status_msg=status_msg, session=task.session)
send(result)
mod = task.task_info["mod"]
if isinstance(mod, dict):
# 一个任务
task_result = exe.run(mod)
dump_result(result.task_uuid, task_result)
if len(task_result.lines) > 30:
task_result.data = None
task_result.lines = task_result.lines[-30:]
ret_code = task_result.return_code
if ret_code != 0:
status_code = TaskStatus.Failed
status_msg = u"单任务执行失败"
else:
status_code = TaskStatus.Success
status_msg = u"单任务执行成功"
result = TaskResult(queue_uuid=queue_uuid, task_uuid=task_uuid, status_code=status_code,
status_msg=status_msg, session=task.session, task_result=task_result)
send(result)
return 0
elif isinstance(mod, list):
# 多个任务
for each in mod:
task_result = exe.run(each)
dump_result(result.task_uuid, task_result)
if len(task_result.lines) > 30:
task_result.data = None
task_result.lines = task_result.lines[-30:]
ret_code = task_result.return_code
if ret_code != 0:
status_code = TaskStatus.Failed
status_msg = u"多任务执行失败"
break
else:
status_code = TaskStatus.Success
status_msg = u"多任务执行成功"
result = TaskResult(queue_uuid=queue_uuid, task_uuid=task_uuid, status_code=status_code,
status_msg=status_msg, session=task.session, task_result=task_result)
send(result)
return 0
else:
# 异常情况 直接返回 - 1
status_code = TaskStatus.UnKnown
status_msg = MSG_DICT.get(status_code)
result = TaskResult(queue_uuid=queue_uuid, task_uuid=task_uuid, status_code=status_code,
status_msg=status_msg, session=task.session)
send(result)
return -1
def run(self):
from TaskManager.Worker.worker import msg_queue
logging.info('msg_loop')
while 1:
event = msg_queue.get()
logging.info(u'MsgLoop Get EventName: {0}, EventData: {1}'.format(
event.event_name,
json.dumps(event.event_data.to_dict(), ensure_ascii=False)))
event_name = event.event_name
callback_list = self.callback_dict.get(event_name)
for each in callback_list:
each(event)
def reconnect_socket(self):
try:
self.socket_client.connect((self.master_host, self.master_port))
logging.info('External Connect To Host: {0}, Port: {1}'.format(
self.master_host, self.master_port))
except socket.error:
logging.warning('External Connect Fail')
return -1
else:
self.hello()
self.health()
return 0
def init_status(task, queue_uuid, task_uuid):
if not worker_pool.vacant():
# 进程池满
logging.warning('TaskUUID: {0}, TaskStatus: {1}'.format(task_uuid, TaskStatus.WorkerWaiting.value))
status_code = TaskStatus.WorkerWaiting
status_msg = MSG_DICT.get(status_code)
result = TaskResult(queue_uuid=queue_uuid, task_uuid=task_uuid, status_code=status_code,
status_msg=status_msg, session=task.session)
send(result)
return -1
else:
ret_code, ret_msg = get_time \
.compare_timestamps(task.trigger_time, task.task_earliest, task.task_latest)
if ret_code == 3:
# 无法执行 直接返回-1
logging.warning('TaskUUID: {0}, TaskStatus: {1}'.format(task_uuid, TaskStatus.TimeRangeExcept.value))
status_code = TaskStatus.TimeRangeExcept
status_msg = MSG_DICT.get(status_code)
result = TaskResult(queue_uuid=queue_uuid, task_uuid=task_uuid, status_code=status_code,
status_msg=status_msg, session=task.session)
send(result)
return -1
elif ret_code == 2:
# 需要等待
logging.info('TaskUUID: {0}, TaskStatus: {1}'.format(task_uuid, TaskStatus.WorkerWaiting.value))
status_code = TaskStatus.TriggerTimeWaiting
status_msg = u'需要等待{0}秒'.format(ret_msg)
result = TaskResult(queue_uuid=queue_uuid, task_uuid=task_uuid, status_code=status_code,
status_msg=status_msg, session=task.session)
send(result)
elif ret_code == 1:
# 该状态不通知Master
# 可以执行
# logging.info('TaskUUID: {0}, TaskStatus: {1}'.format(task_uuid, TaskStatus.Runnable.value))
# status_code = TaskStatus.Runnable
# status_msg = MSG_DICT.get(status_code)
# result = TaskResult(queue_uuid=queue_uuid, task_uuid=task_uuid, status_code=status_code,
# status_msg=status_msg, session=task.session)
# send(result)
pass
else:
# 异常情况 直接返回-1
logging.warning('TaskUUID: {0}, TaskStatus: {1}'.format(task_uuid, TaskStatus.UnKnown.value))
status_code = TaskStatus.UnKnown
status_msg = MSG_DICT.get(status_code)
result = TaskResult(queue_uuid=queue_uuid, task_uuid=task_uuid, status_code=status_code,
status_msg=status_msg, session=task.session)
send(result)
return -1
def reconnect_socket():
"""
socket连接重新连接
:return:
"""
try:
global socket_client, connected_host, connected_port
socket_client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket_client.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
socket_client.connect((connected_host, connected_port))
logging.info('Internal Connect To Host: {0}, Port: {1}'.format(connected_host, connected_port))
except socket.error:
return -1
else:
return 0
def handle(self):
while True:
data = self.request.recv(8192)
if data:
logging.info('Server Receive Len: {0}'.format(len(data)))
ret = self.process(data)
if ret != -1:
ack = '{"ack": "Pass"}'
else:
ack = '{"ack": "Fail"}'
self.request.sendall(ack)
logging.info('Server Send: {0}'.format(ack))
else:
logging.warning('Server Disconnect: {0}'.format(
self.client_address))
break
def init_socket(port):
"""
对于工作进程初始化socket连接
:return:
"""
global socket_client, connected_host, connected_port
connected_host, connected_port = '127.0.0.1', port
socket_client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket_client.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
socket_client = ssl.wrap_socket(
socket_client,
ca_certs=os.path.join(os.path.dirname(__file__), os.pardir, 'SSLCerts', 'ca.crt'),
certfile=os.path.join(os.path.dirname(__file__), os.pardir, 'SSLCerts', 'client.crt'),
keyfile=os.path.join(os.path.dirname(__file__), os.pardir, 'SSLCerts', 'client.key'),
cert_reqs=ssl.CERT_REQUIRED, ssl_version=ssl.PROTOCOL_TLSv1_2)
socket_client.connect((connected_host, connected_port))
logging.info('Client Connect To Host: {0}, Port: {1}'.format(connected_host, connected_port))
def process(data):
try:
task_result = TaskResult.deserial(data)
logging.info(u'Server Receive : {0}'.format(
json.dumps(task_result.to_dict(), ensure_ascii=False)))
task_status = task_result.status_code
if task_status.IsExcepted:
msg_queue.put_event('except', task_result)
if task_status.IsInited:
msg_queue.put_event('init', task_result)
if task_status.IsRunning:
msg_queue.put_event('start', task_result)
if task_status.IsDone:
msg_queue.put_event('end', task_result)
return 0
except DeserialError:
logging.error('TaskResult Deserialize Error')
return -1
def send(self, event):
data = event.event_data
tm_data = TmProtocol('worker', 'master', data)
retry_count = 0
while 1:
try:
self.socket_client.sendall(tm_data.serial())
logging.info(u'Server Send {0}'.format(
json.dumps(tm_data.to_dict(), ensure_ascii=False)))
# ack = self.socket_client.recv(8192)
# todo: 判断ack逻辑
except socket.timeout:
retry_count = retry_count + 1
logging.info('External Timeout, Retry {0}'.format(retry_count))
except socket.error:
retry_count = retry_count + 1
del self.socket_client
self.socket_client = self.init_ssl()
self.reconnect_socket()
else:
break
def send(data):
"""
发送消息
:param data: 未序列化的数据
:return:
"""
if isinstance(data, TaskResult):
dump_data = data.serial()
# 发送数据
while 1:
global socket_client
try:
logging.info('Client Send Len: {0}'.format(len(dump_data)))
socket_client.sendall(dump_data)
logging.info(u'Client Send: {0}'.format(json.dumps(data.to_dict(), ensure_ascii=False)))
ack = socket_client.recv(8192)
logging.info('Client Receive: {0}'.format(ack))
ack = json.loads(ack)
if ack.get('ack') == 'Fail':
raise Exception('TaskResult Deserialize Error')
except socket.error, e:
# 因server端socket异常未正常发送
retry_count = 0
while 1:
retry_count = retry_count + 1
logging.warning('Internal Disconnect, Retry {0}'.format(retry_count))
ret = reconnect_socket()
if ret == 0:
break
else:
time.sleep(5)
logging.warning('Client Send Error, Retry {0}'.format(retry_count))
else:
break
def put_event(self, event_name, event_data):
self.put(Event(event_name, event_data))
logging.info(u'MsgQueue Put EventName: {0}, EventData: {1}'.format(
event_name, json.dumps(event_data.to_dict(), ensure_ascii=False)))
def minus_running_process(self, result):
self.running_process -= 1
logging.info('Running Process Minus 1, Now {0}'.format(self.running_process))
def add_running_process(self):
self.running_process += 1
logging.info('Running Process Add 1, Now {0}'.format(self.running_process))