def message_handler(self, message):
"""
消息处理逻辑，该函数调用具体的消息执行函数，并获取结果放入结果队列中
.. Note:: 由于操作异步执行，因此各子进程执行结果统一放入多进程安全的结果队列，
由主进程统一进程结果的后续处理
:param Message message: 消息对象
:return: 无返回
:rtype: None
"""
try:
log.Logger.setoid(message.operation_id)
log.i("process new message")
log.d("process new message,param:%s" % str(message.params))
ret = self.execute_message(message)
except Exception as e:
log.f("execute_message fail")
ret = "err:{}".format(e)
message = framework.OperationMessage(
"COMPLETE_MESSAGE", message.operation_id, ret)
self._result_queue.put(message)
log.Logger.clearoid()
return
def create_node(self,
path,
value="",
ephemeral=False,
sequence=False,
makepath=False):
"""
根据节点各属性创建节点
:param str path: 节点路径
:param str value: 待存数据
:param bool ephemeral: 是否是临时节点
:param bool sequence: 是否是顺序节点
:param bool makepath: 是否创建父节点
:return: 新创建的节点路径
:rtype: str
:raises: exception.EPNoNodeError 节点不存在
:raises: exception.EPIOError IO异常
"""
node_path = ZkPersistence._run_catch(lambda: (self._client.create(
path, value, None, ephemeral, sequence, makepath)))
log.d("create node success, path:{path}, value:{value}, ephemeral:"
"{ephemeral}, sequence:{sequence}, makepath:{makepath}".format(
path=node_path,
value=value,
ephemeral=ephemeral,
sequence=sequence,
makepath=makepath))
return node_path
def delete_node(self, path):
"""
删除node节点
:param str path: 数据存储路径
:return None
"""
ZkPersistence._run_catch(lambda: (self._client.delete(path=path, recursive=True)))
log.d("delete node success, path:{path}".format(path=path))
def on_persistence(self):
"""
数据持久化操作
:return: 无返回
:rtype: None
"""
self._context.save_context()
log.d("context persistent success")
def disconnect(self):
"""
主动断开持久化请求
:return: 无返回
:rtype: None
"""
ZkPersistence._run_catch(lambda: (self._client.close()))
log.d("disconnect success")
def update_lock(self, is_lock):
"""
更新锁标识
:param bool is_lock: 是否获得锁
:return: 无返回
:rtype: None
"""
self.lock = is_lock
log.d("update context lock, {}".format(self.lock))
def delete_node(self, path):
"""
删除node节点
:param str path: 数据存储路径
:return None
"""
path = self._base + path
import shutil
FilePersistence._run_catch(lambda: (shutil.rmtree(path, True)), path)
log.d("delete node success, path:{path}".format(path=path))
def update_extend(self, params):
"""
更新扩展，增量更新
:param dict params: 更新的参数
:return: 无返回
:rtype: None
"""
self.extend.update(params)
self.save_context()
log.d("update extend success, current extend:{}".format(self.extend))
def del_extend(self, key):
"""
从扩展中删除指定的key
:param str key: 需要删除的字段
:return: 无返回
:rtype: None
"""
del self.extend[key]
self.save_context()
log.d("delete extend success, current extend:{}".format(self.extend))
def disconnect(self):
"""
主动断开持久化请求
:return: 无返回
:rtype: None
"""
self._init = False
with self._lock:
self._ob_paths = {}
self._touch_paths = {}
log.d("disconnect success")
def save_data(self, path, data):
"""
存储数据data到特定的path路径节点
:param str path: 数据存储路径
:param str data: 待存储的数据
:return: None
:raises: exception.EPNoNodeError 节点不存在
:raises: exception.EPIOError 连接异常
"""
ZkPersistence._run_catch(lambda: (self._client.set(path, data)))
log.d("save data success, path:{path}, data:{data}".format(path=path, data=data))
def update_operation(self, operation_id, operation):
"""
更新一个操作
:param str operation_id: 操作id
:param Operation operation: 操作对象
:return: 无返回
:rtype: None
"""
self.operations[operation_id] = operation
self.save_operation(operation)
log.d("update operation success, operation_id:{}".format(operation_id))
def create_operation(self, operation_id, operation):
"""
新增一个操作，一般在感知完成时调用此方法。一个外部事件的整个处理流程，称为一个操作
:param str operation_id: 操作id，作为操作的唯一标识
:param Operation operation: 操作对象
:return: 无返回
:rtype: None
"""
self.operations[operation_id] = operation
self.save_operation(operation)
log.d("create new operation success, operation_id:{}".
format(operation_id))
def delete_operation(self, operation_id):
"""
删除一个操作，一般在一个事件操作结束时调用
:param str operation_id: 操作id
:return: 无返回
:rtype: None
"""
del self.operations[operation_id]
operation_path = config.GuardianConfig.get_persistent_path("operations") + "/" + operation_id
persistence.PersistenceDriver().delete_node(operation_path)
log.d("delete operation from context success, operation_id:{}".
format(operation_id))
def delete_node(self, path, force=False):
"""
删除node节点
:param str path: 数据存储路径
:param bool force: 是否强行删除而不判断节点有效性
:return: 无返回
:rtype: None
:raises: exception.EPNoNodeError 节点不存在
:raises: exception.EPIOError IO异常
"""
self._del_record_when_delnode(path)
self._del_node(path, force)
log.d("delete node success, path:{path}".format(path=path))
def delete_node(self, path, force=False):
"""
删除node节点
:param str path: 数据存储路径
:param bool force: 是否强行删除而不判断节点有效性
:return None
"""
if force:
self._client.delete(path=path, recursive=True)
else:
self._run_catch(lambda:
(self._client.delete(path=path, recursive=True)))
log.d("delete node success, path:{path}".format(path=path))
def create_node(self,
path,
value="",
ephemeral=False,
sequence=False,
makepath=False):
"""
根据节点各属性创建节点
:param str path: 节点路径
:param str value: 待存数据
:param bool ephemeral: 是否是临时节点
:param bool sequence: 是否是顺序节点
:param bool makepath: 是否创建父节点
:return: None
:raises: exception.EPNoNodeError 节点不存在
:raises: exception.EPIOError IO异常
"""
def _createnode(np=path):
node_path, node_name = self._split_node_name(np)
errmsg = {
-1: "Node has ttl or parents-node not exists",
-2: "Node exists(in parents-node record)",
-3: "Node exists"
}
if not self.exists(node_path) and node_path != "/":
if makepath:
self.create_node(node_path, makepath=True)
else:
raise exception.EPNoNodeError(node_path + " not exists")
seq = 1 if sequence else 0
tm = long(time.time()) + self._timeout if ephemeral else 0
ret = self._handle.evalsha(self._new_lua_sha, 2, node_path,
node_name, value, seq, tm)
if ret < 0:
raise exception.EPIOError("redis error when create[%s:%s]:%s" %
(node_path, node_name, errmsg[ret]))
if ephemeral:
self._new_touch(ret)
return ret
ret = self._run_catch(_createnode)
log.d("create node success, path:{path}, value:{value}, ephemeral:"
"{ephemeral}, sequence:{sequence}, makepath:{makepath}".format(
path=ret,
value=value,
ephemeral=ephemeral,
sequence=sequence,
makepath=makepath))
return ret
def init_environment(cls):
"""
初始化Guardian运行环境
:return: 无返回
:rtype: None
:raises EFailedRequest: 状态服务请求异常
"""
guardian_base = config.GuardianConfig.get_persistent_path()
guardian_client_path = config.GuardianConfig.get_persistent_path("alive_clients")
context_path = config.GuardianConfig.get_persistent_path("context")
operations_path = config.GuardianConfig.get_persistent_path("operations")
pd = persistence.PersistenceDriver()
if not pd.exists(guardian_base):
pd.create_node(path=guardian_base, makepath=True)
log.d("persistent node %s created!" % guardian_base)
if not pd.exists(context_path):
pd.create_node(path=context_path)
log.d("persistent node %s created!" % context_path)
if not pd.exists(guardian_client_path):
pd.create_node(path=guardian_client_path)
log.d("persistent node %s created!" % guardian_client_path)
if not pd.exists(operations_path):
pd.create_node(path=operations_path)
log.d("persistent node %s created!" % operations_path)
def create_node(self, path, value="", ephemeral=False, sequence=False, makepath=False):
"""
根据节点各属性创建节点
:param str path: 节点路径
:param str value: 待存数据
:param bool ephemeral: 是否是临时节点
:param bool sequence: 是否是顺序节点
:param bool makepath: 是否创建父节点
:return: None
:raises: exception.EPNoNodeError 节点不存在
:raises: exception.EPIOError IO异常
"""
path = self._base + path
try:
if ephemeral:
# 临时节点，直接用文件来表示
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
if makepath:
os.makedirs(dirname, self._mode)
else:
raise exception.EPNoNodeError
file_path = path
if sequence:
file_path = FilePersistence._seq_file_name(path)
with open(file_path, 'w') as f:
f.write(value)
with self._lock:
self._touch_paths[file_path] = ""
else:
# 实体节点，用目录来表示，数据存放在目录下的.data文件中
if not os.path.exists(path):
if not os.path.exists(os.path.dirname(path)) and not makepath:
raise exception.EPNoNodeError
os.makedirs(path, self._mode)
file_path = "/".join((path, ".data"))
with open(file_path, 'w') as f:
f.write(value)
except exception.EPNoNodeError as e:
log.r(e, "Node not exist:{}".format(os.path.dirname(path)))
except:
log.r(exception.EPIOError(), "Request I/O Error")
log.d("create node success, path:{path}, value:{value}, ephemeral:"
"{ephemeral}, sequence:{sequence}, makepath:{makepath}".format(
path=path, value=value,
ephemeral=ephemeral, sequence=sequence,
makepath=makepath))
def save_operation(self, operation):
"""
持久化状态机信息
:param operation:
:return:
"""
if not self.lock:
log.e("current guardian instance no privilege to save operation")
raise exception.EInvalidOperation(
"current guardian instance no privilege to save operation")
operation_path = config.GuardianConfig.get_persistent_path("operations") + "/" + operation.operation_id
if not persistence.PersistenceDriver().exists(operation_path):
persistence.PersistenceDriver().create_node(path=operation_path)
persistence.PersistenceDriver().save_data(operation_path, pickle.dumps(operation))
log.d("save operation_id:{} success".format(operation.operation_id))
def add_listener(self, watcher):
"""
监听会话状态
:param watcher: 状态监听函数。函数形参为(state)，可能的取值包括"SUSPENDED"、"CONNECTED"、"LOST"
:return: 无返回
:rtype: None
:raises: exception.EPIOError IO异常
"""
# 装饰watcher，将state、type转换为ARK内定义
def dec(zkstate):
state = zkstate
return watcher(state)
ZkPersistence._run_catch(lambda: (self._client.add_listener(watcher and dec)))
log.d("add listener success")
def event_watcher(self, event):
"""
监听事件
:param PersistenceEvent event: 节点状态事件
:return: None
"""
if event.state == persistence.PersistenceEvent.PersistState.CONNECTED \
or event.type == persistence.PersistenceEvent.EventType.CREATED \
or event.type == persistence.PersistenceEvent.EventType.DELETED \
or event.type == persistence.PersistenceEvent.EventType.CHANGED \
or event.type == persistence.PersistenceEvent.EventType.CHILD:
log.d("event change, state:{}".format(event.state))
self.choose_master()
else:
log.d("event unrecognized")
def send(self, message):
"""
发送一个消息到消息泵
.. Note:: 此方法必须在主进程中调用，严禁在子进程中调用send方法，否则可能导致执行记录被覆盖、发送的消息未处理等严重问题。
.. Note:: 该函数涉及到的与消息持久化相关的操作在装饰器中实现，为避免非预期的问题，尽量避免对此方法进行重写操作，如需重写，
需明确可能的行为，并显式添加装饰器
:param Message message: 消息对象
:return: 无返回
:rtype: None
"""
if multiprocessing.current_process().name != "MainProcess":
raise exception.ENotImplement(
"send() only be used in \'MainProcess\'")
self._message_pump.put(message)
log.d("send message to message pump success, message:{}".format(
message.name))
def save_data(self, path, data):
"""
存储数据data到特定的path路径节点
:param str path: 数据存储路径
:param str data: 待存储的数据
:return: 无返回
:rtype: None
:raises: exception.EPNoNodeError 节点不存在
:raises: exception.EPIOError IO异常
"""
handle = self._handle
def _writedata():
# 由于redis不支持事务，所以此处并不会区分节点是否存在，均直接set数据
handle.hset(path, ".data", data)
self._run_catch(_writedata)
log.d("save data success, path:{path}".format(path=path))
def on_decision_message(self, message):
"""
决策处理逻辑
:param Message message: 消息对象
:return: 无返回
:rtype: None
:raises EUnknownEvent: 位置事件异常
"""
log.d("on decision message:{}".format(message.name))
if message.name == "SENSED_MESSAGE":
decided_message = self.decision_logic(message)
self.send(decided_message)
elif message.name == "COMPLETE_MESSAGE":
pass
elif message.name in self._concerned_message_list:
self.on_extend_message(message)
else:
raise exception.EUnknownEvent(
"message type [{}] is not concerned".format(message.name))
def run_next(self):
"""
进行一次状态轮转
.. Note:: 工作流模型中，每个状态处理完成后，下一次需要轮转的状态是不确定的（或者只提供下一个建议执行的状态），因此使用工作流模型，需要自己定义各个状态的 ``check``方法；
状态处理完成后启动对各状态的检查，检查通过的状态，进入处理阶段。
.. Note:: 在某个状态完成后，会从其返回的建议的下一个运行状态开始遍历（如未返回建议状态，则从状态列表中此状态的下一个开始），以提高命中效率
:return: 无返回
:rtype: None
"""
node = self.get_node(self._current_node)
index = self._nodes.index(node)
index_list = range(index, len(self._nodes))
index_list.extend(range(0, index))
for i in index_list:
node = self._nodes[i]
if not node.reentrance and self._nodes_process[node.name]:
continue
else:
ret = node.check(self._session, self._current_node,
self._nodes_process)
log.d("node {} check ret:{}".format(self._current_node, ret))
if ret:
self._nodes_process[node.name] = True
current_node = node.process(
self._session, self._current_node, self._nodes_process)
log.d("node process finished, suggest next "
"node:{}".format(current_node))
if current_node == self._ARK_NODE_END:
self._status = self.Status.FINISHED
elif current_node not in self._nodes_process:
self._current_node = self._nodes[
(i + 1) % len(self._nodes)].name
else:
self._current_node = current_node
return
else:
continue
def save_data(self, path, data):
"""
存储数据data到特定的path路径节点
:param str path: 数据存储路径
:param str data: 待存储的数据
:return: 无返回
:rtype: None
:raises: exception.EPNoNodeError 节点不存在
:raises: exception.EPIOError IO异常
"""
path = self._base + path
def _writedata():
if os.path.isdir(path):
file_path = "/".join((path, ".data"))
else:
file_path = path
with open(file_path, 'w') as f:
return f.write(data)
FilePersistence._run_catch(_writedata, path)
log.d("save data success, path:{path}".format(path=path))
def _reload(self):
"""
重新加载定时列表。加载时会对新的定时列表与上次获取的做比对。仅处理新增或者删除的定时器
"""
while not self._stop_tag:
try:
current_cron = set(self.refresh())
delete_list = self._old_cron - current_cron
add_list = current_cron - self._old_cron
if len(delete_list) != 0:
self._clock.delete_cron(delete_list)
log.d("refresh cron list, delete:{num}".format(
num=len(delete_list)))
if len(add_list) != 0:
self._clock.add_cron(add_list)
log.d("refresh cron list, add:{num}".format(
num=len(add_list)))
self._old_cron = current_cron
except Exception as e:
log.f("reload failed, err")
time.sleep(self._reload_interval)
def run_next(self):
"""
进行一次状态轮转
.. Note:: 状态机模型中，每个状态处理完成后需要返回一个确定的状态，可直接进行处理；若返回的状态不存在，直接抛出异常
:return: 无返回
:rtype: None
:raise ECheckFailed: 检查失败
:raise EUnknownNode: 未知节点
"""
state = self.get_node(self._current_node)
if not state.reentrance and self._nodes_process[state.name]:
raise exception.ECheckFailed(
"node:{} is finished and not reentrance".format(state.name))
ret = state.check(self._session, self._current_node,
self._nodes_process)
log.d("node {} check ret:{}".format(self._current_node, ret))
if ret:
self._nodes_process[state.name] = True
current_state = state.process(self._session, self._current_node,
self._nodes_process)
log.d("node process finished, next node:{}".format(
current_state))
if current_state == self._ARK_NODE_END:
self._current_node = current_state
self._status = self.Status.FINISHED
return
elif current_state not in self._nodes_process:
raise exception.EUnknownNode(
"return state[{}] unkown".format(current_state))
else:
self._current_node = current_state
else:
raise exception.ECheckFailed(
"node:{} check failed".format(state.name))
def save_context(self):
"""
运行数据持久化，当当前Guardian为主（lock属性为True）时，可持久化数据，否则失败
:return: 无返回
:rtype: None
:raises EInvalidOperation: 非法操作
"""
if not self.lock:
log.e("current guardian instance no privilege to save context")
raise exception.EInvalidOperation(
"current guardian instance no privilege to save context")
context_path = config.GuardianConfig.get_persistent_path("context")
context_to_persist = self
operations_tmp = self.operations
context_to_persist.operations = {}
try:
persistence.PersistenceDriver().save_data(context_path, pickle.dumps(context_to_persist))
except Exception as e:
self.operations = operations_tmp
log.r(e, "save context fail")
self.operations = operations_tmp
log.d("save context success")
