def __init__(self):
# init ROS node
rospy.init_node("holo_tf_receiver", anonymous=True)
self.server_ = TcpServer(host="192.168.1.152", port=12345)
self.pub_ = rospy.Publisher('holo_transform',
HoloTransform,
queue_size=10)
# call self.server_.stop before node destroied
rospy.on_shutdown(self.server_.stop)
def test_server(self):
af_inet = AF_INET
sock_stream = SOCK_STREAM
sol_socket = SOL_SOCKET
so_reuseaddr = SO_REUSEADDR
with mock.patch("tcp_server.socket") as mock_socket, \
mock.patch("tcp_server.ClientThread", spec=True) as mock_ClientThread:
host = 'localhost'
port = 5555
tcp_server = TcpServer(host=host, port=port)
self.assertEqual(tcp_server.host, host)
self.assertEqual(tcp_server.port, port)
self.assertIsNone(tcp_server._socket)
self.assertFalse(tcp_server._runnning)
def accept_stub():
ca = [("conn1", "addr1"), ("conn2", "addr2"), ("conn3", "addr3")]
for i in range(len(ca)):
if i == 2:
tcp_server.stop()
yield ca[i]
s_socket = mock.Mock(name="s_sock")
s_socket.accept.side_effect = accept_stub()
mock_socket.socket.return_value = s_socket
mock_socket.AF_INET = af_inet
mock_socket.SOCK_STREAM = sock_stream
mock_socket.SOL_SOCKET = sol_socket
mock_socket.SO_REUSEADDR = so_reuseaddr
tcp_server.run()
mock_socket.socket.assert_called_once_with(af_inet, sock_stream)
tcp_server._socket.setsockopt.assert_called_once_with(sol_socket, so_reuseaddr, 1)
tcp_server._socket.bind.assert_called_once_with((host, port))
tcp_server._socket.listen.assert_called_once_with(5)
self.assertFalse(tcp_server._runnning)
self.assertEqual(tcp_server._socket.accept.call_count, 3)
self.assertEqual(mock_ClientThread.call_count, 3)
inst_client_thread = mock_ClientThread("stub", "stub")
self.assertEqual(inst_client_thread.start.call_count, 3)
tcp_server._socket.close.assert_called_once()
def __init__(self, mode, parent=None):
QObject.__init__(self, parent)
# ---
self.server = TcpServer(mode, self)
self.server.logMessage.connect(self.emit_log)
self.server.readByteArray.connect(self.on_read_bytearray)
self.pending_data = []
self.stored_data = []
# ---
self.client = TcpClient(self)
self.client.logMessage.connect(self.emit_log)
self.client.readByteArray.connect(self.on_read_bytearray_answer)
self.pending_data_answer = []
self.stored_data_answer = []
# ---
self.iserv = u'КАУС->ПРМК'
self.iserv_2 = u'ПРМК->ПРДК'
self.oserv = u'ПРМК->КАУС'
self.oserv_2 = u'ПРДК->ПРМК'
self.iclient = u'ПРДК->ПРМК'
self.oclient = u'ПРМК->ПРДК'
# ---
self.dest_sign = u'ПРМК->КАУС'
self.dest_sign_2 = u'КАУС->ПРМК'
if mode == ImitParams.transmitter_mode:
self.dest_sign = u'ПРДК->ПРМК'
self.dest_sign_2 = u'ПРМК->ПРДК'
# ---
self.tflag = True if mode == ImitParams.transmitter_mode else False
def __init__(self):
# init ROS node
rospy.init_node("holo_sensor_receiver", anonymous=True)
self.server_ = TcpServer(
# get from private domain ~
host=rospy.get_param("~host", default="192.168.1.152"),
port=rospy.get_param("~port", default=8800))
self.pub_ = rospy.Publisher("image_topic", Image, queue_size=5)
self.bridge_ = CvBridge()
# self.pub_ = rospy.Publisher('holo_transform', HoloTransform, queue_size=10)
# call self.server_.stop before node destroied
rospy.on_shutdown(self.server_.stop)
class GameServer:
def __init__(self, ip, port, reactor_num=5):
self.__compute_thread_pool = compute_thread_pool
self.__server = TcpServer(ip, port, reactor_num)
def start(self):
TcpConnection.on_connection_callback = on_connection
TcpConnection.on_message_callback = on_message
TcpConnection.on_close_callback = on_close
self.__compute_thread_pool.start()
self.__server.start()
def close(self):
self.__server.close()
logger.simple_log('正在关闭计算线程池')
self.__compute_thread_pool.stop()
pass
class HoloTransformReceiver(object):
def __init__(self):
# init ROS node
rospy.init_node("holo_tf_receiver", anonymous=True)
self.server_ = TcpServer(host="192.168.1.152", port=12345)
self.pub_ = rospy.Publisher('holo_transform',
HoloTransform,
queue_size=10)
# call self.server_.stop before node destroied
rospy.on_shutdown(self.server_.stop)
def launch(self):
self.server_.launch()
# publish data received from the HoloLens
# We use a thread to launch the publish() function since we
# want to have the ROS handling the system interupts.
# In case, publish() has been blocked.
thread = threading.Thread(target=self.publish_)
thread.start()
rospy.spin()
def publish_(self):
for package in self.server_.fetch_package():
pos, rot = self.unpack_(package)
msg = HoloTransform()
msg.position.x = pos[0]
msg.position.y = pos[1]
msg.position.z = pos[2]
msg.rotation.x = rot[0]
msg.rotation.y = rot[1]
msg.rotation.z = rot[2]
self.pub_.publish(msg)
def unpack_(self, package):
"""
Unpack TCP package
Args:
package (bytes string):
HoloLens position: from 0 ~ 11 bytes
0 - 3: camera position.x
4 - 7: camera position.y
8 - 11: camera position.z
HoloLens orientation (euler angle): from 12 - 27 bytes
12 - 15: euler.x
16 - 19: euler.y
20 - 23: euler.z
"""
position = [
struct.unpack('<f', package[i:i + 4])[0] for i in range(0, 12, 4)
]
euler = [
struct.unpack('<f', package[i:i + 4])[0] for i in range(12, 24, 4)
]
print("Position: {}".format(position))
return position, euler
sys.exit('File torrent not exists')
torrent_file_data = parse_torrent_file(args.torrent_file_path)
info_hash = hashlib.sha1(
json.dumps(torrent_file_data['info']).encode('utf-8')).hexdigest()
my_id = ''.join(
random.choice(string.ascii_letters + string.digits)
for i in range(PEER_ID_SIZE))
main_controller = MainController.start(my_id, info_hash, torrent_file_data,
args.seeder)
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.bind((socket.gethostname(), 0))
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.listen(5)
tracker_client = TrackerClient(my_id, info_hash,
server_socket.getsockname()[0],
server_socket.getsockname()[1],
torrent_file_data['tracker_ip'],
torrent_file_data['tracker_port'],
main_controller)
tracker_client.daemon = True
tracker_client.start()
tcp_server = TcpServer(info_hash, server_socket, main_controller)
tcp_server.daemon = True
tcp_server.start()
def __init__(self, selfNodeAddr, otherNodesAddrs, conf=None):
if conf is None:
self.__conf = SyncObjConf()
else:
self.__conf = conf
self.__conf.validate()
if self.__conf.password is not None:
if not HAS_CRYPTO:
raise ImportError("Please install 'cryptography' module")
self.__encryptor = getEncryptor(self.__conf.password)
else:
self.__encryptor = None
self.__selfNodeAddr = selfNodeAddr
self.__otherNodesAddrs = otherNodesAddrs
self.__unknownConnections = {} # descr => _Connection
self.__raftState = _RAFT_STATE.FOLLOWER
self.__raftCurrentTerm = 0
self.__votedFor = None
self.__votesCount = 0
self.__raftLeader = None
self.__raftElectionDeadline = time.time() + self.__generateRaftTimeout(
)
self.__raftLog = createJournal(self.__conf.journalFile)
if len(self.__raftLog) == 0:
self.__raftLog.add(_bchr(_COMMAND_TYPE.NO_OP), 1,
self.__raftCurrentTerm)
self.__raftCommitIndex = 1
self.__raftLastApplied = 1
self.__raftNextIndex = {}
self.__raftMatchIndex = {}
self.__lastSerializedTime = time.time()
self.__lastSerializedEntry = None
self.__forceLogCompaction = False
self.__leaderCommitIndex = None
self.__onReadyCalled = False
self.__changeClusterIDx = None
self.__noopIDx = None
self.__destroying = False
self.__startTime = time.time()
globalDnsResolver().setTimeouts(self.__conf.dnsCacheTime,
self.__conf.dnsFailCacheTime)
self.__serializer = Serializer(self.__conf.fullDumpFile,
self.__conf.logCompactionBatchSize,
self.__conf.useFork,
self.__conf.serializer,
self.__conf.deserializer,
self.__conf.serializeChecker)
self.__isInitialized = False
self.__lastInitTryTime = 0
self._poller = createPoller(self.__conf.pollerType)
host, port = selfNodeAddr.split(':')
self.__server = TcpServer(
self._poller,
host,
port,
onNewConnection=self.__onNewConnection,
sendBufferSize=self.__conf.sendBufferSize,
recvBufferSize=self.__conf.recvBufferSize,
connectionTimeout=self.__conf.connectionTimeout)
self._methodToID = {}
self._idToMethod = {}
methods = sorted([m for m in dir(self) if callable(getattr(self, m))])
for i, method in enumerate(methods):
self._methodToID[method] = i
self._idToMethod[i] = getattr(self, method)
self.__thread = None
self.__mainThread = None
self.__initialised = None
self.__commandsQueue = Queue.Queue(self.__conf.commandsQueueSize)
self.__nodes = []
self.__newAppendEntriesTime = 0
self.__commandsWaitingCommit = collections.defaultdict(
list) # logID => [(termID, callback), ...]
self.__commandsLocalCounter = 0
self.__commandsWaitingReply = {} # commandLocalCounter => callback
self.__properies = set()
for key in self.__dict__:
self.__properies.add(key)
if self.__conf.autoTick:
self.__mainThread = threading.current_thread()
self.__initialised = threading.Event()
self.__thread = threading.Thread(target=SyncObj._autoTickThread,
args=(weakref.proxy(self), ))
self.__thread.start()
while not self.__initialised.is_set():
pass
else:
self.__initInTickThread()
class SyncObj(object):
def __init__(self, selfNodeAddr, otherNodesAddrs, conf=None):
if conf is None:
self.__conf = SyncObjConf()
else:
self.__conf = conf
self.__conf.validate()
if self.__conf.password is not None:
if not HAS_CRYPTO:
raise ImportError("Please install 'cryptography' module")
self.__encryptor = getEncryptor(self.__conf.password)
else:
self.__encryptor = None
self.__selfNodeAddr = selfNodeAddr
self.__otherNodesAddrs = otherNodesAddrs
self.__unknownConnections = {} # descr => _Connection
self.__raftState = _RAFT_STATE.FOLLOWER
self.__raftCurrentTerm = 0
self.__votedFor = None
self.__votesCount = 0
self.__raftLeader = None
self.__raftElectionDeadline = time.time() + self.__generateRaftTimeout(
)
self.__raftLog = createJournal(self.__conf.journalFile)
if len(self.__raftLog) == 0:
self.__raftLog.add(_bchr(_COMMAND_TYPE.NO_OP), 1,
self.__raftCurrentTerm)
self.__raftCommitIndex = 1
self.__raftLastApplied = 1
self.__raftNextIndex = {}
self.__raftMatchIndex = {}
self.__lastSerializedTime = time.time()
self.__lastSerializedEntry = None
self.__forceLogCompaction = False
self.__leaderCommitIndex = None
self.__onReadyCalled = False
self.__changeClusterIDx = None
self.__noopIDx = None
self.__destroying = False
self.__startTime = time.time()
globalDnsResolver().setTimeouts(self.__conf.dnsCacheTime,
self.__conf.dnsFailCacheTime)
self.__serializer = Serializer(self.__conf.fullDumpFile,
self.__conf.logCompactionBatchSize,
self.__conf.useFork,
self.__conf.serializer,
self.__conf.deserializer,
self.__conf.serializeChecker)
self.__isInitialized = False
self.__lastInitTryTime = 0
self._poller = createPoller(self.__conf.pollerType)
host, port = selfNodeAddr.split(':')
self.__server = TcpServer(
self._poller,
host,
port,
onNewConnection=self.__onNewConnection,
sendBufferSize=self.__conf.sendBufferSize,
recvBufferSize=self.__conf.recvBufferSize,
connectionTimeout=self.__conf.connectionTimeout)
self._methodToID = {}
self._idToMethod = {}
methods = sorted([m for m in dir(self) if callable(getattr(self, m))])
for i, method in enumerate(methods):
self._methodToID[method] = i
self._idToMethod[i] = getattr(self, method)
self.__thread = None
self.__mainThread = None
self.__initialised = None
self.__commandsQueue = Queue.Queue(self.__conf.commandsQueueSize)
self.__nodes = []
self.__newAppendEntriesTime = 0
self.__commandsWaitingCommit = collections.defaultdict(
list) # logID => [(termID, callback), ...]
self.__commandsLocalCounter = 0
self.__commandsWaitingReply = {} # commandLocalCounter => callback
self.__properies = set()
for key in self.__dict__:
self.__properies.add(key)
if self.__conf.autoTick:
self.__mainThread = threading.current_thread()
self.__initialised = threading.Event()
self.__thread = threading.Thread(target=SyncObj._autoTickThread,
args=(weakref.proxy(self), ))
self.__thread.start()
while not self.__initialised.is_set():
pass
else:
self.__initInTickThread()
def _destroy(self):
if self.__conf.autoTick:
self.__destroying = True
else:
self._doDestroy()
def _doDestroy(self):
for node in self.__nodes:
node._destroy()
self.__server.unbind()
self.__raftLog._destroy()
def __initInTickThread(self):
try:
self.__lastInitTryTime = time.time()
self.__server.bind()
self.__nodes = []
for nodeAddr in self.__otherNodesAddrs:
self.__nodes.append(Node(self, nodeAddr))
self.__raftNextIndex[nodeAddr] = self.__getCurrentLogIndex(
) + 1
self.__raftMatchIndex[nodeAddr] = 0
self.__needLoadDumpFile = True
self.__isInitialized = True
except:
LOG_CURRENT_EXCEPTION()
def _addNodeToCluster(self, nodeName, callback=None):
if not self.__conf.dynamicMembershipChange:
raise Exception('dynamicMembershipChange is disabled')
self._applyCommand(cPickle.dumps(['add', nodeName]), callback,
_COMMAND_TYPE.MEMBERSHIP)
def _removeNodeFromCluster(self, nodeName, callback=None):
if not self.__conf.dynamicMembershipChange:
raise Exception('dynamicMembershipChange is disabled')
self._applyCommand(cPickle.dumps(['rem', nodeName]), callback,
_COMMAND_TYPE.MEMBERSHIP)
def _applyCommand(self, command, callback, commandType=None):
try:
if commandType is None:
self.__commandsQueue.put_nowait((command, callback))
else:
self.__commandsQueue.put_nowait(
(_bchr(commandType) + command, callback))
except Queue.Full:
self.__callErrCallback(FAIL_REASON.QUEUE_FULL, callback)
def _checkCommandsToApply(self):
startTime = time.time()
while time.time() - startTime < self.__conf.appendEntriesPeriod:
if self.__raftLeader is None and self.__conf.commandsWaitLeader:
break
try:
command, callback = self.__commandsQueue.get_nowait()
except Queue.Empty:
break
requestNode, requestID = None, None
if isinstance(callback, tuple):
requestNode, requestID = callback
if self.__raftState == _RAFT_STATE.LEADER:
idx, term = self.__getCurrentLogIndex(
) + 1, self.__raftCurrentTerm
if self.__conf.dynamicMembershipChange:
changeClusterRequest = self.__parseChangeClusterRequest(
command)
else:
changeClusterRequest = None
if changeClusterRequest is None or self.__changeCluster(
changeClusterRequest):
self.__raftLog.add(command, idx, term)
if requestNode is None:
if callback is not None:
self.__commandsWaitingCommit[idx].append(
(term, callback))
else:
self.__send(
requestNode, {
'type': 'apply_command_response',
'request_id': requestID,
'log_idx': idx,
'log_term': term,
})
if not self.__conf.appendEntriesUseBatch:
self.__sendAppendEntries()
elif self.__raftLeader is not None:
if requestNode is None:
message = {
'type': 'apply_command',
'command': command,
}
if callback is not None:
self.__commandsLocalCounter += 1
self.__commandsWaitingReply[
self.__commandsLocalCounter] = callback
message['request_id'] = self.__commandsLocalCounter
self.__send(self.__raftLeader, message)
else:
self.__send(
requestNode, {
'type': 'apply_command_response',
'request_id': requestID,
'error': FAIL_REASON.NOT_LEADER,
})
else:
self.__callErrCallback(FAIL_REASON.MISSING_LEADER, callback)
def _autoTickThread(self):
self.__initInTickThread()
self.__initialised.set()
time.sleep(0.1)
try:
while True:
if not self.__mainThread.is_alive():
break
if self.__destroying:
self._doDestroy()
break
self._onTick(self.__conf.autoTickPeriod)
except ReferenceError:
pass
def _onTick(self, timeToWait=0.0):
if not self.__isInitialized:
if time.time(
) >= self.__lastInitTryTime + self.__conf.bindRetryTime:
self.__initInTickThread()
if not self.__isInitialized:
time.sleep(timeToWait)
return
if self.__needLoadDumpFile:
if self.__conf.fullDumpFile is not None and os.path.isfile(
self.__conf.fullDumpFile):
self.__loadDumpFile(clearJournal=False)
self.__needLoadDumpFile = False
if self.__raftState in (_RAFT_STATE.FOLLOWER, _RAFT_STATE.CANDIDATE):
if self.__raftElectionDeadline < time.time(
) and self.__connectedToAnyone():
self.__raftElectionDeadline = time.time(
) + self.__generateRaftTimeout()
self.__raftLeader = None
self.__raftState = _RAFT_STATE.CANDIDATE
self.__raftCurrentTerm += 1
self.__votedFor = self._getSelfNodeAddr()
self.__votesCount = 1
for node in self.__nodes:
node.send({
'type': 'request_vote',
'term': self.__raftCurrentTerm,
'last_log_index': self.__getCurrentLogIndex(),
'last_log_term': self.__getCurrentLogTerm(),
})
self.__onLeaderChanged()
if self.__raftState == _RAFT_STATE.LEADER:
while self.__raftCommitIndex < self.__getCurrentLogIndex():
nextCommitIndex = self.__raftCommitIndex + 1
count = 1
for node in self.__nodes:
if self.__raftMatchIndex[
node.getAddress()] >= nextCommitIndex:
count += 1
if count > (len(self.__nodes) + 1) / 2:
self.__raftCommitIndex = nextCommitIndex
else:
break
self.__leaderCommitIndex = self.__raftCommitIndex
if time.time() > self.__newAppendEntriesTime:
self.__sendAppendEntries()
if self.__raftCommitIndex > self.__raftLastApplied:
count = self.__raftCommitIndex - self.__raftLastApplied
entries = self.__getEntries(self.__raftLastApplied + 1, count)
for entry in entries:
currentTermID = entry[2]
subscribers = self.__commandsWaitingCommit.pop(entry[1], [])
res = self.__doApplyCommand(entry[0])
for subscribeTermID, callback in subscribers:
if subscribeTermID == currentTermID:
callback(res, FAIL_REASON.SUCCESS)
else:
callback(None, FAIL_REASON.DISCARDED)
self.__raftLastApplied += 1
if not self.__onReadyCalled and self.__raftLastApplied == self.__leaderCommitIndex:
if self.__conf.onReady:
self.__conf.onReady()
self.__onReadyCalled = True
self._checkCommandsToApply()
self.__tryLogCompaction()
for node in self.__nodes:
node.connectIfRequired()
self._poller.poll(timeToWait)
def _printStatus(self):
LOG_DEBUG('version', VERSION, REVISION)
LOG_DEBUG('self', self.__selfNodeAddr)
LOG_DEBUG('state:', self.__raftState)
LOG_DEBUG('leader', self.__raftLeader)
LOG_DEBUG('partner nodes', len(self.__nodes))
for n in self.__nodes:
LOG_DEBUG(n.getAddress(), n.getStatus())
LOG_DEBUG('log len:', len(self.__raftLog))
LOG_DEBUG('last applied:', self.__raftLastApplied)
LOG_DEBUG('commit idx:', self.__raftCommitIndex)
LOG_DEBUG('raft term:', self.__raftCurrentTerm)
LOG_DEBUG('next node idx:', self.__raftNextIndex)
LOG_DEBUG('match idx:', self.__raftMatchIndex)
LOG_DEBUG('leader commit idx:', self.__leaderCommitIndex)
LOG_DEBUG('uptime:', int(time.time() - self.__startTime))
LOG_DEBUG('')
def _forceLogCompaction(self):
self.__forceLogCompaction = True
def __doApplyCommand(self, command):
commandType = ord(command[:1])
# Skip no-op and membership change commands
if commandType != _COMMAND_TYPE.REGULAR:
return
command = cPickle.loads(command[1:])
args = []
kwargs = {
'_doApply': True,
}
if not isinstance(command, tuple):
funcID = command
elif len(command) == 2:
funcID, args = command
else:
funcID, args, newKwArgs = command
kwargs.update(newKwArgs)
return self._idToMethod[funcID](*args, **kwargs)
def _onMessageReceived(self, nodeAddr, message):
if message['type'] == 'request_vote':
if message['term'] > self.__raftCurrentTerm:
self.__raftCurrentTerm = message['term']
self.__votedFor = None
self.__raftState = _RAFT_STATE.FOLLOWER
self.__raftLeader = None
if self.__raftState in (_RAFT_STATE.FOLLOWER,
_RAFT_STATE.CANDIDATE):
lastLogTerm = message['last_log_term']
lastLogIdx = message['last_log_index']
if message['term'] >= self.__raftCurrentTerm:
if lastLogTerm < self.__getCurrentLogTerm():
return
if lastLogTerm == self.__getCurrentLogTerm() and \
lastLogIdx < self.__getCurrentLogIndex():
return
if self.__votedFor is not None:
return
self.__votedFor = nodeAddr
self.__raftElectionDeadline = time.time(
) + self.__generateRaftTimeout()
self.__send(nodeAddr, {
'type': 'response_vote',
'term': message['term'],
})
if message['type'] == 'append_entries' and message[
'term'] >= self.__raftCurrentTerm:
self.__raftElectionDeadline = time.time(
) + self.__generateRaftTimeout()
if self.__raftLeader != nodeAddr:
self.__onLeaderChanged()
self.__raftLeader = nodeAddr
if message['term'] > self.__raftCurrentTerm:
self.__raftCurrentTerm = message['term']
self.__votedFor = None
self.__raftState = _RAFT_STATE.FOLLOWER
newEntries = message.get('entries', [])
serialized = message.get('serialized', None)
self.__leaderCommitIndex = leaderCommitIndex = message[
'commit_index']
# Regular append entries
if 'prevLogIdx' in message:
prevLogIdx = message['prevLogIdx']
prevLogTerm = message['prevLogTerm']
prevEntries = self.__getEntries(prevLogIdx)
if not prevEntries:
self.__sendNextNodeIdx(nodeAddr, success=False, reset=True)
return
if prevEntries[0][2] != prevLogTerm:
self.__sendNextNodeIdx(nodeAddr,
nextNodeIdx=prevLogIdx,
success=False,
reset=True)
return
if len(prevEntries) > 1:
# rollback cluster changes
if self.__conf.dynamicMembershipChange:
for entry in reversed(prevEntries[1:]):
clusterChangeRequest = self.__parseChangeClusterRequest(
entry[0])
if clusterChangeRequest is not None:
self.__doChangeCluster(clusterChangeRequest,
reverse=True)
self.__deleteEntriesFrom(prevLogIdx + 1)
for entry in newEntries:
self.__raftLog.add(*entry)
# apply cluster changes
if self.__conf.dynamicMembershipChange:
for entry in newEntries:
clusterChangeRequest = self.__parseChangeClusterRequest(
entry[0])
if clusterChangeRequest is not None:
self.__doChangeCluster(clusterChangeRequest)
nextNodeIdx = prevLogIdx + 1
if newEntries:
nextNodeIdx = newEntries[-1][1]
self.__sendNextNodeIdx(nodeAddr,
nextNodeIdx=nextNodeIdx,
success=True)
# Install snapshot
elif serialized is not None:
if self.__serializer.setTransmissionData(serialized):
self.__loadDumpFile(clearJournal=True)
self.__sendNextNodeIdx(nodeAddr, success=True)
self.__raftCommitIndex = min(leaderCommitIndex,
self.__getCurrentLogIndex())
if message['type'] == 'apply_command':
if 'request_id' in message:
self._applyCommand(message['command'],
(nodeAddr, message['request_id']))
else:
self._applyCommand(message['command'], None)
if message['type'] == 'apply_command_response':
requestID = message['request_id']
error = message.get('error', None)
callback = self.__commandsWaitingReply.pop(requestID, None)
if callback is not None:
if error is not None:
callback(None, error)
else:
idx = message['log_idx']
term = message['log_term']
assert idx > self.__raftLastApplied
self.__commandsWaitingCommit[idx].append((term, callback))
if self.__raftState == _RAFT_STATE.CANDIDATE:
if message['type'] == 'response_vote' and message[
'term'] == self.__raftCurrentTerm:
self.__votesCount += 1
if self.__votesCount > (len(self.__nodes) + 1) / 2:
self.__onBecomeLeader()
if self.__raftState == _RAFT_STATE.LEADER:
if message['type'] == 'next_node_idx':
reset = message['reset']
nextNodeIdx = message['next_node_idx']
success = message['success']
currentNodeIdx = nextNodeIdx - 1
if reset:
self.__raftNextIndex[nodeAddr] = nextNodeIdx
if success:
self.__raftMatchIndex[nodeAddr] = currentNodeIdx
def __callErrCallback(self, err, callback):
if callback is None:
return
if isinstance(callback, tuple):
requestNode, requestID = callback
self.__send(
requestNode, {
'type': 'apply_command_response',
'request_id': requestID,
'error': err,
})
return
callback(None, err)
def __sendNextNodeIdx(self,
nodeAddr,
reset=False,
nextNodeIdx=None,
success=False):
if nextNodeIdx is None:
nextNodeIdx = self.__getCurrentLogIndex() + 1
self.__send(
nodeAddr, {
'type': 'next_node_idx',
'next_node_idx': nextNodeIdx,
'reset': reset,
'success': success,
})
def __generateRaftTimeout(self):
minTimeout = self.__conf.raftMinTimeout
maxTimeout = self.__conf.raftMaxTimeout
return minTimeout + (maxTimeout - minTimeout) * random.random()
def __onNewConnection(self, conn):
descr = conn.fileno()
self.__unknownConnections[descr] = conn
if self.__encryptor:
conn.encryptor = self.__encryptor
conn.setOnMessageReceivedCallback(
functools.partial(self.__onMessageReceived, conn))
conn.setOnDisconnectedCallback(
functools.partial(self.__onDisconnected, conn))
def __onMessageReceived(self, conn, message):
if self.__encryptor and not conn.sendRandKey:
conn.sendRandKey = message
conn.recvRandKey = os.urandom(32)
conn.send(conn.recvRandKey)
return
descr = conn.fileno()
partnerNode = None
for node in self.__nodes:
if node.getAddress() == message:
partnerNode = node
break
if partnerNode is None:
conn.disconnect()
return
partnerNode.onPartnerConnected(conn)
self.__unknownConnections.pop(descr, None)
def __onDisconnected(self, conn):
self.__unknownConnections.pop(conn.fileno(), None)
def __getCurrentLogIndex(self):
return self.__raftLog[-1][1]
def __getCurrentLogTerm(self):
return self.__raftLog[-1][2]
def __getPrevLogIndexTerm(self, nextNodeIndex):
prevIndex = nextNodeIndex - 1
entries = self.__getEntries(prevIndex, 1)
if entries:
return prevIndex, entries[0][2]
return None, None
def __getEntries(self, fromIDx, count=None, maxSizeBytes=None):
firstEntryIDx = self.__raftLog[0][1]
if fromIDx is None or fromIDx < firstEntryIDx:
return []
diff = fromIDx - firstEntryIDx
if count is None:
result = self.__raftLog[diff:]
else:
result = self.__raftLog[diff:diff + count]
if maxSizeBytes is None:
return result
totalSize = 0
i = 0
for i, entry in enumerate(result):
totalSize += len(entry[0])
if totalSize >= maxSizeBytes:
break
return result[:i + 1]
def _isLeader(self):
return self.__raftState == _RAFT_STATE.LEADER
def _getLeader(self):
return self.__raftLeader
def _isReady(self):
return self.__onReadyCalled
def _getTerm(self):
return self.__raftCurrentTerm
def _getRaftLogSize(self):
return len(self.__raftLog)
def __deleteEntriesFrom(self, fromIDx):
firstEntryIDx = self.__raftLog[0][1]
diff = fromIDx - firstEntryIDx
if diff < 0:
return
self.__raftLog.deleteEntriesFrom(diff)
def __deleteEntriesTo(self, toIDx):
firstEntryIDx = self.__raftLog[0][1]
diff = toIDx - firstEntryIDx
if diff < 0:
return
self.__raftLog.deleteEntriesTo(diff)
def __onBecomeLeader(self):
self.__raftLeader = self.__selfNodeAddr
self.__raftState = _RAFT_STATE.LEADER
for node in self.__nodes:
nodeAddr = node.getAddress()
self.__raftNextIndex[nodeAddr] = self.__getCurrentLogIndex() + 1
self.__raftMatchIndex[nodeAddr] = 0
# No-op command after leader election.
idx, term = self.__getCurrentLogIndex() + 1, self.__raftCurrentTerm
self.__raftLog.add(_bchr(_COMMAND_TYPE.NO_OP), idx, term)
self.__noopIDx = idx
if not self.__conf.appendEntriesUseBatch:
self.__sendAppendEntries()
self.__sendAppendEntries()
def __onLeaderChanged(self):
for id in sorted(self.__commandsWaitingReply):
self.__commandsWaitingReply[id](None, FAIL_REASON.LEADER_CHANGED)
self.__commandsWaitingReply = {}
def __sendAppendEntries(self):
self.__newAppendEntriesTime = time.time(
) + self.__conf.appendEntriesPeriod
startTime = time.time()
for node in self.__nodes:
nodeAddr = node.getAddress()
if not node.isConnected():
self.__serializer.cancelTransmisstion(nodeAddr)
continue
sendSingle = True
sendingSerialized = False
nextNodeIndex = self.__raftNextIndex[nodeAddr]
while nextNodeIndex <= self.__getCurrentLogIndex(
) or sendSingle or sendingSerialized:
if nextNodeIndex > self.__raftLog[0][1]:
prevLogIdx, prevLogTerm = self.__getPrevLogIndexTerm(
nextNodeIndex)
entries = []
if nextNodeIndex <= self.__getCurrentLogIndex():
entries = self.__getEntries(
nextNodeIndex, None,
self.__conf.appendEntriesBatchSizeBytes)
self.__raftNextIndex[nodeAddr] = entries[-1][1] + 1
message = {
'type': 'append_entries',
'term': self.__raftCurrentTerm,
'commit_index': self.__raftCommitIndex,
'entries': entries,
'prevLogIdx': prevLogIdx,
'prevLogTerm': prevLogTerm,
}
node.send(message)
else:
transmissionData = self.__serializer.getTransmissionData(
nodeAddr)
message = {
'type': 'append_entries',
'term': self.__raftCurrentTerm,
'commit_index': self.__raftCommitIndex,
'serialized': transmissionData,
}
node.send(message)
if transmissionData is not None:
isLast = transmissionData[2]
if isLast:
self.__raftNextIndex[
nodeAddr] = self.__raftLog[1][1] + 1
sendingSerialized = False
else:
sendingSerialized = True
else:
sendingSerialized = False
nextNodeIndex = self.__raftNextIndex[nodeAddr]
sendSingle = False
delta = time.time() - startTime
if delta > self.__conf.appendEntriesPeriod:
break
def __send(self, nodeAddr, message):
for node in self.__nodes:
if node.getAddress() == nodeAddr:
node.send(message)
break
def __connectedToAnyone(self):
for node in self.__nodes:
if node.getStatus() == NODE_STATUS.CONNECTED:
return True
return False
def _getSelfNodeAddr(self):
return self.__selfNodeAddr
def _getConf(self):
return self.__conf
def _getEncryptor(self):
return self.__encryptor
def __changeCluster(self, request):
if self.__raftLastApplied < self.__noopIDx:
# No-op entry was not commited yet
return False
if self.__changeClusterIDx is not None:
if self.__raftLastApplied >= self.__changeClusterIDx:
self.__changeClusterIDx = None
# Previous cluster change request was not commited yet
if self.__changeClusterIDx is not None:
return False
return self.__doChangeCluster(request)
def __doChangeCluster(self, request, reverse=False):
requestType = request[0]
requestNode = request[1]
if requestType == 'add':
adding = not reverse
elif requestType == 'rem':
adding = reverse
else:
return False
if adding:
newNode = requestNode
# Node already exists in cluster
if newNode == self.__selfNodeAddr or newNode in self.__otherNodesAddrs:
return False
self.__otherNodesAddrs.append(newNode)
self.__nodes.append(Node(self, newNode))
self.__raftNextIndex[newNode] = self.__getCurrentLogIndex() + 1
self.__raftMatchIndex[newNode] = 0
return True
else:
oldNode = requestNode
if oldNode == self.__selfNodeAddr:
return False
if oldNode not in self.__otherNodesAddrs:
return False
for i in xrange(len(self.__nodes)):
if self.__nodes[i].getAddress() == oldNode:
self.__nodes[i]._destroy()
self.__nodes.pop(i)
self.__otherNodesAddrs.pop(i)
del self.__raftNextIndex[oldNode]
return True
return False
def __parseChangeClusterRequest(self, command):
commandType = ord(command[:1])
if commandType != _COMMAND_TYPE.MEMBERSHIP:
return None
return cPickle.loads(command[1:])
def __tryLogCompaction(self):
currTime = time.time()
serializeState, serializeID = self.__serializer.checkSerializing()
if serializeState == SERIALIZER_STATE.SUCCESS:
self.__lastSerializedTime = currTime
self.__deleteEntriesTo(serializeID)
self.__lastSerializedEntry = serializeID
if serializeState == SERIALIZER_STATE.FAILED:
LOG_WARNING("Failed to store full dump")
if serializeState != SERIALIZER_STATE.NOT_SERIALIZING:
return
if len(self.__raftLog) <= self.__conf.logCompactionMinEntries and \
currTime - self.__lastSerializedTime <= self.__conf.logCompactionMinTime and \
not self.__forceLogCompaction:
return
if self.__conf.logCompactionSplit:
allNodes = sorted(self.__otherNodesAddrs + [self.__selfNodeAddr])
nodesCount = len(allNodes)
selfIdx = allNodes.index(self.__selfNodeAddr)
interval = self.__conf.logCompactionMinTime
periodStart = int(currTime) / interval * interval
nodeInterval = float(interval) / nodesCount
nodeIntervalStart = periodStart + selfIdx * nodeInterval
nodeIntervalEnd = nodeIntervalStart + 0.3 * nodeInterval
if currTime < nodeIntervalStart or currTime >= nodeIntervalEnd:
return
self.__forceLogCompaction = False
lastAppliedEntries = self.__getEntries(self.__raftLastApplied - 1, 2)
if len(lastAppliedEntries
) < 2 or lastAppliedEntries[0][1] == self.__lastSerializedEntry:
self.__lastSerializedTime = currTime
return
if self.__conf.serializer is None:
data = dict([(k, self.__dict__[k]) for k in self.__dict__.keys()
if k not in self.__properies])
else:
data = None
cluster = self.__otherNodesAddrs + [self.__selfNodeAddr]
self.__serializer.serialize(
(data, lastAppliedEntries[1], lastAppliedEntries[0], cluster),
lastAppliedEntries[0][1])
def __loadDumpFile(self, clearJournal):
try:
data = self.__serializer.deserialize()
if data[0] is not None:
for k, v in data[0].iteritems():
self.__dict__[k] = v
if clearJournal or \
len(self.__raftLog) < 2 or \
self.__raftLog[0] != data[2] or \
self.__raftLog[1] != data[1]:
self.__raftLog.clear()
self.__raftLog.add(*data[2])
self.__raftLog.add(*data[1])
self.__raftLastApplied = data[1][1]
if self.__conf.dynamicMembershipChange:
self.__otherNodesAddrs = [
node for node in data[3] if node != self.__selfNodeAddr
]
self.__updateClusterConfiguration()
except:
LOG_WARNING('Failed to load full dump')
LOG_CURRENT_EXCEPTION()
def __updateClusterConfiguration(self):
currentNodes = set()
for i in xrange(len(self.__nodes) - 1, -1, -1):
nodeAddr = self.__nodes[i].getAddress()
if nodeAddr not in self.__otherNodesAddrs:
self.__nodes[i]._destroy()
self.__nodes.pop(i)
else:
currentNodes.add(nodeAddr)
for nodeAddr in self.__otherNodesAddrs:
if nodeAddr not in currentNodes:
self.__nodes.append(Node(self, nodeAddr))
self.__raftNextIndex[nodeAddr] = self.__getCurrentLogIndex(
) + 1
self.__raftMatchIndex[nodeAddr] = 0
class Proto(QObject):
__pyqtSignals__ = ('logMessage(QString, int)',
'ready()',
'readyAnswer()')
def __init__(self, mode, parent=None):
QObject.__init__(self, parent)
# ---
self.server = TcpServer(mode, self)
self.server.logMessage.connect(self.emit_log)
self.server.readByteArray.connect(self.on_read_bytearray)
self.pending_data = []
self.stored_data = []
# ---
self.client = TcpClient(self)
self.client.logMessage.connect(self.emit_log)
self.client.readByteArray.connect(self.on_read_bytearray_answer)
self.pending_data_answer = []
self.stored_data_answer = []
# ---
self.iserv = u'КАУС->ПРМК'
self.iserv_2 = u'ПРМК->ПРДК'
self.oserv = u'ПРМК->КАУС'
self.oserv_2 = u'ПРДК->ПРМК'
self.iclient = u'ПРДК->ПРМК'
self.oclient = u'ПРМК->ПРДК'
# ---
self.dest_sign = u'ПРМК->КАУС'
self.dest_sign_2 = u'КАУС->ПРМК'
if mode == ImitParams.transmitter_mode:
self.dest_sign = u'ПРДК->ПРМК'
self.dest_sign_2 = u'ПРМК->ПРДК'
# ---
self.tflag = True if mode == ImitParams.transmitter_mode else False
def pop_pending_data(self):
try:
pdata = self.pending_data.pop(0)
return (pdata[0], pdata[1])
except IndexError:
return None
def pop_pending_data_prdk(self):
try:
pdata = self.pending_data_answer.pop(0)
return (pdata[0], pdata[1])
except IndexError:
return None
def send_data(self, cmd, params):
array, dgram = self.create_datagram(DATAGRAM_ANS4AUS, DEFAULT_SESSION_ID, [cmd] + params)
dest = self.oserv
if self.tflag:
dest = self.oserv_2
self.emit_log(u'%s: %s' % (dest, dgram), 110)
self.server.write(array)
def send_data_prdk(self, cmd, params):
array, dgram = self.create_datagram(DATAGRAM_ANS4AUS, DEFAULT_SESSION_ID, [cmd] + params)
self.emit_log(u'ПРМК->ПРДК: %s' % dgram, 110)
self.client.write(array)
def create_datagram(self, dtype, dservice, info):
array = QByteArray()
array.append(chr(dtype))
info_size = len(info)
hi = (info_size >> 8) & 0xFF
lo = info_size & 0xFF
array.append(chr(hi))
array.append(chr(lo))
array.append(chr(dservice))
for byte in info:
array.append(chr(byte))
array.insert(0, chr(MARKER))
dgram = bytearray2str(array)
return (array, dgram)
def on_read_bytearray_answer(self, bytearray):
# --- переводим QByteArray в список int
ilist = bytearray2intlist(bytearray)
# --- выполняем последовательный поиск МАРКЕРОВ
from_index = 0
marker_index = -1
while True:
try:
marker_index = ilist.index(MARKER, from_index)
# --- если маркер не первый - видимо это завершение предыдущей КДГ
# --- или просто мусор (зависит от сохраненных ранее байтов)
if from_index == 0 and marker_index > 0:
if len(self.stored_data_answer) != 0: # имеются сохраненные данные => это завершение начатого пакета
self.stored_data_answer += ilist[0:marker_index] # прицепляем "хвост"
info = self.check_datagram_validation(self.stored_data_answer, False)
if info and len(info) > 0:
self.pending_data_answer.append((info[0], info[1:]))
self.emit(SIGNAL('readyAnswer()'))
self.stored_data_answer = []
from_index = marker_index + 1
continue
# ---
if len(self.stored_data_answer) != 0:
self.emit_log(u'Данные отброшены (%d) - %s'
% (len(self.stored_data_answer), intlist2str(self.stored_data_answer)))
self.stored_data_answer = []
# ---
kdg = self.check_datagram_validation(ilist[marker_index:], False)
# ---
if not kdg:
from_index = marker_index + 1
continue
# ---
if len(kdg) == 0:
self.stored_data_answer = ilist[marker_index:]
from_index = marker_index + 1
continue
# ---
cmd = kdg[INFO_OFFSET]
params = kdg[INFO_OFFSET + 1:]
pdata = (cmd, params)
self.pending_data_answer.append(pdata)
self.emit(SIGNAL('readyAnswer()'))
from_index = marker_index + len(kdg)
except ValueError:
if marker_index < 0:
if len(self.stored_data_answer) > 0:
self.stored_data_answer += ilist
break
# --- обрабатываем данные от сервера
def on_read_bytearray(self, bytearray):
# --- переводим QByteArray в список int
ilist = bytearray2intlist(bytearray)
# --- выполняем последовательный поиск МАРКЕРОВ
from_index = 0
marker_index = -1
while True:
try:
marker_index = ilist.index(MARKER, from_index)
# --- если маркер не первый - видимо это завершение предыдущей КДГ
# --- или просто мусор (зависит от сохраненных ранее байтов)
if from_index == 0 and marker_index > 0:
if len(self.stored_data) != 0: # имеются сохраненные данные => это завершение начатого пакета
self.stored_data += ilist[0:marker_index] # прицепляем "хвост"
info = self.check_datagram_validation(self.stored_data)
if info and len(info) > 0:
self.pending_data.append((info[0], info[1:]))
self.emit(SIGNAL('ready()'))
self.stored_data = []
from_index = marker_index + 1
continue
# ---
if len(self.stored_data) != 0:
self.emit_log(u'Данные отброшены (%d) - %s'
% (len(self.stored_data), intlist2str(self.stored_data)))
self.stored_data = []
# ---
kdg = self.check_datagram_validation(ilist[marker_index:])
# ---
if not kdg:
from_index = marker_index + 1
continue
# ---
if len(kdg) == 0:
self.stored_data = ilist[marker_index:]
from_index = marker_index + 1
continue
# ---
self.pending_data.append((kdg[INFO_OFFSET], kdg[INFO_OFFSET + 1:]))
self.emit(SIGNAL('ready()'))
from_index = marker_index + len(kdg)
except ValueError:
if marker_index < 0:
if len(self.stored_data) > 0:
self.stored_data += ilist
break
def check_datagram_validation(self, pack, Server_Flag=True):
"""
Проверяем датаграмму на валидность
pack = [int, int, ...]
"""
dgram = intlist2str(pack)
if len(pack) < HEADER_SIZE:
self.emit_log(u'Недостаточная длина пакета - %s' % dgram, 130)
return None
du = self.get_info_length(pack)
df = len(pack) - HEADER_SIZE
if du > df:
return [] # незавершенная кодограмма
dest = self.iserv
if self.tflag:
dest = self.iserv_2
elif not Server_Flag:
dest = self.iclient
self.emit_log(u'%s: %s' % (dest, dgram), 110)
dtype = self.get_datagram_type(pack)
dservice = self.get_service_type(pack)
if dtype == DATAGRAM_NULL:
return None
elif dtype == DATAGRAM_REG4KRS:
if Server_Flag:
self.emit_log(u'Получен пакет регистрации, тип сеанса - %d' % dservice)
array, dgram = self.create_datagram(DATAGRAM_SOK4AUS, DEFAULT_SESSION_ID, [])
dest = self.iserv
if self.tflag:
dest = self.iserv_2
self.emit_log(u'%s: %s' % (dest, dgram), 110)
self.server.write(array)
self.emit_log(u'Сеанс управления открыт, тип сеанса - %d' % DEFAULT_SESSION_ID)
else:
self.emit_log(u'Ошибка регистрации - неожиданное сообщение', 130)
return None
elif dtype == DATAGRAM_CMD4KRS:
kdg = pack[:INFO_OFFSET + du]
return kdg
elif dtype == DATAGRAM_ANS4AUS:
if len(pack) <= HEADER_SIZE:
return None
kdg = pack[:INFO_OFFSET + du]
return kdg
else:
#self.emit_log(u'Ошибка типа входящего сообщения (%d)' % dtype)
return None
def get_datagram_type(self, pack):
return pack[TYPE_OFFSET]
def get_service_type(self, pack):
return pack[SERVICE_OFFSET]
def get_info_length(self, pack):
hi = pack[HIGH_LENGTH_OFFSET] << 8
low = pack[LOW_LENGTH_OFFSET]
length = hi | low
return length
def emit_log(self, text, type_=10):
self.emit(SIGNAL('logMessage(QString, int)'), text, type_)
class HoloSensorReceiver(object):
def __init__(self):
# init ROS node
rospy.init_node("holo_sensor_receiver", anonymous=True)
self.server_ = TcpServer(
# get from private domain ~
host=rospy.get_param("~host", default="192.168.1.152"),
port=rospy.get_param("~port", default=8800))
self.pub_ = rospy.Publisher("image_topic", Image, queue_size=5)
self.bridge_ = CvBridge()
# self.pub_ = rospy.Publisher('holo_transform', HoloTransform, queue_size=10)
# call self.server_.stop before node destroied
rospy.on_shutdown(self.server_.stop)
def launch(self):
self.server_.launch()
# publish data received from the HoloLens
# We use a thread to launch the publish() function since we
# want to have the ROS handling the system interupts.
# In case, publish() has been blocked.
thread = threading.Thread(target=self.publish_)
thread.start()
rospy.spin()
def publish_(self):
for package in self.server_.fetch_package():
width, height, img_bgr = self.unpack_(package)
print("Receive img shape = {}, width = {}, height = {}".format(
img_bgr.shape, width, height))
img_bgr = cv2.resize(img_bgr, (320, 240))
img_bgr = cv2.rotate(img_bgr, cv2.ROTATE_90_CLOCKWISE)
try:
self.pub_.publish(self.bridge_.cv2_to_imgmsg(img_bgr, "bgr8"))
except CvBridgeError as e:
print(e)
def unpack_(self, package):
"""
Unpack TCP package
Args:
package (bytes uint8):
0 - 3: width
4 - 7: height
8 - EOF: image (RGBA)
"""
width = struct.unpack('<I', package[0:4])[0]
height = struct.unpack('<I', package[4:8])[0]
img_bin = package[8:]
img_rgba = np.fromstring(img_bin, dtype=np.uint8).reshape(
(height, width, 4))
img_bgr = cv2.cvtColor(img_rgba, cv2.COLOR_RGBA2BGR)
return width, height, img_bgr
def __init__(self, ip, port, reactor_num=5):
self.__compute_thread_pool = compute_thread_pool
self.__server = TcpServer(ip, port, reactor_num)
def __init__(self, selfNodeAddr, otherNodesAddrs, conf=None):
"""
Main SyncObj class, you should inherit your own class from it.
:param selfNodeAddr: address of the current node server, 'host:port'
:type selfNodeAddr: str
:param otherNodesAddrs: addresses of partner nodes, ['host1:port1', 'host2:port2', ...]
:type otherNodesAddrs: list of str
:param conf: configuration object
:type conf: SyncObjConf
"""
if conf is None:
self.__conf = SyncObjConf()
else:
self.__conf = conf
self.__conf.validate()
if self.__conf.password is not None:
if not HAS_CRYPTO:
raise ImportError("Please install 'cryptography' module")
self.__encryptor = getEncryptor(self.__conf.password)
else:
self.__encryptor = None
self.__selfNodeAddr = selfNodeAddr
self.__otherNodesAddrs = otherNodesAddrs
self.__unknownConnections = {} # descr => _Connection
self.__raftState = _RAFT_STATE.FOLLOWER
self.__raftCurrentTerm = 0
self.__votedFor = None
self.__votesCount = 0
self.__raftLeader = None
self.__raftElectionDeadline = time.time() + self.__generateRaftTimeout()
self.__raftLog = createJournal(self.__conf.journalFile)
if len(self.__raftLog) == 0:
self.__raftLog.add(_bchr(_COMMAND_TYPE.NO_OP), 1, self.__raftCurrentTerm)
self.__raftCommitIndex = 1
self.__raftLastApplied = 1
self.__raftNextIndex = {}
self.__raftMatchIndex = {}
self.__lastSerializedTime = time.time()
self.__lastSerializedEntry = None
self.__forceLogCompaction = False
self.__leaderCommitIndex = None
self.__onReadyCalled = False
self.__changeClusterIDx = None
self.__noopIDx = None
self.__destroying = False
self.__recvTransmission = ''
self.__startTime = time.time()
globalDnsResolver().setTimeouts(self.__conf.dnsCacheTime, self.__conf.dnsFailCacheTime)
self.__serializer = Serializer(self.__conf.fullDumpFile,
self.__conf.logCompactionBatchSize,
self.__conf.useFork,
self.__conf.serializer,
self.__conf.deserializer,
self.__conf.serializeChecker)
self.__isInitialized = False
self.__lastInitTryTime = 0
self._poller = createPoller(self.__conf.pollerType)
if selfNodeAddr is not None:
bindAddr = self.__conf.bindAddress or selfNodeAddr
host, port = bindAddr.split(':')
self.__server = TcpServer(self._poller, host, port, onNewConnection=self.__onNewConnection,
sendBufferSize=self.__conf.sendBufferSize,
recvBufferSize=self.__conf.recvBufferSize,
connectionTimeout=self.__conf.connectionTimeout)
self._methodToID = {}
self._idToMethod = {}
methods = sorted([m for m in dir(self) if callable(getattr(self, m))])
for i, method in enumerate(methods):
self._methodToID[method] = i
self._idToMethod[i] = getattr(self, method)
self.__thread = None
self.__mainThread = None
self.__initialised = None
self.__bindedEvent = threading.Event()
self.__bindRetries = 0
self.__commandsQueue = FastQueue(self.__conf.commandsQueueSize)
if not self.__conf.appendEntriesUseBatch:
self.__pipeNotifier = PipeNotifier(self._poller)
self.__nodes = []
self.__readonlyNodes = []
self.__readonlyNodesCounter = 0
self.__lastReadonlyCheck = 0
self.__newAppendEntriesTime = 0
self.__commandsWaitingCommit = collections.defaultdict(list) # logID => [(termID, callback), ...]
self.__commandsLocalCounter = 0
self.__commandsWaitingReply = {} # commandLocalCounter => callback
self.__properies = set()
for key in self.__dict__:
self.__properies.add(key)
if self.__conf.autoTick:
self.__mainThread = threading.current_thread()
self.__initialised = threading.Event()
self.__thread = threading.Thread(target=SyncObj._autoTickThread, args=(weakref.proxy(self),))
self.__thread.start()
self.__initialised.wait()
# while not self.__initialised.is_set():
# pass
else:
self.__initInTickThread()
class SyncObj(object):
def __init__(self, selfNodeAddr, otherNodesAddrs, conf=None):
"""
Main SyncObj class, you should inherit your own class from it.
:param selfNodeAddr: address of the current node server, 'host:port'
:type selfNodeAddr: str
:param otherNodesAddrs: addresses of partner nodes, ['host1:port1', 'host2:port2', ...]
:type otherNodesAddrs: list of str
:param conf: configuration object
:type conf: SyncObjConf
"""
if conf is None:
self.__conf = SyncObjConf()
else:
self.__conf = conf
self.__conf.validate()
if self.__conf.password is not None:
if not HAS_CRYPTO:
raise ImportError("Please install 'cryptography' module")
self.__encryptor = getEncryptor(self.__conf.password)
else:
self.__encryptor = None
self.__selfNodeAddr = selfNodeAddr
self.__otherNodesAddrs = otherNodesAddrs
self.__unknownConnections = {} # descr => _Connection
self.__raftState = _RAFT_STATE.FOLLOWER
self.__raftCurrentTerm = 0
self.__votedFor = None
self.__votesCount = 0
self.__raftLeader = None
self.__raftElectionDeadline = time.time() + self.__generateRaftTimeout()
self.__raftLog = createJournal(self.__conf.journalFile)
if len(self.__raftLog) == 0:
self.__raftLog.add(_bchr(_COMMAND_TYPE.NO_OP), 1, self.__raftCurrentTerm)
self.__raftCommitIndex = 1
self.__raftLastApplied = 1
self.__raftNextIndex = {}
self.__raftMatchIndex = {}
self.__lastSerializedTime = time.time()
self.__lastSerializedEntry = None
self.__forceLogCompaction = False
self.__leaderCommitIndex = None
self.__onReadyCalled = False
self.__changeClusterIDx = None
self.__noopIDx = None
self.__destroying = False
self.__recvTransmission = ''
self.__startTime = time.time()
globalDnsResolver().setTimeouts(self.__conf.dnsCacheTime, self.__conf.dnsFailCacheTime)
self.__serializer = Serializer(self.__conf.fullDumpFile,
self.__conf.logCompactionBatchSize,
self.__conf.useFork,
self.__conf.serializer,
self.__conf.deserializer,
self.__conf.serializeChecker)
self.__isInitialized = False
self.__lastInitTryTime = 0
self._poller = createPoller(self.__conf.pollerType)
if selfNodeAddr is not None:
bindAddr = self.__conf.bindAddress or selfNodeAddr
host, port = bindAddr.split(':')
self.__server = TcpServer(self._poller, host, port, onNewConnection=self.__onNewConnection,
sendBufferSize=self.__conf.sendBufferSize,
recvBufferSize=self.__conf.recvBufferSize,
connectionTimeout=self.__conf.connectionTimeout)
self._methodToID = {}
self._idToMethod = {}
methods = sorted([m for m in dir(self) if callable(getattr(self, m))])
for i, method in enumerate(methods):
self._methodToID[method] = i
self._idToMethod[i] = getattr(self, method)
self.__thread = None
self.__mainThread = None
self.__initialised = None
self.__bindedEvent = threading.Event()
self.__bindRetries = 0
self.__commandsQueue = FastQueue(self.__conf.commandsQueueSize)
if not self.__conf.appendEntriesUseBatch:
self.__pipeNotifier = PipeNotifier(self._poller)
self.__nodes = []
self.__readonlyNodes = []
self.__readonlyNodesCounter = 0
self.__lastReadonlyCheck = 0
self.__newAppendEntriesTime = 0
self.__commandsWaitingCommit = collections.defaultdict(list) # logID => [(termID, callback), ...]
self.__commandsLocalCounter = 0
self.__commandsWaitingReply = {} # commandLocalCounter => callback
self.__properies = set()
for key in self.__dict__:
self.__properies.add(key)
if self.__conf.autoTick:
self.__mainThread = threading.current_thread()
self.__initialised = threading.Event()
self.__thread = threading.Thread(target=SyncObj._autoTickThread, args=(weakref.proxy(self),))
self.__thread.start()
self.__initialised.wait()
# while not self.__initialised.is_set():
# pass
else:
self.__initInTickThread()
def destroy(self):
"""
Correctly destroy SyncObj. Stop autoTickThread, close connections, etc.
"""
if self.__conf.autoTick:
self.__destroying = True
else:
self._doDestroy()
def waitBinded(self):
"""
Waits until initialized (binded port).
If success - just returns.
If failed to initialized after conf.maxBindRetries - raise SyncObjException.
"""
self.__bindedEvent.wait()
if not self.__isInitialized:
raise SyncObjException('BindError')
def _destroy(self):
self.destroy()
def _doDestroy(self):
for node in self.__nodes:
node._destroy()
for node in self.__readonlyNodes:
node._destroy()
if self.__selfNodeAddr is not None:
self.__server.unbind()
self.__raftLog._destroy()
def __initInTickThread(self):
try:
self.__lastInitTryTime = time.time()
if self.__selfNodeAddr is not None:
self.__server.bind()
shouldConnect = None
else:
shouldConnect = True
self.__nodes = []
for nodeAddr in self.__otherNodesAddrs:
self.__nodes.append(Node(self, nodeAddr, shouldConnect))
self.__raftNextIndex[nodeAddr] = self.__getCurrentLogIndex() + 1
self.__raftMatchIndex[nodeAddr] = 0
self.__needLoadDumpFile = True
self.__isInitialized = True
self.__bindedEvent.set()
except:
self.__bindRetries += 1
if self.__conf.maxBindRetries and self.__bindRetries >= self.__conf.maxBindRetries:
self.__bindedEvent.set()
raise SyncObjException('BindError')
logging.exception('failed to perform initialization')
def addNodeToCluster(self, nodeName, callback = None):
"""Add single node to cluster (dynamic membership changes). Async.
You should wait until node successfully added before adding
next node.
:param nodeName: nodeHost:nodePort
:type nodeName: str
:param callback: will be called on success or fail
:type callback: function(`FAIL_REASON <#pysyncobj.FAIL_REASON>`_, None)
"""
if not self.__conf.dynamicMembershipChange:
raise Exception('dynamicMembershipChange is disabled')
self._applyCommand(cPickle.dumps(['add', nodeName]), callback, _COMMAND_TYPE.MEMBERSHIP)
def removeNodeFromCluster(self, nodeName, callback = None):
"""Remove single node from cluster (dynamic membership changes). Async.
You should wait until node successfully added before adding
next node.
:param nodeName: nodeHost:nodePort
:type nodeName: str
:param callback: will be called on success or fail
:type callback: function(`FAIL_REASON <#pysyncobj.FAIL_REASON>`_, None)
"""
if not self.__conf.dynamicMembershipChange:
raise Exception('dynamicMembershipChange is disabled')
self._applyCommand(cPickle.dumps(['rem', nodeName]), callback, _COMMAND_TYPE.MEMBERSHIP)
def _addNodeToCluster(self, nodeName, callback=None):
self.addNodeToCluster(nodeName, callback)
def _removeNodeFromCluster(self, nodeName, callback=None):
self.removeNodeFromCluster(nodeName, callback)
def _applyCommand(self, command, callback, commandType = None):
try:
if commandType is None:
self.__commandsQueue.put_nowait((command, callback))
else:
self.__commandsQueue.put_nowait((_bchr(commandType) + command, callback))
if not self.__conf.appendEntriesUseBatch:
self.__pipeNotifier.notify()
except Queue.Full:
self.__callErrCallback(FAIL_REASON.QUEUE_FULL, callback)
def _checkCommandsToApply(self):
startTime = time.time()
while time.time() - startTime < self.__conf.appendEntriesPeriod:
if self.__raftLeader is None and self.__conf.commandsWaitLeader:
break
try:
command, callback = self.__commandsQueue.get_nowait()
except Queue.Empty:
break
requestNode, requestID = None, None
if isinstance(callback, tuple):
requestNode, requestID = callback
if self.__raftState == _RAFT_STATE.LEADER:
idx, term = self.__getCurrentLogIndex() + 1, self.__raftCurrentTerm
if self.__conf.dynamicMembershipChange:
changeClusterRequest = self.__parseChangeClusterRequest(command)
else:
changeClusterRequest = None
if changeClusterRequest is None or self.__changeCluster(changeClusterRequest):
self.__raftLog.add(command, idx, term)
if requestNode is None:
if callback is not None:
self.__commandsWaitingCommit[idx].append((term, callback))
else:
self.__send(requestNode, {
'type': 'apply_command_response',
'request_id': requestID,
'log_idx': idx,
'log_term': term,
})
if not self.__conf.appendEntriesUseBatch:
self.__sendAppendEntries()
else:
if requestNode is None:
if callback is not None:
callback(None, FAIL_REASON.REQUEST_DENIED)
else:
self.__send(requestNode, {
'type': 'apply_command_response',
'request_id': requestID,
'error': FAIL_REASON.REQUEST_DENIED,
})
elif self.__raftLeader is not None:
if requestNode is None:
message = {
'type': 'apply_command',
'command': command,
}
if callback is not None:
self.__commandsLocalCounter += 1
self.__commandsWaitingReply[self.__commandsLocalCounter] = callback
message['request_id'] = self.__commandsLocalCounter
self.__send(self.__raftLeader, message)
else:
self.__send(requestNode, {
'type': 'apply_command_response',
'request_id': requestID,
'error': FAIL_REASON.NOT_LEADER,
})
else:
self.__callErrCallback(FAIL_REASON.MISSING_LEADER, callback)
def _autoTickThread(self):
try:
self.__initInTickThread()
except SyncObjException as e:
if e.errorCode == 'BindError':
return
raise
finally:
self.__initialised.set()
time.sleep(0.1)
try:
while True:
if not self.__mainThread.is_alive():
break
if self.__destroying:
self._doDestroy()
break
self._onTick(self.__conf.autoTickPeriod)
except ReferenceError:
pass
def doTick(self, timeToWait=0.0):
"""Performs single tick. Should be called manually if `autoTick <#pysyncobj.SyncObjConf.autoTick>`_ disabled
:param timeToWait: max time to wait for next tick. If zero - perform single tick without waiting for new events.
Otherwise - wait for new socket event and return.
:type timeToWait: float
"""
assert not self.__conf.autoTick
self._onTick(timeToWait)
def _onTick(self, timeToWait=0.0):
if not self.__isInitialized:
if time.time() >= self.__lastInitTryTime + self.__conf.bindRetryTime:
self.__initInTickThread()
if not self.__isInitialized:
time.sleep(timeToWait)
return
if self.__needLoadDumpFile:
if self.__conf.fullDumpFile is not None and os.path.isfile(self.__conf.fullDumpFile):
self.__loadDumpFile(clearJournal=False)
self.__needLoadDumpFile = False
if self.__raftState in (_RAFT_STATE.FOLLOWER, _RAFT_STATE.CANDIDATE) and self.__selfNodeAddr is not None:
if self.__raftElectionDeadline < time.time() and self.__connectedToAnyone():
self.__raftElectionDeadline = time.time() + self.__generateRaftTimeout()
self.__raftLeader = None
self.__raftState = _RAFT_STATE.CANDIDATE
self.__raftCurrentTerm += 1
self.__votedFor = self._getSelfNodeAddr()
self.__votesCount = 1
for node in self.__nodes:
node.send({
'type': 'request_vote',
'term': self.__raftCurrentTerm,
'last_log_index': self.__getCurrentLogIndex(),
'last_log_term': self.__getCurrentLogTerm(),
})
self.__onLeaderChanged()
if self.__votesCount > (len(self.__nodes) + 1) / 2:
self.__onBecomeLeader()
if self.__raftState == _RAFT_STATE.LEADER:
while self.__raftCommitIndex < self.__getCurrentLogIndex():
nextCommitIndex = self.__raftCommitIndex + 1
count = 1
for node in self.__nodes:
if self.__raftMatchIndex[node.getAddress()] >= nextCommitIndex:
count += 1
if count > (len(self.__nodes) + 1) / 2:
self.__raftCommitIndex = nextCommitIndex
else:
break
self.__leaderCommitIndex = self.__raftCommitIndex
needSendAppendEntries = False
if self.__raftCommitIndex > self.__raftLastApplied:
count = self.__raftCommitIndex - self.__raftLastApplied
entries = self.__getEntries(self.__raftLastApplied + 1, count)
for entry in entries:
currentTermID = entry[2]
subscribers = self.__commandsWaitingCommit.pop(entry[1], [])
res = self.__doApplyCommand(entry[0])
for subscribeTermID, callback in subscribers:
if subscribeTermID == currentTermID:
callback(res, FAIL_REASON.SUCCESS)
else:
callback(None, FAIL_REASON.DISCARDED)
self.__raftLastApplied += 1
if not self.__conf.appendEntriesUseBatch:
needSendAppendEntries = True
if self.__raftState == _RAFT_STATE.LEADER:
if time.time() > self.__newAppendEntriesTime or needSendAppendEntries:
self.__sendAppendEntries()
if not self.__onReadyCalled and self.__raftLastApplied == self.__leaderCommitIndex:
if self.__conf.onReady:
self.__conf.onReady()
self.__onReadyCalled = True
self._checkCommandsToApply()
self.__tryLogCompaction()
for node in self.__nodes:
node.connectIfRequired()
if time.time() > self.__lastReadonlyCheck + 1.0:
self.__lastReadonlyCheck = time.time()
newReadonlyNodes = []
for node in self.__readonlyNodes:
if node.isConnected():
newReadonlyNodes.append(node)
else:
self.__raftNextIndex.pop(node, None)
self.__raftMatchIndex.pop(node, None)
node._destroy()
self._poller.poll(timeToWait)
def getStatus(self):
"""Dumps different debug info about cluster to list and return it"""
status = []
status.append(('version', VERSION))
status.append(('revision', REVISION))
status.append(('self', self.__selfNodeAddr))
status.append(('state' , self.__raftState))
status.append(('leader', self.__raftLeader))
status.append(('partner_nodes_count' , len(self.__nodes)))
for n in self.__nodes:
status.append(('partner_node_status_server_'+n.getAddress(), n.getStatus()))
status.append(('readonly_nodes_count', len(self.__readonlyNodes)))
for n in self.__readonlyNodes:
status.append(('readonly_node_status_server_'+n.getAddress(), n.getStatus()))
status.append(('unknown_connections_count', len(self.__unknownConnections)))
status.append(('log_len', len(self.__raftLog)))
status.append(('last_applied', self.__raftLastApplied))
status.append(('commit_idx', self.__raftCommitIndex))
status.append(('raft_term', self.__raftCurrentTerm))
status.append(('next_node_idx_count', len(self.__raftNextIndex)))
for k, v in self.__raftNextIndex.iteritems():
status.append(('next_node_idx_server_'+k, v))
status.append(('match_idx_count', len(self.__raftMatchIndex)))
for k, v in self.__raftMatchIndex.iteritems():
status.append(('match_idx_server_'+k, v))
status.append(('leader_commit_idx', self.__leaderCommitIndex))
status.append(('uptime', int(time.time() - self.__startTime)))
return status
def _getStatus(self):
return self.getStatus()
def printStatus(self):
"""Dumps different debug info about cluster to default logger"""
status = self.getStatus()
for i in status:
logging.info(i[0]+': %s', str(i[1]))
def _printStatus(self):
self.printStatus()
def forceLogCompaction(self):
"""Force to start log compaction (without waiting required time or required number of entries)"""
self.__forceLogCompaction = True
def _forceLogCompaction(self):
self.forceLogCompaction()
def __doApplyCommand(self, command):
commandType = ord(command[:1])
# Skip no-op and membership change commands
if commandType != _COMMAND_TYPE.REGULAR:
return
command = cPickle.loads(command[1:])
args = []
kwargs = {
'_doApply': True,
}
if not isinstance(command, tuple):
funcID = command
elif len(command) == 2:
funcID, args = command
else:
funcID, args, newKwArgs = command
kwargs.update(newKwArgs)
return self._idToMethod[funcID](*args, **kwargs)
def _onMessageReceived(self, nodeAddr, message):
if message['type'] == 'request_vote' and self.__selfNodeAddr is not None:
if message['term'] > self.__raftCurrentTerm:
self.__raftCurrentTerm = message['term']
self.__votedFor = None
self.__raftState = _RAFT_STATE.FOLLOWER
self.__raftLeader = None
if self.__raftState in (_RAFT_STATE.FOLLOWER, _RAFT_STATE.CANDIDATE):
lastLogTerm = message['last_log_term']
lastLogIdx = message['last_log_index']
if message['term'] >= self.__raftCurrentTerm:
if lastLogTerm < self.__getCurrentLogTerm():
return
if lastLogTerm == self.__getCurrentLogTerm() and \
lastLogIdx < self.__getCurrentLogIndex():
return
if self.__votedFor is not None:
return
self.__votedFor = nodeAddr
self.__raftElectionDeadline = time.time() + self.__generateRaftTimeout()
self.__send(nodeAddr, {
'type': 'response_vote',
'term': message['term'],
})
if message['type'] == 'append_entries' and message['term'] >= self.__raftCurrentTerm:
self.__raftElectionDeadline = time.time() + self.__generateRaftTimeout()
if self.__raftLeader != nodeAddr:
self.__onLeaderChanged()
self.__raftLeader = nodeAddr
if message['term'] > self.__raftCurrentTerm:
self.__raftCurrentTerm = message['term']
self.__votedFor = None
self.__raftState = _RAFT_STATE.FOLLOWER
newEntries = message.get('entries', [])
serialized = message.get('serialized', None)
self.__leaderCommitIndex = leaderCommitIndex = message['commit_index']
# Regular append entries
if 'prevLogIdx' in message:
transmission = message.get('transmission', None)
if transmission is not None:
if transmission == 'start':
self.__recvTransmission = message['data']
return
elif transmission == 'process':
self.__recvTransmission += message['data']
return
elif transmission == 'finish':
self.__recvTransmission += message['data']
newEntries = [cPickle.loads(self.__recvTransmission)]
self.__recvTransmission = ''
else:
raise Exception('Wrong transmission type')
prevLogIdx = message['prevLogIdx']
prevLogTerm = message['prevLogTerm']
prevEntries = self.__getEntries(prevLogIdx)
if not prevEntries:
self.__sendNextNodeIdx(nodeAddr, success=False, reset=True)
return
if prevEntries[0][2] != prevLogTerm:
self.__sendNextNodeIdx(nodeAddr, nextNodeIdx = prevLogIdx, success = False, reset=True)
return
if len(prevEntries) > 1:
# rollback cluster changes
if self.__conf.dynamicMembershipChange:
for entry in reversed(prevEntries[1:]):
clusterChangeRequest = self.__parseChangeClusterRequest(entry[0])
if clusterChangeRequest is not None:
self.__doChangeCluster(clusterChangeRequest, reverse=True)
self.__deleteEntriesFrom(prevLogIdx + 1)
for entry in newEntries:
self.__raftLog.add(*entry)
# apply cluster changes
if self.__conf.dynamicMembershipChange:
for entry in newEntries:
clusterChangeRequest = self.__parseChangeClusterRequest(entry[0])
if clusterChangeRequest is not None:
self.__doChangeCluster(clusterChangeRequest)
nextNodeIdx = prevLogIdx + 1
if newEntries:
nextNodeIdx = newEntries[-1][1]
self.__sendNextNodeIdx(nodeAddr, nextNodeIdx=nextNodeIdx, success=True)
# Install snapshot
elif serialized is not None:
if self.__serializer.setTransmissionData(serialized):
self.__loadDumpFile(clearJournal=True)
self.__sendNextNodeIdx(nodeAddr, success=True)
self.__raftCommitIndex = min(leaderCommitIndex, self.__getCurrentLogIndex())
if message['type'] == 'apply_command':
if 'request_id' in message:
self._applyCommand(message['command'], (nodeAddr, message['request_id']))
else:
self._applyCommand(message['command'], None)
if message['type'] == 'apply_command_response':
requestID = message['request_id']
error = message.get('error', None)
callback = self.__commandsWaitingReply.pop(requestID, None)
if callback is not None:
if error is not None:
callback(None, error)
else:
idx = message['log_idx']
term = message['log_term']
assert idx > self.__raftLastApplied
self.__commandsWaitingCommit[idx].append((term, callback))
if self.__raftState == _RAFT_STATE.CANDIDATE:
if message['type'] == 'response_vote' and message['term'] == self.__raftCurrentTerm:
self.__votesCount += 1
if self.__votesCount > (len(self.__nodes) + 1) / 2:
self.__onBecomeLeader()
if self.__raftState == _RAFT_STATE.LEADER:
if message['type'] == 'next_node_idx':
reset = message['reset']
nextNodeIdx = message['next_node_idx']
success = message['success']
currentNodeIdx = nextNodeIdx - 1
if reset:
self.__raftNextIndex[nodeAddr] = nextNodeIdx
if success:
self.__raftMatchIndex[nodeAddr] = currentNodeIdx
def __callErrCallback(self, err, callback):
if callback is None:
return
if isinstance(callback, tuple):
requestNode, requestID = callback
self.__send(requestNode, {
'type': 'apply_command_response',
'request_id': requestID,
'error': err,
})
return
callback(None, err)
def __sendNextNodeIdx(self, nodeAddr, reset=False, nextNodeIdx = None, success = False):
if nextNodeIdx is None:
nextNodeIdx = self.__getCurrentLogIndex() + 1
self.__send(nodeAddr, {
'type': 'next_node_idx',
'next_node_idx': nextNodeIdx,
'reset': reset,
'success': success,
})
def __generateRaftTimeout(self):
minTimeout = self.__conf.raftMinTimeout
maxTimeout = self.__conf.raftMaxTimeout
return minTimeout + (maxTimeout - minTimeout) * random.random()
def __onNewConnection(self, conn):
descr = conn.fileno()
self.__unknownConnections[descr] = conn
if self.__encryptor:
conn.encryptor = self.__encryptor
conn.setOnMessageReceivedCallback(functools.partial(self.__onMessageReceived, conn))
conn.setOnDisconnectedCallback(functools.partial(self.__onDisconnected, conn))
def __utilityCallback(self, res, err, conn, cmd, node):
cmdResult = 'FAIL'
if err == FAIL_REASON.SUCCESS:
cmdResult = 'SUCCESS'
conn.send(cmdResult + ' ' + cmd + ' ' + node)
def __onUtilityMessage(self, conn, message):
if message[0] == 'status':
status = self.getStatus()
data = ''
for i in status:
data += i[0] + ':' + str(i[1]) + '\n'
conn.send(data)
return True
elif message[0] == 'add':
self.addNodeToCluster(message[1], callback=functools.partial(self.__utilityCallback, conn=conn, cmd='ADD', node=message[1]))
return True
elif message[0] == 'remove':
if message[1] == self.__selfNodeAddr:
conn.send('FAIL REMOVE ' + message[1])
else:
self.removeNodeFromCluster(message[1], callback=functools.partial(self.__utilityCallback, conn=conn, cmd='REMOVE', node=message[1]))
return True
return False
def __onMessageReceived(self, conn, message):
if self.__encryptor and not conn.sendRandKey:
conn.sendRandKey = message
conn.recvRandKey = os.urandom(32)
conn.send(conn.recvRandKey)
return
descr = conn.fileno()
if isinstance(message, list) and self.__onUtilityMessage(conn, message):
self.__unknownConnections.pop(descr, None)
return
partnerNode = None
for node in self.__nodes:
if node.getAddress() == message:
partnerNode = node
break
if partnerNode is None and message != 'readonly':
conn.disconnect()
self.__unknownConnections.pop(descr, None)
return
if partnerNode is not None:
partnerNode.onPartnerConnected(conn)
else:
nodeAddr = str(self.__readonlyNodesCounter)
node = Node(self, nodeAddr, shouldConnect=False)
node.onPartnerConnected(conn)
self.__readonlyNodes.append(node)
self.__raftNextIndex[nodeAddr] = self.__getCurrentLogIndex() + 1
self.__raftMatchIndex[nodeAddr] = 0
self.__readonlyNodesCounter += 1
self.__unknownConnections.pop(descr, None)
def __onDisconnected(self, conn):
self.__unknownConnections.pop(conn.fileno(), None)
def __getCurrentLogIndex(self):
return self.__raftLog[-1][1]
def __getCurrentLogTerm(self):
return self.__raftLog[-1][2]
def __getPrevLogIndexTerm(self, nextNodeIndex):
prevIndex = nextNodeIndex - 1
entries = self.__getEntries(prevIndex, 1)
if entries:
return prevIndex, entries[0][2]
return None, None
def __getEntries(self, fromIDx, count=None, maxSizeBytes = None):
firstEntryIDx = self.__raftLog[0][1]
if fromIDx is None or fromIDx < firstEntryIDx:
return []
diff = fromIDx - firstEntryIDx
if count is None:
result = self.__raftLog[diff:]
else:
result = self.__raftLog[diff:diff + count]
if maxSizeBytes is None:
return result
totalSize = 0
i = 0
for i, entry in enumerate(result):
totalSize += len(entry[0])
if totalSize >= maxSizeBytes:
break
return result[:i + 1]
def _isLeader(self):
return self.__raftState == _RAFT_STATE.LEADER
def _getLeader(self):
return self.__raftLeader
def isReady(self):
"""Check if current node is initially synced with others and has an actual data.
:return: True if ready, False otherwise
:rtype: bool
"""
return self.__onReadyCalled
def _isReady(self):
return self.isReady()
def _getTerm(self):
return self.__raftCurrentTerm
def _getRaftLogSize(self):
return len(self.__raftLog)
def __deleteEntriesFrom(self, fromIDx):
firstEntryIDx = self.__raftLog[0][1]
diff = fromIDx - firstEntryIDx
if diff < 0:
return
self.__raftLog.deleteEntriesFrom(diff)
def __deleteEntriesTo(self, toIDx):
firstEntryIDx = self.__raftLog[0][1]
diff = toIDx - firstEntryIDx
if diff < 0:
return
self.__raftLog.deleteEntriesTo(diff)
def __onBecomeLeader(self):
self.__raftLeader = self.__selfNodeAddr
self.__raftState = _RAFT_STATE.LEADER
for node in self.__nodes + self.__readonlyNodes:
nodeAddr = node.getAddress()
self.__raftNextIndex[nodeAddr] = self.__getCurrentLogIndex() + 1
self.__raftMatchIndex[nodeAddr] = 0
# No-op command after leader election.
idx, term = self.__getCurrentLogIndex() + 1, self.__raftCurrentTerm
self.__raftLog.add(_bchr(_COMMAND_TYPE.NO_OP), idx, term)
self.__noopIDx = idx
if not self.__conf.appendEntriesUseBatch:
self.__sendAppendEntries()
self.__sendAppendEntries()
def __onLeaderChanged(self):
for id in sorted(self.__commandsWaitingReply):
self.__commandsWaitingReply[id](None, FAIL_REASON.LEADER_CHANGED)
self.__commandsWaitingReply = {}
def __sendAppendEntries(self):
self.__newAppendEntriesTime = time.time() + self.__conf.appendEntriesPeriod
startTime = time.time()
batchSizeBytes = self.__conf.appendEntriesBatchSizeBytes
for node in self.__nodes + self.__readonlyNodes:
nodeAddr = node.getAddress()
if not node.isConnected():
self.__serializer.cancelTransmisstion(nodeAddr)
continue
sendSingle = True
sendingSerialized = False
nextNodeIndex = self.__raftNextIndex[nodeAddr]
while nextNodeIndex <= self.__getCurrentLogIndex() or sendSingle or sendingSerialized:
if nextNodeIndex > self.__raftLog[0][1]:
prevLogIdx, prevLogTerm = self.__getPrevLogIndexTerm(nextNodeIndex)
entries = []
if nextNodeIndex <= self.__getCurrentLogIndex():
entries = self.__getEntries(nextNodeIndex, None, batchSizeBytes)
self.__raftNextIndex[nodeAddr] = entries[-1][1] + 1
if len(entries) == 1 and len(entries[0][0]) >= batchSizeBytes:
entry = cPickle.dumps(entries[0], -1)
for pos in xrange(0, len(entry), batchSizeBytes):
currData = entry[pos:pos + batchSizeBytes]
if pos == 0:
transmission = 'start'
elif pos + batchSizeBytes >= len(entries[0][0]):
transmission = 'finish'
else:
transmission = 'process'
message = {
'type': 'append_entries',
'transmission': transmission,
'data': currData,
'term': self.__raftCurrentTerm,
'commit_index': self.__raftCommitIndex,
'prevLogIdx': prevLogIdx,
'prevLogTerm': prevLogTerm,
}
node.send(message)
else:
message = {
'type': 'append_entries',
'term': self.__raftCurrentTerm,
'commit_index': self.__raftCommitIndex,
'entries': entries,
'prevLogIdx': prevLogIdx,
'prevLogTerm': prevLogTerm,
}
node.send(message)
else:
transmissionData = self.__serializer.getTransmissionData(nodeAddr)
message = {
'type': 'append_entries',
'term': self.__raftCurrentTerm,
'commit_index': self.__raftCommitIndex,
'serialized': transmissionData,
}
node.send(message)
if transmissionData is not None:
isLast = transmissionData[2]
if isLast:
self.__raftNextIndex[nodeAddr] = self.__raftLog[1][1] + 1
sendingSerialized = False
else:
sendingSerialized = True
else:
sendingSerialized = False
nextNodeIndex = self.__raftNextIndex[nodeAddr]
sendSingle = False
delta = time.time() - startTime
if delta > self.__conf.appendEntriesPeriod:
break
def __send(self, nodeAddr, message):
for node in self.__nodes + self.__readonlyNodes:
if node.getAddress() == nodeAddr:
node.send(message)
break
def __connectedToAnyone(self):
for node in self.__nodes:
if node.getStatus() == NODE_STATUS.CONNECTED:
return True
if not self.__nodes:
return True
return False
def _getSelfNodeAddr(self):
return self.__selfNodeAddr
def _getConf(self):
return self.__conf
def _getEncryptor(self):
return self.__encryptor
def __changeCluster(self, request):
if self.__raftLastApplied < self.__noopIDx:
# No-op entry was not commited yet
return False
if self.__changeClusterIDx is not None:
if self.__raftLastApplied >= self.__changeClusterIDx:
self.__changeClusterIDx = None
# Previous cluster change request was not commited yet
if self.__changeClusterIDx is not None:
return False
return self.__doChangeCluster(request)
def __doChangeCluster(self, request, reverse = False):
requestType = request[0]
requestNode = request[1]
if requestType == 'add':
adding = not reverse
elif requestType == 'rem':
adding = reverse
else:
return False
if self.__selfNodeAddr is not None:
shouldConnect = None
else:
shouldConnect = True
if adding:
newNode = requestNode
# Node already exists in cluster
if newNode == self.__selfNodeAddr or newNode in self.__otherNodesAddrs:
return False
self.__otherNodesAddrs.append(newNode)
self.__nodes.append(Node(self, newNode, shouldConnect))
self.__raftNextIndex[newNode] = self.__getCurrentLogIndex() + 1
self.__raftMatchIndex[newNode] = 0
return True
else:
oldNode = requestNode
if oldNode == self.__selfNodeAddr:
return False
if oldNode not in self.__otherNodesAddrs:
return False
for i in xrange(len(self.__nodes)):
if self.__nodes[i].getAddress() == oldNode:
self.__nodes[i]._destroy()
self.__nodes.pop(i)
self.__otherNodesAddrs.pop(i)
del self.__raftNextIndex[oldNode]
del self.__raftMatchIndex[oldNode]
return True
return False
def __parseChangeClusterRequest(self, command):
commandType = ord(command[:1])
if commandType != _COMMAND_TYPE.MEMBERSHIP:
return None
return cPickle.loads(command[1:])
def __tryLogCompaction(self):
currTime = time.time()
serializeState, serializeID = self.__serializer.checkSerializing()
if serializeState == SERIALIZER_STATE.SUCCESS:
self.__lastSerializedTime = currTime
self.__deleteEntriesTo(serializeID)
self.__lastSerializedEntry = serializeID
if serializeState == SERIALIZER_STATE.FAILED:
logging.warning('Failed to store full dump')
if serializeState != SERIALIZER_STATE.NOT_SERIALIZING:
return
if len(self.__raftLog) <= self.__conf.logCompactionMinEntries and \
currTime - self.__lastSerializedTime <= self.__conf.logCompactionMinTime and \
not self.__forceLogCompaction:
return
if self.__conf.logCompactionSplit:
allNodes = sorted(self.__otherNodesAddrs + [self.__selfNodeAddr])
nodesCount = len(allNodes)
selfIdx = allNodes.index(self.__selfNodeAddr)
interval = self.__conf.logCompactionMinTime
periodStart = int(currTime) / interval * interval
nodeInterval = float(interval) / nodesCount
nodeIntervalStart = periodStart + selfIdx * nodeInterval
nodeIntervalEnd = nodeIntervalStart + 0.3 * nodeInterval
if currTime < nodeIntervalStart or currTime >= nodeIntervalEnd:
return
self.__forceLogCompaction = False
lastAppliedEntries = self.__getEntries(self.__raftLastApplied - 1, 2)
if len(lastAppliedEntries) < 2 or lastAppliedEntries[0][1] == self.__lastSerializedEntry:
self.__lastSerializedTime = currTime
return
if self.__conf.serializer is None:
data = dict([(k, self.__dict__[k]) for k in self.__dict__.keys() if k not in self.__properies])
else:
data = None
cluster = self.__otherNodesAddrs + [self.__selfNodeAddr]
self.__serializer.serialize((data, lastAppliedEntries[1], lastAppliedEntries[0], cluster), lastAppliedEntries[0][1])
def __loadDumpFile(self, clearJournal):
try:
data = self.__serializer.deserialize()
if data[0] is not None:
for k, v in data[0].iteritems():
self.__dict__[k] = v
if clearJournal or \
len(self.__raftLog) < 2 or \
self.__raftLog[0] != data[2] or \
self.__raftLog[1] != data[1]:
self.__raftLog.clear()
self.__raftLog.add(*data[2])
self.__raftLog.add(*data[1])
self.__raftLastApplied = data[1][1]
if self.__conf.dynamicMembershipChange:
self.__otherNodesAddrs = [node for node in data[3] if node != self.__selfNodeAddr]
self.__updateClusterConfiguration()
except:
logging.exception('failed to load full dump')
def __updateClusterConfiguration(self):
currentNodes = set()
for i in xrange(len(self.__nodes) -1, -1, -1):
nodeAddr = self.__nodes[i].getAddress()
if nodeAddr not in self.__otherNodesAddrs:
self.__nodes[i]._destroy()
self.__nodes.pop(i)
else:
currentNodes.add(nodeAddr)
if self.__selfNodeAddr is not None:
shouldConnect = None
else:
shouldConnect = True
for nodeAddr in self.__otherNodesAddrs:
if nodeAddr not in currentNodes:
self.__nodes.append(Node(self, nodeAddr, shouldConnect))
self.__raftNextIndex[nodeAddr] = self.__getCurrentLogIndex() + 1
self.__raftMatchIndex[nodeAddr] = 0
"""
PyPepperChecker module
"""
from tcp_server import TcpServer
from network import Network
OUTPUT_SIZE = 3
INPUT_SIZE = 172 * 229 * 3
print("Building neural network...")
network = Network(INPUT_SIZE, OUTPUT_SIZE)
print "Processing images samples..."
network.process_images()
print "Training network..."
network.train()
print("Running TCP server...")
tcp_server = TcpServer(network)
tcp_server.run()
}
################################################################################
# Main #
################################################################################
script_name = os.path.basename(sys.argv[0])
if len(sys.argv) != 2:
print_usage(script_name)
sys.exit(-1)
service_name = sys.argv[1]
if service_name not in service_table:
print_usage(script_name)
sys.exit(-1)
try:
init_logger(log_file_name, log_max_bytes, log_backup_count)
logger = logging.getLogger()
logger.info(f'SW version: {sw_version}')
tcp_server = TcpServer("svc-server")
tcp_server.run(port, ClientHandlerFactory(service_table[service_name]))
except Exception as ex:
logger.error(traceback.format_exc())
finally:
logger.info('Server stopped.')
logging.shutdown()
def send_serial_client(arduino_conn, arduino_queue, running):
while running:
if not arduino_queue.empty():
arduino_conn.send(arduino_queue.get())
if __name__ == "__main__":
running = True
pc_queue = Queue.Queue()
android_queue = Queue.Queue()
arduino_queue = Queue.Queue()
pc_conn = TcpServer("192.168.7.1", 77)
android_conn = BtServer(4)
arduino_conn = SerialClient("/dev/ttyACM0", 9600)
# arduino_conn = SerialClient("/dev/ttyAMA0", 9600)
t1 = threading.Thread(target=run_tcp_server,
args=(pc_conn, android_queue, arduino_queue, running))
t2 = threading.Thread(target=run_bt_server,
args=(android_conn, pc_queue, arduino_queue, running))
t3 = threading.Thread(target=run_serial_client,
args=(arduino_conn, pc_queue, running))
t4 = threading.Thread(target=send_tcp_server,
args=(pc_conn, pc_queue, running))
t5 = threading.Thread(target=send_bt_server,
args=(android_conn, android_queue, running))
class SyncObj(object):
def __init__(self, selfNodeAddr, otherNodesAddrs, conf=None):
if conf is None:
self.__conf = SyncObjConf()
else:
self.__conf = conf
if self.__conf.password is not None:
if not HAS_CRYPTO:
raise ImportError("Please install 'cryptography' module")
self.__encryptor = getEncryptor(self.__conf.password)
else:
self.__encryptor = None
self.__selfNodeAddr = selfNodeAddr
self.__otherNodesAddrs = otherNodesAddrs
self.__unknownConnections = {} # descr => _Connection
self.__raftState = _RAFT_STATE.FOLLOWER
self.__raftCurrentTerm = 0
self.__votedFor = None
self.__votesCount = 0
self.__raftLeader = None
self.__raftElectionDeadline = time.time() + self.__generateRaftTimeout()
self.__raftLog = [] # (command, logID, term)
self.__raftLog.append(('', 1, self.__raftCurrentTerm))
self.__raftCommitIndex = 1
self.__raftLastApplied = 1
self.__raftNextIndex = {}
self.__raftMatchIndex = {}
self.__lastSerializedTime = time.time()
self.__forceLogCompaction = False
self.__leaderCommitIndex = None
self.__onReadyCalled = False
self.__startTime = time.time()
globalDnsResolver().setTimeouts(self.__conf.dnsCacheTime, self.__conf.dnsFailCacheTime)
self.__serializer = Serializer(self.__conf.fullDumpFile, self.__conf.logCompactionBatchSize)
self.__isInitialized = False
self.__lastInitTryTime = 0
self._poller = createPoller()
host, port = selfNodeAddr.split(':')
self.__server = TcpServer(self._poller, host, port, onNewConnection=self.__onNewConnection,
sendBufferSize=self.__conf.sendBufferSize,
recvBufferSize=self.__conf.recvBufferSize,
connectionTimeout=self.__conf.connectionTimeout)
self._methodToID = {}
self._idToMethod = {}
methods = sorted([m for m in dir(self) if callable(getattr(self, m))])
for i, method in enumerate(methods):
self._methodToID[method] = i
self._idToMethod[i] = getattr(self, method)
self.__thread = None
self.__mainThread = None
self.__initialised = None
self.__commandsQueue = Queue.Queue(self.__conf.commandsQueueSize)
self.__nodes = []
self.__newAppendEntriesTime = 0
self.__commandsWaitingCommit = collections.defaultdict(list) # logID => [(termID, callback), ...]
self.__commandsLocalCounter = 0
self.__commandsWaitingReply = {} # commandLocalCounter => callback
self.__properies = set()
for key in self.__dict__:
self.__properies.add(key)
if self.__conf.autoTick:
self.__mainThread = threading.current_thread()
self.__initialised = threading.Event()
self.__thread = threading.Thread(target=SyncObj._autoTickThread, args=(weakref.proxy(self),))
self.__thread.start()
while not self.__initialised.is_set():
pass
else:
self.__initInTickThread()
def __initInTickThread(self):
try:
self.__lastInitTryTime = time.time()
self.__server.bind()
self.__nodes = []
for nodeAddr in self.__otherNodesAddrs:
self.__nodes.append(Node(self, nodeAddr))
self.__raftNextIndex[nodeAddr] = 0
self.__raftMatchIndex[nodeAddr] = 0
self.__needLoadDumpFile = True
self.__isInitialized = True
except:
LOG_CURRENT_EXCEPTION()
def _applyCommand(self, command, callback):
try:
self.__commandsQueue.put_nowait((command, callback))
except Queue.Full:
self.__callErrCallback(FAIL_REASON.QUEUE_FULL, callback)
def _checkCommandsToApply(self):
startTime = time.time()
while time.time() - startTime < self.__conf.appendEntriesPeriod:
if self.__raftLeader is None and self.__conf.commandsWaitLeader:
break
try:
command, callback = self.__commandsQueue.get_nowait()
except Queue.Empty:
break
requestNode, requestID = None, None
if isinstance(callback, tuple):
requestNode, requestID = callback
if self.__raftState == _RAFT_STATE.LEADER:
idx, term = self.__getCurrentLogIndex() + 1, self.__raftCurrentTerm
self.__raftLog.append((command, idx, term))
if requestNode is None:
if callback is not None:
self.__commandsWaitingCommit[idx].append((term, callback))
else:
self.__send(requestNode, {
'type': 'apply_command_response',
'request_id': requestID,
'log_idx': idx,
'log_term': term,
})
if not self.__conf.appendEntriesUseBatch:
self.__sendAppendEntries()
elif self.__raftLeader is not None:
if requestNode is None:
message = {
'type': 'apply_command',
'command': command,
}
if callback is not None:
self.__commandsLocalCounter += 1
self.__commandsWaitingReply[self.__commandsLocalCounter] = callback
message['request_id'] = self.__commandsLocalCounter
self.__send(self.__raftLeader, message)
else:
self.__send(requestNode, {
'type': 'apply_command_response',
'request_id': requestID,
'error': FAIL_REASON.NOT_LEADER,
})
else:
self.__callErrCallback(FAIL_REASON.MISSING_LEADER, callback)
def _autoTickThread(self):
self.__initInTickThread()
self.__initialised.set()
time.sleep(0.1)
try:
while True:
if not self.__mainThread.is_alive():
break
self._onTick(self.__conf.autoTickPeriod)
except ReferenceError:
pass
def _onTick(self, timeToWait=0.0):
if not self.__isInitialized:
if time.time() >= self.__lastInitTryTime + self.__conf.bindRetryTime:
self.__initInTickThread()
if not self.__isInitialized:
time.sleep(timeToWait)
return
if self.__needLoadDumpFile:
if self.__conf.fullDumpFile is not None and os.path.isfile(self.__conf.fullDumpFile):
self.__loadDumpFile()
self.__needLoadDumpFile = False
if self.__raftState in (_RAFT_STATE.FOLLOWER, _RAFT_STATE.CANDIDATE):
if self.__raftElectionDeadline < time.time() and self.__connectedToAnyone():
self.__raftElectionDeadline = time.time() + self.__generateRaftTimeout()
self.__raftLeader = None
self.__raftState = _RAFT_STATE.CANDIDATE
self.__raftCurrentTerm += 1
self.__votedFor = self._getSelfNodeAddr()
self.__votesCount = 1
for node in self.__nodes:
node.send({
'type': 'request_vote',
'term': self.__raftCurrentTerm,
'last_log_index': self.__getCurrentLogIndex(),
'last_log_term': self.__getCurrentLogTerm(),
})
self.__onLeaderChanged()
if self.__raftState == _RAFT_STATE.LEADER:
while self.__raftCommitIndex < self.__getCurrentLogIndex():
nextCommitIndex = self.__raftCommitIndex + 1
count = 1
for node in self.__nodes:
if self.__raftMatchIndex[node.getAddress()] >= nextCommitIndex:
count += 1
if count > (len(self.__nodes) + 1) / 2:
self.__raftCommitIndex = nextCommitIndex
else:
break
self.__leaderCommitIndex = self.__raftCommitIndex
if time.time() > self.__newAppendEntriesTime:
self.__sendAppendEntries()
if self.__raftCommitIndex > self.__raftLastApplied:
count = self.__raftCommitIndex - self.__raftLastApplied
entries = self.__getEntries(self.__raftLastApplied + 1, count)
for entry in entries:
currentTermID = entry[2]
subscribers = self.__commandsWaitingCommit.pop(entry[1], [])
res = self.__doApplyCommand(entry[0])
for subscribeTermID, callback in subscribers:
if subscribeTermID == currentTermID:
callback(res, FAIL_REASON.SUCCESS)
else:
callback(None, FAIL_REASON.DISCARDED)
self.__raftLastApplied += 1
if not self.__onReadyCalled and self.__raftLastApplied == self.__leaderCommitIndex:
if self.__conf.onReady:
self.__conf.onReady()
self.__onReadyCalled = True
self._checkCommandsToApply()
self.__tryLogCompaction()
for node in self.__nodes:
node.connectIfRequired()
self._poller.poll(timeToWait)
def _getLastCommitIndex(self):
return self.__raftCommitIndex
def _printStatus(self):
LOG_DEBUG('self', self.__selfNodeAddr)
LOG_DEBUG('state:', self.__raftState)
LOG_DEBUG('leader', self.__raftLeader)
LOG_DEBUG('partner nodes', len(self.__nodes))
for n in self.__nodes:
LOG_DEBUG(n.getAddress(), n.getStatus())
LOG_DEBUG('log len:', len(self.__raftLog))
LOG_DEBUG('log size bytes:', len(zlib.compress(cPickle.dumps(self.__raftLog, -1))))
LOG_DEBUG('last applied:', self.__raftLastApplied)
LOG_DEBUG('commit idx:', self.__raftCommitIndex)
LOG_DEBUG('raft term:', self.__raftCurrentTerm)
LOG_DEBUG('next node idx:', self.__raftNextIndex)
LOG_DEBUG('match idx:', self.__raftMatchIndex)
LOG_DEBUG('leader commit idx:', self.__leaderCommitIndex)
LOG_DEBUG('uptime:', int(time.time() - self.__startTime))
LOG_DEBUG('')
def _forceLogCompaction(self):
self.__forceLogCompaction = True
def __doApplyCommand(self, command):
# Skip no-op command
if command == '':
return
command = cPickle.loads(command)
args = []
kwargs = {
'_doApply': True,
}
if not isinstance(command, tuple):
funcID = command
elif len(command) == 2:
funcID, args = command
else:
funcID, args, newKwArgs = command
kwargs.update(newKwArgs)
return self._idToMethod[funcID](*args, **kwargs)
def _onMessageReceived(self, nodeAddr, message):
if message['type'] == 'request_vote':
if message['term'] > self.__raftCurrentTerm:
self.__raftCurrentTerm = message['term']
self.__votedFor = None
self.__raftState = _RAFT_STATE.FOLLOWER
self.__raftLeader = None
if self.__raftState in (_RAFT_STATE.FOLLOWER, _RAFT_STATE.CANDIDATE):
lastLogTerm = message['last_log_term']
lastLogIdx = message['last_log_index']
if message['term'] >= self.__raftCurrentTerm:
if lastLogTerm < self.__getCurrentLogTerm():
return
if lastLogTerm == self.__getCurrentLogTerm() and \
lastLogIdx < self.__getCurrentLogIndex():
return
if self.__votedFor is not None:
return
self.__votedFor = nodeAddr
self.__raftElectionDeadline = time.time() + self.__generateRaftTimeout()
self.__send(nodeAddr, {
'type': 'response_vote',
'term': message['term'],
})
if message['type'] == 'append_entries' and message['term'] >= self.__raftCurrentTerm:
self.__raftElectionDeadline = time.time() + self.__generateRaftTimeout()
if self.__raftLeader != nodeAddr:
self.__onLeaderChanged()
self.__raftLeader = nodeAddr
if message['term'] > self.__raftCurrentTerm:
self.__raftCurrentTerm = message['term']
self.__votedFor = None
self.__raftState = _RAFT_STATE.FOLLOWER
newEntries = message.get('entries', [])
serialized = message.get('serialized', None)
self.__leaderCommitIndex = leaderCommitIndex = message['commit_index']
# Regular append entries
if 'prevLogIdx' in message:
prevLogIdx = message['prevLogIdx']
prevLogTerm = message['prevLogTerm']
prevEntries = self.__getEntries(prevLogIdx)
if not prevEntries:
if prevLogIdx is None or self.__getCurrentLogIndex() is None:
nextNodeIdx = None
else:
nextNodeIdx = min(prevLogIdx, self.__getCurrentLogIndex())
self.__sendNextNodeIdx(nodeAddr, nextNodeIdx = nextNodeIdx, success = False, reset=True)
return
if prevEntries[0][2] != prevLogTerm:
self.__sendNextNodeIdx(nodeAddr, nextNodeIdx = prevLogIdx, success = False, reset=True)
return
if len(prevEntries) > 1:
self.__deleteEntriesFrom(prevLogIdx + 1)
self.__raftLog += newEntries
nextNodeIdx = prevLogIdx + 1
if newEntries:
nextNodeIdx = newEntries[-1][1]
self.__sendNextNodeIdx(nodeAddr, nextNodeIdx=nextNodeIdx, success=True)
# Install snapshot
elif serialized is not None:
if self.__serializer.setTransmissionData(serialized):
self.__loadDumpFile()
self.__sendNextNodeIdx(nodeAddr, success=True)
self.__raftCommitIndex = min(leaderCommitIndex, self.__getCurrentLogIndex())
if message['type'] == 'apply_command':
if 'request_id' in message:
self._applyCommand(message['command'], (nodeAddr, message['request_id']))
else:
self._applyCommand(message['command'], None)
if message['type'] == 'apply_command_response':
requestID = message['request_id']
error = message.get('error', None)
callback = self.__commandsWaitingReply.pop(requestID, None)
if callback is not None:
if error is not None:
callback(None, error)
else:
idx = message['log_idx']
term = message['log_term']
assert idx > self.__raftLastApplied
self.__commandsWaitingCommit[idx].append((term, callback))
if self.__raftState == _RAFT_STATE.CANDIDATE:
if message['type'] == 'response_vote' and message['term'] == self.__raftCurrentTerm:
self.__votesCount += 1
if self.__votesCount > (len(self.__nodes) + 1) / 2:
self.__onBecomeLeader()
if self.__raftState == _RAFT_STATE.LEADER:
if message['type'] == 'next_node_idx':
reset = message['reset']
nextNodeIdx = message['next_node_idx']
success = message['success']
currentNodeIdx = nextNodeIdx - 1
if reset:
self.__raftNextIndex[nodeAddr] = nextNodeIdx
if success:
self.__raftMatchIndex[nodeAddr] = currentNodeIdx
def __callErrCallback(self, err, callback):
if callback is None:
return
if isinstance(callback, tuple):
requestNode, requestID = callback
self.__send(requestNode, {
'type': 'apply_command_response',
'request_id': requestID,
'error': err,
})
return
callback(None, err)
def __sendNextNodeIdx(self, nodeAddr, reset=False, nextNodeIdx = None, success = False):
if nextNodeIdx is None:
nextNodeIdx = self.__getCurrentLogIndex() + 1
self.__send(nodeAddr, {
'type': 'next_node_idx',
'next_node_idx': nextNodeIdx,
'reset': reset,
'success': success,
})
def __generateRaftTimeout(self):
minTimeout = self.__conf.raftMinTimeout
maxTimeout = self.__conf.raftMaxTimeout
return minTimeout + (maxTimeout - minTimeout) * random.random()
def __onNewConnection(self, conn):
descr = conn.fileno()
self.__unknownConnections[descr] = conn
if self.__encryptor:
conn.encryptor = self.__encryptor
conn.setOnMessageReceivedCallback(functools.partial(self.__onMessageReceived, conn))
conn.setOnDisconnectedCallback(functools.partial(self.__onDisconnected, conn))
def __onMessageReceived(self, conn, message):
if self.__encryptor and not conn.sendRandKey:
conn.sendRandKey = message
conn.recvRandKey = os.urandom(32)
conn.send(conn.recvRandKey)
return
descr = conn.fileno()
partnerNode = None
for node in self.__nodes:
if node.getAddress() == message:
partnerNode = node
break
if partnerNode is None:
conn.disconnect()
return
partnerNode.onPartnerConnected(conn)
self.__unknownConnections.pop(descr, None)
def __onDisconnected(self, conn):
self.__unknownConnections.pop(conn.fileno(), None)
def __getCurrentLogIndex(self):
return self.__raftLog[-1][1]
def __getCurrentLogTerm(self):
return self.__raftLog[-1][2]
def __getPrevLogIndexTerm(self, nextNodeIndex):
prevIndex = nextNodeIndex - 1
entries = self.__getEntries(prevIndex, 1)
if entries:
return prevIndex, entries[0][2]
return None, None
def __getEntries(self, fromIDx, count=None, maxSizeBytes = None):
firstEntryIDx = self.__raftLog[0][1]
if fromIDx is None or fromIDx < firstEntryIDx:
return []
diff = fromIDx - firstEntryIDx
if count is None:
result = self.__raftLog[diff:]
else:
result = self.__raftLog[diff:diff + count]
if maxSizeBytes is None:
return result
totalSize = 0
i = 0
for i, entry in enumerate(result):
totalSize += len(entry[0])
if totalSize >= maxSizeBytes:
break
return result[:i + 1]
def _isLeader(self):
return self.__raftState == _RAFT_STATE.LEADER
def _getLeader(self):
return self.__raftLeader
def _isReady(self):
return self.__onReadyCalled
def _getTerm(self):
return self.__raftCurrentTerm
def _getRaftLogSize(self):
return len(self.__raftLog)
def __deleteEntriesFrom(self, fromIDx):
firstEntryIDx = self.__raftLog[0][1]
diff = fromIDx - firstEntryIDx
if diff < 0:
return
self.__raftLog = self.__raftLog[:diff]
def __deleteEntriesTo(self, toIDx):
firstEntryIDx = self.__raftLog[0][1]
diff = toIDx - firstEntryIDx
if diff < 0:
return
self.__raftLog = self.__raftLog[diff:]
def __onBecomeLeader(self):
self.__raftLeader = self.__selfNodeAddr
self.__raftState = _RAFT_STATE.LEADER
for node in self.__nodes:
nodeAddr = node.getAddress()
self.__raftNextIndex[nodeAddr] = self.__getCurrentLogIndex() + 1
self.__raftMatchIndex[nodeAddr] = 0
# No-op command after leader election.
self._applyCommand('', None)
self.__sendAppendEntries()
def __onLeaderChanged(self):
for id in sorted(self.__commandsWaitingReply):
self.__commandsWaitingReply[id](None, FAIL_REASON.LEADER_CHANGED)
self.__commandsWaitingReply = {}
def __sendAppendEntries(self):
self.__newAppendEntriesTime = time.time() + self.__conf.appendEntriesPeriod
startTime = time.time()
for node in self.__nodes:
nodeAddr = node.getAddress()
if not node.isConnected():
self.__serializer.cancelTransmisstion(nodeAddr)
continue
sendSingle = True
sendingSerialized = False
nextNodeIndex = self.__raftNextIndex[nodeAddr]
while nextNodeIndex <= self.__getCurrentLogIndex() or sendSingle or sendingSerialized:
if nextNodeIndex >= self.__raftLog[0][1]:
prevLogIdx, prevLogTerm = self.__getPrevLogIndexTerm(nextNodeIndex)
entries = []
if nextNodeIndex <= self.__getCurrentLogIndex():
entries = self.__getEntries(nextNodeIndex, None, self.__conf.appendEntriesBatchSizeBytes)
self.__raftNextIndex[nodeAddr] = entries[-1][1] + 1
message = {
'type': 'append_entries',
'term': self.__raftCurrentTerm,
'commit_index': self.__raftCommitIndex,
'entries': entries,
'prevLogIdx': prevLogIdx,
'prevLogTerm': prevLogTerm,
}
node.send(message)
else:
transmissionData = self.__serializer.getTransmissionData(nodeAddr)
message = {
'type': 'append_entries',
'term': self.__raftCurrentTerm,
'commit_index': self.__raftCommitIndex,
'serialized': transmissionData,
}
node.send(message)
if transmissionData is not None:
isLast = transmissionData[2]
if isLast:
self.__raftNextIndex[nodeAddr] = self.__raftLog[0][1]
sendingSerialized = False
else:
sendingSerialized = True
else:
sendingSerialized = False
nextNodeIndex = self.__raftNextIndex[nodeAddr]
sendSingle = False
delta = time.time() - startTime
if delta > self.__conf.appendEntriesPeriod:
break
def __send(self, nodeAddr, message):
for node in self.__nodes:
if node.getAddress() == nodeAddr:
node.send(message)
break
def __connectedToAnyone(self):
for node in self.__nodes:
if node.getStatus() == NODE_STATUS.CONNECTED:
return True
return False
def _getSelfNodeAddr(self):
return self.__selfNodeAddr
def _getConf(self):
return self.__conf
def _getEncryptor(self):
return self.__encryptor
def __tryLogCompaction(self):
currTime = time.time()
serializeState, serializeID = self.__serializer.checkSerializing()
if serializeState == SERIALIZER_STATE.SUCCESS:
self.__lastSerializedTime = currTime
self.__deleteEntriesTo(serializeID)
if serializeState == SERIALIZER_STATE.FAILED:
LOG_WARNING("Failed to store full dump")
if serializeState != SERIALIZER_STATE.NOT_SERIALIZING:
return
if len(self.__raftLog) <= self.__conf.logCompactionMinEntries and \
currTime - self.__lastSerializedTime <= self.__conf.logCompactionMinTime and \
not self.__forceLogCompaction:
return
self.__forceLogCompaction = False
lastAppliedEntries = self.__getEntries(self.__raftLastApplied - 1, 2)
if len(lastAppliedEntries) < 2:
return
data = dict([(k, self.__dict__[k]) for k in self.__dict__.keys() if k not in self.__properies])
self.__serializer.serialize((data, lastAppliedEntries[1], lastAppliedEntries[0]), lastAppliedEntries[0][1])
def __loadDumpFile(self):
try:
data = self.__serializer.deserialize()
for k, v in data[0].iteritems():
self.__dict__[k] = v
self.__raftLog = [data[2], data[1]]
self.__raftLastApplied = data[1][1]
except:
LOG_WARNING('Failed to load full dump')
LOG_CURRENT_EXCEPTION()
def __init__(self, selfNodeAddr, otherNodesAddrs, conf=None):
if conf is None:
self.__conf = SyncObjConf()
else:
self.__conf = conf
if self.__conf.password is not None:
if not HAS_CRYPTO:
raise ImportError("Please install 'cryptography' module")
self.__encryptor = getEncryptor(self.__conf.password)
else:
self.__encryptor = None
self.__selfNodeAddr = selfNodeAddr
self.__otherNodesAddrs = otherNodesAddrs
self.__unknownConnections = {} # descr => _Connection
self.__raftState = _RAFT_STATE.FOLLOWER
self.__raftCurrentTerm = 0
self.__votedFor = None
self.__votesCount = 0
self.__raftLeader = None
self.__raftElectionDeadline = time.time() + self.__generateRaftTimeout()
self.__raftLog = [] # (command, logID, term)
self.__raftLog.append(('', 1, self.__raftCurrentTerm))
self.__raftCommitIndex = 1
self.__raftLastApplied = 1
self.__raftNextIndex = {}
self.__raftMatchIndex = {}
self.__lastSerializedTime = time.time()
self.__forceLogCompaction = False
self.__leaderCommitIndex = None
self.__onReadyCalled = False
self.__startTime = time.time()
globalDnsResolver().setTimeouts(self.__conf.dnsCacheTime, self.__conf.dnsFailCacheTime)
self.__serializer = Serializer(self.__conf.fullDumpFile, self.__conf.logCompactionBatchSize)
self.__isInitialized = False
self.__lastInitTryTime = 0
self._poller = createPoller()
host, port = selfNodeAddr.split(':')
self.__server = TcpServer(self._poller, host, port, onNewConnection=self.__onNewConnection,
sendBufferSize=self.__conf.sendBufferSize,
recvBufferSize=self.__conf.recvBufferSize,
connectionTimeout=self.__conf.connectionTimeout)
self._methodToID = {}
self._idToMethod = {}
methods = sorted([m for m in dir(self) if callable(getattr(self, m))])
for i, method in enumerate(methods):
self._methodToID[method] = i
self._idToMethod[i] = getattr(self, method)
self.__thread = None
self.__mainThread = None
self.__initialised = None
self.__commandsQueue = Queue.Queue(self.__conf.commandsQueueSize)
self.__nodes = []
self.__newAppendEntriesTime = 0
self.__commandsWaitingCommit = collections.defaultdict(list) # logID => [(termID, callback), ...]
self.__commandsLocalCounter = 0
self.__commandsWaitingReply = {} # commandLocalCounter => callback
self.__properies = set()
for key in self.__dict__:
self.__properies.add(key)
if self.__conf.autoTick:
self.__mainThread = threading.current_thread()
self.__initialised = threading.Event()
self.__thread = threading.Thread(target=SyncObj._autoTickThread, args=(weakref.proxy(self),))
self.__thread.start()
while not self.__initialised.is_set():
pass
else:
self.__initInTickThread()
class SyncObj(object):
def __init__(self, selfNodeAddr, otherNodesAddrs, conf=None):
"""
Main SyncObj class, you should inherit your own class from it.
:param selfNodeAddr: address of the current node server, 'host:port'
:type selfNodeAddr: str
:param otherNodesAddrs: addresses of partner nodes, ['host1:port1', 'host2:port2', ...]
:type otherNodesAddrs: list of str
:param conf: configuration object
:type conf: SyncObjConf
"""
if conf is None:
self.__conf = SyncObjConf()
else:
self.__conf = conf
self.__conf.validate()
if self.__conf.password is not None:
if not HAS_CRYPTO:
raise ImportError("Please install 'cryptography' module")
self.__encryptor = getEncryptor(self.__conf.password)
else:
self.__encryptor = None
self.__selfNodeAddr = selfNodeAddr
self.__otherNodesAddrs = otherNodesAddrs
self.__unknownConnections = {} # descr => _Connection
self.__raftState = _RAFT_STATE.FOLLOWER
self.__raftCurrentTerm = 0
self.__votedFor = None
self.__votesCount = 0
self.__raftLeader = None
self.__raftElectionDeadline = time.time() + self.__generateRaftTimeout(
)
self.__raftLog = createJournal(self.__conf.journalFile)
if len(self.__raftLog) == 0:
self.__raftLog.add(_bchr(_COMMAND_TYPE.NO_OP), 1,
self.__raftCurrentTerm)
self.__raftCommitIndex = 1
self.__raftLastApplied = 1
self.__raftNextIndex = {}
self.__raftMatchIndex = {}
self.__lastSerializedTime = time.time()
self.__lastSerializedEntry = None
self.__forceLogCompaction = False
self.__leaderCommitIndex = None
self.__onReadyCalled = False
self.__changeClusterIDx = None
self.__noopIDx = None
self.__destroying = False
self.__recvTransmission = ''
self.__startTime = time.time()
globalDnsResolver().setTimeouts(self.__conf.dnsCacheTime,
self.__conf.dnsFailCacheTime)
self.__serializer = Serializer(self.__conf.fullDumpFile,
self.__conf.logCompactionBatchSize,
self.__conf.useFork,
self.__conf.serializer,
self.__conf.deserializer,
self.__conf.serializeChecker)
self.__isInitialized = False
self.__lastInitTryTime = 0
self._poller = createPoller(self.__conf.pollerType)
if selfNodeAddr is not None:
bindAddr = self.__conf.bindAddress or selfNodeAddr
host, port = bindAddr.split(':')
self.__server = TcpServer(
self._poller,
host,
port,
onNewConnection=self.__onNewConnection,
sendBufferSize=self.__conf.sendBufferSize,
recvBufferSize=self.__conf.recvBufferSize,
connectionTimeout=self.__conf.connectionTimeout)
self._methodToID = {}
self._idToMethod = {}
methods = sorted([m for m in dir(self) if callable(getattr(self, m))])
for i, method in enumerate(methods):
self._methodToID[method] = i
self._idToMethod[i] = getattr(self, method)
self.__thread = None
self.__mainThread = None
self.__initialised = None
self.__commandsQueue = FastQueue(self.__conf.commandsQueueSize)
if not self.__conf.appendEntriesUseBatch:
self.__pipeNotifier = PipeNotifier(self._poller)
self.__nodes = []
self.__readonlyNodes = []
self.__readonlyNodesCounter = 0
self.__lastReadonlyCheck = 0
self.__newAppendEntriesTime = 0
self.__commandsWaitingCommit = collections.defaultdict(
list) # logID => [(termID, callback), ...]
self.__commandsLocalCounter = 0
self.__commandsWaitingReply = {} # commandLocalCounter => callback
self.__properies = set()
for key in self.__dict__:
self.__properies.add(key)
if self.__conf.autoTick:
self.__mainThread = threading.current_thread()
self.__initialised = threading.Event()
self.__thread = threading.Thread(target=SyncObj._autoTickThread,
args=(weakref.proxy(self), ))
self.__thread.start()
while not self.__initialised.is_set():
pass
else:
self.__initInTickThread()
def destroy(self):
"""
Correctly destroy SyncObj. Stop autoTickThread, close connections, etc.
"""
if self.__conf.autoTick:
self.__destroying = True
else:
self._doDestroy()
def _destroy(self):
self.destroy()
def _doDestroy(self):
for node in self.__nodes:
node._destroy()
for node in self.__readonlyNodes:
node._destroy()
if self.__selfNodeAddr is not None:
self.__server.unbind()
self.__raftLog._destroy()
def __initInTickThread(self):
try:
self.__lastInitTryTime = time.time()
if self.__selfNodeAddr is not None:
self.__server.bind()
shouldConnect = None
else:
shouldConnect = True
self.__nodes = []
for nodeAddr in self.__otherNodesAddrs:
self.__nodes.append(Node(self, nodeAddr, shouldConnect))
self.__raftNextIndex[nodeAddr] = self.__getCurrentLogIndex(
) + 1
self.__raftMatchIndex[nodeAddr] = 0
self.__needLoadDumpFile = True
self.__isInitialized = True
except:
logging.exception('failed to perform initialization')
def addNodeToCluster(self, nodeName, callback=None):
"""Add single node to cluster (dynamic membership changes). Async.
You should wait until node successfully added before adding
next node.
:param nodeName: nodeHost:nodePort
:type nodeName: str
:param callback: will be called on success or fail
:type callback: function(`FAIL_REASON <#pysyncobj.FAIL_REASON>`_, None)
"""
if not self.__conf.dynamicMembershipChange:
raise Exception('dynamicMembershipChange is disabled')
self._applyCommand(cPickle.dumps(['add', nodeName]), callback,
_COMMAND_TYPE.MEMBERSHIP)
def removeNodeFromCluster(self, nodeName, callback=None):
"""Remove single node from cluster (dynamic membership changes). Async.
You should wait until node successfully added before adding
next node.
:param nodeName: nodeHost:nodePort
:type nodeName: str
:param callback: will be called on success or fail
:type callback: function(`FAIL_REASON <#pysyncobj.FAIL_REASON>`_, None)
"""
if not self.__conf.dynamicMembershipChange:
raise Exception('dynamicMembershipChange is disabled')
self._applyCommand(cPickle.dumps(['rem', nodeName]), callback,
_COMMAND_TYPE.MEMBERSHIP)
def _addNodeToCluster(self, nodeName, callback=None):
self.addNodeToCluster(nodeName, callback)
def _removeNodeFromCluster(self, nodeName, callback=None):
self.removeNodeFromCluster(nodeName, callback)
def _applyCommand(self, command, callback, commandType=None):
try:
if commandType is None:
self.__commandsQueue.put_nowait((command, callback))
else:
self.__commandsQueue.put_nowait(
(_bchr(commandType) + command, callback))
if not self.__conf.appendEntriesUseBatch:
self.__pipeNotifier.notify()
except Queue.Full:
self.__callErrCallback(FAIL_REASON.QUEUE_FULL, callback)
def _checkCommandsToApply(self):
startTime = time.time()
while time.time() - startTime < self.__conf.appendEntriesPeriod:
if self.__raftLeader is None and self.__conf.commandsWaitLeader:
break
try:
command, callback = self.__commandsQueue.get_nowait()
except Queue.Empty:
break
requestNode, requestID = None, None
if isinstance(callback, tuple):
requestNode, requestID = callback
if self.__raftState == _RAFT_STATE.LEADER:
idx, term = self.__getCurrentLogIndex(
) + 1, self.__raftCurrentTerm
if self.__conf.dynamicMembershipChange:
changeClusterRequest = self.__parseChangeClusterRequest(
command)
else:
changeClusterRequest = None
if changeClusterRequest is None or self.__changeCluster(
changeClusterRequest):
self.__raftLog.add(command, idx, term)
if requestNode is None:
if callback is not None:
self.__commandsWaitingCommit[idx].append(
(term, callback))
else:
self.__send(
requestNode, {
'type': 'apply_command_response',
'request_id': requestID,
'log_idx': idx,
'log_term': term,
})
if not self.__conf.appendEntriesUseBatch:
self.__sendAppendEntries()
else:
if requestNode is None:
if callback is not None:
callback(None, FAIL_REASON.REQUEST_DENIED)
else:
self.__send(
requestNode, {
'type': 'apply_command_response',
'request_id': requestID,
'error': FAIL_REASON.REQUEST_DENIED,
})
elif self.__raftLeader is not None:
if requestNode is None:
message = {
'type': 'apply_command',
'command': command,
}
if callback is not None:
self.__commandsLocalCounter += 1
self.__commandsWaitingReply[
self.__commandsLocalCounter] = callback
message['request_id'] = self.__commandsLocalCounter
self.__send(self.__raftLeader, message)
else:
self.__send(
requestNode, {
'type': 'apply_command_response',
'request_id': requestID,
'error': FAIL_REASON.NOT_LEADER,
})
else:
self.__callErrCallback(FAIL_REASON.MISSING_LEADER, callback)
def _autoTickThread(self):
self.__initInTickThread()
self.__initialised.set()
time.sleep(0.1)
try:
while True:
if not self.__mainThread.is_alive():
break
if self.__destroying:
self._doDestroy()
break
self._onTick(self.__conf.autoTickPeriod)
except ReferenceError:
pass
def doTick(self, timeToWait=0.0):
"""Performs single tick. Should be called manually if `autoTick <#pysyncobj.SyncObjConf.autoTick>`_ disabled
:param timeToWait: max time to wait for next tick. If zero - perform single tick without waiting for new events.
Otherwise - wait for new socket event and return.
:type timeToWait: float
"""
assert not self.__conf.autoTick
self._onTick(timeToWait)
def _onTick(self, timeToWait=0.0):
if not self.__isInitialized:
if time.time(
) >= self.__lastInitTryTime + self.__conf.bindRetryTime:
self.__initInTickThread()
if not self.__isInitialized:
time.sleep(timeToWait)
return
if self.__needLoadDumpFile:
if self.__conf.fullDumpFile is not None and os.path.isfile(
self.__conf.fullDumpFile):
self.__loadDumpFile(clearJournal=False)
self.__needLoadDumpFile = False
if self.__raftState in (_RAFT_STATE.FOLLOWER, _RAFT_STATE.CANDIDATE
) and self.__selfNodeAddr is not None:
if self.__raftElectionDeadline < time.time(
) and self.__connectedToAnyone():
self.__raftElectionDeadline = time.time(
) + self.__generateRaftTimeout()
self.__raftLeader = None
self.__raftState = _RAFT_STATE.CANDIDATE
self.__raftCurrentTerm += 1
self.__votedFor = self._getSelfNodeAddr()
self.__votesCount = 1
for node in self.__nodes:
node.send({
'type': 'request_vote',
'term': self.__raftCurrentTerm,
'last_log_index': self.__getCurrentLogIndex(),
'last_log_term': self.__getCurrentLogTerm(),
})
self.__onLeaderChanged()
if self.__votesCount > (len(self.__nodes) + 1) / 2:
self.__onBecomeLeader()
if self.__raftState == _RAFT_STATE.LEADER:
while self.__raftCommitIndex < self.__getCurrentLogIndex():
nextCommitIndex = self.__raftCommitIndex + 1
count = 1
for node in self.__nodes:
if self.__raftMatchIndex[
node.getAddress()] >= nextCommitIndex:
count += 1
if count > (len(self.__nodes) + 1) / 2:
self.__raftCommitIndex = nextCommitIndex
else:
break
self.__leaderCommitIndex = self.__raftCommitIndex
needSendAppendEntries = False
if self.__raftCommitIndex > self.__raftLastApplied:
count = self.__raftCommitIndex - self.__raftLastApplied
entries = self.__getEntries(self.__raftLastApplied + 1, count)
for entry in entries:
currentTermID = entry[2]
subscribers = self.__commandsWaitingCommit.pop(entry[1], [])
res = self.__doApplyCommand(entry[0])
for subscribeTermID, callback in subscribers:
if subscribeTermID == currentTermID:
callback(res, FAIL_REASON.SUCCESS)
else:
callback(None, FAIL_REASON.DISCARDED)
self.__raftLastApplied += 1
if not self.__conf.appendEntriesUseBatch:
needSendAppendEntries = True
if self.__raftState == _RAFT_STATE.LEADER:
if time.time(
) > self.__newAppendEntriesTime or needSendAppendEntries:
self.__sendAppendEntries()
if not self.__onReadyCalled and self.__raftLastApplied == self.__leaderCommitIndex:
if self.__conf.onReady:
self.__conf.onReady()
self.__onReadyCalled = True
self._checkCommandsToApply()
self.__tryLogCompaction()
for node in self.__nodes:
node.connectIfRequired()
if time.time() > self.__lastReadonlyCheck + 1.0:
self.__lastReadonlyCheck = time.time()
newReadonlyNodes = []
for node in self.__readonlyNodes:
if node.isConnected():
newReadonlyNodes.append(node)
else:
self.__raftNextIndex.pop(node, None)
self.__raftMatchIndex.pop(node, None)
node._destroy()
self._poller.poll(timeToWait)
def getStatus(self):
"""Dumps different debug info about cluster to list and return it"""
status = []
status.append(('version', VERSION))
status.append(('revision', REVISION))
status.append(('self', self.__selfNodeAddr))
status.append(('state', self.__raftState))
status.append(('leader', self.__raftLeader))
status.append(('partner_nodes_count', len(self.__nodes)))
for n in self.__nodes:
status.append(('partner_node_status_server_' + n.getAddress(),
n.getStatus()))
status.append(('readonly_nodes_count', len(self.__readonlyNodes)))
for n in self.__readonlyNodes:
status.append(('readonly_node_status_server_' + n.getAddress(),
n.getStatus()))
status.append(
('unknown_connections_count', len(self.__unknownConnections)))
status.append(('log_len', len(self.__raftLog)))
status.append(('last_applied', self.__raftLastApplied))
status.append(('commit_idx', self.__raftCommitIndex))
status.append(('raft_term', self.__raftCurrentTerm))
status.append(('next_node_idx_count', len(self.__raftNextIndex)))
for k, v in self.__raftNextIndex.iteritems():
status.append(('next_node_idx_server_' + k, v))
status.append(('match_idx_count', len(self.__raftMatchIndex)))
for k, v in self.__raftMatchIndex.iteritems():
status.append(('match_idx_server_' + k, v))
status.append(('leader_commit_idx', self.__leaderCommitIndex))
status.append(('uptime', int(time.time() - self.__startTime)))
return status
def _getStatus(self):
return self.getStatus()
def printStatus(self):
"""Dumps different debug info about cluster to default logger"""
status = self.getStatus()
for i in status:
logging.info(i[0] + ': %s', str(i[1]))
def _printStatus(self):
self.printStatus()
def forceLogCompaction(self):
"""Force to start log compaction (without waiting required time or required number of entries)"""
self.__forceLogCompaction = True
def _forceLogCompaction(self):
self.forceLogCompaction()
def __doApplyCommand(self, command):
commandType = ord(command[:1])
# Skip no-op and membership change commands
if commandType != _COMMAND_TYPE.REGULAR:
return
command = cPickle.loads(command[1:])
args = []
kwargs = {
'_doApply': True,
}
if not isinstance(command, tuple):
funcID = command
elif len(command) == 2:
funcID, args = command
else:
funcID, args, newKwArgs = command
kwargs.update(newKwArgs)
return self._idToMethod[funcID](*args, **kwargs)
def _onMessageReceived(self, nodeAddr, message):
if message[
'type'] == 'request_vote' and self.__selfNodeAddr is not None:
if message['term'] > self.__raftCurrentTerm:
self.__raftCurrentTerm = message['term']
self.__votedFor = None
self.__raftState = _RAFT_STATE.FOLLOWER
self.__raftLeader = None
if self.__raftState in (_RAFT_STATE.FOLLOWER,
_RAFT_STATE.CANDIDATE):
lastLogTerm = message['last_log_term']
lastLogIdx = message['last_log_index']
if message['term'] >= self.__raftCurrentTerm:
if lastLogTerm < self.__getCurrentLogTerm():
return
if lastLogTerm == self.__getCurrentLogTerm() and \
lastLogIdx < self.__getCurrentLogIndex():
return
if self.__votedFor is not None:
return
self.__votedFor = nodeAddr
self.__raftElectionDeadline = time.time(
) + self.__generateRaftTimeout()
self.__send(nodeAddr, {
'type': 'response_vote',
'term': message['term'],
})
if message['type'] == 'append_entries' and message[
'term'] >= self.__raftCurrentTerm:
self.__raftElectionDeadline = time.time(
) + self.__generateRaftTimeout()
if self.__raftLeader != nodeAddr:
self.__onLeaderChanged()
self.__raftLeader = nodeAddr
if message['term'] > self.__raftCurrentTerm:
self.__raftCurrentTerm = message['term']
self.__votedFor = None
self.__raftState = _RAFT_STATE.FOLLOWER
newEntries = message.get('entries', [])
serialized = message.get('serialized', None)
self.__leaderCommitIndex = leaderCommitIndex = message[
'commit_index']
# Regular append entries
if 'prevLogIdx' in message:
transmission = message.get('transmission', None)
if transmission is not None:
if transmission == 'start':
self.__recvTransmission = message['data']
return
elif transmission == 'process':
self.__recvTransmission += message['data']
return
elif transmission == 'finish':
self.__recvTransmission += message['data']
newEntries = [cPickle.loads(self.__recvTransmission)]
self.__recvTransmission = ''
else:
raise Exception('Wrong transmission type')
prevLogIdx = message['prevLogIdx']
prevLogTerm = message['prevLogTerm']
prevEntries = self.__getEntries(prevLogIdx)
if not prevEntries:
self.__sendNextNodeIdx(nodeAddr, success=False, reset=True)
return
if prevEntries[0][2] != prevLogTerm:
self.__sendNextNodeIdx(nodeAddr,
nextNodeIdx=prevLogIdx,
success=False,
reset=True)
return
if len(prevEntries) > 1:
# rollback cluster changes
if self.__conf.dynamicMembershipChange:
for entry in reversed(prevEntries[1:]):
clusterChangeRequest = self.__parseChangeClusterRequest(
entry[0])
if clusterChangeRequest is not None:
self.__doChangeCluster(clusterChangeRequest,
reverse=True)
self.__deleteEntriesFrom(prevLogIdx + 1)
for entry in newEntries:
self.__raftLog.add(*entry)
# apply cluster changes
if self.__conf.dynamicMembershipChange:
for entry in newEntries:
clusterChangeRequest = self.__parseChangeClusterRequest(
entry[0])
if clusterChangeRequest is not None:
self.__doChangeCluster(clusterChangeRequest)
nextNodeIdx = prevLogIdx + 1
if newEntries:
nextNodeIdx = newEntries[-1][1]
self.__sendNextNodeIdx(nodeAddr,
nextNodeIdx=nextNodeIdx,
success=True)
# Install snapshot
elif serialized is not None:
if self.__serializer.setTransmissionData(serialized):
self.__loadDumpFile(clearJournal=True)
self.__sendNextNodeIdx(nodeAddr, success=True)
self.__raftCommitIndex = min(leaderCommitIndex,
self.__getCurrentLogIndex())
if message['type'] == 'apply_command':
if 'request_id' in message:
self._applyCommand(message['command'],
(nodeAddr, message['request_id']))
else:
self._applyCommand(message['command'], None)
if message['type'] == 'apply_command_response':
requestID = message['request_id']
error = message.get('error', None)
callback = self.__commandsWaitingReply.pop(requestID, None)
if callback is not None:
if error is not None:
callback(None, error)
else:
idx = message['log_idx']
term = message['log_term']
assert idx > self.__raftLastApplied
self.__commandsWaitingCommit[idx].append((term, callback))
if self.__raftState == _RAFT_STATE.CANDIDATE:
if message['type'] == 'response_vote' and message[
'term'] == self.__raftCurrentTerm:
self.__votesCount += 1
if self.__votesCount > (len(self.__nodes) + 1) / 2:
self.__onBecomeLeader()
if self.__raftState == _RAFT_STATE.LEADER:
if message['type'] == 'next_node_idx':
reset = message['reset']
nextNodeIdx = message['next_node_idx']
success = message['success']
currentNodeIdx = nextNodeIdx - 1
if reset:
self.__raftNextIndex[nodeAddr] = nextNodeIdx
if success:
self.__raftMatchIndex[nodeAddr] = currentNodeIdx
def __callErrCallback(self, err, callback):
if callback is None:
return
if isinstance(callback, tuple):
requestNode, requestID = callback
self.__send(
requestNode, {
'type': 'apply_command_response',
'request_id': requestID,
'error': err,
})
return
callback(None, err)
def __sendNextNodeIdx(self,
nodeAddr,
reset=False,
nextNodeIdx=None,
success=False):
if nextNodeIdx is None:
nextNodeIdx = self.__getCurrentLogIndex() + 1
self.__send(
nodeAddr, {
'type': 'next_node_idx',
'next_node_idx': nextNodeIdx,
'reset': reset,
'success': success,
})
def __generateRaftTimeout(self):
minTimeout = self.__conf.raftMinTimeout
maxTimeout = self.__conf.raftMaxTimeout
return minTimeout + (maxTimeout - minTimeout) * random.random()
def __onNewConnection(self, conn):
descr = conn.fileno()
self.__unknownConnections[descr] = conn
if self.__encryptor:
conn.encryptor = self.__encryptor
conn.setOnMessageReceivedCallback(
functools.partial(self.__onMessageReceived, conn))
conn.setOnDisconnectedCallback(
functools.partial(self.__onDisconnected, conn))
def __utilityCallback(self, res, err, conn, cmd, node):
cmdResult = 'FAIL'
if err == FAIL_REASON.SUCCESS:
cmdResult = 'SUCCESS'
conn.send(cmdResult + ' ' + cmd + ' ' + node)
def __onUtilityMessage(self, conn, message):
if message[0] == 'status':
status = self.getStatus()
data = ''
for i in status:
data += i[0] + ':' + str(i[1]) + '\n'
conn.send(data)
return True
elif message[0] == 'add':
self.addNodeToCluster(message[1],
callback=functools.partial(
self.__utilityCallback,
conn=conn,
cmd='ADD',
node=message[1]))
return True
elif message[0] == 'remove':
if message[1] == self.__selfNodeAddr:
conn.send('FAIL REMOVE ' + message[1])
else:
self.removeNodeFromCluster(message[1],
callback=functools.partial(
self.__utilityCallback,
conn=conn,
cmd='REMOVE',
node=message[1]))
return True
return False
def __onMessageReceived(self, conn, message):
if self.__encryptor and not conn.sendRandKey:
conn.sendRandKey = message
conn.recvRandKey = os.urandom(32)
conn.send(conn.recvRandKey)
return
descr = conn.fileno()
if isinstance(message, list) and self.__onUtilityMessage(
conn, message):
self.__unknownConnections.pop(descr, None)
return
partnerNode = None
for node in self.__nodes:
if node.getAddress() == message:
partnerNode = node
break
if partnerNode is None and message != 'readonly':
conn.disconnect()
self.__unknownConnections.pop(descr, None)
return
if partnerNode is not None:
partnerNode.onPartnerConnected(conn)
else:
nodeAddr = str(self.__readonlyNodesCounter)
node = Node(self, nodeAddr, shouldConnect=False)
node.onPartnerConnected(conn)
self.__readonlyNodes.append(node)
self.__raftNextIndex[nodeAddr] = self.__getCurrentLogIndex() + 1
self.__raftMatchIndex[nodeAddr] = 0
self.__readonlyNodesCounter += 1
self.__unknownConnections.pop(descr, None)
def __onDisconnected(self, conn):
self.__unknownConnections.pop(conn.fileno(), None)
def __getCurrentLogIndex(self):
return self.__raftLog[-1][1]
def __getCurrentLogTerm(self):
return self.__raftLog[-1][2]
def __getPrevLogIndexTerm(self, nextNodeIndex):
prevIndex = nextNodeIndex - 1
entries = self.__getEntries(prevIndex, 1)
if entries:
return prevIndex, entries[0][2]
return None, None
def __getEntries(self, fromIDx, count=None, maxSizeBytes=None):
firstEntryIDx = self.__raftLog[0][1]
if fromIDx is None or fromIDx < firstEntryIDx:
return []
diff = fromIDx - firstEntryIDx
if count is None:
result = self.__raftLog[diff:]
else:
result = self.__raftLog[diff:diff + count]
if maxSizeBytes is None:
return result
totalSize = 0
i = 0
for i, entry in enumerate(result):
totalSize += len(entry[0])
if totalSize >= maxSizeBytes:
break
return result[:i + 1]
def _isLeader(self):
return self.__raftState == _RAFT_STATE.LEADER
def _getLeader(self):
return self.__raftLeader
def isReady(self):
"""Check if current node is initially synced with others and has an actual data.
:return: True if ready, False otherwise
:rtype: bool
"""
return self.__onReadyCalled
def _isReady(self):
return self.isReady()
def _getTerm(self):
return self.__raftCurrentTerm
def _getRaftLogSize(self):
return len(self.__raftLog)
def __deleteEntriesFrom(self, fromIDx):
firstEntryIDx = self.__raftLog[0][1]
diff = fromIDx - firstEntryIDx
if diff < 0:
return
self.__raftLog.deleteEntriesFrom(diff)
def __deleteEntriesTo(self, toIDx):
firstEntryIDx = self.__raftLog[0][1]
diff = toIDx - firstEntryIDx
if diff < 0:
return
self.__raftLog.deleteEntriesTo(diff)
def __onBecomeLeader(self):
self.__raftLeader = self.__selfNodeAddr
self.__raftState = _RAFT_STATE.LEADER
for node in self.__nodes + self.__readonlyNodes:
nodeAddr = node.getAddress()
self.__raftNextIndex[nodeAddr] = self.__getCurrentLogIndex() + 1
self.__raftMatchIndex[nodeAddr] = 0
# No-op command after leader election.
idx, term = self.__getCurrentLogIndex() + 1, self.__raftCurrentTerm
self.__raftLog.add(_bchr(_COMMAND_TYPE.NO_OP), idx, term)
self.__noopIDx = idx
if not self.__conf.appendEntriesUseBatch:
self.__sendAppendEntries()
self.__sendAppendEntries()
def __onLeaderChanged(self):
for id in sorted(self.__commandsWaitingReply):
self.__commandsWaitingReply[id](None, FAIL_REASON.LEADER_CHANGED)
self.__commandsWaitingReply = {}
def __sendAppendEntries(self):
self.__newAppendEntriesTime = time.time(
) + self.__conf.appendEntriesPeriod
startTime = time.time()
batchSizeBytes = self.__conf.appendEntriesBatchSizeBytes
for node in self.__nodes + self.__readonlyNodes:
nodeAddr = node.getAddress()
if not node.isConnected():
self.__serializer.cancelTransmisstion(nodeAddr)
continue
sendSingle = True
sendingSerialized = False
nextNodeIndex = self.__raftNextIndex[nodeAddr]
while nextNodeIndex <= self.__getCurrentLogIndex(
) or sendSingle or sendingSerialized:
if nextNodeIndex > self.__raftLog[0][1]:
prevLogIdx, prevLogTerm = self.__getPrevLogIndexTerm(
nextNodeIndex)
entries = []
if nextNodeIndex <= self.__getCurrentLogIndex():
entries = self.__getEntries(nextNodeIndex, None,
batchSizeBytes)
self.__raftNextIndex[nodeAddr] = entries[-1][1] + 1
if len(entries) == 1 and len(
entries[0][0]) >= batchSizeBytes:
entry = cPickle.dumps(entries[0], -1)
for pos in xrange(0, len(entry), batchSizeBytes):
currData = entry[pos:pos + batchSizeBytes]
if pos == 0:
transmission = 'start'
elif pos + batchSizeBytes >= len(entries[0][0]):
transmission = 'finish'
else:
transmission = 'process'
message = {
'type': 'append_entries',
'transmission': transmission,
'data': currData,
'term': self.__raftCurrentTerm,
'commit_index': self.__raftCommitIndex,
'prevLogIdx': prevLogIdx,
'prevLogTerm': prevLogTerm,
}
node.send(message)
else:
message = {
'type': 'append_entries',
'term': self.__raftCurrentTerm,
'commit_index': self.__raftCommitIndex,
'entries': entries,
'prevLogIdx': prevLogIdx,
'prevLogTerm': prevLogTerm,
}
node.send(message)
else:
transmissionData = self.__serializer.getTransmissionData(
nodeAddr)
message = {
'type': 'append_entries',
'term': self.__raftCurrentTerm,
'commit_index': self.__raftCommitIndex,
'serialized': transmissionData,
}
node.send(message)
if transmissionData is not None:
isLast = transmissionData[2]
if isLast:
self.__raftNextIndex[
nodeAddr] = self.__raftLog[1][1] + 1
sendingSerialized = False
else:
sendingSerialized = True
else:
sendingSerialized = False
nextNodeIndex = self.__raftNextIndex[nodeAddr]
sendSingle = False
delta = time.time() - startTime
if delta > self.__conf.appendEntriesPeriod:
break
def __send(self, nodeAddr, message):
for node in self.__nodes + self.__readonlyNodes:
if node.getAddress() == nodeAddr:
node.send(message)
break
def __connectedToAnyone(self):
for node in self.__nodes:
if node.getStatus() == NODE_STATUS.CONNECTED:
return True
if not self.__nodes:
return True
return False
def _getSelfNodeAddr(self):
return self.__selfNodeAddr
def _getConf(self):
return self.__conf
def _getEncryptor(self):
return self.__encryptor
def __changeCluster(self, request):
if self.__raftLastApplied < self.__noopIDx:
# No-op entry was not commited yet
return False
if self.__changeClusterIDx is not None:
if self.__raftLastApplied >= self.__changeClusterIDx:
self.__changeClusterIDx = None
# Previous cluster change request was not commited yet
if self.__changeClusterIDx is not None:
return False
return self.__doChangeCluster(request)
def __doChangeCluster(self, request, reverse=False):
requestType = request[0]
requestNode = request[1]
if requestType == 'add':
adding = not reverse
elif requestType == 'rem':
adding = reverse
else:
return False
if self.__selfNodeAddr is not None:
shouldConnect = None
else:
shouldConnect = True
if adding:
newNode = requestNode
# Node already exists in cluster
if newNode == self.__selfNodeAddr or newNode in self.__otherNodesAddrs:
return False
self.__otherNodesAddrs.append(newNode)
self.__nodes.append(Node(self, newNode, shouldConnect))
self.__raftNextIndex[newNode] = self.__getCurrentLogIndex() + 1
self.__raftMatchIndex[newNode] = 0
return True
else:
oldNode = requestNode
if oldNode == self.__selfNodeAddr:
return False
if oldNode not in self.__otherNodesAddrs:
return False
for i in xrange(len(self.__nodes)):
if self.__nodes[i].getAddress() == oldNode:
self.__nodes[i]._destroy()
self.__nodes.pop(i)
self.__otherNodesAddrs.pop(i)
del self.__raftNextIndex[oldNode]
del self.__raftMatchIndex[oldNode]
return True
return False
def __parseChangeClusterRequest(self, command):
commandType = ord(command[:1])
if commandType != _COMMAND_TYPE.MEMBERSHIP:
return None
return cPickle.loads(command[1:])
def __tryLogCompaction(self):
currTime = time.time()
serializeState, serializeID = self.__serializer.checkSerializing()
if serializeState == SERIALIZER_STATE.SUCCESS:
self.__lastSerializedTime = currTime
self.__deleteEntriesTo(serializeID)
self.__lastSerializedEntry = serializeID
if serializeState == SERIALIZER_STATE.FAILED:
logging.warning('Failed to store full dump')
if serializeState != SERIALIZER_STATE.NOT_SERIALIZING:
return
if len(self.__raftLog) <= self.__conf.logCompactionMinEntries and \
currTime - self.__lastSerializedTime <= self.__conf.logCompactionMinTime and \
not self.__forceLogCompaction:
return
if self.__conf.logCompactionSplit:
allNodes = sorted(self.__otherNodesAddrs + [self.__selfNodeAddr])
nodesCount = len(allNodes)
selfIdx = allNodes.index(self.__selfNodeAddr)
interval = self.__conf.logCompactionMinTime
periodStart = int(currTime) / interval * interval
nodeInterval = float(interval) / nodesCount
nodeIntervalStart = periodStart + selfIdx * nodeInterval
nodeIntervalEnd = nodeIntervalStart + 0.3 * nodeInterval
if currTime < nodeIntervalStart or currTime >= nodeIntervalEnd:
return
self.__forceLogCompaction = False
lastAppliedEntries = self.__getEntries(self.__raftLastApplied - 1, 2)
if len(lastAppliedEntries
) < 2 or lastAppliedEntries[0][1] == self.__lastSerializedEntry:
self.__lastSerializedTime = currTime
return
if self.__conf.serializer is None:
data = dict([(k, self.__dict__[k]) for k in self.__dict__.keys()
if k not in self.__properies])
else:
data = None
cluster = self.__otherNodesAddrs + [self.__selfNodeAddr]
self.__serializer.serialize(
(data, lastAppliedEntries[1], lastAppliedEntries[0], cluster),
lastAppliedEntries[0][1])
def __loadDumpFile(self, clearJournal):
try:
data = self.__serializer.deserialize()
if data[0] is not None:
for k, v in data[0].iteritems():
self.__dict__[k] = v
if clearJournal or \
len(self.__raftLog) < 2 or \
self.__raftLog[0] != data[2] or \
self.__raftLog[1] != data[1]:
self.__raftLog.clear()
self.__raftLog.add(*data[2])
self.__raftLog.add(*data[1])
self.__raftLastApplied = data[1][1]
if self.__conf.dynamicMembershipChange:
self.__otherNodesAddrs = [
node for node in data[3] if node != self.__selfNodeAddr
]
self.__updateClusterConfiguration()
except:
logging.exception('failed to load full dump')
def __updateClusterConfiguration(self):
currentNodes = set()
for i in xrange(len(self.__nodes) - 1, -1, -1):
nodeAddr = self.__nodes[i].getAddress()
if nodeAddr not in self.__otherNodesAddrs:
self.__nodes[i]._destroy()
self.__nodes.pop(i)
else:
currentNodes.add(nodeAddr)
if self.__selfNodeAddr is not None:
shouldConnect = None
else:
shouldConnect = True
for nodeAddr in self.__otherNodesAddrs:
if nodeAddr not in currentNodes:
self.__nodes.append(Node(self, nodeAddr, shouldConnect))
self.__raftNextIndex[nodeAddr] = self.__getCurrentLogIndex(
) + 1
self.__raftMatchIndex[nodeAddr] = 0
