def create_paxos(tr, node_id):
def on_stale(last_accepted_id):
pass
def inject_node(rec):
rec.extra['node'] = node_id
_logger_group = LoggerGroup(processor=inject_node)
tr.paxos = Paxos(
tr,
on_learn=tr.learn,
on_stale=on_stale,
logger_group=_logger_group,
)
def __init__(self, config):
self.log = Logger('lockfactory')
self.interface = config.LOCK_INTERFACE or '127.0.0.1'
self.port = config.LOCK_PORT or 4001
def inject_node(rec):
rec.extra['node'] = self.port
self._logger_group = LoggerGroup(processor=inject_node)
self._logger_group.add_logger(self.log)
self.log.debug('creating lock factory')
self.paxos = Paxos(
self,
on_learn=self.on_learn,
on_prepare=self.on_prepare,
on_stale=lambda last_accepted_id: self.set_stale(True),
logger_group=self._logger_group,
)
self.neighbours = config.NODES
self._first_connect_delay = config.FIRST_CONNECT_DELAY or 0
self.master = None
self._stale = False
self._delayed_reconnect = None
# used to prevent server from reconnecting when it was
# closed in a unittest
self._closed = False
# this flag is used to prevent recursion in _reconnect method
self._reconnecting = False
self.connections = {}
self._all_connections = []
# list of deferreds to be called when
# connections with all other nodes will be established
self._connection_waiters = []
# same for sync complete event
self._sync_completion_waiters = []
# state
self._epoch = 0 # received commands counter
self._log = []
# a buffer for learn commands if they come out of order.
self._learn_queue = []
# actual data storage
self._keys = {}
# last successful Paxos ID
self.last_accepted_iteration = 0
self.state = []
self.callbacks = []
# map paxos-id -> proposer
self._proposers = {}
# this buffer is used to keep messages
# when node is stale
self._paxos_messages_buffer = deque()
self.add_callback('^paxos_.*$', self._process_paxos_messages)
self.syncronizer = Syncronizer(self)
self.add_callback('^sync_subscribe$',
self.syncronizer.on_sync_subscribe)
self.add_callback('^sync_unsubscribe$',
self.syncronizer.on_sync_subscribe)
self.add_callback('^sync_snapshot .*$',
self.syncronizer.on_sync_snapshot)
self.log.debug('Opening the port %s:%s' % (self.interface, self.port))
self._port_listener = reactor.listenTCP(self.port,
self,
interface=self.interface)
self.web_server = server.Site(Root(self))
self.http_interface = config.HTTP_INTERFACE or '127.0.0.1'
self.http_port = config.HTTP_PORT or 9001
self._webport_listener = reactor.listenTCP(
self.http_port,
self.web_server,
interface=self.http_interface,
)
class LockFactory(ClientFactory):
protocol = LockProtocol
def __init__(self, config):
self.log = Logger('lockfactory')
self.interface = config.LOCK_INTERFACE or '127.0.0.1'
self.port = config.LOCK_PORT or 4001
def inject_node(rec):
rec.extra['node'] = self.port
self._logger_group = LoggerGroup(processor=inject_node)
self._logger_group.add_logger(self.log)
self.log.debug('creating lock factory')
self.paxos = Paxos(
self,
on_learn=self.on_learn,
on_prepare=self.on_prepare,
on_stale=lambda last_accepted_id: self.set_stale(True),
logger_group=self._logger_group,
)
self.neighbours = config.NODES
self._first_connect_delay = config.FIRST_CONNECT_DELAY or 0
self.master = None
self._stale = False
self._delayed_reconnect = None
# used to prevent server from reconnecting when it was
# closed in a unittest
self._closed = False
# this flag is used to prevent recursion in _reconnect method
self._reconnecting = False
self.connections = {}
self._all_connections = []
# list of deferreds to be called when
# connections with all other nodes will be established
self._connection_waiters = []
# same for sync complete event
self._sync_completion_waiters = []
# state
self._epoch = 0 # received commands counter
self._log = []
# a buffer for learn commands if they come out of order.
self._learn_queue = []
# actual data storage
self._keys = {}
# last successful Paxos ID
self.last_accepted_iteration = 0
self.state = []
self.callbacks = []
# map paxos-id -> proposer
self._proposers = {}
# this buffer is used to keep messages
# when node is stale
self._paxos_messages_buffer = deque()
self.add_callback('^paxos_.*$', self._process_paxos_messages)
self.syncronizer = Syncronizer(self)
self.add_callback('^sync_subscribe$',
self.syncronizer.on_sync_subscribe)
self.add_callback('^sync_unsubscribe$',
self.syncronizer.on_sync_subscribe)
self.add_callback('^sync_snapshot .*$',
self.syncronizer.on_sync_snapshot)
self.log.debug('Opening the port %s:%s' % (self.interface, self.port))
self._port_listener = reactor.listenTCP(self.port,
self,
interface=self.interface)
self.web_server = server.Site(Root(self))
self.http_interface = config.HTTP_INTERFACE or '127.0.0.1'
self.http_port = config.HTTP_PORT or 9001
self._webport_listener = reactor.listenTCP(
self.http_port,
self.web_server,
interface=self.http_interface,
)
def close(self):
self._closed = True
d1 = self._port_listener.stopListening()
d2 = self._webport_listener.stopListening()
if self._delayed_reconnect is not None:
stop_waiting(self._delayed_reconnect)
self.disconnect()
def add_callback(self, regex, callback):
self.callbacks.append((re.compile(regex), callback))
def remove_callback(self, callback):
self.callbacks = filter(lambda x: x[1] != callback, self.callbacks)
def find_callback(self, line):
for regex, callback in self.callbacks:
if regex.match(line) != None:
return callback
def get_status(self):
"""Returns a list of tuples (param_name, value)."""
if self.master is not None:
master = ':'.join(map(str, self.master.http))
else:
master = 'None'
return [
('bind', '%s:%s' % (self.interface, self.port)),
('master', master),
('stale', self._stale),
('current_id', self.paxos.id),
('max_seen_id', self.paxos.max_seen_id),
('last_accepted_id', self.paxos.last_accepted_id),
('num_connections', len(self.connections)),
('quorum_size', self.quorum_size),
('log_size', len(self._log)),
('epoch', self._epoch),
]
def get_key(self, key):
d = Deferred()
def cb():
if key not in self._keys:
raise KeyNotFound('Key "%s" not found' % key)
return self._keys[key]
d.addCallback(cb)
return d
def set_key(self, key, value):
value = 'set-key %s "%s"' % (key, escape(value))
return self.paxos.propose(value)
def del_key(self, key):
value = 'del-key %s' % key
return self.paxos.propose(value)
def add_connection(self, conn):
self.connections[conn.other_side] = conn
self._notify_waiters_if_needed()
def _notify_waiters_if_needed(self):
num_disconnected = len(self.neighbours) - len(self.connections)
if num_disconnected == 0:
for waiter in self._connection_waiters:
waiter.callback(True)
self._connection_waiters = []
def remove_connection(self, conn):
for key, value in self.connections.items():
if value == conn:
self.log.info('Connection to the %s:%s (%s) lost.' %
(conn.other_side[0], conn.other_side[1],
conn.transport.getPeer()))
del self.connections[key]
break
def when_connected(self):
d = Deferred()
self._connection_waiters.append(d)
self._notify_waiters_if_needed()
return d
def when_sync_completed(self):
d = Deferred()
if self.get_stale():
# sync in progress
self._sync_completion_waiters.append(d)
else:
# we are not a stale node
d.callback(True)
return d
def disconnect(self):
for conn in self._all_connections:
if conn.connected:
conn.transport.loseConnection()
def startFactory(self):
self.log.info('factory started at %s:%s' % (self.interface, self.port))
if self._first_connect_delay > 0:
def delay_connect():
# delay connection to other server
# this is needed to start few test servers
# on the same machine without errors
self._delayed_reconnect = reactor.callLater(
self._first_connect_delay, self._reconnect)
reactor.callWhenRunning(delay_connect)
else:
reactor.callWhenRunning(self._reconnect)
def _reconnect(self):
if not self._closed and not self._reconnecting:
try:
self._reconnecting = True
for host, port in self.neighbours:
if (host, port) not in self.connections:
self.log.info('reconnecting to %s:%s' % (host, port))
reactor.connectTCP(host, port, self)
self._delayed_reconnect = reactor.callLater(
RECONNECT_INTERVAL, self._reconnect)
finally:
self._reconnecting = False
def startedConnecting(self, connector):
self.log.info('Started to connect to another server: %s:%s' %
(connector.host, connector.port))
def buildProtocol(self, addr):
conn = addr.host, addr.port
result = ClientFactory.buildProtocol(self, addr)
result.other_side = conn
def on_connect():
"""This callback will be called when actual connection happened."""
self._all_connections.append(result)
if addr.port in map(itemgetter(1), self.neighbours):
self.log.info('Connected to another server: %s:%s' % conn)
self.add_connection(result)
else:
self.log.info(
'Connection from another server accepted: %s:%s' % conn)
def on_disconnect():
self.remove_connection(result)
result.on_connect = on_connect
result.on_disconnect = on_disconnect
return result
def clientConnectionFailed(self, connector, reason):
self.log.info('Connection to %s:%s failed. Reason: %s' %
(connector.host, connector.port, reason))
# BEGIN Paxos Transport methods
def broadcast(self, line):
for connection in self.connections.values():
connection.sendLine(line)
return len(self.connections)
@property
def quorum_size(self):
return max(2, len(self.connections) / 2 + 1)
def on_learn(self, num, value, client):
"""First callback in the paxos result accepting chain."""
self.log.info('factory.on_learn %s %s' % (len(self._log) + 1, value))
self._log.append(value)
self._epoch += 1
splitted = shlex.split(value)
command_name, args = splitted[0], splitted[1:]
command = '_log_cmd_' + command_name.replace('-', '_')
cmd = getattr(self, command)
try:
return cmd(*args)
except Exception, e:
self.log.error('command "%s" failed: %s' % (command_name, e))
raise
import logbook from logbook import LoggerGroup, CRITICAL logger_group = LoggerGroup() logger_group.level = CRITICAL
def __init__(self, config):
self.log = Logger('lockfactory')
self.interface = config.LOCK_INTERFACE or '127.0.0.1'
self.port = config.LOCK_PORT or 4001
def inject_node(rec):
rec.extra['node'] = self.port
self._logger_group = LoggerGroup(processor=inject_node)
self._logger_group.add_logger(self.log)
self.log.debug('creating lock factory')
self.paxos = Paxos(
self,
on_learn=self.on_learn,
on_prepare=self.on_prepare,
on_stale=lambda last_accepted_id: self.set_stale(True),
logger_group=self._logger_group,
)
self.neighbours = config.NODES
self._first_connect_delay = config.FIRST_CONNECT_DELAY or 0
self.master = None
self._stale = False
self._delayed_reconnect = None
# used to prevent server from reconnecting when it was
# closed in a unittest
self._closed = False
# this flag is used to prevent recursion in _reconnect method
self._reconnecting = False
self.connections = {}
self._all_connections = []
# list of deferreds to be called when
# connections with all other nodes will be established
self._connection_waiters = []
# same for sync complete event
self._sync_completion_waiters = []
# state
self._epoch = 0 # received commands counter
self._log = []
# a buffer for learn commands if they come out of order.
self._learn_queue = []
# actual data storage
self._keys = {}
# last successful Paxos ID
self.last_accepted_iteration = 0
self.state = []
self.callbacks = []
# map paxos-id -> proposer
self._proposers = {}
# this buffer is used to keep messages
# when node is stale
self._paxos_messages_buffer = deque()
self.add_callback('^paxos_.*$', self._process_paxos_messages)
self.syncronizer = Syncronizer(self)
self.add_callback('^sync_subscribe$', self.syncronizer.on_sync_subscribe)
self.add_callback('^sync_unsubscribe$', self.syncronizer.on_sync_subscribe)
self.add_callback('^sync_snapshot .*$', self.syncronizer.on_sync_snapshot)
self.log.debug('Opening the port %s:%s' % (self.interface, self.port))
self._port_listener = reactor.listenTCP(self.port, self, interface=self.interface)
self.web_server = server.Site(Root(self))
self.http_interface = config.HTTP_INTERFACE or '127.0.0.1'
self.http_port = config.HTTP_PORT or 9001
self._webport_listener = reactor.listenTCP(
self.http_port,
self.web_server,
interface = self.http_interface,
)
class LockFactory(ClientFactory):
protocol = LockProtocol
def __init__(self, config):
self.log = Logger('lockfactory')
self.interface = config.LOCK_INTERFACE or '127.0.0.1'
self.port = config.LOCK_PORT or 4001
def inject_node(rec):
rec.extra['node'] = self.port
self._logger_group = LoggerGroup(processor=inject_node)
self._logger_group.add_logger(self.log)
self.log.debug('creating lock factory')
self.paxos = Paxos(
self,
on_learn=self.on_learn,
on_prepare=self.on_prepare,
on_stale=lambda last_accepted_id: self.set_stale(True),
logger_group=self._logger_group,
)
self.neighbours = config.NODES
self._first_connect_delay = config.FIRST_CONNECT_DELAY or 0
self.master = None
self._stale = False
self._delayed_reconnect = None
# used to prevent server from reconnecting when it was
# closed in a unittest
self._closed = False
# this flag is used to prevent recursion in _reconnect method
self._reconnecting = False
self.connections = {}
self._all_connections = []
# list of deferreds to be called when
# connections with all other nodes will be established
self._connection_waiters = []
# same for sync complete event
self._sync_completion_waiters = []
# state
self._epoch = 0 # received commands counter
self._log = []
# a buffer for learn commands if they come out of order.
self._learn_queue = []
# actual data storage
self._keys = {}
# last successful Paxos ID
self.last_accepted_iteration = 0
self.state = []
self.callbacks = []
# map paxos-id -> proposer
self._proposers = {}
# this buffer is used to keep messages
# when node is stale
self._paxos_messages_buffer = deque()
self.add_callback('^paxos_.*$', self._process_paxos_messages)
self.syncronizer = Syncronizer(self)
self.add_callback('^sync_subscribe$', self.syncronizer.on_sync_subscribe)
self.add_callback('^sync_unsubscribe$', self.syncronizer.on_sync_subscribe)
self.add_callback('^sync_snapshot .*$', self.syncronizer.on_sync_snapshot)
self.log.debug('Opening the port %s:%s' % (self.interface, self.port))
self._port_listener = reactor.listenTCP(self.port, self, interface=self.interface)
self.web_server = server.Site(Root(self))
self.http_interface = config.HTTP_INTERFACE or '127.0.0.1'
self.http_port = config.HTTP_PORT or 9001
self._webport_listener = reactor.listenTCP(
self.http_port,
self.web_server,
interface = self.http_interface,
)
def close(self):
self._closed = True
d1 = self._port_listener.stopListening()
d2 = self._webport_listener.stopListening()
if self._delayed_reconnect is not None:
stop_waiting(self._delayed_reconnect)
self.disconnect()
def add_callback(self, regex, callback):
self.callbacks.append((re.compile(regex), callback))
def remove_callback(self, callback):
self.callbacks = filter(lambda x: x[1] != callback, self.callbacks)
def find_callback(self, line):
for regex, callback in self.callbacks:
if regex.match(line) != None:
return callback
def get_status(self):
"""Returns a list of tuples (param_name, value)."""
if self.master is not None:
master = ':'.join(map(str, self.master.http))
else:
master = 'None'
return [
('bind', '%s:%s' % (self.interface, self.port)),
('master', master),
('stale', self._stale),
('current_id', self.paxos.id),
('max_seen_id', self.paxos.max_seen_id),
('last_accepted_id', self.paxos.last_accepted_id),
('num_connections', len(self.connections)),
('quorum_size', self.quorum_size),
('log_size', len(self._log)),
('epoch', self._epoch),
]
def get_key(self, key):
d = Deferred()
def cb():
if key not in self._keys:
raise KeyNotFound('Key "%s" not found' % key)
return self._keys[key]
d.addCallback(cb)
return d
def set_key(self, key, value):
value = 'set-key %s "%s"' % (key, escape(value))
return self.paxos.propose(value)
def del_key(self, key):
value = 'del-key %s' % key
return self.paxos.propose(value)
def add_connection(self, conn):
self.connections[conn.other_side] = conn
self._notify_waiters_if_needed()
def _notify_waiters_if_needed(self):
num_disconnected = len(self.neighbours) - len(self.connections)
if num_disconnected == 0:
for waiter in self._connection_waiters:
waiter.callback(True)
self._connection_waiters = []
def remove_connection(self, conn):
for key, value in self.connections.items():
if value == conn:
self.log.info(
'Connection to the %s:%s (%s) lost.' % (
conn.other_side[0],
conn.other_side[1],
conn.transport.getPeer()
)
)
del self.connections[key]
break
def when_connected(self):
d = Deferred()
self._connection_waiters.append(d)
self._notify_waiters_if_needed()
return d
def when_sync_completed(self):
d = Deferred()
if self.get_stale():
# sync in progress
self._sync_completion_waiters.append(d)
else:
# we are not a stale node
d.callback(True)
return d
def disconnect(self):
for conn in self._all_connections:
if conn.connected:
conn.transport.loseConnection()
def startFactory(self):
self.log.info('factory started at %s:%s' % (self.interface, self.port))
if self._first_connect_delay > 0:
def delay_connect():
# delay connection to other server
# this is needed to start few test servers
# on the same machine without errors
self._delayed_reconnect = reactor.callLater(self._first_connect_delay, self._reconnect)
reactor.callWhenRunning(delay_connect)
else:
reactor.callWhenRunning(self._reconnect)
def _reconnect(self):
if not self._closed and not self._reconnecting:
try:
self._reconnecting = True
for host, port in self.neighbours:
if (host, port) not in self.connections:
self.log.info('reconnecting to %s:%s' % (host, port))
reactor.connectTCP(host, port, self)
self._delayed_reconnect = reactor.callLater(RECONNECT_INTERVAL, self._reconnect)
finally:
self._reconnecting = False
def startedConnecting(self, connector):
self.log.info('Started to connect to another server: %s:%s' % (
connector.host,
connector.port
))
def buildProtocol(self, addr):
conn = addr.host, addr.port
result = ClientFactory.buildProtocol(self, addr)
result.other_side = conn
def on_connect():
"""This callback will be called when actual connection happened."""
self._all_connections.append(result)
if addr.port in map(itemgetter(1), self.neighbours):
self.log.info('Connected to another server: %s:%s' % conn)
self.add_connection(result)
else:
self.log.info('Connection from another server accepted: %s:%s' % conn)
def on_disconnect():
self.remove_connection(result)
result.on_connect = on_connect
result.on_disconnect = on_disconnect
return result
def clientConnectionFailed(self, connector, reason):
self.log.info('Connection to %s:%s failed. Reason: %s' % (
connector.host,
connector.port,
reason
))
# BEGIN Paxos Transport methods
def broadcast(self, line):
for connection in self.connections.values():
connection.sendLine(line)
return len(self.connections)
@property
def quorum_size(self):
return max(2, len(self.connections)/ 2 + 1)
def on_learn(self, num, value, client):
"""First callback in the paxos result accepting chain."""
self.log.info('factory.on_learn %s %s' % (len(self._log) + 1, value))
self._log.append(value)
self._epoch += 1
splitted = shlex.split(value)
command_name, args = splitted[0], splitted[1:]
command = '_log_cmd_' + command_name.replace('-', '_')
cmd = getattr(self, command)
try:
return cmd(*args)
except Exception, e:
self.log.error('command "%s" failed: %s' % (command_name, e))
raise
import sys from logbook import Logger, LoggerGroup, StreamHandler from logbook import WARNING, DEBUG logger_group = LoggerGroup() logger_group.level = WARNING StreamHandler(sys.stdout).push_application()