def _process(self, message):
bytes_ = ''.join(message.serialize())
self._lock.acquire()
try:
start = time.time()
try_count = 0.0
backoff_period = 0.2
call_succeeded = False
retry_period = ArakoonClientConfig.getNoMasterRetryPeriod()
deadline = start + retry_period
while not call_succeeded and time.time() < deadline:
try:
# Send on wire
connection = self._send_message(self._node_id, bytes_)
return utils.read_blocking(message.receive(),
connection.read)
except (errors.NotMaster, ArakoonNoMaster):
self.master_id = None
self.drop_connections()
sleep_period = backoff_period * try_count
if time.time() + sleep_period > deadline:
raise
try_count += 1.0
logger.warning(
'Master not found, retrying in {0:.2f} seconds'.format(
sleep_period))
time.sleep(sleep_period)
finally:
self._lock.release()
def _process(self, message, node_id=None, retry=True):
bytes_ = "".join(message.serialize())
self._lock.acquire()
try:
start = time.time()
tryCount = 0.0
backoffPeriod = 0.2
retryPeriod = ArakoonClientConfig.getNoMasterRetryPeriod()
deadline = start + retryPeriod
while True:
try:
# Send on wire
if node_id is None:
connection = self._send_to_master(bytes_)
else:
connection = self._send_message(node_id, bytes_)
return utils.read_blocking(message.receive(), connection.read)
except (errors.NotMaster, ArakoonNoMaster, ArakoonNotConnected, ArakoonSockReadNoBytes):
self.master_id = None
self.drop_connections()
sleepPeriod = backoffPeriod * tryCount
if retry and time.time() + sleepPeriod <= deadline:
tryCount += 1.0
LOGGER.warning("Master not found, retrying in %0.2f seconds" % sleepPeriod)
time.sleep(sleepPeriod)
else:
raise
finally:
self._lock.release()
def _process(self, message):
bytes_ = ''.join(message.serialize())
self._lock.acquire()
try:
start = time.time()
try_count = 0.0
backoff_period = 0.2
call_succeeded = False
retry_period = ArakoonClientConfig.getNoMasterRetryPeriod()
deadline = start + retry_period
while not call_succeeded and time.time() < deadline:
try:
# Send on wire
connection = self._send_message(self._node_id, bytes_)
return utils.read_blocking(message.receive(), connection.read)
except (errors.NotMaster, ArakoonNoMaster):
self.master_id = None
self.drop_connections()
sleep_period = backoff_period * try_count
if time.time() + sleep_period > deadline:
raise
try_count += 1.0
logger.warning('Master not found, retrying in {0:.2f} seconds'.format(sleep_period))
time.sleep(sleep_period)
finally:
self._lock.release()
def _get_master_id_from_node(self, node_id):
command = protocol.WhoMaster()
data = ''.join(command.serialize())
connection = self._send_message(node_id, data)
receiver = command.receive()
return utils.read_blocking(receiver, connection.read)
def _get_master_id_from_node(self, node_id):
command = protocol.WhoMaster()
data = "".join(command.serialize())
connection = self._send_message(node_id, data)
receiver = command.receive()
return utils.read_blocking(receiver, connection.read)
def parse_nursery_routing(read):
'''Parse nursery routing information from a given stream
:param read: Function returning the requested amount of data
:type read: `callable` of `int -> str`
'''
is_leaf = utils.read_blocking(protocol.BOOL.receive(), read)
if is_leaf:
cluster = utils.read_blocking(protocol.STRING.receive(), read)
return LeafNode(cluster)
else:
boundary = utils.read_blocking(protocol.STRING.receive(), read)
left = parse_nursery_routing(read)
right = parse_nursery_routing(read)
return InternalNode(boundary, left, right)
def parse_nursery_routing(read):
'''Parse nursery routing information from a given stream
:param read: Function returning the requested amount of data
:type read: `callable` of `int -> str`
'''
is_leaf = utils.read_blocking(protocol.BOOL.receive(), read)
if is_leaf:
cluster = utils.read_blocking(protocol.STRING.receive(), read)
return LeafNode(cluster)
else:
boundary = utils.read_blocking(protocol.STRING.receive(), read)
left = parse_nursery_routing(read)
right = parse_nursery_routing(read)
return InternalNode(boundary, left, right)
def receive(self):
buffer_receiver = protocol.STRING.receive()
request = buffer_receiver.next() #pylint: disable=E1101
while isinstance(request, protocol.Request):
value = yield request
request = buffer_receiver.send(value) #pylint: disable=E1101
if not isinstance(request, protocol.Result):
raise TypeError
read = StringIO.StringIO(request.value).read
routing = parse_nursery_routing(read)
config_count = utils.read_blocking(protocol.UINT32.receive(), read)
configs = {}
for _ in xrange(config_count):
cluster_id = utils.read_blocking(protocol.STRING.receive(), read)
cluster_size = utils.read_blocking(protocol.UINT32.receive(), read)
config = {}
for _ in xrange(cluster_size):
node_id = utils.read_blocking(protocol.STRING.receive(), read)
ips = utils.read_blocking(
protocol.List(protocol.STRING).receive(), read)
port = utils.read_blocking(protocol.UINT32.receive(), read)
config[node_id] = (tuple(ips), port)
configs[cluster_id] = config
yield protocol.Result(NurseryConfig(routing, configs))
def receive(self):
buffer_receiver = protocol.STRING.receive()
request = buffer_receiver.next() #pylint: disable=E1101
while isinstance(request, protocol.Request):
value = yield request
request = buffer_receiver.send(value) #pylint: disable=E1101
if not isinstance(request, protocol.Result):
raise TypeError
read = StringIO.StringIO(request.value).read
routing = parse_nursery_routing(read)
config_count = utils.read_blocking(protocol.UINT32.receive(), read)
configs = {}
for _ in xrange(config_count):
cluster_id = utils.read_blocking(protocol.STRING.receive(), read)
cluster_size = utils.read_blocking(protocol.UINT32.receive(), read)
config = {}
for _ in xrange(cluster_size):
node_id = utils.read_blocking(protocol.STRING.receive(), read)
ips = utils.read_blocking(
protocol.List(protocol.STRING).receive(), read)
port = utils.read_blocking(protocol.UINT32.receive(), read)
config[node_id] = (tuple(ips), port)
configs[cluster_id] = config
yield protocol.Result(NurseryConfig(routing, configs))
def _process(self, message, node_id=None, retry=True):
bytes_ = ''.join(message.serialize())
self._lock.acquire()
try:
start = time.time()
tryCount = 0.0
backoffPeriod = 0.2
retryPeriod = ArakoonClientConfig.getNoMasterRetryPeriod()
deadline = start + retryPeriod
while True:
try:
# Send on wire
if node_id is None:
connection = self._send_to_master(bytes_)
else:
connection = self._send_message(node_id, bytes_)
return utils.read_blocking(message.receive(),
connection.read)
except (errors.NotMaster, ArakoonNoMaster, ArakoonNotConnected,
ArakoonSockReadNoBytes):
self.master_id = None
self.drop_connections()
sleepPeriod = backoffPeriod * tryCount
if retry and time.time() + sleepPeriod <= deadline:
tryCount += 1.0
LOGGER.warning(
'Master not found, retrying in %0.2f seconds' % \
sleepPeriod)
time.sleep(sleepPeriod)
else:
raise
finally:
self._lock.release()
def _process(self, message):
bytes_ = ''.join(message.serialize())
self._lock.acquire()
try:
start = time.time()
tryCount = 0.0
backoffPeriod = 0.2
callSucceeded = False
retryPeriod = ArakoonClientConfig.getNoMasterRetryPeriod()
deadline = start + retryPeriod
while not callSucceeded and time.time() < deadline:
try:
# Send on wire
connection = self._send_to_master(bytes_)
return utils.read_blocking(message.receive(),
connection.read)
except (errors.NotMaster,
ArakoonNoMaster,
ArakoonSockReadNoBytes):
self.master_id = None
self.drop_connections()
sleepPeriod = backoffPeriod * tryCount
if time.time() + sleepPeriod > deadline:
raise
tryCount += 1.0
LOGGER.warning(
'Master not found, retrying in %0.2f seconds' % \
sleepPeriod)
time.sleep(sleepPeriod)
finally:
self._lock.release()
def receive(self):
buffer_receiver = STRING.receive()
request = buffer_receiver.next() #pylint: disable=E1101
while isinstance(request, Request):
value = yield request
request = buffer_receiver.send(value) #pylint: disable=E1101
if not isinstance(request, Result):
raise TypeError
read = StringIO.StringIO(request.value).read
class NamedField(Type):
'''NamedField type'''
FIELD_TYPE_INT = 1
FIELD_TYPE_INT64 = 2
FIELD_TYPE_FLOAT = 3
FIELD_TYPE_STRING = 4
FIELD_TYPE_LIST = 5
def check(self, value):
raise NotImplementedError('NamedFields can\'t be checked')
def serialize(self, value):
raise NotImplementedError('NamedFields can\'t be serialized')
@classmethod
def receive(cls):
type_receiver = INT32.receive()
request = type_receiver.next() #pylint: disable=E1101
while isinstance(request, Request):
value = yield request
#pylint: disable=E1101
request = type_receiver.send(value)
if not isinstance(request, Result):
raise TypeError
type_ = request.value
name_receiver = STRING.receive()
request = name_receiver.next() #pylint: disable=E1101
while isinstance(request, Request):
value = yield request
#pylint: disable=E1101
request = name_receiver.send(value)
if not isinstance(request, Result):
raise TypeError
name = request.value
if type_ == cls.FIELD_TYPE_INT:
value_receiver = INT32.receive()
elif type_ == cls.FIELD_TYPE_INT64:
value_receiver = INT64.receive()
elif type_ == cls.FIELD_TYPE_FLOAT:
value_receiver = FLOAT.receive()
elif type_ == cls.FIELD_TYPE_STRING:
value_receiver = STRING.receive()
elif type_ == cls.FIELD_TYPE_LIST:
value_receiver = List(NamedField).receive()
else:
raise ValueError('Unknown named field type %d' % type_)
request = value_receiver.next() #pylint: disable=E1103
while isinstance(request, Request):
value = yield request
#pylint: disable=E1103
request = value_receiver.send(value)
if not isinstance(request, Result):
raise TypeError
value = request.value
if type_ == cls.FIELD_TYPE_LIST:
result = dict()
map(result.update, value) #pylint: disable=W0141
value = result
yield Result({name: value})
result = utils.read_blocking(NamedField.receive(), read)
if 'arakoon_stats' not in result:
raise ValueError('Missing expected \'arakoon_stats\' value')
yield Result(result['arakoon_stats'])
def _process(self, message): #pylint: disable=R0912
bytes_ = StringIO.StringIO(''.join(message.serialize())).read
# Helper
recv = lambda type_: utils.read_blocking(type_.receive(), bytes_)
command = recv(protocol.UINT32)
def handle_hello():
'''Handle a "hello" command'''
_ = recv(protocol.STRING)
_ = recv(protocol.STRING)
for rbytes in protocol.UINT32.serialize(protocol.RESULT_SUCCESS):
yield rbytes
for rbytes in protocol.STRING.serialize(self.VERSION):
yield rbytes
def handle_exists():
'''Handle an "exists" command'''
_ = recv(protocol.BOOL)
key = recv(protocol.STRING)
for rbytes in protocol.UINT32.serialize(protocol.RESULT_SUCCESS):
yield rbytes
for rbytes in protocol.BOOL.serialize(key in self._values):
yield rbytes
def handle_who_master():
'''Handle a "who_master" command'''
for rbytes in protocol.UINT32.serialize(protocol.RESULT_SUCCESS):
yield rbytes
for rbytes in protocol.Option(protocol.STRING).serialize(
self.MASTER):
yield rbytes
def handle_get():
'''Handle a "get" command'''
_ = recv(protocol.BOOL)
key = recv(protocol.STRING)
if key not in self._values:
for rbytes in protocol.UINT32.serialize(errors.NotFound.CODE):
yield rbytes
for rbytes in protocol.STRING.serialize(key):
yield rbytes
else:
for rbytes in protocol.UINT32.serialize(
protocol.RESULT_SUCCESS):
yield rbytes
for rbytes in protocol.STRING.serialize(self._values[key]):
yield rbytes
def handle_set():
'''Handle a "set" command'''
key = recv(protocol.STRING)
value = recv(protocol.STRING)
self._values[key] = value
for rbytes in protocol.UINT32.serialize(protocol.RESULT_SUCCESS):
yield rbytes
def handle_delete():
'''Handle a "delete" command'''
key = recv(protocol.STRING)
if key not in self._values:
for rbytes in protocol.UINT32.serialize(errors.NotFound.CODE):
yield rbytes
for rbytes in protocol.STRING.serialize(key):
yield rbytes
else:
del self._values[key]
for rbytes in protocol.UINT32.serialize(
protocol.RESULT_SUCCESS):
yield rbytes
def handle_prefix_keys():
'''Handle a "prefix_keys" command'''
_ = recv(protocol.BOOL)
prefix = recv(protocol.STRING)
max_elements = recv(protocol.UINT32)
matches = [
key for key in self._values.iterkeys()
if key.startswith(prefix)
]
matches = matches if max_elements < 0 else matches[:max_elements]
for rbytes in protocol.UINT32.serialize(protocol.RESULT_SUCCESS):
yield rbytes
for rbytes in protocol.List(protocol.STRING).serialize(matches):
yield rbytes
def handle_test_and_set():
'''Handle a "test_and_set" command'''
key = recv(protocol.STRING)
test_value = recv(protocol.Option(protocol.STRING))
set_value = recv(protocol.Option(protocol.STRING))
# Key doesn't exist and test_value is not None -> NotFound
if key not in self._values and test_value is not None:
for rbytes in protocol.UINT32.serialize(errors.NotFound.CODE):
yield rbytes
for rbytes in protocol.STRING.serialize(key):
yield rbytes
return
# Key doesn't exist and test_value is None -> create
if key not in self._values and test_value is None:
self._values[key] = set_value
for rbytes in protocol.UINT32.serialize(
protocol.RESULT_SUCCESS):
yield rbytes
for rbytes in protocol.Option(protocol.STRING).serialize(None):
yield rbytes
return
# Key exists
orig_value = self._values[key]
# Need to update?
if test_value == orig_value:
if set_value is not None:
self._values[key] = set_value
else:
del self._values[key]
# Return original value
for rbytes in protocol.UINT32.serialize(protocol.RESULT_SUCCESS):
yield rbytes
for rbytes in protocol.Option(
protocol.STRING).serialize(orig_value):
yield rbytes
handlers = {
protocol.Hello.TAG: handle_hello,
protocol.Exists.TAG: handle_exists,
protocol.WhoMaster.TAG: handle_who_master,
protocol.Get.TAG: handle_get,
protocol.Set.TAG: handle_set,
protocol.Delete.TAG: handle_delete,
protocol.PrefixKeys.TAG: handle_prefix_keys,
protocol.TestAndSet.TAG: handle_test_and_set,
}
if command in handlers:
result = StringIO.StringIO(''.join(handlers[command]()))
else:
result = StringIO.StringIO()
result.write(struct.pack('<I', errors.UnknownFailure.CODE))
result.write(struct.pack('<I', 0))
result.seek(0)
return utils.read_blocking(message.receive(), result.read)
def receive(self):
buffer_receiver = STRING.receive()
request = buffer_receiver.next() #pylint: disable=E1101
while isinstance(request, Request):
value = yield request
request = buffer_receiver.send(value) #pylint: disable=E1101
if not isinstance(request, Result):
raise TypeError
read = StringIO.StringIO(request.value).read
class NamedField(Type):
'''NamedField type'''
FIELD_TYPE_INT = 1
FIELD_TYPE_INT64 = 2
FIELD_TYPE_FLOAT = 3
FIELD_TYPE_STRING = 4
FIELD_TYPE_LIST = 5
def check(self, value):
raise NotImplementedError('NamedFields can\'t be checked')
def serialize(self, value):
raise NotImplementedError('NamedFields can\'t be serialized')
@classmethod
def receive(cls):
type_receiver = INT32.receive()
request = type_receiver.next() #pylint: disable=E1101
while isinstance(request, Request):
value = yield request
#pylint: disable=E1101
request = type_receiver.send(value)
if not isinstance(request, Result):
raise TypeError
type_ = request.value
name_receiver = STRING.receive()
request = name_receiver.next() #pylint: disable=E1101
while isinstance(request, Request):
value = yield request
#pylint: disable=E1101
request = name_receiver.send(value)
if not isinstance(request, Result):
raise TypeError
name = request.value
if type_ == cls.FIELD_TYPE_INT:
value_receiver = INT32.receive()
elif type_ == cls.FIELD_TYPE_INT64:
value_receiver = INT64.receive()
elif type_ == cls.FIELD_TYPE_FLOAT:
value_receiver = FLOAT.receive()
elif type_ == cls.FIELD_TYPE_STRING:
value_receiver = STRING.receive()
elif type_ == cls.FIELD_TYPE_LIST:
value_receiver = List(NamedField).receive()
else:
raise ValueError('Unknown named field type %d' % type_)
request = value_receiver.next() #pylint: disable=E1103
while isinstance(request, Request):
value = yield request
#pylint: disable=E1103
request = value_receiver.send(value)
if not isinstance(request, Result):
raise TypeError
value = request.value
if type_ == cls.FIELD_TYPE_LIST:
result = dict()
map(result.update, value) #pylint: disable=W0141
value = result
yield Result({name: value})
result = utils.read_blocking(NamedField.receive(), read)
if 'arakoon_stats' not in result:
raise ValueError('Missing expected \'arakoon_stats\' value')
yield Result(result['arakoon_stats'])
def _process(self, message): #pylint: disable=R0912
bytes_ = StringIO.StringIO(''.join(message.serialize())).read
# Helper
recv = lambda type_: utils.read_blocking(type_.receive(), bytes_)
command = recv(protocol.UINT32)
def handle_hello():
'''Handle a "hello" command'''
_ = recv(protocol.STRING)
_ = recv(protocol.STRING)
for rbytes in protocol.UINT32.serialize(
protocol.RESULT_SUCCESS):
yield rbytes
for rbytes in protocol.STRING.serialize(self.VERSION):
yield rbytes
def handle_exists():
'''Handle an "exists" command'''
_ = recv(protocol.BOOL)
key = recv(protocol.STRING)
for rbytes in protocol.UINT32.serialize(
protocol.RESULT_SUCCESS):
yield rbytes
for rbytes in protocol.BOOL.serialize(key in self._values):
yield rbytes
def handle_who_master():
'''Handle a "who_master" command'''
for rbytes in protocol.UINT32.serialize(
protocol.RESULT_SUCCESS):
yield rbytes
for rbytes in protocol.Option(protocol.STRING).serialize(
self.MASTER):
yield rbytes
def handle_get():
'''Handle a "get" command'''
_ = recv(protocol.BOOL)
key = recv(protocol.STRING)
if key not in self._values:
for rbytes in protocol.UINT32.serialize(
errors.NotFound.CODE):
yield rbytes
for rbytes in protocol.STRING.serialize(key):
yield rbytes
else:
for rbytes in protocol.UINT32.serialize(
protocol.RESULT_SUCCESS):
yield rbytes
for rbytes in protocol.STRING.serialize(self._values[key]):
yield rbytes
def handle_set():
'''Handle a "set" command'''
key = recv(protocol.STRING)
value = recv(protocol.STRING)
self._values[key] = value
for rbytes in protocol.UINT32.serialize(
protocol.RESULT_SUCCESS):
yield rbytes
def handle_delete():
'''Handle a "delete" command'''
key = recv(protocol.STRING)
if key not in self._values:
for rbytes in protocol.UINT32.serialize(
errors.NotFound.CODE):
yield rbytes
for rbytes in protocol.STRING.serialize(key):
yield rbytes
else:
del self._values[key]
for rbytes in protocol.UINT32.serialize(
protocol.RESULT_SUCCESS):
yield rbytes
def handle_prefix_keys():
'''Handle a "prefix_keys" command'''
_ = recv(protocol.BOOL)
prefix = recv(protocol.STRING)
max_elements = recv(protocol.UINT32)
matches = [key for key in self._values.iterkeys()
if key.startswith(prefix)]
matches = matches if max_elements < 0 else matches[:max_elements]
for rbytes in protocol.UINT32.serialize(
protocol.RESULT_SUCCESS):
yield rbytes
for rbytes in protocol.List(protocol.STRING).serialize(matches):
yield rbytes
def handle_test_and_set():
'''Handle a "test_and_set" command'''
key = recv(protocol.STRING)
test_value = recv(protocol.Option(protocol.STRING))
set_value = recv(protocol.Option(protocol.STRING))
# Key doesn't exist and test_value is not None -> NotFound
if key not in self._values and test_value is not None:
for rbytes in protocol.UINT32.serialize(
errors.NotFound.CODE):
yield rbytes
for rbytes in protocol.STRING.serialize(key):
yield rbytes
return
# Key doesn't exist and test_value is None -> create
if key not in self._values and test_value is None:
self._values[key] = set_value
for rbytes in protocol.UINT32.serialize(
protocol.RESULT_SUCCESS):
yield rbytes
for rbytes in protocol.Option(protocol.STRING).serialize(None):
yield rbytes
return
# Key exists
orig_value = self._values[key]
# Need to update?
if test_value == orig_value:
if set_value is not None:
self._values[key] = set_value
else:
del self._values[key]
# Return original value
for rbytes in protocol.UINT32.serialize(
protocol.RESULT_SUCCESS):
yield rbytes
for rbytes in protocol.Option(protocol.STRING).serialize(
orig_value):
yield rbytes
handlers = {
protocol.Hello.TAG: handle_hello,
protocol.Exists.TAG: handle_exists,
protocol.WhoMaster.TAG: handle_who_master,
protocol.Get.TAG: handle_get,
protocol.Set.TAG: handle_set,
protocol.Delete.TAG: handle_delete,
protocol.PrefixKeys.TAG: handle_prefix_keys,
protocol.TestAndSet.TAG: handle_test_and_set,
}
if command in handlers:
result = StringIO.StringIO(''.join(handlers[command]()))
else:
result = StringIO.StringIO()
result.write(struct.pack('<I', errors.UnknownFailure.CODE))
result.write(struct.pack('<I', 0))
result.seek(0)
return utils.read_blocking(message.receive(), result.read)