def _update_capabilities():
member_path = self._path_member(group_id, self._member_id)
index, data = self._client.kv.get(member_path)
if not data:
raise coordination.MemberNotJoined(group_id, self._member_id)
# no need to Check-And-Set here, latest write wins
self._client.kv.put(member_path, utils.dumps(capabilities))
def _update_capabilities():
cap, metadata = self.client.get(encoded_member)
if cap is None:
raise coordination.MemberNotJoined(group_id, self._member_id)
self.client.put(encoded_member, utils.dumps(capabilities),
lease=self._membership_lease)
def _update_capabilities():
prefix_member = self._prefix_group(group_id) + self._member_id
result = self.client.get(prefix_member)
if not result:
raise coordination.MemberNotJoined(group_id, self._member_id)
self.client.put(prefix_member, utils.dumps(capabilities),
lease=self._membership_lease)
def _update_group_metadata(self, path, group_id):
details = {
u'group_id': utils.to_binary(group_id, encoding="utf8")
}
details[u'encoded'] = details[u"group_id"] != group_id
details_blob = utils.dumps(details)
with open(path, "wb") as fh:
fh.write(details_blob)
def _update_group_metadata(self, path, group_id):
details = {u'group_id': utils.to_binary(group_id, encoding="utf8")}
details[u'encoded'] = details[u"group_id"] != group_id
details_blob = utils.dumps(details)
fd, name = tempfile.mkstemp("tooz", dir=self._tmpdir)
with os.fdopen(fd, "wb") as fh:
fh.write(details_blob)
os.rename(name, path)
def _update_capabilities():
prefix_member = self._prefix_group(group_id) + self._member_id
result = self.client.get(prefix_member)
if not result:
raise coordination.MemberNotJoined(group_id, self._member_id)
self.client.put(prefix_member, utils.dumps(capabilities),
lease=self._membership_lease)
def _update_group_metadata(self, path, group_id):
details = {
u'group_id': utils.to_binary(group_id, encoding="utf8")
}
details[u'encoded'] = details[u"group_id"] != group_id
details_blob = utils.dumps(details)
fd, name = tempfile.mkstemp("tooz", dir=self._tmpdir)
with os.fdopen(fd, "wb") as fh:
fh.write(details_blob)
os.rename(name, path)
def _do_create_group():
if os.path.isdir(group_dir):
raise coordination.GroupAlreadyExist(group_id)
else:
details = {
'group_id': group_id,
}
details_blob = utils.dumps(details)
utils.ensure_tree(group_dir)
with open(group_meta_path, "wb") as fh:
fh.write(details_blob)
def _do_create_group():
if os.path.isdir(group_dir):
raise coordination.GroupAlreadyExist(group_id)
else:
details = {
'group_id': group_id,
}
details_blob = utils.dumps(details)
utils.ensure_tree(group_dir)
with open(group_meta_path, "wb") as fh:
fh.write(details_blob)
def _do_join_group():
if not os.path.isdir(group_dir):
raise coordination.GroupNotCreated(group_id)
if os.path.isfile(me_path):
raise coordination.MemberAlreadyExist(group_id,
self._member_id)
details = {
'capabilities': capabilities,
'joined_on': datetime.datetime.now(),
'member_id': self._member_id,
}
details_blob = utils.dumps(details)
with open(me_path, "wb") as fh:
fh.write(details_blob)
self._joined_groups.add(group_id)
def _do_join_group():
if not os.path.isdir(group_dir):
raise coordination.GroupNotCreated(group_id)
if os.path.isfile(me_path):
raise coordination.MemberAlreadyExist(group_id,
self._member_id)
details = {
'capabilities': capabilities,
'joined_on': datetime.datetime.now(),
'member_id': self._member_id,
}
details_blob = utils.dumps(details)
with open(me_path, "wb") as fh:
fh.write(details_blob)
self._joined_groups.add(group_id)
def _do_join_group():
if not os.path.exists(os.path.join(group_dir, ".metadata")):
raise coordination.GroupNotCreated(group_id)
if os.path.isfile(me_path):
raise coordination.MemberAlreadyExist(group_id,
self._member_id)
details = {
u'capabilities': capabilities,
u'joined_on': datetime.datetime.now(),
u'member_id': utils.to_binary(self._member_id,
encoding="utf-8")
}
details[u'encoded'] = details[u"member_id"] != self._member_id
details_blob = utils.dumps(details)
with open(me_path, "wb") as fh:
fh.write(details_blob)
self._joined_groups.add(group_id)
def _do_join_group():
if not os.path.isdir(group_dir):
raise coordination.GroupNotCreated(group_id)
if os.path.isfile(me_path):
raise coordination.MemberAlreadyExist(group_id,
self._member_id)
details = {
u'capabilities': capabilities,
u'joined_on': datetime.datetime.now(),
u'member_id': utils.to_binary(self._member_id,
encoding="utf-8")
}
details[u'encoded'] = details[u"member_id"] != self._member_id
details_blob = utils.dumps(details)
with open(me_path, "wb") as fh:
fh.write(details_blob)
self._joined_groups.add(group_id)
def _join_group():
# lock the group so that it doesn't get deleted while we join
with self._get_group_lock(group_id):
group_path = self._path_group(group_id)
member_path = self._path_member(group_id, self._member_id)
group, members = self._get_group_members(group_path)
if not group:
raise coordination.GroupNotCreated(group_id)
for m in members:
if m["Key"] == member_path:
raise coordination.MemberAlreadyExist(
group_id, self._member_id)
# create with Check-And-Set index 0 will only succeed if the
# key doesn't exit
self._client.kv.put(member_path,
utils.dumps(capabilities),
cas=0)
self._joined_groups.add(group_id)
def _join_group():
prefix_group = self._prefix_group(group_id)
prefix_member = prefix_group + self._member_id
members = self.client.get_prefix(prefix_group)
encoded_member = _encode(prefix_member)
group_metadata = None
for cap, metadata in members:
if metadata['key'] == prefix_member:
raise coordination.MemberAlreadyExist(group_id,
self._member_id)
if metadata['key'] == prefix_group:
group_metadata = metadata
if group_metadata is None:
raise coordination.GroupNotCreated(group_id)
encoded_group = self._encode_group_id(group_id)
txn = {
'compare': [{
'key': encoded_group,
'result': 'EQUAL',
'target': 'VERSION',
'version': int(group_metadata['version'])
}],
'success': [{
'request_put': {
'key': encoded_member,
'value': _encode(utils.dumps(capabilities)),
'lease': self._membership_lease.id
}
}],
'failure': []
}
result = self.client.transaction(txn)
if not result.get('succeeded'):
raise _retry.TryAgain
else:
self._joined_groups.add(group_id)
def _join_group():
prefix_group = self._prefix_group(group_id)
prefix_member = prefix_group + self._member_id
members = self.client.get_prefix(prefix_group)
encoded_member = _encode(prefix_member)
group_metadata = None
for cap, metadata in members:
if metadata['key'] == prefix_member:
raise coordination.MemberAlreadyExist(group_id,
self._member_id)
if metadata['key'] == prefix_group:
group_metadata = metadata
if group_metadata is None:
raise coordination.GroupNotCreated(group_id)
encoded_group = self._encode_group_id(group_id)
txn = {
'compare': [{
'key': encoded_group,
'result': 'EQUAL',
'target': 'VERSION',
'version': int(group_metadata['version'])
}],
'success': [{
'request_put': {
'key': encoded_member,
'value': _encode(utils.dumps(capabilities)),
'lease': self._membership_lease.id
}
}],
'failure': []
}
result = self.client.transaction(txn)
if not result.get('succeeded'):
raise _retry.TryAgain
else:
self._joined_groups.add(group_id)
def join_partitioned_group(
self, group_id,
weight=1,
partitions=partitioner.Partitioner.DEFAULT_PARTITION_NUMBER):
"""Join a group and get a partitioner.
A partitioner allows to distribute a bunch of objects across several
members using a consistent hash ring. Each object gets assigned (at
least) one member responsible for it. It's then possible to check which
object is owned by any member of the group.
This method also creates if necessary, and joins the group with the
selected weight.
:param group_id: The group to create a partitioner for.
:param weight: The weight to use in the hashring for this node.
:param partitions: The number of partitions to create.
:return: A :py:class:`~tooz.partitioner.Partitioner` object.
"""
self.join_group_create(
group_id, capabilities=utils.dumps({'weight': weight}))
return partitioner.Partitioner(self, group_id, partitions=partitions)
def _join_group():
members = list(self.client.get_prefix(encoded_group))
encoded_member = self._encode_group_member_id(
group_id, self._member_id)
group_metadata = None
for cap, metadata in members:
if metadata.key == encoded_member:
raise coordination.MemberAlreadyExist(group_id,
self._member_id)
if metadata.key == encoded_group:
group_metadata = metadata
if group_metadata is None:
raise coordination.GroupNotCreated(group_id)
status, results = self.client.transaction(
# This comparison makes sure the group has not been deleted in
# the mean time
compare=[
self.client.transactions.version(encoded_group) ==
group_metadata.version
],
success=[
self.client.transactions.put(encoded_member,
utils.dumps(capabilities),
lease=self._membership_lease)
],
failure=[],
)
if not status:
# TODO(jd) There's a small optimization doable by getting the
# current range on failure and passing it to this function as
# the first arg when retrying to avoid redoing a get_prefix()
raise _retry.TryAgain
def _join_group():
members = list(self.client.get_prefix(encoded_group))
encoded_member = self._encode_group_member_id(
group_id, self._member_id)
group_metadata = None
for cap, metadata in members:
if metadata.key == encoded_member:
raise coordination.MemberAlreadyExist(
group_id, self._member_id)
if metadata.key == encoded_group:
group_metadata = metadata
if group_metadata is None:
raise coordination.GroupNotCreated(group_id)
status, results = self.client.transaction(
# This comparison makes sure the group has not been deleted in
# the mean time
compare=[
self.client.transactions.version(encoded_group) ==
group_metadata.version
],
success=[
self.client.transactions.put(encoded_member,
utils.dumps(capabilities),
lease=self._membership_lease)
],
failure=[],
)
if not status:
# TODO(jd) There's a small optimization doable by getting the
# current range on failure and passing it to this function as
# the first arg when retrying to avoid redoing a get_prefix()
raise _retry.TryAgain
def _update_group_metadata(self, path, group_id):
details = {u'group_id': utils.to_binary(group_id, encoding="utf8")}
details[u'encoded'] = details[u"group_id"] != group_id
details_blob = utils.dumps(details)
with open(path, "wb") as fh:
fh.write(details_blob)
def _msgpack_serializer(key, value):
if isinstance(value, six.binary_type):
return value, 1
return utils.dumps(value), 2
def _dumps(data):
return utils.dumps(data)
def _msgpack_serializer(key, value):
if isinstance(value, bytes):
return value, 1
return utils.dumps(value), 2
def _msgpack_serializer(key, value):
if isinstance(value, six.binary_type):
return value, 1
return utils.dumps(value), 2
def _dumps(data):
return utils.dumps(data)
