def _update_capabilities(): member_path = self._path_member(group_id, self._member_id) index, data = self._client.kv.get(member_path) if not data: raise coordination.MemberNotJoined(group_id, self._member_id) # no need to Check-And-Set here, latest write wins self._client.kv.put(member_path, utils.dumps(capabilities))
def _update_capabilities(): cap, metadata = self.client.get(encoded_member) if cap is None: raise coordination.MemberNotJoined(group_id, self._member_id) self.client.put(encoded_member, utils.dumps(capabilities), lease=self._membership_lease)
def _update_capabilities(): prefix_member = self._prefix_group(group_id) + self._member_id result = self.client.get(prefix_member) if not result: raise coordination.MemberNotJoined(group_id, self._member_id) self.client.put(prefix_member, utils.dumps(capabilities), lease=self._membership_lease)
def _update_group_metadata(self, path, group_id): details = { u'group_id': utils.to_binary(group_id, encoding="utf8") } details[u'encoded'] = details[u"group_id"] != group_id details_blob = utils.dumps(details) with open(path, "wb") as fh: fh.write(details_blob)
def _update_group_metadata(self, path, group_id): details = {u'group_id': utils.to_binary(group_id, encoding="utf8")} details[u'encoded'] = details[u"group_id"] != group_id details_blob = utils.dumps(details) fd, name = tempfile.mkstemp("tooz", dir=self._tmpdir) with os.fdopen(fd, "wb") as fh: fh.write(details_blob) os.rename(name, path)
def _update_group_metadata(self, path, group_id): details = { u'group_id': utils.to_binary(group_id, encoding="utf8") } details[u'encoded'] = details[u"group_id"] != group_id details_blob = utils.dumps(details) fd, name = tempfile.mkstemp("tooz", dir=self._tmpdir) with os.fdopen(fd, "wb") as fh: fh.write(details_blob) os.rename(name, path)
def _do_create_group(): if os.path.isdir(group_dir): raise coordination.GroupAlreadyExist(group_id) else: details = { 'group_id': group_id, } details_blob = utils.dumps(details) utils.ensure_tree(group_dir) with open(group_meta_path, "wb") as fh: fh.write(details_blob)
def _do_join_group(): if not os.path.isdir(group_dir): raise coordination.GroupNotCreated(group_id) if os.path.isfile(me_path): raise coordination.MemberAlreadyExist(group_id, self._member_id) details = { 'capabilities': capabilities, 'joined_on': datetime.datetime.now(), 'member_id': self._member_id, } details_blob = utils.dumps(details) with open(me_path, "wb") as fh: fh.write(details_blob) self._joined_groups.add(group_id)
def _do_join_group(): if not os.path.exists(os.path.join(group_dir, ".metadata")): raise coordination.GroupNotCreated(group_id) if os.path.isfile(me_path): raise coordination.MemberAlreadyExist(group_id, self._member_id) details = { u'capabilities': capabilities, u'joined_on': datetime.datetime.now(), u'member_id': utils.to_binary(self._member_id, encoding="utf-8") } details[u'encoded'] = details[u"member_id"] != self._member_id details_blob = utils.dumps(details) with open(me_path, "wb") as fh: fh.write(details_blob) self._joined_groups.add(group_id)
def _do_join_group(): if not os.path.isdir(group_dir): raise coordination.GroupNotCreated(group_id) if os.path.isfile(me_path): raise coordination.MemberAlreadyExist(group_id, self._member_id) details = { u'capabilities': capabilities, u'joined_on': datetime.datetime.now(), u'member_id': utils.to_binary(self._member_id, encoding="utf-8") } details[u'encoded'] = details[u"member_id"] != self._member_id details_blob = utils.dumps(details) with open(me_path, "wb") as fh: fh.write(details_blob) self._joined_groups.add(group_id)
def _join_group(): # lock the group so that it doesn't get deleted while we join with self._get_group_lock(group_id): group_path = self._path_group(group_id) member_path = self._path_member(group_id, self._member_id) group, members = self._get_group_members(group_path) if not group: raise coordination.GroupNotCreated(group_id) for m in members: if m["Key"] == member_path: raise coordination.MemberAlreadyExist( group_id, self._member_id) # create with Check-And-Set index 0 will only succeed if the # key doesn't exit self._client.kv.put(member_path, utils.dumps(capabilities), cas=0) self._joined_groups.add(group_id)
def _join_group(): prefix_group = self._prefix_group(group_id) prefix_member = prefix_group + self._member_id members = self.client.get_prefix(prefix_group) encoded_member = _encode(prefix_member) group_metadata = None for cap, metadata in members: if metadata['key'] == prefix_member: raise coordination.MemberAlreadyExist(group_id, self._member_id) if metadata['key'] == prefix_group: group_metadata = metadata if group_metadata is None: raise coordination.GroupNotCreated(group_id) encoded_group = self._encode_group_id(group_id) txn = { 'compare': [{ 'key': encoded_group, 'result': 'EQUAL', 'target': 'VERSION', 'version': int(group_metadata['version']) }], 'success': [{ 'request_put': { 'key': encoded_member, 'value': _encode(utils.dumps(capabilities)), 'lease': self._membership_lease.id } }], 'failure': [] } result = self.client.transaction(txn) if not result.get('succeeded'): raise _retry.TryAgain else: self._joined_groups.add(group_id)
def join_partitioned_group( self, group_id, weight=1, partitions=partitioner.Partitioner.DEFAULT_PARTITION_NUMBER): """Join a group and get a partitioner. A partitioner allows to distribute a bunch of objects across several members using a consistent hash ring. Each object gets assigned (at least) one member responsible for it. It's then possible to check which object is owned by any member of the group. This method also creates if necessary, and joins the group with the selected weight. :param group_id: The group to create a partitioner for. :param weight: The weight to use in the hashring for this node. :param partitions: The number of partitions to create. :return: A :py:class:`~tooz.partitioner.Partitioner` object. """ self.join_group_create( group_id, capabilities=utils.dumps({'weight': weight})) return partitioner.Partitioner(self, group_id, partitions=partitions)
def _join_group(): members = list(self.client.get_prefix(encoded_group)) encoded_member = self._encode_group_member_id( group_id, self._member_id) group_metadata = None for cap, metadata in members: if metadata.key == encoded_member: raise coordination.MemberAlreadyExist(group_id, self._member_id) if metadata.key == encoded_group: group_metadata = metadata if group_metadata is None: raise coordination.GroupNotCreated(group_id) status, results = self.client.transaction( # This comparison makes sure the group has not been deleted in # the mean time compare=[ self.client.transactions.version(encoded_group) == group_metadata.version ], success=[ self.client.transactions.put(encoded_member, utils.dumps(capabilities), lease=self._membership_lease) ], failure=[], ) if not status: # TODO(jd) There's a small optimization doable by getting the # current range on failure and passing it to this function as # the first arg when retrying to avoid redoing a get_prefix() raise _retry.TryAgain
def _join_group(): members = list(self.client.get_prefix(encoded_group)) encoded_member = self._encode_group_member_id( group_id, self._member_id) group_metadata = None for cap, metadata in members: if metadata.key == encoded_member: raise coordination.MemberAlreadyExist( group_id, self._member_id) if metadata.key == encoded_group: group_metadata = metadata if group_metadata is None: raise coordination.GroupNotCreated(group_id) status, results = self.client.transaction( # This comparison makes sure the group has not been deleted in # the mean time compare=[ self.client.transactions.version(encoded_group) == group_metadata.version ], success=[ self.client.transactions.put(encoded_member, utils.dumps(capabilities), lease=self._membership_lease) ], failure=[], ) if not status: # TODO(jd) There's a small optimization doable by getting the # current range on failure and passing it to this function as # the first arg when retrying to avoid redoing a get_prefix() raise _retry.TryAgain
def _update_group_metadata(self, path, group_id): details = {u'group_id': utils.to_binary(group_id, encoding="utf8")} details[u'encoded'] = details[u"group_id"] != group_id details_blob = utils.dumps(details) with open(path, "wb") as fh: fh.write(details_blob)
def _msgpack_serializer(key, value): if isinstance(value, six.binary_type): return value, 1 return utils.dumps(value), 2
def _dumps(data): return utils.dumps(data)
def _msgpack_serializer(key, value): if isinstance(value, bytes): return value, 1 return utils.dumps(value), 2