def _ShardedOp(self, key_value_tuples, shard_execute_fn):
"""Perform an operation on each shard in the cluster, and aggregate the
results. The callback will be invoked for each shard, with the keys, virtual
keys, and values the belong to that shard.
Args:
key_value_tuples - List of tuples of (key, value) for the operation.
shard_execute_fn - Callback for executing the desired operation in each
shard. The signature is f(shard, keys, vkeys, values) => Operation
Returns:
A CompountOperation with the results of the queries. The response_value
field is the combined response_value of each sub-operation (assuming all
the sub-operations succeeded and returned a response_value)
"""
# Compute the shard info for all the keys, and group them by the shard in
# which they reside.
vkeys_by_shard = defaultdict(list)
for i, (key, val) in enumerate(key_value_tuples):
(shard_num, vkey) = self._GetShardInfo(key)
vkeys_by_shard[shard_num].append((i, key, vkey, val))
# Split the request for each shard into a separate greenlet to allow the
# requests to happen in parallel.
op = CompoundOperation()
responses = []
def _ShardGreenletWraper(shard_num, vkey_tuples):
indices, keys, vkeys, values = [list(i) for i in zip(*vkey_tuples)]
try:
sub_op = shard_execute_fn(self.shards[shard_num], keys, vkeys,
values)
except Exception:
sub_op = Operation(success=False,
traceback=traceback.format_exc())
# Aggregate the results.
op.AddOp(sub_op)
if sub_op.response_value:
responses.extend(zip(indices, sub_op.response_value))
greenlets = [
gevent.spawn(_ShardGreenletWraper, shard_num, vkey_tuples)
for shard_num, vkey_tuples in vkeys_by_shard.iteritems()
]
gevent.joinall(greenlets)
# Sort the combined responses back into the request order.
responses = sorted(responses, key=lambda r: r[0])
op.response_value = [r[1] for r in responses]
return op
def _ShardedOp(self, key_value_tuples, shard_execute_fn):
"""Perform an operation on each shard in the cluster, and aggregate the
results. The callback will be invoked for each shard, with the keys, virtual
keys, and values the belong to that shard.
Args:
key_value_tuples - List of tuples of (key, value) for the operation.
shard_execute_fn - Callback for executing the desired operation in each
shard. The signature is f(shard, keys, vkeys, values) => Operation
Returns:
A CompountOperation with the results of the queries. The response_value
field is the combined response_value of each sub-operation (assuming all
the sub-operations succeeded and returned a response_value)
"""
# Compute the shard info for all the keys, and group them by the shard in
# which they reside.
vkeys_by_shard = defaultdict(list)
for i, (key, val) in enumerate(key_value_tuples):
(shard_num, vkey) = self._GetShardInfo(key)
vkeys_by_shard[shard_num].append((i, key, vkey, val))
# Split the request for each shard into a separate greenlet to allow the
# requests to happen in parallel.
op = CompoundOperation()
responses = []
def _ShardGreenletWraper(shard_num, vkey_tuples):
indices, keys, vkeys, values = [list(i) for i in zip(*vkey_tuples)]
try:
sub_op = shard_execute_fn(self.shards[shard_num], keys, vkeys, values)
except Exception:
sub_op = Operation(success=False, traceback=traceback.format_exc())
# Aggregate the results.
op.AddOp(sub_op)
if sub_op.response_value:
responses.extend(zip(indices, sub_op.response_value))
greenlets = [
gevent.spawn(_ShardGreenletWraper, shard_num, vkey_tuples)
for shard_num, vkey_tuples in vkeys_by_shard.iteritems()]
gevent.joinall(greenlets)
# Sort the combined responses back into the request order.
responses = sorted(responses, key=lambda r: r[0])
op.response_value = [r[1] for r in responses]
return op
def _ShardCheckAndSet(self, shard, keys, vkeys, values):
op = CompoundOperation()
# Split the request into batches to avoid locking too many keys at once.
greenlets = [
gevent.spawn(self._ShardCheckAndSetBatch, shard,
keys[i:i + CAS_BATCH_SIZE],
vkeys[i:i + CAS_BATCH_SIZE],
values[i:i + CAS_BATCH_SIZE])
for i in xrange(0, len(keys), CAS_BATCH_SIZE)
]
gevent.joinall(greenlets)
# Get the results and combine them into a single Operation.
[op.AddOp(g.get()) for g in greenlets]
op.response_value = []
for sub_op in op.sub_operations:
if sub_op.success:
op.response_value.extend(sub_op.response_value)
return op
def _ShardCheckAndSet(self, shard, keys, vkeys, values):
op = CompoundOperation()
# Split the request into batches to avoid locking too many keys at once.
greenlets = [
gevent.spawn(
self._ShardCheckAndSetBatch,
shard,
keys[i:i+CAS_BATCH_SIZE],
vkeys[i:i+CAS_BATCH_SIZE],
values[i:i+CAS_BATCH_SIZE])
for i in xrange(0, len(keys), CAS_BATCH_SIZE)]
gevent.joinall(greenlets)
# Get the results and combine them into a single Operation.
[op.AddOp(g.get()) for g in greenlets]
op.response_value = []
for sub_op in op.sub_operations:
if sub_op.success:
op.response_value.extend(sub_op.response_value)
return op
def _UpdateIdSets(self, id_tuples):
"""Update the sets of Client IDs and Taba Names.
Args:
id_tuples - List of tuples of the form (client_id, name).
Returns:
Operation object with the query results.
"""
op = CompoundOperation()
# Build a map of Client ID to Taba Names
client_id_names_map = defaultdict(list)
for client_id, name in id_tuples:
client_id_names_map[client_id].append(name)
try:
# Add the Client IDs to the Set of all Client IDs.
op_clients = self.ClientIdsAdd(client_id_names_map.keys())
op.AddOp(op_clients)
# Add all the Taba Names to the Set of all names across all Client IDs.
all_names = set()
all_names.update(*client_id_names_map.values())
op_names_all = self.TabaNamesForAllAdd(all_names)
op.AddOp(op_names_all)
# Add the Taba Names for each Client ID.
for client_id, names in client_id_names_map.iteritems():
op_names = self.TabaNamesForClientAdd(client_id, names)
op.AddOp(op_names)
except Exception:
LOG.error("Error updating Client/Name sets")
LOG.error(traceback.format_exc())
op.AddOp(Operation(success=False,
traceback=traceback.format_exc()))
return op
