def test_basic(self):
timer = 0
self.config.next_nodes = self.servers
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'v1'),
timer)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client.run(timer)
self.run_servers(timer)
self.assertEqual(self.server_apps[0]._store['k1'], 'v1')
self.assertEqual(self.server_apps[1]._store['k1'], 'v1')
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
0)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client.run(timer)
self.run_servers(timer)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
1)
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k1'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client.run(timer)
self.run_servers(timer)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
1)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET],
1)
self.assertEqual(self.stats.cache_hits, 1)
self.assertEqual(self.stats.cache_misses, 0)
self.client_app._execute(kv.Operation(kv.Operation.Type.DEL, 'k1'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client.run(timer)
self.run_servers(timer)
self.assertFalse('k1' in self.server_apps[0]._store)
self.assertFalse('k1' in self.server_apps[1]._store)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
1)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET],
1)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.DEL],
1)
self.assertEqual(self.stats.cache_hits, 1)
self.assertEqual(self.stats.cache_misses, 0)
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k1'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client.run(timer)
self.run_servers(timer)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET],
2)
self.assertEqual(self.stats.cache_hits, 1)
self.assertEqual(self.stats.cache_misses, 1)
def test_basic_migration(self):
timer = 5
self.server_apps[1]._store['k6'] = 'v6'
self.server_apps[1]._store['k7'] = 'v7'
self.server_apps[1]._store['k8'] = 'v8'
self.server_apps[2]._store['kk'] = 'vk'
self.config.install_key('k6')
self.config.key_rates['k6'] = memcachekv.KeyRate(4, 1000000)
self.config.install_key('k7')
self.config.key_rates['k7'] = memcachekv.KeyRate(2, 1000000)
self.config.install_key('k8')
self.config.key_rates['k8'] = memcachekv.KeyRate(2, 1000000)
self.config.install_key('kk')
self.config.key_rates['kk'] = memcachekv.KeyRate(4, 1000000)
self.config.ploads[1] = 8
self.config.iloads[1] = 3
self.config.ploads[2] = 4
self.config.iloads[2] = 1
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k6'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_servers(timer)
self.assertEqual(self.server_apps[2]._store['k7'], 'v7')
self.assertEqual(self.server_apps[2]._store['k8'], 'v8')
self.assertFalse('k6' in self.server_apps[2]._store)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET], 1)
self.config.c = 100 # prevent migration
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k6', 'V6'),
timer)
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k7', 'V7'),
timer)
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k8', 'V8'),
timer)
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'kk', 'Vk'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + 3*param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_servers(timer)
self.assertEqual(self.server_apps[1]._store['k6'], 'V6')
self.assertEqual(self.server_apps[1]._store['k7'], 'v7')
self.assertEqual(self.server_apps[1]._store['k8'], 'v8')
self.assertEqual(self.server_apps[2]._store['k7'], 'V7')
self.assertEqual(self.server_apps[2]._store['k8'], 'V8')
self.assertEqual(self.server_apps[2]._store['kk'], 'Vk')
self.assertFalse('k6' in self.server_apps[2]._store)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET], 1)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT], 4)
def test_migration(self):
timer = 0
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'v1'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_servers(timer)
self.assertEqual(self.server_apps[0]._store['k1'], 'v1')
self.assertFalse('k1' in self.server_apps[1]._store)
for _ in range(2):
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k1'),
timer)
timer += param.MAX_PROPG_DELAY + 2 * param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_servers(timer)
self.assertFalse('k1' in self.server_apps[1]._store)
timer += param.MAX_PROPG_DELAY + 2 * param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_servers(timer)
self.assertEqual(self.server_apps[0]._store['k1'], 'v1')
self.assertEqual(self.server_apps[1]._store['k1'], 'v1')
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET], 2)
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'v2'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_servers(timer)
self.assertEqual(self.server_apps[0]._store['k1'], 'v1')
self.assertEqual(self.server_apps[1]._store['k1'], 'v2')
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT], 2)
for _ in range(2):
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k1'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + 2 * param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_servers(timer)
self.assertEqual(self.server_apps[0]._store['k1'], 'v1')
self.assertEqual(self.server_apps[1]._store['k1'], 'v2')
self.assertEqual(self.server_apps[2]._store['k1'], 'v2')
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'v3'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_servers(timer)
self.assertEqual(self.server_apps[0]._store['k1'], 'v1')
self.assertEqual(self.server_apps[1]._store['k1'], 'v2')
self.assertEqual(self.server_apps[2]._store['k1'], 'v3')
def test_rounding_migration(self):
timer = 5
self.server_apps[0]._store['k5'] = 'v5'
self.server_apps[0]._store['k4'] = 'v4'
self.server_apps[0]._store['k2'] = 'v2'
self.server_apps[3]._store['k1'] = 'v1'
self.config.install_key('k5')
self.config.key_rates['k5'] = memcachekv.KeyRate(2, 1000000)
self.config.install_key('k4')
self.config.key_rates['k4'] = memcachekv.KeyRate(2, 1000000)
self.config.install_key('k2')
self.config.key_rates['k2'] = memcachekv.KeyRate(4, 1000000)
self.config.install_key('k1')
self.config.key_rates['k1'] = memcachekv.KeyRate(4, 1000000)
self.config.ploads[0] = 8
self.config.iloads[0] = 3
self.config.ploads[3] = 4
self.config.iloads[3] = 1
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k5'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_servers(timer)
self.assertEqual(self.server_apps[1]._store['k5'], 'v5')
self.assertEqual(self.server_apps[1]._store['k4'], 'v4')
self.assertFalse('k2' in self.server_apps[1]._store)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET], 1)
self.config.c = 100 # prevent migration
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k5', 'V5'),
timer)
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k4', 'V4'),
timer)
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k2', 'V2'),
timer)
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'V1'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + 3*param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_servers(timer)
self.assertEqual(self.server_apps[0]._store['k2'], 'V2')
self.assertEqual(self.server_apps[0]._store['k5'], 'v5')
self.assertEqual(self.server_apps[0]._store['k4'], 'v4')
self.assertEqual(self.server_apps[1]._store['k5'], 'V5')
self.assertEqual(self.server_apps[1]._store['k4'], 'V4')
self.assertEqual(self.server_apps[3]._store['k1'], 'V1')
self.assertFalse('k2' in self.server_apps[1]._store)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET], 1)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT], 4)
def __init__(self):
self.ops = [(kv.Operation(kv.Operation.Type.PUT,
"k1",
"v1"),
0),
(kv.Operation(kv.Operation.Type.PUT,
"k2",
"v2"),
round(0.5*param.propg_delay())),
(kv.Operation(kv.Operation.Type.GET,
"k1"),
round(1.2*param.propg_delay())),
(kv.Operation(kv.Operation.Type.GET,
"k3"),
round(1.7*param.propg_delay())),
(kv.Operation(kv.Operation.Type.PUT,
"k3",
"v3"),
round(2.5*param.propg_delay())),
(kv.Operation(kv.Operation.Type.GET,
"k3"),
round(3*param.propg_delay())),
(kv.Operation(kv.Operation.Type.GET,
"k2"),
round(3.9*param.propg_delay())),
(kv.Operation(kv.Operation.Type.DEL,
"k1"),
round(4.3*param.propg_delay())),
(kv.Operation(kv.Operation.Type.GET,
"k1"),
round(5*param.propg_delay()))]
def test_basic(self):
timer = 0
op = kv.Operation(kv.Operation.Type.PUT, "k1", "v1")
request = kv.DBRequest(self.client, timer, op)
self.client.send_message(request, self.server, timer)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC + kv.DB_REQUEST_LTC
self.server.run(timer)
self.client.run(timer)
self.assertEqual(self.server_app._store['k1'], 'v1')
self.assertEqual(len(self.client_app.received_replies), 0)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.server.run(timer)
self.client.run(timer)
self.assertEqual(len(self.client_app.received_replies), 1)
reply = self.client_app.received_replies[0]
self.assertEqual(reply.operation, op)
self.assertEqual(reply.result, kv.Result.OK)
op = kv.Operation(kv.Operation.Type.GET, "k1")
request = kv.DBRequest(self.client, timer, op)
self.client.send_message(request, self.server, timer)
timer += 2 * (param.MAX_PROPG_DELAY +
param.MAX_PKT_PROC_LTC) + kv.DB_REQUEST_LTC
self.server.run(timer)
self.client.run(timer)
self.assertEqual(len(self.client_app.received_replies), 2)
reply = self.client_app.received_replies[1]
self.assertEqual(reply.operation, op)
self.assertEqual(reply.result, kv.Result.OK)
self.assertEqual(reply.value, "v1")
op = kv.Operation(kv.Operation.Type.DEL, "k1")
request = kv.DBRequest(self.client, timer, op)
self.client.send_message(request, self.server, timer)
timer += 2 * (param.MAX_PROPG_DELAY +
param.MAX_PKT_PROC_LTC) + kv.DB_REQUEST_LTC
self.server.run(timer)
self.client.run(timer)
self.assertEqual(len(self.client_app.received_replies), 3)
reply = self.client_app.received_replies[2]
self.assertEqual(reply.operation, op)
self.assertEqual(reply.result, kv.Result.OK)
op = kv.Operation(kv.Operation.Type.GET, "k1")
request = kv.DBRequest(self.client, timer, op)
self.client.send_message(request, self.server, timer)
timer += 2 * (param.MAX_PROPG_DELAY +
param.MAX_PKT_PROC_LTC) + kv.DB_REQUEST_LTC
self.server.run(timer)
self.client.run(timer)
self.assertEqual(len(self.client_app.received_replies), 4)
reply = self.client_app.received_replies[3]
self.assertEqual(reply.operation, op)
self.assertEqual(reply.result, kv.Result.NOT_FOUND)
def test_basic(self):
self.server_apps[0]._store['k1'] = 'v1'
self.server_apps[0]._store['k2'] = 'v2'
self.server_apps[0]._store['k3'] = 'v3'
self.server_apps[0]._store['k4'] = 'v4'
self.config.key_node_map['k1'] = 0
self.config.migration_requests = [memcachekv.MemcacheKVRequest.MigrationRequest(['k1', 'k2'],
self.servers[1]),
memcachekv.MemcacheKVRequest.MigrationRequest(['k3'],
self.servers[2]),
memcachekv.MemcacheKVRequest.MigrationRequest(['k4'],
self.servers[3])]
self.assertFalse('k1' in self.server_apps[1]._store)
self.assertFalse('k2' in self.server_apps[1]._store)
self.assertFalse('k3' in self.server_apps[2]._store)
self.assertFalse('k4' in self.server_apps[3]._store)
timer = 0
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k1'), timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client.run(timer)
self.run_servers(timer)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET],
1)
self.assertEqual(self.server_apps[1]._store['k1'], 'v1')
self.assertEqual(self.server_apps[1]._store['k2'], 'v2')
self.assertEqual(self.server_apps[2]._store['k3'], 'v3')
self.assertEqual(self.server_apps[3]._store['k4'], 'v4')
def _process_message(self, message, time):
if isinstance(message, MemcacheKVRequest):
if self._config.report_load:
count = self._key_request_counter.get(message.operation.key,
0) + 1
self._key_request_counter[message.operation.key] = count
result, value = self._execute_op(message.operation)
reply = MemcacheKVReply(src=self._node,
req_id=message.req_id,
result=result,
value=value)
self._node.send_message(reply, message.src, time)
if message.migration_requests is not None:
for request in message.migration_requests:
if len(request.keys) > 0:
ops = []
for key in request.keys:
ops.append(
kv.Operation(op_type=kv.Operation.Type.PUT,
key=key,
value=self._store.get(key, "")))
msg = MemcacheMigrationRequest(ops)
self._node.send_message(msg, request.dst, time)
elif isinstance(message, MemcacheMigrationRequest):
for op in message.ops:
self._execute_op(op)
else:
raise ValueError("Invalid message type")
def test_basic(self):
result, value = self.kvapp._execute_op(kv.Operation(kv.Operation.Type.PUT, 'k1', 'v1'))
self.assertEqual(result, kv.Result.OK)
self.assertEqual(len(value), 0)
result, value = self.kvapp._execute_op(kv.Operation(kv.Operation.Type.GET, 'k1'))
self.assertEqual(result, kv.Result.OK)
self.assertEqual(value, 'v1')
result, value = self.kvapp._execute_op(kv.Operation(kv.Operation.Type.GET, 'k2'))
self.assertEqual(result, kv.Result.NOT_FOUND)
self.assertEqual(len(value), 0)
result, value = self.kvapp._execute_op(kv.Operation(kv.Operation.Type.DEL, 'k1'))
self.assertEqual(result, kv.Result.OK)
self.assertEqual(len(value), 0)
result, value = self.kvapp._execute_op(kv.Operation(kv.Operation.Type.GET, 'k1'))
self.assertEqual(result, kv.Result.NOT_FOUND)
self.assertEqual(len(value), 0)
def _execute(self, op, time):
"""
Always send the operation to a remote node. Client nodes
in a MemcacheKV are stateless, and do not store kv pairs.
If the key is replicated on multiple cache nodes
(key_to_nodes returns multiple nodes), pick one node in
random for GET requests, and send to all replicated nodes
for DEL requests. PUT request handling depends on the
write_mode.
"""
mapped_nodes = self._config.key_to_nodes(op.key, op.op_type)
pending_req = self.PendingRequest(operation=op, time=time)
msg = MemcacheKVRequest(
src=self._node,
req_id=self._next_req_id,
operation=op,
migration_requests=mapped_nodes.migration_requests)
if op.op_type == kv.Operation.Type.GET:
node = random.choice(mapped_nodes.dst_nodes)
self._node.send_message(msg, node, time)
self._config.report_op_send(node, op, time)
elif op.op_type == kv.Operation.Type.PUT:
write_nodes = []
inval_nodes = []
if self._config.write_mode == WriteMode.ANYNODE:
write_nodes = [random.choice(mapped_nodes.dst_nodes)]
elif self._config.write_mode == WriteMode.UPDATE:
write_nodes = mapped_nodes.dst_nodes
elif self._config.write_mode == WriteMode.INVALIDATE:
write_nodes = mapped_nodes.dst_nodes[:1]
inval_nodes = mapped_nodes.dst_nodes[1:]
else:
raise ValueError("Invalid write mode")
for node in write_nodes:
self._node.send_message(msg, node, time)
self._config.report_op_send(node, op, time)
inval_msg = MemcacheKVRequest(src=self._node,
req_id=self._next_req_id,
operation=kv.Operation(
op_type=kv.Operation.Type.DEL,
key=op.key))
for node in inval_nodes:
self._node.send_message(inval_msg, node, time)
self._config.report_op_send(node, inval_msg.operation, time)
pending_req.expected_acks = len(write_nodes) + len(inval_nodes)
elif op.op_type == kv.Operation.Type.DEL:
for node in mapped_nodes.dst_nodes:
self._node.send_message(msg, node, time)
self._config.report_op_send(node, op, time)
pending_req.expected_acks = len(mapped_nodes.dst_nodes)
else:
raise ValueError("Invalid operation type")
self._pending_requests[self._next_req_id] = pending_req
self._next_req_id += 1
def test_basic(self):
timer = 0
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'v1'),
timer)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client.run(timer)
self.server.run(timer)
self.assertEqual(self.server_app._store['k1'], 'v1')
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
0)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client.run(timer)
self.server.run(timer)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
1)
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k1'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client.run(timer)
self.server.run(timer)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
1)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET],
1)
self.assertEqual(self.stats.cache_hits, 1)
self.assertEqual(self.stats.cache_misses, 0)
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k2'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client.run(timer)
self.server.run(timer)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
1)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET],
2)
self.assertEqual(self.stats.cache_hits, 1)
self.assertEqual(self.stats.cache_misses, 1)
self.assertEqual(len(self.client_app._store), 0)
self.assertEqual(len(self.server_app._store), 1)
def test_basic(self):
timer = 0
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'v1'), timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_servers(timer)
self.assertEqual(self.server_apps[0]._store['k1'], 'v1')
self.assertEqual(self.server_apps[1]._store['k1'], 'v1')
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k1'), timer)
self.assertEqual(len(self.cache_nodes[0]._inflight_messages), 1)
self.assertEqual(len(self.cache_nodes[1]._inflight_messages), 0)
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k1'), timer)
self.assertEqual(len(self.cache_nodes[0]._inflight_messages), 1)
self.assertEqual(len(self.cache_nodes[1]._inflight_messages), 1)
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k1'), timer)
self.assertEqual(len(self.cache_nodes[0]._inflight_messages), 2)
self.assertEqual(len(self.cache_nodes[1]._inflight_messages), 1)
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k1'), timer)
self.assertEqual(len(self.cache_nodes[0]._inflight_messages), 2)
self.assertEqual(len(self.cache_nodes[1]._inflight_messages), 2)
def test_basic(self):
self.config.c = 100 # prevent migration
timer = 0
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'v1'),
timer)
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k2', 'v2'),
timer)
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k6', 'v6'),
timer)
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'kk', 'vk'),
timer)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_servers(timer)
timer += param.MAX_PROPG_DELAY + 4 * param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_servers(timer)
self.assertEqual(self.server_apps[3]._store['k1'], 'v1')
self.assertEqual(self.server_apps[0]._store['k2'], 'v2')
self.assertEqual(self.server_apps[1]._store['k6'], 'v6')
self.assertEqual(self.server_apps[2]._store['kk'], 'vk')
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT], 4)
def next_operation(self):
if self._key_type == KeyType.UNIFORM:
key = random.choice(self._keys)
elif self._key_type == KeyType.ZIPF:
key = self._keys[self.zipf_key_index()]
else:
raise ValueError("Invalide key distribution type")
op_choice = random.uniform(0, 1)
op_type = None
if op_choice < self._get_ratio:
if self._initkey:
if key not in self._initialized_keys:
op_type = kv.Operation.Type.PUT
else:
op_type = kv.Operation.Type.GET
else:
op_type = kv.Operation.Type.GET
elif op_choice < self._get_ratio + self._put_ratio:
op_type = kv.Operation.Type.PUT
else:
op_type = kv.Operation.Type.DEL
op = None
if op_type == kv.Operation.Type.PUT:
if self._initkey:
self._initialized_keys.add(key)
op = kv.Operation(op_type, key, self._value)
else:
op = kv.Operation(op_type, key)
if self._interval_type == IntervalType.UNIFORM:
self._timer += self._mean_interval
elif self._interval_type == IntervalType.POISS:
self._timer += random.expovariate(1 / self._mean_interval)
else:
raise ValueError("Invalid interval distribution type")
return (op, self._timer)
def _check_load_and_migrate(self, key, time):
# If key is already migrated (but the routing is not updated yet),
# do not migrate again
if key in self._migrated_keys:
return
# Check if node is overloaded. If yes, migrate key value pair
# to another node.
total_iload = sum(self._config.iloads.values())
expected_iload = (self._config.c * total_iload) / len(
self._config.cache_nodes)
total_pload = sum(self._config.ploads.values())
expected_pload = (self._config.c * total_pload) / len(
self._config.cache_nodes)
if self._config.iloads[self._node.id] <= expected_iload:
return
if self._config.ploads[self._node.id] <= expected_pload:
return
for dst_node_id in sorted(self._config.ploads,
key=self._config.ploads.get):
if self._config.ploads[dst_node_id] > expected_pload:
return
if self._config.iloads[dst_node_id] <= expected_iload:
break
assert dst_node_id != self._node.id
key_rate = self._config.key_rates.get(key, KeyRate())
self._config.ploads[self._node.id] -= key_rate.rate()
self._config.ploads[dst_node_id] += key_rate.rate()
request = MemcacheMigrationRequest(ops=[
kv.Operation(op_type=kv.Operation.Type.PUT,
key=key,
value=self._store.get(key, ""))
])
self._node.send_message(request, self._config.cache_nodes[dst_node_id],
time)
self._migrated_keys.add(key)
def test_write_modes(self):
timer = 0
# ANYNODE write mode
self.config.write_mode = memcachekv.WriteMode.ANYNODE
self.config.next_nodes = self.servers
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'v1'),
timer)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client.run(timer)
self.run_servers(timer)
if 'k1' in self.server_apps[0]._store:
self.assertEqual(self.server_apps[0]._store['k1'], 'v1')
self.assertFalse('k1' in self.server_apps[1]._store)
self.config.next_nodes = [self.servers[1]]
if 'k1' in self.server_apps[1]._store:
self.assertEqual(self.server_apps[1]._store['k1'], 'v1')
self.assertFalse('k1' in self.server_apps[0]._store)
self.config.next_nodes = [self.servers[0]]
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
0)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client.run(timer)
self.run_servers(timer)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
1)
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'v1'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client.run(timer)
self.run_servers(timer)
self.assertEqual(self.server_apps[0]._store['k1'], 'v1')
self.assertEqual(self.server_apps[1]._store['k1'], 'v1')
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
2)
self.config.next_nodes = self.servers
self.client_app._execute(kv.Operation(kv.Operation.Type.DEL, 'k1'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client.run(timer)
self.run_servers(timer)
self.assertFalse('k1' in self.server_apps[0]._store)
self.assertFalse('k1' in self.server_apps[1]._store)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
2)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.DEL],
1)
# INVALIDATE write mode
self.config.write_mode = memcachekv.WriteMode.INVALIDATE
self.config.next_nodes = [self.servers[0]]
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'v1'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client.run(timer)
self.run_servers(timer)
self.config.next_nodes = [self.servers[1]]
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'v1'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client.run(timer)
self.run_servers(timer)
self.assertEqual(self.server_apps[0]._store['k1'], 'v1')
self.assertEqual(self.server_apps[1]._store['k1'], 'v1')
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
4)
self.config.next_nodes = self.servers
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'v2'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client.run(timer)
self.run_servers(timer)
self.assertEqual(self.server_apps[0]._store['k1'], 'v2')
self.assertFalse('k1' in self.server_apps[1]._store)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
5)
self.config.write_mode = memcachekv.WriteMode.UPDATE
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'v3'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client.run(timer)
self.run_servers(timer)
self.assertEqual(self.server_apps[0]._store['k1'], 'v3')
self.assertEqual(self.server_apps[1]._store['k1'], 'v3')
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
6)
self.config.write_mode = memcachekv.WriteMode.INVALIDATE
self.config.next_nodes = [self.servers[1], self.servers[0]]
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'v4'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client.run(timer)
self.run_servers(timer)
self.assertEqual(self.server_apps[1]._store['k1'], 'v4')
self.assertFalse('k1' in self.server_apps[0]._store)
def test_basic(self):
timer = 0
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'v1'),
timer)
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k2', 'v2'),
timer)
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k3', 'v3'),
timer)
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k4', 'v4'),
timer)
for node in self.cache_nodes:
self.assertEqual(len(node._inflight_messages), 1)
for _ in range(1):
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k1'),
timer)
self.assertEqual(len(self.cache_nodes[0]._inflight_messages), 2)
self.assertEqual(len(self.cache_nodes[1]._inflight_messages), 1)
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k1'),
timer)
self.assertEqual(len(self.cache_nodes[0]._inflight_messages), 3)
self.assertEqual(len(self.cache_nodes[1]._inflight_messages), 1)
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k1'),
timer)
self.assertEqual(len(self.cache_nodes[0]._inflight_messages), 3)
self.assertEqual(len(self.cache_nodes[1]._inflight_messages), 2)
for _ in range(1):
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k4'),
timer)
self.assertEqual(len(self.cache_nodes[3]._inflight_messages), 2)
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k4'),
timer)
self.assertEqual(len(self.cache_nodes[3]._inflight_messages), 3)
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k4'),
timer)
self.assertEqual(len(self.cache_nodes[3]._inflight_messages), 4)
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k4'),
timer)
self.assertEqual(len(self.cache_nodes[3]._inflight_messages), 5)
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k4'),
timer)
self.assertEqual(len(self.cache_nodes[3]._inflight_messages), 5)
self.assertEqual(len(self.cache_nodes[0]._inflight_messages), 4)
timer += param.MAX_PROPG_DELAY + 5 * param.MAX_PKT_PROC_LTC
self.run_servers(timer)
# Process migration messages
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.run_servers(timer)
self.client_node.run(timer)
for node in self.cache_nodes:
self.assertEqual(len(node._inflight_messages), 0)
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k4'),
timer)
self.assertEqual(len(self.cache_nodes[3]._inflight_messages), 0)
self.assertEqual(len(self.cache_nodes[0]._inflight_messages), 1)
def test_basic(self):
timer = 0
# PUT k1 v1
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'v1'),
timer)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_cache_nodes(timer)
self.assertEqual(self.server_apps[0]._store['k1'], 'v1')
self.assertTrue(0 in self.server_apps[0]._directory['k1'].sharers)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_cache_nodes(timer)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
1)
# GET k1 (node 0)
self.config.next_cache_node = 0
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k1'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_cache_nodes(timer)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET],
1)
self.assertEqual(self.stats.cache_hits, 1)
# PUT k2 v2
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k2', 'v2'),
timer)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_cache_nodes(timer)
self.assertTrue(0 in self.server_apps[0]._directory['k2'].sharers)
self.assertEqual(self.server_apps[0]._store['k2'], 'v2')
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
2)
# GET k1 (node 1)
self.config.next_cache_node = 1
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k1'),
timer)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_cache_nodes(timer)
self.assertFalse(1 in self.server_apps[0]._directory['k1'].sharers)
self.assertFalse('k1' in self.server_apps[1]._store)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET],
1)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_cache_nodes(timer)
self.assertTrue(1 in self.server_apps[0]._directory['k1'].sharers)
self.assertFalse('k1' in self.server_apps[1]._store)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET],
1)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_cache_nodes(timer)
self.assertEqual(self.server_apps[1]._store['k1'], 'v1')
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET],
1)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_cache_nodes(timer)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET],
2)
self.assertEqual(self.stats.cache_hits, 2)
# GET k1 (node 2)
self.config.next_cache_node = 2
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k1'),
timer)
for _ in range(4):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_cache_nodes(timer)
self.assertTrue(2 in self.server_apps[0]._directory['k1'].sharers)
self.assertEqual(self.server_apps[2]._store['k1'], 'v1')
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET],
3)
self.assertEqual(self.stats.cache_hits, 3)
# PUT k1 vv1
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'vv1'),
timer)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_cache_nodes(timer)
self.assertEqual(self.server_apps[0]._directory['k1'].sharers, {0, 1, 2})
self.assertEqual(self.server_apps[0]._store['k1'], 'vv1')
for i in range(1, 3):
self.assertEqual(self.server_apps[i]._store['k1'], 'v1')
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
2)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_cache_nodes(timer)
for i in range(3):
self.assertEqual(self.server_apps[i]._store['k1'], 'vv1')
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
2)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_cache_nodes(timer)
self.assertEqual(self.server_apps[0]._directory['k1'].sharers, {0, 1, 2})
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT],
3)
# DEL k1
self.client_app._execute(kv.Operation(kv.Operation.Type.DEL, 'k1'),
timer)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_cache_nodes(timer)
self.assertEqual(len(self.server_apps[0]._directory['k1'].sharers), 0)
self.assertFalse('k1' in self.server_apps[0]._store)
for i in range(1, 3):
self.assertEqual(self.server_apps[i]._store['k1'], 'vv1')
self.assertEqual(self.stats.received_replies[kv.Operation.Type.DEL],
0)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_cache_nodes(timer)
for i in range(3):
self.assertFalse('k1' in self.server_apps[i]._store)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.DEL],
0)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_cache_nodes(timer)
self.assertEqual(len(self.server_apps[0]._directory['k1'].sharers), 0)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.DEL],
1)
# GET k2 (node 2)
self.config.next_cache_node = 2
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k1'),
timer)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_cache_nodes(timer)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET],
3)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_cache_nodes(timer)
self.assertEqual(len(self.server_apps[0]._directory['k1'].sharers), 0)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET],
3)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_cache_nodes(timer)
self.assertFalse('k1' in self.server_apps[2]._store)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET],
3)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.client_node.run(timer)
self.run_cache_nodes(timer)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET],
4)
self.assertEqual(self.stats.cache_hits, 3)
self.assertEqual(self.stats.cache_misses, 1)
def test_basic(self):
timer = 0
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'v1'),
timer)
self.assertEqual(len(self.cache_nodes[0]._inflight_messages), 1)
for _ in range(2):
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.run_servers(timer)
self.client_node.run(timer)
# Migrate 'k1' from node 0 to node 2
self.assertFalse('k1' in self.server_apps[2]._store)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET], 0)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT], 1)
self.client_app._execute(kv.Operation(kv.Operation.Type.GET, 'k1'),
timer)
self.assertEqual(len(self.cache_nodes[0]._inflight_messages), 1)
self.assertEqual(len(self.cache_nodes[2]._inflight_messages), 0)
self.config.iloads[0] = 2
self.config.iloads[1] = 1
self.config.iloads[2] = 1
self.config.iloads[3] = 1
self.config.ploads[0] = 3
self.config.ploads[1] = 2
self.config.ploads[2] = 0
self.config.ploads[3] = 1
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.run_servers(timer)
self.assertEqual(len(self.cache_nodes[2]._inflight_messages), 1)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.run_servers(timer)
self.client_node.run(timer)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.GET], 1)
self.assertEqual(self.server_apps[2]._store['k1'], 'v1')
self.assertEqual(self.config.key_node_map['k1'], 2)
# Migrate 'k1' from node 2 to node 3
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT], 1)
self.assertFalse('k1' in self.server_apps[3]._store)
self.client_app._execute(kv.Operation(kv.Operation.Type.PUT, 'k1', 'v2'),
timer)
self.assertEqual(len(self.cache_nodes[0]._inflight_messages), 0)
self.assertEqual(len(self.cache_nodes[2]._inflight_messages), 1)
self.config.iloads[0] = 2
self.config.iloads[1] = 1
self.config.iloads[2] = 4
self.config.iloads[3] = 3
self.config.ploads[0] = 3
self.config.ploads[1] = 3
self.config.ploads[2] = 5
self.config.ploads[3] = 1
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.run_servers(timer)
self.assertEqual(len(self.cache_nodes[3]._inflight_messages), 1)
timer += param.MAX_PROPG_DELAY + param.MAX_PKT_PROC_LTC
self.run_servers(timer)
self.client_node.run(timer)
self.assertEqual(self.stats.received_replies[kv.Operation.Type.PUT], 2)
self.assertEqual(self.server_apps[0]._store['k1'], 'v1')
self.assertEqual(self.server_apps[2]._store['k1'], 'v2')
self.assertEqual(self.server_apps[3]._store['k1'], 'v2')
self.assertEqual(self.config.key_node_map['k1'], 3)
