def test_set_get(self, setup):
""" Test that set returns OK and get returns value set """
self.nodes = setup.get("nodes")
storage_types = self.verify_started()
for stype in storage_types:
for i in ["aa", "bb", "", None, True, False, 0, 1, 2, [0, 1], {"xx": 10}]:
key = str(i)
for y in range(3):
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.set("test1-", key + str(y), i, CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10)
# Verify response is CalvinResponse object with OK status
print "set response", self.get_ans, stype, key, y
assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK
for x in range(3):
if stype == "notstarted" and x != y:
# Not started storage is never connected to other nodes storage
continue
self.done = False
self.get_ans = None
self.nodes[stype][x].storage.get("test1-", key + str(y), CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10)
print "get response", self.get_ans, stype, key, x
# Verify we read what is written
assert self.get_ans == i
def test_avail_valid(self):
"""
Test valid values for CPU avail.
Verify if storage is as expected
"""
values = [0, 25, 50, 75, 100]
for i in values:
# verify set return
self.done = False
self.cpu.set_avail(i, CalvinCB(self.cb))
yield wait_for(self._test_done)
assert self.get_ans == True
# verify nodeCpuAvail in DB
self.done = False
self.storage.get(prefix="nodeCpuAvail-",
key=self.node.id,
cb=CalvinCB(self.cb))
yield wait_for(self._test_done)
assert self.get_ans == i
# verify index ok and present for level i
self.done = False
self.storage.get_index(index=self.CPUAVAIL_INDEX_BASE +
map(str, values[:values.index(i) + 1]),
root_prefix_level=2,
cb=CalvinCB(self.cb2))
yield wait_for(self._test_done)
assert self.node.id in self.get_ans
def test_set_delete_get(self, setup):
""" Test that set returns OK, delete return OK and get returns 404 response """
self.nodes = setup.get("nodes")
storage_types = self.verify_started()
for stype in storage_types:
for i in ["aa", "bb", "cc", "", None, True, False, 0, 1, 2, [0, 1], {"xx": 10}]:
for y in range(3):
key = "t2" + str(i)
if i != "cc":
# "cc" is missing without being set and deleted
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.set("test2-", key + str(y), i, CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10)
# Verify response is CalvinResponse object with OK status
assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK
print "set response", self.get_ans, stype, i, y
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.delete("test2-", key + str(y), CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10)
# Verify response is CalvinResponse object with OK status
assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK
print "delete response", self.get_ans, stype, i, y
for x in range(3):
if stype == "notstarted" and x != y:
# Not started storage is never connected to other nodes storage
continue
self.done = False
self.get_ans = None
self.nodes[stype][x].storage.get("test2-", key + str(y), CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10)
print "get response", self.get_ans, stype, i, x
# Verify the response is 404
assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.NOT_FOUND
def test_stop_node(self):
"""
Verify if indexes are cleared after node stop
Old value must be erased from indexes
"""
self.mem.set_avail(25, CalvinCB(self.cb))
yield wait_for(self._test_done)
assert self.get_ans == True
self.done = False
self.storage.get_index(index=self.MEMAVAIL_INDEX_BASE + ['0', '25'], root_prefix_level=2, cb=CalvinCB(self.cb2))
yield wait_for(self._test_done)
assert self.node.id in self.get_ans
self.mem.stop()
self.storage.delete_node(self.node)
# nodeMemAvail must not exist
self.done = False
self.storage.get(prefix="nodeMemAvail-", key=self.node.id, cb=CalvinCB(self.cb))
yield wait_for(self._test_done)
assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.NOT_FOUND
# node id must not be present at level 25
self.done = False
self.storage.get_index(index=self.MEMAVAIL_INDEX_BASE + ['0', '25'], root_prefix_level=2, cb=CalvinCB(self.cb2))
yield wait_for(self._test_done)
assert self.get_ans == []
# no node in total indexes
self.done = False
self.storage.get_index(index=self.MEMTOTAL_INDEX_BASE + ['1K'], root_prefix_level=2, cb=CalvinCB(self.cb2))
yield wait_for(self._test_done)
assert self.get_ans == []
def test_append_multi_remove_getconcat(self, setup):
""" Test that appending multiple values returns OK, remove a value returns OK
and get_concat returns the unique values (set) in a list """
self.nodes = setup.get("nodes")
storage_types = self.verify_started()
for stype in storage_types:
for i in [("aa", "aa"), ("bb", 2, "ff"), (0, 1, 2, 3)]: # values must be hashable (i.e. not list or dict)
key = str(i)
for y in range(3):
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.append("test5-", key + str(y), i, CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10)
# Verify response is CalvinResponse object with OK status
print "append response", self.get_ans, stype, key, y
assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.remove("test5-", key + str(y), i[1:], CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10)
# Verify response is CalvinResponse object with OK status
print "append response", self.get_ans, stype, key, y
assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK
for x in range(3):
if stype == "notstarted" and x != y:
# Not started storage is never connected to other nodes storage
continue
self.done = False
self.get_ans = None
self.nodes[stype][x].storage.get_concat("test5-", key + str(y), CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10)
print "get_concat response", self.get_ans, stype, key, x
# Verify we read what is written
answer = set(i) - set(i[1:])
assert set(self.get_ans) == answer
def test_application_functions(self):
self.q = Queue.Queue()
def cb(key, value):
self.q.put({"key": key, "value": value})
application = appmanager.Application(
calvinuuid.uuid('APP'), "test_app",
[calvinuuid.uuid('ACTOR'),
calvinuuid.uuid('ACTOR')], calvinuuid.uuid("NODE"), None)
self.storage.add_application(application, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
# print value
self.storage.get_application(application.id, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
# print value
assert value["key"] == application.id and value["value"][
"name"] == application.name
self.storage.delete_application(application.id, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert application.id not in self.storage.localstore
def test_append_delete_get_concat(self, setup):
""" Test that append returns OK, delete return OK and get returns empty set """
self.nodes = setup.get("nodes")
storage_types = self.verify_started()
for stype in storage_types:
for i in ["aa", None, True, False, 0, 1, 2]:
for y in range(3):
key = "t3" + str(i)
if i != "cc":
# "cc" is missing without being set and deleted
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.append("test6-", key + str(y), [i], CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10, test_part="append"+key+str(y))
# Verify response is CalvinResponse object with OK status
assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK
print "append response", self.get_ans, stype, i, y
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.delete("test6-", key + str(y), CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10, test_part="delete"+key+str(y))
# Verify response is CalvinResponse object with OK status
assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK
print "delete response", self.get_ans, stype, i, y
for x in range(3):
if stype == "notstarted" and x != y:
# Not started storage is never connected to other nodes storage
continue
self.done = False
self.get_ans = None
self.nodes[stype][x].storage.get_concat("test6-", key + str(y), CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10, test_part="get_concat"+key+str(y))
print "get_concat response", self.get_ans, stype, i, x
# Verify the response is empty list (no difference between emptied or deleted)
assert self.get_ans == []
def test_node_functions(self):
self.q = Queue.Queue()
def cb(key, value):
self.q.put({"key": key, "value": value})
node = calvin.tests.TestNode(["127.0.0.1:5000"])
self.storage.add_node(node, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.OK
self.storage.get_node(node.id, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value["value"] == {u'attributes': {u'indexed_public': [], u'public': {}}, u'control_uris': [u'127.0.0.1:5000'], u'uris': [u'127.0.0.1:5000']}
self.storage.delete_node(node, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.OK
self.storage.get_node(node.id, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.NOT_FOUND
def test_application_functions(self):
self.q = Queue.Queue()
def cb(key, value):
self.q.put({"key": key, "value": value})
application = appmanager.Application(calvinuuid.uuid(
'APP'), "test_app", [calvinuuid.uuid('ACTOR'), calvinuuid.uuid('ACTOR')], calvinuuid.uuid("NODE"), None)
self.storage.add_application(application, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.OK
self.storage.get_application(application.id, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value["value"]["name"] == application.name
self.storage.delete_application(application.id, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.OK
self.storage.get_application(application.id, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.NOT_FOUND
def test_node_functions(self):
self.q = Queue.Queue()
def cb(key, value):
self.q.put({"key": key, "value": value})
node = calvin.tests.TestNode(["127.0.0.1:5000"])
self.storage.add_node(node)
self.storage.get_node(node_id=node.id, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value["key"] == node.id and value["value"] == {
u'attributes': {
u'indexed_public': [],
u'public': {}
},
u'control_uris': [u'127.0.0.1:5000'],
'uris': node.uris
}
self.storage.delete_node(node, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value
assert node.id not in self.storage.localstore
def test_actor_functions(self):
self.q = Queue.Queue()
def cb(key, value):
self.q.put({"key": key, "value": value})
port1 = calvin.tests.TestPort("out", "out")
port2 = calvin.tests.TestPort("in", "in", )
port1.peers = [("local", port2.id)]
port2.peers = [("local", port1.id)]
actor = calvin.tests.TestActor("actor1", "type1", {}, {port1.name: port1})
self.storage.add_actor(actor, calvinuuid.uuid("NODE"), cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.OK
self.storage.get_actor(actor.id, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value["value"]["name"] == actor.name
self.storage.delete_actor(actor.id, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.OK
self.storage.get_actor(actor.id, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.NOT_FOUND
def test_application_functions(self):
self.q = Queue.Queue()
def cb(key, value):
self.q.put({"key": key, "value": value})
application = appmanager.Application(
calvinuuid.uuid('APP'), "test_app",
[calvinuuid.uuid('ACTOR'),
calvinuuid.uuid('ACTOR')], calvinuuid.uuid("NODE"), None)
self.storage.add_application(application, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse
) and value["value"] == calvinresponse.OK
self.storage.get_application(application.id, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value["value"]["name"] == application.name
self.storage.delete_application(application.id, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse
) and value["value"] == calvinresponse.OK
self.storage.get_application(application.id, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse
) and value["value"] == calvinresponse.NOT_FOUND
def test_application_functions(self):
self.q = Queue.Queue()
def cb(key, value):
self.q.put({"key": key, "value": value})
application = appmanager.Application(calvinuuid.uuid(
'APP'), "test_app", [calvinuuid.uuid('ACTOR'), calvinuuid.uuid('ACTOR')], calvinuuid.uuid("NODE"), None)
self.storage.add_application(application, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
# print value
self.storage.get_application(
application.id, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
# print value
assert value["key"] == application.id and value[
"value"]["name"] == application.name
self.storage.delete_application(application.id, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert application.id not in self.storage.localstore
def test_append_multi_getconcat(self, setup):
""" Test that appending multiple values returns OK and get_concat returns the unique values (set) in a list """
self.nodes = setup.get("nodes")
storage_types = self.verify_started()
for stype in storage_types:
for i in [("aa", "aa"), ("bb", 2, "ff"), (True, ""), (False, True)
]: # values must be hashable (i.e. not list or dict)
key = str(i)
for y in range(3):
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.append(
"test4-", key + str(y), i, CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10)
# Verify response is CalvinResponse object with OK status
print "append response", self.get_ans, stype, key, y
assert isinstance(self.get_ans,
calvinresponse.CalvinResponse
) and self.get_ans == calvinresponse.OK
for x in range(3):
if stype == "notstarted" and x != y:
# Not started storage is never connected to other nodes storage
continue
self.done = False
self.get_ans = None
self.nodes[stype][x].storage.get_concat(
"test4-", key + str(y), CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10)
print "get_concat response", self.get_ans, stype, key, x
# Verify we read what is written
assert set(self.get_ans) == set(i)
def test_total_valid(self):
"""
Test valid values for total RAM.
Verify if storage is as expected
"""
values = ["1K", "100K", "1M", "100M", "1G", "10G"]
for i in values:
# verify set return
self.done = False
self.node.attributes = AttributeResolver(
{"indexed_public": {
"memTotal": i
}})
self.storage.add_node(self.node, cb=self.cb)
yield wait_for(self._test_done)
assert isinstance(self.get_ans, calvinresponse.CalvinResponse
) and self.get_ans == calvinresponse.OK
# verify index ok and present for level i
self.done = False
self.storage.get_index(index=self.MEMTOTAL_INDEX_BASE +
map(str, values[:values.index(i) + 1]),
root_prefix_level=2,
cb=CalvinCB(self.cb2))
yield wait_for(self._test_done)
assert self.node.id in self.get_ans
def test_set_get(self, setup):
""" Test that set returns OK and get returns value set """
self.nodes = setup.get("nodes")
storage_types = self.verify_started()
for stype in storage_types:
for i in [
"aa", "bb", "", None, True, False, 0, 1, 2, [0, 1], {
"xx": 10
}
]:
key = str(i)
for y in range(3):
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.set("test1-", key + str(y), i,
CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10)
# Verify response is CalvinResponse object with OK status
print "set response", self.get_ans, stype, key, y
assert isinstance(self.get_ans,
calvinresponse.CalvinResponse
) and self.get_ans == calvinresponse.OK
for x in range(3):
if stype == "notstarted" and x != y:
# Not started storage is never connected to other nodes storage
continue
self.done = False
self.get_ans = None
self.nodes[stype][x].storage.get(
"test1-", key + str(y), CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10)
print "get response", self.get_ans, stype, key, x
# Verify we read what is written
assert self.get_ans == i
def test_append_getconcat(self, setup):
""" Test that single value append returns OK and get_concat returns appended value in list """
self.nodes = setup.get("nodes")
storage_types = self.verify_started()
for stype in storage_types:
for i in ["aa", "bb", "", True, False, 0, 1, 2]: # values must be hashable (i.e. not list or dict)
key = str(i)
for y in range(3):
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.append("test3-", key + str(y), [i], CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10)
# Verify response is CalvinResponse object with OK status
print "append response", self.get_ans, stype, key, y
assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK
for x in range(3):
if stype == "notstarted" and x != y:
# Not started storage is never connected to other nodes storage
continue
self.done = False
self.get_ans = None
self.nodes[stype][x].storage.get_concat("test3-", key + str(y), CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10)
print "get_concat response", self.get_ans, stype, key, x
# Verify we read what is written
assert self.get_ans == [i]
def test_append_delete_append_get_concat(self, setup):
""" Test that append returns OK, delete return OK and get returns second append """
self.nodes = setup.get("nodes")
storage_types = self.verify_started()
for stype in storage_types:
for i in ["aa"]:
for y in range(3):
key = "t3" + str(i)
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.append(
"test7-", key + str(y), [i], CalvinCB(self.cb))
yield wait_for(self._test_done,
timeout=10,
test_part="append" + key + str(y))
# Verify response is CalvinResponse object with OK status
assert isinstance(self.get_ans,
calvinresponse.CalvinResponse
) and self.get_ans == calvinresponse.OK
print "append response", self.get_ans, stype, i, y
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.delete(
"test7-", key + str(y), CalvinCB(self.cb))
yield wait_for(self._test_done,
timeout=10,
test_part="delete" + key + str(y))
# Verify response is CalvinResponse object with OK status
assert isinstance(self.get_ans,
calvinresponse.CalvinResponse
) and self.get_ans == calvinresponse.OK
print "delete response", self.get_ans, stype, i, y
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.append(
"test7-", key + str(y), [i + "2"], CalvinCB(self.cb))
yield wait_for(self._test_done,
timeout=10,
test_part="append2" + key + str(y))
# Verify response is CalvinResponse object with OK status
assert isinstance(self.get_ans,
calvinresponse.CalvinResponse
) and self.get_ans == calvinresponse.OK
print "append2 response", self.get_ans, stype, i, y
for x in range(3):
if stype == "notstarted" and x != y:
# Not started storage is never connected to other nodes storage
continue
self.done = False
self.get_ans = None
self.nodes[stype][x].storage.get_concat(
"test7-", key + str(y), CalvinCB(self.cb))
yield wait_for(self._test_done,
timeout=10,
test_part="get_concat" + key + str(y))
print "get_concat response", self.get_ans, stype, i, x
# Verify the response is second appended value
assert self.get_ans == [i + "2"]
def test_add_remove_get_index(self, setup):
""" Test that add_index returns OK, remove_index returns OK and get_index returns appended value in list for index hierarchies """
self.nodes = setup.get("nodes")
storage_types = self.verify_started()
for stype in storage_types:
for y in range(3):
for i in [(["aa", "bb", "", "dd"], ["xx", "kk", "ll"], "+"),
(["aa", "bb", "", "ee"], ["yy", "kk", "ll"], "+"),
(["aa", "bb"], ["zz", "mm", "oo"], "+"),
(["aa", "ff", "", "dd"], ["xx"], "+"),
(["aa", "bb", "", "dd"], ["kk", "ll"], "-"),
(["aa", "bb", "", "ee"], ["yy", "kk", "ll"], "-"),
(["aa", "bb"], ["oo"], "-"),
(["aa", "gg", "", "dd"], ["xx"], "-")]:
self.done = False
self.get_ans = None
index = copy.copy(i[0])
index[0] = "test2" + index[0] + str(y)
# root_prefix_level is default 2 hence 2 first are combined
if i[2] == "+":
self.nodes[stype][y].storage.add_index(index,
i[1],
cb=CalvinCB(
self.cb2))
else:
self.nodes[stype][y].storage.remove_index(
index, i[1], cb=CalvinCB(self.cb2))
yield wait_for(self._test_done, timeout=10)
# Verify response is CalvinResponse object with OK status
print "add" if i[
2] == "+" else "remove", " response", self.get_ans, stype, index, y
assert isinstance(self.get_ans,
calvinresponse.CalvinResponse
) and self.get_ans == calvinresponse.OK
for x in range(3):
if stype == "notstarted" and x != y:
# Not started storage is never connected to other nodes storage
continue
for i in [(["aa"], []), (["gg"], []), (["aa", "gg"], []),
(["aa", "bb"], ["xx", "zz", "mm"]),
(["aa", "bb", ""], ["xx"]), (["aa",
"ff"], ["xx"]),
(["aa", "bb", "", "dd"], ["xx"]),
(["aa", "bb", "", "ee"], []),
(["aa", "ff", "", "dd"], ["xx"])]:
self.done = False
self.get_ans = None
index = copy.copy(i[0])
index[0] = "test2" + index[0] + str(y)
self.nodes[stype][x].storage.get_index(index,
cb=CalvinCB(
self.cb2))
yield wait_for(self._test_done, timeout=10)
print "get_index response", self.get_ans, stype, index, x
# Verify we read what is written if too short prefix or not existing we should get [].
assert set(self.get_ans) == set(i[1])
def test_add_remove_get_index(self, setup):
""" Test that add_index returns OK, remove_index returns OK and get_index returns appended value in list for index hierarchies """
self.nodes = setup.get("nodes")
storage_types = self.verify_started()
for stype in storage_types:
for y in range(3):
for i in [(["aa", "bb", "", "dd"], ["xx", "kk", "ll"], "+"),
(["aa", "bb", "", "ee"], ["yy", "kk", "ll"], "+"),
(["aa", "bb"], ["zz", "mm", "oo"], "+"),
(["aa", "ff", "", "dd"], ["xx"], "+"),
(["aa", "bb", "", "dd"], ["kk", "ll"], "-"),
(["aa", "bb", "", "ee"], ["yy", "kk", "ll"], "-"),
(["aa", "bb"], ["oo"], "-"),
(["aa", "gg", "", "dd"], ["xx"], "-")
]:
self.done = False
self.get_ans = None
index = copy.copy(i[0])
index[0] = "test2" + index[0] + str(y)
# root_prefix_level is default 2 hence 2 first are combined
if i[2] == "+":
self.nodes[stype][y].storage.add_index(index, i[1], cb=CalvinCB(self.cb2))
else:
self.nodes[stype][y].storage.remove_index(index, i[1], cb=CalvinCB(self.cb2))
yield wait_for(self._test_done, timeout=10)
# Verify response is CalvinResponse object with OK status
print "add" if i[2] == "+" else "remove", " response", self.get_ans, stype, index, y
assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK
for x in range(3):
if stype == "notstarted" and x != y:
# Not started storage is never connected to other nodes storage
continue
for i in [(["aa"], []),
(["gg"], []),
(["aa", "gg"], []),
(["aa", "bb"], ["xx", "zz", "mm"]),
(["aa", "bb", ""], ["xx"]),
(["aa", "ff"], ["xx"]),
(["aa", "bb", "", "dd"], ["xx"]),
(["aa", "bb", "", "ee"], []),
(["aa", "ff", "", "dd"], ["xx"])]:
self.done = False
self.get_ans = None
index = copy.copy(i[0])
index[0] = "test2" + index[0] + str(y)
self.nodes[stype][x].storage.get_index(index, cb=CalvinCB(self.cb2))
yield wait_for(self._test_done, timeout=10)
print "get_index response", self.get_ans, stype, index, x
# Verify we read what is written if too short prefix or not existing we should get [].
assert set(self.get_ans) == set(i[1])
def test_avail_change(self):
"""
Verify if indexes are ok after a change in CPU avail.
Old value must be erased from indexes
"""
self.cpu.set_avail(50)
self.cpu.set_avail(25, CalvinCB(self.cb))
yield wait_for(self._test_done)
assert self.get_ans == True
# node id must not be present at level 50, only at 25
self.done = False
self.storage.get_index(index=self.CPUAVAIL_INDEX_BASE + ['0', '25', '50'], root_prefix_level=2, cb=CalvinCB(self.cb2))
yield wait_for(self._test_done)
assert self.get_ans == []
def test_set_delete_get(self, setup):
""" Test that set returns OK, delete return OK and get returns 404 response """
self.nodes = setup.get("nodes")
storage_types = self.verify_started()
for stype in storage_types:
for i in [
"aa", "bb", "cc", "", None, True, False, 0, 1, 2, [0, 1], {
"xx": 10
}
]:
for y in range(3):
key = "t2" + str(i)
if i != "cc":
# "cc" is missing without being set and deleted
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.set(
"test2-", key + str(y), i, CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10)
# Verify response is CalvinResponse object with OK status
assert isinstance(
self.get_ans, calvinresponse.CalvinResponse
) and self.get_ans == calvinresponse.OK
print "set response", self.get_ans, stype, i, y
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.delete(
"test2-", key + str(y), CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10)
# Verify response is CalvinResponse object with OK status
assert isinstance(
self.get_ans, calvinresponse.CalvinResponse
) and self.get_ans == calvinresponse.OK
print "delete response", self.get_ans, stype, i, y
for x in range(3):
if stype == "notstarted" and x != y:
# Not started storage is never connected to other nodes storage
continue
self.done = False
self.get_ans = None
self.nodes[stype][x].storage.get(
"test2-", key + str(y), CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10)
print "get response", self.get_ans, stype, i, x
# Verify the response is 404
assert isinstance(
self.get_ans, calvinresponse.CalvinResponse
) and self.get_ans == calvinresponse.NOT_FOUND
def test_append_delete_get_concat(self, setup):
""" Test that append returns OK, delete return OK and get returns empty set """
self.nodes = setup.get("nodes")
storage_types = self.verify_started()
for stype in storage_types:
for i in ["aa", None, True, False, 0, 1, 2]:
for y in range(3):
key = "t3" + str(i)
if i != "cc":
# "cc" is missing without being set and deleted
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.append(
"test6-", key + str(y), [i], CalvinCB(self.cb))
yield wait_for(self._test_done,
timeout=10,
test_part="append" + key + str(y))
# Verify response is CalvinResponse object with OK status
assert isinstance(
self.get_ans, calvinresponse.CalvinResponse
) and self.get_ans == calvinresponse.OK
print "append response", self.get_ans, stype, i, y
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.delete(
"test6-", key + str(y), CalvinCB(self.cb))
yield wait_for(self._test_done,
timeout=10,
test_part="delete" + key + str(y))
# Verify response is CalvinResponse object with OK status
assert isinstance(
self.get_ans, calvinresponse.CalvinResponse
) and self.get_ans == calvinresponse.OK
print "delete response", self.get_ans, stype, i, y
for x in range(3):
if stype == "notstarted" and x != y:
# Not started storage is never connected to other nodes storage
continue
self.done = False
self.get_ans = None
self.nodes[stype][x].storage.get_concat(
"test6-", key + str(y), CalvinCB(self.cb))
yield wait_for(self._test_done,
timeout=10,
test_part="get_concat" + key + str(y))
print "get_concat response", self.get_ans, stype, i, x
# Verify the response is empty list (no difference between emptied or deleted)
assert self.get_ans == []
def test_avail_invalid(self):
"""
Verify invalid values for CPU avail
"""
for i in [-1, 40, 101]:
self.done = False
self.cpu.set_avail(i, CalvinCB(self.cb))
yield wait_for(self._test_done)
assert self.get_ans == False
def test_avail_invalid(self):
"""
Verify invalid values for CPU avail
"""
for i in [-1, 40, 101]:
self.done = False
self.cpu.set_avail(i, CalvinCB(self.cb))
yield wait_for(self._test_done)
assert self.get_ans == False
def test_avail_change(self):
"""
Verify if indexes are ok after a change in RAM avail.
Old value must be erased from indexes
"""
self.mem.set_avail(50)
self.mem.set_avail(25, CalvinCB(self.cb))
yield wait_for(self._test_done)
assert self.get_ans == True
# node id must not be present at level 50, only at 25
self.done = False
self.storage.get_index(index=self.MEMAVAIL_INDEX_BASE +
['0', '25', '50'],
root_prefix_level=2,
cb=CalvinCB(self.cb2))
yield wait_for(self._test_done)
assert self.get_ans == []
def test_stop_node(self):
"""
Verify if indexes are cleared after node stop
Old value must be erased from indexes
"""
self.mem.set_avail(25, CalvinCB(self.cb))
yield wait_for(self._test_done)
assert self.get_ans == True
self.done = False
self.storage.get_index(index=self.MEMAVAIL_INDEX_BASE + ['0', '25'],
root_prefix_level=2,
cb=CalvinCB(self.cb2))
yield wait_for(self._test_done)
assert self.node.id in self.get_ans
self.mem.stop()
self.storage.delete_node(self.node)
# nodeMemAvail must not exist
self.done = False
self.storage.get(prefix="nodeMemAvail-",
key=self.node.id,
cb=CalvinCB(self.cb))
yield wait_for(self._test_done)
assert isinstance(self.get_ans, calvinresponse.CalvinResponse
) and self.get_ans == calvinresponse.NOT_FOUND
# node id must not be present at level 25
self.done = False
self.storage.get_index(index=self.MEMAVAIL_INDEX_BASE + ['0', '25'],
root_prefix_level=2,
cb=CalvinCB(self.cb2))
yield wait_for(self._test_done)
assert self.get_ans == []
# no node in total indexes
self.done = False
self.storage.get_index(index=self.MEMTOTAL_INDEX_BASE + ['1K'],
root_prefix_level=2,
cb=CalvinCB(self.cb2))
yield wait_for(self._test_done)
assert self.get_ans == []
def test_node_functions(self):
self.q = Queue.Queue()
def cb(key, value):
self.q.put({"key": key, "value": value})
node = calvin.tests.TestNode(["127.0.0.1:5000"])
self.storage.add_node(node)
self.storage.get_node(node_id=node.id, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value["key"] == node.id and value["value"] == {u'attributes': {u'indexed_public': [], u'public': {}},
u'control_uris': [u'127.0.0.1:5000'], 'uris': node.uris}
self.storage.delete_node(node, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value
assert node.id not in self.storage.localstore
def test_total_valid(self):
"""
Test valid values for total RAM.
Verify if storage is as expected
"""
values = ["1K", "100K", "1M", "100M", "1G", "10G"]
for i in values:
# verify set return
self.done = False
self.node.attributes = AttributeResolver({"indexed_public": {"memTotal": i }})
self.storage.add_node(self.node, cb=self.cb)
yield wait_for(self._test_done)
assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK
# verify index ok and present for level i
self.done = False
self.storage.get_index(index=self.MEMTOTAL_INDEX_BASE + map(str, values[:values.index(i)+1]), root_prefix_level=2, cb=CalvinCB(self.cb2))
yield wait_for(self._test_done)
assert self.node.id in self.get_ans
def test_actor_functions(self):
self.q = Queue.Queue()
def cb(key, value):
self.q.put({"key": key, "value": value})
port1 = calvin.tests.TestPort("out", "out")
port2 = calvin.tests.TestPort(
"in",
"in",
)
port1.peers = [("local", port2.id)]
port2.peers = [("local", port1.id)]
actor = calvin.tests.TestActor("actor1", "type1", {},
{port1.name: port1})
self.storage.add_actor(actor, calvinuuid.uuid("NODE"), cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse
) and value["value"] == calvinresponse.OK
self.storage.get_actor(actor.id, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value["value"]["name"] == actor.name
self.storage.delete_actor(actor.id, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse
) and value["value"] == calvinresponse.OK
self.storage.get_actor(actor.id, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse
) and value["value"] == calvinresponse.NOT_FOUND
def test_append_delete_append_get_concat(self, setup):
""" Test that append returns OK, delete return OK and get returns second append """
self.nodes = setup.get("nodes")
storage_types = self.verify_started()
for stype in storage_types:
for i in ["aa"]:
for y in range(3):
key = "t3" + str(i)
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.append("test7-", key + str(y), [i], CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10, test_part="append"+key+str(y))
# Verify response is CalvinResponse object with OK status
assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK
print "append response", self.get_ans, stype, i, y
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.delete("test7-", key + str(y), CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10, test_part="delete"+key+str(y))
# Verify response is CalvinResponse object with OK status
assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK
print "delete response", self.get_ans, stype, i, y
self.done = False
self.get_ans = None
self.nodes[stype][y].storage.append("test7-", key + str(y), [i+"2"], CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10, test_part="append2"+key+str(y))
# Verify response is CalvinResponse object with OK status
assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK
print "append2 response", self.get_ans, stype, i, y
for x in range(3):
if stype == "notstarted" and x != y:
# Not started storage is never connected to other nodes storage
continue
self.done = False
self.get_ans = None
self.nodes[stype][x].storage.get_concat("test7-", key + str(y), CalvinCB(self.cb))
yield wait_for(self._test_done, timeout=10, test_part="get_concat"+key+str(y))
print "get_concat response", self.get_ans, stype, i, x
# Verify the response is second appended value
assert self.get_ans == [i+"2"]
def test_node_functions(self):
self.q = Queue.Queue()
def cb(key, value):
self.q.put({"key": key, "value": value})
node = calvin.tests.TestNode(["127.0.0.1:5000"])
self.storage.add_node(node, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse
) and value["value"] == calvinresponse.OK
self.storage.get_node(node.id, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value["value"] == {
u'attributes': {
u'indexed_public': [],
u'public': {}
},
u'control_uris': [u'127.0.0.1:5000'],
u'uris': [u'127.0.0.1:5000']
}
self.storage.delete_node(node, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse
) and value["value"] == calvinresponse.OK
self.storage.get_node(node.id, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse
) and value["value"] == calvinresponse.NOT_FOUND
def test_avail_valid(self):
"""
Test valid values for CPU avail.
Verify if storage is as expected
"""
values = [0, 25, 50, 75, 100]
for i in values:
# verify set return
self.done = False
self.cpu.set_avail(i, CalvinCB(self.cb))
yield wait_for(self._test_done)
assert self.get_ans == True
# verify nodeCpuAvail in DB
self.done = False
self.storage.get(prefix="nodeCpuAvail-", key=self.node.id, cb=CalvinCB(self.cb))
yield wait_for(self._test_done)
assert self.get_ans == i
# verify index ok and present for level i
self.done = False
self.storage.get_index(index=self.CPUAVAIL_INDEX_BASE + map(str, values[:values.index(i)+1]), root_prefix_level=2, cb=CalvinCB(self.cb2))
yield wait_for(self._test_done)
assert self.node.id in self.get_ans
def test_late_start(self):
self.q = Queue.Queue()
self.q2 = Queue.Queue()
def cb(key, value):
self.q2.put({"key": key, "value": value})
def started_cb(started):
self.q.put(started)
cb, d = create_callback()
self.storage = storage.Storage(DummyNode())
self.storage.start('', cb)
self.storage2 = storage.Storage(DummyNode())
self.storage.set("test", "1", 1, None)
self.storage.set("test", "2", 2, None)
self.storage.set("test", "3", 3, None)
assert "test1" in self.storage.localstore
assert "test2" in self.storage.localstore
assert "test3" in self.storage.localstore
cb2, d = create_callback()
self.storage2.start('', cb2)
value = yield d
# print value[0][0]
assert value[0][0]
yield wait_for(self.storage.localstore.keys,
condition=lambda x: not x())
assert not self.storage.localstore.keys()
cb, d = create_callback()
self.storage.get("test", "3", cb)
a, kw = yield d
assert a[0] == "3" and a[1] == 3
self.storage.stop()
self.storage2.stop()
def test_late_start(self):
self.q = Queue.Queue()
self.q2 = Queue.Queue()
def cb(key, value):
self.q2.put({"key": key, "value": value})
def started_cb(started):
self.q.put(started)
cb, d = create_callback(timeout=2)
self.storage = storage.Storage(DummyNode())
self.storage.start('', cb)
self.storage2 = storage.Storage(DummyNode())
self.storage.set("test", "1", 1, None)
self.storage.set("test", "2", 2, None)
self.storage.set("test", "3", 3, None)
assert "test1" in self.storage.localstore
assert "test2" in self.storage.localstore
assert "test3" in self.storage.localstore
cb2, d = create_callback(timeout=2)
self.storage2.start('', cb2)
value = yield d
# print value
assert value[0][0]
yield wait_for(self.storage.localstore.keys, condition=lambda x: not x())
assert not self.storage.localstore.keys()
cb, d = create_callback()
self.storage.get("test", "3", cb)
a, kw = yield d
assert a[0] == "3" and a[1] == 3
self.storage.stop()
self.storage2.stop()
def test_actor_and_port_functions(self):
self.q = Queue.Queue()
def cb(key, value):
self.q.put({"key": key, "value": value})
port1 = calvin.tests.TestPort("out", "out")
port2 = calvin.tests.TestPort("in", "in", )
port1.peers = [("local", port2.id)]
port2.peers = [("local", port1.id)]
actor1 = calvin.tests.TestActor("actor1", "type1", {}, {port1.name: port1})
actor2 = calvin.tests.TestActor("actor2", "type2", {port2.name: port2}, {})
self.storage.add_actor(actor1, calvinuuid.uuid("NODE"))
value = self.storage.get_actor(actor1.id, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value["key"] == actor1.id and value[
"value"]["name"] == actor1.name
assert value["value"]["name"] == actor1.name
assert value["value"]["type"] == actor1._type
assert value["value"]["inports"] == []
assert value["value"]["outports"][0]["id"] == port1.id
value = self.storage.get_port(port1.id, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value["key"] == port1.id
assert value["value"]["name"] == port1.name
assert value["value"]["peers"] == [("local", port2.id)]
self.storage.add_actor(actor2, calvinuuid.uuid("NODE"))
value = self.storage.get_actor(actor2.id, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value["key"] == actor2.id
assert value["value"]["name"] == actor2.name
assert value["value"]["type"] == actor2._type
assert value["value"]["inports"][0]["id"] == port2.id
assert value["value"]["outports"] == []
value = self.storage.get_port(port2.id, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value["key"] == port2.id
assert value["value"]["name"] == port2.name
assert value["value"]["peers"] == [("local", port1.id)]
self.storage.delete_actor(actor1.id, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value
assert actor1.id not in self.storage.localstore
self.storage.delete_port(port1.id, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value
assert port1.id not in self.storage.localstore
def test_actor_and_port_functions(self):
self.q = Queue.Queue()
def cb(key, value):
self.q.put({"key": key, "value": value})
port1 = calvin.tests.TestPort("out", "out")
port2 = calvin.tests.TestPort(
"in",
"in",
)
port1.peers = [("local", port2.id)]
port2.peers = [("local", port1.id)]
actor1 = calvin.tests.TestActor("actor1", "type1", {},
{port1.name: port1})
actor2 = calvin.tests.TestActor("actor2", "type2", {port2.name: port2},
{})
self.storage.add_actor(actor1, calvinuuid.uuid("NODE"))
value = self.storage.get_actor(actor1.id, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value["key"] == actor1.id and value["value"][
"name"] == actor1.name
assert value["value"]["name"] == actor1.name
assert value["value"]["type"] == actor1._type
assert value["value"]["inports"] == []
assert value["value"]["outports"][0]["id"] == port1.id
value = self.storage.get_port(port1.id, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value["key"] == port1.id
assert value["value"]["name"] == port1.name
assert value["value"]["peers"] == [["local", port2.id]]
self.storage.add_actor(actor2, calvinuuid.uuid("NODE"))
value = self.storage.get_actor(actor2.id, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value["key"] == actor2.id
assert value["value"]["name"] == actor2.name
assert value["value"]["type"] == actor2._type
assert value["value"]["inports"][0]["id"] == port2.id
assert value["value"]["outports"] == []
value = self.storage.get_port(port2.id, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value["key"] == port2.id
assert value["value"]["name"] == port2.name
assert value["value"]["peers"] == [["local", port1.id]]
self.storage.delete_actor(actor1.id, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value
assert actor1.id not in self.storage.localstore
self.storage.delete_port(port1.id, cb=CalvinCB(func=cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value
assert port1.id not in self.storage.localstore
