def test_set_get(self, setup): """ Test that set returns OK and get returns value set """ self.nodes = setup.get("nodes") storage_types = self.verify_started() for stype in storage_types: for i in ["aa", "bb", "", None, True, False, 0, 1, 2, [0, 1], {"xx": 10}]: key = str(i) for y in range(3): self.done = False self.get_ans = None self.nodes[stype][y].storage.set("test1-", key + str(y), i, CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10) # Verify response is CalvinResponse object with OK status print "set response", self.get_ans, stype, key, y assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK for x in range(3): if stype == "notstarted" and x != y: # Not started storage is never connected to other nodes storage continue self.done = False self.get_ans = None self.nodes[stype][x].storage.get("test1-", key + str(y), CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10) print "get response", self.get_ans, stype, key, x # Verify we read what is written assert self.get_ans == i
def test_avail_valid(self): """ Test valid values for CPU avail. Verify if storage is as expected """ values = [0, 25, 50, 75, 100] for i in values: # verify set return self.done = False self.cpu.set_avail(i, CalvinCB(self.cb)) yield wait_for(self._test_done) assert self.get_ans == True # verify nodeCpuAvail in DB self.done = False self.storage.get(prefix="nodeCpuAvail-", key=self.node.id, cb=CalvinCB(self.cb)) yield wait_for(self._test_done) assert self.get_ans == i # verify index ok and present for level i self.done = False self.storage.get_index(index=self.CPUAVAIL_INDEX_BASE + map(str, values[:values.index(i) + 1]), root_prefix_level=2, cb=CalvinCB(self.cb2)) yield wait_for(self._test_done) assert self.node.id in self.get_ans
def test_set_delete_get(self, setup): """ Test that set returns OK, delete return OK and get returns 404 response """ self.nodes = setup.get("nodes") storage_types = self.verify_started() for stype in storage_types: for i in ["aa", "bb", "cc", "", None, True, False, 0, 1, 2, [0, 1], {"xx": 10}]: for y in range(3): key = "t2" + str(i) if i != "cc": # "cc" is missing without being set and deleted self.done = False self.get_ans = None self.nodes[stype][y].storage.set("test2-", key + str(y), i, CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10) # Verify response is CalvinResponse object with OK status assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK print "set response", self.get_ans, stype, i, y self.done = False self.get_ans = None self.nodes[stype][y].storage.delete("test2-", key + str(y), CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10) # Verify response is CalvinResponse object with OK status assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK print "delete response", self.get_ans, stype, i, y for x in range(3): if stype == "notstarted" and x != y: # Not started storage is never connected to other nodes storage continue self.done = False self.get_ans = None self.nodes[stype][x].storage.get("test2-", key + str(y), CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10) print "get response", self.get_ans, stype, i, x # Verify the response is 404 assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.NOT_FOUND
def test_stop_node(self): """ Verify if indexes are cleared after node stop Old value must be erased from indexes """ self.mem.set_avail(25, CalvinCB(self.cb)) yield wait_for(self._test_done) assert self.get_ans == True self.done = False self.storage.get_index(index=self.MEMAVAIL_INDEX_BASE + ['0', '25'], root_prefix_level=2, cb=CalvinCB(self.cb2)) yield wait_for(self._test_done) assert self.node.id in self.get_ans self.mem.stop() self.storage.delete_node(self.node) # nodeMemAvail must not exist self.done = False self.storage.get(prefix="nodeMemAvail-", key=self.node.id, cb=CalvinCB(self.cb)) yield wait_for(self._test_done) assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.NOT_FOUND # node id must not be present at level 25 self.done = False self.storage.get_index(index=self.MEMAVAIL_INDEX_BASE + ['0', '25'], root_prefix_level=2, cb=CalvinCB(self.cb2)) yield wait_for(self._test_done) assert self.get_ans == [] # no node in total indexes self.done = False self.storage.get_index(index=self.MEMTOTAL_INDEX_BASE + ['1K'], root_prefix_level=2, cb=CalvinCB(self.cb2)) yield wait_for(self._test_done) assert self.get_ans == []
def test_append_multi_remove_getconcat(self, setup): """ Test that appending multiple values returns OK, remove a value returns OK and get_concat returns the unique values (set) in a list """ self.nodes = setup.get("nodes") storage_types = self.verify_started() for stype in storage_types: for i in [("aa", "aa"), ("bb", 2, "ff"), (0, 1, 2, 3)]: # values must be hashable (i.e. not list or dict) key = str(i) for y in range(3): self.done = False self.get_ans = None self.nodes[stype][y].storage.append("test5-", key + str(y), i, CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10) # Verify response is CalvinResponse object with OK status print "append response", self.get_ans, stype, key, y assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK self.done = False self.get_ans = None self.nodes[stype][y].storage.remove("test5-", key + str(y), i[1:], CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10) # Verify response is CalvinResponse object with OK status print "append response", self.get_ans, stype, key, y assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK for x in range(3): if stype == "notstarted" and x != y: # Not started storage is never connected to other nodes storage continue self.done = False self.get_ans = None self.nodes[stype][x].storage.get_concat("test5-", key + str(y), CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10) print "get_concat response", self.get_ans, stype, key, x # Verify we read what is written answer = set(i) - set(i[1:]) assert set(self.get_ans) == answer
def test_application_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) application = appmanager.Application( calvinuuid.uuid('APP'), "test_app", [calvinuuid.uuid('ACTOR'), calvinuuid.uuid('ACTOR')], calvinuuid.uuid("NODE"), None) self.storage.add_application(application, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) # print value self.storage.get_application(application.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) # print value assert value["key"] == application.id and value["value"][ "name"] == application.name self.storage.delete_application(application.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert application.id not in self.storage.localstore
def test_append_delete_get_concat(self, setup): """ Test that append returns OK, delete return OK and get returns empty set """ self.nodes = setup.get("nodes") storage_types = self.verify_started() for stype in storage_types: for i in ["aa", None, True, False, 0, 1, 2]: for y in range(3): key = "t3" + str(i) if i != "cc": # "cc" is missing without being set and deleted self.done = False self.get_ans = None self.nodes[stype][y].storage.append("test6-", key + str(y), [i], CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10, test_part="append"+key+str(y)) # Verify response is CalvinResponse object with OK status assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK print "append response", self.get_ans, stype, i, y self.done = False self.get_ans = None self.nodes[stype][y].storage.delete("test6-", key + str(y), CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10, test_part="delete"+key+str(y)) # Verify response is CalvinResponse object with OK status assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK print "delete response", self.get_ans, stype, i, y for x in range(3): if stype == "notstarted" and x != y: # Not started storage is never connected to other nodes storage continue self.done = False self.get_ans = None self.nodes[stype][x].storage.get_concat("test6-", key + str(y), CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10, test_part="get_concat"+key+str(y)) print "get_concat response", self.get_ans, stype, i, x # Verify the response is empty list (no difference between emptied or deleted) assert self.get_ans == []
def test_node_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) node = calvin.tests.TestNode(["127.0.0.1:5000"]) self.storage.add_node(node, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.OK self.storage.get_node(node.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["value"] == {u'attributes': {u'indexed_public': [], u'public': {}}, u'control_uris': [u'127.0.0.1:5000'], u'uris': [u'127.0.0.1:5000']} self.storage.delete_node(node, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.OK self.storage.get_node(node.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.NOT_FOUND
def test_application_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) application = appmanager.Application(calvinuuid.uuid( 'APP'), "test_app", [calvinuuid.uuid('ACTOR'), calvinuuid.uuid('ACTOR')], calvinuuid.uuid("NODE"), None) self.storage.add_application(application, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.OK self.storage.get_application(application.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["value"]["name"] == application.name self.storage.delete_application(application.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.OK self.storage.get_application(application.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.NOT_FOUND
def test_node_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) node = calvin.tests.TestNode(["127.0.0.1:5000"]) self.storage.add_node(node) self.storage.get_node(node_id=node.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["key"] == node.id and value["value"] == { u'attributes': { u'indexed_public': [], u'public': {} }, u'control_uris': [u'127.0.0.1:5000'], 'uris': node.uris } self.storage.delete_node(node, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value assert node.id not in self.storage.localstore
def test_actor_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) port1 = calvin.tests.TestPort("out", "out") port2 = calvin.tests.TestPort("in", "in", ) port1.peers = [("local", port2.id)] port2.peers = [("local", port1.id)] actor = calvin.tests.TestActor("actor1", "type1", {}, {port1.name: port1}) self.storage.add_actor(actor, calvinuuid.uuid("NODE"), cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.OK self.storage.get_actor(actor.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["value"]["name"] == actor.name self.storage.delete_actor(actor.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.OK self.storage.get_actor(actor.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.NOT_FOUND
def test_application_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) application = appmanager.Application( calvinuuid.uuid('APP'), "test_app", [calvinuuid.uuid('ACTOR'), calvinuuid.uuid('ACTOR')], calvinuuid.uuid("NODE"), None) self.storage.add_application(application, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse ) and value["value"] == calvinresponse.OK self.storage.get_application(application.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["value"]["name"] == application.name self.storage.delete_application(application.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse ) and value["value"] == calvinresponse.OK self.storage.get_application(application.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse ) and value["value"] == calvinresponse.NOT_FOUND
def test_application_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) application = appmanager.Application(calvinuuid.uuid( 'APP'), "test_app", [calvinuuid.uuid('ACTOR'), calvinuuid.uuid('ACTOR')], calvinuuid.uuid("NODE"), None) self.storage.add_application(application, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) # print value self.storage.get_application( application.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) # print value assert value["key"] == application.id and value[ "value"]["name"] == application.name self.storage.delete_application(application.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert application.id not in self.storage.localstore
def test_append_multi_getconcat(self, setup): """ Test that appending multiple values returns OK and get_concat returns the unique values (set) in a list """ self.nodes = setup.get("nodes") storage_types = self.verify_started() for stype in storage_types: for i in [("aa", "aa"), ("bb", 2, "ff"), (True, ""), (False, True) ]: # values must be hashable (i.e. not list or dict) key = str(i) for y in range(3): self.done = False self.get_ans = None self.nodes[stype][y].storage.append( "test4-", key + str(y), i, CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10) # Verify response is CalvinResponse object with OK status print "append response", self.get_ans, stype, key, y assert isinstance(self.get_ans, calvinresponse.CalvinResponse ) and self.get_ans == calvinresponse.OK for x in range(3): if stype == "notstarted" and x != y: # Not started storage is never connected to other nodes storage continue self.done = False self.get_ans = None self.nodes[stype][x].storage.get_concat( "test4-", key + str(y), CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10) print "get_concat response", self.get_ans, stype, key, x # Verify we read what is written assert set(self.get_ans) == set(i)
def test_total_valid(self): """ Test valid values for total RAM. Verify if storage is as expected """ values = ["1K", "100K", "1M", "100M", "1G", "10G"] for i in values: # verify set return self.done = False self.node.attributes = AttributeResolver( {"indexed_public": { "memTotal": i }}) self.storage.add_node(self.node, cb=self.cb) yield wait_for(self._test_done) assert isinstance(self.get_ans, calvinresponse.CalvinResponse ) and self.get_ans == calvinresponse.OK # verify index ok and present for level i self.done = False self.storage.get_index(index=self.MEMTOTAL_INDEX_BASE + map(str, values[:values.index(i) + 1]), root_prefix_level=2, cb=CalvinCB(self.cb2)) yield wait_for(self._test_done) assert self.node.id in self.get_ans
def test_set_get(self, setup): """ Test that set returns OK and get returns value set """ self.nodes = setup.get("nodes") storage_types = self.verify_started() for stype in storage_types: for i in [ "aa", "bb", "", None, True, False, 0, 1, 2, [0, 1], { "xx": 10 } ]: key = str(i) for y in range(3): self.done = False self.get_ans = None self.nodes[stype][y].storage.set("test1-", key + str(y), i, CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10) # Verify response is CalvinResponse object with OK status print "set response", self.get_ans, stype, key, y assert isinstance(self.get_ans, calvinresponse.CalvinResponse ) and self.get_ans == calvinresponse.OK for x in range(3): if stype == "notstarted" and x != y: # Not started storage is never connected to other nodes storage continue self.done = False self.get_ans = None self.nodes[stype][x].storage.get( "test1-", key + str(y), CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10) print "get response", self.get_ans, stype, key, x # Verify we read what is written assert self.get_ans == i
def test_append_getconcat(self, setup): """ Test that single value append returns OK and get_concat returns appended value in list """ self.nodes = setup.get("nodes") storage_types = self.verify_started() for stype in storage_types: for i in ["aa", "bb", "", True, False, 0, 1, 2]: # values must be hashable (i.e. not list or dict) key = str(i) for y in range(3): self.done = False self.get_ans = None self.nodes[stype][y].storage.append("test3-", key + str(y), [i], CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10) # Verify response is CalvinResponse object with OK status print "append response", self.get_ans, stype, key, y assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK for x in range(3): if stype == "notstarted" and x != y: # Not started storage is never connected to other nodes storage continue self.done = False self.get_ans = None self.nodes[stype][x].storage.get_concat("test3-", key + str(y), CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10) print "get_concat response", self.get_ans, stype, key, x # Verify we read what is written assert self.get_ans == [i]
def test_append_delete_append_get_concat(self, setup): """ Test that append returns OK, delete return OK and get returns second append """ self.nodes = setup.get("nodes") storage_types = self.verify_started() for stype in storage_types: for i in ["aa"]: for y in range(3): key = "t3" + str(i) self.done = False self.get_ans = None self.nodes[stype][y].storage.append( "test7-", key + str(y), [i], CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10, test_part="append" + key + str(y)) # Verify response is CalvinResponse object with OK status assert isinstance(self.get_ans, calvinresponse.CalvinResponse ) and self.get_ans == calvinresponse.OK print "append response", self.get_ans, stype, i, y self.done = False self.get_ans = None self.nodes[stype][y].storage.delete( "test7-", key + str(y), CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10, test_part="delete" + key + str(y)) # Verify response is CalvinResponse object with OK status assert isinstance(self.get_ans, calvinresponse.CalvinResponse ) and self.get_ans == calvinresponse.OK print "delete response", self.get_ans, stype, i, y self.done = False self.get_ans = None self.nodes[stype][y].storage.append( "test7-", key + str(y), [i + "2"], CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10, test_part="append2" + key + str(y)) # Verify response is CalvinResponse object with OK status assert isinstance(self.get_ans, calvinresponse.CalvinResponse ) and self.get_ans == calvinresponse.OK print "append2 response", self.get_ans, stype, i, y for x in range(3): if stype == "notstarted" and x != y: # Not started storage is never connected to other nodes storage continue self.done = False self.get_ans = None self.nodes[stype][x].storage.get_concat( "test7-", key + str(y), CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10, test_part="get_concat" + key + str(y)) print "get_concat response", self.get_ans, stype, i, x # Verify the response is second appended value assert self.get_ans == [i + "2"]
def test_add_remove_get_index(self, setup): """ Test that add_index returns OK, remove_index returns OK and get_index returns appended value in list for index hierarchies """ self.nodes = setup.get("nodes") storage_types = self.verify_started() for stype in storage_types: for y in range(3): for i in [(["aa", "bb", "", "dd"], ["xx", "kk", "ll"], "+"), (["aa", "bb", "", "ee"], ["yy", "kk", "ll"], "+"), (["aa", "bb"], ["zz", "mm", "oo"], "+"), (["aa", "ff", "", "dd"], ["xx"], "+"), (["aa", "bb", "", "dd"], ["kk", "ll"], "-"), (["aa", "bb", "", "ee"], ["yy", "kk", "ll"], "-"), (["aa", "bb"], ["oo"], "-"), (["aa", "gg", "", "dd"], ["xx"], "-")]: self.done = False self.get_ans = None index = copy.copy(i[0]) index[0] = "test2" + index[0] + str(y) # root_prefix_level is default 2 hence 2 first are combined if i[2] == "+": self.nodes[stype][y].storage.add_index(index, i[1], cb=CalvinCB( self.cb2)) else: self.nodes[stype][y].storage.remove_index( index, i[1], cb=CalvinCB(self.cb2)) yield wait_for(self._test_done, timeout=10) # Verify response is CalvinResponse object with OK status print "add" if i[ 2] == "+" else "remove", " response", self.get_ans, stype, index, y assert isinstance(self.get_ans, calvinresponse.CalvinResponse ) and self.get_ans == calvinresponse.OK for x in range(3): if stype == "notstarted" and x != y: # Not started storage is never connected to other nodes storage continue for i in [(["aa"], []), (["gg"], []), (["aa", "gg"], []), (["aa", "bb"], ["xx", "zz", "mm"]), (["aa", "bb", ""], ["xx"]), (["aa", "ff"], ["xx"]), (["aa", "bb", "", "dd"], ["xx"]), (["aa", "bb", "", "ee"], []), (["aa", "ff", "", "dd"], ["xx"])]: self.done = False self.get_ans = None index = copy.copy(i[0]) index[0] = "test2" + index[0] + str(y) self.nodes[stype][x].storage.get_index(index, cb=CalvinCB( self.cb2)) yield wait_for(self._test_done, timeout=10) print "get_index response", self.get_ans, stype, index, x # Verify we read what is written if too short prefix or not existing we should get []. assert set(self.get_ans) == set(i[1])
def test_add_remove_get_index(self, setup): """ Test that add_index returns OK, remove_index returns OK and get_index returns appended value in list for index hierarchies """ self.nodes = setup.get("nodes") storage_types = self.verify_started() for stype in storage_types: for y in range(3): for i in [(["aa", "bb", "", "dd"], ["xx", "kk", "ll"], "+"), (["aa", "bb", "", "ee"], ["yy", "kk", "ll"], "+"), (["aa", "bb"], ["zz", "mm", "oo"], "+"), (["aa", "ff", "", "dd"], ["xx"], "+"), (["aa", "bb", "", "dd"], ["kk", "ll"], "-"), (["aa", "bb", "", "ee"], ["yy", "kk", "ll"], "-"), (["aa", "bb"], ["oo"], "-"), (["aa", "gg", "", "dd"], ["xx"], "-") ]: self.done = False self.get_ans = None index = copy.copy(i[0]) index[0] = "test2" + index[0] + str(y) # root_prefix_level is default 2 hence 2 first are combined if i[2] == "+": self.nodes[stype][y].storage.add_index(index, i[1], cb=CalvinCB(self.cb2)) else: self.nodes[stype][y].storage.remove_index(index, i[1], cb=CalvinCB(self.cb2)) yield wait_for(self._test_done, timeout=10) # Verify response is CalvinResponse object with OK status print "add" if i[2] == "+" else "remove", " response", self.get_ans, stype, index, y assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK for x in range(3): if stype == "notstarted" and x != y: # Not started storage is never connected to other nodes storage continue for i in [(["aa"], []), (["gg"], []), (["aa", "gg"], []), (["aa", "bb"], ["xx", "zz", "mm"]), (["aa", "bb", ""], ["xx"]), (["aa", "ff"], ["xx"]), (["aa", "bb", "", "dd"], ["xx"]), (["aa", "bb", "", "ee"], []), (["aa", "ff", "", "dd"], ["xx"])]: self.done = False self.get_ans = None index = copy.copy(i[0]) index[0] = "test2" + index[0] + str(y) self.nodes[stype][x].storage.get_index(index, cb=CalvinCB(self.cb2)) yield wait_for(self._test_done, timeout=10) print "get_index response", self.get_ans, stype, index, x # Verify we read what is written if too short prefix or not existing we should get []. assert set(self.get_ans) == set(i[1])
def test_avail_change(self): """ Verify if indexes are ok after a change in CPU avail. Old value must be erased from indexes """ self.cpu.set_avail(50) self.cpu.set_avail(25, CalvinCB(self.cb)) yield wait_for(self._test_done) assert self.get_ans == True # node id must not be present at level 50, only at 25 self.done = False self.storage.get_index(index=self.CPUAVAIL_INDEX_BASE + ['0', '25', '50'], root_prefix_level=2, cb=CalvinCB(self.cb2)) yield wait_for(self._test_done) assert self.get_ans == []
def test_set_delete_get(self, setup): """ Test that set returns OK, delete return OK and get returns 404 response """ self.nodes = setup.get("nodes") storage_types = self.verify_started() for stype in storage_types: for i in [ "aa", "bb", "cc", "", None, True, False, 0, 1, 2, [0, 1], { "xx": 10 } ]: for y in range(3): key = "t2" + str(i) if i != "cc": # "cc" is missing without being set and deleted self.done = False self.get_ans = None self.nodes[stype][y].storage.set( "test2-", key + str(y), i, CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10) # Verify response is CalvinResponse object with OK status assert isinstance( self.get_ans, calvinresponse.CalvinResponse ) and self.get_ans == calvinresponse.OK print "set response", self.get_ans, stype, i, y self.done = False self.get_ans = None self.nodes[stype][y].storage.delete( "test2-", key + str(y), CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10) # Verify response is CalvinResponse object with OK status assert isinstance( self.get_ans, calvinresponse.CalvinResponse ) and self.get_ans == calvinresponse.OK print "delete response", self.get_ans, stype, i, y for x in range(3): if stype == "notstarted" and x != y: # Not started storage is never connected to other nodes storage continue self.done = False self.get_ans = None self.nodes[stype][x].storage.get( "test2-", key + str(y), CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10) print "get response", self.get_ans, stype, i, x # Verify the response is 404 assert isinstance( self.get_ans, calvinresponse.CalvinResponse ) and self.get_ans == calvinresponse.NOT_FOUND
def test_append_delete_get_concat(self, setup): """ Test that append returns OK, delete return OK and get returns empty set """ self.nodes = setup.get("nodes") storage_types = self.verify_started() for stype in storage_types: for i in ["aa", None, True, False, 0, 1, 2]: for y in range(3): key = "t3" + str(i) if i != "cc": # "cc" is missing without being set and deleted self.done = False self.get_ans = None self.nodes[stype][y].storage.append( "test6-", key + str(y), [i], CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10, test_part="append" + key + str(y)) # Verify response is CalvinResponse object with OK status assert isinstance( self.get_ans, calvinresponse.CalvinResponse ) and self.get_ans == calvinresponse.OK print "append response", self.get_ans, stype, i, y self.done = False self.get_ans = None self.nodes[stype][y].storage.delete( "test6-", key + str(y), CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10, test_part="delete" + key + str(y)) # Verify response is CalvinResponse object with OK status assert isinstance( self.get_ans, calvinresponse.CalvinResponse ) and self.get_ans == calvinresponse.OK print "delete response", self.get_ans, stype, i, y for x in range(3): if stype == "notstarted" and x != y: # Not started storage is never connected to other nodes storage continue self.done = False self.get_ans = None self.nodes[stype][x].storage.get_concat( "test6-", key + str(y), CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10, test_part="get_concat" + key + str(y)) print "get_concat response", self.get_ans, stype, i, x # Verify the response is empty list (no difference between emptied or deleted) assert self.get_ans == []
def test_avail_invalid(self): """ Verify invalid values for CPU avail """ for i in [-1, 40, 101]: self.done = False self.cpu.set_avail(i, CalvinCB(self.cb)) yield wait_for(self._test_done) assert self.get_ans == False
def test_avail_change(self): """ Verify if indexes are ok after a change in RAM avail. Old value must be erased from indexes """ self.mem.set_avail(50) self.mem.set_avail(25, CalvinCB(self.cb)) yield wait_for(self._test_done) assert self.get_ans == True # node id must not be present at level 50, only at 25 self.done = False self.storage.get_index(index=self.MEMAVAIL_INDEX_BASE + ['0', '25', '50'], root_prefix_level=2, cb=CalvinCB(self.cb2)) yield wait_for(self._test_done) assert self.get_ans == []
def test_stop_node(self): """ Verify if indexes are cleared after node stop Old value must be erased from indexes """ self.mem.set_avail(25, CalvinCB(self.cb)) yield wait_for(self._test_done) assert self.get_ans == True self.done = False self.storage.get_index(index=self.MEMAVAIL_INDEX_BASE + ['0', '25'], root_prefix_level=2, cb=CalvinCB(self.cb2)) yield wait_for(self._test_done) assert self.node.id in self.get_ans self.mem.stop() self.storage.delete_node(self.node) # nodeMemAvail must not exist self.done = False self.storage.get(prefix="nodeMemAvail-", key=self.node.id, cb=CalvinCB(self.cb)) yield wait_for(self._test_done) assert isinstance(self.get_ans, calvinresponse.CalvinResponse ) and self.get_ans == calvinresponse.NOT_FOUND # node id must not be present at level 25 self.done = False self.storage.get_index(index=self.MEMAVAIL_INDEX_BASE + ['0', '25'], root_prefix_level=2, cb=CalvinCB(self.cb2)) yield wait_for(self._test_done) assert self.get_ans == [] # no node in total indexes self.done = False self.storage.get_index(index=self.MEMTOTAL_INDEX_BASE + ['1K'], root_prefix_level=2, cb=CalvinCB(self.cb2)) yield wait_for(self._test_done) assert self.get_ans == []
def test_node_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) node = calvin.tests.TestNode(["127.0.0.1:5000"]) self.storage.add_node(node) self.storage.get_node(node_id=node.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["key"] == node.id and value["value"] == {u'attributes': {u'indexed_public': [], u'public': {}}, u'control_uris': [u'127.0.0.1:5000'], 'uris': node.uris} self.storage.delete_node(node, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value assert node.id not in self.storage.localstore
def test_total_valid(self): """ Test valid values for total RAM. Verify if storage is as expected """ values = ["1K", "100K", "1M", "100M", "1G", "10G"] for i in values: # verify set return self.done = False self.node.attributes = AttributeResolver({"indexed_public": {"memTotal": i }}) self.storage.add_node(self.node, cb=self.cb) yield wait_for(self._test_done) assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK # verify index ok and present for level i self.done = False self.storage.get_index(index=self.MEMTOTAL_INDEX_BASE + map(str, values[:values.index(i)+1]), root_prefix_level=2, cb=CalvinCB(self.cb2)) yield wait_for(self._test_done) assert self.node.id in self.get_ans
def test_actor_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) port1 = calvin.tests.TestPort("out", "out") port2 = calvin.tests.TestPort( "in", "in", ) port1.peers = [("local", port2.id)] port2.peers = [("local", port1.id)] actor = calvin.tests.TestActor("actor1", "type1", {}, {port1.name: port1}) self.storage.add_actor(actor, calvinuuid.uuid("NODE"), cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse ) and value["value"] == calvinresponse.OK self.storage.get_actor(actor.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["value"]["name"] == actor.name self.storage.delete_actor(actor.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse ) and value["value"] == calvinresponse.OK self.storage.get_actor(actor.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse ) and value["value"] == calvinresponse.NOT_FOUND
def test_append_delete_append_get_concat(self, setup): """ Test that append returns OK, delete return OK and get returns second append """ self.nodes = setup.get("nodes") storage_types = self.verify_started() for stype in storage_types: for i in ["aa"]: for y in range(3): key = "t3" + str(i) self.done = False self.get_ans = None self.nodes[stype][y].storage.append("test7-", key + str(y), [i], CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10, test_part="append"+key+str(y)) # Verify response is CalvinResponse object with OK status assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK print "append response", self.get_ans, stype, i, y self.done = False self.get_ans = None self.nodes[stype][y].storage.delete("test7-", key + str(y), CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10, test_part="delete"+key+str(y)) # Verify response is CalvinResponse object with OK status assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK print "delete response", self.get_ans, stype, i, y self.done = False self.get_ans = None self.nodes[stype][y].storage.append("test7-", key + str(y), [i+"2"], CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10, test_part="append2"+key+str(y)) # Verify response is CalvinResponse object with OK status assert isinstance(self.get_ans, calvinresponse.CalvinResponse) and self.get_ans == calvinresponse.OK print "append2 response", self.get_ans, stype, i, y for x in range(3): if stype == "notstarted" and x != y: # Not started storage is never connected to other nodes storage continue self.done = False self.get_ans = None self.nodes[stype][x].storage.get_concat("test7-", key + str(y), CalvinCB(self.cb)) yield wait_for(self._test_done, timeout=10, test_part="get_concat"+key+str(y)) print "get_concat response", self.get_ans, stype, i, x # Verify the response is second appended value assert self.get_ans == [i+"2"]
def test_node_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) node = calvin.tests.TestNode(["127.0.0.1:5000"]) self.storage.add_node(node, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse ) and value["value"] == calvinresponse.OK self.storage.get_node(node.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["value"] == { u'attributes': { u'indexed_public': [], u'public': {} }, u'control_uris': [u'127.0.0.1:5000'], u'uris': [u'127.0.0.1:5000'] } self.storage.delete_node(node, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse ) and value["value"] == calvinresponse.OK self.storage.get_node(node.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse ) and value["value"] == calvinresponse.NOT_FOUND
def test_avail_valid(self): """ Test valid values for CPU avail. Verify if storage is as expected """ values = [0, 25, 50, 75, 100] for i in values: # verify set return self.done = False self.cpu.set_avail(i, CalvinCB(self.cb)) yield wait_for(self._test_done) assert self.get_ans == True # verify nodeCpuAvail in DB self.done = False self.storage.get(prefix="nodeCpuAvail-", key=self.node.id, cb=CalvinCB(self.cb)) yield wait_for(self._test_done) assert self.get_ans == i # verify index ok and present for level i self.done = False self.storage.get_index(index=self.CPUAVAIL_INDEX_BASE + map(str, values[:values.index(i)+1]), root_prefix_level=2, cb=CalvinCB(self.cb2)) yield wait_for(self._test_done) assert self.node.id in self.get_ans
def test_late_start(self): self.q = Queue.Queue() self.q2 = Queue.Queue() def cb(key, value): self.q2.put({"key": key, "value": value}) def started_cb(started): self.q.put(started) cb, d = create_callback() self.storage = storage.Storage(DummyNode()) self.storage.start('', cb) self.storage2 = storage.Storage(DummyNode()) self.storage.set("test", "1", 1, None) self.storage.set("test", "2", 2, None) self.storage.set("test", "3", 3, None) assert "test1" in self.storage.localstore assert "test2" in self.storage.localstore assert "test3" in self.storage.localstore cb2, d = create_callback() self.storage2.start('', cb2) value = yield d # print value[0][0] assert value[0][0] yield wait_for(self.storage.localstore.keys, condition=lambda x: not x()) assert not self.storage.localstore.keys() cb, d = create_callback() self.storage.get("test", "3", cb) a, kw = yield d assert a[0] == "3" and a[1] == 3 self.storage.stop() self.storage2.stop()
def test_late_start(self): self.q = Queue.Queue() self.q2 = Queue.Queue() def cb(key, value): self.q2.put({"key": key, "value": value}) def started_cb(started): self.q.put(started) cb, d = create_callback(timeout=2) self.storage = storage.Storage(DummyNode()) self.storage.start('', cb) self.storage2 = storage.Storage(DummyNode()) self.storage.set("test", "1", 1, None) self.storage.set("test", "2", 2, None) self.storage.set("test", "3", 3, None) assert "test1" in self.storage.localstore assert "test2" in self.storage.localstore assert "test3" in self.storage.localstore cb2, d = create_callback(timeout=2) self.storage2.start('', cb2) value = yield d # print value assert value[0][0] yield wait_for(self.storage.localstore.keys, condition=lambda x: not x()) assert not self.storage.localstore.keys() cb, d = create_callback() self.storage.get("test", "3", cb) a, kw = yield d assert a[0] == "3" and a[1] == 3 self.storage.stop() self.storage2.stop()
def test_actor_and_port_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) port1 = calvin.tests.TestPort("out", "out") port2 = calvin.tests.TestPort("in", "in", ) port1.peers = [("local", port2.id)] port2.peers = [("local", port1.id)] actor1 = calvin.tests.TestActor("actor1", "type1", {}, {port1.name: port1}) actor2 = calvin.tests.TestActor("actor2", "type2", {port2.name: port2}, {}) self.storage.add_actor(actor1, calvinuuid.uuid("NODE")) value = self.storage.get_actor(actor1.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["key"] == actor1.id and value[ "value"]["name"] == actor1.name assert value["value"]["name"] == actor1.name assert value["value"]["type"] == actor1._type assert value["value"]["inports"] == [] assert value["value"]["outports"][0]["id"] == port1.id value = self.storage.get_port(port1.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["key"] == port1.id assert value["value"]["name"] == port1.name assert value["value"]["peers"] == [("local", port2.id)] self.storage.add_actor(actor2, calvinuuid.uuid("NODE")) value = self.storage.get_actor(actor2.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["key"] == actor2.id assert value["value"]["name"] == actor2.name assert value["value"]["type"] == actor2._type assert value["value"]["inports"][0]["id"] == port2.id assert value["value"]["outports"] == [] value = self.storage.get_port(port2.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["key"] == port2.id assert value["value"]["name"] == port2.name assert value["value"]["peers"] == [("local", port1.id)] self.storage.delete_actor(actor1.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value assert actor1.id not in self.storage.localstore self.storage.delete_port(port1.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value assert port1.id not in self.storage.localstore
def test_actor_and_port_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) port1 = calvin.tests.TestPort("out", "out") port2 = calvin.tests.TestPort( "in", "in", ) port1.peers = [("local", port2.id)] port2.peers = [("local", port1.id)] actor1 = calvin.tests.TestActor("actor1", "type1", {}, {port1.name: port1}) actor2 = calvin.tests.TestActor("actor2", "type2", {port2.name: port2}, {}) self.storage.add_actor(actor1, calvinuuid.uuid("NODE")) value = self.storage.get_actor(actor1.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["key"] == actor1.id and value["value"][ "name"] == actor1.name assert value["value"]["name"] == actor1.name assert value["value"]["type"] == actor1._type assert value["value"]["inports"] == [] assert value["value"]["outports"][0]["id"] == port1.id value = self.storage.get_port(port1.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["key"] == port1.id assert value["value"]["name"] == port1.name assert value["value"]["peers"] == [["local", port2.id]] self.storage.add_actor(actor2, calvinuuid.uuid("NODE")) value = self.storage.get_actor(actor2.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["key"] == actor2.id assert value["value"]["name"] == actor2.name assert value["value"]["type"] == actor2._type assert value["value"]["inports"][0]["id"] == port2.id assert value["value"]["outports"] == [] value = self.storage.get_port(port2.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["key"] == port2.id assert value["value"]["name"] == port2.name assert value["value"]["peers"] == [["local", port1.id]] self.storage.delete_actor(actor1.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value assert actor1.id not in self.storage.localstore self.storage.delete_port(port1.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value assert port1.id not in self.storage.localstore