def test_routing_structure(): with NixServer(modules=[module]) as server: client = NixClient(server.get_address()) result = client.call(module, "list_routes", {}, 2000) # print(json.dumps(result.data, indent=4, sort_keys=True)) assert_false(result.is_rejected(), "not rejected") assert_false(result.is_abandoned(), "not abandoned") assert_true(result.is_replied(), "is replied") assert_equal(expected_routing, result.data) routing = result.data.get("routing") auth_error_code = 101 # routes with access mod != ANY require api key -> code 101 for route in routing: if routing[route]["access_modifier"] != "ANY": response = client.call(module, route, {}, 2000) if routing[route]["processing_type"] != "VOID": status_code = response.data.get("@status_code") assert_equal(status_code, auth_error_code) # test API_PRIVATE with invalid key (public) -> code 101 for route in routing: am = routing[route]["access_modifier"] if am == "API_PRIVATE" and route != "list_routes": params = {"@api_key" : development_key_public} response = client.call(module, route, params, 2000) status_code = response.data.get("@status_code") assert_equal(status_code, 101)
def test_nodes(): test_node = "unittest-01" test_address = "tcp://127.0.0.1:33333" with NixServer(modules=[module]) as server: client = NixClient(server.get_address()) params = {"@api_key" : development_key} # testing 'bind' route invalid_params = params.copy() response = client.call(module, "node/bind", invalid_params, 2000) assert_equal( response.status_code(), 501, "Code 501 - indicates invalid parameters" ) # valid params.update({ "nodename" : test_node, "address" : test_address }) response = client.call(module, "node/bind", params, 2000) assert_true(response.is_status_ok(), "Bind succeeded") # testing 'resolve' route params = {"@api_key" : development_key} invalid_params = params.copy() response = client.call(module, "node/resolve", invalid_params, 2000) assert_true(response.is_status_fail(), "Resolve fails without node") # valid params.update({"nodename" : test_node}) response = client.call(module, "node/resolve", params, 2000) assert_true(response.is_status_ok(), "Is resolved ok?") assert_equal( response.data[test_node], test_address, "Is resolved address correct?" ) # testing 'unbind' route response = client.send_one_way(module, "node/unbind", params) time.sleep(0.2) # after unbind node cannot be resolved response = client.call(module, "node/resolve", params, 2000) assert_true(response.is_status_fail(), "Is node unbound?")
def test_routing_structure(): with NixServer(modules=[module]) as server: client = NixClient(server.get_address()) result = client.call(module, "list_routes", {}, 2000) # print(json.dumps(result.data, indent=4, sort_keys=True)) assert_false(result.is_rejected(), "not rejected") assert_false(result.is_abandoned(), "not abandoned") assert_true(result.is_replied(), "is replied") assert_equal(expected_routing, result.data)
def test_basic(): with NixServer(modules=["cache"]) as server: client = NixClient(server.get_address()) result = client.call("Invalid", "invalid", {}, 2000) assert_true(result.is_rejected(), "Invalid route (reject)") result = client.call("Cache", "store", {}, 2000) assert_true(result.is_status_fail(), "Is failed?") assert_equal(result.status_code(), 101, "Auth error") params = { "@api_key" : development_key, "key" : "key", "value" : [] } result = client.call("Cache", "store", params, 2000) assert_true(result.is_status_ok(), "succeeded with auth") assert_equal(result.status_code(), 0, "success indicated by 0")
def test_service(): test_key = "unittest" test_value = {"a_string": "some string"} with NixServer(modules=[module]) as server: client = NixClient(server.get_address()) params = {"@api_key": development_key} # testing 'store' route invalid_params = params.copy() response = client.call(module, "store", invalid_params, 2000) assert_equal(response.status_code(), 501, "Code 501 - indicates invalid parameters") # valid params.update({"key": test_key, "value": test_value}) response = client.call(module, "store", params, 2000) assert_true(response.is_status_ok(), "Store succeeded") # # testing 'retrieve' route params = {"@api_key": development_key} invalid_params = params.copy() response = client.call(module, "retrieve", invalid_params, 2000) assert_true(response.is_status_fail(), "Retrieve fails without key") # valid params = {"@api_key": development_key, "key": test_key} response = client.call(module, "retrieve", params, 2000) assert_true(response.is_status_ok(), "Is retrieve call ok?") assert_equal(response.data.get(test_key), test_value, "Is retrieved value correct?") # testing 'remove' route response = client.send_one_way(module, "remove", params) time.sleep(0.2) # after remove key cannot be retrieved response = client.call(module, "retrieve", params, 2000) assert_true(response.is_status_fail(), "Is key removed?")
def run(self, event): client = NixClient(self._server_address, verbose=False) cache_key = "cache-key-%d-" % threading.current_thread().ident cache_value = "Random value %d: " % threading.current_thread().ident while not event.is_set(): time.sleep(random.randint(1, 2) / 100.0) # print("this is", threading.current_thread().ident) params = { "@api_key" : self._api_key, "key" : "test", # cache_key, "value" : "test", # cache_value, } response = client.call("Cache", "store", params, 2000) if response.is_status_fail(): print("%s %d" % ("Store failed", threading.current_thread().ident)) response = client.call("Cache", "retrieve", params, 5000) if response.is_status_fail(): print("Retrieve failed", response.data) print(response.data) response.data.get(cache_key) == cache_value # client.send_one_way("Cache", "remove", params) print("CacheClient finished")
def test_basic(): with NixServer(modules=server_modules) as server: params = { "@api_key": development_key, "server": True, "stats": True, "module_stats": True, "routing": True, } client = NixClient(server.get_address()) response = client.call("Status", "status", params) # print(json.dumps(response.data, indent=4, sort_keys=True)) assert_true(response.is_status_ok(), "Got server status") data = response.data assert_true(data.get("nodename", ""), "has nodename") assert_true(data.get("server", {}).get("routing", ""), "has routing") assert_true(data.get("stats", {}), "stats") for name in server_modules_names: assert_true(data["stats"].get(name), "has stats for: %s" % name)
if __name__ == "__main__": event = threading.Event() signal.signal(signal.SIGINT, lambda: event.set()) signal.signal(signal.SIGTERM, lambda: event.set()) threads = [] with NixServer(modules=SERVER_MODULES) as server: for i in range(0, 50): for cls in [ResolverClient, CacheClient]: worker = cls(server.get_address(), DEVELOPMENT_KEY) t = threading.Thread(target=worker.run, args=(event, )) t.start() threads.append(t) try: client = NixClient(server.get_address()) while True: params = { "@api_key": DEVELOPMENT_KEY, "stats": True, "module_stats": True, } response = client.call("Status", "status", params) print(json.dumps(response.data, indent=4, sort_keys=True)) time.sleep(1) except Exception as e: event.set() for t in threads: print("join") t.join()
def test_services(): test_service = "some_implementation" test_nodename1 = "service-node-1" test_address1 = "tcp://1.1.1.1:11111" test_nodename2 = "service-node-2" test_address2 = "tcp://2.2.2.2:22222" with NixServer(modules=[module]) as server: client = NixClient(server.get_address()) params = {"@api_key" : development_key} invalid_params = params.copy() response = client.call(module, "service/bind", invalid_params, 2000) assert_equal( response.status_code(), 501, "Code 501 - indicates invalid parameters" ) # fail bind when node is not yet known invalid_params = params.copy() invalid_params.update({ "service" : "test", "nodename" : test_nodename1 }) response = client.call(module, "service/bind", params, 2000) assert_true( response.is_status_fail(), "bind service fails (node not bound yet)" ) # bind nodes node_bind_params = params.copy() node_bind_params.update({ "nodename" : test_nodename1, "address" : test_address1 }) response = client.call(module, "node/bind", node_bind_params) assert_true(response.is_status_ok(), "node 1 bound") node_bind_params.update({ "nodename" : test_nodename2, "address" : test_address2 }) response = client.call(module, "node/bind", node_bind_params) assert_true(response.is_status_ok(), "node 2 bound") # first node params.update({ "service" : test_service, "nodename" : test_nodename1 }) response = client.call(module, "service/bind", params, 2000) assert_true(response.is_status_ok(), "1st bind service ok") response = client.call(module, "service/resolve", params, 2000) assert_true(response.is_status_ok(), "resolved after 1st bind") assert_equal( response.data.get("addresses", []), [test_address1], "resolved to a single address (1 elem array)" ) # second node params.update({ "service" : test_service, "nodename" : test_nodename2 }) response = client.call(module, "service/bind", params, 2000) assert_true(response.is_status_ok(), "2nd bind service ok") response = client.call(module, "service/resolve", params, 2000) assert_true(response.is_status_ok(), "resolved after 2nd bind") assert_equal( response.data.get("addresses", []), [test_address1, test_address2], "resolved both addresses single address (1 elem array)" ) response = client.call(module, "service/unbind", params, 2000) assert_true(response.is_status_ok(), "unbind 2nd address") response = client.call(module, "service/resolve", params, 2000) assert_true(response.is_status_ok(), "resolved after unbinding") assert_equal( response.data.get("addresses", []), [test_address1], "resolved a single address after unbind (1 elem array)" ) params.update({ "nodename" : test_nodename1 }) response = client.call(module, "node/unbind", params, 2000) assert_true(response.is_status_ok(), "unbind node1") params.update({ "nodename" : test_nodename2 }) response = client.call(module, "node/unbind", params, 2000) assert_true(response.is_status_ok(), "unbind node1") response = client.call(module, "service/resolve", params, 2000) assert_true( response.is_status_fail(), "Unbinding nodes unbinds services" ) assert_equal(response.status_msg(), "Unknown service.")
def test_submit(): with NixServer(modules=["job-queue", "resolver"]) as server: client = NixClient(server.get_address()) params = {"@api_key": development_key} # testing 'submit' route response = client.call(module, "job/submit", params, 100) assert_true(response.is_status_fail(), "Fails without params") job_module = "test" job_action = "test" job_param_name = "param" job_param_value = "πœę©ß←" # first: create the queue create_params = { "@api_key": development_key_admin, "queue": job_module, "create": { "size": 256, } } response = client.call(module, "manage", create_params) assert_true(response.is_status_ok(), "created queue") params.update({ "@origin_node": server.get_nodename(), "module": job_module, "action": job_action, "parameters": { job_param_name: job_param_value } }) response = client.call(module, "job/submit", params, 1000) assert_true(response.is_status_ok(), "Submit ok") JOB_ID = response.data["@job_id"] assert_true(len(JOB_ID)) params = {"@api_key": development_key, "@job_id": JOB_ID} response = client.call(module, "job/result/get", params, 100) assert_true(response.is_status_fail(), "Fail - job not started") params = { "@api_key": development_key, "module": "test" # queue name } response = client.call(module, "job/work/get", params, 1000) assert_true(response.is_status_ok(), "Got job to process") # this does not happen in real world, but this is a test # and this is the only submitted job assert_equal(response.data["@job_id"], JOB_ID, "Job id") # --- assert_equal(response.data["@origin_node"], server.get_nodename(), "Origin node name set") job_parameters = response.data["parameters"] job_payload = response.data # no need to test module - it was a name of the queue assert_equal(job_payload["action"], job_action, "action") assert_equal(job_parameters.get(job_param_name), job_param_value, "API key") # simulate some processing total = 121 for processed in range(0, total, 20): params = { "@api_key": development_key, "@job_id": JOB_ID, "progress": float(processed) / total * 100 } client.send_one_way(module, "job/progress/set", params) time.sleep(0.01) response = client.call(module, "job/result/get", params, 100) if processed < total: assert_equal(response.status_code(), 2, "Got progress report (code 2: not_ready)") params = { "@api_key": development_key, "@job_id": JOB_ID, "parameters": { job_param_name: "Result: %s" % job_param_value } } response = client.call(module, "job/result/set", params, 100) assert_true(response.is_status_ok(), "set response") params = { "@api_key": development_key, "@job_id": JOB_ID, } response = client.call(module, "job/result/get", params, 100) assert_true(response.is_status_ok(), "got result") data = response.data assert_equal(data["@api_key"], development_key, "result: api key") assert_equal(data["@job_id"], JOB_ID, "result: job_id") assert_equal( data.get("parameters", {}).get(job_param_name, ""), "Result: %s" % job_param_value, "result: job value")
def test_queue_management(): with NixServer(modules=["job-queue"]) as server: client = NixClient(server.get_address()) params = { "@api_key": development_key, } invalid_auth_params = { "@api_key": development_key_public, } response = client.call(module, "manage", invalid_auth_params) assert_equal(response.status_code(), 101, "Auth error") status_params = { "@api_key": development_key_admin, } response = client.call(module, "status", status_params, 2000) assert_true(response.is_status_ok(), "got status") status = response.data assert_equal(status["total_completed"], 0, "no jobs done yey") assert_equal(status["results_awaiting"], 0, "no results awaiting") assert_equal(status["in_progress"], 0, "no jobs in progress") assert_equal(status["total_pending"], 0, "no pending jobs") queue_name_1 = "unittest-queue-%d" % os.getpid() queue_name_2 = "second-queue-%d" % os.getpid() create_params = { "@api_key": development_key_admin, "queue": queue_name_1, "create": { "size": 256, } } response = client.call(module, "manage", create_params) assert_true(response.is_status_ok(), "created queue") create_params["queue"] = queue_name_2 response = client.call(module, "manage", create_params) assert_true(response.is_status_ok(), "created second queue") response = client.call(module, "manage", create_params) assert_true(response.is_status_fail(), "queue already exists") response = client.call(module, "status", status_params) assert_true(response.is_status_ok(), "got status") status = response.data assert_equal(status["queues"][queue_name_1]["closed"], False, "created queue is not closed") assert_equal(status["queues"][queue_name_2]["closed"], False, "second created queue is not closed") assert_equal(status["queues"][queue_name_1]["pending"], 0, "no jobs in created queue") assert_equal(status["queues"][queue_name_2]["pending"], 0, "no jobs in second created queue") # FIRST queue - submit jobs submitted_jobs = [] batch_size = 10 submit_params = params.copy() for i in range(0, batch_size): submit_params.update({ "module": queue_name_1, "action": "dummy", "parameters": [i, "a string", True, { "test": [] }, None] }) response = client.call(module, "job/submit", submit_params) assert_true(response.is_status_ok(), "submitted job") job_id = response.data["@job_id"] assert_true(len(job_id), "got job id") submitted_jobs.append(job_id) assert_equal(len(set(submitted_jobs)), batch_size, "submitted jobs have unique ids") response = client.call(module, "status", status_params) assert_true(response.is_status_ok(), "got status") status = response.data assert_equal(status["total_pending"], batch_size, "number of pending jobs") # SECOND queue - submit jobs for i in range(0, batch_size): submit_params.update({ "module": queue_name_2, "action": "dummy", "parameters": [i, "a string", True, { "test": [] }, None] }) response = client.call(module, "job/submit", submit_params) assert_true(response.is_status_ok(), "submitted job") response = client.call(module, "status", status_params) assert_true(response.is_status_ok(), "got status") status = response.data assert_equal(status["queues"][queue_name_2]["pending"], batch_size, "second queue got jobs") assert_equal(status["total_pending"], batch_size * 2, "number of pending jobs") # Close second queue switch_params = params.copy() switch_params["queue"] = queue_name_2 switch_params["enable"] = False response = client.call(module, "manage", switch_params) assert_true(response.is_status_ok(), "close queue call") response = client.call(module, "status", status_params) assert_true(response.is_status_ok(), "got status after close") status = response.data assert_equal(status["queues"][queue_name_2]["closed"], True, "second queue is now closed") # Reopen second queue switch_params = params.copy() switch_params["queue"] = queue_name_2 switch_params["enable"] = True response = client.call(module, "manage", switch_params) assert_true(response.is_status_ok(), "close queue call") response = client.call(module, "status", status_params) assert_true(response.is_status_ok(), "got status after reopen") status = response.data assert_equal(status["queues"][queue_name_2]["closed"], False, "second queue is now enabled") assert_equal(status["queues"][queue_name_2]["pending"], batch_size, "second queue still has pending jobs") # clear jobs in SECOND queue clear_params = params.copy() clear_params["queue"] = queue_name_2 clear_params["clear"] = True response = client.call(module, "manage", clear_params) assert_true(response.is_status_ok(), "close queue call") response = client.call(module, "status", status_params) assert_true(response.is_status_ok(), "got status after reopen") status = response.data assert_equal(status["queues"][queue_name_2]["pending"], 0, "second queue cleared") # remove SECOND queue remove_params = params.copy() remove_params["queue"] = queue_name_2 remove_params["remove"] = True response = client.call(module, "manage", remove_params) assert_true(response.is_status_ok(), "remove queue call") response = client.call(module, "status", status_params) assert_true(response.is_status_ok(), "got status after remove") status = response.data assert_false(queue_name_2 in status["queues"], "second queue removed") params = { "@api_key": development_key, "server": True, "stats": True, "module_stats": True, "routing": True, }