def test_routing_structure():
with NixServer(modules=[module]) as server:
client = NixClient(server.get_address())
result = client.call(module, "list_routes", {}, 2000)
# print(json.dumps(result.data, indent=4, sort_keys=True))
assert_false(result.is_rejected(), "not rejected")
assert_false(result.is_abandoned(), "not abandoned")
assert_true(result.is_replied(), "is replied")
assert_equal(expected_routing, result.data)
routing = result.data.get("routing")
auth_error_code = 101
# routes with access mod != ANY require api key -> code 101
for route in routing:
if routing[route]["access_modifier"] != "ANY":
response = client.call(module, route, {}, 2000)
if routing[route]["processing_type"] != "VOID":
status_code = response.data.get("@status_code")
assert_equal(status_code, auth_error_code)
# test API_PRIVATE with invalid key (public) -> code 101
for route in routing:
am = routing[route]["access_modifier"]
if am == "API_PRIVATE" and route != "list_routes":
params = {"@api_key" : development_key_public}
response = client.call(module, route, params, 2000)
status_code = response.data.get("@status_code")
assert_equal(status_code, 101)
def test_routing_structure():
with NixServer(modules=[module]) as server:
client = NixClient(server.get_address())
result = client.call(module, "list_routes", {}, 2000)
# print(json.dumps(result.data, indent=4, sort_keys=True))
assert_false(result.is_rejected(), "not rejected")
assert_false(result.is_abandoned(), "not abandoned")
assert_true(result.is_replied(), "is replied")
assert_equal(expected_routing, result.data)
def test_basic():
with NixServer(modules=["cache"]) as server:
client = NixClient(server.get_address())
result = client.call("Invalid", "invalid", {}, 2000)
assert_true(result.is_rejected(), "Invalid route (reject)")
result = client.call("Cache", "store", {}, 2000)
assert_true(result.is_status_fail(), "Is failed?")
assert_equal(result.status_code(), 101, "Auth error")
params = {
"@api_key" : development_key,
"key" : "key",
"value" : []
}
result = client.call("Cache", "store", params, 2000)
assert_true(result.is_status_ok(), "succeeded with auth")
assert_equal(result.status_code(), 0, "success indicated by 0")
def test_basic():
with NixServer(modules=server_modules) as server:
params = {
"@api_key": development_key,
"server": True,
"stats": True,
"module_stats": True,
"routing": True,
}
client = NixClient(server.get_address())
response = client.call("Status", "status", params)
# print(json.dumps(response.data, indent=4, sort_keys=True))
assert_true(response.is_status_ok(), "Got server status")
data = response.data
assert_true(data.get("nodename", ""), "has nodename")
assert_true(data.get("server", {}).get("routing", ""), "has routing")
assert_true(data.get("stats", {}), "stats")
for name in server_modules_names:
assert_true(data["stats"].get(name), "has stats for: %s" % name)
def run(self, event):
client = NixClient(self._server_address, verbose=False)
loop = 0
while not event.is_set():
for loop in range(0, 20):
if loop == 20:
self.remove_own_nodes(client)
continue
elif loop % 4 == 0:
self.register_node(client, loop)
self.resolve_random_node(client)
print("Signaled")
def test_nodes():
test_node = "unittest-01"
test_address = "tcp://127.0.0.1:33333"
with NixServer(modules=[module]) as server:
client = NixClient(server.get_address())
params = {"@api_key" : development_key}
# testing 'bind' route
invalid_params = params.copy()
response = client.call(module, "node/bind", invalid_params, 2000)
assert_equal(
response.status_code(),
501,
"Code 501 - indicates invalid parameters"
)
# valid
params.update({
"nodename" : test_node,
"address" : test_address
})
response = client.call(module, "node/bind", params, 2000)
assert_true(response.is_status_ok(), "Bind succeeded")
# testing 'resolve' route
params = {"@api_key" : development_key}
invalid_params = params.copy()
response = client.call(module, "node/resolve", invalid_params, 2000)
assert_true(response.is_status_fail(), "Resolve fails without node")
# valid
params.update({"nodename" : test_node})
response = client.call(module, "node/resolve", params, 2000)
assert_true(response.is_status_ok(), "Is resolved ok?")
assert_equal(
response.data[test_node],
test_address,
"Is resolved address correct?"
)
# testing 'unbind' route
response = client.send_one_way(module, "node/unbind", params)
time.sleep(0.2)
# after unbind node cannot be resolved
response = client.call(module, "node/resolve", params, 2000)
assert_true(response.is_status_fail(), "Is node unbound?")
def run(self, event):
client = NixClient(self._server_address, verbose=False)
cache_key = "cache-key-%d-" % threading.current_thread().ident
cache_value = "Random value %d: " % threading.current_thread().ident
while not event.is_set():
time.sleep(random.randint(1, 2) / 100.0)
# print("this is", threading.current_thread().ident)
params = {
"@api_key" : self._api_key,
"key" : "test", # cache_key,
"value" : "test", # cache_value,
}
response = client.call("Cache", "store", params, 2000)
if response.is_status_fail():
print("%s %d" % ("Store failed", threading.current_thread().ident))
response = client.call("Cache", "retrieve", params, 5000)
if response.is_status_fail():
print("Retrieve failed", response.data)
print(response.data)
response.data.get(cache_key) == cache_value
# client.send_one_way("Cache", "remove", params)
print("CacheClient finished")
def test_service():
test_key = "unittest"
test_value = {"a_string": "some string"}
with NixServer(modules=[module]) as server:
client = NixClient(server.get_address())
params = {"@api_key": development_key}
# testing 'store' route
invalid_params = params.copy()
response = client.call(module, "store", invalid_params, 2000)
assert_equal(response.status_code(), 501,
"Code 501 - indicates invalid parameters")
# valid
params.update({"key": test_key, "value": test_value})
response = client.call(module, "store", params, 2000)
assert_true(response.is_status_ok(), "Store succeeded")
# # testing 'retrieve' route
params = {"@api_key": development_key}
invalid_params = params.copy()
response = client.call(module, "retrieve", invalid_params, 2000)
assert_true(response.is_status_fail(), "Retrieve fails without key")
# valid
params = {"@api_key": development_key, "key": test_key}
response = client.call(module, "retrieve", params, 2000)
assert_true(response.is_status_ok(), "Is retrieve call ok?")
assert_equal(response.data.get(test_key), test_value,
"Is retrieved value correct?")
# testing 'remove' route
response = client.send_one_way(module, "remove", params)
time.sleep(0.2)
# after remove key cannot be retrieved
response = client.call(module, "retrieve", params, 2000)
assert_true(response.is_status_fail(), "Is key removed?")
if __name__ == "__main__":
event = threading.Event()
signal.signal(signal.SIGINT, lambda: event.set())
signal.signal(signal.SIGTERM, lambda: event.set())
threads = []
with NixServer(modules=SERVER_MODULES) as server:
for i in range(0, 50):
for cls in [ResolverClient, CacheClient]:
worker = cls(server.get_address(), DEVELOPMENT_KEY)
t = threading.Thread(target=worker.run, args=(event, ))
t.start()
threads.append(t)
try:
client = NixClient(server.get_address())
while True:
params = {
"@api_key": DEVELOPMENT_KEY,
"stats": True,
"module_stats": True,
}
response = client.call("Status", "status", params)
print(json.dumps(response.data, indent=4, sort_keys=True))
time.sleep(1)
except Exception as e:
event.set()
for t in threads:
print("join")
t.join()
def test_services():
test_service = "some_implementation"
test_nodename1 = "service-node-1"
test_address1 = "tcp://1.1.1.1:11111"
test_nodename2 = "service-node-2"
test_address2 = "tcp://2.2.2.2:22222"
with NixServer(modules=[module]) as server:
client = NixClient(server.get_address())
params = {"@api_key" : development_key}
invalid_params = params.copy()
response = client.call(module, "service/bind", invalid_params, 2000)
assert_equal(
response.status_code(),
501,
"Code 501 - indicates invalid parameters"
)
# fail bind when node is not yet known
invalid_params = params.copy()
invalid_params.update({
"service" : "test",
"nodename" : test_nodename1
})
response = client.call(module, "service/bind", params, 2000)
assert_true(
response.is_status_fail(),
"bind service fails (node not bound yet)"
)
# bind nodes
node_bind_params = params.copy()
node_bind_params.update({
"nodename" : test_nodename1,
"address" : test_address1
})
response = client.call(module, "node/bind", node_bind_params)
assert_true(response.is_status_ok(), "node 1 bound")
node_bind_params.update({
"nodename" : test_nodename2,
"address" : test_address2
})
response = client.call(module, "node/bind", node_bind_params)
assert_true(response.is_status_ok(), "node 2 bound")
# first node
params.update({
"service" : test_service,
"nodename" : test_nodename1
})
response = client.call(module, "service/bind", params, 2000)
assert_true(response.is_status_ok(), "1st bind service ok")
response = client.call(module, "service/resolve", params, 2000)
assert_true(response.is_status_ok(), "resolved after 1st bind")
assert_equal(
response.data.get("addresses", []),
[test_address1],
"resolved to a single address (1 elem array)"
)
# second node
params.update({
"service" : test_service,
"nodename" : test_nodename2
})
response = client.call(module, "service/bind", params, 2000)
assert_true(response.is_status_ok(), "2nd bind service ok")
response = client.call(module, "service/resolve", params, 2000)
assert_true(response.is_status_ok(), "resolved after 2nd bind")
assert_equal(
response.data.get("addresses", []),
[test_address1, test_address2],
"resolved both addresses single address (1 elem array)"
)
response = client.call(module, "service/unbind", params, 2000)
assert_true(response.is_status_ok(), "unbind 2nd address")
response = client.call(module, "service/resolve", params, 2000)
assert_true(response.is_status_ok(), "resolved after unbinding")
assert_equal(
response.data.get("addresses", []),
[test_address1],
"resolved a single address after unbind (1 elem array)"
)
params.update({
"nodename" : test_nodename1
})
response = client.call(module, "node/unbind", params, 2000)
assert_true(response.is_status_ok(), "unbind node1")
params.update({
"nodename" : test_nodename2
})
response = client.call(module, "node/unbind", params, 2000)
assert_true(response.is_status_ok(), "unbind node1")
response = client.call(module, "service/resolve", params, 2000)
assert_true(
response.is_status_fail(),
"Unbinding nodes unbinds services"
)
assert_equal(response.status_msg(), "Unknown service.")
def test_submit():
with NixServer(modules=["job-queue", "resolver"]) as server:
client = NixClient(server.get_address())
params = {"@api_key": development_key}
# testing 'submit' route
response = client.call(module, "job/submit", params, 100)
assert_true(response.is_status_fail(), "Fails without params")
job_module = "test"
job_action = "test"
job_param_name = "param"
job_param_value = "πœę©ß←"
# first: create the queue
create_params = {
"@api_key": development_key_admin,
"queue": job_module,
"create": {
"size": 256,
}
}
response = client.call(module, "manage", create_params)
assert_true(response.is_status_ok(), "created queue")
params.update({
"@origin_node": server.get_nodename(),
"module": job_module,
"action": job_action,
"parameters": {
job_param_name: job_param_value
}
})
response = client.call(module, "job/submit", params, 1000)
assert_true(response.is_status_ok(), "Submit ok")
JOB_ID = response.data["@job_id"]
assert_true(len(JOB_ID))
params = {"@api_key": development_key, "@job_id": JOB_ID}
response = client.call(module, "job/result/get", params, 100)
assert_true(response.is_status_fail(), "Fail - job not started")
params = {
"@api_key": development_key,
"module": "test" # queue name
}
response = client.call(module, "job/work/get", params, 1000)
assert_true(response.is_status_ok(), "Got job to process")
# this does not happen in real world, but this is a test
# and this is the only submitted job
assert_equal(response.data["@job_id"], JOB_ID, "Job id")
# ---
assert_equal(response.data["@origin_node"], server.get_nodename(),
"Origin node name set")
job_parameters = response.data["parameters"]
job_payload = response.data
# no need to test module - it was a name of the queue
assert_equal(job_payload["action"], job_action, "action")
assert_equal(job_parameters.get(job_param_name), job_param_value,
"API key")
# simulate some processing
total = 121
for processed in range(0, total, 20):
params = {
"@api_key": development_key,
"@job_id": JOB_ID,
"progress": float(processed) / total * 100
}
client.send_one_way(module, "job/progress/set", params)
time.sleep(0.01)
response = client.call(module, "job/result/get", params, 100)
if processed < total:
assert_equal(response.status_code(), 2,
"Got progress report (code 2: not_ready)")
params = {
"@api_key": development_key,
"@job_id": JOB_ID,
"parameters": {
job_param_name: "Result: %s" % job_param_value
}
}
response = client.call(module, "job/result/set", params, 100)
assert_true(response.is_status_ok(), "set response")
params = {
"@api_key": development_key,
"@job_id": JOB_ID,
}
response = client.call(module, "job/result/get", params, 100)
assert_true(response.is_status_ok(), "got result")
data = response.data
assert_equal(data["@api_key"], development_key, "result: api key")
assert_equal(data["@job_id"], JOB_ID, "result: job_id")
assert_equal(
data.get("parameters", {}).get(job_param_name, ""),
"Result: %s" % job_param_value, "result: job value")
def test_queue_management():
with NixServer(modules=["job-queue"]) as server:
client = NixClient(server.get_address())
params = {
"@api_key": development_key,
}
invalid_auth_params = {
"@api_key": development_key_public,
}
response = client.call(module, "manage", invalid_auth_params)
assert_equal(response.status_code(), 101, "Auth error")
status_params = {
"@api_key": development_key_admin,
}
response = client.call(module, "status", status_params, 2000)
assert_true(response.is_status_ok(), "got status")
status = response.data
assert_equal(status["total_completed"], 0, "no jobs done yey")
assert_equal(status["results_awaiting"], 0, "no results awaiting")
assert_equal(status["in_progress"], 0, "no jobs in progress")
assert_equal(status["total_pending"], 0, "no pending jobs")
queue_name_1 = "unittest-queue-%d" % os.getpid()
queue_name_2 = "second-queue-%d" % os.getpid()
create_params = {
"@api_key": development_key_admin,
"queue": queue_name_1,
"create": {
"size": 256,
}
}
response = client.call(module, "manage", create_params)
assert_true(response.is_status_ok(), "created queue")
create_params["queue"] = queue_name_2
response = client.call(module, "manage", create_params)
assert_true(response.is_status_ok(), "created second queue")
response = client.call(module, "manage", create_params)
assert_true(response.is_status_fail(), "queue already exists")
response = client.call(module, "status", status_params)
assert_true(response.is_status_ok(), "got status")
status = response.data
assert_equal(status["queues"][queue_name_1]["closed"], False,
"created queue is not closed")
assert_equal(status["queues"][queue_name_2]["closed"], False,
"second created queue is not closed")
assert_equal(status["queues"][queue_name_1]["pending"], 0,
"no jobs in created queue")
assert_equal(status["queues"][queue_name_2]["pending"], 0,
"no jobs in second created queue")
# FIRST queue - submit jobs
submitted_jobs = []
batch_size = 10
submit_params = params.copy()
for i in range(0, batch_size):
submit_params.update({
"module":
queue_name_1,
"action":
"dummy",
"parameters": [i, "a string", True, {
"test": []
}, None]
})
response = client.call(module, "job/submit", submit_params)
assert_true(response.is_status_ok(), "submitted job")
job_id = response.data["@job_id"]
assert_true(len(job_id), "got job id")
submitted_jobs.append(job_id)
assert_equal(len(set(submitted_jobs)), batch_size,
"submitted jobs have unique ids")
response = client.call(module, "status", status_params)
assert_true(response.is_status_ok(), "got status")
status = response.data
assert_equal(status["total_pending"], batch_size,
"number of pending jobs")
# SECOND queue - submit jobs
for i in range(0, batch_size):
submit_params.update({
"module":
queue_name_2,
"action":
"dummy",
"parameters": [i, "a string", True, {
"test": []
}, None]
})
response = client.call(module, "job/submit", submit_params)
assert_true(response.is_status_ok(), "submitted job")
response = client.call(module, "status", status_params)
assert_true(response.is_status_ok(), "got status")
status = response.data
assert_equal(status["queues"][queue_name_2]["pending"], batch_size,
"second queue got jobs")
assert_equal(status["total_pending"], batch_size * 2,
"number of pending jobs")
# Close second queue
switch_params = params.copy()
switch_params["queue"] = queue_name_2
switch_params["enable"] = False
response = client.call(module, "manage", switch_params)
assert_true(response.is_status_ok(), "close queue call")
response = client.call(module, "status", status_params)
assert_true(response.is_status_ok(), "got status after close")
status = response.data
assert_equal(status["queues"][queue_name_2]["closed"], True,
"second queue is now closed")
# Reopen second queue
switch_params = params.copy()
switch_params["queue"] = queue_name_2
switch_params["enable"] = True
response = client.call(module, "manage", switch_params)
assert_true(response.is_status_ok(), "close queue call")
response = client.call(module, "status", status_params)
assert_true(response.is_status_ok(), "got status after reopen")
status = response.data
assert_equal(status["queues"][queue_name_2]["closed"], False,
"second queue is now enabled")
assert_equal(status["queues"][queue_name_2]["pending"], batch_size,
"second queue still has pending jobs")
# clear jobs in SECOND queue
clear_params = params.copy()
clear_params["queue"] = queue_name_2
clear_params["clear"] = True
response = client.call(module, "manage", clear_params)
assert_true(response.is_status_ok(), "close queue call")
response = client.call(module, "status", status_params)
assert_true(response.is_status_ok(), "got status after reopen")
status = response.data
assert_equal(status["queues"][queue_name_2]["pending"], 0,
"second queue cleared")
# remove SECOND queue
remove_params = params.copy()
remove_params["queue"] = queue_name_2
remove_params["remove"] = True
response = client.call(module, "manage", remove_params)
assert_true(response.is_status_ok(), "remove queue call")
response = client.call(module, "status", status_params)
assert_true(response.is_status_ok(), "got status after remove")
status = response.data
assert_false(queue_name_2 in status["queues"], "second queue removed")
params = {
"@api_key": development_key,
"server": True,
"stats": True,
"module_stats": True,
"routing": True,
}