def test_http_server_plan_history(re_manager, fastapi_server): # noqa F811
# Select very short plan
plan = {
"item": {
"name": "count",
"args": [["det1", "det2"]],
"item_type": "plan"
}
}
request_to_json("post", "/queue/item/add", json=plan)
request_to_json("post", "/queue/item/add", json=plan)
request_to_json("post", "/queue/item/add", json=plan)
request_to_json("post", "/environment/open")
assert wait_for_environment_to_be_created(10), "Timeout"
request_to_json("post", "/queue/start")
ttime.sleep(5)
resp1 = request_to_json("get", "/history/get")
assert len(resp1["items"]) == 3
assert resp1["items"][0]["name"] == "count"
resp2 = request_to_json("post", "/history/clear")
assert resp2["success"] is True
resp3 = request_to_json("get", "/history/get")
assert resp3["items"] == []
def test_http_server_re_runs(re_manager, fastapi_server, suffix,
expected_n_items): # noqa F811
"""
Basic test for ``/re/run/...`` API. The API is tested on a single run plan.
"""
resp1 = request_to_json("post", "/queue/item/add", json={"item": _plan3})
assert resp1["success"] is True
assert resp1["qsize"] == 1
request_to_json("post", "/environment/open")
assert wait_for_environment_to_be_created(10), "Timeout"
request_to_json("post", "/queue/start")
ttime.sleep(2)
status = request_to_json("get", "/status")
assert status["manager_state"] == "executing_queue"
run_list_uid = status["run_list_uid"]
assert isinstance(run_list_uid, str)
req = "/re/runs" + suffix
resp2 = request_to_json("get", req)
assert resp2["success"] is True
assert len(resp2["run_list"]) == expected_n_items
assert resp2["run_list_uid"] == run_list_uid
assert wait_for_manager_state_idle(30), "Timeout"
def test_http_server_queue_start_handler(re_manager,
fastapi_server): # noqa F811
add_plans_to_queue()
resp1 = request_to_json("post", "/queue/start")
assert resp1 == {
"success": False,
"msg": "RE Worker environment does not exist."
}
resp2 = request_to_json("post", "/environment/open")
assert resp2 == {"success": True, "msg": ""}
resp2a = request_to_json("get", "/queue/get")
assert len(resp2a["items"]) == 3
assert resp2a["running_item"] == {}
assert wait_for_environment_to_be_created(10), "Timeout"
resp3 = request_to_json("post", "/queue/start")
assert resp3 == {"success": True, "msg": ""}
ttime.sleep(1)
# The plan is currently being executed. 'get_queue' is expected to return currently executed plan.
resp4 = request_to_json("get", "/queue/get")
assert len(resp4["items"]) == 2
assert resp4["running_item"][
"name"] == "count" # Check name of the running plan
ttime.sleep(25) # Wait until all plans are executed
resp4 = request_to_json("get", "/queue/get")
assert len(resp4["items"]) == 0
assert resp2a["running_item"] == {}
def test_http_server_secure_1(monkeypatch, re_manager_cmd, fastapi_server_fs,
test_mode): # noqa: F811
"""
Test operation of HTTP server with enabled encryption. Security of HTTP server can be enabled
only by setting the environment variable to the value of the public key.
"""
public_key, private_key = generate_new_zmq_key_pair()
if test_mode == "none":
# No encryption
pass
elif test_mode == "ev":
# Set server private key using environment variable
monkeypatch.setenv("QSERVER_ZMQ_PRIVATE_KEY",
private_key) # RE Manager
monkeypatch.setenv("QSERVER_ZMQ_PUBLIC_KEY", public_key) # HTTP server
set_qserver_zmq_public_key(
monkeypatch, server_public_key=public_key) # For test functions
else:
raise RuntimeError(f"Unrecognized test mode '{test_mode}'")
fastapi_server_fs()
re_manager_cmd([])
resp1 = request_to_json("post", "/queue/item/add", json={"item": _plan1})
assert resp1["success"] is True, str(resp1)
resp2 = request_to_json("post", "/queue/item/add", json={"item": _plan2})
assert resp2["success"] is True, str(resp2)
resp3 = request_to_json("get", "/plans/allowed")
assert isinstance(resp3["plans_allowed"], dict)
assert len(resp3["plans_allowed"]) > 0
resp4 = request_to_json("get", "/devices/allowed")
assert isinstance(resp4["devices_allowed"], dict)
assert len(resp4["devices_allowed"]) > 0
resp5 = request_to_json("post", "/environment/open")
assert resp5["success"] is True
assert wait_for_environment_to_be_created(10)
resp6 = request_to_json("get", "/status")
assert resp6["items_in_queue"] == 2
assert resp6["items_in_history"] == 0
resp7 = request_to_json("post", "/queue/start")
assert resp7["success"] is True
wait_for_queue_execution_to_complete(20)
resp8 = request_to_json("get", "/status")
assert resp8["items_in_queue"] == 0
assert resp8["items_in_history"] == 2
# Close the environment
resp9 = request_to_json("post", "/environment/close")
assert resp9 == {"success": True, "msg": ""}
wait_for_manager_state_idle(10)
def test_http_server_re_pause_continue_handlers(
re_manager,
fastapi_server,
option_pause,
option_continue # noqa F811
):
resp1 = request_to_json("post", "/environment/open")
assert resp1 == {"success": True, "msg": ""}
assert wait_for_environment_to_be_created(10), "Timeout"
resp2 = request_to_json(
"post",
"/queue/item/add",
json={
"item": {
"name": "count",
"args": [["det1", "det2"]],
"kwargs": {
"num": 10,
"delay": 1
},
"item_type": "plan",
}
},
)
assert resp2["success"] is True
assert resp2["qsize"] == 1
assert resp2["item"]["name"] == "count"
assert resp2["item"]["args"] == [["det1", "det2"]]
assert "item_uid" in resp2["item"]
resp3 = request_to_json("post", "/queue/start")
assert resp3 == {"success": True, "msg": ""}
ttime.sleep(
3.5
) # Let some time pass before pausing the plan (fractional number of seconds)
resp3a = request_to_json("post",
"/re/pause",
json={"option": option_pause})
assert resp3a == {"msg": "", "success": True}
ttime.sleep(2) # TODO: API is needed
resp3b = request_to_json("get", "/queue/get")
assert len(
resp3b["items"]) == 0 # The plan is paused, but it is not in the queue
assert resp3b["running_item"] != {} # Running plan is set
resp4 = request_to_json("post", f"/re/{option_continue}")
assert resp4 == {"msg": "", "success": True}
ttime.sleep(15) # TODO: we need to wait for plan completion
resp4a = request_to_json("get", "/queue/get")
# The plan returns to the queue if it is stopped
assert len(resp4a["items"]) == 0 if option_continue == "resume" else 1
assert resp4a["running_item"] == {}
def test_http_server_manager_stop_handler_1(re_manager, fastapi_server,
option): # noqa F811
request_to_json("post", "/environment/open")
assert wait_for_environment_to_be_created(10), "Timeout"
kwargs = {"json": {"option": option} if option else {}}
resp1 = request_to_json("post", "/manager/stop", **kwargs)
assert resp1["success"] is True
assert re_manager.check_if_stopped() is True
def test_http_server_open_environment_handler(re_manager,
fastapi_server): # noqa F811
resp1 = request_to_json("post", "/environment/open")
assert resp1 == {"success": True, "msg": ""}
assert wait_for_environment_to_be_created(10), "Timeout"
resp2 = request_to_json("post", "/environment/open")
assert resp2 == {
"success": False,
"msg": "RE Worker environment already exists."
}
def test_http_server_manager_kill(re_manager, fastapi_server): # noqa F811
request_to_json("post", "/environment/open")
assert wait_for_environment_to_be_created(10), "Timeout"
resp = request_to_json("post", "/test/manager/kill")
assert resp["success"] is False
assert "ZMQ communication error:" in resp["msg"]
ttime.sleep(10)
resp = request_to_json("get", "/status")
assert resp["msg"] == "RE Manager"
assert resp["manager_state"] == "idle"
assert resp["items_in_queue"] == 0
assert resp["running_item_uid"] is None
assert resp["worker_environment_exists"] is True
def test_http_server_close_print_db_uids_handler(re_manager,
fastapi_server): # noqa F811
add_plans_to_queue()
resp1 = request_to_json("post", "/environment/open")
assert resp1 == {"success": True, "msg": ""}
assert wait_for_environment_to_be_created(10), "Timeout"
resp2 = request_to_json("post", "/queue/start")
assert resp2 == {"success": True, "msg": ""}
ttime.sleep(15)
resp2a = request_to_json("get", "/queue/get")
assert len(resp2a["items"]) == 0
assert resp2a["running_item"] == {}
def test_http_server_queue_stop(re_manager, fastapi_server,
deactivate): # noqa F811
"""
Methods ``queue_stop_activate`` and ``queue_stop_deactivate``.
"""
add_plans_to_queue()
request_to_json("post", "/environment/open")
assert wait_for_environment_to_be_created(10), "Timeout"
# Queue is not running, so the request is expected to fail
resp1 = request_to_json("post", "/queue/stop")
assert resp1["success"] is False
status = request_to_json("get", "/status")
assert status["queue_stop_pending"] is False
request_to_json("post", "/queue/start")
ttime.sleep(2)
status = request_to_json("get", "/status")
assert status["manager_state"] == "executing_queue"
resp2 = request_to_json("post", "/queue/stop")
assert resp2["success"] is True
status = request_to_json("get", "/status")
assert status["queue_stop_pending"] is True
if deactivate:
ttime.sleep(1)
resp3 = request_to_json("post", "/queue/stop/cancel")
assert resp3["success"] is True
status = request_to_json("get", "/status")
assert status["queue_stop_pending"] is False
ttime.sleep(15)
status = request_to_json("get", "/status")
assert status["manager_state"] == "idle"
assert status["items_in_queue"] == (0 if deactivate else 2)
assert status["items_in_history"] == (3 if deactivate else 1)
assert status["running_item_uid"] is None
assert status["worker_environment_exists"] is True
assert status["queue_stop_pending"] is False
def test_http_server_close_environment_handler(re_manager,
fastapi_server): # noqa F811
resp1 = request_to_json("post", "/environment/open")
assert resp1 == {"success": True, "msg": ""}
assert wait_for_environment_to_be_created(10), "Timeout"
resp2 = request_to_json("post", "/environment/close")
assert resp2 == {"success": True, "msg": ""}
ttime.sleep(
3
) # TODO: API needed to test if environment is closed. Use delay for now.
resp3 = request_to_json("post", "/environment/close")
assert resp3 == {
"success": False,
"msg": "RE Worker environment does not exist."
}
def test_http_server_set_zmq_address_1(monkeypatch, re_manager_cmd,
fastapi_server_fs): # noqa: F811
"""
Test if ZMQ address of RE Manager is passed to the HTTP server using 'QSERVER_ZMQ_ADDRESS' environment
variable. Start RE Manager and HTTP server with ZMQ address for control communication channel
different from default address, add and execute a plan.
"""
# Change ZMQ address to use port 60616 instead of the default port 60615.
zmq_server_address = "tcp://localhost:60616"
monkeypatch.setenv("QSERVER_ZMQ_ADDRESS", zmq_server_address) # RE Manager
fastapi_server_fs()
set_qserver_zmq_address(monkeypatch, zmq_server_address=zmq_server_address)
re_manager_cmd(["--zmq-addr", "tcp://*:60616"])
# Now execute a plan to make sure everything works as expected
resp1 = request_to_json("post", "/queue/item/add", json={"item": _plan1})
assert resp1["success"] is True, str(resp1)
resp5 = request_to_json("post", "/environment/open")
assert resp5["success"] is True
assert wait_for_environment_to_be_created(10)
resp6 = request_to_json("get", "/status")
assert resp6["items_in_queue"] == 1
assert resp6["items_in_history"] == 0
resp7 = request_to_json("post", "/queue/start")
assert resp7["success"] is True
wait_for_queue_execution_to_complete(20)
resp8 = request_to_json("get", "/status")
assert resp8["items_in_queue"] == 0
assert resp8["items_in_history"] == 1
# Close the environment
resp9 = request_to_json("post", "/environment/close")
assert resp9 == {"success": True, "msg": ""}
wait_for_manager_state_idle(10)
def test_http_server_manager_stop_handler_2(re_manager, fastapi_server,
option): # noqa F811
add_plans_to_queue()
request_to_json("post", "/environment/open")
assert wait_for_environment_to_be_created(10), "Timeout"
request_to_json("post", "/queue/start")
ttime.sleep(2)
resp = request_to_json("get", "/status")
assert resp["msg"] == "RE Manager"
assert resp["manager_state"] == "executing_queue"
assert resp["items_in_queue"] == 2
assert resp["running_item_uid"] is not None
assert resp["items_in_history"] == 0
assert resp["worker_environment_exists"] is True
# Attempt to stop
kwargs = {"json": {"option": option} if option else {}}
resp1 = request_to_json("post", "/manager/stop", **kwargs)
assert resp1["success"] == (option == "safe_off")
if option == "safe_off":
assert re_manager.check_if_stopped() is True
else:
# The queue is expected to be running
ttime.sleep(15)
resp = request_to_json("get", "/status")
assert resp["msg"] == "RE Manager"
assert resp["manager_state"] == "idle"
assert resp["items_in_queue"] == 0
assert resp["items_in_history"] == 3
assert resp["running_item_uid"] is None
assert resp["worker_environment_exists"] is True
def test_http_server_queue_item_get_remove_handler_3(
re_manager, fastapi_server): # noqa F811
"""
Get and remove elements using plan UID. Successful and failing cases.
Note: the test is derived from ZMQ API test ``test_zmq_api_queue_item_get_remove_3()``
"""
request_to_json("post", "/queue/item/add", json={"item": _plan3})
request_to_json("post", "/queue/item/add", json={"item": _plan2})
request_to_json("post", "/queue/item/add", json={"item": _plan1})
resp1 = request_to_json("get", "/queue/get")
plans_in_queue = resp1["items"]
assert len(plans_in_queue) == 3
# Get and then remove plan 2 from the queue
uid = plans_in_queue[1]["item_uid"]
resp2a = request_to_json("post", "/queue/item/get", json={"uid": uid})
assert resp2a["item"]["item_uid"] == plans_in_queue[1]["item_uid"]
assert resp2a["item"]["name"] == plans_in_queue[1]["name"]
assert resp2a["item"]["args"] == plans_in_queue[1]["args"]
resp2b = request_to_json("post", "/queue/item/remove", json={"uid": uid})
assert resp2b["item"]["item_uid"] == plans_in_queue[1]["item_uid"]
assert resp2b["item"]["name"] == plans_in_queue[1]["name"]
assert resp2b["item"]["args"] == plans_in_queue[1]["args"]
# Start the first plan (this removes it from the queue)
# Also the rest of the operations will be performed on a running queue.
resp3 = request_to_json("post", "/environment/open")
assert resp3["success"] is True
assert wait_for_environment_to_be_created(10)
resp4 = request_to_json("post", "/queue/start")
assert resp4["success"] is True
ttime.sleep(1)
uid = plans_in_queue[0]["item_uid"]
resp5a = request_to_json("post", "/queue/item/get", json={"uid": uid})
assert resp5a["success"] is False
assert "is currently running" in resp5a["msg"]
resp5b = request_to_json("post", "/queue/item/remove", json={"uid": uid})
assert resp5b["success"] is False
assert "Can not remove an item which is currently running" in resp5b["msg"]
uid = "nonexistent"
resp6a = request_to_json("post", "/queue/item/get", json={"uid": uid})
assert resp6a["success"] is False
assert "not in the queue" in resp6a["msg"]
resp6b = request_to_json("post", "/queue/item/remove", json={"uid": uid})
assert resp6b["success"] is False
assert "not in the queue" in resp6b["msg"]
# Remove the last entry
uid = plans_in_queue[2]["item_uid"]
resp7a = request_to_json("post", "/queue/item/get", json={"uid": uid})
assert resp7a["success"] is True
resp7b = request_to_json("post", "/queue/item/remove", json={"uid": uid})
assert resp7b["success"] is True
ttime.sleep(10) # TODO: wait for the queue processing to be completed
state = request_to_json("get", "/status")
assert state["items_in_queue"] == 0
assert state["items_in_history"] == 1
def test_http_server_queue_upload_spreasheet_1(re_manager, fastapi_server_fs,
tmp_path,
monkeypatch): # noqa F811
"""
Test for ``/queue/upload/spreadsheet`` API: generate .xlsx file, upload it to the server, verify
the contents of the queue, run the queue and verify that the required number of plans were successfully
completed.
"""
monkeypatch.setenv(
"BLUESKY_HTTPSERVER_CUSTOM_MODULE",
"bluesky_queueserver.server.tests.http_custom_proc_functions",
prepend=False,
)
fastapi_server_fs()
plan_params = [["count", 5, 1], ["count", 6, 0.5]]
col_names = ["name", "num", "delay"]
ss_path, plans_expected = _create_test_excel_file1(tmp_path,
plan_params=plan_params,
col_names=col_names)
# Send the Excel file to the server
files = {"spreadsheet": open(ss_path, "rb")}
resp1 = request_to_json("post", "/queue/upload/spreadsheet", files=files)
assert resp1["success"] is True, str(resp1)
items1 = resp1["items"]
results1 = resp1["results"]
assert len(items1) == len(plans_expected), str(items1)
for p, p_exp in zip(items1, plans_expected):
for k, v in p_exp.items():
assert k in p
assert v == p[k]
assert len(results1) == len(plans_expected), str(results1)
assert all([_["success"] is True for _ in results1]), str(results1)
assert all([_["msg"] == "" for _ in results1]), str(results1)
# Verify that the queue contains correct plans
resp2 = request_to_json("get", "/queue/get")
assert resp2["success"] is True
assert resp2["running_item"] == {}
queue = resp2["items"]
assert len(queue) == len(plans_expected), str(queue)
for p, p_exp in zip(queue, plans_expected):
for k, v in p_exp.items():
assert k in p
assert v == p[k]
resp3 = request_to_json("post", "/environment/open")
assert resp3["success"] is True
assert wait_for_environment_to_be_created(10)
resp4 = request_to_json("post", "/queue/start")
assert resp4["success"] is True
assert wait_for_queue_execution_to_complete(60)
resp5 = request_to_json("get", "/status")
assert resp5["items_in_queue"] == 0
assert resp5["items_in_history"] == len(plans_expected)
resp6 = request_to_json("post", "/environment/close")
assert resp6 == {"success": True, "msg": ""}
assert wait_for_manager_state_idle(10)
def test_http_server_queue_upload_spreasheet_4(
re_manager,
fastapi_server_fs,
tmp_path,
monkeypatch,
use_custom # noqa F811
):
"""
Test for ``/queue/upload/spreadsheet`` API. Pass the spreadsheet to the default processing function
either directly (use_custom=False) or first pass it to the custom processing function which
rejects the spreadsheet by returning ``None``. If custom processing function returns ``None``, then
the spreadsheet is passed to the default function.
NOTE: currently the default processing function is not implemented and the request returns error message.
The test will have to be modified, when the function is implemented.
"""
if use_custom:
monkeypatch.setenv(
"BLUESKY_HTTPSERVER_CUSTOM_MODULE",
"bluesky_queueserver.server.tests.http_custom_proc_functions",
prepend=False,
)
fastapi_server_fs()
ss_path = create_excel_file_from_plan_list(tmp_path,
plan_list=plan_list_sample)
plans_expected = [
_ for _ in plan_list_sample if isinstance(_["name"], str)
]
# Send the Excel file to the server
params = {"files": {"spreadsheet": open(ss_path, "rb")}}
if use_custom:
params["data"] = {"data_type": "process_with_default_function"}
resp1 = request_to_json("post", "/queue/upload/spreadsheet", **params)
assert resp1["success"] is True, str(resp1)
assert "items" in resp1, str(resp1)
assert "results" in resp1, str(resp1)
assert len(resp1["results"]) == len(plans_expected), str(resp1)
# Verify that the queue contains correct plans
resp2 = request_to_json("get", "/queue/get")
assert resp2["success"] is True
assert resp2["running_item"] == {}
queue = resp2["items"]
assert len(queue) == len(plans_expected), str(queue)
for p, p_exp in zip(queue, plans_expected):
for k, v in p_exp.items():
assert k in p
assert v == p[k]
resp3 = request_to_json("post", "/environment/open")
assert resp3["success"] is True
assert wait_for_environment_to_be_created(10)
resp4 = request_to_json("post", "/queue/start")
assert resp4["success"] is True
assert wait_for_queue_execution_to_complete(60)
resp5 = request_to_json("get", "/status")
assert resp5["items_in_queue"] == 0
assert resp5["items_in_history"] == len(plans_expected)
resp6 = request_to_json("post", "/environment/close")
assert resp6 == {"success": True, "msg": ""}
assert wait_for_manager_state_idle(10)
