def __init__(self,
                 zmq_server_address=None,
                 user_name="GUI Client",
                 user_group="admin"):
        self._client = ZMQCommSendThreads(
            zmq_server_address=zmq_server_address)
        self.set_map_param_labels_to_keys()

        # User name and group are hard coded for now
        self._user_name = user_name
        self._user_group = user_group

        self._re_manager_status = {}
        self._re_manager_connected = None
        self._re_manager_status_time = time.time()
        # Minimum period of status update (avoid excessive call frequency)
        self._re_manager_status_update_period = 0.2

        self._allowed_devices = {}
        self._allowed_plans = {}
        self._plan_queue_items = []
        # Dictionary key: item uid, value: item pos in queue:
        self._plan_queue_items_pos = {}
        self._running_item = {}
        self._plan_queue_uid = ""
        self._run_list = []
        self._run_list_uid = ""
        self._plan_history_items = []
        self._plan_history_uid = ""

        # UID of the selected queue item, "" if no items are selected
        self._selected_queue_item_uid = ""
        # History items are addressed by position (there could be repeated UIDs in the history)
        #   Items in the history can not be moved or deleted, only added to the bottom, so
        #   using positions is consistent.
        self._selected_history_item_pos = -1

        # TODO: in the future the list of allowed instructions should be requested from the server
        self._allowed_instructions = {
            "queue_stop": {
                "name": "queue_stop",
                "description": "Stop execution of the queue.",
            }
        }

        self.events = EmitterGroup(
            source=self,
            status_changed=Event,
            plan_queue_changed=Event,
            running_item_changed=Event,
            plan_history_changed=Event,
            allowed_devices_changed=Event,
            allowed_plans_changed=Event,
            queue_item_selection_changed=Event,
            history_item_selection_changed=Event,
        )
Example #2
0
    def __init__(self, worker_address=None):
        self._client = ZMQCommSendThreads(zmq_server_address=worker_address)

        self._re_manager_status = {}
        self._re_manager_connected = None
        self._re_manager_status_time = time.time()
        # Minimum period of status update (avoid excessive call frequency)
        self._re_manager_status_update_period = 0.2

        self.events = EmitterGroup(
            source=self,
            status_changed=Event,
        )
Example #3
0
def test_ZMQCommSendThreads_2(is_blocking, encryption_enabled):
    """
    Basic test of ZMQCommSendThreads class: two consecutive communications with the
    server both in blocking and non-blocking mode.
    """
    public_key, _, server_kwargs = _gen_server_keys(
        encryption_enabled=encryption_enabled)

    thread = threading.Thread(target=_zmq_server_2msg, kwargs=server_kwargs)
    thread.start()

    zmq_comm = ZMQCommSendThreads(
        server_public_key=public_key if encryption_enabled else None)
    method, params = "testing", {"p1": 10, "p2": "abc"}

    msg_recv, msg_recv_err = {}, ""

    for val in (10, 20):
        if is_blocking:
            msg_recv = zmq_comm.send_message(method=method, params=params)
        else:
            done = False

            def cb(msg, msg_err):
                nonlocal msg_recv, msg_recv_err, done
                msg_recv = msg
                msg_recv_err = msg_err
                done = True

            zmq_comm.send_message(method=method, params=params, cb=cb)
            # Implement primitive polling of 'done' flag
            while not done:
                ttime.sleep(0.1)

        assert msg_recv["success"] is True, str(msg_recv)
        assert msg_recv["some_data"] == val, str(msg_recv)
        assert msg_recv["msg_in"] == {
            "method": method,
            "params": params
        }, str(msg_recv)
        assert msg_recv_err == ""

    thread.join()
def test_ZMQCommSendThreads_1(is_blocking):
    """
    Basic test of ZMQCommSendThreads class: single communication with the
    server both in blocking and non-blocking mode.
    """

    thread = threading.Thread(target=_zmq_server_1msg)
    thread.start()

    zmq_comm = ZMQCommSendThreads()
    method, params = "testing", {"p1": 10, "p2": "abc"}

    msg_recv, msg_recv_err = {}, ""

    if is_blocking:
        msg_recv = zmq_comm.send_message(method=method, params=params)
    else:
        done = False

        def cb(msg, msg_err):
            nonlocal msg_recv, msg_recv_err, done
            msg_recv = msg
            msg_recv_err = msg_err
            done = True

        zmq_comm.send_message(method=method, params=params, cb=cb)
        # Implement primitive polling of 'done' flag
        while not done:
            ttime.sleep(0.1)

    assert msg_recv["success"] is True, str(msg_recv)
    assert msg_recv["some_data"] == 10, str(msg_recv)
    assert msg_recv["msg_in"] == {
        "method": method,
        "params": params
    }, str(msg_recv)
    assert msg_recv_err == ""

    thread.join()
def main():
    with gui_qt("Example App"):
        worker_address, message_bus_address = sys.argv[1:]
        dispatcher = RemoteDispatcher(message_bus_address)
        client = ZMQCommSendThreads(zmq_server_address=worker_address)

        # NOTE: this example starts only if RE Manager is idle and the queue is cleared.
        #   Those are optional steps that ensure that the code in this example is executed correctly.

        # Check if RE Worker environment already exists and RE manager is idle.
        status = client.send_message(method="status")
        if status["manager_state"] != "idle":
            raise RuntimeError(
                f"RE Manager state must be 'idle': current state: {status['manager_state']}"
            )

        # Clear the queue.
        response = client.send_message(method="queue_clear")
        if not response["success"]:
            raise RuntimeError(
                f"Failed to clear the plan queue: {response['msg']}")

        # Open the new environment only if it does not exist.
        if not status["worker_environment_exists"]:
            # Initiate opening of RE Worker environment
            response = client.send_message(method="environment_open")
            if not response["success"]:
                raise RuntimeError(
                    f"Failed to open RE Worker environment: {response['msg']}")

            # Wait for the environment to be created.
            t_timeout = 10
            t_stop = time.time() + t_timeout
            while True:
                status2 = client.send_message(method="status")
                if (status2["worker_environment_exists"]
                        and status2["manager_state"] == "idle"):
                    break
                if time.time() > t_stop:
                    raise RuntimeError(
                        "Failed to start RE Worker: timeout occurred")
                time.sleep(0.5)

        # Add plan to queue
        response = client.send_message(
            method="queue_item_add",
            params={
                "plan": {
                    "name": "scan",
                    "args": [["det"], "motor", -5, 5, 11]
                },
                "user": "",
                "user_group": "admin",
            },
        )
        if not response["success"]:
            raise RuntimeError(
                f"Failed to add plan to the queue: {response['msg']}")

        model = Lines("motor", ["det"], max_runs=3)
        dispatcher.subscribe(stream_documents_into_runs(model.add_run))
        view = QtFigure(model.figure)
        view.show()
        dispatcher.start()

        response = client.send_message(method="queue_start")
        if not response["success"]:
            raise RuntimeError(f"Failed to start the queue: {response['msg']}")
Example #6
0
def test_ZMQCommSendThreads_3(is_blocking, encryption_enabled):
    """
    Testing protection of '_zmq_communicate` with lock. In this test the function
    ``send_message` is called twice so that the second call is submitted before
    the response to the first message is received. The server waits for 0.1 seconds
    before responding to the 1st message to emulate delay in processing. Since
    ``_zmq_communicate`` is protected by a lock, the second request will not
    be sent until the first message is processed.
    """
    public_key, _, server_kwargs = _gen_server_keys(
        encryption_enabled=encryption_enabled)

    thread = threading.Thread(target=_zmq_server_2msg_delay1,
                              kwargs=server_kwargs)
    thread.start()

    zmq_comm = ZMQCommSendThreads(
        server_public_key=public_key if encryption_enabled else None)
    method, params = "testing", {"p1": 10, "p2": "abc"}

    msg_recv, msg_recv_err = [], []
    n_done = 0

    def cb(msg, msg_err):
        nonlocal msg_recv, msg_recv_err, n_done
        msg_recv.append(msg)
        msg_recv_err.append(msg_err)
        n_done += 1

    vals = (10, 20)

    if is_blocking:
        # In case of blocking call the lock can only be tested using threads
        def thread_request():
            nonlocal msg_recv, msg_recv_err, n_done
            _ = zmq_comm.send_message(method=method, params=params)
            msg_recv.append(_)
            msg_recv_err.append("")
            n_done += 1

        th_request = threading.Thread(target=thread_request)
        th_request.start()

        # Call the same function in the main thread (send 2nd message to the server)
        thread_request()

        th_request.join()

    else:
        for n in range(len(vals)):
            zmq_comm.send_message(method=method, params=params, cb=cb)

    while n_done < 2:
        ttime.sleep(0.1)

    for n, val in enumerate(vals):
        assert msg_recv[n]["success"] is True, str(msg_recv)
        assert msg_recv[n]["some_data"] == val, str(msg_recv)
        assert msg_recv[n]["msg_in"] == {
            "method": method,
            "params": params
        }, str(msg_recv)
        assert msg_recv_err[n] == ""

    thread.join()
Example #7
0
def test_ZMQCommSendThreads_5_fail():
    """
    Invalid public key
    """
    with pytest.raises(ValueError):
        ZMQCommSendThreads(server_public_key="abc")
Example #8
0
def test_ZMQCommSendThreads_4(is_blocking, raise_exception,
                              delay_between_reads, encryption_enabled):
    """
    ZMQCommSendThreads: Timeout at the server.
    """
    public_key, _, server_kwargs = _gen_server_keys(
        encryption_enabled=encryption_enabled)

    thread = threading.Thread(target=_zmq_server_delay2, kwargs=server_kwargs)
    thread.start()

    zmq_comm = ZMQCommSendThreads(
        server_public_key=public_key if encryption_enabled else None)
    method, params = "testing", {"p1": 10, "p2": "abc"}

    msg_recv, msg_recv_err = {}, ""

    for val in (10, 20):
        if is_blocking:
            if (raise_exception in (True, None)) and (val == 10):
                # The case when timeout is expected for blocking operation
                with pytest.raises(CommTimeoutError, match="timeout occurred"):
                    zmq_comm.send_message(method=method,
                                          params=params,
                                          raise_exceptions=raise_exception)
            else:
                msg_recv = zmq_comm.send_message(
                    method=method,
                    params=params,
                    raise_exceptions=raise_exception)
        else:
            done = False

            def cb(msg, msg_err):
                nonlocal msg_recv, msg_recv_err, done
                msg_recv = msg
                msg_recv_err = msg_err
                done = True

            zmq_comm.send_message(method=method,
                                  params=params,
                                  cb=cb,
                                  raise_exceptions=raise_exception)

            # Implement primitive polling of 'done' flag
            while not done:
                ttime.sleep(0.1)

        if val == 10:
            if is_blocking:
                if raise_exception not in (True, None):
                    assert msg_recv["success"] is False, str(msg_recv)
                    assert "timeout occurred" in msg_recv["msg"], str(msg_recv)
            else:
                assert msg_recv == {}
                assert "timeout occurred" in msg_recv_err

            # Delay between consecutive reads. Test cases when read is initiated before and
            #   after the server restored operation and sent the response.
            ttime.sleep(delay_between_reads)

        else:
            assert msg_recv["success"] is True, str(msg_recv)
            assert msg_recv["some_data"] == val, str(msg_recv)
            assert msg_recv["msg_in"] == {
                "method": method,
                "params": params
            }, str(msg_recv)
            assert msg_recv_err == ""

    thread.join()
Example #9
0
class RunEngineClient:
    def __init__(self, worker_address=None):
        self._client = ZMQCommSendThreads(zmq_server_address=worker_address)

        self._re_manager_status = {}
        self._re_manager_connected = None
        self._re_manager_status_time = time.time()
        # Minimum period of status update (avoid excessive call frequency)
        self._re_manager_status_update_period = 0.2

        self.events = EmitterGroup(
            source=self,
            status_changed=Event,
        )

    @property
    def re_manager_status(self):
        return self._re_manager_status

    @property
    def re_manager_accessible(self):
        return self._re_manager_connected

    def clear(self):
        # Clear the queue.
        response = self._client.send_message(method="queue_clear")
        if not response["success"]:
            raise RuntimeError(
                f"Failed to clear the plan queue: {response['msg']}")

    def clear_connection_status(self):
        """
        This function is not expected to clear 'status', only 'self._re_manager_connected'.
        """
        self._re_manager_connected = None
        self.events.status_changed(
            status=self._re_manager_status,
            is_connected=self._re_manager_connected,
        )

    def load_re_manager_status(self, *, enforce=False):
        if enforce or (time.time() - self._re_manager_status_time >
                       self._re_manager_status_update_period):
            status = self._re_manager_status.copy()
            accessible = self._re_manager_connected
            try:
                new_manager_status = self._client.send_message(
                    method="status", raise_exceptions=True)
                self._re_manager_status.clear()
                self._re_manager_status.update(new_manager_status)
                self._re_manager_connected = True
            except CommTimeoutError:
                self._re_manager_connected = False
            if (status != self._re_manager_status) or (
                    accessible != self._re_manager_connected):
                # Status changed. Initiate the updates
                self.events.status_changed(
                    status=self._re_manager_status,
                    is_connected=self._re_manager_connected,
                )

    # ============================================================================
    #                  Operations with RE Environment

    def environment_open(self, timeout=0):
        """
        Open RE Worker environment. Blocks until operation is complete or timeout expires.
        If ``timeout=0``, then the function blocks until operation is complete.

        Parameters
        ----------
        timeout : float
            maximum time for the operation. Exception is raised if timeout expires.
            If ``timeout=0``, the function blocks until operation is complete.

        Returns
        -------
        None
        """
        # Check if RE Worker environment already exists and RE manager is idle.
        self.load_re_manager_status()
        status = self._re_manager_status
        if status["manager_state"] != "idle":
            raise RuntimeError(
                f"RE Manager state must be 'idle': current state: {status['manager_state']}"
            )
        if status["worker_environment_exists"]:
            raise RuntimeError("RE Worker environment already exists")

        # Initiate opening of RE Worker environment
        response = self._client.send_message(method="environment_open")
        if not response["success"]:
            raise RuntimeError(
                f"Failed to open RE Worker environment: {response['msg']}")

        # Wait for the environment to be created.
        if timeout:
            t_stop = time.time() + timeout
        while True:
            self.load_re_manager_status()
            status2 = self._re_manager_status
            if (status2["worker_environment_exists"]
                    and status2["manager_state"] == "idle"):
                break
            if timeout and (time.time() > t_stop):
                raise RuntimeError(
                    "Failed to start RE Worker: timeout occurred")
            time.sleep(0.5)

    def environment_close(self, timeout=0):
        """
        Close RE Worker environment. Blocks until operation is complete or timeout expires.
        If ``timeout=0``, then the function blocks until operation is complete.

        Parameters
        ----------
        timeout : float
            maximum time for the operation. Exception is raised if timeout expires.
            If ``timeout=0``, the function blocks until operation is complete.

        Returns
        -------
        None
        """
        # Check if RE Worker environment already exists and RE manager is idle.
        self.load_re_manager_status()
        status = self._re_manager_status
        if status["manager_state"] != "idle":
            raise RuntimeError(
                f"RE Manager state must be 'idle': current state: {status['manager_state']}"
            )
        if not status["worker_environment_exists"]:
            raise RuntimeError("RE Worker environment does not exist")

        # Initiate opening of RE Worker environment
        response = self._client.send_message(method="environment_close")
        if not response["success"]:
            raise RuntimeError(
                f"Failed to close RE Worker environment: {response['msg']}")

        # Wait for the environment to be created.
        if timeout:
            t_stop = time.time() + timeout
        while True:
            self.load_re_manager_status()
            status2 = self._re_manager_status
            if (not status2["worker_environment_exists"]
                    and status2["manager_state"] == "idle"):
                break
            if timeout and (time.time() > t_stop):
                raise RuntimeError(
                    "Failed to start RE Worker: timeout occurred")
            time.sleep(0.5)

    def environment_destroy(self, timeout=0):
        """
        Destroy (unresponsive) RE Worker environment. The function is intended for the cases when
        the environment is unresponsive and can not be stopped using ``environment_close``.
        Blocks until operation is complete or timeout expires. If ``timeout=0``, then the function
        blocks until operation is complete.

        Parameters
        ----------
        timeout : float
            maximum time for the operation. Exception is raised if timeout expires.
            If ``timeout=0``, the function blocks until operation is complete.

        Returns
        -------
        None
        """
        # Check if RE Worker environment already exists and RE manager is idle.
        self.load_re_manager_status()
        status = self._re_manager_status
        if not status["worker_environment_exists"]:
            raise RuntimeError("RE Worker environment does not exist")

        # Initiate opening of RE Worker environment
        response = self._client.send_message(method="environment_destroy")
        if not response["success"]:
            raise RuntimeError(
                f"Failed to destroy RE Worker environment: {response['msg']}")

        # Wait for the environment to be created.
        if timeout:
            t_stop = time.time() + timeout
        while True:
            self.load_re_manager_status()
            status2 = self._re_manager_status
            if (not status2["worker_environment_exists"]
                    and status2["manager_state"] == "idle"):
                break
            if timeout and (time.time() > t_stop):
                raise RuntimeError(
                    "Failed to start RE Worker: timeout occurred")
            time.sleep(0.5)

    # ============================================================================
    #                        Queue Control

    def queue_start(self):
        response = self._client.send_message(method="queue_start")
        if not response["success"]:
            raise RuntimeError(f"Failed to start the queue: {response['msg']}")

    def queue_stop(self):
        response = self._client.send_message(method="queue_stop")
        if not response["success"]:
            raise RuntimeError(
                f"Failed to request stopping the queue: {response['msg']}")

    def queue_stop_cancel(self):
        response = self._client.send_message(method="queue_stop_cancel")
        if not response["success"]:
            raise RuntimeError(
                f"Failed to cancel request to stop the queue: {response['msg']}"
            )

    # ============================================================================
    #                        RE Control

    def _wait_for_completion(self,
                             *,
                             condition,
                             msg="complete operation",
                             timeout=0):
        if timeout:
            t_stop = time.time() + timeout

        while True:
            self.load_re_manager_status()
            status = self._re_manager_status
            if condition(status):
                break
            if timeout and (time.time() > t_stop):
                raise RuntimeError(f"Failed to {msg}: timeout occurred")
            time.sleep(0.5)

    def re_pause(self, timeout=0, *, option):
        """
        Pause execution of a plan.

        Parameters
        ----------
        timeout : float
            maximum time for the operation. Exception is raised if timeout expires.
            If ``timeout=0``, the function blocks until operation is complete.

        option : str
            "immediate" or "deferred"
        Returns
        -------
        None
        """

        # Initiate opening of RE Worker environment
        response = self._client.send_message(method="re_pause",
                                             params={"option": option})
        if not response["success"]:
            raise RuntimeError(
                f"Failed to pause the running plan: {response['msg']}")

        def condition(status):
            return status["manager_state"] in ("idle", "paused")

        self._wait_for_completion(condition=condition,
                                  msg="pause the running plan",
                                  timeout=timeout)

    def re_resume(self, timeout=0):
        """
        Pause execution of a plan.

        Parameters
        ----------
        timeout : float
            maximum time for the operation. Exception is raised if timeout expires.
            If ``timeout=0``, the function blocks until operation is complete.

        Returns
        -------
        None
        """

        # Initiate opening of RE Worker environment
        response = self._client.send_message(method="re_resume")
        if not response["success"]:
            raise RuntimeError(
                f"Failed to resume the running plan: {response['msg']}")

        def condition(status):
            return status["manager_state"] in ("idle", "executing_queue")

        self._wait_for_completion(condition=condition,
                                  msg="resume execution of the plan",
                                  timeout=timeout)

    def _re_continue_plan(self, *, action, timeout=0):

        if action not in ("stop", "abort", "halt"):
            raise RuntimeError(f"Unrecognized action '{action}'")

        method = f"re_{action}"

        response = self._client.send_message(method=method)
        if not response["success"]:
            raise RuntimeError(
                f"Failed to {action} the running plan: {response['msg']}")

        def condition(status):
            return status["manager_state"] == "idle"

        self._wait_for_completion(condition=condition,
                                  msg=f"{action} the plan",
                                  timeout=timeout)

    def re_stop(self, timeout=0):
        self._re_continue_plan(action="stop", timeout=timeout)

    def re_abort(self, timeout=0):
        self._re_continue_plan(action="abort", timeout=timeout)

    def re_halt(self, timeout=0):
        self._re_continue_plan(action="halt", timeout=timeout)

    def add(self, plan_name, plan_args):
        # Add plan to queue
        response = self._client.send_message(
            method="queue_item_add",
            params={
                "plan": {
                    "name": plan_name,
                    "args": plan_args
                },
                "user": "",
                "user_group": "admin",
            },
        )
        if not response["success"]:
            raise RuntimeError(
                f"Failed to add plan to the queue: {response['msg']}")
class RunEngineClient:
    """
    Parameters
    ----------
    zmq_server_address : str or None
        Address of ZMQ server (Run Engine Manager). If None, then the default address defined
        in RE Manager code is used. (Default address is ``tcp://localhost:60615``).
    user_name : str
        Name of the user submitting the plan. The name is saved as a parameter of the queue item
        and identifies the user submitting the plan (may be important in multiuser systems).
    user_group : str
        Name of the user group. User group is saved as a parameter of a queue item. Each user group
        can be assigned permissions to use a restricted set of plans and pass a restricted set of
        devices as plan parameters. Groups and group permissions are defined in the file
        ``user_group_permissions.yaml`` (see documentation for RE Manager).
    """
    def __init__(self,
                 zmq_server_address=None,
                 user_name="GUI Client",
                 user_group="admin"):
        self._client = ZMQCommSendThreads(
            zmq_server_address=zmq_server_address)
        self.set_map_param_labels_to_keys()

        # User name and group are hard coded for now
        self._user_name = user_name
        self._user_group = user_group

        self._re_manager_status = {}
        self._re_manager_connected = None
        self._re_manager_status_time = time.time()
        # Minimum period of status update (avoid excessive call frequency)
        self._re_manager_status_update_period = 0.2

        self._allowed_devices = {}
        self._allowed_plans = {}
        self._plan_queue_items = []
        # Dictionary key: item uid, value: item pos in queue:
        self._plan_queue_items_pos = {}
        self._running_item = {}
        self._plan_queue_uid = ""
        self._run_list = []
        self._run_list_uid = ""
        self._plan_history_items = []
        self._plan_history_uid = ""

        # UID of the selected queue item, "" if no items are selected
        self._selected_queue_item_uid = ""
        # History items are addressed by position (there could be repeated UIDs in the history)
        #   Items in the history can not be moved or deleted, only added to the bottom, so
        #   using positions is consistent.
        self._selected_history_item_pos = -1

        # TODO: in the future the list of allowed instructions should be requested from the server
        self._allowed_instructions = {
            "queue_stop": {
                "name": "queue_stop",
                "description": "Stop execution of the queue.",
            }
        }

        self.events = EmitterGroup(
            source=self,
            status_changed=Event,
            plan_queue_changed=Event,
            running_item_changed=Event,
            plan_history_changed=Event,
            allowed_devices_changed=Event,
            allowed_plans_changed=Event,
            queue_item_selection_changed=Event,
            history_item_selection_changed=Event,
        )

    @property
    def re_manager_status(self):
        return self._re_manager_status

    @property
    def re_manager_connected(self):
        return self._re_manager_connected

    def clear(self):
        # Clear the queue.
        response = self._client.send_message(method="queue_clear")
        if not response["success"]:
            raise RuntimeError(
                f"Failed to clear the plan queue: {response['msg']}")

    def clear_connection_status(self):
        """
        This function is not expected to clear 'status', only 'self._re_manager_connected'.
        """
        self._re_manager_connected = None
        self.events.status_changed(
            status=self._re_manager_status,
            is_connected=self._re_manager_connected,
        )

    def manager_connecting_ops(self):
        """
        Sequence of additional operations that should be performed while connecting to RE Manager.
        """
        self.load_allowed_devices()
        self.load_allowed_plans()
        self.load_plan_queue()
        self.load_plan_history()

    def load_re_manager_status(self, *, unbuffered=False):
        if unbuffered or (time.time() - self._re_manager_status_time >
                          self._re_manager_status_update_period):
            status = self._re_manager_status.copy()
            accessible = self._re_manager_connected
            try:
                new_manager_status = self._client.send_message(
                    method="status", raise_exceptions=True)
                self._re_manager_status.clear()
                self._re_manager_status.update(new_manager_status)
                self._re_manager_connected = True

                new_queue_uid = self._re_manager_status.get(
                    "plan_queue_uid", "")
                if new_queue_uid != self._plan_queue_uid:
                    self.load_plan_queue()
                new_run_list_uid = self._re_manager_status.get(
                    "run_list_uid", "")
                if new_run_list_uid != self._run_list_uid:
                    self.load_run_list()
                new_history_uid = self._re_manager_status.get(
                    "plan_history_uid", "")
                if new_history_uid != self._plan_history_uid:
                    self.load_plan_history()

            except CommTimeoutError:
                self._re_manager_connected = False
            if (status != self._re_manager_status) or (
                    accessible != self._re_manager_connected):
                # Status changed. Initiate the updates
                self.events.status_changed(
                    status=self._re_manager_status,
                    is_connected=self._re_manager_connected,
                )

    def load_allowed_devices(self):
        try:
            result = self._client.send_message(
                method="devices_allowed",
                params={"user_group": self._user_group},
                raise_exceptions=True,
            )
            if result["success"] is False:
                raise RuntimeError(
                    f"Failed to load list of allowed devices: {result['msg']}")
            self._allowed_devices.clear()
            self._allowed_devices.update(result["devices_allowed"])
            self.events.allowed_devices_changed(
                allowed_devices=self._allowed_devices)
        except Exception as ex:
            print(f"Exception: {ex}")

    def load_allowed_plans(self):
        try:
            result = self._client.send_message(
                method="plans_allowed",
                params={"user_group": self._user_group},
                raise_exceptions=True,
            )
            if result["success"] is False:
                raise RuntimeError(
                    f"Failed to load list of allowed plans: {result['msg']}")
            self._allowed_plans.clear()
            self._allowed_plans.update(result["plans_allowed"])
            self.events.allowed_plans_changed(
                allowed_plans=self._allowed_plans)
        except Exception as ex:
            print(f"Exception: {ex}")

    def load_plan_queue(self):
        try:
            result = self._client.send_message(method="queue_get",
                                               raise_exceptions=True)
            if result["success"] is False:
                raise RuntimeError(f"Failed to load queue: {result['msg']}")
            self._plan_queue_items.clear()
            self._plan_queue_items.extend(result["items"])
            self._running_item.clear()
            self._running_item.update(result["running_item"])
            self._plan_queue_uid = result["plan_queue_uid"]

            # The dictionary that relates item uids and their positions in the queue.
            #   Used to speed up computations during queue operations.
            self._plan_queue_items_pos = {
                item["item_uid"]: n
                for n, item in enumerate(self._plan_queue_items)
                if "item_uid" in item
            }

            # Deselect queue item if it is not present in the queue
            #   Selection will be cleared when the table is reloaded, so save it in local variable
            selected_uid = self.selected_queue_item_uid
            if self.queue_item_uid_to_pos(selected_uid) < 0:
                selected_uid = ""

            # Update the representation of the queue
            self.events.plan_queue_changed(
                plan_queue_items=self._plan_queue_items,
                selected_item_uid=selected_uid,
            )
            self.events.running_item_changed(
                running_item=self._running_item,
                run_list=self._run_list,
            )

        except Exception as ex:
            print(f"Exception: {ex}")

    def load_run_list(self):
        try:
            result = self._client.send_message(method="re_runs",
                                               raise_exceptions=True)
            if result["success"] is False:
                raise RuntimeError(f"Failed to load run_list: {result['msg']}")
            self._run_list.clear()
            self._run_list.extend(result["run_list"])
            self._run_list_uid = result["run_list_uid"]

            self.events.running_item_changed(
                running_item=self._running_item,
                run_list=self._run_list,
            )

        except Exception as ex:
            print(f"Exception: {ex}")

    def load_plan_history(self):
        try:
            result = self._client.send_message(method="history_get",
                                               raise_exceptions=True)
            if result["success"] is False:
                raise RuntimeError(f"Failed to load history: {result['msg']}")
            self._plan_history_items.clear()
            self._plan_history_items.extend(result["items"])
            self._plan_history_uid = result["plan_history_uid"]

            # Deselect queue history if it does not exist in the queue
            #   Selection will be cleared when the table is reloaded, so save it in local variable
            selected_item_pos = self.selected_history_item_pos
            if selected_item_pos >= len(self._plan_history_items):
                selected_item_pos = -1

            self.events.plan_history_changed(
                plan_history_items=self._plan_history_items.copy(),
                selected_item_pos=selected_item_pos,
            )

        except Exception as ex:
            print(f"Exception: {ex}")

    # ============================================================================
    #                       Useful functions
    def get_allowed_plan_parameters(self, *, name):
        """
        Returns the dictionary of parameters for the plan with name ``name`` from
        the list of allowed plans. Returns ``None`` if plan is not found in the list

        Parameters
        ----------
        name : str
            name of the plan

        Returns
        -------
        dict or None
            dictionary of plan parameters or ``None`` if the plan is not in the list.
        """
        return self._allowed_plans.get(name, None)

    def get_allowed_instruction_parameters(self, *, name):
        """
        Returns the dictionary of parameters for the instruction with name ``name`` from
        the list of allowed instructions. Returns ``None`` if plan is not found in the list

        Parameters
        ----------
        name : str
            name of the instruction

        Returns
        -------
        dict or None
            dictionary of instruction parameters or ``None`` if the plan is not in the list.
        """
        return self._allowed_instructions.get(name, None)

    def extract_descriptions_from_item_parameters(self, *, item_parameters):
        """
        Extract descriptions from the dictionary of item parameters. The descriptions
        includes: item name/description, each parameter name/type/description.
        The descriptions are presented in humanly readable text form, which is convenient
        for user presentation.

        TODO: potentially move this function to Queue Server code (file 'profile_ops.py')

        Parameters
        ----------
        item_parameters : dict
            dictionary of item parameters (element of the list of allowed plans/instructions)

        Returns
        -------
        dict
            dictionary of descriptions
        """
        if not item_parameters:
            return {}

        descriptions = {}

        # Plan description
        item_name = item_parameters.get("name", "")
        item_description = item_parameters.get("description", None)
        item_description = item_description if item_description else ""
        item_description = str(item_description)

        descriptions = {
            "name": item_name,
            "description": item_description,
            "parameters": {},
        }

        parameters = item_parameters.get("parameters", {})
        for p in parameters:
            p_name = p.get("name", "")

            p_type, p_description, p_default = None, None, None

            custom = p.get("custom", None)
            if custom is not None:
                p_type = custom.get("annotation", None)
                if p_type is not None:
                    devices = custom.get("devices", "")
                    if devices:
                        p_type = str(p_type) + "\n" + pprint.pformat(devices)

                p_description = custom.get("description", None)

            if p_type is None:
                p_type = p.get("annotation", "")

            if p_description is None:
                p_description = p.get("description", None)
            p_description = p_description if p_description else ""
            p_description = str(p_description)

            try:
                d_tmp = p["default"]
                p_default = str(d_tmp)
                if p_default:
                    p_default = (f"'{p_default}'"
                                 if isinstance(d_tmp, str) else p_default)
            except Exception:
                p_default = ""

            descriptions["parameters"][p_name] = {}
            descriptions["parameters"][p_name]["name"] = p_name
            descriptions["parameters"][p_name]["type"] = p_type
            descriptions["parameters"][p_name]["description"] = p_description
            descriptions["parameters"][p_name]["default"] = p_default

        return descriptions

    def format_item_parameter_descriptions(self,
                                           *,
                                           item_descriptions,
                                           use_html=True):
        """
        Format plan parameter descriptions obtained from ``extract_descriptions_from_plan_parameters``.
        Returns description of the plan and each parameter represented as formatted strings
        containing plan name/description and parameter name/type/description. The text is ready
        for presentation in user interfaces.

        TODO: potentially move this function or its modification to
              Queue Server code (file 'profile_ops.py')
        """

        if not item_descriptions:
            return {}

        start_bold = "<b>" if use_html else ""
        stop_bold = "</b>" if use_html else ""
        start_it = "<i>" if use_html else ""
        stop_it = "</i>" if use_html else ""
        new_line = "<br>" if use_html else ""

        not_available = "Description is not available"

        descriptions = {}

        item_name = item_descriptions.get("name", "")
        item_description = item_descriptions.get("description", "")
        item_description = item_description.replace("\n", new_line)
        s = (
            f"{start_it}Name:{stop_it} {start_bold}{item_name}{stop_bold}{new_line}"
            f"{item_description}")

        descriptions["description"] = s if s else not_available

        descriptions["parameters"] = {}
        for _, p in item_descriptions["parameters"].items():
            p_name = p["name"] or "-"
            p_type = p["type"] or "-"
            p_default = p["default"] or "-"
            p_description = p["description"]
            p_description = p_description.replace("\n", new_line)
            s = (
                f"{start_it}Name:{stop_it} {start_bold}{p_name}{stop_bold}{new_line}"
                f"{start_it}Type:{stop_it} {start_bold}{p_type}{stop_bold}{new_line}"
                f"{start_it}Default:{stop_it} {start_bold}{p_default}{stop_bold}{new_line}"
                f"{p_description}")

            descriptions["parameters"][p_name] = s if s else not_available

        return descriptions

    def get_allowed_plan_names(self):
        return list(self._allowed_plans.keys()) if self._allowed_plans else []

    def get_allowed_instruction_names(self):
        return list(("queue_stop", ))

    # ============================================================================
    #                         Item representation

    def set_map_param_labels_to_keys(self, *, map_dict=None):
        """
        Set mapping between labels and item dictionary keys. Map is a dictionary where
        keys are label names (e.g. names of the columns of a table) and dictionaries are
        tuples of keys that show the location of the parameter in item dictionary, e.g.
        ``{"STATUS": ("result", "exit_status")}``. In most practical cases this function
        should not be called at all.

        Parameters
        ----------
        map_dict : dict or None
            Map dictionary or None to use the default dictionary

        Returns
        -------
        None
        """
        if (map_dict is not None) and not isinstance(map_dict,
                                                     collections.abc.Mapping):
            raise ValueError(
                f"Incorrect type ('{type(map_dict)}') of the parameter 'map'. 'None' or 'dict' is expected"
            )

        _default_map = {
            "": ("item_type", ),
            "Name": ("name", ),
            "Parameters": ("kwargs", ),
            "USER": ("user", ),
            "GROUP": ("user_group", ),
            "STATUS": ("result", "exit_status"),
        }
        map_dict = map_dict if (map_dict is not None) else _default_map
        self._map_column_labels_to_keys = map_dict

    def get_bound_item_arguments(self, item):
        item_args = item.get("args", [])
        item_kwargs = item.get("kwargs", {})
        item_type = item.get("item_type", None)
        item_name = item.get("name", None)

        try:
            if item_type == "plan":
                plan_parameters = self._allowed_plans.get(item_name, None)
                if plan_parameters is None:
                    raise RuntimeError(
                        f"Plan '{item_name}' is not in the list of allowed plans"
                    )
                bound_arguments = bind_plan_arguments(
                    plan_args=item_args,
                    plan_kwargs=item_kwargs,
                    plan_parameters=plan_parameters,
                )
                # If the arguments were bound successfully, then replace 'args' and 'kwargs'.
                item_args = []
                item_kwargs = bound_arguments.arguments
        except Exception:
            # print(
            #     f"Failed to bind arguments (item_type='{item_type}', "
            #     f"item_name='{item_name}'). Exception: {ex}"
            # )
            pass

        return item_args, item_kwargs

    def get_item_value_for_label(self, *, item, label, as_str=True):
        """
        Returns parameter value of the item for given label (e.g. table column name). Returns
        value represented as a string if `as_str=True`, otherwise returns value itself. Raises
        `KeyError` if the label or parameter is not found. It is not guaranteed that item
        dictionaries always contain all parameters, so exception does not indicate an error
        and should be processed by application.

        Parameters
        ----------
        item : dict
            Dictionary containing item parameters
        label : str
            Label (e.g. table column name)
        as_str : boolean
            ``True`` - return string representation of the value, otherwise return the value

        Returns
        -------
        str
            column value represented as a string

        Raises
        ------
        KeyError
            label or parameter is not found in the dictionary
        """
        try:
            key_seq = self._map_column_labels_to_keys[label]
        except KeyError:
            raise KeyError("Label 'label' is not found in the map dictionary")

        # Follow the path in the dictionary. 'KeyError' exception is raised if a key does not exist
        try:
            value = item
            if (len(key_seq) == 1) and (key_seq[-1] in ("args", "kwargs")):
                # Special case: combine args and kwargs to be displayed in one column
                value = {
                    "args": value.get("args", []),
                    "kwargs": value.get("kwargs", {}),
                }
            else:
                for key in key_seq:
                    value = value[key]
        except KeyError:
            raise KeyError(
                f"Parameter with keys {key_seq} is not found in the item dictionary"
            )

        if as_str:
            key = key_seq[-1]

            s = ""
            if key in ("args", "kwargs"):
                value["args"], value["kwargs"] = self.get_bound_item_arguments(
                    item)

                s_args, s_kwargs = "", ""
                if value["args"] and isinstance(value["args"],
                                                collections.abc.Iterable):
                    s_args = ", ".join(f"{v}" for v in value["args"])
                if value["kwargs"] and isinstance(value["kwargs"],
                                                  collections.abc.Mapping):
                    s_kwargs = ", ".join(f"{k}: {v}"
                                         for k, v in value["kwargs"].items())
                s = ", ".join([_ for _ in [s_args, s_kwargs] if _])

            elif key == "args":
                if value and isinstance(value, collections.abc.Iterable):
                    s = ", ".join(f"{v}" for v in value)

            elif key == "item_type":
                # Print capitalized first letter of the item type ('P' or 'I')
                s_tmp = str(value)
                if s_tmp:
                    s = s_tmp[0].upper()

            else:
                s = str(value)

        else:
            s = value

        return s

    # ============================================================================
    #                         Queue operations

    @property
    def selected_queue_item_uid(self):
        return self._selected_queue_item_uid

    @selected_queue_item_uid.setter
    def selected_queue_item_uid(self, item_uid):
        if self._selected_queue_item_uid != item_uid:
            self._selected_queue_item_uid = item_uid
            self.events.queue_item_selection_changed(
                selected_item_uid=item_uid)

    def queue_item_uid_to_pos(self, item_uid):
        # Returns -1 if item was not found
        return self._plan_queue_items_pos.get(item_uid, -1)

    def queue_item_pos_to_uid(self, n_item):
        try:
            item_uid = self._plan_queue_items[n_item]["item_uid"]
        except Exception:
            item_uid = ""
        return item_uid

    def queue_item_by_uid(self, item_uid):
        """
        Returns deep copy of the item based on item UID or None if the item was not found.

        Parameters
        ----------
        item_uid : str
            UID of an item. If ``item_uid=""`` then None will be returned

        Returns
        -------
        dict or None
            Dictionary of item parameters or ``None`` if the item was not found
        """
        if item_uid:
            sel_item_pos = self.queue_item_uid_to_pos(item_uid)
            if sel_item_pos >= 0:
                return copy.deepcopy(self._plan_queue_items[sel_item_pos])
        return None

    def queue_item_move_up(self):
        """
        Move plan up in the queue by one positon
        """
        item_uid = self.selected_queue_item_uid
        n_item = self.queue_item_uid_to_pos(item_uid)
        n_items = len(self._plan_queue_items)
        if item_uid and (n_items > 1) and (n_item > 0):
            n_item_above = n_item - 1
            item_uid_above = self.queue_item_pos_to_uid(n_item_above)
            response = self._client.send_message(
                method="queue_item_move",
                params={
                    "uid": item_uid,
                    "before_uid": item_uid_above
                },
            )
            self.load_re_manager_status(unbuffered=True)
            if not response["success"]:
                raise RuntimeError(
                    f"Failed to move the item: {response['msg']}")

    def queue_item_move_down(self):
        """
        Move plan down in the queue by one positon
        """
        item_uid = self.selected_queue_item_uid
        n_item = self.queue_item_uid_to_pos(item_uid)
        n_items = len(self._plan_queue_items)
        if item_uid and (n_items > 1) and (0 <= n_item < n_items - 1):
            n_item_below = n_item + 1
            item_uid_below = self.queue_item_pos_to_uid(n_item_below)
            response = self._client.send_message(
                method="queue_item_move",
                params={
                    "uid": item_uid,
                    "after_uid": item_uid_below
                },
            )
            self.load_re_manager_status(unbuffered=True)
            if not response["success"]:
                raise RuntimeError(
                    f"Failed to move the item: {response['msg']}")

    def queue_item_move_in_place_of(self, item_uid_to_replace):
        """
        Replace plan with given UID with the selected plan
        """
        item_uid = self.selected_queue_item_uid
        n_item = self.queue_item_uid_to_pos(item_uid)
        n_item_to_replace = self.queue_item_uid_to_pos(item_uid_to_replace)
        if (item_uid and item_uid_to_replace and (n_item_to_replace >= 0)
                and (item_uid != item_uid_to_replace)):
            location = "before_uid" if (
                n_item_to_replace < n_item) else "after_uid"
            response = self._client.send_message(
                method="queue_item_move",
                params={
                    "uid": item_uid,
                    location: item_uid_to_replace
                },
            )
            self.load_re_manager_status(unbuffered=True)
            if not response["success"]:
                raise RuntimeError(
                    f"Failed to move the item: {response['msg']}")

    def queue_item_move_to_top(self):
        """
        Move plan to top of the queue
        """
        item_uid = self.selected_queue_item_uid
        if item_uid:
            response = self._client.send_message(method="queue_item_move",
                                                 params={
                                                     "uid": item_uid,
                                                     "pos_dest": "front"
                                                 })
            self.load_re_manager_status(unbuffered=True)
            if not response["success"]:
                raise RuntimeError(
                    f"Failed to move the item: {response['msg']}")

    def queue_item_move_to_bottom(self):
        """
        Move plan to top of the queue
        """
        item_uid = self.selected_queue_item_uid
        if item_uid:
            response = self._client.send_message(method="queue_item_move",
                                                 params={
                                                     "uid": item_uid,
                                                     "pos_dest": "back"
                                                 })
            self.load_re_manager_status(unbuffered=True)
            if not response["success"]:
                raise RuntimeError(
                    f"Failed to move the item: {response['msg']}")

    def queue_item_remove(self):
        """
        Delete item from queue
        """
        item_uid = self.selected_queue_item_uid
        if item_uid:
            # Find and set UID of an item that will be selected once the current item is removed
            n_item = self.queue_item_uid_to_pos(item_uid)
            n_items = len(self._plan_queue_items)
            if n_items <= 1:
                n_sel_item_new = -1
            elif n_item < n_items - 1:
                n_sel_item_new = n_item + 1
            else:
                n_sel_item_new = n_item - 1
            self.selected_queue_item_uid = self.queue_item_pos_to_uid(
                n_sel_item_new)

            response = self._client.send_message(method="queue_item_remove",
                                                 params={"uid": item_uid})
            self.load_re_manager_status(unbuffered=True)
            if not response["success"]:
                raise RuntimeError(f"Failed to delete item: {response['msg']}")

    def queue_clear(self):
        """
        Clear the plan queue
        """
        response = self._client.send_message(method="queue_clear", )
        self.load_re_manager_status(unbuffered=True)
        if not response["success"]:
            raise RuntimeError(f"Failed to clear the queue: {response['msg']}")

    def queue_mode_loop_enable(self, enable):
        """
        Enable or disable LOOP mode of the queue
        """
        response = self._client.send_message(method="queue_mode_set",
                                             params={"mode": {
                                                 "loop": enable
                                             }})
        self.load_re_manager_status(unbuffered=True)
        if not response["success"]:
            raise RuntimeError(
                f"Failed to change plan queue mode: {response['msg']}")

    def queue_item_copy_to_queue(self):
        """
        Copy currently selected item to queue. Item is supposed to be selected in the plan queue.
        """
        sel_item_uid = self._selected_queue_item_uid
        sel_item_pos = self.queue_item_uid_to_pos(sel_item_uid)
        if sel_item_uid and (sel_item_pos >= 0):
            item = self._plan_queue_items[sel_item_pos]
            self.queue_item_add(item=item)

    def queue_item_add(self, *, item, params=None):
        """
        Add item to queue. This function should be called by all widgets that add items to queue.
        The new item is inserted after the selected item or to the back of the queue in case
        no item is selected. Optional dictionary `params` may be used to override the default
        behavior. E.g. ``params={"pos": "front"}`` will add the item to the font of the queue.
        See the documentation for ``queue_item_add`` 0MQ API of Queue Server.
        The new item becomes the selected item.
        """
        sel_item_uid = self._selected_queue_item_uid
        queue_is_empty = not len(self._plan_queue_items)
        if not params:
            if queue_is_empty or not sel_item_uid:
                # Push button to the back of the queue
                params = {}
            else:
                params = {"after_uid": sel_item_uid}

        # We are submitting a plan as a new plan, so all unnecessary data will be stripped
        #   and new item UID will be assigned.
        request_params = {
            "item": item,
            "user": self._user_name,
            "user_group": self._user_group,
        }
        request_params.update(params)
        response = self._client.send_message(method="queue_item_add",
                                             params=request_params)
        self.load_re_manager_status(unbuffered=True)
        if not response["success"]:
            raise RuntimeError(
                f"Failed to add item to the queue: {response['msg']}")
        else:
            try:
                # The 'item' and 'item_uid' should always be included in the returned item in case of success.
                sel_item_uid = response["item"]["item_uid"]
            except KeyError as ex:
                print(
                    f"Item or item UID is not found in the server response {pprint.pformat(response)}. "
                    f"Can not update item selection in the queue table. Exception: {ex}"
                )
            self.selected_queue_item_uid = sel_item_uid

    def queue_item_update(self, *, item):
        """
        Update the existing plan in the queue. This function should be called by all widgets
        that are used to modify (edit) the existing queue items. The items are distinguished by
        item UID, so item UID in the submitted ``item`` must match UID of the existing queue item
        that is replaced. The modified item becomes a selected item.
        """
        # We are submitting a plan as a new plan, so all unnecessary data will be stripped
        #   and new item UID will be assigned.
        request_params = {
            "item": item,
            "user": self._user_name,
            "user_group": self._user_group,
            "replace": True,  # Generates new UID
        }
        response = self._client.send_message(method="queue_item_update",
                                             params=request_params)
        self.load_re_manager_status(unbuffered=True)
        if not response["success"]:
            raise RuntimeError(
                f"Failed to add item to the queue: {response['msg']}")
        else:
            try:
                # The 'item' and 'item_uid' should always be included in the returned item in case of success.
                sel_item_uid = response["item"]["item_uid"]
            except KeyError as ex:
                print(
                    f"Item or item UID is not found in the server response {pprint.pformat(response)}. "
                    f"Can not update item selection in the queue table. Exception: {ex}"
                )
            self.selected_queue_item_uid = sel_item_uid

    # ============================================================================
    #                         History operations

    @property
    def selected_history_item_pos(self):
        return self._selected_history_item_pos

    @selected_history_item_pos.setter
    def selected_history_item_pos(self, item_pos):
        if self._selected_history_item_pos != item_pos:
            self._selected_history_item_pos = item_pos
            self.events.history_item_selection_changed(
                selected_item_pos=item_pos)

    def history_item_add_to_queue(self):
        """Copy the selected plan from history to the end of the queue"""
        selected_item_pos = self.selected_history_item_pos
        if selected_item_pos >= 0:
            history_item = self._plan_history_items[selected_item_pos]
            self.queue_item_add(item=history_item)

    def history_clear(self):
        """
        Clear history
        """
        response = self._client.send_message(method="history_clear", )
        self.load_re_manager_status(unbuffered=True)
        if not response["success"]:
            raise RuntimeError(
                f"Failed to clear the history: {response['msg']}")

    # ============================================================================
    #                     Operations with running item

    def running_item_add_to_queue(self):
        """Copy the selected plan from history to the end of the queue"""
        if self._running_item:
            running_item = self._running_item.copy()
            self.queue_item_add(item=running_item)

    # ============================================================================
    #                  Operations with RE Environment

    def environment_open(self, timeout=0):
        """
        Open RE Worker environment. Blocks until operation is complete or timeout expires.
        If ``timeout=0``, then the function blocks until operation is complete.

        Parameters
        ----------
        timeout : float
            maximum time for the operation. Exception is raised if timeout expires.
            If ``timeout=0``, the function blocks until operation is complete.

        Returns
        -------
        None
        """
        # Check if RE Worker environment already exists and RE manager is idle.
        self.load_re_manager_status()
        status = self._re_manager_status
        if status["manager_state"] != "idle":
            raise RuntimeError(
                f"RE Manager state must be 'idle': current state: {status['manager_state']}"
            )
        if status["worker_environment_exists"]:
            raise RuntimeError("RE Worker environment already exists")

        # Initiate opening of RE Worker environment
        response = self._client.send_message(method="environment_open")
        if not response["success"]:
            raise RuntimeError(
                f"Failed to open RE Worker environment: {response['msg']}")

        # Wait for the environment to be created.
        if timeout:
            t_stop = time.time() + timeout
        while True:
            self.load_re_manager_status()
            status2 = self._re_manager_status
            if (status2["worker_environment_exists"]
                    and status2["manager_state"] == "idle"):
                break
            if timeout and (time.time() > t_stop):
                raise RuntimeError(
                    "Failed to start RE Worker: timeout occurred")
            time.sleep(0.5)

    def environment_close(self, timeout=0):
        """
        Close RE Worker environment. Blocks until operation is complete or timeout expires.
        If ``timeout=0``, then the function blocks until operation is complete.

        Parameters
        ----------
        timeout : float
            maximum time for the operation. Exception is raised if timeout expires.
            If ``timeout=0``, the function blocks until operation is complete.

        Returns
        -------
        None
        """
        # Check if RE Worker environment already exists and RE manager is idle.
        self.load_re_manager_status()
        status = self._re_manager_status
        if status["manager_state"] != "idle":
            raise RuntimeError(
                f"RE Manager state must be 'idle': current state: {status['manager_state']}"
            )
        if not status["worker_environment_exists"]:
            raise RuntimeError("RE Worker environment does not exist")

        # Initiate opening of RE Worker environment
        response = self._client.send_message(method="environment_close")
        if not response["success"]:
            raise RuntimeError(
                f"Failed to close RE Worker environment: {response['msg']}")

        # Wait for the environment to be created.
        if timeout:
            t_stop = time.time() + timeout
        while True:
            self.load_re_manager_status()
            status2 = self._re_manager_status
            if (not status2["worker_environment_exists"]
                    and status2["manager_state"] == "idle"):
                break
            if timeout and (time.time() > t_stop):
                raise RuntimeError(
                    "Failed to start RE Worker: timeout occurred")
            time.sleep(0.5)

    def environment_destroy(self, timeout=0):
        """
        Destroy (unresponsive) RE Worker environment. The function is intended for the cases when
        the environment is unresponsive and can not be stopped using ``environment_close``.
        Blocks until operation is complete or timeout expires. If ``timeout=0``, then the function
        blocks until operation is complete.

        Parameters
        ----------
        timeout : float
            maximum time for the operation. Exception is raised if timeout expires.
            If ``timeout=0``, the function blocks until operation is complete.

        Returns
        -------
        None
        """
        # Check if RE Worker environment already exists and RE manager is idle.
        self.load_re_manager_status()
        status = self._re_manager_status
        if not status["worker_environment_exists"]:
            raise RuntimeError("RE Worker environment does not exist")

        # Initiate opening of RE Worker environment
        response = self._client.send_message(method="environment_destroy")
        if not response["success"]:
            raise RuntimeError(
                f"Failed to destroy RE Worker environment: {response['msg']}")

        # Wait for the environment to be created.
        if timeout:
            t_stop = time.time() + timeout
        while True:
            self.load_re_manager_status()
            status2 = self._re_manager_status
            if (not status2["worker_environment_exists"]
                    and status2["manager_state"] == "idle"):
                break
            if timeout and (time.time() > t_stop):
                raise RuntimeError(
                    "Failed to start RE Worker: timeout occurred")
            time.sleep(0.5)

    # ============================================================================
    #                        Queue Control

    def queue_start(self):
        response = self._client.send_message(method="queue_start")
        if not response["success"]:
            raise RuntimeError(f"Failed to start the queue: {response['msg']}")

    def queue_stop(self):
        response = self._client.send_message(method="queue_stop")
        if not response["success"]:
            raise RuntimeError(
                f"Failed to request stopping the queue: {response['msg']}")

    def queue_stop_cancel(self):
        response = self._client.send_message(method="queue_stop_cancel")
        if not response["success"]:
            raise RuntimeError(
                f"Failed to cancel request to stop the queue: {response['msg']}"
            )

    # ============================================================================
    #                        RE Control

    def _wait_for_completion(self,
                             *,
                             condition,
                             msg="complete operation",
                             timeout=0):
        if timeout:
            t_stop = time.time() + timeout

        while True:
            self.load_re_manager_status()
            status = self._re_manager_status
            if condition(status):
                break
            if timeout and (time.time() > t_stop):
                raise RuntimeError(f"Failed to {msg}: timeout occurred")
            time.sleep(0.5)

    def re_pause(self, timeout=0, *, option):
        """
        Pause execution of a plan.

        Parameters
        ----------
        timeout : float
            maximum time for the operation. Exception is raised if timeout expires.
            If ``timeout=0``, the function blocks until operation is complete.

        option : str
            "immediate" or "deferred"
        Returns
        -------
        None
        """

        # Initiate opening of RE Worker environment
        response = self._client.send_message(method="re_pause",
                                             params={"option": option})
        if not response["success"]:
            raise RuntimeError(
                f"Failed to pause the running plan: {response['msg']}")

        def condition(status):
            return status["manager_state"] in ("idle", "paused")

        self._wait_for_completion(condition=condition,
                                  msg="pause the running plan",
                                  timeout=timeout)

    def re_resume(self, timeout=0):
        """
        Pause execution of a plan.

        Parameters
        ----------
        timeout : float
            maximum time for the operation. Exception is raised if timeout expires.
            If ``timeout=0``, the function blocks until operation is complete.

        Returns
        -------
        None
        """

        # Initiate opening of RE Worker environment
        response = self._client.send_message(method="re_resume")
        if not response["success"]:
            raise RuntimeError(
                f"Failed to resume the running plan: {response['msg']}")

        def condition(status):
            return status["manager_state"] in ("idle", "executing_queue")

        self._wait_for_completion(condition=condition,
                                  msg="resume execution of the plan",
                                  timeout=timeout)

    def _re_continue_plan(self, *, action, timeout=0):

        if action not in ("stop", "abort", "halt"):
            raise RuntimeError(f"Unrecognized action '{action}'")

        method = f"re_{action}"

        response = self._client.send_message(method=method)
        if not response["success"]:
            raise RuntimeError(
                f"Failed to {action} the running plan: {response['msg']}")

        def condition(status):
            return status["manager_state"] == "idle"

        self._wait_for_completion(condition=condition,
                                  msg=f"{action} the plan",
                                  timeout=timeout)

    def re_stop(self, timeout=0):
        self._re_continue_plan(action="stop", timeout=timeout)

    def re_abort(self, timeout=0):
        self._re_continue_plan(action="abort", timeout=timeout)

    def re_halt(self, timeout=0):
        self._re_continue_plan(action="halt", timeout=timeout)

    def add(self, plan_name, plan_args):
        # Add plan to queue
        response = self._client.send_message(
            method="queue_item_add",
            params={
                "plan": {
                    "name": plan_name,
                    "args": plan_args
                },
                "user": "",
                "user_group": "admin",
            },
        )
        if not response["success"]:
            raise RuntimeError(
                f"Failed to add plan to the queue: {response['msg']}")