def run(self):
"""
Overrides the `run()` function of the `multiprocessing.Process` class. Called
by the `start` method.
"""
success = True
logging.basicConfig(level=logging.WARNING)
logging.getLogger(__name__).setLevel(self._log_level)
from .plan_monitoring import RunList, CallbackRegisterRun
self._active_run_list = RunList(
) # Initialization should be done before communication is enabled.
self._comm_to_manager.add_method(self._request_state_handler,
"request_state")
self._comm_to_manager.add_method(self._request_plan_report_handler,
"request_plan_report")
self._comm_to_manager.add_method(self._request_run_list_handler,
"request_run_list")
self._comm_to_manager.add_method(self._command_close_env_handler,
"command_close_env")
self._comm_to_manager.add_method(self._command_confirm_exit_handler,
"command_confirm_exit")
self._comm_to_manager.add_method(self._command_run_plan_handler,
"command_run_plan")
self._comm_to_manager.add_method(self._command_pause_plan_handler,
"command_pause_plan")
self._comm_to_manager.add_method(self._command_continue_plan_handler,
"command_continue_plan")
self._comm_to_manager.add_method(self._command_reset_worker_handler,
"command_reset_worker")
self._comm_to_manager.start()
self._exit_event = threading.Event()
self._exit_confirmed_event = threading.Event()
self._re_report_lock = threading.Lock()
from bluesky import RunEngine
from bluesky.run_engine import get_bluesky_event_loop
from bluesky.callbacks.best_effort import BestEffortCallback
from bluesky_kafka import Publisher as kafkaPublisher
from bluesky.utils import PersistentDict
from .profile_tools import global_user_namespace
# TODO: TC - Do you think that the following code may be included in RE.__init__()
# (for Python 3.8 and above)
# Setting the default event loop is needed to make the code work with Python 3.8.
loop = get_bluesky_event_loop()
asyncio.set_event_loop(loop)
try:
keep_re = self._config["keep_re"]
startup_dir = self._config.get("startup_dir", None)
startup_module_name = self._config.get("startup_module_name", None)
startup_script_path = self._config.get("startup_script_path", None)
self._re_namespace = load_worker_startup_code(
startup_dir=startup_dir,
startup_module_name=startup_module_name,
startup_script_path=startup_script_path,
keep_re=keep_re,
)
if keep_re and ("RE" not in self._re_namespace):
raise RuntimeError(
"Run Engine is not created in the startup code and 'keep_re' option is activated."
)
self._existing_plans = plans_from_nspace(self._re_namespace)
self._existing_devices = devices_from_nspace(self._re_namespace)
logger.info("Startup code was loaded completed.")
except Exception as ex:
logger.exception(
"Failed to start RE Worker environment. Error while loading startup code: %s.",
str(ex),
)
success = False
# Load lists of allowed plans and devices
logger.info("Loading the lists of allowed plans and devices ...")
path_pd = self._config["existing_plans_and_devices_path"]
path_ug = self._config["user_group_permissions_path"]
try:
self._allowed_plans, self._allowed_devices = load_allowed_plans_and_devices(
path_existing_plans_and_devices=path_pd,
path_user_group_permissions=path_ug)
except Exception as ex:
logger.exception(
"Error occurred while loading lists of allowed plans and devices from '%s': %s",
path_pd, str(ex))
if success:
logger.info("Instantiating and configuring Run Engine ...")
try:
# Make RE namespace available to the plan code.
global_user_namespace.set_user_namespace(
user_ns=self._re_namespace, use_ipython=False)
if self._config["keep_re"]:
# Copy references from the namespace
self._RE = self._re_namespace["RE"]
self._db = self._re_namespace.get("RE", None)
else:
# Instantiate a new Run Engine and Data Broker (if needed)
md = {}
if self._config["use_persistent_metadata"]:
# This code is temporarily copied from 'nslsii' before better solution for keeping
# continuous sequence Run ID is found. TODO: continuous sequence of Run IDs.
directory = os.path.expanduser("~/.config/bluesky/md")
os.makedirs(directory, exist_ok=True)
md = PersistentDict(directory)
self._RE = RunEngine(md)
self._re_namespace["RE"] = self._RE
def factory(name, doc):
# Documents from each run are routed to an independent
# instance of BestEffortCallback
bec = BestEffortCallback()
return [bec], []
# Subscribe to Best Effort Callback in the way that works with multi-run plans.
rr = RunRouter([factory])
self._RE.subscribe(rr)
# Subscribe RE to databroker if config file name is provided
self._db = None
if "databroker" in self._config:
config_name = self._config["databroker"].get(
"config", None)
if config_name:
logger.info(
"Subscribing RE to Data Broker using configuration '%s'.",
config_name)
from databroker import Broker
self._db = Broker.named(config_name)
self._re_namespace["db"] = self._db
self._RE.subscribe(self._db.insert)
# Subscribe Run Engine to 'CallbackRegisterRun'. This callback is used internally
# by the worker process to keep track of the runs that are open and closed.
run_reg_cb = CallbackRegisterRun(
run_list=self._active_run_list)
self._RE.subscribe(run_reg_cb)
if "kafka" in self._config:
logger.info(
"Subscribing to Kafka: topic '%s', servers '%s'",
self._config["kafka"]["topic"],
self._config["kafka"]["bootstrap"],
)
kafka_publisher = kafkaPublisher(
topic=self._config["kafka"]["topic"],
bootstrap_servers=self._config["kafka"]["bootstrap"],
key="kafka-unit-test-key",
# work with a single broker
producer_config={
"acks": 1,
"enable.idempotence": False,
"request.timeout.ms": 5000
},
serializer=partial(msgpack.dumps, default=mpn.encode),
)
self._RE.subscribe(kafka_publisher)
if "zmq_data_proxy_addr" in self._config:
from bluesky.callbacks.zmq import Publisher
publisher = Publisher(self._config["zmq_data_proxy_addr"])
self._RE.subscribe(publisher)
self._execution_queue = queue.Queue()
self._state["environment_state"] = "ready"
except BaseException as ex:
success = False
logger.exception(
"Error occurred while initializing the environment: %s.",
str(ex))
if success:
logger.info("RE Environment is ready.")
self._execute_in_main_thread()
else:
self._exit_event.set()
logger.info("Environment is waiting to be closed ...")
self._state["environment_state"] = "closing"
# Wait until confirmation is received from RE Manager
while not self._exit_confirmed_event.is_set():
ttime.sleep(0.02)
self._RE = None
self._comm_to_manager.stop()
logger.info("Run Engine environment was closed successfully.")
ip = IPython.get_ipython()
hclient = happi.Client(path='/usr/local/share/happi/test_db.json')
db = databroker.catalog['MAD']
RE = RunEngine()
bec = BestEffortCallback()
zmq_publisher = zmqPublisher("127.0.0.1:4567")
kafka_publisher = kafkaPublisher(
topic="mad.bluesky.documents",
bootstrap_servers="127.0.0.1:9092",
key="kafka-unit-test-key",
# work with a single broker
producer_config={
"acks": 1,
"enable.idempotence": False,
"request.timeout.ms": 5000,
},
serializer=partial(msgpack.dumps, default=mpn.encode),
)
logger = logging.getLogger("databroker")
logger.setLevel("DEBUG")
handler = logging.StreamHandler()
handler.setLevel("DEBUG")
logger.addHandler(handler)
RE.subscribe(zmq_publisher)
RE.subscribe(kafka_publisher)
RE.subscribe(bec)