def migrate_metadata():
"""
Copy metadata from (old) sqlite-backed file to (new) directory of msgpack.
"""
from bluesky.utils import get_history, PersistentDict
old_md = get_history()
directory = os.path.expanduser('~/.config/bluesky/md')
os.makedirs(directory, exist_ok=True)
new_md = PersistentDict(directory)
new_md.update(old_md)
old_md = None
md_path = get_md_path()
if not os.path.exists(md_path):
logger.info(
"New directory to store RE.md between sessions: %s",
md_path)
os.makedirs(md_path)
from bluesky.utils import get_history
old_md = get_history()
from bluesky import RunEngine
RE = RunEngine()
from bluesky.utils import PersistentDict
RE.md = PersistentDict(md_path)
if old_md is not None:
logger.info('migrating RE.md storage to PersistentDict')
RE.md.update(old_md)
# keep track of callback subscriptions
callback_db = {}
# set up databroker
import databroker
db = databroker.Broker.named('mongoCat')
callback_db['Broker'] = RE.subscribe(db.insert)
# Set up SupplementalData.
from bluesky import SupplementalData
sd = SupplementalData()
super().__init__(self._dump, self._load, self._file)
@property
def directory(self):
return self._directory
def __repr__(self):
return f"<{self.__class__.__name__} {dict(self)!r}>"
@staticmethod
def _dump(obj):
"Encode as msgpack using numpy-aware encoder."
# See https://github.com/msgpack/msgpack-python#string-and-binary-type
# for more on use_bin_type.
return msgpack.packb(obj,
default=msgpack_numpy.encode,
use_bin_type=True)
@staticmethod
def _load(file):
return msgpack.unpackb(file,
object_hook=msgpack_numpy.decode,
raw=False)
# runengine_metadata_dir = appdirs.user_data_dir(appname="bluesky") / Path("runengine-metadata")
runengine_metadata_dir = Path(
'/nsls2/xf05id1/shared/config/runengine-metadata')
RE.md = PersistentDict(runengine_metadata_dir)
# Following scripts to be run as ipython startup scripts.
# Add contents to ~/.ipython/yourprofile/startup
"""connect with Bluesky"""
from bluesky import RunEngine
RE = RunEngine()
from bluesky.utils import PersistentDict
RE.md = PersistentDict('./metadata.md')
# from ssrltools.utils import setup_user_metadata
# user_md = setup_user_metadata()
# RE.md.update(user_md)
# Import matplotlib and put it in interactive mode.
import matplotlib.pyplot as plt
plt.ion()
# Optional: set any metadata that rarely changes. in 60-metadata.py
# convenience imports
from bluesky.callbacks import *
import bluesky.plans as bp
import bluesky.plan_stubs as bps
import bluesky.preprocessors as bpp
from time import sleep
import numpy as np
import bluesky.magics
# diagnostics
def run(self):
"""
Overrides the `run()` function of the `multiprocessing.Process` class. Called
by the `start` method.
"""
success = True
logging.basicConfig(level=logging.WARNING)
logging.getLogger(__name__).setLevel(self._log_level)
from .plan_monitoring import RunList, CallbackRegisterRun
self._active_run_list = RunList(
) # Initialization should be done before communication is enabled.
self._comm_to_manager.add_method(self._request_state_handler,
"request_state")
self._comm_to_manager.add_method(self._request_plan_report_handler,
"request_plan_report")
self._comm_to_manager.add_method(self._request_run_list_handler,
"request_run_list")
self._comm_to_manager.add_method(self._command_close_env_handler,
"command_close_env")
self._comm_to_manager.add_method(self._command_confirm_exit_handler,
"command_confirm_exit")
self._comm_to_manager.add_method(self._command_run_plan_handler,
"command_run_plan")
self._comm_to_manager.add_method(self._command_pause_plan_handler,
"command_pause_plan")
self._comm_to_manager.add_method(self._command_continue_plan_handler,
"command_continue_plan")
self._comm_to_manager.add_method(self._command_reset_worker_handler,
"command_reset_worker")
self._comm_to_manager.start()
self._exit_event = threading.Event()
self._exit_confirmed_event = threading.Event()
self._re_report_lock = threading.Lock()
from bluesky import RunEngine
from bluesky.run_engine import get_bluesky_event_loop
from bluesky.callbacks.best_effort import BestEffortCallback
from bluesky_kafka import Publisher as kafkaPublisher
from bluesky.utils import PersistentDict
from .profile_tools import global_user_namespace
# TODO: TC - Do you think that the following code may be included in RE.__init__()
# (for Python 3.8 and above)
# Setting the default event loop is needed to make the code work with Python 3.8.
loop = get_bluesky_event_loop()
asyncio.set_event_loop(loop)
try:
keep_re = self._config["keep_re"]
startup_dir = self._config.get("startup_dir", None)
startup_module_name = self._config.get("startup_module_name", None)
startup_script_path = self._config.get("startup_script_path", None)
self._re_namespace = load_worker_startup_code(
startup_dir=startup_dir,
startup_module_name=startup_module_name,
startup_script_path=startup_script_path,
keep_re=keep_re,
)
if keep_re and ("RE" not in self._re_namespace):
raise RuntimeError(
"Run Engine is not created in the startup code and 'keep_re' option is activated."
)
self._existing_plans = plans_from_nspace(self._re_namespace)
self._existing_devices = devices_from_nspace(self._re_namespace)
logger.info("Startup code was loaded completed.")
except Exception as ex:
logger.exception(
"Failed to start RE Worker environment. Error while loading startup code: %s.",
str(ex),
)
success = False
# Load lists of allowed plans and devices
logger.info("Loading the lists of allowed plans and devices ...")
path_pd = self._config["existing_plans_and_devices_path"]
path_ug = self._config["user_group_permissions_path"]
try:
self._allowed_plans, self._allowed_devices = load_allowed_plans_and_devices(
path_existing_plans_and_devices=path_pd,
path_user_group_permissions=path_ug)
except Exception as ex:
logger.exception(
"Error occurred while loading lists of allowed plans and devices from '%s': %s",
path_pd, str(ex))
if success:
logger.info("Instantiating and configuring Run Engine ...")
try:
# Make RE namespace available to the plan code.
global_user_namespace.set_user_namespace(
user_ns=self._re_namespace, use_ipython=False)
if self._config["keep_re"]:
# Copy references from the namespace
self._RE = self._re_namespace["RE"]
self._db = self._re_namespace.get("RE", None)
else:
# Instantiate a new Run Engine and Data Broker (if needed)
md = {}
if self._config["use_persistent_metadata"]:
# This code is temporarily copied from 'nslsii' before better solution for keeping
# continuous sequence Run ID is found. TODO: continuous sequence of Run IDs.
directory = os.path.expanduser("~/.config/bluesky/md")
os.makedirs(directory, exist_ok=True)
md = PersistentDict(directory)
self._RE = RunEngine(md)
self._re_namespace["RE"] = self._RE
def factory(name, doc):
# Documents from each run are routed to an independent
# instance of BestEffortCallback
bec = BestEffortCallback()
return [bec], []
# Subscribe to Best Effort Callback in the way that works with multi-run plans.
rr = RunRouter([factory])
self._RE.subscribe(rr)
# Subscribe RE to databroker if config file name is provided
self._db = None
if "databroker" in self._config:
config_name = self._config["databroker"].get(
"config", None)
if config_name:
logger.info(
"Subscribing RE to Data Broker using configuration '%s'.",
config_name)
from databroker import Broker
self._db = Broker.named(config_name)
self._re_namespace["db"] = self._db
self._RE.subscribe(self._db.insert)
# Subscribe Run Engine to 'CallbackRegisterRun'. This callback is used internally
# by the worker process to keep track of the runs that are open and closed.
run_reg_cb = CallbackRegisterRun(
run_list=self._active_run_list)
self._RE.subscribe(run_reg_cb)
if "kafka" in self._config:
logger.info(
"Subscribing to Kafka: topic '%s', servers '%s'",
self._config["kafka"]["topic"],
self._config["kafka"]["bootstrap"],
)
kafka_publisher = kafkaPublisher(
topic=self._config["kafka"]["topic"],
bootstrap_servers=self._config["kafka"]["bootstrap"],
key="kafka-unit-test-key",
# work with a single broker
producer_config={
"acks": 1,
"enable.idempotence": False,
"request.timeout.ms": 5000
},
serializer=partial(msgpack.dumps, default=mpn.encode),
)
self._RE.subscribe(kafka_publisher)
if "zmq_data_proxy_addr" in self._config:
from bluesky.callbacks.zmq import Publisher
publisher = Publisher(self._config["zmq_data_proxy_addr"])
self._RE.subscribe(publisher)
self._execution_queue = queue.Queue()
self._state["environment_state"] = "ready"
except BaseException as ex:
success = False
logger.exception(
"Error occurred while initializing the environment: %s.",
str(ex))
if success:
logger.info("RE Environment is ready.")
self._execute_in_main_thread()
else:
self._exit_event.set()
logger.info("Environment is waiting to be closed ...")
self._state["environment_state"] = "closing"
# Wait until confirmation is received from RE Manager
while not self._exit_confirmed_event.is_set():
ttime.sleep(0.02)
self._RE = None
self._comm_to_manager.stop()
logger.info("Run Engine environment was closed successfully.")
def configure_base(
user_ns,
broker_name,
*,
bec=True,
epics_context=False,
magics=True,
mpl=True,
configure_logging=True,
pbar=True,
ipython_exc_logging=True,
):
"""
Perform base setup and instantiation of important objects.
This factory function instantiates essential objects to data collection
environments at NSLS-II and adds them to the current namespace. In some
cases (documented below), it will check whether certain variables already
exist in the user name space, and will avoid creating them if so. The
following are added:
* ``RE`` -- a RunEngine
This is created only if an ``RE`` instance does not currently exist in
the namespace.
* ``db`` -- a Broker (from "databroker"), subscribe to ``RE``
* ``bec`` -- a BestEffortCallback, subscribed to ``RE``
* ``peaks`` -- an alias for ``bec.peaks``
* ``sd`` -- a SupplementalData preprocessor, added to ``RE.preprocessors``
* ``pbar_maanger`` -- a ProgressBarManager, set as the ``RE.waiting_hook``
And it performs some low-level configuration:
* creates a context in ophyd's control layer (``ophyd.setup_ophyd()``)
* turns on interactive plotting (``matplotlib.pyplot.ion()``)
* bridges the RunEngine and Qt event loops
(``bluesky.utils.install_kicker()``)
* logs ERROR-level log message from ophyd to the standard out
Parameters
----------
user_ns: dict
a namespace --- for example, ``get_ipython().user_ns``
broker_name : Union[str, Broker]
Name of databroker configuration or a Broker instance.
bec : boolean, optional
True by default. Set False to skip BestEffortCallback.
epics_context : boolean, optional
True by default. Set False to skip ``setup_ophyd()``.
magics : boolean, optional
True by default. Set False to skip registration of custom IPython
magics.
mpl : boolean, optional
True by default. Set False to skip matplotlib ``ion()`` at event-loop
bridging.
configure_logging : boolean, optional
True by default. Set False to skip INFO-level logging to
/var/logs/bluesky/bluesky.log.
pbar : boolean, optional
True by default. Set false to skip ProgressBarManager.
ipython_exc_logging : boolean, optional
True by default. Exception stack traces will be written to IPython
log file when IPython logging is enabled.
Returns
-------
names : list
list of names added to the namespace
Examples
--------
Configure IPython for CHX.
>>>> configure_base(get_ipython().user_ns, 'chx');
"""
ns = {} # We will update user_ns with this at the end.
# Protect against double-subscription.
SENTINEL = "__nslsii_configure_base_has_been_run"
if user_ns.get(SENTINEL):
raise RuntimeError(
"configure_base should only be called once per process.")
ns[SENTINEL] = True
# Set up a RunEngine and use metadata backed by files on disk.
from bluesky import RunEngine, __version__ as bluesky_version
if LooseVersion(bluesky_version) >= LooseVersion("1.6.0"):
# current approach using PersistentDict
from bluesky.utils import PersistentDict
directory = os.path.expanduser("~/.config/bluesky/md")
os.makedirs(directory, exist_ok=True)
md = PersistentDict(directory)
else:
# legacy approach using HistoryDict
from bluesky.utils import get_history
md = get_history()
# if RunEngine already defined grab it
# useful when users make their own custom RunEngine
if "RE" in user_ns:
RE = user_ns["RE"]
else:
RE = RunEngine(md)
ns["RE"] = RE
# Set up SupplementalData.
# (This is a no-op until devices are added to it,
# so there is no need to provide a 'skip_sd' switch.)
from bluesky import SupplementalData
sd = SupplementalData()
RE.preprocessors.append(sd)
ns["sd"] = sd
if isinstance(broker_name, str):
# Set up a Broker.
from databroker import Broker
db = Broker.named(broker_name)
ns["db"] = db
else:
db = broker_name
RE.subscribe(db.insert)
if pbar:
# Add a progress bar.
from bluesky.utils import ProgressBarManager
pbar_manager = ProgressBarManager()
RE.waiting_hook = pbar_manager
ns["pbar_manager"] = pbar_manager
if magics:
# Register bluesky IPython magics.
from bluesky.magics import BlueskyMagics
get_ipython().register_magics(BlueskyMagics)
if bec:
# Set up the BestEffortCallback.
from bluesky.callbacks.best_effort import BestEffortCallback
_bec = BestEffortCallback()
RE.subscribe(_bec)
ns["bec"] = _bec
ns["peaks"] = _bec.peaks # just as alias for less typing
if mpl:
# Import matplotlib and put it in interactive mode.
import matplotlib.pyplot as plt
ns["plt"] = plt
plt.ion()
# Make plots update live while scans run.
if LooseVersion(bluesky_version) < LooseVersion("1.6.0"):
from bluesky.utils import install_kicker
install_kicker()
if epics_context:
# Create a context in the underlying EPICS client.
from ophyd import setup_ophyd
setup_ophyd()
if configure_logging:
configure_bluesky_logging(ipython=get_ipython())
if ipython_exc_logging:
# IPython logging must be enabled separately
from nslsii.common.ipynb.logutils import log_exception
configure_ipython_exc_logging(exception_logger=log_exception,
ipython=get_ipython())
# always configure %xmode minimal
get_ipython().magic("xmode minimal")
# convenience imports
# some of the * imports are for 'back-compatibility' of a sort -- we have
# taught BL staff to expect LiveTable and LivePlot etc. to be in their
# namespace
import numpy as np
ns["np"] = np
import bluesky.callbacks
ns["bc"] = bluesky.callbacks
import_star(bluesky.callbacks, ns)
import bluesky.plans
ns["bp"] = bluesky.plans
import_star(bluesky.plans, ns)
import bluesky.plan_stubs
ns["bps"] = bluesky.plan_stubs
import_star(bluesky.plan_stubs, ns)
# special-case the commonly-used mv / mvr and its aliases mov / movr4
ns["mv"] = bluesky.plan_stubs.mv
ns["mvr"] = bluesky.plan_stubs.mvr
ns["mov"] = bluesky.plan_stubs.mov
ns["movr"] = bluesky.plan_stubs.movr
import bluesky.preprocessors
ns["bpp"] = bluesky.preprocessors
import bluesky.callbacks.broker
import_star(bluesky.callbacks.broker, ns)
import bluesky.simulators
import_star(bluesky.simulators, ns)
user_ns.update(ns)
return list(ns)
# import nslsii
# Register bluesky IPython magics.
#from bluesky.magics import BlueskyMagics
#get_ipython().register_magics(BlueskyMagics)
#nslsii.configure_base(get_ipython().user_ns, 'amx')
import bluesky.plans as bp
from bluesky.run_engine import RunEngine
from bluesky.utils import get_history, PersistentDict
RE = RunEngine()
beamline = os.environ["BEAMLINE_ID"]
configdir = os.environ['CONFIGDIR']
RE.md = PersistentDict('%s%s_bluesky_config' % (configdir, beamline))
from databroker import Broker
db = Broker.named(beamline)
RE.subscribe(db.insert)
# from bluesky.callbacks.best_effort import BestEffortCallback
# bec = BestEffortCallback()
# RE.subscribe(bec)
# convenience imports
# from ophyd.commands import *
from bluesky.callbacks import *
# from bluesky.spec_api import *
# from bluesky.global_state import gs, abort, stop, resume
# from databroker import (DataBroker as db, get_events, get_images,
from ophyd import Device, Component, EpicsSignal
from ophyd.signal import EpicsSignalBase
from ophyd.areadetector.filestore_mixins import resource_factory
import uuid
import os
from pathlib import Path
import numpy as np
from IPython import get_ipython
# Set up a RunEngine and use metadata backed by a sqlite file.
from bluesky import RunEngine
from bluesky.utils import PersistentDict
RE = RunEngine({})
RE.md = PersistentDict(str(Path("~/.bluesky_history").expanduser()))
# Set up SupplementalData.
from bluesky import SupplementalData
sd = SupplementalData()
RE.preprocessors.append(sd)
# Set up a Broker.
from databroker import Broker
db = Broker.named("temp")
# and subscribe it to the RunEngine
RE.subscribe(db.insert)
import os
import warnings
# convenience imports
import bluesky.plans as bp
import bluesky.plan_stubs as bps
import bluesky.preprocessors as bpp
import numpy as np
from ..session_logs import logger
logger.info(__file__)
# Set up a RunEngine and use metadata-backed PersistentDict
RE = RunEngine({})
RE.md = PersistentDict(
os.path.join(os.environ["HOME"], ".config", "Bluesky_RunEngine_md")
)
# keep track of callback subscriptions
callback_db = {}
# Connect with mongodb database.
db = databroker.catalog["mongodb_config"]
# Subscribe metadatastore to documents.
# If this is removed, data is not saved to metadatastore.
callback_db["db"] = RE.subscribe(db.v1.insert)
# Set up SupplementalData.
sd = SupplementalData()
RE.preprocessors.append(sd)