def __init__(self, address, *, prefix=b"", deserializer=pickle.loads, parent=None): super().__init__(parent) if isinstance(prefix, str): raise ValueError("prefix must be bytes, not string") if b" " in prefix: raise ValueError("prefix {!r} may not contain b' '".format(prefix)) self._prefix = prefix if isinstance(address, str): address = address.split(":", maxsplit=1) self._deserializer = deserializer self.address = (address[0], int(address[1])) self._context = zmq.Context() self._socket = self._context.socket(zmq.SUB) url = "tcp://%s:%d" % self.address self._socket.connect(url) self._socket.setsockopt_string(zmq.SUBSCRIBE, "") self._task = None self.closed = False self._timer = QTimer(self) self._dispatcher = Dispatcher() self.subscribe = self._dispatcher.subscribe self._waiting_for_start = True
def __init__(self): # Public dispatcher for callbacks self.dispatcher = Dispatcher() # Local caches for internal use self.seq_count = 0 # Maintain our own sequence count for this stream self.raw_descriptors = dict() # Store raw descriptors for use later self._stream_start_uid = None # Generated start doc uid self._descriptors = dict() # Dictionary of sent descriptors
class ControlGui(QtWidgets.QWidget): def __init__(self, queue, teleport, *scan_widgets, live_widget=None, **kwargs): super().__init__(**kwargs) self.label = label = StartLabel() self.queue = queue self.teleport = teleport self.md_parameters = MetaDataEntry(name='Metadata') self.md_widget = ParameterTree() self.md_widget.setParameters(self.md_parameters) outmost_layout = QtWidgets.QHBoxLayout() input_layout = QtWidgets.QVBoxLayout() outmost_layout.addLayout(input_layout) input_layout.addWidget(label) self.tabs = TabScanSelector(*scan_widgets) input_layout.addWidget(self.tabs) for sw in scan_widgets: sw.md_parameters = self.md_parameters self.go_button = QtWidgets.QPushButton('SCAN!') self.md_button = QtWidgets.QPushButton('edit metadata') input_layout.addWidget(self.md_button) input_layout.addWidget(self.go_button) self.teleport.name_doc.connect(label.doc_consumer) self.cbr = Dispatcher() self.bec = BestEffortCallback() self.teleport.name_doc.connect( lambda name, doc: self.cbr.process(DocumentNames(name), doc)) self.cbr.subscribe(self.bec) def runner(): self.queue.put(self.tabs.get_plan()) self.go_button.clicked.connect(runner) self.md_button.clicked.connect(self.md_widget.show) if live_widget is None: live_widget = LivePlaceholder() self.live_widget = live_widget self.teleport.name_doc.connect(live_widget.doc_consumer) outmost_layout.addWidget(live_widget) self.setLayout(outmost_layout)
def __init__(self, address, *, prefix=b"", deserializer=pickle.loads): if isinstance(prefix, str): raise ValueError("prefix must be bytes, not string") if b" " in prefix: raise ValueError("prefix {!r} may not contain b' '".format(prefix)) self._prefix = prefix if isinstance(address, str): address = address.split(":", maxsplit=1) self._deserializer = deserializer self.address = (address[0], int(address[1])) self.closed = False self._thread = None self._process = None self._result_queue = multiprocessing.Queue() self._kill_worker = multiprocessing.Queue() self._dispatcher = Dispatcher() self.subscribe = self._dispatcher.subscribe self._waiting_for_start = True
class RemoteDispatcher(QObject): """ Dispatch documents received over the network from a 0MQ proxy. This is designed to be run in a Qt application. Parameters ---------- address : tuple | str Address of a running 0MQ proxy, given either as a string like ``'127.0.0.1:5567'`` or as a tuple like ``('127.0.0.1', 5567)`` prefix : bytes, optional User-defined bytestring used to distinguish between multiple Publishers. If set, messages without this prefix will be ignored. If unset, no mesages will be ignored. deserializer: function, optional optional function to deserialize data. Default is pickle.loads Examples -------- Print all documents generated by remote RunEngines. >>> d = RemoteDispatcher(('localhost', 5568)) >>> d.subscribe(stream_documents_into_runs(model.add_run)) >>> d.start() # launches periodic workers on background threads >>> d.stop() # stop launching workers """ def __init__(self, address, *, prefix=b"", deserializer=pickle.loads, parent=None): super().__init__(parent) if isinstance(prefix, str): raise ValueError("prefix must be bytes, not string") if b" " in prefix: raise ValueError("prefix {!r} may not contain b' '".format(prefix)) self._prefix = prefix if isinstance(address, str): address = address.split(":", maxsplit=1) self._deserializer = deserializer self.address = (address[0], int(address[1])) self._context = zmq.Context() self._socket = self._context.socket(zmq.SUB) url = "tcp://%s:%d" % self.address self._socket.connect(url) self._socket.setsockopt_string(zmq.SUBSCRIBE, "") self._task = None self.closed = False self._timer = QTimer(self) self._dispatcher = Dispatcher() self.subscribe = self._dispatcher.subscribe self._waiting_for_start = True def _receive_data(self): our_prefix = self._prefix # local var to save an attribute lookup # TODO Pull on the socket more than once here, until it blocks, to # ensure we do not get more and more behind over time. message = self._socket.recv() prefix, name, doc = message.split(b" ", 2) name = name.decode() if (not our_prefix) or prefix == our_prefix: if self._waiting_for_start: if name == "start": self._waiting_for_start = False else: # We subscribed midstream and are seeing documents for # which we do not have the full run. Wait for a 'start' # doc. return doc = self._deserializer(doc) return name, doc def start(self): if self.closed: raise RuntimeError("This RemoteDispatcher has already been " "started and interrupted. Create a fresh " "instance with {}".format(repr(self))) self._work_loop() def _work_loop(self): worker = create_worker(self._receive_data, ) # Schedule this method to be run again after a brief wait. worker.finished.connect( lambda: self._timer.singleShot(LOADING_LATENCY, self._work_loop)) worker.returned.connect(self._process_result) worker.start() def _process_result(self, result): if result is None: return name, doc = result self._dispatcher.process(DocumentNames[name], doc) def stop(self): self.closed = True
class RemoteDispatcher: """ Dispatch documents received over the network from a 0MQ proxy. This is designed to be run in a Jupyter kernel. Parameters ---------- address : tuple | str Address of a running 0MQ proxy, given either as a string like ``'127.0.0.1:5567'`` or as a tuple like ``('127.0.0.1', 5567)`` prefix : bytes, optional User-defined bytestring used to distinguish between multiple Publishers. If set, messages without this prefix will be ignored. If unset, no mesages will be ignored. deserializer: function, optional optional function to deserialize data. Default is pickle.loads Examples -------- Print all documents generated by remote RunEngines. >>> d = RemoteDispatcher(('localhost', 5568)) >>> d.subscribe(stream_documents_into_runs(model.add_run)) >>> d.start() # starts a thread and a long-running subprocess >>> d.stop() # stops them and blocks until they stop """ def __init__(self, address, *, prefix=b"", deserializer=pickle.loads): if isinstance(prefix, str): raise ValueError("prefix must be bytes, not string") if b" " in prefix: raise ValueError("prefix {!r} may not contain b' '".format(prefix)) self._prefix = prefix if isinstance(address, str): address = address.split(":", maxsplit=1) self._deserializer = deserializer self.address = (address[0], int(address[1])) self.closed = False self._thread = None self._process = None self._result_queue = multiprocessing.Queue() self._kill_worker = multiprocessing.Queue() self._dispatcher = Dispatcher() self.subscribe = self._dispatcher.subscribe self._waiting_for_start = True def _receive_data(self): our_prefix = self._prefix # local var to save an attribute lookup # TODO Pull on the socket more than once here, until it blocks, to # ensure we do not get more and more behind over time. message = self._socket.recv() prefix, name, doc = message.split(b" ", 2) name = name.decode() if (not our_prefix) or prefix == our_prefix: if self._waiting_for_start: if name == "start": self._waiting_for_start = False else: # We subscribed midstream and are seeing documents for # which we do not have the full run. Wait for a 'start' # doc. return doc = self._deserializer(doc) return name, doc def start(self): if self.closed: raise RuntimeError( "This RemoteDispatcher has already been " "started and interrupted. Create a fresh " "instance with {}".format(repr(self)) ) self._process = multiprocessing.Process( target=self._zmq_worker, args=(self._kill_worker, self._result_queue) ) self._process.start() self._thread = threading.Thread(target=self._dispatcher_worker) self._thread.start() def _zmq_worker(self, kill_worker, result_queue): "This runs in a subprocess to avoid stepping on Jupyter's use of zmq." self._context = zmq.Context() self._socket = self._context.socket(zmq.SUB) url = "tcp://%s:%d" % self.address self._socket.connect(url) self._socket.setsockopt_string(zmq.SUBSCRIBE, "") # http://stackoverflow.com/questions/7538988/zeromq-how-to-prevent-infinite-wait#comment68021160_7540299 # self._socket.setsockopt(zmq.RCVTIMEO, 100) # miliseconds # self._socket.setsockopt(zmq.LINGER, 0) while kill_worker.empty(): result = self._receive_data() if result is not None: result_queue.put(result) def _dispatcher_worker(self): "This runs in a thread." while not self.closed: try: result = self._result_queue.get(timeout=0.1) except Empty: continue name, doc = result self._dispatcher.process(DocumentNames[name], doc) def stop(self): self.closed = True self._kill_worker.put(object()) self._thread.join() self._process.join()
class LiveDispatcher(CallbackBase): """ A secondary event stream of processed data The LiveDipatcher base implementation does not change any of the data emitted, this task is left to sub-classes, but instead handles reimplementing a secondary event stream that fits the same schema demanded by the RunEngine itself. In order to reduce the work done by these processed data pipelines, the LiveDispatcher handles the nitty-gritty details of formatting the event documents. This includes creating new uids, numbering events and creating descriptors. The LiveDispatcher can be subscribed to using the same syntax as the RunEngine, effectively creating a small chain of callbacks .. code:: # Create our dispatcher ld = LiveDispatcher() # Subscribe it to receive events from the RunEgine RE.subscribe(ld) # Subscribe any callbacks we desire to second stream ld.subscribe(LivePlot('det', x='motor')) """ def __init__(self): # Public dispatcher for callbacks self.dispatcher = Dispatcher() # Local caches for internal use self.seq_count = 0 # Maintain our own sequence count for this stream self.raw_descriptors = dict() # Store raw descriptors for use later self._stream_start_uid = None # Generated start doc uid self._descriptors = dict() # Dictionary of sent descriptors def start(self, doc, _md=None): """Receive a raw start document, re-emit it for the modified stream""" self._stream_start_uid = new_uid() _md = _md or dict() # Create a new start document with a new uid, start time, and the uid # of the original start document. Preserve the rest of the metadata # that we retrieved from the start document md = ChainMap({'uid': self._stream_start_uid, 'original_run_uid': doc['uid'], 'time': ttime.time()}, _md, doc) # Dispatch the start document for anyone subscribed to our Dispatcher self.emit(DocumentNames.start, dict(md)) super().start(doc) def descriptor(self, doc): """Store a descriptor""" self.raw_descriptors[doc['uid']] = doc super().descriptor(doc) def event(self, doc, **kwargs): """ Receive an event document from the raw stream. This should be reimplemented by a subclass. Parameters ---------- doc : event kwargs: All keyword arguments are passed to :meth:`.process_event` """ self.process_event(doc, **kwargs) return super().event(doc) def process_event(self, doc, stream_name='primary', id_args=None, config=None): """ Process a modified event document then emit it for the modified stream This will pass an Event document to the dispatcher. If we have received a new event descriptor from the original stream, or we have recieved a new set of `id_args` or `descriptor_id` , a new descriptor document is first issued and passed through to the dispatcher. When issuing a new event, the new descriptor is given a new source field. Parameters ---------- doc : event stream_name : str, optional String identifier for a particular stream id_args : tuple, optional Additional tuple of hashable objects to identify the stream config: dict, optional Additional configuration information to be included in the event descriptor Notes ----- Any callback subscribed to the `Dispatcher` will receive these event streams. If nothing is subscribed, these documents will not go anywhere. """ id_args = id_args or (doc['descriptor'],) config = config or dict() # Determine the descriptor id desc_id = frozenset((tuple(doc['data'].keys()), stream_name, id_args)) # If we haven't described this configuration # Send a new document to our subscribers if (stream_name not in self._descriptors or desc_id not in self._descriptors[stream_name]): # Create a new description document for the output of the stream data_keys = dict() # Parse the event document creating a new description. If the key # existed in the original source description, just assume that it # is the same type, units and shape. Otherwise do some # investigation raw_desc = self.raw_descriptors.get(doc['descriptor'], {}) for key, val in doc['data'].items(): # Described priorly if key in raw_desc['data_keys']: key_desc = raw_desc['data_keys'][key] # String key elif isinstance(val, str): key_desc = {'dtype': 'string', 'shape': []} # Iterable elif isinstance(val, Iterable): key_desc = {'dtype': 'array', 'shape': np.shape(val)} # Number else: key_desc = {'dtype': 'number', 'shape': []} # Modify the source key_desc['source'] = 'Stream' # Store in our new descriptor data_keys[key] = key_desc # Create our complete description document desc = ChainMap({'uid': new_uid(), 'time': ttime.time(), 'run_start': self._stream_start_uid, 'data_keys': data_keys, 'configuration': config, 'object_keys': {'stream': list(data_keys.keys())}}, raw_desc) # Store information about our descriptors desc = dict(desc) if stream_name not in self._descriptors: self._descriptors[stream_name] = dict() self._descriptors[stream_name][desc_id] = desc # Emit the document to all subscribers self.emit(DocumentNames.descriptor, desc) # Clean the Event document produced by graph network. The data is left # untouched, but the relevant uids, timestamps, seq_num are modified so # that this event is not confused with the raw data stream self.seq_count += 1 desc_uid = self._descriptors[stream_name][desc_id]['uid'] current_time = ttime.time() evt = ChainMap({'uid': new_uid(), 'descriptor': desc_uid, 'timestamps': dict((key, current_time) for key in doc['data'].keys()), 'seq_num': self.seq_count, 'time': current_time}, doc) # Emit the event document self.emit(DocumentNames.event, dict(evt)) def stop(self, doc, _md=None): """Receive a raw stop document, re-emit it for the modified stream""" # Create a new stop document with a new_uid, pointing to the correct # start document uid, and tally the number of events we have emitted. # The rest of the stop information is passed on to the next callback _md = _md or dict() num_events = dict((stream, len(self._descriptors[stream])) for stream in self._descriptors.keys()) md = ChainMap(dict(run_start=self._stream_start_uid, time=ttime.time(), uid=new_uid(), num_events=num_events), doc) self.emit(DocumentNames.stop, dict(md)) # Clear the local caches for the run self.seq_count = 0 self.raw_descriptors.clear() self._descriptors.clear() self._stream_start_uid = None super().stop(doc) def emit(self, name, doc): """Check the document schema and send to the dispatcher""" jsonschema.validate(doc, schemas[name]) self.dispatcher.process(name, doc) def subscribe(self, func, name='all'): """Convenience function for dispatcher subscription""" return self.dispatcher.subscribe(func, name) def unsubscribe(self, token): """Convenience function for dispatcher un-subscription""" self.dispatcher.unsubscribe(token)