def recreate_from_others(self, method_metas, without=None): if without is None: without = [] defaults = OrderedDict() elements = OrderedDict() required = [] # Populate the intermediate data structures for method_meta in method_metas: for element in method_meta.takes.elements: if element not in without: # Serialize it to copy it serialized = method_meta.takes.elements[element].to_dict() elements[element] = serialized if element in method_meta.takes.required and element not in required: required.append(element) if element in method_meta.defaults: defaults.pop(element, None) defaults[element] = method_meta.defaults[element] # TODO: what about returns? # remove required args that are now defaulted required = [r for r in required if r not in defaults] # Update ourself from these structures takes = MapMeta() takes.set_elements(ElementMap(elements)) takes.set_required(required) self.set_takes(takes) self.set_defaults(defaults)
def recreate_from_others(self, method_metas, without=()): defaults = OrderedDict() elements = OrderedDict() required = [] # Populate the intermediate data structures for method_meta in method_metas: for element in method_meta.takes.elements: if element not in without: # Serialize it to copy it serialized = method_meta.takes.elements[element].to_dict() elements[element] = serialized if element in method_meta.takes.required and \ element not in required: required.append(element) if element in method_meta.defaults: defaults.pop(element, None) defaults[element] = method_meta.defaults[element] # TODO: what about returns? # remove required args that are now defaulted required = [r for r in required if r not in defaults] # Update ourself from these structures takes = MapMeta() takes.set_elements(elements) takes.set_required(required) self.set_takes(takes) self.set_defaults(defaults)
class BlockModel(Model): """Data Model for a Block""" def __init__(self) -> None: # Make a new call_types dict so we don't modify for all instances self.call_types = OrderedDict() self.meta = self.set_endpoint_data("meta", BlockMeta()) def set_endpoint_data( self, name: str, value: Union[AttributeModel, MethodModel, BlockMeta], ) -> Any: name = deserialize_object(name, str) if name == "meta": value = deserialize_object(value, BlockMeta) else: value = deserialize_object(value, (AttributeModel, MethodModel)) with self.notifier.changes_squashed: if name in self.call_types: # Stop the old Model notifying getattr(self, name).set_notifier_path(Model.notifier, []) else: anno = Anno("Field").set_typ(type(value)) self.call_types[name] = anno value.set_notifier_path(self.notifier, self.path + [name]) setattr(self, name, value) # Tell the notifier what changed self.notifier.add_squashed_change(self.path + [name], value) self._update_fields() return value def _update_fields(self): self.meta.set_fields([x for x in self.call_types if x != "meta"]) def remove_endpoint(self, name: str) -> None: with self.notifier.changes_squashed: getattr(self, name).set_notifier_path(Model.notifier, []) self.call_types.pop(name) delattr(self, name) self._update_fields() self.notifier.add_squashed_delete(self.path + [name])
def execute(self, args): self.log_debug("Execute %s method called on [%s] with: %s", self._method, self._block, args) self.log_debug("Structure: %s", args.getStructureDict()) # Acquire the lock with self._lock: try: # We now need to create the Post message and execute it endpoint = [self._block, self._method] request = Post(None, self._server.q, endpoint, self.parse_variants(args.toDict(True))) request.set_id(self._id) self._server.process.q.put(request) # Now wait for the Post reply self.log_debug("Waiting for reply") self.wait_for_reply(timeout=None) self.log_debug("Reply received %s %s", type(self._response), self._response) response_dict = OrderedDict() if isinstance(self._response, Return): response_dict = self._response["value"] self.log_debug("Response value : %s", response_dict) elif isinstance(self._response, Error): response_dict = self._response.to_dict() response_dict.pop("id") if not response_dict: pv_object = pvaccess.PvObject(OrderedDict(), 'malcolm:core/Map:1.0') else: #pv_object = self._server.dict_to_structure(response_dict) #self.log_debug("Pv Object structure created") #self.log_debug("%s", self._server.strip_type_id(response_dict)) #pv_object.set(self._server.strip_type_id(response_dict)) pv_object = self._server.dict_to_pv_object(response_dict) self.log_debug("Pv Object value set: %s", pv_object) # Add this RPC to the purge list #self._server.register_dead_rpc(self._id) return pv_object except Exception: self.log_exception("Request %s failed", self._request)
class Process(Loggable): """Hosts a number of Blocks, distributing requests between them""" def __init__(self, name, sync_factory): self.set_logger_name(name) self.name = name self.sync_factory = sync_factory self.q = self.create_queue() self._blocks = OrderedDict() # block_name -> Block self._controllers = OrderedDict() # block_name -> Controller self._block_state_cache = Cache() self._recv_spawned = None self._other_spawned = [] # lookup of all Subscribe requests, ordered to guarantee subscription # notification ordering # {Request.generate_key(): Subscribe} self._subscriptions = OrderedDict() self.comms = [] self._client_comms = OrderedDict() # client comms -> list of blocks self._handle_functions = { Post: self._forward_block_request, Put: self._forward_block_request, Get: self._handle_get, Subscribe: self._handle_subscribe, Unsubscribe: self._handle_unsubscribe, BlockChanges: self._handle_block_changes, BlockRespond: self._handle_block_respond, BlockAdd: self._handle_block_add, BlockList: self._handle_block_list, AddSpawned: self._add_spawned, } self.create_process_block() def recv_loop(self): """Service self.q, distributing the requests to the right block""" while True: request = self.q.get() self.log_debug("Received request %s", request) if request is PROCESS_STOP: # Got the sentinel, stop immediately break try: self._handle_functions[type(request)](request) except Exception as e: # pylint:disable=broad-except self.log_exception("Exception while handling %s", request) try: request.respond_with_error(str(e)) except Exception: pass def add_comms(self, comms): assert not self._recv_spawned, \ "Can't add comms when process has been started" self.comms.append(comms) def start(self): """Start the process going""" self._recv_spawned = self.sync_factory.spawn(self.recv_loop) for comms in self.comms: comms.start() def stop(self, timeout=None): """Stop the process and wait for it to finish Args: timeout (float): Maximum amount of time to wait for each spawned process. None means forever """ assert self._recv_spawned, "Process not started" self.q.put(PROCESS_STOP) for comms in self.comms: comms.stop() # Wait for recv_loop to complete first self._recv_spawned.wait(timeout=timeout) # Now wait for anything it spawned to complete for s, _ in self._other_spawned: s.wait(timeout=timeout) # Garbage collect the syncfactory del self.sync_factory def _forward_block_request(self, request): """Lookup target Block and spawn block.handle_request(request) Args: request (Request): The message that should be passed to the Block """ block_name = request.endpoint[0] block = self._blocks[block_name] spawned = self.sync_factory.spawn(block.handle_request, request) self._add_spawned(AddSpawned(spawned, block.handle_request)) def create_queue(self): """ Create a queue using sync_factory object Returns: Queue: New queue """ return self.sync_factory.create_queue() def create_lock(self): """ Create a lock using sync_factory object Returns: New lock object """ return self.sync_factory.create_lock() def spawn(self, function, *args, **kwargs): """Calls SyncFactory.spawn()""" def catching_function(): try: function(*args, **kwargs) except Exception: self.log_exception( "Exception calling %s(*%s, **%s)", function, args, kwargs) raise spawned = self.sync_factory.spawn(catching_function) request = AddSpawned(spawned, function) self.q.put(request) return spawned def _add_spawned(self, request): spawned = self._other_spawned self._other_spawned = [] spawned.append((request.spawned, request.function)) # Filter out the spawned that have completed to stop memory leaks for sp, f in spawned: if not sp.ready(): self._other_spawned.append((sp, f)) def get_client_comms(self, block_name): for client_comms, blocks in list(self._client_comms.items()): if block_name in blocks: return client_comms def create_process_block(self): self.process_block = Block() # TODO: add a meta here children = OrderedDict() children["blocks"] = StringArrayMeta( description="Blocks hosted by this Process" ).make_attribute([]) children["remoteBlocks"] = StringArrayMeta( description="Blocks reachable via ClientComms" ).make_attribute([]) self.process_block.replace_endpoints(children) self.process_block.set_process_path(self, [self.name]) self.add_block(self.process_block, self) def update_block_list(self, client_comms, blocks): self.q.put(BlockList(client_comms=client_comms, blocks=blocks)) def _handle_block_list(self, request): self._client_comms[request.client_comms] = request.blocks remotes = [] for blocks in self._client_comms.values(): remotes += [b for b in blocks if b not in remotes] self.process_block["remoteBlocks"].set_value(remotes) def _handle_block_changes(self, request): """Update subscribers with changes and applies stored changes to the cached structure""" # update cached dict subscription_changes = self._block_state_cache.apply_changes( *request.changes) # Send out the changes for subscription, changes in subscription_changes.items(): if subscription.delta: # respond with the filtered changes subscription.respond_with_delta(changes) else: # respond with the structure of everything # below the endpoint d = self._block_state_cache.walk_path(subscription.endpoint) subscription.respond_with_update(d) def report_changes(self, *changes): self.q.put(BlockChanges(changes=list(changes))) def block_respond(self, response, response_queue): self.q.put(BlockRespond(response, response_queue)) def _handle_block_respond(self, request): """Push the response to the required queue""" request.response_queue.put(request.response) def add_block(self, block, controller): """Add a block to be hosted by this process Args: block (Block): The block to be added controller (Controller): Its controller """ path = block.process_path assert len(path) == 1, \ "Expected block %r to have %r as parent, got path %r" % \ (block, self, path) name = path[0] assert name not in self._blocks, \ "There is already a block called %r" % name request = BlockAdd(block=block, controller=controller, name=name) if self._recv_spawned: # Started, so call in Process thread self.q.put(request) else: # Not started yet so we are safe to add in this thread self._handle_block_add(request) def _handle_block_add(self, request): """Add a block to be hosted by this process""" assert request.name not in self._blocks, \ "There is already a block called %r" % request.name self._blocks[request.name] = request.block self._controllers[request.name] = request.controller serialized = request.block.to_dict() change_request = BlockChanges([[[request.name], serialized]]) self._handle_block_changes(change_request) # Regenerate list of blocks self.process_block["blocks"].set_value(list(self._blocks)) def get_block(self, block_name): try: return self._blocks[block_name] except KeyError: if block_name in self.process_block.remoteBlocks: return self.make_client_block(block_name) else: raise def make_client_block(self, block_name): params = ClientController.MethodMeta.prepare_input_map( mri=block_name) controller = ClientController(self, {}, params) return controller.block def get_controller(self, block_name): return self._controllers[block_name] def _handle_subscribe(self, request): """Add a new subscriber and respond with the current sub-structure state""" key = request.generate_key() assert key not in self._subscriptions, \ "Subscription on %s already exists" % (key,) self._subscriptions[key] = request self._block_state_cache.add_subscriber(request, request.endpoint) d = self._block_state_cache.walk_path(request.endpoint) self.log_debug("Initial subscription value %s", d) if request.delta: request.respond_with_delta([[[], d]]) else: request.respond_with_update(d) def _handle_unsubscribe(self, request): """Remove a subscriber and respond with success or error""" key = request.generate_key() try: subscription = self._subscriptions.pop(key) except KeyError: request.respond_with_error( "No subscription found for %s" % (key,)) else: self._block_state_cache.remove_subscriber( subscription, subscription.endpoint) request.respond_with_return() def _handle_get(self, request): d = self._block_state_cache.walk_path(request.endpoint) request.respond_with_return(d)
class PandABlocksManagerController(ManagerController): def __init__(self, mri, # type: AMri config_dir, # type: AConfigDir hostname="localhost", # type: AHostname port=8888, # type: APort initial_design="", # type: AInitialDesign description="", # type: ADescription use_git=True, # type: AUseGit doc_url_base=DOC_URL_BASE, # type: ADocUrlBase poll_period=0.1 # type: APollPeriod ): # type: (...) -> None super(PandABlocksManagerController, self).__init__( mri, config_dir, initial_design, description, use_git) self._poll_period = poll_period self._doc_url_base = doc_url_base # {block_name: BlockData} self._blocks_data = {} # {block_name: {field_name: Part}} self._blocks_parts = OrderedDict() # src_attr -> [dest_attr] self._listening_attrs = {} # lut elements to be displayed or not # {fnum: {id: visible}} self._lut_elements = {} # changes left over from last time self.changes = OrderedDict() # The PandABlock client that does the comms self.client = PandABlocksClient(hostname, port, Queue) # Filled in on reset self._stop_queue = None self._poll_spawned = None def do_init(self): # start the poll loop and make block parts first to fill in our parts # before calling _set_block_children() self.start_poll_loop() super(PandABlocksManagerController, self).do_init() def start_poll_loop(self): # queue to listen for stop events if not self.client.started: self._stop_queue = Queue() if self.client.started: self.client.stop() self.client.start(self.process.spawn, socket) if not self._blocks_parts: self._make_blocks_parts() if self._poll_spawned is None: self._poll_spawned = self.process.spawn(self._poll_loop) def do_disable(self): super(PandABlocksManagerController, self).do_disable() self.stop_poll_loop() def do_reset(self): self.start_poll_loop() super(PandABlocksManagerController, self).do_reset() def _poll_loop(self): """At self.poll_period poll for changes""" next_poll = time.time() while True: next_poll += self._poll_period timeout = next_poll - time.time() if timeout < 0: timeout = 0 try: return self._stop_queue.get(timeout=timeout) except TimeoutError: # No stop, no problem pass try: self.handle_changes(self.client.get_changes()) except Exception: # TODO: should fault here? self.log.exception("Error while getting changes") def stop_poll_loop(self): if self._poll_spawned: self._stop_queue.put(None) self._poll_spawned.wait() self._poll_spawned = None if self.client.started: self.client.stop() def _make_blocks_parts(self): # {block_name_without_number: BlockData} self._blocks_data = OrderedDict() self._blocks_parts = OrderedDict() for block_rootname, block_data in self.client.get_blocks_data().items(): block_names = [] if block_data.number == 1: block_names.append(block_rootname) else: for i in range(block_data.number): block_names.append("%s%d" % (block_rootname, i + 1)) for block_name in block_names: self._blocks_data[block_name] = block_data self._make_parts(block_name, block_data) # Handle the initial set of changes to get an initial value self.handle_changes(self.client.get_changes()) # Then once more to let bit_outs toggle back self.handle_changes(()) assert not self.changes, "There are still changes %s" % self.changes def _make_child_controller(self, parts, mri): controller = BasicController(mri=mri) if mri.endswith("PCAP"): parts.append(PandABlocksActionPart( self.client, "*PCAP", "ARM", "Arm position capture", [])) parts.append(PandABlocksActionPart( self.client, "*PCAP", "DISARM", "Disarm position capture", [])) for part in parts: controller.add_part(part) return controller def _make_corresponding_part(self, block_name, mri): part = ChildPart(name=block_name, mri=mri, stateful=False) return part def _make_parts(self, block_name, block_data): mri = "%s:%s" % (self.mri, block_name) # Defer creation of parts to a block maker maker = PandABlocksMaker( self.client, block_name, block_data, self._doc_url_base) # Make the child controller and add it to the process controller = self._make_child_controller(maker.parts.values(), mri) self.process.add_controller(controller, timeout=5) # Store the parts so we can update them with the poller self._blocks_parts[block_name] = maker.parts # Make the corresponding part for us child_part = self._make_corresponding_part(block_name, mri) self.add_part(child_part) def _set_lut_icon(self, block_name): icon_attr = self._blocks_parts[block_name]["icon"].attr with open(os.path.join(SVG_DIR, "LUT.svg")) as f: svg_text = f.read() fnum = int(self.client.get_field(block_name, "FUNC.RAW"), 0) invis = self._get_lut_icon_elements(fnum) # https://stackoverflow.com/a/8998773 ET.register_namespace('', "http://www.w3.org/2000/svg") root = ET.fromstring(svg_text) for i in invis: # Find the first parent which has a child with id i parent = root.find('.//*[@id=%r]/..' % i) # Find the child and remove it child = parent.find('./*[@id=%r]' % i) parent.remove(child) svg_text = et_to_string(root) icon_attr.set_value(svg_text) def _get_lut_icon_elements(self, fnum): if not self._lut_elements: # Generate the lut element table # Do the general case funcs funcs = [("AND", operator.and_), ("OR", operator.or_)] for func, op in funcs: for nargs in (2, 3, 4, 5): # 2**nargs permutations for permutation in range(2 ** nargs): self._calc_visibility(func, op, nargs, permutation) # Add in special cases for NOT for ninp in "ABCDE": invis = {"AND", "OR", "LUT"} for inp in "ABCDE": if inp != ninp: invis.add(inp) invis.add("not%s" % inp) self._lut_elements[~LUT_CONSTANTS[ninp] & (2 ** 32 - 1)] = invis # And catchall for LUT in 0 invis = {"AND", "OR", "NOT"} for inp in "ABCDE": invis.add("not%s" % inp) self._lut_elements[0] = invis return self._lut_elements.get(fnum, self._lut_elements[0]) def _calc_visibility(self, func, op, nargs, permutations): # Visibility dictionary defaults invis = {"AND", "OR", "LUT", "NOT"} invis.remove(func) args = [] for i, inp in enumerate("EDCBA"): # xxxxx where x is 0 or 1 # EDCBA negations = format(permutations, '05b') if (5 - i) > nargs: # invisible invis.add(inp) invis.add("not%s" % inp) else: # visible if negations[i] == "1": args.append(~LUT_CONSTANTS[inp] & (2 ** 32 - 1)) else: invis.add("not%s" % inp) args.append(LUT_CONSTANTS[inp]) # Insert into table fnum = op(args[0], args[1]) for a in args[2:]: fnum = op(fnum, a) self._lut_elements[fnum] = invis def handle_changes(self, changes): for k, v in changes: self.changes[k] = v block_changes = OrderedDict() for full_field, val in list(self.changes.items()): block_name, field_name = full_field.split(".", 1) block_changes.setdefault(block_name, []).append(( field_name, full_field, val)) for block_name, field_changes in block_changes.items(): # Squash changes block_mri = "%s:%s" % (self.mri, block_name) try: block_controller = self.process.get_controller(block_mri) except ValueError: self.log.debug("Block %s not known", block_name) for _, full_field, _ in field_changes: self.changes.pop(full_field) else: with block_controller.changes_squashed: self.do_field_changes(block_name, field_changes) def do_field_changes(self, block_name, field_changes): for field_name, full_field, val in field_changes: ret = self.update_attribute(block_name, field_name, val) if ret is not None: self.changes[full_field] = ret else: self.changes.pop(full_field) # If it was LUT.FUNC then recalculate icon if block_name.startswith("LUT") and field_name == "FUNC": self._set_lut_icon(block_name) def update_attribute(self, block_name, field_name, value): ret = None parts = self._blocks_parts[block_name] if field_name not in parts: self.log.debug("Block %s has no field %s", block_name, field_name) return ret part = parts[field_name] attr = part.attr field_data = self._blocks_data[block_name].fields.get(field_name, None) if value == Exception: # TODO: set error self.log.warning("Field %s.%s in error", block_name, field_name) value = None # Cheaper than isinstance if attr.meta.typeid == TableMeta.typeid: value = part.table_from_list(value) elif attr.meta.typeid == BooleanMeta.typeid: value = bool(int(value)) is_bit_out = field_data and field_data.field_type == "bit_out" if is_bit_out and attr.value is value: # make bit_out things toggle while changing ret = value value = not value else: value = attr.meta.validate(value) # Update the value of our attribute and anyone listening ts = TimeStamp() attr.set_value_alarm_ts(value, Alarm.ok, ts) for dest_attr in self._listening_attrs.get(attr, []): dest_attr.set_value_alarm_ts(value, Alarm.ok, ts) # if we changed the value of a mux, update the slaved values if field_data and field_data.field_type in ("bit_mux", "pos_mux"): current_part = parts[field_name + ".CURRENT"] current_attr = current_part.attr self._update_current_attr( current_attr, value, ts, field_data.field_type) # if we changed a pos_out, its SCALE or OFFSET, update its scaled value root_field_name = field_name.split(".")[0] field_data = self._blocks_data[block_name].fields[root_field_name] if field_data.field_type == "pos_out": scale = parts[root_field_name + ".SCALE"].attr.value offset = parts[root_field_name + ".OFFSET"].attr.value scaled = parts[root_field_name].attr.value * scale + offset parts[root_field_name + ".SCALED"].attr.set_value_alarm_ts( scaled, Alarm.ok, ts) return ret def _update_current_attr(self, current_attr, mux_val, ts, field_type): # Remove the old current_attr from all lists for mux_set in self._listening_attrs.values(): try: mux_set.remove(current_attr) except KeyError: pass # add it to the list of things that need to update try: val = MUX_CONSTANT_VALUES[field_type][mux_val] except KeyError: mon_block_name, mon_field_name = mux_val.split(".", 1) mon_parts = self._blocks_parts[mon_block_name] out_attr = mon_parts[mon_field_name].attr self._listening_attrs.setdefault(out_attr, set()).add(current_attr) # update it to the right value val = out_attr.value current_attr.set_value_alarm_ts(val, Alarm.ok, ts)
class Process(Loggable): """Hosts a number of Blocks, distributing requests between them""" def __init__(self, name, sync_factory): self.set_logger_name(name) self.name = name self.sync_factory = sync_factory self.q = self.create_queue() self._blocks = OrderedDict() # block_name -> Block self._controllers = OrderedDict() # block_name -> Controller self._block_state_cache = Cache() self._recv_spawned = None self._other_spawned = [] # lookup of all Subscribe requests, ordered to guarantee subscription # notification ordering # {Request.generate_key(): Subscribe} self._subscriptions = OrderedDict() self.comms = [] self._client_comms = OrderedDict() # client comms -> list of blocks self._handle_functions = { Post: self._forward_block_request, Put: self._forward_block_request, Get: self._handle_get, Subscribe: self._handle_subscribe, Unsubscribe: self._handle_unsubscribe, BlockChanges: self._handle_block_changes, BlockRespond: self._handle_block_respond, BlockAdd: self._handle_block_add, BlockList: self._handle_block_list, AddSpawned: self._add_spawned, } self.create_process_block() def recv_loop(self): """Service self.q, distributing the requests to the right block""" while True: request = self.q.get() self.log_debug("Received request %s", request) if request is PROCESS_STOP: # Got the sentinel, stop immediately break try: self._handle_functions[type(request)](request) except Exception as e: # pylint:disable=broad-except self.log_exception("Exception while handling %s", request) try: request.respond_with_error(str(e)) except Exception: pass def add_comms(self, comms): assert not self._recv_spawned, \ "Can't add comms when process has been started" self.comms.append(comms) def start(self): """Start the process going""" self._recv_spawned = self.sync_factory.spawn(self.recv_loop) for comms in self.comms: comms.start() def stop(self, timeout=None): """Stop the process and wait for it to finish Args: timeout (float): Maximum amount of time to wait for each spawned process. None means forever """ assert self._recv_spawned, "Process not started" self.q.put(PROCESS_STOP) for comms in self.comms: comms.stop() # Wait for recv_loop to complete first self._recv_spawned.wait(timeout=timeout) # Now wait for anything it spawned to complete for s, _ in self._other_spawned: s.wait(timeout=timeout) # Garbage collect the syncfactory del self.sync_factory def _forward_block_request(self, request): """Lookup target Block and spawn block.handle_request(request) Args: request (Request): The message that should be passed to the Block """ block_name = request.endpoint[0] block = self._blocks[block_name] spawned = self.sync_factory.spawn(block.handle_request, request) self._add_spawned(AddSpawned(spawned, block.handle_request)) def create_queue(self): """ Create a queue using sync_factory object Returns: Queue: New queue """ return self.sync_factory.create_queue() def create_lock(self): """ Create a lock using sync_factory object Returns: New lock object """ return self.sync_factory.create_lock() def spawn(self, function, *args, **kwargs): """Calls SyncFactory.spawn()""" def catching_function(): try: function(*args, **kwargs) except Exception: self.log_exception("Exception calling %s(*%s, **%s)", function, args, kwargs) raise spawned = self.sync_factory.spawn(catching_function) request = AddSpawned(spawned, function) self.q.put(request) return spawned def _add_spawned(self, request): spawned = self._other_spawned self._other_spawned = [] spawned.append((request.spawned, request.function)) # Filter out the spawned that have completed to stop memory leaks for sp, f in spawned: if not sp.ready(): self._other_spawned.append((sp, f)) def get_client_comms(self, block_name): for client_comms, blocks in list(self._client_comms.items()): if block_name in blocks: return client_comms def create_process_block(self): self.process_block = Block() # TODO: add a meta here children = OrderedDict() children["blocks"] = StringArrayMeta( description="Blocks hosted by this Process").make_attribute([]) children["remoteBlocks"] = StringArrayMeta( description="Blocks reachable via ClientComms").make_attribute([]) self.process_block.replace_endpoints(children) self.process_block.set_process_path(self, [self.name]) self.add_block(self.process_block, self) def update_block_list(self, client_comms, blocks): self.q.put(BlockList(client_comms=client_comms, blocks=blocks)) def _handle_block_list(self, request): self._client_comms[request.client_comms] = request.blocks remotes = [] for blocks in self._client_comms.values(): remotes += [b for b in blocks if b not in remotes] self.process_block["remoteBlocks"].set_value(remotes) def _handle_block_changes(self, request): """Update subscribers with changes and applies stored changes to the cached structure""" # update cached dict subscription_changes = self._block_state_cache.apply_changes( *request.changes) # Send out the changes for subscription, changes in subscription_changes.items(): if subscription.delta: # respond with the filtered changes subscription.respond_with_delta(changes) else: # respond with the structure of everything # below the endpoint d = self._block_state_cache.walk_path(subscription.endpoint) subscription.respond_with_update(d) def report_changes(self, *changes): self.q.put(BlockChanges(changes=list(changes))) def block_respond(self, response, response_queue): self.q.put(BlockRespond(response, response_queue)) def _handle_block_respond(self, request): """Push the response to the required queue""" request.response_queue.put(request.response) def add_block(self, block, controller): """Add a block to be hosted by this process Args: block (Block): The block to be added controller (Controller): Its controller """ path = block.process_path assert len(path) == 1, \ "Expected block %r to have %r as parent, got path %r" % \ (block, self, path) name = path[0] assert name not in self._blocks, \ "There is already a block called %r" % name request = BlockAdd(block=block, controller=controller, name=name) if self._recv_spawned: # Started, so call in Process thread self.q.put(request) else: # Not started yet so we are safe to add in this thread self._handle_block_add(request) def _handle_block_add(self, request): """Add a block to be hosted by this process""" assert request.name not in self._blocks, \ "There is already a block called %r" % request.name self._blocks[request.name] = request.block self._controllers[request.name] = request.controller serialized = request.block.to_dict() change_request = BlockChanges([[[request.name], serialized]]) self._handle_block_changes(change_request) # Regenerate list of blocks self.process_block["blocks"].set_value(list(self._blocks)) def get_block(self, block_name): try: return self._blocks[block_name] except KeyError: if block_name in self.process_block.remoteBlocks: return self.make_client_block(block_name) else: raise def make_client_block(self, block_name): params = ClientController.MethodMeta.prepare_input_map(mri=block_name) controller = ClientController(self, {}, params) return controller.block def get_controller(self, block_name): return self._controllers[block_name] def _handle_subscribe(self, request): """Add a new subscriber and respond with the current sub-structure state""" key = request.generate_key() assert key not in self._subscriptions, \ "Subscription on %s already exists" % (key,) self._subscriptions[key] = request self._block_state_cache.add_subscriber(request, request.endpoint) d = self._block_state_cache.walk_path(request.endpoint) self.log_debug("Initial subscription value %s", d) if request.delta: request.respond_with_delta([[[], d]]) else: request.respond_with_update(d) def _handle_unsubscribe(self, request): """Remove a subscriber and respond with success or error""" key = request.generate_key() try: subscription = self._subscriptions.pop(key) except KeyError: request.respond_with_error("No subscription found for %s" % (key, )) else: self._block_state_cache.remove_subscriber(subscription, subscription.endpoint) request.respond_with_return() def _handle_get(self, request): d = self._block_state_cache.walk_path(request.endpoint) request.respond_with_return(d)
class PandABlocksManagerController(ManagerController): def __init__(self, process, parts, params): super(PandABlocksManagerController, self).__init__(process, parts, params) # {block_name: BlockData} self._blocks_data = {} # {block_name: {field_name: Part}} self._blocks_parts = OrderedDict() # src_attr -> [dest_attr] self._listening_attrs = {} # (block_name, src_field_name) -> [dest_field_name] self._scale_offset_fields = {} # full_src_field -> [full_dest_field] self._mirrored_fields = {} # fields that need to inherit UNITS, SCALE and OFFSET from upstream self._inherit_scale = {} self._inherit_offset = {} # lut elements to be displayed or not # {fnum: {id: visible}} self._lut_elements = {} # changes left over from last time self.changes = OrderedDict() # The PandABlock client that does the comms self.client = PandABlocksClient(params.hostname, params.port, Queue) # Filled in on reset self._stop_queue = None self._poll_spawned = None def do_init(self): # start the poll loop and make block parts first to fill in our parts # before calling _set_block_children() self.start_poll_loop() super(PandABlocksManagerController, self).do_init() def start_poll_loop(self): # queue to listen for stop events if not self.client.started: self._stop_queue = Queue() if self.client.started: self.client.stop() from socket import socket if self.use_cothread: cothread = maybe_import_cothread() if cothread: from cothread.cosocket import socket self.client.start(self.spawn, socket) if not self._blocks_parts: self._make_blocks_parts() if self._poll_spawned is None: self._poll_spawned = self.spawn(self._poll_loop) def do_disable(self): super(PandABlocksManagerController, self).do_disable() self.stop_poll_loop() def do_reset(self): self.start_poll_loop() super(PandABlocksManagerController, self).do_reset() def _poll_loop(self): """At 10Hz poll for changes""" next_poll = time.time() while True: next_poll += 0.1 timeout = next_poll - time.time() if timeout < 0: timeout = 0 try: return self._stop_queue.get(timeout=timeout) except TimeoutError: # No stop, no problem pass try: self.handle_changes(self.client.get_changes()) except Exception: # TODO: should fault here? self.log.exception("Error while getting changes") def stop_poll_loop(self): if self._poll_spawned: self._stop_queue.put(None) self._poll_spawned.wait() self._poll_spawned = None if self.client.started: self.client.stop() def _make_blocks_parts(self): # {block_name_without_number: BlockData} self._blocks_data = OrderedDict() self._blocks_parts = OrderedDict() for block_rootname, block_data in self.client.get_blocks_data().items( ): block_names = [] if block_data.number == 1: block_names.append(block_rootname) else: for i in range(block_data.number): block_names.append("%s%d" % (block_rootname, i + 1)) for block_name in block_names: self._blocks_data[block_name] = block_data self._make_parts(block_name, block_data) # Handle the initial set of changes to get an initial value self.handle_changes(self.client.get_changes()) # Then once more to let bit_outs toggle back self.handle_changes({}) assert not self.changes, "There are still changes %s" % self.changes def _make_child_controller(self, parts, mri): controller = call_with_params(BasicController, self.process, parts, mri=mri) return controller def _make_corresponding_part(self, block_name, mri): part = call_with_params(ChildPart, name=block_name, mri=mri) return part def _make_parts(self, block_name, block_data): mri = "%s:%s" % (self.params.mri, block_name) # Defer creation of parts to a block maker maker = PandABlocksMaker(self.client, block_name, block_data) # Make the child controller and add it to the process controller = self._make_child_controller(maker.parts.values(), mri) self.process.add_controller(mri, controller) # Store the parts so we can update them with the poller self._blocks_parts[block_name] = maker.parts # setup param pos on a block with pos_out to inherit SCALE OFFSET UNITS pos_fields = [] pos_out_fields = [] pos_mux_inp_fields = [] for field_name, field_data in block_data.fields.items(): if field_name == "INP" and field_data.field_type == "pos_mux": pos_mux_inp_fields.append(field_name) elif field_data.field_type == "pos_out": pos_out_fields.append(field_name) elif field_data.field_subtype in ("pos", "relative_pos"): pos_fields.append(field_name) # Make sure pos_fields can get SCALE from somewhere if pos_fields: sources = pos_mux_inp_fields + pos_out_fields assert len(sources) == 1, \ "Expected one source of SCALE and OFFSET for %s, got %s" % ( pos_fields, sources) for field_name in pos_fields: self._map_scale_offset(block_name, sources[0], field_name) # Make the corresponding part for us child_part = self._make_corresponding_part(block_name, mri) self.add_part(child_part) def _map_scale_offset(self, block_name, src_field, dest_field): self._scale_offset_fields.setdefault((block_name, src_field), []).append(dest_field) if src_field == "INP": # mapping based on what it is connected to, defer return for suff in ("SCALE", "OFFSET", "UNITS"): full_src_field = "%s.%s.%s" % (block_name, src_field, suff) full_dest_field = "%s.%s.%s" % (block_name, dest_field, suff) self._mirrored_fields.setdefault(full_src_field, []).append(full_dest_field) def _set_lut_icon(self, block_name): icon_attr = self._blocks_parts[block_name]["icon"].attr with open(os.path.join(SVG_DIR, "LUT.svg")) as f: svg_text = f.read() fnum = int(self.client.get_field(block_name, "FUNC.RAW")) invis = self._get_lut_icon_elements(fnum) root = ET.fromstring(svg_text) for i in invis: # Find the first parent which has a child with id i parent = root.find('.//*[@id=%r]/..' % i) # Find the child and remove it child = parent.find('./*[@id=%r]' % i) parent.remove(child) svg_text = et_to_string(root) icon_attr.set_value(svg_text) def _get_lut_icon_elements(self, fnum): if not self._lut_elements: # Generate the lut element table # Do the general case funcs funcs = [("AND", operator.and_), ("OR", operator.or_)] for func, op in funcs: for nargs in (2, 3, 4, 5): # 2**nargs permutations for permutation in range(2**nargs): self._calc_visibility(func, op, nargs, permutation) # Add in special cases for NOT for ninp in "ABCDE": invis = {"AND", "OR", "LUT"} for inp in "ABCDE": if inp != ninp: invis.add(inp) invis.add("not%s" % inp) self._lut_elements[~LUT_CONSTANTS[ninp] & (2**32 - 1)] = invis # And catchall for LUT in 0 invis = {"AND", "OR", "NOT"} for inp in "ABCDE": invis.add("not%s" % inp) self._lut_elements[0] = invis return self._lut_elements.get(fnum, self._lut_elements[0]) def _calc_visibility(self, func, op, nargs, permutations): # Visibility dictionary defaults invis = {"AND", "OR", "LUT", "NOT"} invis.remove(func) args = [] for i, inp in enumerate("EDCBA"): # xxxxx where x is 0 or 1 # EDCBA negations = format(permutations, '05b') if (5 - i) > nargs: # invisible invis.add(inp) invis.add("not%s" % inp) else: # visible if negations[i] == "1": args.append(~LUT_CONSTANTS[inp] & (2**32 - 1)) else: invis.add("not%s" % inp) args.append(LUT_CONSTANTS[inp]) # Insert into table fnum = op(args[0], args[1]) for a in args[2:]: fnum = op(fnum, a) self._lut_elements[fnum] = invis def handle_changes(self, changes): for k, v in changes.items(): self.changes[k] = v for full_field, val in list(self.changes.items()): # If we have a mirrored field then fire off a request for dest_field in self._mirrored_fields.get(full_field, []): self.client.send("%s=%s\n" % (dest_field, val)) block_name, field_name = full_field.split(".", 1) ret = self.update_attribute(block_name, field_name, val) if ret is not None: self.changes[full_field] = ret else: self.changes.pop(full_field) # If it was LUT.FUNC then recalculate icon if block_name.startswith("LUT") and field_name == "FUNC": self._set_lut_icon(block_name) def update_attribute(self, block_name, field_name, val): ret = None if block_name not in self._blocks_parts: self.log.debug("Block %s not known", block_name) return parts = self._blocks_parts[block_name] if field_name not in parts: self.log.debug("Block %s has no field %s", block_name, field_name) return part = parts[field_name] attr = part.attr field_data = self._blocks_data[block_name].fields.get(field_name, None) if val == Exception: # TODO: set error val = None elif isinstance(attr.meta, BooleanMeta): val = bool(int(val)) is_bit_out = field_data and field_data.field_type == "bit_out" if is_bit_out and val == attr.value: # make bit_out things toggle while changing ret = val val = not val elif isinstance(attr.meta, TableMeta): val = part.table_from_list(val) # Update the value of our attribute and anyone listening attr.set_value(val) for dest_attr in self._listening_attrs.get(attr, []): dest_attr.set_value(val) # if we changed the value of a mux, update the slaved values if field_data and field_data.field_type in ("bit_mux", "pos_mux"): current_part = parts[field_name + ".CURRENT"] current_attr = current_part.attr self._update_current_attr(current_attr, val) if field_data.field_type == "pos_mux" and field_name == "INP": # all param pos fields should inherit scale and offset for dest_field_name in self._scale_offset_fields.get( (block_name, field_name), []): self._update_scale_offset_mapping(block_name, dest_field_name, val) return ret def _update_scale_offset_mapping(self, block_name, field_name, mux_val): # Find the fields that depend on this input field_data = self._blocks_data[block_name].fields.get(field_name, None) if field_data.field_subtype == "relative_pos": suffs = ("SCALE", "UNITS") else: suffs = ("SCALE", "OFFSET", "UNITS") for suff in suffs: full_src_field = "%s.%s" % (mux_val, suff) full_dest_field = "%s.%s.%s" % (block_name, field_name, suff) # Remove mirrored fields that are already in lists for field_list in self._mirrored_fields.values(): try: field_list.remove(full_dest_field) except ValueError: pass self._mirrored_fields.setdefault(full_src_field, []).append(full_dest_field) # update it to the right value if mux_val == "ZERO": value = dict(SCALE=1, OFFSET=0, UNITS="")[suff] else: mon_block_name, mon_field_name = mux_val.split(".", 1) mon_parts = self._blocks_parts[mon_block_name] src_attr = mon_parts["%s.%s" % (mon_field_name, suff)].attr value = src_attr.value self.client.send("%s=%s\n" % (full_dest_field, value)) def _update_current_attr(self, current_attr, mux_val): # Remove the old current_attr from all lists for mux_list in self._listening_attrs.values(): try: mux_list.remove(current_attr) except ValueError: pass # add it to the list of things that need to update if mux_val == "ZERO": current_attr.set_value(0) elif mux_val == "ONE": current_attr.set_value(1) else: mon_block_name, mon_field_name = mux_val.split(".", 1) mon_parts = self._blocks_parts[mon_block_name] out_attr = mon_parts[mon_field_name].attr self._listening_attrs.setdefault(out_attr, []).append(current_attr) # update it to the right value current_attr.set_value(out_attr.value)
class PandABoxPoller(Spawnable, Loggable): def __init__(self, process, control): self.set_logger_name("PandABoxPoller(%s)" % control.hostname) self.process = process self.control = control # block_name -> BlockData self._block_data = {} # block_name -> {field_name: Part} self._parts = OrderedDict() # src_attr -> [dest_attr] self._listening_attrs = {} # (block_name, src_field_name) -> [dest_field_name] self._scale_offset_fields = {} # full_src_field -> [full_dest_field] self._mirrored_fields = {} # changes left over from last time self.changes = OrderedDict() # fields that need to inherit UNITS, SCALE and OFFSET from upstream self._inherit_scale = {} self._inherit_offset = {} self.q = process.create_queue() self.add_spawn_function(self.poll_loop, self.make_default_stop_func(self.q)) def make_panda_block(self, mri, block_name, block_data, parts=None, area_detector=False): # Validate and store block_data self._store_block_data(block_name, block_data) # Defer creation of parts to a block maker maker = PandABoxBlockMaker(self.process, self.control, block_name, block_data, area_detector) # Add in any extras we are passed if parts: for part in parts: maker.parts[part.name] = part # Make a controller params = DefaultController.MethodMeta.prepare_input_map(mri=mri) controller = DefaultController( self.process, maker.parts.values(), params) block = controller.block self._parts[block_name] = maker.parts # Set the initial block_url self._set_icon_url(block_name) return block def _set_icon_url(self, block_name): icon_attr = self._parts[block_name]["icon"].attr fname = block_name.rstrip("0123456789") if fname == "LUT": # TODO: Get fname from func pass # TODO: make relative url = "http://localhost:8080/path/to/%s" % fname icon_attr.set_value(url) def _store_block_data(self, block_name, block_data): self._block_data[block_name] = block_data # setup param pos on a block with pos_out to inherit SCALE OFFSET UNITS pos_fields = [] pos_out_fields = [] pos_mux_inp_fields = [] for field_name, field_data in block_data.fields.items(): if field_name == "INP" and field_data.field_type == "pos_mux": pos_mux_inp_fields.append(field_name) elif field_data.field_type == "pos_out": pos_out_fields.append(field_name) elif field_data.field_subtype in ("pos", "relative_pos"): pos_fields.append(field_name) # Make sure pos_fields can get SCALE from somewhere if pos_fields: sources = pos_mux_inp_fields + pos_out_fields assert len(sources) == 1, \ "Expected one source of SCALE and OFFSET for %s, got %s" % ( pos_fields, sources) for field_name in pos_fields: self._map_scale_offset(block_name, sources[0], field_name) def _map_scale_offset(self, block_name, src_field, dest_field): self._scale_offset_fields.setdefault( (block_name, src_field), []).append(dest_field) if src_field == "INP": # mapping based on what it is connected to, defer return for suff in ("SCALE", "OFFSET", "UNITS"): full_src_field = "%s.%s.%s" % (block_name, src_field, suff) full_dest_field = "%s.%s.%s" % (block_name, dest_field, suff) self._mirrored_fields.setdefault(full_src_field, []).append( full_dest_field) def poll_loop(self): """At 10Hz poll for changes""" next_poll = time.time() while True: next_poll += 0.1 timeout = next_poll - time.time() if timeout < 0: timeout = 0 try: message = self.q.get(timeout=timeout) if message is Spawnable.STOP: break except queue.Empty: # No problem pass try: self.handle_changes(self.control.get_changes()) except Exception: self.log_exception("Error while getting changes") def handle_changes(self, changes): for k, v in changes.items(): self.changes[k] = v for full_field, val in list(self.changes.items()): # If we have a mirrored field then fire off a request for dest_field in self._mirrored_fields.get(full_field, []): self.control.send("%s=%s\n" % (dest_field, val)) block_name, field_name = full_field.split(".", 1) ret = self.update_attribute(block_name, field_name, val) if ret is not None: self.changes[full_field] = ret else: self.changes.pop(full_field) # If it was LUT.FUNC then recalculate icon if block_name.startswith("LUT") and field_name == "FUNC": self._set_icon_url(block_name) def update_attribute(self, block_name, field_name, val): ret = None if block_name not in self._parts: self.log_debug("Block %s not known", block_name) return parts = self._parts[block_name] if field_name not in parts: self.log_debug("Block %s has no field %s", block_name, field_name) return part = parts[field_name] attr = part.attr field_data = self._block_data[block_name].fields.get(field_name, None) if val == Exception: # TODO: set error val = None elif isinstance(attr.meta, BooleanMeta): val = bool(int(val)) is_bit_out = field_data and field_data.field_type == "bit_out" if is_bit_out and val == attr.value: # make bit_out things toggle while changing ret = val val = not val elif isinstance(attr.meta, TableMeta): val = part.table_from_list(val) # Update the value of our attribute and anyone listening attr.set_value(val) for dest_attr in self._listening_attrs.get(attr, []): dest_attr.set_value(val) # if we changed the value of a mux, update the slaved values if field_data and field_data.field_type in ("bit_mux", "pos_mux"): val_part = parts[field_name + ".VAL"] val_attr = val_part.attr self._update_val_attr(val_attr, val) if field_data.field_type == "pos_mux" and field_name == "INP": # all param pos fields should inherit scale and offset for dest_field_name in self._scale_offset_fields.get( (block_name, field_name), []): self._update_scale_offset_mapping( block_name, dest_field_name, val) return ret def _update_scale_offset_mapping(self, block_name, field_name, mux_val): # Find the fields that depend on this input field_data = self._block_data[block_name].fields.get(field_name, None) if field_data.field_subtype == "relative_pos": suffs = ("SCALE", "UNITS") else: suffs = ("SCALE", "OFFSET", "UNITS") for suff in suffs: full_src_field = "%s.%s" % (mux_val, suff) full_dest_field = "%s.%s.%s" % (block_name, field_name, suff) # Remove mirrored fields that are already in lists for field_list in self._mirrored_fields.values(): try: field_list.remove(full_dest_field) except ValueError: pass self._mirrored_fields.setdefault(full_src_field, []).append( full_dest_field) # update it to the right value if mux_val == "ZERO": value = dict(SCALE=1, OFFSET=0, UNITS="")[suff] else: mon_block_name, mon_field_name = mux_val.split(".", 1) mon_parts = self._parts[mon_block_name] src_attr = mon_parts["%s.%s" % (mon_field_name, suff)].attr value = src_attr.value self.control.send("%s=%s\n" % (full_dest_field, value)) def _update_val_attr(self, val_attr, mux_val): # Remove the old val_attr from all lists for mux_list in self._listening_attrs.values(): try: mux_list.remove(val_attr) except ValueError: pass # add it to the list of things that need to update if mux_val == "ZERO": val_attr.set_value(0) elif mux_val == "ONE": val_attr.set_value(1) else: mon_block_name, mon_field_name = mux_val.split(".", 1) mon_parts = self._parts[mon_block_name] out_attr = mon_parts[mon_field_name].attr self._listening_attrs.setdefault(out_attr, []).append(val_attr) # update it to the right value val_attr.set_value(out_attr.value)