async def shutdown(self):
"""Stops pollers and shuts down all remaining tasks."""
if not self.is_bootloader:
log.info("Stopping positioners and shutting down.")
await self.stop_trajectory()
log.debug("Stopping all pollers.")
if self.pollers:
await self.pollers.stop()
await self.chiller.stop()
await self.alerts.stop()
log.debug("Cancelling all pending tasks and shutting down.")
tasks = [
task for task in asyncio.all_tasks(loop=self.loop)
if task is not asyncio.current_task(loop=self.loop)
]
list(map(lambda task: task.cancel(), tasks))
await asyncio.gather(*tasks, return_exceptions=True)
self.loop.stop()
del self.__class__._instance[self.__class__]
async def lock(
self,
stop_trajectories: bool = True,
by: Optional[List[int]] = None,
do_warn: bool = True,
snapshot: bool = True,
):
"""Locks the `.FPS` and prevents commands to be sent.
Parameters
----------
stop_trajectories
Whether to stop trajectories when locking. This will not
clear any collided flags.
"""
self._locked = True
if do_warn:
warnings.warn("Locking the FPS.", JaegerUserWarning)
if stop_trajectories:
await self.stop_trajectory()
if by:
self.locked_by += by
await self.update_status()
if jaeger.actor_instance:
jaeger.actor_instance.write(
"e",
{
"locked": True,
"locked_by": self.locked_by
},
)
if snapshot:
if self.locked_by is not None:
highlight = self.locked_by[0]
else:
highlight = None
log.debug(
f"Saving snapshot with highlight {highlight} ({self.locked_by})"
)
filename = await self.save_snapshot(highlight=highlight)
warnings.warn(f"Snapshot for locked FPS: {filename}",
JaegerUserWarning)
def get_robot_grid(fps: FPS | None,
seed: int | None = None,
collision_buffer=None):
"""Returns a new robot grid with the destination set to the lattice position.
If an initialised instance of the FPS is available, disabled robots will be
set offline in the grid at their current positions.
"""
from kaiju.robotGrid import RobotGridCalib
if seed is None:
t = 1000 * time.time()
seed = int(int(t) % 2**32 / 1000)
kaiju_config = config["kaiju"]
ang_step = kaiju_config["ang_step"]
collision_buffer = collision_buffer or kaiju_config["collision_buffer"]
alpha0, beta0 = kaiju_config["lattice_position"]
epsilon = ang_step * kaiju_config["epsilon_factor"]
if collision_buffer < 1.5:
raise JaegerError("Invalid collision buffer < 1.5.")
log.debug(
f"Creating RobotGridCalib with stepSize={ang_step}, epsilon={epsilon}."
)
robot_grid = RobotGridCalib(stepSize=ang_step, epsilon=epsilon, seed=seed)
robot_grid.setCollisionBuffer(collision_buffer)
# TODO: This is a bit hacky. Kaiju doesn't have a collisionBuffer anymore
# as collision buffers are per robot, but I want to keep this information
# for when I dump and reload robot grids.
robot_grid.collisionBuffer = collision_buffer
for robot in robot_grid.robotDict.values():
if fps is not None and robot.id in fps and fps[
robot.id].disabled is True:
positioner = fps[robot.id]
robot.setDestinationAlphaBeta(positioner.alpha, positioner.beta)
robot.setAlphaBeta(positioner.alpha, positioner.beta)
robot.isOffline = True
else:
robot.setDestinationAlphaBeta(alpha0, beta0)
robot.setAlphaBeta(alpha0, beta0)
return robot_grid
async def start(self: T) -> T:
self.stop()
itype = self.interface_type
InterfaceClass: Type[Bus_co] = INTERFACES[itype]["class"]
self.multibus = INTERFACES[itype]["multibus"]
if not isinstance(self.channels, (list, tuple)):
self.channels = [self.channels]
for channel in self.channels:
iargs = "".join(
[f" {k}={repr(v)}" for k, v in self.interface_args.items()])
log.debug(
f"creating interface {itype}, channel={channel!r}{iargs}.")
try:
interface = InterfaceClass(channel, **self.interface_args)
result = await interface.open()
if result is False:
raise ConnectionError()
self.interfaces.append(interface)
except ConnectionResetError:
log.error(
f"connection to {itype}:{channel} failed. "
"Possibly another instance is connected to the device.")
except (socket.timeout, ConnectionError, ConnectionRefusedError,
OSError):
log.error(f"connection to {itype}:{channel} failed.")
except Exception as ee:
raise ee.__class__(
f"connection to {itype}:{channel} failed: {ee}.")
self.command_queue = asyncio.Queue()
self._command_queue_task = asyncio.create_task(
self._process_command_queue())
self.notifier = Notifier(
listeners=[self._process_reply_queue],
buses=self.interfaces,
)
self._started = True
return self
def jaeger(
ctx,
config_file,
profile,
verbose,
quiet,
ieb,
sextant,
virtual,
npositioners,
allow_host,
no_lock,
):
"""CLI for the SDSS-V focal plane system.
If called without subcommand starts the actor.
"""
if allow_host is False:
hostname = socket.getfqdn()
if hostname.endswith("apo.nmsu.edu") or hostname.endswith("lco.cl"):
if not hostname.startswith("sdss5-fps"):
raise RuntimeError(
"At the observatories jaeger must run on sdss5-fps.")
if verbose > 0 and quiet:
raise click.UsageError("--quiet and --verbose are mutually exclusive.")
if config_file:
config.load(config_file)
if no_lock:
config["fps"]["use_lock"] = False
actor_config = config.get("actor", {})
if verbose == 1:
log.sh.setLevel(logging.INFO)
actor_config["verbose"] = logging.INFO
elif verbose == 2:
log.sh.setLevel(logging.DEBUG)
actor_config["verbose"] = logging.DEBUG
elif verbose >= 3:
log.sh.setLevel(logging.DEBUG)
can_log.sh.setLevel(logging.DEBUG)
actor_config["verbose"] = logging.DEBUG
if quiet:
log.handlers.remove(log.sh)
warnings.simplefilter("ignore")
actor_config["verbose"] = 100
if sextant:
sextant_file = os.path.join(os.path.dirname(__file__),
"etc/sextant.yaml")
config["files"]["ieb_config"] = sextant_file
log.debug(
f"Using internal IEB sextant onfiguration file {sextant_file}.")
if sextant or "sextants/" in config["files"]["ieb_config"]:
enable_low_temperature = False
else:
enable_low_temperature = True
if virtual is True:
profile = "virtual"
ctx.obj = FPSWrapper(
profile,
ieb=ieb,
npositioners=npositioners,
enable_low_temperature=enable_low_temperature,
)
def send_command(
self,
command: str | int | CommandID | Command,
positioner_ids: int | List[int] | None = None,
data: Any = None,
now: bool = False,
**kwargs,
) -> Command:
"""Sends a command to the bus.
Parameters
----------
command
The ID of the command, either as the integer value, a string,
or the `.CommandID` flag. Alternatively, the `.Command` to send.
positioner_ids
The positioner IDs to command, or zero for broadcast. If `None`,
sends the command to all FPS non-disabled positioners.
data
The bytes to send. See `.Command` for details on the format.
interface
The index in the interface list for the interface to use. Only
relevant in case of a multibus interface. If `None`, the positioner
to bus map will be used.
bus
The bus within the interface to be used. Only relevant in case of
a multibus interface. If `None`, the positioner to bus map will
be used.
now
If `True`, the command is sent to the CAN network immediately,
skipping the command queue. No tracking is done for this command.
It should only be used for emergency and shutdown commands.
kwargs
Extra arguments to be passed to the command.
Returns
-------
command
The command sent to the bus. The command needs to be awaited
before it is considered done.
"""
if not isinstance(self.can, JaegerCAN) or self.can._started is False:
raise JaegerError("CAN connection not established.")
if positioner_ids is None:
positioner_ids = [p for p in self if not self[p].disabled]
if not isinstance(command, Command):
command_flag = CommandID(command)
CommandClass = command_flag.get_command_class()
assert CommandClass, "CommandClass not defined"
command = CommandClass(positioner_ids, data=data, **kwargs)
assert isinstance(command, Command)
broadcast = command.is_broadcast
pids = command.positioner_ids
if broadcast:
if any([self[pos].disabled for pos in self]) and not command.safe:
raise JaegerError("Some positioners are disabled.")
else:
if any([self[pid].disabled for pid in pids]) and not command.safe:
raise JaegerError("Some commanded positioners are disabled.")
if not broadcast and not all([p in self for p in pids]):
raise JaegerError("Some positioners are not connected.")
# Check if we are in bootloader mode.
in_boot = [p for p in pids if p != 0 and self[p].is_bootloader()]
if (broadcast and self.is_bootloader()) or (not broadcast
and any(in_boot)):
if not command.bootloader:
raise JaegerError(
f"Cannot send command {command.command_id.name!r} "
"while in bootloader mode.")
command_name = command.name
command_uid = command.command_uid
header = f"({command_name}, {command_uid}): "
if self.locked:
if command.safe:
log.debug(f"FPS is locked but {command_name} is safe.")
else:
command.cancel(silent=True)
raise FPSLockedError("FPS is locked.")
elif command.move_command and self.moving:
command.cancel(silent=True)
raise JaegerError("Cannot send move command while the "
"FPS is moving. Use FPS.stop_trajectory() "
"to stop the FPS.")
if command.status.is_done:
raise JaegerError(header + "trying to send a done command.")
if not now:
assert self.can.command_queue
self.can.command_queue.put_nowait(command)
can_log.debug(header + "added command to CAN processing queue.")
else:
self.can.send_messages(command)
can_log.debug(header + "sent command to CAN immediately.")
return command
def __post_init__(self):
super().__init__()
# Start file logger
start_file_loggers(start_log=True, start_can=False)
if config.CONFIG_FILE:
log.debug(f"Using configuration from {config.CONFIG_FILE}")
else:
warnings.warn("Unknown configuration file.", JaegerUserWarning)
self.loop = asyncio.get_event_loop()
self.loop.set_exception_handler(log.asyncio_exception_handler)
# The mapping between positioners and buses.
self.positioner_to_bus: Dict[int, Tuple[BusABC, int | None]] = {}
self.pid_lock: LockFile | None = None
self._locked = False
self.locked_by: List[int] = []
if self.ieb is None or self.ieb is True:
self.ieb = config["files"]["ieb_config"]
if isinstance(self.ieb, pathlib.Path):
self.ieb = str(self.ieb)
if isinstance(self.ieb, (str, dict)):
if isinstance(self.ieb, str):
self.ieb = os.path.expanduser(os.path.expandvars(str(
self.ieb)))
if not os.path.isabs(self.ieb):
self.ieb = os.path.join(os.path.dirname(__file__),
self.ieb)
try:
self.ieb = IEB.from_config(self.ieb)
except FileNotFoundError:
warnings.warn(
f"IEB configuration file {self.ieb} cannot be loaded.",
JaegerUserWarning,
)
self.ieb = None
elif self.ieb is False:
self.ieb = None
else:
raise ValueError(f"Invalid input value for ieb {self.ieb!r}.")
assert isinstance(self.ieb, IEB) or self.ieb is None
self.__status_event = asyncio.Event()
self.__temperature_task: asyncio.Task | None = None
self.observatory = config["observatory"]
self.alerts = AlertsBot(self)
self.chiller = ChillerBot(self)
self._configuration: BaseConfiguration | None = None
self._previous_configurations: list[BaseConfiguration] = []
self._preloaded_configuration: BaseConfiguration | None = None
# Position and status pollers
self.pollers = PollerList([
Poller(
"status",
self.update_status,
delay=config["fps"]["status_poller_delay"],
loop=self.loop,
),
Poller(
"position",
self.update_position,
delay=config["fps"]["position_poller_delay"],
loop=self.loop,
),
])
def get_path_pair(
robot_grid: Optional[RobotGridCalib] = None,
data: Optional[dict] = None,
path_generation_mode: str = "greedy",
ignore_did_fail: bool = False,
explode_deg: float = 5,
explode_positioner_id: int | None = None,
speed=None,
smooth_points=None,
path_delay=None,
collision_shrink=None,
stop_if_deadlock: bool = False,
ignore_initial_collisions: bool = False,
) -> tuple:
"""Runs ``pathGenGreedy`` and returns the to and from destination paths.
Parameters
----------
robot_grid
The Kaiju ``RobotGridCalib`` instance to decollide.
data
A dictionary of data that can be used to reload a Kaiju robot grid
using `.load_robot_grid`. This is useful if the function is being
run in an executor.
path_generation_mode
Defines the path generation algorithm to use.
Either ``greedy`` or ``explode`` or ``explode_one``.
ignore_did_fail
Generate paths even if path generation failed (i.e., deadlocks).
explode_deg
Degrees for ``pathGenExplode``.
explode_positioner_id
The positioner to explode.
speed, smooth_points, path_delay, collision_shrink
Kaiju parameters to pass to ``getPathPair``. Otherwise uses the default
configuration values.
stop_if_deadlock
If `True`, detects deadlocks early in the path and returns shorter
trajectories (at the risk of some false positive deadlocks).
ignore_initial_collisions
If `True`, does not fail if the initial state is collided. To be used
only for offsets.
Returns
-------
paths
A tuple with the to destination path, from destination path, whether path
generation failed, and the list of deadlocks
"""
if robot_grid is not None and data is not None:
raise JaegerError("robot_grid and data are mutually exclusive.")
if data is not None:
set_destination = False if path_generation_mode == "explode" else True
robot_grid = load_robot_grid(data, set_destination=set_destination)
assert robot_grid is not None
if path_generation_mode == "explode":
log.debug(f"Running pathGenExplode with explode_deg={explode_deg}.")
robot_grid.pathGenExplode(explode_deg)
deadlocks = []
elif path_generation_mode == "explode_one":
log.debug(f"Running pathGenExplodeOne with explode_deg={explode_deg}, "
f"explode_positioner_id={explode_positioner_id}.")
robot_grid.pathGenExplodeOne(explode_deg, explode_positioner_id)
deadlocks = []
else:
log.debug(
f"Running pathGenGreedy with stopIfDeadlock={stop_if_deadlock}.")
robot_grid.pathGenGreedy(
stopIfDeadlock=stop_if_deadlock,
ignoreInitialCollisions=ignore_initial_collisions,
)
# Check for deadlocks.
deadlocks = robot_grid.deadlockedRobots()
if robot_grid.didFail and ignore_did_fail is False:
return (None, None, robot_grid.didFail, deadlocks)
speed = speed or config["kaiju"]["speed"]
smooth_points = smooth_points or config["kaiju"]["smooth_points"]
collision_shrink = collision_shrink or config["kaiju"]["collision_shrink"]
path_delay = path_delay or config["kaiju"]["path_delay"]
log.debug(
f"Running getPathPair with speed={speed}, smoothPoints={smooth_points}, "
f"collisionShrink={collision_shrink}, pathDelay={path_delay}.")
to_destination, from_destination = robot_grid.getPathPair(
speed=speed,
smoothPoints=smooth_points,
collisionShrink=collision_shrink,
pathDelay=path_delay,
)
return (
to_destination,
from_destination,
robot_grid.didFail,
deadlocks,
)
async def calibrate_positioner(
fps, positioner_id, motors=True, datums=True, cogging=True
):
"""Runs the calibration process and saves it to the internal memory.
Parameters
----------
fps : .FPS
The instance of `.FPS` that will receive the trajectory.
positioner_id : int
The ID of the positioner to calibrate.
motors : bool
Whether to perform the motor calibration.
datums : bool
Whether to perform the datums calibration.
cogging : bool
Whether to perform the cogging calibration (may take more
than one hour).
Raises
------
JaegerError
If encounters a problem during the process.
Examples
--------
::
>>> fps = FPS()
>>> await fps.initialise()
# Calibrate positioner 31.
>>> await calibrate_positioner(fps, 31)
"""
log.info(f"Calibrating positioner {positioner_id}.")
if positioner_id not in fps.positioners:
raise JaegerError(f"Positioner {positioner_id} not found.")
positioner = fps[positioner_id]
if fps.pollers.running:
log.debug("Stopping pollers")
await fps.pollers.stop()
if motors:
log.info("Starting motor calibration.")
cmd = await fps.send_command(
CommandID.START_MOTOR_CALIBRATION,
positioner_ids=positioner_id,
)
if cmd.status.failed:
raise JaegerError("Motor calibration failed.")
await asyncio.sleep(1)
await positioner.wait_for_status(
[
PS.DISPLACEMENT_COMPLETED,
PS.MOTOR_ALPHA_CALIBRATED,
PS.MOTOR_BETA_CALIBRATED,
]
)
else:
log.warning("Skipping motor calibration.")
if datums:
log.info("Starting datum calibration.")
cmd = await fps.send_command(
CommandID.START_DATUM_CALIBRATION,
positioner_ids=positioner_id,
)
if cmd.status.failed:
raise JaegerError("Datum calibration failed.")
await asyncio.sleep(1)
await positioner.wait_for_status(
[
PS.DISPLACEMENT_COMPLETED,
PS.DATUM_ALPHA_CALIBRATED,
PS.DATUM_BETA_CALIBRATED,
]
)
else:
log.warning("Skipping datum calibration.")
if cogging:
log.info("Starting cogging calibration.")
cmd = await fps.send_command(
CommandID.START_COGGING_CALIBRATION,
positioner_ids=positioner_id,
)
if cmd.status.failed:
raise JaegerError("Cogging calibration failed.")
await asyncio.sleep(1)
await positioner.wait_for_status(
[PS.COGGING_ALPHA_CALIBRATED, PS.COGGING_BETA_CALIBRATED]
)
else:
log.warning("Skipping cogging calibration.")
if motors or datums or cogging:
log.info("Saving calibration.")
cmd = await fps.send_command(
CommandID.SAVE_INTERNAL_CALIBRATION,
positioner_ids=positioner_id,
)
if cmd.status.failed:
raise JaegerError("Saving calibration failed.")
log.info(f"Positioner {positioner_id} has been calibrated.")
return
async def send_trajectory(
fps: FPS,
trajectories: str | pathlib.Path | TrajectoryDataType,
use_sync_line: bool | None = None,
send_trajectory: bool = True,
start_trajectory: bool = True,
command: Optional[CluCommand[JaegerActor]] = None,
dump: bool | str = True,
extra_dump_data: dict[str, Any] = {},
) -> Trajectory:
"""Sends a set of trajectories to the positioners.
This is a low-level function that raises errors when a problem is
encountered. Most users should use `.FPS.send_trajectory` instead.
`.send_trajectory` automates `.Trajectory` by calling the different
methods in order, but provides less control.
Parameters
----------
fps
The instance of `.FPS` that will receive the trajectory.
trajectories
Either a path to a YAML file to read or a dictionary with the
trajectories. In either case the format must be a dictionary in
which the keys are the ``positioner_ids`` and each value is a
dictionary containing two keys: ``alpha`` and ``beta``, each
pointing to a list of tuples ``(position, time)``, where
``position`` is in degrees and ``time`` is in seconds.
use_sync_line
If `True`, the SYNC line will be used to synchronise the beginning of
all trajectories. Otherwise a `.START_TRAJECTORY` command will be sent
over the CAN network. If `None`, defaults to the configuration parameter.
send_trajectory
If `True`, sends the trajectory to the positioners and returns the
`.Trajectory` instance.
start_trajectory
If `True`, runs the trajectory after sending it. Otherwise, returns
the `.Trajectory` instance after sending the data. Ignored if
`send_trajectory=False`.
command
A command to which to output informational messages.
dump
Whether to dump a JSON with the trajectory to disk. If `True`,
the trajectory is stored in ``/data/logs/jaeger/trajectories/<MJD>`` with
a unique sequence for each trajectory. A string can be passed with a
custom path. The dump file is created only if `.Trajectory.start` is
called, regardless of whether the trajectory succeeds.
extra_dump_data
A dictionary with additional parameters to add to the dump JSON.
Raises
------
TrajectoryError
If encounters a problem sending the trajectory.
FPSLockedError
If the FPS is locked.
Examples
--------
::
>>> fps = FPS()
>>> await fps.initialise()
# Send a trajectory with two points for positioner 4
>>> await fps.send_trajectory({4: {'alpha': [(90, 0), (91, 3)],
'beta': [(20, 0), (23, 4)]}})
"""
traj = Trajectory(
fps,
trajectories,
dump=dump,
extra_dump_data=extra_dump_data,
)
if use_sync_line is None:
use_sync_line = config["fps"]["use_sync_line"]
if use_sync_line:
if not isinstance(fps.ieb, IEB) or fps.ieb.disabled:
raise TrajectoryError(
"IEB is not connected. Cannot use SYNC line.", traj)
if (await fps.ieb.get_device("sync").read())[0] == "closed":
raise TrajectoryError("The SYNC line is on high.", traj)
if send_trajectory is False:
return traj
msg = "Sending trajectory data."
log.debug(msg)
if command:
command.debug(msg)
try:
await traj.send()
except TrajectoryError as err:
raise TrajectoryError(
f"Something went wrong sending the trajectory: {err}",
err.trajectory,
)
finally:
if traj.dump_file:
traj.dump_trajectory()
if command:
command.debug(trajectory_dump_file=traj.dump_file)
msg = f"Trajectory sent in {traj.data_send_time:.1f} seconds."
log.debug(msg)
if command:
command.debug(msg)
log.info(
f"Expected time to complete trajectory: {traj.move_time:.2f} seconds.")
if command and traj.move_time:
command.info(move_time=round(traj.move_time, 2))
if start_trajectory is False:
return traj
msg = "Starting trajectory ..."
log.info(msg)
if command:
command.info(msg)
try:
await traj.start(use_sync_line=use_sync_line)
except TrajectoryError as err:
if traj.start_time is not None and traj.end_time is not None:
elapsed = traj.end_time - traj.start_time
elapsed_msg = f" Trajectory failed {elapsed:.2f} seconds after start."
else:
elapsed_msg = ""
raise TrajectoryError(
f"Something went wrong during the trajectory: {err}.{elapsed_msg}",
err.trajectory,
)
finally:
if traj.dump_file and command:
command.debug(trajectory_dump_file=traj.dump_file)
if command:
command.info(folded=(await fps.is_folded()))
msg = "All positioners have reached their destinations."
log.info(msg)
if command:
command.info(msg)
return traj