class Gripper(Device):
"""Base gripper class."""
state = State(default='released')
@background
@check(source='gripped', target='released')
async def release(self):
"""
release()
Release an object.
"""
await self._release()
@background
@check(source='released', target='gripped')
async def grip(self):
"""
grip()
Grip an object.
"""
await self._grip()
async def _release(self):
"""The actual release implementation."""
raise NotImplementedError
async def _grip(self):
"""The actual grip implementation."""
raise NotImplementedError
class FooDevice(BaseDevice):
state = State(default='standby')
no_write = Parameter()
foo = Quantity(q.m, check=check(source='*', target='moved'))
bar = Quantity(q.m)
test = Quantity(q.m, fset=_test_setter, fget=_test_getter)
def __init__(self, default):
super(FooDevice, self).__init__()
self._value = default
self._param_value = 0 * q.mm
self._test_value = 0 * q.mm
async def _get_foo(self):
return self._value
@transition(target='moved')
async def _set_foo(self, value):
self._value = value
async def _get_bar(self):
return 5 * q.m
param = Parameter(fget=_get_foo, fset=_set_foo)
async def _get_param(self):
return self._param_value
async def _set_param(self, value):
self._param_value = value
param = Parameter(fget=_get_param, fset=_set_param)
class Shutter(Device):
"""Shutter device class implementation."""
state = State(default="open")
def __init__(self):
super(Shutter, self).__init__()
@async
@check(source='closed', target='open')
def open(self):
"""open()
Open the shutter."""
self._open()
@async
@check(source='open', target='closed')
def close(self):
"""close()
Close the shutter."""
self._close()
def _open(self):
raise AccessorNotImplementedError
def _close(self):
raise AccessorNotImplementedError
def _abort(self):
self._close()
class Signal(Device):
"""Base device for binary signals, e.g. TTL trigger signals and similar."""
state = State(default='off')
@check(source='off', target='on')
def on(self):
"""Switch the signal on."""
self._on()
@check(source='on', target='off')
def off(self):
"""Switch the signal off."""
self._off()
@check(source='off', target='off')
def trigger(self, duration=10 * q.ms):
"""Generate a trigger signal of *duration*."""
self.on()
time.sleep(duration.to(q.s).magnitude)
self.off()
def _on(self):
"""Implementation."""
raise NotImplementedError
def _off(self):
"""Implementation."""
raise NotImplementedError
class SomeDevice(Device):
state = State(default='standby')
def __init__(self):
super(SomeDevice, self).__init__()
self.velocity = STOP_VELOCITY
self.faulty = False
self._error = False
@transition(target='standby')
async def make_transition(self):
if self.faulty:
raise RuntimeError
@check(source='standby', target='moving')
@transition(target='moving')
async def start_moving(self, velocity):
self.velocity = velocity
@check(source='*', target='standby')
@transition(target='standby')
async def stop_moving(self):
self.velocity = STOP_VELOCITY
@check(source='standby', target='standby')
@transition(immediate='moving', target='standby')
async def move_some_time(self, velocity, duration):
self.velocity = velocity
await asyncio.sleep(duration)
self.velocity = STOP_VELOCITY
@check(source=['standby', 'moving'], target='standby')
@transition(target='standby')
async def stop_no_matter_what(self):
self.velocity = STOP_VELOCITY
async def _get_state(self):
if self.faulty:
raise RuntimeError
if self._error:
return 'error'
return 'standby' if not self.velocity else 'moving'
@check(source='*', target=['standby', 'moving'])
async def set_velocity(self, velocity):
self.velocity = velocity
@check(source='*', target=['ok', 'error'])
async def make_error(self):
self._error = True
raise StateError('error')
@check(source='error', target='standby')
@transition(target='standby')
async def reset(self):
self._error = False
class DerivedDevice(BaseDevice):
state = State(default='standby')
def __init__(self):
super(DerivedDevice, self).__init__()
@check(source='standby', target='in-derived')
@transition(target='in-derived')
async def switch_derived(self):
pass
class BufferedMixin(Device):
"""A camera that stores the frames in an internal buffer"""
state = State(default='standby')
def readout_buffer(self, *args, **kwargs):
return self._readout_real(*args, **kwargs)
async def _readout_real(self, *args, **kwargs):
raise AccessorNotImplementedError
class Monochromator(Device):
"""Monochromator device which is used to filter the beam in order to
get a very narrow energy bandwidth.
.. py:attribute:: energy
Monochromatic energy in electron volts.
.. py:attribute:: wavelength
Monochromatic wavelength in meters.
"""
state = State(default="standby")
energy = Quantity(q.eV, help="Energy")
wavelength = Quantity(q.nanometer, help="Wavelength")
async def _get_energy(self):
try:
return await self._get_energy_real()
except AccessorNotImplementedError:
return wavelength_to_energy(await self._get_wavelength_real())
@check(source="standby", target="standby")
async def _set_energy(self, energy):
try:
await self._set_energy_real(energy)
except AccessorNotImplementedError:
await self._set_wavelength_real(energy_to_wavelength(energy))
async def _get_wavelength(self):
try:
return await self._get_wavelength_real()
except AccessorNotImplementedError:
return energy_to_wavelength(await self._get_energy_real())
@check(source="standby", target="standby")
async def _set_wavelength(self, wavelength):
try:
await self._set_wavelength_real(wavelength)
except AccessorNotImplementedError:
await self._set_energy_real(wavelength_to_energy(wavelength))
async def _get_energy_real(self):
raise AccessorNotImplementedError
async def _set_energy_real(self, energy):
raise AccessorNotImplementedError
async def _get_wavelength_real(self):
raise AccessorNotImplementedError
async def _set_wavelength_real(self, wavelength):
raise AccessorNotImplementedError
class AttenuatorBox(Device):
'''Attenuator box base class'''
state = State(default='standby')
def __init__(self):
super(AttenuatorBox, self).__init__()
async def _set_attenuator(self, att):
raise AccessorNotImplementedError
async def _get_attenuator(self):
raise AccessorNotImplementedError
class LensChanger(Device):
'''Lens changer base class.'''
state = State(default='standby')
def __init__(self):
super(LensChanger, self).__init__()
async def _set_objective(self, objective):
raise AccessorNotImplementedError
async def _get_objective(self):
raise AccessorNotImplementedError
class RealDevice(Device):
state = State()
def __init__(self):
super(RealDevice, self).__init__()
self._state = 'standby'
def change_state(self):
self._state = 'moved'
async def _get_state(self):
return self._state
class ElementSelector(Device):
"""ElementSelector base class"""
state = State(default='standby')
def __init__(self):
super().__init__()
async def _set_element(self, element):
raise AccessorNotImplementedError
async def _get_element(self):
raise AccessorNotImplementedError
class Acquisition(Parameterizable):
"""
An acquisition acquires data, gets it and sends it to consumers.
.. py:attribute:: producer
a callable with no arguments which returns a generator yielding data items once called.
.. py:attribute:: consumers
a list of callables with no arguments which return a coroutine consuming the data once
started, can be empty.
.. py:attribute:: acquire
a coroutine function which acquires the data, takes no arguments, can be None.
"""
state = State(default='standby')
def __init__(self, name, producer, consumers=None, acquire=None):
self.name = name
self.producer = producer
self.consumers = [] if consumers is None else consumers
# Don't bother with checking this for None later
if acquire and not asyncio.iscoroutinefunction(acquire):
raise TypeError('acquire must be a coroutine function')
self.acquire = acquire
Parameterizable.__init__(self)
@background
@check(source=['standby', 'error'], target='standby')
@transition(immediate='running', target='standby')
async def __call__(self):
"""Run the acquisition, i.e. acquire the data and connect the producer and consumers."""
LOG.debug(f"Running acquisition '{self.name}'")
consumers = self.consumers
if not consumers:
LOG.debug(f"`{self.name}' has no consumers, using null")
consumers = [null]
if self.acquire:
await self.acquire()
coros = broadcast(self.producer(), *consumers)
await asyncio.gather(*coros, return_exceptions=False)
def __repr__(self):
return "Acquisition({})".format(self.name)
class BaseDevice(Device):
state = State(default='standby')
def __init__(self):
super(BaseDevice, self).__init__()
@check(source='standby', target='in-base')
@transition(target='in-base')
async def switch_base(self):
pass
@check(source='*', target='standby')
@transition(target='standby')
async def reset(self):
pass
class Signal(Device):
"""Base device for binary signals, e.g. TTL trigger signals and similar."""
state = State(default='off')
@background
@check(source='off', target='on')
async def on(self):
"""
on()
Switch the signal on.
"""
await self._on()
@background
@check(source='on', target='off')
async def off(self):
"""
off()
Switch the signal off.
"""
await self._off()
@background
@check(source='off', target='off')
async def trigger(self, duration=10 * q.ms):
"""
trigger(duration=10*q.ms)
Generate a trigger signal of *duration*.
"""
await self.on()
await asyncio.sleep(duration.to(q.s).magnitude)
await self.off()
async def _on(self):
"""Implementation."""
raise NotImplementedError
async def _off(self):
"""Implementation."""
raise NotImplementedError
class RotationMotor(_PositionMixin):
"""
One-dimensional rotational motor.
.. attribute:: position
Position of the motor in angular units.
"""
async def _get_state(self):
raise NotImplementedError
state = State(default='standby')
position = Quantity(q.deg,
help="Angular position",
check=check(source=['hard-limit', 'standby'],
target=['hard-limit', 'standby']))
def __init__(self):
super(RotationMotor, self).__init__()
class ContinuousRotationMotor(RotationMotor, _VelocityMixin):
"""
One-dimensional rotational motor with adjustable velocity.
.. attribute:: velocity
Current velocity in angle per time unit.
"""
def __init__(self):
super(ContinuousRotationMotor, self).__init__()
async def _get_state(self):
raise NotImplementedError
state = State(default='standby')
velocity = Quantity(q.deg / q.s,
help="Angular velocity",
check=check(source=['hard-limit', 'standby', 'moving'],
target=['moving', 'standby']))
class LinearMotor(_PositionMixin):
"""
One-dimensional linear motor.
.. attribute:: position
Position of the motor in length units.
"""
def __init__(self):
super(LinearMotor, self).__init__()
def _get_state(self):
raise NotImplementedError
state = State(default='standby')
position = Quantity(q.mm,
help="Position",
check=check(source=['hard-limit', 'standby'],
target=['hard-limit', 'standby']))
class Pump(Device):
"""A pumping device."""
state = State(default='standby')
flow_rate = Quantity(q.l / q.s, help="Flow rate")
def __init__(self):
super(Pump, self).__init__()
@async
@check(source='standby', target='pumping')
def start(self):
"""
start()
Start pumping.
"""
self._start()
@async
@check(source='pumping', target='standby')
def stop(self):
"""
stop()
Stop pumping.
"""
self._stop()
def _get_flow_rate(self):
raise AccessorNotImplementedError
def _set_flow_rate(self, flow_rate):
raise AccessorNotImplementedError
def _start(self):
raise NotImplementedError
def _stop(self):
raise NotImplementedError
class ContinuousLinearMotor(LinearMotor):
"""
One-dimensional linear motor with adjustable velocity.
.. attribute:: velocity
Current velocity in length per time unit.
"""
def __init__(self):
super(ContinuousLinearMotor, self).__init__()
def _get_state(self):
raise NotImplementedError
def _cancel_velocity(self):
self._abort()
state = State(default='standby')
velocity = Quantity(q.mm / q.s,
help="Linear velocity",
check=check(source=['hard-limit', 'standby', 'moving'],
target=['moving', 'standby']))
class Camera(Device):
"""Base class for remotely controllable cameras.
.. py:attribute:: frame-rate
Frame rate of acquisition in q.count per time unit.
"""
trigger_sources = Bunch(['AUTO', 'SOFTWARE', 'EXTERNAL'])
trigger_types = Bunch(['EDGE', 'LEVEL'])
state = State(default='standby')
frame_rate = Quantity(1 / q.second, help="Frame frequency")
trigger_source = Parameter(help="Trigger source")
def __init__(self):
super(Camera, self).__init__()
self.convert = identity
@background
@check(source='standby', target='recording')
async def start_recording(self):
"""
start_recording()
Start recording frames.
"""
await self._record_real()
@background
@check(source='recording', target='standby')
async def stop_recording(self):
"""
stop_recording()
Stop recording frames.
"""
await self._stop_real()
@contextlib.asynccontextmanager
async def recording(self):
"""
recording()
A context manager for starting and stopping the camera.
In general it is used with the ``async with`` keyword like this::
async with camera.recording():
frame = await camera.grab()
"""
await self.start_recording()
try:
yield
finally:
LOG.log(AIODEBUG, 'stop recording in recording()')
await self.stop_recording()
@background
async def trigger(self):
"""Trigger a frame if possible."""
await self._trigger_real()
@background
async def grab(self):
"""Return a NumPy array with data of the current frame."""
return self.convert(await self._grab_real())
async def stream(self):
"""
stream()
Grab frames continuously yield them. This is an async generator.
"""
await self.set_trigger_source(self.trigger_sources.AUTO)
await self.start_recording()
while await self.get_state() == 'recording':
yield await self.grab()
async def _get_trigger_source(self):
raise AccessorNotImplementedError
async def _set_trigger_source(self, source):
raise AccessorNotImplementedError
async def _record_real(self):
raise AccessorNotImplementedError
async def _stop_real(self):
raise AccessorNotImplementedError
async def _trigger_real(self):
raise AccessorNotImplementedError
async def _grab_real(self):
raise AccessorNotImplementedError
class Camera(Device):
"""Base class for remotely controllable cameras.
.. py:attribute:: frame-rate
Frame rate of acquisition in q.count per time unit.
"""
trigger_modes = Bunch(['AUTO', 'SOFTWARE', 'EXTERNAL'])
state = State(default='standby')
frame_rate = Quantity(1.0 / q.second, help="Frame frequency")
trigger_mode = Parameter(help="Trigger mode")
def __init__(self):
super(Camera, self).__init__()
self.convert = identity
@check(source='standby', target='recording')
def start_recording(self):
"""Start recording frames."""
self._record_real()
@check(source='recording', target='standby')
def stop_recording(self):
"""Stop recording frames."""
self._stop_real()
@contextlib.contextmanager
def recording(self):
"""
A context manager for starting and stopping the camera.
In general it is used with the ``with`` keyword like this::
with camera.recording():
frame = camera.grab()
"""
self.start_recording()
try:
yield
finally:
self.stop_recording()
def trigger(self):
"""Trigger a frame if possible."""
self._trigger_real()
def grab(self):
"""Return a NumPy array with data of the current frame."""
return self.convert(self._grab_real())
@async
def grab_async(self):
return self.grab()
@async
def stream(self, consumer):
"""Grab frames continuously and send them to *consumer*, which
is a coroutine.
"""
self.trigger_mode = self.trigger_modes.AUTO
self.start_recording()
while self.state == 'recording':
consumer.send(self.grab())
def _get_trigger_mode(self):
raise AccessorNotImplementedError
def _set_trigger_mode(self, mode):
raise AccessorNotImplementedError
def _record_real(self):
raise AccessorNotImplementedError
def _stop_real(self):
raise AccessorNotImplementedError
def _trigger_real(self):
raise AccessorNotImplementedError
def _grab_real(self):
raise AccessorNotImplementedError
class DummyDevice(Device):
"""A dummy device."""
position = Quantity(unit=q.mm)
sleep_time = Quantity(unit=q.s)
# Value with a check-decorated setter before it is bound to instance, so still a function
value = Parameter()
# value with check wrapping the bound method
cvalue = Parameter(check=check(source='*', target='standby'))
# Value set elsewhere
evalue = Parameter(fset=set_evalue,
fget=get_evalue,
check=check(source='*', target='standby'))
slow = Parameter()
state = State(default='standby')
def __init__(self, slow=None):
super(DummyDevice, self).__init__()
self._position = 1 * q.mm
self._value = 0
self._slow = slow
self._sleep_time = 0.5 * q.s
async def _get_sleep_time(self):
return self._sleep_time
async def _set_sleep_time(self, value):
self._sleep_time = value
async def _get_position(self):
return self._position
async def _set_position(self, value):
self._position = value
async def _get_slow(self):
try:
LOG.log(AIODEBUG, 'get slow start %s', self._slow)
await asyncio.sleep((await self.get_sleep_time()).magnitude)
LOG.log(AIODEBUG, 'get slow finish %s', self._slow)
return self._slow
except asyncio.CancelledError:
LOG.log(AIODEBUG, 'get slow cancelled %s', self._slow)
raise
except KeyboardInterrupt:
# do not scream
LOG.debug("KeyboardInterrupt caught while getting")
async def _set_slow(self, value):
try:
LOG.log(AIODEBUG, 'set slow start %s', value)
await asyncio.sleep((await self.get_sleep_time()).magnitude)
LOG.log(AIODEBUG, 'set slow finish %s', value)
self._slow = value
except asyncio.CancelledError:
LOG.log(AIODEBUG, 'set slow cancelled %s', value)
raise
async def _get_value(self):
"""Get the real value."""
return self._value
async def _get_target_value(self):
"""Get the real value."""
return self._value + 1
@check(source='standby', target=['standby', 'hard-limit'])
@transition(immediate='moving', target='standby')
async def _set_value(self, value):
"""The real value setter."""
self._value = value
async def _get_cvalue(self):
"""The real value setter."""
return self._value
async def _set_cvalue(self, value):
"""The real value setter."""
self._value = value
@background
async def do_nothing(self, value=None):
"""Do nothing. For testing task canellation."""
await self._do_nothing(value=value)
async def _do_nothing(self, value=None):
LOG.log(AIODEBUG, f'Start doing nothing: {value}')
try:
await asyncio.sleep(1)
LOG.log(AIODEBUG, f'Stop doing nothing: {value}')
return value
except asyncio.CancelledError:
LOG.log(AIODEBUG, f'Doing nothing cancelled: {value}')
raise
async def _emergency_stop(self):
LOG.debug('Emergency stop on a dummy device')
await asyncio.sleep(0.5)
self._state_value = 'aborted'
class BadDevice(Device):
state = State()
class CustomizedDevice(Device):
async def state_getter(self):
return 'custom'
state = State(fget=state_getter)
class ImplicitSoftwareDevice(Device):
state = State(default='standby')
class XRayTube(Device):
"""
A base x-ray tube class.
"""
voltage = Quantity(q.kV)
current = Quantity(q.uA)
power = Quantity(q.W)
state = State(default='off')
def __init__(self):
super(XRayTube, self).__init__()
async def _get_state(self):
raise AccessorNotImplementedError
async def _get_voltage(self):
raise AccessorNotImplementedError
async def _set_voltage(self, voltage):
raise AccessorNotImplementedError
async def _get_current(self):
raise AccessorNotImplementedError
async def _set_current(self, current):
raise AccessorNotImplementedError
async def _get_power(self):
return (await self.get_voltage() * await self.get_current()).to(q.W)
@background
@check(source='off', target='on')
async def on(self):
"""
on()
Enables the x-ray tube.
"""
await self._on()
@background
@check(source='on', target='off')
async def off(self):
"""
off()
Disables the x-ray tube.
"""
await self._off()
async def _on(self):
"""
Implementation of on().
"""
raise NotImplementedError
async def _off(self):
"""
Implementation of off().
"""
raise NotImplementedError
class Experiment(Parameterizable):
"""
Experiment base class. An experiment can be run multiple times with the output data and log
stored on disk. You can prepare every run by :meth:`.prepare` and finsh the run by
:meth:`.finish`. These methods do nothing by default. They can be useful e.g. if you need to
reinitialize some experiment parts or want to attach some logging output.
.. py:attribute:: acquisitions
:noindex:
A list of acquisitions this experiment is composed of
.. py:attribute:: walker
A :class:`concert.storage.Walker` descends to a data set specific for every run if given
.. py:attribute:: separate_scans
If True, *walker* does not descend to data sets based on specific runs
.. py:attribute:: name_fmt
Since experiment can be run multiple times each iteration will have a separate entry
on the disk. The entry consists of a name and a number of the current iteration, so the
parameter is a formattable string.
"""
iteration = Parameter()
separate_scans = Parameter()
name_fmt = Parameter()
state = State(default='standby')
log_level = Selection(['critical', 'error', 'warning', 'info', 'debug'])
def __init__(self,
acquisitions,
walker=None,
separate_scans=True,
name_fmt='scan_{:>04}'):
self._acquisitions = []
for acquisition in acquisitions:
self.add(acquisition)
self.walker = walker
self._separate_scans = separate_scans
self._name_fmt = name_fmt
self._iteration = 1
self.log = LOG
Parameterizable.__init__(self)
if separate_scans and walker:
# The data is not supposed to be overwritten, so find an iteration which
# hasn't been used yet
while self.walker.exists(self._name_fmt.format(self._iteration)):
self._iteration += 1
async def _get_iteration(self):
return self._iteration
async def _set_iteration(self, iteration):
self._iteration = iteration
async def _get_separate_scans(self):
return self._separate_scans
async def _set_separate_scans(self, separate_scans):
self._separate_scans = separate_scans
async def _get_name_fmt(self):
return self._name_fmt
async def _set_name_fmt(self, fmt):
self._name_fmt = fmt
async def _get_log_level(self):
return logging.getLevelName(self.log.getEffectiveLevel()).lower()
async def _set_log_level(self, level):
self.log.setLevel(level.upper())
async def prepare(self):
"""Gets executed before every experiment run."""
pass
async def finish(self):
"""Gets executed after every experiment run."""
pass
@property
def acquisitions(self):
"""Acquisitions is a read-only attribute which has to be manipulated by explicit methods
provided by this class.
"""
return tuple(self._acquisitions)
def add(self, acquisition):
"""
Add *acquisition* to the acquisition list and make it accessible as
an attribute::
frames = Acquisition(...)
experiment.add(frames)
# This is possible
experiment.frames
"""
self._acquisitions.append(acquisition)
setattr(self, acquisition.name, acquisition)
def remove(self, acquisition):
"""Remove *acquisition* from experiment."""
self._acquisitions.remove(acquisition)
delattr(self, acquisition.name)
def swap(self, first, second):
"""
Swap acquisition *first* with *second*. If there are more occurences
of either of them then the ones which are found first in the acquisitions
list are swapped.
"""
if first not in self._acquisitions or second not in self._acquisitions:
raise ValueError(
"Both acquisitions must be part of the experiment")
first_index = self._acquisitions.index(first)
second_index = self._acquisitions.index(second)
self._acquisitions[first_index] = second
self._acquisitions[second_index] = first
def get_acquisition(self, name):
"""
Get acquisition by its *name*. In case there are more like it, the first
one is returned.
"""
for acq in self._acquisitions:
if acq.name == name:
return acq
raise ExperimentError(
"Acquisition with name `{}' not found".format(name))
async def acquire(self):
"""
Acquire data by running the acquisitions. This is the method which implements
the data acquisition and should be overriden if more functionality is required,
unlike :meth:`~.Experiment.run`.
"""
for acq in wrap_iterable(self._acquisitions):
if await self.get_state() != 'running':
break
await acq()
@background
@check(source=['standby', 'error'], target='standby')
@transition(immediate='running', target='standby')
async def run(self):
start_time = time.time()
handler = None
iteration = await self.get_iteration()
separate_scans = await self.get_separate_scans()
try:
if self.walker:
if separate_scans:
self.walker.descend(
(await self.get_name_fmt()).format(iteration))
if os.path.exists(self.walker.current):
# We might have a dummy walker which doesn't create the directory
handler = logging.FileHandler(
os.path.join(self.walker.current, 'experiment.log'))
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s '
'- %(message)s')
handler.setFormatter(formatter)
self.log.addHandler(handler)
self.log.info(await self.info_table)
LOG.debug('Experiment iteration %d start', iteration)
await self.prepare()
await self.acquire()
except asyncio.CancelledError:
# This is normal, no special state needed -> standby
LOG.warn('Experiment cancelled')
except Exception as e:
# Something bad happened and we can't know what, so set the state to error
LOG.warn(f"Error `{e}' while running experiment")
raise StateError('error', msg=str(e))
except KeyboardInterrupt:
LOG.warn('Experiment cancelled by keyboard interrupt')
self._state_value = 'standby'
raise
finally:
try:
await self.finish()
except Exception as e:
LOG.warn(f"Error `{e}' while finalizing experiment")
raise StateError('error', msg=str(e))
finally:
if separate_scans and self.walker:
self.walker.ascend()
LOG.debug('Experiment iteration %d duration: %.2f s',
iteration,
time.time() - start_time)
if handler:
handler.close()
self.log.removeHandler(handler)
await self.set_iteration(iteration + 1)
class GeneralBackprojectManager(Parameterizable):
"""Manage 3D recnstruction by back projection. The manager stores darks, flats and projections
and spreads them to back projection workers in as many batches as needed in order not to
overflow GPU memory.
.. py:attribute:: args
:class:`.GeneralBackprojectArgs` instance with arguments for reconstruction
.. py:attribute:: average_normalization
if False, use only one dark and flat image from their streams, otherwise average all of them
.. py:attribute:: regions
User defined regions (batches) for reconstruction, if None, batches are determined
automatically
.. py:attribute:: copy_inputs
if True copy images before they are inserted into UFO
"""
state = State(default='standby')
def __init__(self, args, average_normalization=True, regions=None, copy_inputs=False):
super().__init__()
self.args = args
self.regions = regions
self.copy_inputs = copy_inputs
self.projections = None
self._resources = []
self.volume = None
self.average_normalization = average_normalization
self.darks = []
self.flats = []
self._darks_condition = asyncio.Condition()
self._flats_condition = asyncio.Condition()
# Conditions for darks and flats need to signal that no more images will come and *done* is
# how they do it
self._darks_condition.done = False
self._flats_condition.done = False
self._num_received_projections = 0
self._num_processed_projections = 0
self._producer_condition = asyncio.Condition()
self._processing_task = None
self._regions = None
@property
def num_received_projections(self):
"""Number of received projections."""
return self._num_received_projections
@property
def num_processed_projections(self):
"""Number of projections sent to backprojectors."""
return self._num_processed_projections
def _update(self):
"""Update the regions and volume sizes based on changed args or region."""
st = time.perf_counter()
x_region, y_region, z_region = get_reconstruction_regions(self.args)
if not self._resources:
self._resources = [Ufo.Resources()]
gpus = np.array(self._resources[0].get_gpu_nodes())
gpu_indices = np.array(self.args.gpus or list(range(len(gpus))))
if min(gpu_indices) < 0 or max(gpu_indices) > len(gpus) - 1:
raise ValueError('--gpus contains invalid indices')
gpus = gpus[gpu_indices]
if self.regions is None:
self._regions = make_runs(gpus, gpu_indices, x_region, y_region, z_region,
DTYPE_CL_SIZE[self.args.store_type],
slices_per_device=self.args.slices_per_device,
slice_memory_coeff=self.args.slice_memory_coeff,
data_splitting_policy=self.args.data_splitting_policy,
num_gpu_threads=self.args.num_gpu_threads)
else:
self._regions = self.regions
offset = 0
for batch in self._regions:
for i, region in batch:
if len(self._resources) < len(batch):
self._resources.append(Ufo.Resources())
offset += len(np.arange(*region))
if self.args.slice_metric:
shape = (offset,)
else:
shape = (offset, len(np.arange(*y_region)), len(np.arange(*x_region)))
if self.volume is None or shape != self.volume.shape:
self.volume = np.empty(shape, dtype=np.float32)
LOG.log(PERFDEBUG, 'Backprojector manager update duration: %g s', time.perf_counter() - st)
async def _produce(self):
"""Produce projections for backprojectors."""
for i in range(self.args.number):
async with self._producer_condition:
await self._producer_condition.wait_for(lambda: self._num_received_projections > i)
yield self.projections[i]
self._num_processed_projections = i + 1
async def _consume(self, offset, producer):
"""Consume slices from individual backprojectors."""
i = 0
async for item in producer:
self.volume[offset + i] = item
i += 1
def find_parameters(self, parameters, projections=None, metrics=('sag',), regions=None,
iterations=1, fwhm=0, minimize=(True,), z=None, method='powell',
method_options=None, guesses=None, bounds=None, store=True):
"""Find reconstruction parameters. *parameters* (see
:attr:`.GeneralBackprojectArgs.z_parameters`) are the names of the parameters which should
be found, *projections* are the input data and if not specified, the ones from last
reconstruction are used. *z* specifies the height in which the parameter is looked for. If
*store* is True, the found parameter values are stored in the reconstruction arguments.
Optimization is done either brute-force if *regions* are not specified or one of the scipy
minimization methods is used, see below.
If *regions* are specified, they are reconstructed for the corresponding parameters and a
metric from *metrics* list is applied. Thus, first parameter in *parameters* is
reconstructed within the first region in *regions* and the first metric (see
:attr:`.GeneralBackprojectArgs.slice_metrics`) in *metrics* is applied and so on. If
*metrics* is of length 1 then it is applied to all parameters. *minimize* is a tuple
specifying whether each parameter in the list should be minimized (True) or maximized
(False). After every parameter is processed, the parameter optimization result is stored and
the next parameter is optimized in such a way, that the result of the optimization of the
previous parameter already takes place. *iterations* specifies how many times are all the
parameters reconstructed. *fwhm* specifies the full width half maximum of the gaussian
window used to filter out the low frequencies in the metric, which is useful when the region
for a metric is large. If the *fwhm* is specified, the region must be at least 4 * fwhm
large. If *fwhm* is 0 no filtering is done.
If *regions* is not specified, :func:`scipy.minimize` is used to find the parameter, where
the optimization method is given by the *method* parameter, *method_options* are passed as
*options* to the minimize function and *guesses* are initial guesses in the order of the
*parameters* list. If *bounds* are given, they represent the domains where to look for
parameters, they are (min, max) tuples, also in the order of the *parameters* list. See
documentation of :func:`scipy.minimize` for the list of minimization methods which support
bounds specification. In this approach only the first in *metrics* is taken into account
because the optimization happens on all parameters simultaneously, the same holds for
*minimize*.
"""
if projections is None:
if self.projections is None:
raise GeneralBackprojectManagerError('*projections* must be specified if no '
' reconstructions have been done yet')
projections = self.projections
orig_args = self.args
self.args = copy.deepcopy(self.args)
if regions is None:
# No region specified, do a real optimization on the parameters vector
from scipy import optimize
def score(vector):
for (parameter, value) in zip(parameters, vector):
setattr(self.args, parameter.replace('-', '_'), [value])
run_in_loop(self.backproject(async_generate(projections)))
result = sgn * self.volume[0]
LOG.info('Optimization vector: %s, result: %g', vector, result)
return result
self.args.z_parameter = 'z'
z = z or 0
self.args.region = [z, z + 1, 1.]
self.args.slice_metric = metrics[0]
sgn = 1 if minimize[0] else -1
if guesses is None:
guesses = []
for parameter in parameters:
if parameter == 'center-position-x':
guesses.append(self.args.width / 2)
else:
guesses.append(0.)
LOG.info('Guesses: %s', guesses)
result = optimize.minimize(score, guesses, method=method, bounds=bounds,
options=method_options)
LOG.info('%s', result.message)
result = result.x
else:
# Regions specified, reconstruct given regions for given parameters and simply search
# for extrema of the given metrics
self.args.z = z or 0
if fwhm:
for region in regions:
if len(np.arange(*region)) < 4 * fwhm:
raise ValueError('All regions must be at least 4 * fwhm large '
'when fwhm is specified')
result = []
if len(metrics) == 1:
metrics = metrics * len(parameters)
if len(minimize) == 1:
minimize = minimize * len(parameters)
for i in range(iterations):
for (parameter, region, metric, minim) in zip(parameters, regions,
metrics, minimize):
self.args.slice_metric = metric
self.args.z_parameter = parameter
self.args.region = region
run_in_loop(self.backproject(async_generate(projections)))
sgn = 1 if minim else -1
values = self.volume
if fwhm:
values = filter_low_frequencies(values, fwhm=fwhm)[2 * int(fwhm):
-2 * int(fwhm)]
param_result = (np.argmin(sgn * values) + 2 * fwhm) * region[2] + region[0]
setattr(self.args, parameter.replace('-', '_'), [param_result])
if i == iterations - 1:
result.append(param_result)
LOG.info('Optimizing %s, region: %s, metric: %s, minimize: %s, result: %g',
parameter, region, metric, minim, param_result)
LOG.info('Optimization result: %s', result)
if store:
for (parameter, value) in zip(parameters, result):
setattr(orig_args, parameter.replace('-', '_'), [value])
self.args = orig_args
return result
async def _copy_projections(self, producer):
def copy_projection(projection):
self.projections[self._num_received_projections] = projection
async for projection in producer:
if self._num_received_projections == 0:
if not self.args.width:
(self.args.height, self.args.width) = projection.shape
elif (self.args.height, self.args.width) != projection.shape:
raise GeneralBackprojectManagerError('Projections have different '
'shape from normalization images')
in_shape = (self.args.number,) + projection.shape
if (self.projections is None or in_shape != self.projections.shape
or projection.dtype != self.projections.dtype):
self.projections = np.empty(in_shape, dtype=projection.dtype)
if self._num_received_projections < self.args.number:
await run_in_executor(copy_projection, projection)
self._num_received_projections += 1
async with self._producer_condition:
self._producer_condition.notify_all()
async def _distribute(self, reuse_normalization=False, do_normalization=False):
"""Distribute projections to multiple batches which may run on multiple GPUs. If
*reuse_normalization* is True just feed the workers with the stored darks and flats, do not
expect new streams. If *do_normalization* is True send darks and flats to backprojectors.
"""
LOG.debug('Processing start')
st = time.perf_counter()
self._update()
async def start_one(batch_index, region_index):
"""Start one backprojector with a specific GPU ID in a separate thread."""
# first slice offset
offset = 0
for i in range(batch_index):
for j, region in self._regions[i]:
offset += len(np.arange(*region))
batch = self._regions[batch_index]
offset += sum([len(np.arange(*reg)) for j, reg in batch[:region_index]])
gpu_index, region = self._regions[batch_index][region_index]
bp = GeneralBackproject(self.args,
resources=self._resources[region_index],
gpu_index=gpu_index,
do_normalization=do_normalization,
region=region,
copy_inputs=self.copy_inputs)
if do_normalization:
if reuse_normalization:
darks = self.darks if self.average_normalization else self.darks[:1]
flats = self.flats if self.average_normalization else self.flats[:1]
darks_generator = async_generate(darks)
flats_generator = async_generate(flats)
else:
darks_generator = self._produce_normalization(self.darks,
self._darks_condition)
flats_generator = self._produce_normalization(self.flats,
self._flats_condition)
await asyncio.gather(bp.average_darks(darks_generator),
bp.average_flats(flats_generator))
await self._consume(offset, bp(self._produce()))
LOG.debug('Reconstructing %d batches: %s', len(self._regions), self._regions)
for batch_index in range(len(self._regions)):
coros = []
for region_index in range(len(self._regions[batch_index])):
coros.append(start_one(batch_index, region_index))
await asyncio.gather(*coros)
# Process results
duration = time.perf_counter() - st
LOG.log(PERFDEBUG, 'Backprojectors duration: %.2f s', duration)
in_size = self.projections.nbytes / 2 ** 20
out_size = self.volume.nbytes / 2 ** 20
LOG.log(PERFDEBUG, 'Input size: %g GB, output size: %g GB', in_size / 1024,
out_size / 1024)
LOG.log(PERFDEBUG, 'Performance: %.2f GUPS (In: %.2f MB/s, out: %.2f MB/s)',
self.volume.size * self.projections.shape[0] * 1e-9 / duration,
in_size / duration, out_size / duration)
async def _produce_normalization(self, images, condition):
i = 0
while True:
async with condition:
# Either there is an image to consume, or last image has come and we have consumed
# all of them
await condition.wait_for(lambda: len(images) > i
or condition.done and i == len(images))
if condition.done and i == len(images):
break
yield images[i]
i += 1
async def _copy_normalization(self, producer, images, condition):
if not self._processing_task:
# When we start a new backprojection job we need to clear both darks and flats because
# we don't know which acquisition will be done first. This must be done here so that the
# first image is not lost, which would be the case if we called it where the actual
# _distribute is called.
self.reset()
try:
async for image in producer:
if not self.args.width:
(self.args.height, self.args.width) = image.shape
if not self._processing_task:
# Start averaging before projection stream starts
self._processing_task = start(self._distribute(reuse_normalization=False,
do_normalization=True))
async with condition:
images.append(image)
condition.notify_all()
if not self.average_normalization:
# We store only one
break
# Last image has arrived, signal by *done* and notify waiting coros
async with condition:
condition.done = True
condition.notify_all()
except asyncio.CancelledError:
if self._processing_task and not self._processing_task.cancelled():
self._processing_task.cancel()
self._processing_task = None
raise
def reset(self):
"""Reset state, clearing all pre-processing steps but keep projections and slices intact."""
LOG.debug('Reset')
if self._processing_task and not self._processing_task.done():
raise GeneralBackprojectManagerError('Reconstruction running, cannot reset')
del self.darks[:]
del self.flats[:]
self._darks_condition.done = False
self._flats_condition.done = False
self._processing_task = None
self._num_received_projections = self._num_processed_projections = 0
@background
@check(source='standby', target='*')
@transition(immediate='running', target='standby')
async def update_darks(self, producer):
"""Get new darks from *producer*. Immediately start the reconstruction so that averaging
starts.
"""
await self._copy_normalization(producer, self.darks, self._darks_condition)
@background
@check(source='standby', target='*')
@transition(immediate='running', target='standby')
async def update_flats(self, producer):
"""Get new flats from *producer*. Immediately start the reconstruction so that averaging
starts.
"""
await self._copy_normalization(producer, self.flats, self._flats_condition)
@background
@check(source='standby', target='*')
async def backproject(self, producer):
"""Backproject projections from *producer*."""
self._state_value = 'running'
self._num_received_projections = self._num_processed_projections = 0
try:
if not self._processing_task:
self._processing_task = start(self._distribute(reuse_normalization=True,
do_normalization=self.darks
and self.flats))
await asyncio.gather(self._copy_projections(producer), self._processing_task)
except asyncio.CancelledError:
if self._processing_task and not self._processing_task.cancelled():
self._processing_task.cancel()
finally:
self._processing_task = None
self._state_value = 'standby'