def test_ar_export(self):
filename = "test-ar.csv"
# Create AR data
md = {
model.MD_SW_VERSION: "1.0-test",
model.MD_HW_NAME: "fake ccd",
model.MD_DESCRIPTION: "AR",
model.MD_ACQ_TYPE: model.MD_AT_AR,
model.MD_ACQ_DATE: time.time(),
model.MD_BPP: 12,
model.MD_BINNING: (1, 1), # px, px
model.MD_SENSOR_PIXEL_SIZE: (13e-6, 13e-6), # m/px
model.MD_PIXEL_SIZE: (2e-5, 2e-5), # m/px
model.MD_POS: (1.2e-3, -30e-3), # m
model.MD_EXP_TIME: 1.2, # s
model.MD_AR_POLE: (253.1, 65.1),
model.MD_LENS_MAG: 0.4, # ratio
}
md0 = dict(md)
data0 = model.DataArray(1500 + numpy.zeros((512, 512), dtype=numpy.uint16), md0)
md1 = dict(md)
md1[model.MD_POS] = (1.5e-3, -30e-3)
md1[model.MD_BASELINE] = 300 # AR background should take this into account
data1 = model.DataArray(3345 + numpy.zeros((512, 512), dtype=numpy.uint16), md1)
# Create AR stream
ars = stream.StaticARStream("test", [data0, data1])
ars.point.value = md1[model.MD_POS]
# Convert to exportable RGB image
exdata = img.ar_to_export_data([ars], raw=True)
# shape = raw data + theta/phi axes values
self.assertGreater(exdata.shape[0], 50)
self.assertGreater(exdata.shape[1], 50)
# Save into a CSV file
exporter = dataio.get_converter("CSV")
exporter.export(filename, exdata)
st = os.stat(filename) # this test also that the file is created
self.assertGreater(st.st_size, 100)
# clean up
try:
os.remove(filename)
except Exception:
pass
def data_to_static_streams(data):
""" Split the given data into static streams
Args:
data: (list of DataArrays or DataArrayShadows) Data to be split
Returns:
(list) A list of Stream instances
"""
result_streams = []
# AR data is special => all merged in one big stream
ar_data = []
logging.debug("Processing %s data arrays", len(data))
# Add each data as a stream of the correct type
for d in data:
acqtype = d.metadata.get(model.MD_ACQ_TYPE)
# Hack for not displaying Anchor region data
# TODO: store and use acquisition type with MD_ACQ_TYPE?
if acqtype == model.MD_AT_ANCHOR or d.metadata.get(
model.MD_DESCRIPTION) == "Anchor region":
continue
dims = d.metadata.get(model.MD_DIMS, "CTZYX"[-d.ndim::])
pxs = d.metadata.get(model.MD_PIXEL_SIZE)
ti = dims.find("T") # -1 if not found
ci = dims.find("C") # -1 if not found
if ((MD_WL_LIST in d.metadata and (ci >= 0 and d.shape[ci] > 1))
or (ci >= 0 and d.shape[ci] >= 5
) # No metadata, but looks like a spectrum
):
if MD_TIME_LIST in d.metadata and (ti >= 0 and d.shape[ti] > 1):
# Streak camera data. Create a temporal spectrum
name = d.metadata.get(model.MD_DESCRIPTION,
"Temporal Spectrum")
klass = stream.StaticSpectrumStream
else:
# Spectrum: either it's obvious according to metadata, or no metadata
# but lots of wavelengths, so no other way to display
# Note: this is also temporal spectrum data acquired with mirror and focus mode (so no time/wl info)
# TODO: maybe drop the check for TIME_LIST and WL_LIST
name = d.metadata.get(model.MD_DESCRIPTION, "Spectrum")
klass = stream.StaticSpectrumStream
elif ((MD_TIME_LIST in d.metadata and ti >= 0 and d.shape[ti] > 1)
or (ti >= 5 and d.shape[ti] >= 5)):
# Time data (with XY)
name = d.metadata.get(model.MD_DESCRIPTION, "Time")
klass = stream.StaticSpectrumStream
elif model.MD_AR_POLE in d.metadata:
# AR data
ar_data.append(d)
continue
elif model.MD_IN_WL in d.metadata and model.MD_OUT_WL in d.metadata:
# No explicit way to distinguish between Brightfield and Fluo,
# so guess it's Brightfield if excitation wl is large (> 100 nm)
in_wl = d.metadata.get(model.MD_IN_WL, (0, 0))
if in_wl[1] - in_wl[0] < 100e-9:
# Fluo
name = d.metadata.get(model.MD_DESCRIPTION, "Filtered colour")
klass = stream.StaticFluoStream
else:
# Brightfield
name = d.metadata.get(model.MD_DESCRIPTION, "Brightfield")
klass = stream.StaticBrightfieldStream
elif model.MD_IN_WL in d.metadata: # only MD_IN_WL
name = d.metadata.get(model.MD_DESCRIPTION, "Brightfield")
klass = stream.StaticBrightfieldStream
elif model.MD_OUT_WL in d.metadata: # only MD_OUT_WL
name = d.metadata.get(model.MD_DESCRIPTION, "Cathodoluminescence")
klass = stream.StaticCLStream
elif model.MD_USER_TINT in d.metadata: # User requested a colour => fallback to FluoStream
name = d.metadata.get(model.MD_DESCRIPTION, "Filtered colour")
klass = stream.StaticFluoStream
elif dims in ("CYX", "YXC") and d.shape[ci] in (3, 4):
# Only decide it's RGB as last resort, because most microscopy data is not RGB
name = d.metadata.get(model.MD_DESCRIPTION, "RGB data")
klass = stream.RGBStream
else:
# Now, either it's a flat greyscale image and we decide it's a SEM image,
# or it's gone too weird and we try again on flat images
if numpy.prod(
d.shape[:-2]) != 1 and pxs is not None and len(pxs) != 3:
# FIXME: doesn't work currently if d is a DAS
subdas = _split_planes(d)
logging.info("Reprocessing data of shape %s into %d sub-data",
d.shape, len(subdas))
if len(subdas) > 30:
logging.error(
"The data seems to have %d sub-data, limiting it to the first 10",
len(subdas))
subdas = subdas[:10]
if len(subdas) > 1:
result_streams.extend(data_to_static_streams(subdas))
continue
name = d.metadata.get(model.MD_DESCRIPTION, "Electrons")
klass = stream.StaticSEMStream
if issubclass(klass, stream.Static2DStream):
# FIXME: doesn't work currently if d is a DAS
if numpy.prod(d.shape[:-3]) != 1:
logging.warning(
"Dropping dimensions from the data %s of shape %s", name,
d.shape)
# T Z X Y
# d[0,0] -> d[0,0,:,:]
d = d[(0, ) * (d.ndim - 2)]
stream_instance = klass(name, d)
result_streams.append(stream_instance)
# Add one global AR stream
if ar_data:
result_streams.append(stream.StaticARStream("Angular", ar_data))
return result_streams
def data_to_static_streams(cls, data):
""" Split the given data into static streams
:param data: (list of DataArrays) Data to be split
:return: (list) A list of Stream instances
"""
result_streams = []
# AR data is special => all merged in one big stream
ar_data = []
# Add each data as a stream of the correct type
for d in data:
# Streams only support 2D data (e.g., no multiple channels like RGB)
# except for spectra which have a 3rd dimensions on dim 5.
# So if that's the case => separate into one stream per channel
channels_data = cls._split_channels(d)
for channel_data in channels_data:
# TODO: be more clever to detect the type of stream
if (model.MD_WL_LIST in channel_data.metadata
or model.MD_WL_POLYNOMIAL in channel_data.metadata
or (len(channel_data.shape) >= 5
and channel_data.shape[-5] > 1)):
name = channel_data.metadata.get(model.MD_DESCRIPTION,
"Spectrum")
klass = acqstream.StaticSpectrumStream
elif model.MD_AR_POLE in channel_data.metadata:
# AR data
ar_data.append(channel_data)
continue
elif ((model.MD_IN_WL in channel_data.metadata
and model.MD_OUT_WL in channel_data.metadata)
or model.MD_USER_TINT in channel_data.metadata):
# No explicit way to distinguish between Brightfield and Fluo,
# so guess it's Brightfield iif:
# * No tint
# * (and) Large band for excitation wl (> 100 nm)
in_wl = d.metadata[model.MD_IN_WL]
if (model.MD_USER_TINT in channel_data.metadata
or in_wl[1] - in_wl[0] < 100e-9):
# Fluo
name = channel_data.metadata.get(
model.MD_DESCRIPTION, "Filtered colour")
klass = acqstream.StaticFluoStream
else:
# Brightfield
name = channel_data.metadata.get(
model.MD_DESCRIPTION, "Brightfield")
klass = acqstream.StaticBrightfieldStream
elif model.MD_IN_WL in channel_data.metadata: # no MD_OUT_WL
name = channel_data.metadata.get(model.MD_DESCRIPTION,
"Brightfield")
klass = acqstream.StaticBrightfieldStream
else:
name = channel_data.metadata.get(model.MD_DESCRIPTION,
"Secondary electrons")
klass = acqstream.StaticSEMStream
result_streams.append(klass(name, channel_data))
# Add one global AR stream
if ar_data:
result_streams.append(acqstream.StaticARStream("Angular", ar_data))
return result_streams
def test_ar(self):
"""Test StaticARStream"""
# AR background data
md = {
model.MD_SW_VERSION: "1.0-test",
model.MD_HW_NAME: "fake ccd",
model.MD_DESCRIPTION: "AR",
model.MD_ACQ_DATE: time.time(),
model.MD_BPP: 12,
model.MD_BINNING: (1, 1), # px, px
model.MD_SENSOR_PIXEL_SIZE: (13e-6, 13e-6), # m/px
model.MD_PIXEL_SIZE: (2e-5, 2e-5), # m/px
model.MD_POS: (1.2e-3, -30e-3), # m
model.MD_EXP_TIME: 1.2, # s
model.MD_AR_POLE: (253.1, 65.1),
model.MD_LENS_MAG: 0.4, # ratio
}
# AR data
md0 = dict(md)
data0 = model.DataArray(
1500 + numpy.zeros((512, 1024), dtype=numpy.uint16), md0)
md1 = dict(md)
md1[model.MD_POS] = (1.5e-3, -30e-3)
md1[model.
MD_BASELINE] = 300 # AR background should take this into account
data1 = model.DataArray(
6345 + numpy.zeros((512, 1024), dtype=numpy.uint16), md1)
logging.info("setting up stream")
ars = stream.StaticARStream("test", [data0, data1])
# Control AR projection
im2d0 = ars.image.value
# Check it's a RGB DataArray
self.assertEqual(im2d0.shape[2], 3)
logging.info("changing AR pos")
# Wait a bit to be sure the projection doesn't get postponed to later
time.sleep(0.2)
# change position
for p in ars.point.choices:
if p != (None, None) and p != ars.point.value:
ars.point.value = p
break
else:
self.fail("Failed to find a second point in AR")
im2d1 = ars.image.value
# Check it's a RGB DataArray
self.assertEqual(im2d1.shape[2], 3)
self.assertFalse(im2d0 is im2d1)
logging.info("testing image background correction")
# test background correction from image
dcalib = numpy.ones((1, 1, 1, 512, 1024), dtype=numpy.uint16)
calib = model.DataArray(dcalib, md)
time.sleep(0.2)
ars.background.value = calib
numpy.testing.assert_equal(ars.background.value, calib[0, 0, 0])
im2dc = ars.image.value
# Check it's a RGB DataArray
self.assertEqual(im2dc.shape[2], 3)
self.assertFalse(im2d1 is im2dc)
def data_to_static_streams(data):
""" Split the given data into static streams
Args:
data: (list of DataArrays or DataArrayShadows) Data to be split
Returns:
(list) A list of Stream instances
"""
result_streams = []
# AR data is special => all merged in one big stream
ar_data = []
logging.debug("Processing %s data arrays", len(data))
# Add each data as a stream of the correct type
for d in data:
acqtype = d.metadata.get(model.MD_ACQ_TYPE)
# Hack for not displaying Anchor region data
# TODO: store and use acquisition type with MD_ACQ_TYPE?
if acqtype == model.MD_AT_ANCHOR or d.metadata.get(model.MD_DESCRIPTION) == "Anchor region":
continue
dims = d.metadata.get(model.MD_DIMS, "CTZYX"[-d.ndim::])
ti = dims.find("T") # -1 if not found
ci = dims.find("C") # -1 if not found
if (((model.MD_WL_LIST in d.metadata or
model.MD_WL_POLYNOMIAL in d.metadata) and
(ci >= 0 and d.shape[ci] > 1)
) or
(ci >= 0 and d.shape[ci] >= 5)
):
# Spectrum: either it's obvious according to metadata, or no metadata
# but lots of wavelengths, so no other way to display
name = d.metadata.get(model.MD_DESCRIPTION, "Spectrum")
klass = stream.StaticSpectrumStream
elif model.MD_PIXEL_DUR in d.metadata and ti >= 0 and d.shape[ti] > 1:
# Time data (with XY)
logging.info("Converting time data into spectrum data")
# HACK: for now we don't have a good static stream and GUI tools for
# showing data with time, but it's pretty much the same as a spectrum
# (expected it's on the 4th dim, in s, instead of 5th dim in m).
# FIXME: make the StaticSpectrumStream more generic, to support any
# 3D data (ie, dYX).
i3d = [0] * (d.ndim - 2) + [slice(None), slice(None)]
i3d[ti] = slice(None)
sda = d[tuple(i3d)] # basically, d[0, :, 0, :, :] for CTZYX
if sda.size != d.size:
logging.warning("Attempted to reduce data to TYX, but data had shape %s", d.shape)
d = sda
d.metadata[model.MD_DIMS] = "TYX"
# Convert linear scale (PIXEL_DUR + TIME_OFFSET) to WL_LIST
pd = d.metadata[model.MD_PIXEL_DUR]
to = d.metadata.get(model.MD_TIME_OFFSET, 0)
n = sda.shape[0]
tv = numpy.linspace(to, to + pd * (n - 1), n)
d.metadata[model.MD_WL_LIST] = tv
name = d.metadata.get(model.MD_DESCRIPTION, "Time")
klass = stream.StaticSpectrumStream
elif model.MD_AR_POLE in d.metadata:
# AR data
ar_data.append(d)
continue
elif ((model.MD_IN_WL in d.metadata and model.MD_OUT_WL in d.metadata) or
model.MD_USER_TINT in d.metadata
):
# No explicit way to distinguish between Brightfield and Fluo,
# so guess it's Brightfield iif:
# * No tint
# * (and) Large band for excitation wl (> 100 nm)
in_wl = d.metadata.get(model.MD_IN_WL, (0, 0))
if model.MD_USER_TINT in d.metadata or in_wl[1] - in_wl[0] < 100e-9:
# Fluo
name = d.metadata.get(model.MD_DESCRIPTION, "Filtered colour")
klass = stream.StaticFluoStream
else:
# Brightfield
name = d.metadata.get(model.MD_DESCRIPTION, "Brightfield")
klass = stream.StaticBrightfieldStream
elif model.MD_IN_WL in d.metadata: # only MD_IN_WL
name = d.metadata.get(model.MD_DESCRIPTION, "Brightfield")
klass = stream.StaticBrightfieldStream
elif model.MD_OUT_WL in d.metadata: # only MD_OUT_WL
name = d.metadata.get(model.MD_DESCRIPTION, "Cathodoluminescence")
klass = stream.StaticCLStream
elif dims in ("CYX", "YXC") and d.shape[ci] in (3, 4):
# Only decide it's RGB as last resort, because most microscopy data is not RGB
name = d.metadata.get(model.MD_DESCRIPTION, "RGB data")
klass = stream.RGBStream
else:
# Now, either it's a flat greyscale image and we decide it's a SEM image,
# or it's gone too weird and we try again on flat images
if numpy.prod(d.shape[:-2]) != 1:
# FIXME: doesn't work currently if d is a DAS
subdas = _split_planes(d)
logging.info("Reprocessing data of shape %s into %d sub-data",
d.shape, len(subdas))
if len(subdas) > 1:
result_streams.extend(data_to_static_streams(subdas))
continue
name = d.metadata.get(model.MD_DESCRIPTION, "Secondary electrons")
klass = stream.StaticSEMStream
if issubclass(klass, stream.Static2DStream):
# FIXME: doesn't work currently if d is a DAS
if numpy.prod(d.shape[:-2]) != 1:
logging.warning("Dropping dimensions from the data %s of shape %s",
name, d.shape)
d = d[-2, -1]
stream_instance = klass(name, d)
result_streams.append(stream_instance)
# Add one global AR stream
if ar_data:
result_streams.append(stream.StaticARStream("Angular", ar_data))
return result_streams
def test_big_ar_export(self):
# Create AR data
md = {
model.MD_SW_VERSION: "1.0-test",
model.MD_HW_NAME: "fake ccd",
model.MD_DESCRIPTION: "AR",
model.MD_ACQ_TYPE: model.MD_AT_AR,
model.MD_ACQ_DATE: time.time(),
model.MD_BPP: 12,
model.MD_BINNING: (1, 1), # px, px
model.MD_SENSOR_PIXEL_SIZE: (13e-6, 13e-6), # m/px
model.MD_PIXEL_SIZE: (4e-5, 4e-5), # m/px
model.MD_POS: (1.2e-3, -30e-3), # m
model.MD_EXP_TIME: 1.2, # s
model.MD_AR_POLE: (500, 500),
}
md0 = dict(md)
data0 = model.DataArray(
1500 + numpy.zeros((1080, 1024), dtype=numpy.uint16), md0)
md1 = dict(md)
md1[model.MD_POS] = (1.5e-3, -30e-3)
md1[model.
MD_BASELINE] = 300 # AR background should take this into account
data1 = model.DataArray(
500 + numpy.zeros((1080, 1024), dtype=numpy.uint16), md1)
# Create AR stream
ars = stream.StaticARStream("test", [data0])
ars.point.value = md0[model.MD_POS]
# Wait for the projection to be computed
tend = time.time() + 90
while ars.image.value is None:
self.assertLess(time.time(), tend, "Timeout during AR computation")
time.sleep(0.1)
# Convert to exportable RGB image
exdata = img.ar_to_export_data([ars], raw=False)
# shape = RGBA
self.assertGreater(exdata.shape[0], 200)
self.assertGreater(exdata.shape[1], 200)
self.assertEqual(exdata.shape[2], 4)
# The top-left corner should be white
numpy.testing.assert_equal(exdata[0, 0], [255, 255, 255, 255])
# There should be some non-white data
self.assertTrue(numpy.any(exdata != 255))
# Save into a PNG file
exporter = dataio.get_converter("PNG")
exporter.export(self.FILENAME_PR, exdata)
st = os.stat(
self.FILENAME_PR) # this test also that the file is created
self.assertGreater(st.st_size, 1000)
# Convert to equirectangular (RAW) image
exdata = img.ar_to_export_data([ars], raw=True)
# shape = raw data + theta/phi axes values
self.assertGreater(exdata.shape[0], 50)
self.assertGreater(exdata.shape[1], 50)
# Save into a CSV file
exporter = dataio.get_converter("CSV")
exporter.export(self.FILENAME_RAW, exdata)
st = os.stat(
self.FILENAME_RAW) # this test also that the file is created
self.assertGreater(st.st_size, 100)
# Create AR stream with background image
ars.background.value = data1
# Convert to equirectangular (RAW) image
exdata = img.ar_to_export_data([ars], raw=True)
# shape = raw data + theta/phi axes values
self.assertGreater(exdata.shape[0], 50)
self.assertGreater(exdata.shape[1], 50)