def _preprocessData(self, n, data, i):
"""
Pre-process the raw data, just after it was received from the detector.
:param n: (0<=int) The detector/stream index.
:param data: (DataArray) The data as received from the detector, from _onData(),
and with MD_POS updated to the current position of the e-beam.
:param i: (int, int) The iteration number in X, Y.
:returns: (value) The value as needed by _assembleFinalData.
"""
if n != self._ccd_idx:
return super(SECOMCLSEMMDStream, self)._preprocessData(n, data, i)
ccd_roi = self.ccd_roi
data = data[ccd_roi[1]: ccd_roi[3] + 1, ccd_roi[0]: ccd_roi[2] + 1] # crop
cpos = self._get_center_pos(data, self.ccd_roi)
sname = self._streams[n].name.value
data.metadata[model.MD_DESCRIPTION] = sname
# update center position of optical image (should be the same for all optical images)
data.metadata[model.MD_POS] = cpos
# Hack: To avoid memory issues, we save the optical image immediately after being acquired.
# Thus, we do not keep all the images in cache until the end of the acquisition.
fn = self.filename.value
logging.debug("Will save CL data to %s", fn)
fn_prefix, fn_ext = os.path.splitext(self.filename.value)
self.save_data(data,
prefix=fn_prefix,
xres=self.repetition.value[0],
yres=self.repetition.value[1],
xstepsize=self._getPixelSize()[0] * 1e9,
ystepsize=self._getPixelSize()[1] * 1e9,
xpos=i[1]+1, # start counting with 1
ypos=i[0]+1,
type="optical"
)
# Return something, but not the data to avoid data being cached.
return model.DataArray(numpy.array([0]))
def test_multiple_tiles(self):
def getSubData(dast, zoom, rect):
x1, y1, x2, y2 = rect
tiles = []
for x in range(x1, x2 + 1):
tiles_column = []
for y in range(y1, y2 + 1):
tiles_column.append(dast.getTile(x, y, zoom))
tiles.append(tiles_column)
return tiles
FILENAME = u"test" + tiff.EXTENSIONS[0]
POS = (5.0, 7.0)
size = (2000, 1000)
md = {
model.MD_DIMS: 'YX',
model.MD_POS: POS,
model.MD_PIXEL_SIZE: (1e-6, 1e-6),
}
arr = numpy.arange(size[0] * size[1], dtype=numpy.uint8).reshape(size[::-1])
data = model.DataArray(arr, metadata=md)
# export
tiff.export(FILENAME, data, pyramid=True)
rdata = tiff.open_data(FILENAME)
tiles = getSubData(rdata.content[0], 0, (0, 0, 7, 3))
merged_img = img.mergeTiles(tiles)
self.assertEqual(merged_img.shape, (1000, 2000))
self.assertEqual(merged_img.metadata[model.MD_POS], POS)
tiles = getSubData(rdata.content[0], 0, (0, 0, 3, 1))
merged_img = img.mergeTiles(tiles)
self.assertEqual(merged_img.shape, (512, 1024))
numpy.testing.assert_almost_equal(merged_img.metadata[model.MD_POS], (4.999512, 7.000244))
del rdata
os.remove(FILENAME)
def test_wl_list(self):
shape = (220, 1, 1, 50, 400)
dtype = numpy.dtype("uint16")
wl_orig = (400e-9 + numpy.arange(shape[0]) * 10e-9).tolist()
metadata = {
model.MD_SW_VERSION: "1.0-test",
model.MD_HW_NAME: "fake spec",
model.MD_DESCRIPTION: "test3d",
model.MD_ACQ_DATE: time.time(),
model.MD_BPP: 12,
model.MD_BINNING: (1, 1), # px, px
model.MD_PIXEL_SIZE: (1e-6, 2e-5), # m/px
model.MD_WL_LIST: wl_orig,
model.MD_POS: (1e-3, -30e-3), # m
model.MD_EXP_TIME: 1.2, # s
}
da = model.DataArray(numpy.zeros(shape, dtype), metadata)
wl = spectrum.get_wavelength_per_pixel(da)
self.assertEqual(len(wl), shape[0])
self.assertEqual(wl, wl_orig)
def test_rgb(self):
"""
Test downscaling an RGB in YXC format
"""
# X=1024, Y=512
size = (512, 1024, 3)
background = 58
img_in = numpy.zeros(size, dtype="uint8") + background
# watermark
img_in[246:266, 502:522, 0] = 50
img_in[246:266, 502:522, 1] = 100
img_in[246:266, 502:522, 2] = 150
img_in = model.DataArray(img_in)
img_in.metadata[model.MD_DIMS] = "YXC"
out = img.rescale_hq(img_in, (256, 512, 3))
self.assertEqual(out.shape, (256, 512, 3))
self.assertEqual(out.dtype, img_in.dtype)
# Check watermark. Should be no interpolation between color channels
self.assertEqual(50, out[128, 256, 0])
self.assertEqual(100, out[128, 256, 1])
self.assertEqual(150, out[128, 256, 2])
def testExportAR(self):
"""Try simple AR export"""
size = (90, 360)
dtype = numpy.float
metadata = {
model.MD_DESCRIPTION: "Angle-resolved",
model.MD_ACQ_TYPE: model.MD_AT_AR
}
data = model.DataArray(numpy.zeros(size, dtype), metadata)
data[...] = 26.1561
data[10, 10] = 10
# export
csv.export(FILENAME, data)
# check it's here
st = os.stat(FILENAME) # this test also that the file is created
self.assertGreater(st.st_size, 100)
raised = False
try:
pycsv.reader(open(FILENAME, 'rb'))
except IOError:
raised = True
self.assertFalse(raised, 'Failed to read csv file')
# test intensity value is at correct position
file = pycsv.reader(open(FILENAME, 'r'))
a = numpy.zeros((91, 361))
index = 0
for line in file:
if index == 0:
a[index] = 0.0
else:
a[index] = line
index += 1
# test intensity for same px as defined above is also different when reading back
# (+1 as we add a line for theta/phi MD to the array when exporting)
self.assertEqual(a[11][11], 10)
def _groupImages(das):
"""
Group images into larger ndarray, to follow the HDF5 SVI flavour.
In practice, this only consists in merging data for multiple channels into
one, and ordering/extending the shape to CTZYX.
das (list of DataArray): all the images
returns :
acq (list of DataArrays): each group of data, with the (general) metadata
metadatas (list of (list of dict, or None)): for each item of acq, either
None if the metadata is fully in acq or one metadata per channel.
"""
# For each image: adjust dimensions
adas = [_adjustDimensions(da) for da in das]
# For each image, if C = 1, try to merge it to an existing group
groups = _findImageGroups(adas)
acq, mds = [], []
# For each group:
# * if alone, do nothing
# * if many, merge along C
for g in groups:
if len(g) == 1:
acq.append(g[0])
mds.append(None)
else:
# merge along C (always axis 0)
gdata = numpy.concatenate(g, axis=0)
md = [d.metadata for d in g]
# merge metadata
# TODO: might need to be more clever for some metadata (eg, ACQ_DATE)
gmd = {}
map(gmd.update, md)
gdata = model.DataArray(gdata, gmd)
acq.append(gdata)
mds.append(md)
return acq, mds
def testExportSpectrumLineNoWL(self):
"""Try simple spectrum-line export"""
size = (1340, 6)
dtype = numpy.float
md = {
model.MD_PIXEL_SIZE: (None, 4.2e-06),
model.MD_ACQ_TYPE: model.MD_AT_SPECTRUM
}
data = model.DataArray(numpy.zeros(size, dtype), md)
# export
csv.export(FILENAME, data)
# check it's here
st = os.stat(FILENAME) # this test also that the file is created
self.assertGreater(st.st_size, 5)
raised = False
try:
pycsv.reader(open(FILENAME, 'rb'))
except IOError:
raised = True
self.assertFalse(raised, 'Failed to read csv file')
def _assemble_correlator_data(self, cordata, resolution, roi, stepsize):
"""
Assemble time-correlator data and metadata
"""
#get metadata, no need to ask directly to the component because the metadata is already embedded in the first dataset
md = cordata[0].metadata.copy()
xres, yres = resolution
md[model.MD_PIXEL_SIZE] = stepsize
md[model.MD_POS] = self._get_center_pxs(resolution, roi, cordata[0])
md[model.MD_DESCRIPTION] = "Time correlator"
#force exposure time metadata to be full time on the pixel rather than dwelltime/nDC
md[model.MD_DWELL_TIME] = self.dwellTime.value
logging.debug("Assembling correlator data")
full_cordata = model.DataArray(cordata, metadata=md)
# reshaping matrix. This is probably a silly way but it works
full_cordata = full_cordata.swapaxes(0, 3)
full_cordata = full_cordata.swapaxes(1, 2)
# Check XY ordering
full_cordata = numpy.reshape(full_cordata, [1, full_cordata.shape[1], 1, yres, xres])
#full_cordata = full_cordata.swapaxes(3, 4)
full_cordata.metadata[model.MD_DIMS] = "CTZYX"
return full_cordata
def correct_data(self, dlg):
"""
Remove the spikes of self._spec_stream
"""
# We keep the original raw data in a special ._orig_raw, and will put the
# corrected data in .raw (so that the correction is displayed). If the
# runs the correction again, it will be done on the original data (so
# the correction is run from scratch every time, and not from the data
# already cleaned-up).
try:
raw_spec_dat = self._spec_stream._orig_raw
except AttributeError: # No orig_raw yet
raw_spec_dat = self._spec_stream.raw[0]
self._spec_stream._orig_raw = raw_spec_dat
corrected_spec, npixels, nspikes = self.removespikes_spec(raw_spec_dat)
full_cordata = model.DataArray(corrected_spec, metadata=raw_spec_dat.metadata)
self._spec_stream.raw[0] = full_cordata
self._force_update_spec(self._spec_stream)
self._update_spike_pix(npixels, nspikes)
self._update_save_button()
def assemble_tiles(self, shape, data, roi, pxs):
"""
Convert a series of tiles acquisitions into an image (2D)
shape (2 x 0<ints): Number of tiles in the output (Y, X)
data (ndarray of shape N, T, S): the values,
ordered in blocks of TxS with X first, then Y. N = Y*X.
Each element along N is tiled on the final data.
roi (4 0<=floats<=1): ROI relative to the SEM FoV used to compute the
spots positions
pxs (0<float): distance (in m) between 2 tile centers, used to compute the
spots positions
return (DataArray of shape Y*T, X*S): the data with the correct metadata
"""
N, T, S = data.shape
if T == 1 and S == 1:
# fast path: the data is already ordered
arr = data
# reshape to get a 2D image
arr.shape = shape
else:
# need to reorder data by tiles
Y, X = shape
# change N to Y, X
arr = data.reshape((Y, X, T, S))
# change to Y, T, X, S by moving the "T" axis
arr = numpy.rollaxis(arr, 2, 1)
# and apply the change in memory (= 1 copy)
arr = numpy.ascontiguousarray(arr)
# reshape to apply the tiles
arr.shape = (Y * T, X * S)
# set the metadata
phys_roi = self.convert_roi_ratio_to_phys(roi)
center = ((phys_roi[0] + phys_roi[2]) / 2,
(phys_roi[1] + phys_roi[3]) / 2)
md = {model.MD_POS: center,
model.MD_PIXEL_SIZE: (pxs / S, pxs / T)}
return model.DataArray(arr, md)
def get_pixel_spectrum(self):
"""
Return the (0D) spectrum belonging to the selected pixel.
See get_spectrum_range() to know the wavelength values for each index of
the spectrum dimension
return (None or DataArray with 1 dimension): the spectrum of the given
pixel or None if no spectrum is selected.
"""
if self.selected_pixel.value == (None, None):
return None
x, y = self.selected_pixel.value
spec2d = self._calibrated[:, 0, 0, :, :] # same data but remove useless dims
# We treat width as the diameter of the circle which contains the center
# of the pixels to be taken into account
width = self.selectionWidth.value
if width == 1: # short-cut for simple case
return spec2d[:, y, x]
# There are various ways to do it with numpy. As typically the spectrum
# dimension is big, and the number of pixels to sum is small, it seems
# the easiest way is to just do some kind of "clever" mean. Using a
# masked array would also work, but that'd imply having a huge mask.
radius = width / 2
n = 0
# TODO: use same cleverness as mean() for dtype?
datasum = numpy.zeros(spec2d.shape[0], dtype=numpy.float64)
# Scan the square around the point, and only pick the points in the circle
for px in range(max(0, int(x - radius)),
min(int(x + radius) + 1, spec2d.shape[-1])):
for py in range(max(0, int(y - radius)),
min(int(y + radius) + 1, spec2d.shape[-2])):
if math.hypot(x - px, y - py) <= radius:
n += 1
datasum += spec2d[:, py, px]
mean = datasum / n
return model.DataArray(mean.astype(spec2d.dtype))
def testExportOnePage(self):
# create a simple greyscale image
size = (256, 512) # (width, height)
dtype = numpy.uint16
data = model.DataArray(numpy.zeros(size[::-1], dtype))
white = (12, 52) # non symmetric position
# less that 2**15 so that we don't have problem with PIL.getpixel() always returning an signed int
data[white[::-1]] = 124
# export
hdf5.export(FILENAME, data)
# check it's here
st = os.stat(FILENAME) # this test also that the file is created
self.assertGreater(st.st_size, 0)
f = h5py.File(FILENAME, "r")
# need to transform to a full numpy.array just to remove the dimensions
im = numpy.array(f["Acquisition0/ImageData/Image"])
im.shape = im.shape[3:5]
self.assertEqual(im.shape, data.shape)
self.assertEqual(im[white[-1:-3:-1]], data[white[-1:-3:-1]])
def setUp(self):
data = numpy.ones((251, 1, 1, 200, 300), dtype="uint16")
data[:, 0, 0, :, 3] = range(200)
data[:, 0, 0, :, 3] *= 3
data[:, 0, 0, 1, 3] = range(251)
data[2, 0, 0, :, :] = range(300)
data[200, 0, 0, 2, :] = range(300)
wld = 433e-9 + numpy.array(range(data.shape[0])) * 0.1e-9
md = {model.MD_SW_VERSION: "1.0-test",
model.MD_HW_NAME: "fake ccd",
model.MD_DESCRIPTION: "Spectrum",
model.MD_ACQ_DATE: time.time(),
model.MD_BPP: 12,
model.MD_PIXEL_SIZE: (2e-6, 2e-6), # m/px
model.MD_POS: (-0.001203511795256, -0.000295338300158), # m
model.MD_EXP_TIME: 0.2, # s
model.MD_LENS_MAG: 60, # ratio
model.MD_WL_LIST: wld,
}
self.spec_data = model.DataArray(data, md)
self.spec_stream = stream.StaticSpectrumStream("test spec", self.spec_data)
self.spec_stream.selected_pixel.value = (3, 1)
def _thumbFromHDF5(filename):
"""
Read thumbnails from an HDF5 file.
Expects to find them as IMAGE in Preview/Image.
return (list of model.DataArray)
"""
f = h5py.File(filename, "r")
thumbs = []
# look for the Preview directory
try:
grp = f["Preview"]
except KeyError:
# no thumbnail
return thumbs
# scan for images
for name, ds in grp.items():
# an image? (== has the attribute CLASS: IMAGE)
if isinstance(ds, h5py.Dataset) and ds.attrs.get("CLASS") == "IMAGE":
try:
da = model.DataArray(_read_image_dataset(ds))
except Exception:
logging.info("Skipping image '%s' which couldn't be read.",
name)
continue
if name == "Image":
try:
da.metadata = _read_image_info(grp)
except Exception:
logging.debug(
"Failed to parse metadata of acquisition '%s'", name)
continue
thumbs.append(da)
return thumbs
def _assembleAnchorData(self, data_list):
"""
Take all the data acquired for the anchor region
data_list (list of N DataArray of shape 2D (Y, X)): all the anchor data
return (DataArray of shape (1, N, 1, Y, X))
"""
assert len(data_list) > 0
assert data_list[0].ndim == 2
# extend the shape to TZ dimensions to allow the concatenation on T
for d in data_list:
d.shape = (1, 1) + d.shape
anchor_data = numpy.concatenate(data_list)
anchor_data.shape = (1,) + anchor_data.shape
# copy the metadata from the first image (which contains the original
# position of the anchor region, without drift correction)
md = data_list[0].metadata.copy()
md[model.MD_DESCRIPTION] = "Anchor region"
md[model.MD_AD_LIST] = tuple(d.metadata[model.MD_ACQ_DATE] for d in data_list)
return model.DataArray(anchor_data, metadata=md)
def testUnicodeName(self):
"""Try filename not fitting in ascii"""
# create a simple greyscale image
size = (256, 512)
dtype = numpy.uint16
data = model.DataArray(numpy.zeros(size[::-1], dtype))
white = (12, 52) # non symmetric position
# less that 2**15 so that we don't have problem with PIL.getpixel() always returning an signed int
data[white[::-1]] = 124
fn = u"𝔸𝔹ℂ" + FILENAME
# export
tiff.export(fn, data)
# check it's here
st = os.stat(fn) # this test also that the file is created
self.assertGreater(st.st_size, 0)
im = Image.open(fn)
self.assertEqual(im.format, "TIFF")
self.assertEqual(im.size, size)
self.assertEqual(im.getpixel(white), 124)
os.remove(fn)
def test_25d(self):
"""
Test downscaling an 2.5D image (YXC, with C=14)
"""
# X=1024, Y=512
size = (512, 1024, 14)
background = 58
img_in = numpy.zeros(size, dtype=numpy.float) + background
# watermark
img_in[246:266, 502:522, 0] = 50
img_in[246:266, 502:522, 1] = 100
img_in[246:266, 502:522, 2] = 150
img_in[246:266, 502:522, 3] = 255 # Alpha
img_in = model.DataArray(img_in)
img_in.metadata[model.MD_DIMS] = "YXC"
out = img.rescale_hq(img_in, (256, 512, 14))
self.assertEqual(out.shape, (256, 512, 14))
self.assertEqual(out.dtype, img_in.dtype)
# Check watermark. Should be no interpolation between color channels
self.assertEqual(50, out[128, 256, 0])
self.assertEqual(100, out[128, 256, 1])
self.assertEqual(150, out[128, 256, 2])
self.assertEqual(255, out[128, 256, 3])
def testExportSpectrum(self):
"""Try simple spectrum export"""
size = (150, )
dtype = numpy.uint16
md = {
model.MD_WL_LIST: numpy.linspace(536e-9, 650e-9, size[0]).tolist(),
model.MD_ACQ_TYPE: model.MD_AT_SPECTRUM
}
data = model.DataArray(numpy.zeros(size, dtype), md)
data += 56
# export
csv.export(FILENAME, data)
# check it's here
st = os.stat(FILENAME) # this test also that the file is created
self.assertGreater(st.st_size, 150)
raised = False
try:
pycsv.reader(open(FILENAME, 'rb'))
except IOError:
raised = True
self.assertFalse(raised, 'Failed to read csv file')
def _updateImage(self):
raw = self.stream.raw[0]
metadata = self.stream._find_metadata(raw.metadata)
raw = img.ensure2DImage(raw) # Remove extra dimensions (of length 1)
grayscale_im = preprocess(raw, self._invert, self._flip, self._crop,
self._gaussian_sigma, self._eqhis)
rgb_im = img.DataArray2RGB(grayscale_im)
if self._kp:
rgb_im = cv2.drawKeypoints(rgb_im,
self._kp,
None,
color=(30, 30, 255),
flags=0)
if self._mkp:
rgb_im = cv2.drawKeypoints(rgb_im,
self._mkp,
None,
color=(0, 255, 0),
flags=0)
rgb_im = model.DataArray(rgb_im, metadata)
rgb_im.flags.writeable = False
self.image.value = rgb_im
def testExportMultiPage(self):
# create a simple greyscale image
size = (512, 256)
white = (12, 52) # non symmetric position
dtype = numpy.uint16
ldata = []
self.no_of_images = 2
metadata = [
{
model.MD_IN_WL: (500e-9, 520e-9), # m
},
{
model.MD_EXP_TIME: 1.2, # s
},
]
# Add wavelength metadata just to group them
for i in range(self.no_of_images):
a = model.DataArray(numpy.zeros(size[::-1], dtype), metadata[i])
a[white[::-1]] = 124 + i
ldata.append(a)
# export
stiff.export(FILENAME, ldata)
tokens = FILENAME.split(".0.", 1)
# Iterate through the files generated
for i in range(self.no_of_images):
fname = tokens[0] + "." + str(i) + "." + tokens[1]
# check it's here
st = os.stat(fname) # this test also that the file is created
self.assertGreater(st.st_size, 0)
im = Image.open(fname)
self.assertEqual(im.format, "TIFF")
self.assertEqual(im.size, size)
self.assertEqual(im.getpixel(white), 124 + i)
del im
def _onCompletedData(self, n, raw_das):
# Only override for the CCD data
if n < len(self._streams) - 1:
r = super(SEMCLCCDStream, self)._onCompletedData(n, raw_das)
return r
# Same as sem data, but without computing the data position from the
# CCD metadata
md = self._ccd_md.copy()
sem_data = self._raw[0] # _onCompletedData() should be called in order
md[model.MD_POS] = sem_data.metadata[model.MD_POS]
md[model.MD_DESCRIPTION] = self._streams[n].name.value
# Make sure it doesn't contain metadata related to AR
for k in (model.MD_AR_POLE, model.MD_AR_FOCUS_DISTANCE,
model.MD_AR_HOLE_DIAMETER, model.MD_AR_PARABOLA_F,
model.MD_AR_XMAX, model.MD_ROTATION):
md.pop(k, None)
try:
# handle sub-pixels (aka fuzzing)
sem_shape = sem_data.shape[-1:-3:-1] # 1,1,1,Y,X -> X, Y
rep = self.repetition.value
tile_shape = (sem_shape[0] / rep[0], sem_shape[1] / rep[1])
pxs = (sem_data.metadata[model.MD_PIXEL_SIZE][0] * tile_shape[0],
sem_data.metadata[model.MD_PIXEL_SIZE][1] * tile_shape[1])
md[model.MD_PIXEL_SIZE] = pxs
except KeyError:
logging.warning("Metadata missing from the SEM data")
# concatenate data into one big array of (number of pixels,1)
flat_list = [ar.flatten() for ar in raw_das]
rep_one = numpy.concatenate(flat_list)
# reshape to (Y, X)
rep_one.shape = rep[::-1]
rep_one = model.DataArray(rep_one, metadata=md)
self._raw.append(rep_one)
def test_marking_line_overlay(self):
cnvs = miccanvas.TwoDPlotCanvas(self.panel)
mlol = cnvs.markline_overlay
self.add_control(cnvs, wx.EXPAND, proportion=1, clear=True)
rgb = numpy.empty((30, 200, 3), dtype=numpy.uint8)
data = model.DataArray(rgb)
cnvs.set_2d_data(data,
unit_x='m',
unit_y='m',
range_x=[200e-9, 500e-9],
range_y=[0, 20e-6])
test.gui_loop()
mlol.val.value = (201e-9, 10e-6)
cnvs.Refresh()
test.gui_loop(0.5)
mlol.orientation = vol.MarkingLineOverlay.HORIZONTAL
cnvs.Refresh()
test.gui_loop(0.5)
mlol.orientation = vol.MarkingLineOverlay.VERTICAL
mlol.val.value = (301e-9, 12e-6)
cnvs.Refresh()
test.gui_loop(0.5)
mlol.orientation = vol.MarkingLineOverlay.HORIZONTAL | vol.MarkingLineOverlay.VERTICAL
mlol.val.value = (401e-9, 20e-6)
cnvs.Refresh()
test.gui_loop(0.5)
# Out of the range
mlol.val.value = (0, 0)
cnvs.Refresh()
test.gui_loop(0.5)
def ARBackgroundSubtract(data):
"""
Subtracts the "baseline" (i.e. the average intensity of the background) from the data.
This function can be called before AngleResolved2Polar in order to take a better data output.
data (model.DataArray): The DataArray with the data. Must be 2D.
Can have metadata MD_BASELINE to indicate the average 0 value. If not,
it must have metadata MD_PIXEL_SIZE and MD_AR_POLE
returns (model.DataArray): Filtered data
"""
baseline = 0
try:
# If available, use the baseline from the metadata, as it's much faster
baseline = data.metadata[model.MD_BASELINE]
except KeyError:
# If baseline is not provided we calculate it, taking the average intensity of the
# background (i.e. the pixels that are outside the half circle)
try:
pxs = data.metadata[model.MD_PIXEL_SIZE]
pole_pos = data.metadata[model.MD_AR_POLE]
except KeyError:
raise ValueError("Metadata required: MD_PIXEL_SIZE, MD_AR_POLE.")
circle_mask = _CreateMirrorMask(data, pxs, pole_pos, hole=False)
masked_image = ma.array(data, mask=circle_mask)
# Calculate the average value of the outside pixels
baseline = masked_image.mean()
# Clip values that will result to negative numbers
# after the subtraction
ret_data = numpy.where(data < baseline, baseline, data)
# Subtract background
ret_data -= baseline
result = model.DataArray(ret_data, data.metadata)
return result
def assemble_cube(self, shape, specs):
"""
Assemble all the spectrum data together
shape (int,int)
specs (list of DataArray of one dimension): must be in order X/Y
return DataArray (3 dimensions): spectral cube
"""
# create a cube out of the spectral data acquired
# dimensions must be wavelength, 1, 1, Y, X
assert len(specs) == numpy.prod(shape)
# each element of specs has a shape of (N)
# reshape to (N, 1)
for s in specs:
s.shape += (1, )
# concatenate into one big array of (N, Y*X)
spect_data = numpy.concatenate(specs, axis=1)
# reshape to (N, 1, 1, Y, X)
spect_data.shape = (spect_data.shape[0], 1, 1, shape[1], shape[0])
# copy the metadata from the first point
spect_data = model.DataArray(spect_data, metadata=specs[0].metadata)
return spect_data
def _dataFromSVIHDF5(f):
"""
Read microscopy data from an HDF5 file using the SVI convention.
Expects to find them as IMAGE in XXX/ImageData/Image + XXX/PhysicalData.
f (h5py.File): the root of the file
return (list of model.DataArray)
"""
data = []
for obj in f.values():
# find all the expected and interesting objects
try:
svidata = obj["SVIData"]
imagedata = obj["ImageData"]
image = imagedata["Image"]
physicaldata = obj["PhysicalData"]
except KeyError:
continue # not conforming => try next object
# Read the raw data
try:
nd = _read_image_dataset(image)
except Exception:
logging.exception("Failed to read data of acquisition '%s'",
obj.name)
# TODO: read more metadata
try:
da = model.DataArray(nd, metadata=_read_image_info(imagedata))
except Exception:
logging.exception("Failed to parse metadata of acquisition '%s'",
obj.name)
das = _parse_physical_data(physicaldata, da)
data.extend(das)
return data
def get(self):
da = model.DataArray([1e-12], {model.MD_ACQ_DATE: time.time()})
return da
def test_get_next_pixels(self):
det = Fake0DDetector("test")
pca = ProbeCurrentAcquirer(det)
# Period = dt => every pixel
pca.period.value = 0.1
np = pca.start(0.1, (10, 10))
scan_px = np
while scan_px < 10 * 10:
self.assertEqual(np, 1) # don't check the last call
da = model.DataArray([0] * np, {model.MD_ACQ_DATE: time.time()})
np = pca.next([da])
scan_px += np
pca.next([da]) # one last time
# Period = dt + epsilon => every pixel
pca.period.value = 0.10001
np = pca.start(0.1, (10, 10))
scan_px = np
while scan_px < 10 * 10:
self.assertEqual(np, 1) # don't check the last call
da = model.DataArray([0] * np, {model.MD_ACQ_DATE: time.time()})
np = pca.next([da])
scan_px += np
pca.next([da]) # one last time
# Period < dt => every pixel
pca.period.value = 0.05
np = pca.start(0.1, (10, 10))
scan_px = np
while scan_px < 10 * 10:
self.assertEqual(np, 1) # don't check the last call
da = model.DataArray([0] * np, {model.MD_ACQ_DATE: time.time()})
np = pca.next([da])
scan_px += np
pca.next([da]) # one last time
# Period = 2.5 * dt => alternatively every 2 and 3 pixels
pca.period.value = 0.1 * 2.5
np = pca.start(0.1, (10, 10))
scan_px = np
while scan_px < 10 * 10:
self.assertIn(np, (2, 3)) # don't check the last call
da = model.DataArray([0] * np, {model.MD_ACQ_DATE: time.time()})
np = pca.next([da])
scan_px += np
pca.next([da]) # one last time
# Period = dt * 5 => every 5 px
pca.period.value = 0.5
np = pca.start(0.1, (10, 10))
scan_px = np
while scan_px < 10 * 10:
self.assertEqual(np, 5) # don't check the last call
da = model.DataArray([0] * np, {model.MD_ACQ_DATE: time.time()})
np = pca.next([da])
scan_px += np
pca.next([da]) # one last time
# Period > dt * shape => at first and last
pca.period.value = 100
np = pca.start(0.1, (10, 10))
scan_px = np
while scan_px < 10 * 10:
self.assertEqual(np, 10 * 10) # don't check the last call
da = model.DataArray([0] * np, {model.MD_ACQ_DATE: time.time()})
np = pca.next([da])
scan_px += np
pca.next([da]) # one last time
# Period = dt * shape / 2 => at first, middle and last
pca.period.value = 0.1 * 10 * 5
np = pca.start(0.1, (10, 10))
scan_px = np
while scan_px < 10 * 10:
self.assertEqual(np, 10 * 10 / 2) # don't check the last call
da = model.DataArray([0] * np, {model.MD_ACQ_DATE: time.time()})
np = pca.next([da])
scan_px += np
pca.next([da]) # one last time
# Short period, on a large shape
pca.period.value = 4900e-6 # A little less than every 10 lines
np = pca.start(1e-6, (700, 500))
assert 9 * 500 <= np <= 4900
scan_px = np
while scan_px < 700 * 500:
da = model.DataArray([0] * np, {model.MD_ACQ_DATE: time.time()})
np = pca.next([da])
left = 700 * 500 - scan_px
if left < 4900:
assert np <= left
else:
assert 9 * 500 <= np <= 4900
scan_px += np
pca.next([da]) # one last time
def _runAcquisition(self, future):
self._data = []
self._md = {}
wls = self.startWavelength.value
wle = self.endWavelength.value
res = self.numberOfPixels.value
dt = self.dwellTime.value
trig = self._detector.softwareTrigger
df = self._detector.data
# Prepare the hardware
self._emitter.resolution.value = (1, 1) # Force one pixel only
self._emitter.translation.value = self.emtTranslation.value
self._emitter.dwellTime.value = dt
df.synchronizedOn(trig)
df.subscribe(self._on_mchr_data)
wllist = []
if wle == wls:
res = 1
if res <= 1:
res = 1
wli = 0
else:
wli = (wle - wls) / (res - 1)
try:
for i in range(res):
left = (res - i) * (dt + 0.05)
future.set_progress(end=time.time() + left)
cwl = wls + i * wli # requested value
self._sgr.moveAbs({"wavelength": cwl}).result()
if future._acq_state == CANCELLED:
raise CancelledError()
cwl = self._sgr.position.value["wavelength"] # actual value
logging.info("Acquiring point %d/%d @ %s", i + 1, res,
units.readable_str(cwl, unit="m", sig=3))
self._pt_acq.clear()
trig.notify()
if not self._pt_acq.wait(dt * 5 + 1):
raise IOError("Timeout waiting for the data")
if future._acq_state == CANCELLED:
raise CancelledError()
wllist.append(cwl)
# Done
df.unsubscribe(self._on_mchr_data)
df.synchronizedOn(None)
# Convert the sequence of data into one spectrum in a DataArray
if wls > wle: # went backward? => sort back the spectrum
logging.debug(
"Inverting spectrum as acquisition went from %g to %g m",
wls, wls)
self._data.reverse()
wllist.reverse()
na = numpy.array(self._data) # keeps the dtype
na.shape += (1, 1, 1, 1) # make it 5th dim to indicate a channel
md = self._md
md[model.MD_WL_LIST] = wllist
if model.MD_OUT_WL in md:
# The MD_OUT_WL on the monochromator contains the current cw, which we don't want
del md[model.MD_OUT_WL]
# MD_POS should already be at the correct position (from the e-beam metadata)
# MD_PIXEL_SIZE is not meaningful but handy for the display in Odemis
# (it's the size of the square on top of the SEM survey => BIG!)
sempxs = self._emitter.pixelSize.value
md[model.MD_PIXEL_SIZE] = (sempxs[0] * 50, sempxs[1] * 50)
spec = model.DataArray(na, md)
with future._acq_lock:
if future._acq_state == CANCELLED:
raise CancelledError()
future._acq_state = FINISHED
return [spec]
except CancelledError:
raise # Just don't log the exception
except Exception:
logging.exception("Failure during monochromator scan")
finally:
# In case it was stopped before the end
df.unsubscribe(self._on_mchr_data)
df.synchronizedOn(None)
future._acq_done.set()
def testReadMDFluo(self):
"""
Checks that we can read back the metadata of a fluoresence image
The OME-TIFF file will contain just one big array, but three arrays
should be read back with the right data.
"""
metadata = [
{
model.MD_SW_VERSION: "1.0-test",
model.MD_HW_NAME: "fake hw",
model.MD_DESCRIPTION: "brightfield",
model.MD_ACQ_DATE: time.time(),
model.MD_BPP: 12,
model.MD_BINNING: (1, 1), # px, px
model.MD_PIXEL_SIZE: (1e-6, 1e-6), # m/px
model.MD_POS: (13.7e-3, -30e-3), # m
model.MD_EXP_TIME: 1.2, # s
model.MD_IN_WL: (400e-9, 630e-9), # m
model.MD_OUT_WL: (400e-9, 630e-9), # m
},
{
model.MD_SW_VERSION: "1.0-test",
model.MD_HW_NAME: "fake hw",
model.MD_DESCRIPTION: "blue dye",
model.MD_ACQ_DATE: time.time() + 10,
model.MD_BPP: 12,
model.MD_BINNING: (1, 1), # px, px
model.MD_PIXEL_SIZE: (1e-6, 1e-6), # m/px
model.MD_POS: (13.7e-3, -30e-3), # m
model.MD_EXP_TIME: 1.2, # s
model.MD_IN_WL: (500e-9, 520e-9), # m
model.MD_OUT_WL: (600e-9, 630e-9), # m
},
{
model.MD_SW_VERSION: "1.0-test",
model.MD_HW_NAME: "fake hw",
model.MD_DESCRIPTION: "green dye",
model.MD_ACQ_DATE: time.time() + 20,
model.MD_BPP: 12,
model.MD_BINNING: (1, 1), # px, px
model.MD_PIXEL_SIZE: (1e-6, 1e-6), # m/px
model.MD_POS: (13.7e-3, -3e-3), # m
model.MD_EXP_TIME: 1, # s
model.MD_IN_WL: (600e-9, 620e-9), # m
model.MD_OUT_WL: (620e-9, 650e-9), # m
},
]
# create 3 greyscale images of same size
size = (512, 256)
dtype = numpy.dtype("uint16")
ldata = []
for i, md in enumerate(metadata):
a = model.DataArray(numpy.zeros(size[::-1], dtype), md)
a[i, i] = i # "watermark" it
ldata.append(a)
# thumbnail : small RGB completely red
tshape = (size[1] // 8, size[0] // 8, 3)
tdtype = numpy.uint8
thumbnail = model.DataArray(numpy.zeros(tshape, tdtype))
thumbnail[:, :, 1] += 255 # green
# export
hdf5.export(FILENAME, ldata, thumbnail)
# check it's here
st = os.stat(FILENAME) # this test also that the file is created
self.assertGreater(st.st_size, 0)
# check data
rdata = hdf5.read_data(FILENAME)
self.assertEqual(len(rdata), len(ldata))
# TODO: rdata and ldata don't have to be in the same order
for i, im in enumerate(rdata):
md = metadata[i]
self.assertEqual(im.metadata[model.MD_DESCRIPTION],
md[model.MD_DESCRIPTION])
self.assertAlmostEqual(im.metadata[model.MD_POS][0],
md[model.MD_POS][0])
self.assertAlmostEqual(im.metadata[model.MD_POS][1],
md[model.MD_POS][1])
self.assertAlmostEqual(im.metadata[model.MD_PIXEL_SIZE][0],
md[model.MD_PIXEL_SIZE][0])
self.assertAlmostEqual(im.metadata[model.MD_PIXEL_SIZE][1],
md[model.MD_PIXEL_SIZE][1])
iwl = im.metadata[model.MD_IN_WL] # nm
self.assertTrue((md[model.MD_IN_WL][0] <= iwl[0]
and iwl[1] <= md[model.MD_IN_WL][1]))
owl = im.metadata[model.MD_OUT_WL] # nm
self.assertTrue((md[model.MD_OUT_WL][0] <= owl[0]
and owl[1] <= md[model.MD_OUT_WL][1]))
# SVI HDF5 only records one acq time per T dimension
# so only check for the first channel
if i == 0:
self.assertAlmostEqual(im.metadata[model.MD_ACQ_DATE],
md[model.MD_ACQ_DATE],
delta=1)
# SVI HDF5 doesn't this metadata:
# self.assertEqual(im.metadata[model.MD_BPP], md[model.MD_BPP])
# self.assertEqual(im.metadata[model.MD_BINNING], md[model.MD_BINNING])
self.assertEqual(im.metadata[model.MD_EXP_TIME],
md[model.MD_EXP_TIME])
# check thumbnail
rthumbs = hdf5.read_thumbnail(FILENAME)
self.assertEqual(len(rthumbs), 1)
im = rthumbs[0]
self.assertEqual(im.shape, tshape)
self.assertEqual(im[0, 0].tolist(), [0, 255, 0])
def testReadMDAR(self):
"""
Checks that we can read back the metadata of an Angular Resolved image
"""
metadata = [
{
model.MD_SW_VERSION: "1.0-test",
model.MD_HW_NAME: "fake hw",
model.MD_DESCRIPTION: "sem survey",
model.MD_ACQ_DATE: time.time(),
model.MD_BPP: 12,
model.MD_BINNING: (1, 2), # px, px
model.MD_PIXEL_SIZE: (1e-6, 2e-5), # m/px
model.MD_POS: (1e-3, -30e-3), # m
model.MD_EXP_TIME: 1.2, # s
model.MD_LENS_MAG: 1200, # ratio
},
{
model.MD_SW_VERSION: "1.0-test",
model.MD_HW_NAME: "fake ccd",
model.MD_DESCRIPTION: "AR",
model.MD_ACQ_DATE: time.time(),
model.MD_BPP: 12,
model.MD_BINNING: (1, 1), # px, px
model.MD_SENSOR_PIXEL_SIZE: (13e-6, 13e-6), # m/px
model.MD_PIXEL_SIZE: (1e-6, 2e-5), # m/px
model.MD_POS: (1.2e-3, -30e-3), # m
model.MD_EXP_TIME: 1.2, # s
model.MD_AR_POLE: (253.1, 65.1),
model.MD_LENS_MAG: 60, # ratio
},
{
model.MD_SW_VERSION: "1.0-test",
model.MD_HW_NAME: "fake ccd",
model.MD_DESCRIPTION: "AR",
model.MD_ACQ_DATE: time.time(),
model.MD_BPP: 12,
model.MD_BINNING: (1, 1), # px, px
model.MD_SENSOR_PIXEL_SIZE: (13e-6, 13e-6), # m/px
model.MD_PIXEL_SIZE: (1e-6, 2e-5), # m/px
model.MD_POS: (1e-3, -30e-3), # m
model.MD_EXP_TIME: 1.2, # s
model.MD_AR_POLE: (253.1, 65.1),
model.MD_LENS_MAG: 60, # ratio
},
]
# create 2 simple greyscale images
sizes = [(512, 256), (500, 400), (500, 400)
] # different sizes to ensure different acquisitions
dtype = numpy.dtype("uint16")
ldata = []
for s, md in zip(sizes, metadata):
a = model.DataArray(numpy.zeros(s[::-1], dtype), md)
ldata.append(a)
# thumbnail : small RGB completely red
tshape = (sizes[0][1] // 8, sizes[0][0] // 8, 3)
tdtype = numpy.uint8
thumbnail = model.DataArray(numpy.zeros(tshape, tdtype))
thumbnail[:, :, 1] += 255 # green
# export
hdf5.export(FILENAME, ldata, thumbnail)
# check it's here
st = os.stat(FILENAME) # this test also that the file is created
self.assertGreater(st.st_size, 0)
# check data
rdata = hdf5.read_data(FILENAME)
self.assertEqual(len(rdata), len(ldata))
for im, md in zip(rdata, metadata):
self.assertEqual(im.metadata[model.MD_DESCRIPTION],
md[model.MD_DESCRIPTION])
self.assertEqual(im.metadata[model.MD_POS], md[model.MD_POS])
self.assertEqual(im.metadata[model.MD_PIXEL_SIZE],
md[model.MD_PIXEL_SIZE])
self.assertEqual(im.metadata[model.MD_ACQ_DATE],
md[model.MD_ACQ_DATE])
if model.MD_AR_POLE in md:
self.assertEqual(im.metadata[model.MD_AR_POLE],
md[model.MD_AR_POLE])
if model.MD_LENS_MAG in md:
self.assertEqual(im.metadata[model.MD_LENS_MAG],
md[model.MD_LENS_MAG])
# check thumbnail
rthumbs = hdf5.read_thumbnail(FILENAME)
self.assertEqual(len(rthumbs), 1)
im = rthumbs[0]
self.assertEqual(im.shape, tshape)
self.assertEqual(im[0, 0].tolist(), [0, 255, 0])