def _preprocessData(self, n, data, i): """ Pre-process the raw data, just after it was received from the detector. :param n: (0<=int) The detector/stream index. :param data: (DataArray) The data as received from the detector, from _onData(), and with MD_POS updated to the current position of the e-beam. :param i: (int, int) The iteration number in X, Y. :returns: (value) The value as needed by _assembleFinalData. """ if n != self._ccd_idx: return super(SECOMCLSEMMDStream, self)._preprocessData(n, data, i) ccd_roi = self.ccd_roi data = data[ccd_roi[1]: ccd_roi[3] + 1, ccd_roi[0]: ccd_roi[2] + 1] # crop cpos = self._get_center_pos(data, self.ccd_roi) sname = self._streams[n].name.value data.metadata[model.MD_DESCRIPTION] = sname # update center position of optical image (should be the same for all optical images) data.metadata[model.MD_POS] = cpos # Hack: To avoid memory issues, we save the optical image immediately after being acquired. # Thus, we do not keep all the images in cache until the end of the acquisition. fn = self.filename.value logging.debug("Will save CL data to %s", fn) fn_prefix, fn_ext = os.path.splitext(self.filename.value) self.save_data(data, prefix=fn_prefix, xres=self.repetition.value[0], yres=self.repetition.value[1], xstepsize=self._getPixelSize()[0] * 1e9, ystepsize=self._getPixelSize()[1] * 1e9, xpos=i[1]+1, # start counting with 1 ypos=i[0]+1, type="optical" ) # Return something, but not the data to avoid data being cached. return model.DataArray(numpy.array([0]))
def test_multiple_tiles(self): def getSubData(dast, zoom, rect): x1, y1, x2, y2 = rect tiles = [] for x in range(x1, x2 + 1): tiles_column = [] for y in range(y1, y2 + 1): tiles_column.append(dast.getTile(x, y, zoom)) tiles.append(tiles_column) return tiles FILENAME = u"test" + tiff.EXTENSIONS[0] POS = (5.0, 7.0) size = (2000, 1000) md = { model.MD_DIMS: 'YX', model.MD_POS: POS, model.MD_PIXEL_SIZE: (1e-6, 1e-6), } arr = numpy.arange(size[0] * size[1], dtype=numpy.uint8).reshape(size[::-1]) data = model.DataArray(arr, metadata=md) # export tiff.export(FILENAME, data, pyramid=True) rdata = tiff.open_data(FILENAME) tiles = getSubData(rdata.content[0], 0, (0, 0, 7, 3)) merged_img = img.mergeTiles(tiles) self.assertEqual(merged_img.shape, (1000, 2000)) self.assertEqual(merged_img.metadata[model.MD_POS], POS) tiles = getSubData(rdata.content[0], 0, (0, 0, 3, 1)) merged_img = img.mergeTiles(tiles) self.assertEqual(merged_img.shape, (512, 1024)) numpy.testing.assert_almost_equal(merged_img.metadata[model.MD_POS], (4.999512, 7.000244)) del rdata os.remove(FILENAME)
def test_wl_list(self): shape = (220, 1, 1, 50, 400) dtype = numpy.dtype("uint16") wl_orig = (400e-9 + numpy.arange(shape[0]) * 10e-9).tolist() metadata = { model.MD_SW_VERSION: "1.0-test", model.MD_HW_NAME: "fake spec", model.MD_DESCRIPTION: "test3d", model.MD_ACQ_DATE: time.time(), model.MD_BPP: 12, model.MD_BINNING: (1, 1), # px, px model.MD_PIXEL_SIZE: (1e-6, 2e-5), # m/px model.MD_WL_LIST: wl_orig, model.MD_POS: (1e-3, -30e-3), # m model.MD_EXP_TIME: 1.2, # s } da = model.DataArray(numpy.zeros(shape, dtype), metadata) wl = spectrum.get_wavelength_per_pixel(da) self.assertEqual(len(wl), shape[0]) self.assertEqual(wl, wl_orig)
def test_rgb(self): """ Test downscaling an RGB in YXC format """ # X=1024, Y=512 size = (512, 1024, 3) background = 58 img_in = numpy.zeros(size, dtype="uint8") + background # watermark img_in[246:266, 502:522, 0] = 50 img_in[246:266, 502:522, 1] = 100 img_in[246:266, 502:522, 2] = 150 img_in = model.DataArray(img_in) img_in.metadata[model.MD_DIMS] = "YXC" out = img.rescale_hq(img_in, (256, 512, 3)) self.assertEqual(out.shape, (256, 512, 3)) self.assertEqual(out.dtype, img_in.dtype) # Check watermark. Should be no interpolation between color channels self.assertEqual(50, out[128, 256, 0]) self.assertEqual(100, out[128, 256, 1]) self.assertEqual(150, out[128, 256, 2])
def testExportAR(self): """Try simple AR export""" size = (90, 360) dtype = numpy.float metadata = { model.MD_DESCRIPTION: "Angle-resolved", model.MD_ACQ_TYPE: model.MD_AT_AR } data = model.DataArray(numpy.zeros(size, dtype), metadata) data[...] = 26.1561 data[10, 10] = 10 # export csv.export(FILENAME, data) # check it's here st = os.stat(FILENAME) # this test also that the file is created self.assertGreater(st.st_size, 100) raised = False try: pycsv.reader(open(FILENAME, 'rb')) except IOError: raised = True self.assertFalse(raised, 'Failed to read csv file') # test intensity value is at correct position file = pycsv.reader(open(FILENAME, 'r')) a = numpy.zeros((91, 361)) index = 0 for line in file: if index == 0: a[index] = 0.0 else: a[index] = line index += 1 # test intensity for same px as defined above is also different when reading back # (+1 as we add a line for theta/phi MD to the array when exporting) self.assertEqual(a[11][11], 10)
def _groupImages(das): """ Group images into larger ndarray, to follow the HDF5 SVI flavour. In practice, this only consists in merging data for multiple channels into one, and ordering/extending the shape to CTZYX. das (list of DataArray): all the images returns : acq (list of DataArrays): each group of data, with the (general) metadata metadatas (list of (list of dict, or None)): for each item of acq, either None if the metadata is fully in acq or one metadata per channel. """ # For each image: adjust dimensions adas = [_adjustDimensions(da) for da in das] # For each image, if C = 1, try to merge it to an existing group groups = _findImageGroups(adas) acq, mds = [], [] # For each group: # * if alone, do nothing # * if many, merge along C for g in groups: if len(g) == 1: acq.append(g[0]) mds.append(None) else: # merge along C (always axis 0) gdata = numpy.concatenate(g, axis=0) md = [d.metadata for d in g] # merge metadata # TODO: might need to be more clever for some metadata (eg, ACQ_DATE) gmd = {} map(gmd.update, md) gdata = model.DataArray(gdata, gmd) acq.append(gdata) mds.append(md) return acq, mds
def testExportSpectrumLineNoWL(self): """Try simple spectrum-line export""" size = (1340, 6) dtype = numpy.float md = { model.MD_PIXEL_SIZE: (None, 4.2e-06), model.MD_ACQ_TYPE: model.MD_AT_SPECTRUM } data = model.DataArray(numpy.zeros(size, dtype), md) # export csv.export(FILENAME, data) # check it's here st = os.stat(FILENAME) # this test also that the file is created self.assertGreater(st.st_size, 5) raised = False try: pycsv.reader(open(FILENAME, 'rb')) except IOError: raised = True self.assertFalse(raised, 'Failed to read csv file')
def _assemble_correlator_data(self, cordata, resolution, roi, stepsize): """ Assemble time-correlator data and metadata """ #get metadata, no need to ask directly to the component because the metadata is already embedded in the first dataset md = cordata[0].metadata.copy() xres, yres = resolution md[model.MD_PIXEL_SIZE] = stepsize md[model.MD_POS] = self._get_center_pxs(resolution, roi, cordata[0]) md[model.MD_DESCRIPTION] = "Time correlator" #force exposure time metadata to be full time on the pixel rather than dwelltime/nDC md[model.MD_DWELL_TIME] = self.dwellTime.value logging.debug("Assembling correlator data") full_cordata = model.DataArray(cordata, metadata=md) # reshaping matrix. This is probably a silly way but it works full_cordata = full_cordata.swapaxes(0, 3) full_cordata = full_cordata.swapaxes(1, 2) # Check XY ordering full_cordata = numpy.reshape(full_cordata, [1, full_cordata.shape[1], 1, yres, xres]) #full_cordata = full_cordata.swapaxes(3, 4) full_cordata.metadata[model.MD_DIMS] = "CTZYX" return full_cordata
def correct_data(self, dlg): """ Remove the spikes of self._spec_stream """ # We keep the original raw data in a special ._orig_raw, and will put the # corrected data in .raw (so that the correction is displayed). If the # runs the correction again, it will be done on the original data (so # the correction is run from scratch every time, and not from the data # already cleaned-up). try: raw_spec_dat = self._spec_stream._orig_raw except AttributeError: # No orig_raw yet raw_spec_dat = self._spec_stream.raw[0] self._spec_stream._orig_raw = raw_spec_dat corrected_spec, npixels, nspikes = self.removespikes_spec(raw_spec_dat) full_cordata = model.DataArray(corrected_spec, metadata=raw_spec_dat.metadata) self._spec_stream.raw[0] = full_cordata self._force_update_spec(self._spec_stream) self._update_spike_pix(npixels, nspikes) self._update_save_button()
def assemble_tiles(self, shape, data, roi, pxs): """ Convert a series of tiles acquisitions into an image (2D) shape (2 x 0<ints): Number of tiles in the output (Y, X) data (ndarray of shape N, T, S): the values, ordered in blocks of TxS with X first, then Y. N = Y*X. Each element along N is tiled on the final data. roi (4 0<=floats<=1): ROI relative to the SEM FoV used to compute the spots positions pxs (0<float): distance (in m) between 2 tile centers, used to compute the spots positions return (DataArray of shape Y*T, X*S): the data with the correct metadata """ N, T, S = data.shape if T == 1 and S == 1: # fast path: the data is already ordered arr = data # reshape to get a 2D image arr.shape = shape else: # need to reorder data by tiles Y, X = shape # change N to Y, X arr = data.reshape((Y, X, T, S)) # change to Y, T, X, S by moving the "T" axis arr = numpy.rollaxis(arr, 2, 1) # and apply the change in memory (= 1 copy) arr = numpy.ascontiguousarray(arr) # reshape to apply the tiles arr.shape = (Y * T, X * S) # set the metadata phys_roi = self.convert_roi_ratio_to_phys(roi) center = ((phys_roi[0] + phys_roi[2]) / 2, (phys_roi[1] + phys_roi[3]) / 2) md = {model.MD_POS: center, model.MD_PIXEL_SIZE: (pxs / S, pxs / T)} return model.DataArray(arr, md)
def get_pixel_spectrum(self): """ Return the (0D) spectrum belonging to the selected pixel. See get_spectrum_range() to know the wavelength values for each index of the spectrum dimension return (None or DataArray with 1 dimension): the spectrum of the given pixel or None if no spectrum is selected. """ if self.selected_pixel.value == (None, None): return None x, y = self.selected_pixel.value spec2d = self._calibrated[:, 0, 0, :, :] # same data but remove useless dims # We treat width as the diameter of the circle which contains the center # of the pixels to be taken into account width = self.selectionWidth.value if width == 1: # short-cut for simple case return spec2d[:, y, x] # There are various ways to do it with numpy. As typically the spectrum # dimension is big, and the number of pixels to sum is small, it seems # the easiest way is to just do some kind of "clever" mean. Using a # masked array would also work, but that'd imply having a huge mask. radius = width / 2 n = 0 # TODO: use same cleverness as mean() for dtype? datasum = numpy.zeros(spec2d.shape[0], dtype=numpy.float64) # Scan the square around the point, and only pick the points in the circle for px in range(max(0, int(x - radius)), min(int(x + radius) + 1, spec2d.shape[-1])): for py in range(max(0, int(y - radius)), min(int(y + radius) + 1, spec2d.shape[-2])): if math.hypot(x - px, y - py) <= radius: n += 1 datasum += spec2d[:, py, px] mean = datasum / n return model.DataArray(mean.astype(spec2d.dtype))
def testExportOnePage(self): # create a simple greyscale image size = (256, 512) # (width, height) dtype = numpy.uint16 data = model.DataArray(numpy.zeros(size[::-1], dtype)) white = (12, 52) # non symmetric position # less that 2**15 so that we don't have problem with PIL.getpixel() always returning an signed int data[white[::-1]] = 124 # export hdf5.export(FILENAME, data) # check it's here st = os.stat(FILENAME) # this test also that the file is created self.assertGreater(st.st_size, 0) f = h5py.File(FILENAME, "r") # need to transform to a full numpy.array just to remove the dimensions im = numpy.array(f["Acquisition0/ImageData/Image"]) im.shape = im.shape[3:5] self.assertEqual(im.shape, data.shape) self.assertEqual(im[white[-1:-3:-1]], data[white[-1:-3:-1]])
def setUp(self): data = numpy.ones((251, 1, 1, 200, 300), dtype="uint16") data[:, 0, 0, :, 3] = range(200) data[:, 0, 0, :, 3] *= 3 data[:, 0, 0, 1, 3] = range(251) data[2, 0, 0, :, :] = range(300) data[200, 0, 0, 2, :] = range(300) wld = 433e-9 + numpy.array(range(data.shape[0])) * 0.1e-9 md = {model.MD_SW_VERSION: "1.0-test", model.MD_HW_NAME: "fake ccd", model.MD_DESCRIPTION: "Spectrum", model.MD_ACQ_DATE: time.time(), model.MD_BPP: 12, model.MD_PIXEL_SIZE: (2e-6, 2e-6), # m/px model.MD_POS: (-0.001203511795256, -0.000295338300158), # m model.MD_EXP_TIME: 0.2, # s model.MD_LENS_MAG: 60, # ratio model.MD_WL_LIST: wld, } self.spec_data = model.DataArray(data, md) self.spec_stream = stream.StaticSpectrumStream("test spec", self.spec_data) self.spec_stream.selected_pixel.value = (3, 1)
def _thumbFromHDF5(filename): """ Read thumbnails from an HDF5 file. Expects to find them as IMAGE in Preview/Image. return (list of model.DataArray) """ f = h5py.File(filename, "r") thumbs = [] # look for the Preview directory try: grp = f["Preview"] except KeyError: # no thumbnail return thumbs # scan for images for name, ds in grp.items(): # an image? (== has the attribute CLASS: IMAGE) if isinstance(ds, h5py.Dataset) and ds.attrs.get("CLASS") == "IMAGE": try: da = model.DataArray(_read_image_dataset(ds)) except Exception: logging.info("Skipping image '%s' which couldn't be read.", name) continue if name == "Image": try: da.metadata = _read_image_info(grp) except Exception: logging.debug( "Failed to parse metadata of acquisition '%s'", name) continue thumbs.append(da) return thumbs
def _assembleAnchorData(self, data_list): """ Take all the data acquired for the anchor region data_list (list of N DataArray of shape 2D (Y, X)): all the anchor data return (DataArray of shape (1, N, 1, Y, X)) """ assert len(data_list) > 0 assert data_list[0].ndim == 2 # extend the shape to TZ dimensions to allow the concatenation on T for d in data_list: d.shape = (1, 1) + d.shape anchor_data = numpy.concatenate(data_list) anchor_data.shape = (1,) + anchor_data.shape # copy the metadata from the first image (which contains the original # position of the anchor region, without drift correction) md = data_list[0].metadata.copy() md[model.MD_DESCRIPTION] = "Anchor region" md[model.MD_AD_LIST] = tuple(d.metadata[model.MD_ACQ_DATE] for d in data_list) return model.DataArray(anchor_data, metadata=md)
def testUnicodeName(self): """Try filename not fitting in ascii""" # create a simple greyscale image size = (256, 512) dtype = numpy.uint16 data = model.DataArray(numpy.zeros(size[::-1], dtype)) white = (12, 52) # non symmetric position # less that 2**15 so that we don't have problem with PIL.getpixel() always returning an signed int data[white[::-1]] = 124 fn = u"𝔸𝔹ℂ" + FILENAME # export tiff.export(fn, data) # check it's here st = os.stat(fn) # this test also that the file is created self.assertGreater(st.st_size, 0) im = Image.open(fn) self.assertEqual(im.format, "TIFF") self.assertEqual(im.size, size) self.assertEqual(im.getpixel(white), 124) os.remove(fn)
def test_25d(self): """ Test downscaling an 2.5D image (YXC, with C=14) """ # X=1024, Y=512 size = (512, 1024, 14) background = 58 img_in = numpy.zeros(size, dtype=numpy.float) + background # watermark img_in[246:266, 502:522, 0] = 50 img_in[246:266, 502:522, 1] = 100 img_in[246:266, 502:522, 2] = 150 img_in[246:266, 502:522, 3] = 255 # Alpha img_in = model.DataArray(img_in) img_in.metadata[model.MD_DIMS] = "YXC" out = img.rescale_hq(img_in, (256, 512, 14)) self.assertEqual(out.shape, (256, 512, 14)) self.assertEqual(out.dtype, img_in.dtype) # Check watermark. Should be no interpolation between color channels self.assertEqual(50, out[128, 256, 0]) self.assertEqual(100, out[128, 256, 1]) self.assertEqual(150, out[128, 256, 2]) self.assertEqual(255, out[128, 256, 3])
def testExportSpectrum(self): """Try simple spectrum export""" size = (150, ) dtype = numpy.uint16 md = { model.MD_WL_LIST: numpy.linspace(536e-9, 650e-9, size[0]).tolist(), model.MD_ACQ_TYPE: model.MD_AT_SPECTRUM } data = model.DataArray(numpy.zeros(size, dtype), md) data += 56 # export csv.export(FILENAME, data) # check it's here st = os.stat(FILENAME) # this test also that the file is created self.assertGreater(st.st_size, 150) raised = False try: pycsv.reader(open(FILENAME, 'rb')) except IOError: raised = True self.assertFalse(raised, 'Failed to read csv file')
def _updateImage(self): raw = self.stream.raw[0] metadata = self.stream._find_metadata(raw.metadata) raw = img.ensure2DImage(raw) # Remove extra dimensions (of length 1) grayscale_im = preprocess(raw, self._invert, self._flip, self._crop, self._gaussian_sigma, self._eqhis) rgb_im = img.DataArray2RGB(grayscale_im) if self._kp: rgb_im = cv2.drawKeypoints(rgb_im, self._kp, None, color=(30, 30, 255), flags=0) if self._mkp: rgb_im = cv2.drawKeypoints(rgb_im, self._mkp, None, color=(0, 255, 0), flags=0) rgb_im = model.DataArray(rgb_im, metadata) rgb_im.flags.writeable = False self.image.value = rgb_im
def testExportMultiPage(self): # create a simple greyscale image size = (512, 256) white = (12, 52) # non symmetric position dtype = numpy.uint16 ldata = [] self.no_of_images = 2 metadata = [ { model.MD_IN_WL: (500e-9, 520e-9), # m }, { model.MD_EXP_TIME: 1.2, # s }, ] # Add wavelength metadata just to group them for i in range(self.no_of_images): a = model.DataArray(numpy.zeros(size[::-1], dtype), metadata[i]) a[white[::-1]] = 124 + i ldata.append(a) # export stiff.export(FILENAME, ldata) tokens = FILENAME.split(".0.", 1) # Iterate through the files generated for i in range(self.no_of_images): fname = tokens[0] + "." + str(i) + "." + tokens[1] # check it's here st = os.stat(fname) # this test also that the file is created self.assertGreater(st.st_size, 0) im = Image.open(fname) self.assertEqual(im.format, "TIFF") self.assertEqual(im.size, size) self.assertEqual(im.getpixel(white), 124 + i) del im
def _onCompletedData(self, n, raw_das): # Only override for the CCD data if n < len(self._streams) - 1: r = super(SEMCLCCDStream, self)._onCompletedData(n, raw_das) return r # Same as sem data, but without computing the data position from the # CCD metadata md = self._ccd_md.copy() sem_data = self._raw[0] # _onCompletedData() should be called in order md[model.MD_POS] = sem_data.metadata[model.MD_POS] md[model.MD_DESCRIPTION] = self._streams[n].name.value # Make sure it doesn't contain metadata related to AR for k in (model.MD_AR_POLE, model.MD_AR_FOCUS_DISTANCE, model.MD_AR_HOLE_DIAMETER, model.MD_AR_PARABOLA_F, model.MD_AR_XMAX, model.MD_ROTATION): md.pop(k, None) try: # handle sub-pixels (aka fuzzing) sem_shape = sem_data.shape[-1:-3:-1] # 1,1,1,Y,X -> X, Y rep = self.repetition.value tile_shape = (sem_shape[0] / rep[0], sem_shape[1] / rep[1]) pxs = (sem_data.metadata[model.MD_PIXEL_SIZE][0] * tile_shape[0], sem_data.metadata[model.MD_PIXEL_SIZE][1] * tile_shape[1]) md[model.MD_PIXEL_SIZE] = pxs except KeyError: logging.warning("Metadata missing from the SEM data") # concatenate data into one big array of (number of pixels,1) flat_list = [ar.flatten() for ar in raw_das] rep_one = numpy.concatenate(flat_list) # reshape to (Y, X) rep_one.shape = rep[::-1] rep_one = model.DataArray(rep_one, metadata=md) self._raw.append(rep_one)
def test_marking_line_overlay(self): cnvs = miccanvas.TwoDPlotCanvas(self.panel) mlol = cnvs.markline_overlay self.add_control(cnvs, wx.EXPAND, proportion=1, clear=True) rgb = numpy.empty((30, 200, 3), dtype=numpy.uint8) data = model.DataArray(rgb) cnvs.set_2d_data(data, unit_x='m', unit_y='m', range_x=[200e-9, 500e-9], range_y=[0, 20e-6]) test.gui_loop() mlol.val.value = (201e-9, 10e-6) cnvs.Refresh() test.gui_loop(0.5) mlol.orientation = vol.MarkingLineOverlay.HORIZONTAL cnvs.Refresh() test.gui_loop(0.5) mlol.orientation = vol.MarkingLineOverlay.VERTICAL mlol.val.value = (301e-9, 12e-6) cnvs.Refresh() test.gui_loop(0.5) mlol.orientation = vol.MarkingLineOverlay.HORIZONTAL | vol.MarkingLineOverlay.VERTICAL mlol.val.value = (401e-9, 20e-6) cnvs.Refresh() test.gui_loop(0.5) # Out of the range mlol.val.value = (0, 0) cnvs.Refresh() test.gui_loop(0.5)
def ARBackgroundSubtract(data): """ Subtracts the "baseline" (i.e. the average intensity of the background) from the data. This function can be called before AngleResolved2Polar in order to take a better data output. data (model.DataArray): The DataArray with the data. Must be 2D. Can have metadata MD_BASELINE to indicate the average 0 value. If not, it must have metadata MD_PIXEL_SIZE and MD_AR_POLE returns (model.DataArray): Filtered data """ baseline = 0 try: # If available, use the baseline from the metadata, as it's much faster baseline = data.metadata[model.MD_BASELINE] except KeyError: # If baseline is not provided we calculate it, taking the average intensity of the # background (i.e. the pixels that are outside the half circle) try: pxs = data.metadata[model.MD_PIXEL_SIZE] pole_pos = data.metadata[model.MD_AR_POLE] except KeyError: raise ValueError("Metadata required: MD_PIXEL_SIZE, MD_AR_POLE.") circle_mask = _CreateMirrorMask(data, pxs, pole_pos, hole=False) masked_image = ma.array(data, mask=circle_mask) # Calculate the average value of the outside pixels baseline = masked_image.mean() # Clip values that will result to negative numbers # after the subtraction ret_data = numpy.where(data < baseline, baseline, data) # Subtract background ret_data -= baseline result = model.DataArray(ret_data, data.metadata) return result
def assemble_cube(self, shape, specs): """ Assemble all the spectrum data together shape (int,int) specs (list of DataArray of one dimension): must be in order X/Y return DataArray (3 dimensions): spectral cube """ # create a cube out of the spectral data acquired # dimensions must be wavelength, 1, 1, Y, X assert len(specs) == numpy.prod(shape) # each element of specs has a shape of (N) # reshape to (N, 1) for s in specs: s.shape += (1, ) # concatenate into one big array of (N, Y*X) spect_data = numpy.concatenate(specs, axis=1) # reshape to (N, 1, 1, Y, X) spect_data.shape = (spect_data.shape[0], 1, 1, shape[1], shape[0]) # copy the metadata from the first point spect_data = model.DataArray(spect_data, metadata=specs[0].metadata) return spect_data
def _dataFromSVIHDF5(f): """ Read microscopy data from an HDF5 file using the SVI convention. Expects to find them as IMAGE in XXX/ImageData/Image + XXX/PhysicalData. f (h5py.File): the root of the file return (list of model.DataArray) """ data = [] for obj in f.values(): # find all the expected and interesting objects try: svidata = obj["SVIData"] imagedata = obj["ImageData"] image = imagedata["Image"] physicaldata = obj["PhysicalData"] except KeyError: continue # not conforming => try next object # Read the raw data try: nd = _read_image_dataset(image) except Exception: logging.exception("Failed to read data of acquisition '%s'", obj.name) # TODO: read more metadata try: da = model.DataArray(nd, metadata=_read_image_info(imagedata)) except Exception: logging.exception("Failed to parse metadata of acquisition '%s'", obj.name) das = _parse_physical_data(physicaldata, da) data.extend(das) return data
def get(self): da = model.DataArray([1e-12], {model.MD_ACQ_DATE: time.time()}) return da
def test_get_next_pixels(self): det = Fake0DDetector("test") pca = ProbeCurrentAcquirer(det) # Period = dt => every pixel pca.period.value = 0.1 np = pca.start(0.1, (10, 10)) scan_px = np while scan_px < 10 * 10: self.assertEqual(np, 1) # don't check the last call da = model.DataArray([0] * np, {model.MD_ACQ_DATE: time.time()}) np = pca.next([da]) scan_px += np pca.next([da]) # one last time # Period = dt + epsilon => every pixel pca.period.value = 0.10001 np = pca.start(0.1, (10, 10)) scan_px = np while scan_px < 10 * 10: self.assertEqual(np, 1) # don't check the last call da = model.DataArray([0] * np, {model.MD_ACQ_DATE: time.time()}) np = pca.next([da]) scan_px += np pca.next([da]) # one last time # Period < dt => every pixel pca.period.value = 0.05 np = pca.start(0.1, (10, 10)) scan_px = np while scan_px < 10 * 10: self.assertEqual(np, 1) # don't check the last call da = model.DataArray([0] * np, {model.MD_ACQ_DATE: time.time()}) np = pca.next([da]) scan_px += np pca.next([da]) # one last time # Period = 2.5 * dt => alternatively every 2 and 3 pixels pca.period.value = 0.1 * 2.5 np = pca.start(0.1, (10, 10)) scan_px = np while scan_px < 10 * 10: self.assertIn(np, (2, 3)) # don't check the last call da = model.DataArray([0] * np, {model.MD_ACQ_DATE: time.time()}) np = pca.next([da]) scan_px += np pca.next([da]) # one last time # Period = dt * 5 => every 5 px pca.period.value = 0.5 np = pca.start(0.1, (10, 10)) scan_px = np while scan_px < 10 * 10: self.assertEqual(np, 5) # don't check the last call da = model.DataArray([0] * np, {model.MD_ACQ_DATE: time.time()}) np = pca.next([da]) scan_px += np pca.next([da]) # one last time # Period > dt * shape => at first and last pca.period.value = 100 np = pca.start(0.1, (10, 10)) scan_px = np while scan_px < 10 * 10: self.assertEqual(np, 10 * 10) # don't check the last call da = model.DataArray([0] * np, {model.MD_ACQ_DATE: time.time()}) np = pca.next([da]) scan_px += np pca.next([da]) # one last time # Period = dt * shape / 2 => at first, middle and last pca.period.value = 0.1 * 10 * 5 np = pca.start(0.1, (10, 10)) scan_px = np while scan_px < 10 * 10: self.assertEqual(np, 10 * 10 / 2) # don't check the last call da = model.DataArray([0] * np, {model.MD_ACQ_DATE: time.time()}) np = pca.next([da]) scan_px += np pca.next([da]) # one last time # Short period, on a large shape pca.period.value = 4900e-6 # A little less than every 10 lines np = pca.start(1e-6, (700, 500)) assert 9 * 500 <= np <= 4900 scan_px = np while scan_px < 700 * 500: da = model.DataArray([0] * np, {model.MD_ACQ_DATE: time.time()}) np = pca.next([da]) left = 700 * 500 - scan_px if left < 4900: assert np <= left else: assert 9 * 500 <= np <= 4900 scan_px += np pca.next([da]) # one last time
def _runAcquisition(self, future): self._data = [] self._md = {} wls = self.startWavelength.value wle = self.endWavelength.value res = self.numberOfPixels.value dt = self.dwellTime.value trig = self._detector.softwareTrigger df = self._detector.data # Prepare the hardware self._emitter.resolution.value = (1, 1) # Force one pixel only self._emitter.translation.value = self.emtTranslation.value self._emitter.dwellTime.value = dt df.synchronizedOn(trig) df.subscribe(self._on_mchr_data) wllist = [] if wle == wls: res = 1 if res <= 1: res = 1 wli = 0 else: wli = (wle - wls) / (res - 1) try: for i in range(res): left = (res - i) * (dt + 0.05) future.set_progress(end=time.time() + left) cwl = wls + i * wli # requested value self._sgr.moveAbs({"wavelength": cwl}).result() if future._acq_state == CANCELLED: raise CancelledError() cwl = self._sgr.position.value["wavelength"] # actual value logging.info("Acquiring point %d/%d @ %s", i + 1, res, units.readable_str(cwl, unit="m", sig=3)) self._pt_acq.clear() trig.notify() if not self._pt_acq.wait(dt * 5 + 1): raise IOError("Timeout waiting for the data") if future._acq_state == CANCELLED: raise CancelledError() wllist.append(cwl) # Done df.unsubscribe(self._on_mchr_data) df.synchronizedOn(None) # Convert the sequence of data into one spectrum in a DataArray if wls > wle: # went backward? => sort back the spectrum logging.debug( "Inverting spectrum as acquisition went from %g to %g m", wls, wls) self._data.reverse() wllist.reverse() na = numpy.array(self._data) # keeps the dtype na.shape += (1, 1, 1, 1) # make it 5th dim to indicate a channel md = self._md md[model.MD_WL_LIST] = wllist if model.MD_OUT_WL in md: # The MD_OUT_WL on the monochromator contains the current cw, which we don't want del md[model.MD_OUT_WL] # MD_POS should already be at the correct position (from the e-beam metadata) # MD_PIXEL_SIZE is not meaningful but handy for the display in Odemis # (it's the size of the square on top of the SEM survey => BIG!) sempxs = self._emitter.pixelSize.value md[model.MD_PIXEL_SIZE] = (sempxs[0] * 50, sempxs[1] * 50) spec = model.DataArray(na, md) with future._acq_lock: if future._acq_state == CANCELLED: raise CancelledError() future._acq_state = FINISHED return [spec] except CancelledError: raise # Just don't log the exception except Exception: logging.exception("Failure during monochromator scan") finally: # In case it was stopped before the end df.unsubscribe(self._on_mchr_data) df.synchronizedOn(None) future._acq_done.set()
def testReadMDFluo(self): """ Checks that we can read back the metadata of a fluoresence image The OME-TIFF file will contain just one big array, but three arrays should be read back with the right data. """ metadata = [ { model.MD_SW_VERSION: "1.0-test", model.MD_HW_NAME: "fake hw", model.MD_DESCRIPTION: "brightfield", model.MD_ACQ_DATE: time.time(), model.MD_BPP: 12, model.MD_BINNING: (1, 1), # px, px model.MD_PIXEL_SIZE: (1e-6, 1e-6), # m/px model.MD_POS: (13.7e-3, -30e-3), # m model.MD_EXP_TIME: 1.2, # s model.MD_IN_WL: (400e-9, 630e-9), # m model.MD_OUT_WL: (400e-9, 630e-9), # m }, { model.MD_SW_VERSION: "1.0-test", model.MD_HW_NAME: "fake hw", model.MD_DESCRIPTION: "blue dye", model.MD_ACQ_DATE: time.time() + 10, model.MD_BPP: 12, model.MD_BINNING: (1, 1), # px, px model.MD_PIXEL_SIZE: (1e-6, 1e-6), # m/px model.MD_POS: (13.7e-3, -30e-3), # m model.MD_EXP_TIME: 1.2, # s model.MD_IN_WL: (500e-9, 520e-9), # m model.MD_OUT_WL: (600e-9, 630e-9), # m }, { model.MD_SW_VERSION: "1.0-test", model.MD_HW_NAME: "fake hw", model.MD_DESCRIPTION: "green dye", model.MD_ACQ_DATE: time.time() + 20, model.MD_BPP: 12, model.MD_BINNING: (1, 1), # px, px model.MD_PIXEL_SIZE: (1e-6, 1e-6), # m/px model.MD_POS: (13.7e-3, -3e-3), # m model.MD_EXP_TIME: 1, # s model.MD_IN_WL: (600e-9, 620e-9), # m model.MD_OUT_WL: (620e-9, 650e-9), # m }, ] # create 3 greyscale images of same size size = (512, 256) dtype = numpy.dtype("uint16") ldata = [] for i, md in enumerate(metadata): a = model.DataArray(numpy.zeros(size[::-1], dtype), md) a[i, i] = i # "watermark" it ldata.append(a) # thumbnail : small RGB completely red tshape = (size[1] // 8, size[0] // 8, 3) tdtype = numpy.uint8 thumbnail = model.DataArray(numpy.zeros(tshape, tdtype)) thumbnail[:, :, 1] += 255 # green # export hdf5.export(FILENAME, ldata, thumbnail) # check it's here st = os.stat(FILENAME) # this test also that the file is created self.assertGreater(st.st_size, 0) # check data rdata = hdf5.read_data(FILENAME) self.assertEqual(len(rdata), len(ldata)) # TODO: rdata and ldata don't have to be in the same order for i, im in enumerate(rdata): md = metadata[i] self.assertEqual(im.metadata[model.MD_DESCRIPTION], md[model.MD_DESCRIPTION]) self.assertAlmostEqual(im.metadata[model.MD_POS][0], md[model.MD_POS][0]) self.assertAlmostEqual(im.metadata[model.MD_POS][1], md[model.MD_POS][1]) self.assertAlmostEqual(im.metadata[model.MD_PIXEL_SIZE][0], md[model.MD_PIXEL_SIZE][0]) self.assertAlmostEqual(im.metadata[model.MD_PIXEL_SIZE][1], md[model.MD_PIXEL_SIZE][1]) iwl = im.metadata[model.MD_IN_WL] # nm self.assertTrue((md[model.MD_IN_WL][0] <= iwl[0] and iwl[1] <= md[model.MD_IN_WL][1])) owl = im.metadata[model.MD_OUT_WL] # nm self.assertTrue((md[model.MD_OUT_WL][0] <= owl[0] and owl[1] <= md[model.MD_OUT_WL][1])) # SVI HDF5 only records one acq time per T dimension # so only check for the first channel if i == 0: self.assertAlmostEqual(im.metadata[model.MD_ACQ_DATE], md[model.MD_ACQ_DATE], delta=1) # SVI HDF5 doesn't this metadata: # self.assertEqual(im.metadata[model.MD_BPP], md[model.MD_BPP]) # self.assertEqual(im.metadata[model.MD_BINNING], md[model.MD_BINNING]) self.assertEqual(im.metadata[model.MD_EXP_TIME], md[model.MD_EXP_TIME]) # check thumbnail rthumbs = hdf5.read_thumbnail(FILENAME) self.assertEqual(len(rthumbs), 1) im = rthumbs[0] self.assertEqual(im.shape, tshape) self.assertEqual(im[0, 0].tolist(), [0, 255, 0])
def testReadMDAR(self): """ Checks that we can read back the metadata of an Angular Resolved image """ metadata = [ { model.MD_SW_VERSION: "1.0-test", model.MD_HW_NAME: "fake hw", model.MD_DESCRIPTION: "sem survey", model.MD_ACQ_DATE: time.time(), model.MD_BPP: 12, model.MD_BINNING: (1, 2), # px, px model.MD_PIXEL_SIZE: (1e-6, 2e-5), # m/px model.MD_POS: (1e-3, -30e-3), # m model.MD_EXP_TIME: 1.2, # s model.MD_LENS_MAG: 1200, # ratio }, { model.MD_SW_VERSION: "1.0-test", model.MD_HW_NAME: "fake ccd", model.MD_DESCRIPTION: "AR", model.MD_ACQ_DATE: time.time(), model.MD_BPP: 12, model.MD_BINNING: (1, 1), # px, px model.MD_SENSOR_PIXEL_SIZE: (13e-6, 13e-6), # m/px model.MD_PIXEL_SIZE: (1e-6, 2e-5), # m/px model.MD_POS: (1.2e-3, -30e-3), # m model.MD_EXP_TIME: 1.2, # s model.MD_AR_POLE: (253.1, 65.1), model.MD_LENS_MAG: 60, # ratio }, { model.MD_SW_VERSION: "1.0-test", model.MD_HW_NAME: "fake ccd", model.MD_DESCRIPTION: "AR", model.MD_ACQ_DATE: time.time(), model.MD_BPP: 12, model.MD_BINNING: (1, 1), # px, px model.MD_SENSOR_PIXEL_SIZE: (13e-6, 13e-6), # m/px model.MD_PIXEL_SIZE: (1e-6, 2e-5), # m/px model.MD_POS: (1e-3, -30e-3), # m model.MD_EXP_TIME: 1.2, # s model.MD_AR_POLE: (253.1, 65.1), model.MD_LENS_MAG: 60, # ratio }, ] # create 2 simple greyscale images sizes = [(512, 256), (500, 400), (500, 400) ] # different sizes to ensure different acquisitions dtype = numpy.dtype("uint16") ldata = [] for s, md in zip(sizes, metadata): a = model.DataArray(numpy.zeros(s[::-1], dtype), md) ldata.append(a) # thumbnail : small RGB completely red tshape = (sizes[0][1] // 8, sizes[0][0] // 8, 3) tdtype = numpy.uint8 thumbnail = model.DataArray(numpy.zeros(tshape, tdtype)) thumbnail[:, :, 1] += 255 # green # export hdf5.export(FILENAME, ldata, thumbnail) # check it's here st = os.stat(FILENAME) # this test also that the file is created self.assertGreater(st.st_size, 0) # check data rdata = hdf5.read_data(FILENAME) self.assertEqual(len(rdata), len(ldata)) for im, md in zip(rdata, metadata): self.assertEqual(im.metadata[model.MD_DESCRIPTION], md[model.MD_DESCRIPTION]) self.assertEqual(im.metadata[model.MD_POS], md[model.MD_POS]) self.assertEqual(im.metadata[model.MD_PIXEL_SIZE], md[model.MD_PIXEL_SIZE]) self.assertEqual(im.metadata[model.MD_ACQ_DATE], md[model.MD_ACQ_DATE]) if model.MD_AR_POLE in md: self.assertEqual(im.metadata[model.MD_AR_POLE], md[model.MD_AR_POLE]) if model.MD_LENS_MAG in md: self.assertEqual(im.metadata[model.MD_LENS_MAG], md[model.MD_LENS_MAG]) # check thumbnail rthumbs = hdf5.read_thumbnail(FILENAME) self.assertEqual(len(rthumbs), 1) im = rthumbs[0] self.assertEqual(im.shape, tshape) self.assertEqual(im[0, 0].tolist(), [0, 255, 0])