def mergeHeights(height_filename, channel_filenames): """ channel_filenames is a list in order from shortest wavelength to the longest wavelength. """ # # FIXME: We are just loading all the data at once, which could # problematic for really large files. # height_data = None for i, elt in enumerate(channel_filenames): i3_data = readinsight3.loadI3File(elt) if height_data is None: height_data = numpy.zeros((i3_data.size, len(channel_filenames)), dtype=numpy.float32) height_data[:, i] = i3_data['h'] # User feedback. if (height_data.shape[0] > 5): for i in range(5): print(height_data[i, :]) # Save the heights. numpy.save(height_filename, height_data)
def __init__(self, parameters = None, **kwds): super(DataWriterI3, self).__init__(**kwds) raise Exception("Using the Insight3 format for analysis is deprecated!") self.pixel_size = parameters.getAttr("pixel_size") # # If the i3 file already exists, read it in, write it # out to prepare for starting the analysis from the # end of what currently exists. # # FIXME: If the existing file is really large there # could be problems here as we're going to load # the whole thing into memory. # if(os.path.exists(self.filename)): print("Found", self.filename) i3data_in = readinsight3.loadI3File(self.filename) if (i3data_in is None) or (i3data_in.size == 0): self.start_frame = 0 else: self.start_frame = int(numpy.max(i3data_in['fr'])) print(" Starting analysis at frame:", self.start_frame) self.i3data = writeinsight3.I3Writer(self.filename) if (self.start_frame > 0): self.i3data.addMolecules(i3data_in) self.total_peaks = i3data_in['x'].size else: self.start_frame = 0 self.i3data = writeinsight3.I3Writer(self.filename)
def test_hdf5_to_bin_2(): """ Test tracks conversion. """ peaks = {"x" : numpy.zeros(10), "y" : numpy.ones(10)} h5_name = storm_analysis.getPathOutputTest("test_sa_hdf5.hdf5") storm_analysis.removeFile(h5_name) # Write data. with saH5Py.SAH5Py(h5_name, is_existing = False) as h5: h5.addMetadata("<settings/>") h5.setMovieInformation(256, 256, 10, "XYZZY") h5.setPixelSize(100.0) h5.addTracks(peaks) # Convert. i3_name = storm_analysis.getPathOutputTest("test_mlist.bin") storm_analysis.removeFile(i3_name) hdf5ToBin.hdf5ToBin(h5_name, i3_name) # Load Insight3 file and check values. i3_data = readinsight3.loadI3File(i3_name, verbose = False) assert(numpy.allclose(peaks["x"], i3_data['x'] - 1.0)) assert(numpy.allclose(peaks["y"], i3_data['y'] - 1.0)) assert(numpy.allclose(i3_data['fr'], numpy.ones(10)))
def pixelBias(filename, n_bins = 1000, normalized = True, i3_field = "x"): i3_data = readinsight3.loadI3File(filename) xv = numpy.fmod(i3_data[i3_field], 1.0) [xp_hist, x_bins] = numpy.histogram(xv, bins = n_bins, range = (0.0, 1.0), normed = normalized) x_centers = 0.5 * (x_bins[1:] + x_bins[:-1]) return [x_centers, xp_hist]
def simulate(self, dax_file, bin_file, n_frames): # # Initialization. # dax_data = daxwriter.DaxWriter(dax_file, self.x_size, self.y_size) i3_data_in = readinsight3.loadI3File(bin_file) out_fname_base = dax_file[:-4] i3_data_out = writeinsight3.I3Writer(out_fname_base + "_olist.bin") sim_settings = open(out_fname_base + "_sim_params.txt", "w") # # Create the user-specified class instances that will do # most of the actual work of the simulation. # bg = self.bg_factory(sim_settings, self.x_size, self.y_size, i3_data_in) cam = self.cam_factory(sim_settings, self.x_size, self.y_size, i3_data_in) pp = self.pphys_factory(sim_settings, self.x_size, self.y_size, i3_data_in) psf = self.psf_factory(sim_settings, self.x_size, self.y_size, i3_data_in) sim_settings.write(json.dumps({"simulation" : {"bin_file" : bin_file, "x_size" : str(self.x_size), "y_size" : str(self.y_size)}}) + "\n") # # Generate the simulated movie. # for i in range(n_frames): # Generate the new image. image = numpy.zeros((self.x_size, self.y_size)) cur_i3 = pp.getEmitters(i) print("Frame", i, cur_i3['x'].size, "emitters") # Background image += bg.getBackground(i) cur_i3 = bg.getEmitterBackground(cur_i3) # Foreground image += psf.getPSFs(cur_i3) # Camera image = cam.readImage(image) # Save the image. dax_data.addFrame(image) # Save the molecule locations. cur_i3['fr'] = i + 1 i3_data_out.addMolecules(cur_i3) dax_data.close() i3_data_out.close() sim_settings.close()
def test_good_i3(): mlist_name = storm_analysis.getPathOutputTest("test_i3_io_mlist.bin") # Create data. locs = i3dtype.createDefaultI3Data(100) # Save the data. with writeinsight3.I3Writer(mlist_name) as i3w: i3w.addMolecules(locs) # Read the data. locs = readinsight3.loadI3File(mlist_name) assert(locs.shape[0] == 100)
def test_write_read_1(): """ Test writing and reading. """ bin_name = storm_analysis.getPathOutputTest("test_insight3io.bin") i3_locs = i3dtype.createDefaultI3Data(10) i3dtype.posSet(i3_locs, 'x', 10.0) with writeinsight3.I3Writer(bin_name) as i3: i3.addMolecules(i3_locs) i3_in = readinsight3.loadI3File(bin_name, verbose = False) assert(numpy.allclose(i3_locs['x'], i3_in['x']))
def test_bad_i3(): mlist_name = storm_analysis.getPathOutputTest("test_i3_io_mlist.bin") # Create data. locs = i3dtype.createDefaultI3Data(100) # Save the data. i3w = writeinsight3.I3Writer(mlist_name) i3w.addMolecules(locs) i3w.fp.close() # Read the data. locs = readinsight3.loadI3File(mlist_name) assert(locs is None)
def splitPeaks(mlist_filename, params_filename): parameters = params.ParametersMultiplane().initFromFile(params_filename) # Load the plane to plane mapping data. mappings = {} if parameters.hasAttr("mapping"): if os.path.exists(parameters.getAttr("mapping")): with open(parameters.getAttr("mapping"), 'rb') as fp: mappings = pickle.load(fp) else: print("No mapping file parameter, nothing to do") # Load frame offset information. frame_offsets = list( map(parameters.getAttr, mpUtilC.getOffsetAttrs(parameters))) print(frame_offsets) # Load the molecule list. ch0_data = readinsight3.loadI3File(mlist_filename) # Map to other channels. basename = mlist_filename[:-4] channel = 1 m_key = "0_1_" while (m_key + "x") in mappings: chn_data = ch0_data.copy() # Map x. tx = mappings[m_key + "x"] chn_data['x'] = tx[0] + ch0_data['x'] * tx[1] + ch0_data['y'] * tx[2] # Map y. ty = mappings[m_key + "y"] chn_data['y'] = ty[0] + ch0_data['x'] * ty[1] + ch0_data['y'] * ty[2] # Map frame. chn_data[ 'fr'] = ch0_data['fr'] - frame_offsets[0] + frame_offsets[channel] with writeinsight3.I3Writer(basename + "_ch" + str(channel) + ".bin") as i3w: i3w.addMolecules(chn_data) channel += 1 m_key = "0_" + str(channel) + "_"
def makeTreeAndQuadsFromI3File(i3_filename, min_size=None, max_size=None, max_neighbors=10): """ Make a KD tree and a list of quads from an Insight3 file. Note: This file should probably only have localizations for a single frame. """ i3_data = readinsight3.loadI3File(i3_filename) # Warning if there is more than 1 frame in the data. if (len(numpy.unique(i3_data['fr'])) > 1): print("Warning: Localizations in multiple frames detected!") return makeTreeAndQuads(i3_data['xc'], i3_data['yc'], min_size=min_size, max_size=max_size, max_neighbors=max_neighbors)
def _test_l1h(): # Test L1H. movie_name = storm_analysis.getData("test/data/test_l1h.dax") settings = storm_analysis.getData("test/data/test_l1h.xml") hres = storm_analysis.getPathOutputTest("test_l1h_list.hres") mlist = storm_analysis.getPathOutputTest("test_l1h_list.bin") storm_analysis.removeFile(hres) storm_analysis.removeFile(mlist) from storm_analysis.L1H.cs_analysis import analyze analyze(movie_name, settings, hres, mlist) # Verify number of localizations found. # # FIXME: Change L1H to use the HDF5 format. # num_locs = readinsight3.loadI3File(mlist)["x"].size if not veri.verifyIsCloseEnough(num_locs, 1986): raise Exception("L1H did not find the expected number of localizations.")
def test_good_i3_metadata(): mlist_name = storm_analysis.getPathOutputTest("test_i3_io_mlist.bin") # Create data. locs = i3dtype.createDefaultI3Data(100) # Save data and metadata. i3w = writeinsight3.I3Writer(mlist_name) i3w.addMolecules(locs) etree = ElementTree.Element("xml") test = ElementTree.SubElement(etree, "test") test.text = "test" i3w.closeWithMetadata(ElementTree.tostring(etree, 'ISO-8859-1')) # Read the data. locs = readinsight3.loadI3File(mlist_name) assert(locs.shape[0] == 100) # Read the metadata. metadata = readinsight3.loadI3Metadata(mlist_name) assert(metadata.find("test").text == "test")
def _test_l1h(): # Test L1H. movie_name = storm_analysis.getData("test/data/test_l1h.dax") settings = storm_analysis.getData("test/data/test_l1h.xml") hres = storm_analysis.getPathOutputTest("test_l1h_list.hres") mlist = storm_analysis.getPathOutputTest("test_l1h_list.bin") storm_analysis.removeFile(hres) storm_analysis.removeFile(mlist) from storm_analysis.L1H.cs_analysis import analyze analyze(movie_name, settings, hres, mlist) # Verify number of localizations found. # # FIXME: Change L1H to use the HDF5 format. # num_locs = readinsight3.loadI3File(mlist)["x"].size if not veri.verifyIsCloseEnough(num_locs, 1986): raise Exception( "L1H did not find the expected number of localizations.")
def measurePSF(movie_name, zfile_name, movie_mlist, psf_name, want2d=False, aoi_size=12, z_range=750.0, z_step=50.0): """ The actual z range is 2x z_range (i.e. from -z_range to z_range). """ # Load dax file, z offset file and molecule list file. dax_data = datareader.inferReader(movie_name) z_offsets = None if os.path.exists(zfile_name): try: z_offsets = numpy.loadtxt(zfile_name, ndmin=2)[:, 1] except IndexError: z_offsets = None print("z offsets were not loaded.") i3_data = readinsight3.loadI3File(movie_mlist) if want2d: print("Measuring 2D PSF") else: print("Measuring 3D PSF") # # Go through the frames identifying good peaks and adding them # to the average psf. For 3D molecule z positions are rounded to # the nearest 50nm. # z_mid = int(z_range / z_step) max_z = 2 * z_mid + 1 average_psf = numpy.zeros((max_z, 4 * aoi_size, 4 * aoi_size)) peaks_used = 0 totals = numpy.zeros(max_z) [dax_x, dax_y, dax_l] = dax_data.filmSize() for curf in range(dax_l): # Select localizations in current frame & not near the edges. mask = (i3_data['fr'] == curf + 1) & (i3_data['x'] > aoi_size) & ( i3_data['x'] < (dax_x - aoi_size - 1)) & (i3_data['y'] > aoi_size) & (i3_data['y'] < (dax_y - aoi_size - 1)) xr = i3_data['x'][mask] yr = i3_data['y'][mask] # Use the z offset file if it was specified, otherwise use localization z positions. if z_offsets is None: if (curf == 0): print("Using fit z locations.") zr = i3_data['z'][mask] else: if (curf == 0): print("Using z offset file.") zr = numpy.ones(xr.size) * z_offsets[curf] ht = i3_data['h'][mask] # Remove localizations that are too close to each other. in_peaks = numpy.zeros((xr.size, util_c.getNPeakPar())) in_peaks[:, util_c.getXCenterIndex()] = xr in_peaks[:, util_c.getYCenterIndex()] = yr in_peaks[:, util_c.getZCenterIndex()] = zr in_peaks[:, util_c.getHeightIndex()] = ht out_peaks = util_c.removeNeighbors(in_peaks, 2 * aoi_size) #out_peaks = util_c.removeNeighbors(in_peaks, aoi_size) print(curf, "peaks in", in_peaks.shape[0], ", peaks out", out_peaks.shape[0]) # Use remaining localizations to calculate spline. image = dax_data.loadAFrame(curf).astype(numpy.float64) xr = out_peaks[:, util_c.getXCenterIndex()] yr = out_peaks[:, util_c.getYCenterIndex()] zr = out_peaks[:, util_c.getZCenterIndex()] ht = out_peaks[:, util_c.getHeightIndex()] for i in range(xr.size): xf = xr[i] yf = yr[i] zf = zr[i] xi = int(xf) yi = int(yf) if want2d: zi = 0 else: zi = int(round(zf / z_step) + z_mid) # check the z is in range if (zi > -1) and (zi < max_z): # get localization image mat = image[xi - aoi_size:xi + aoi_size, yi - aoi_size:yi + aoi_size] # zoom in by 2x psf = scipy.ndimage.interpolation.zoom(mat, 2.0) # re-center image psf = scipy.ndimage.interpolation.shift( psf, (-2.0 * (xf - xi), -2.0 * (yf - yi)), mode='nearest') # add to average psf accumulator average_psf[zi, :, :] += psf totals[zi] += 1 # Force PSF to be zero (on average) at the boundaries. for i in range(max_z): edge = numpy.concatenate((average_psf[i, 0, :], average_psf[i, -1, :], average_psf[i, :, 0], average_psf[i, :, -1])) average_psf[i, :, :] -= numpy.mean(edge) # Normalize the PSF. if want2d: max_z = 1 for i in range(max_z): print(i, totals[i]) if (totals[i] > 0.0): average_psf[i, :, :] = average_psf[i, :, :] / numpy.sum( numpy.abs(average_psf[i, :, :])) average_psf = average_psf / numpy.max(average_psf) # Save PSF (in image form). if True: import storm_analysis.sa_library.daxwriter as daxwriter dxw = daxwriter.DaxWriter( os.path.join(os.path.dirname(psf_name), "psf.dax"), average_psf.shape[1], average_psf.shape[2]) for i in range(max_z): dxw.addFrame(1000.0 * average_psf[i, :, :] + 100) dxw.close() # Save PSF. if want2d: psf_dict = {"psf": average_psf[0, :, :], "type": "2D"} else: cur_z = -z_range z_vals = [] for i in range(max_z): z_vals.append(cur_z) cur_z += z_step psf_dict = { "psf": average_psf, "pixel_size": 0.080, # 1/2 the camera pixel size in nm. "type": "3D", "zmin": -z_range, "zmax": z_range, "zvals": z_vals } with open(psf_name, 'wb') as fp: pickle.dump(psf_dict, fp)
def psfZStack(movie_name, i3_filename, zstack_name, scmos_cal=None, aoi_size=8, driftx=0.0, drifty=0.0): """ driftx, drifty are in units of pixels per frame, (bead x last frame - bead x first frame)/n_frames. """ # Load movie. movie_data = datareader.inferReader(movie_name) [movie_x, movie_y, movie_len] = movie_data.filmSize() # Load localizations. i3_data = readinsight3.loadI3File(i3_filename) x = i3_data["x"] y = i3_data["y"] # Load sCMOS calibration data. gain = numpy.ones((movie_y, movie_x)) offset = numpy.zeros((movie_y, movie_x)) if scmos_cal is not None: [offset, variance, gain] = numpy.load(scmos_cal) gain = 1.0 / gain z_stack = numpy.zeros((4 * aoi_size, 4 * aoi_size, movie_len)) for i in range(movie_len): if ((i % 50) == 0): print("Processing frame", i) # # Subtract pixel offset and convert to units of photo-electrons. # frame = (movie_data.loadAFrame(i) - offset) * gain # # Subtract estimated background. This assumes that the image is # mostly background and that the background is uniform. # frame = frame - numpy.median(frame) for j in range(x.size): xf = x[j] + driftx * float(i) yf = y[j] + drifty * float(i) xi = int(xf) yi = int(yf) im_slice = frame[xi - aoi_size:xi + aoi_size, yi - aoi_size:yi + aoi_size] im_slice_up = scipy.ndimage.interpolation.zoom(im_slice, 2.0) im_slice_up = scipy.ndimage.interpolation.shift( im_slice_up, (-2.0 * (xf - xi), -2.0 * (yf - yi)), mode='nearest') z_stack[:, :, i] += im_slice_up # Normalize by the number of localizations. z_stack = z_stack / float(x.size) print("max intensity", numpy.amax(z_stack)) # Save z_stack. numpy.save(zstack_name + ".npy", z_stack) # Save (normalized) z_stack as tif for inspection purposes. z_stack = z_stack / numpy.amax(z_stack) z_stack = z_stack.astype(numpy.float32) with tifffile.TiffWriter(zstack_name + ".tif") as tf: for i in range(movie_len): tf.save(z_stack[:, :, i])
if (len(sys.argv) != 4): print( "usage: homotopy_psf <dax_file, input> <bin_file, input> <npy_file, output>" ) exit() # Minimum number of peaks to calculate the PSF from. min_peaks = 300 # Half width of the aoi size in pixels. aoi_size = 8 # Load dax file and corresponding molecule list file. dax_data = datareader.inferReader(sys.argv[1]) i3_data = readinsight3.loadI3File(sys.argv[2]) # Go through the frames identifying good peaks and adding them # to the average psf average_psf = numpy.zeros((4 * aoi_size, 4 * aoi_size)) curf = 1 peaks_used = 0 total = 0.0 [dax_x, dax_y, dax_l] = dax_data.filmSize() while (curf < dax_l) and (peaks_used < min_peaks): # Select localizations in current frame & not near the edges. mask = (i3_data['fr'] == curf) & (i3_data['x'] > aoi_size) & ( i3_data['x'] < (dax_y - aoi_size - 1)) & (i3_data['y'] > aoi_size) & ( i3_data['y'] < (dax_x - aoi_size - 1)) xr = i3_data['x'][mask]
def peakFinding(find_peaks, movie_file, mlist_file, parameters): # open files for input & output movie_data = datareader.inferReader(movie_file) [movie_x,movie_y,movie_l] = movie_data.filmSize() # if the i3 file already exists, read it in, # write it out & start the analysis from the # end. total_peaks = 0 if(os.path.exists(mlist_file)): print("Found", mlist_file) i3data_in = readinsight3.loadI3File(mlist_file) try: curf = int(numpy.max(i3data_in['fr'])) except ValueError: curf = 0 print(" Starting analysis at frame:", curf) i3data = writeinsight3.I3Writer(mlist_file) if (curf > 0): i3data.addMolecules(i3data_in) total_peaks = i3data_in['x'].size else: curf = 0 i3data = writeinsight3.I3Writer(mlist_file) # process parameters if parameters.hasAttr("start_frame"): if (parameters.getAttr("start_frame")>=curf) and (parameters.getAttr("start_frame")<movie_l): curf = parameters.getAttr("start_frame") if parameters.hasAttr("max_frame"): if (parameters.getAttr("max_frame")>0) and (parameters.getAttr("max_frame")<movie_l): movie_l = parameters.getAttr("max_frame") static_bg_estimator = None if (parameters.getAttr("static_background_estimate", 0) > 0): print("Using static background estimator.") static_bg_estimator = static_background.StaticBGEstimator(movie_data, start_frame = curf, sample_size = parameters.getAttr("static_background_estimate")) # analyze the movie # catch keyboard interrupts & "gracefully" exit. try: while(curf<movie_l): #for j in range(l): # Set up the analysis. image = movie_data.loadAFrame(curf) - parameters.getAttr("baseline") mask = (image < 1.0) if (numpy.sum(mask) > 0): print(" Removing negative values in frame", curf) image[mask] = 1.0 # Find and fit the peaks. if static_bg_estimator is not None: bg_estimate = static_bg_estimator.estimateBG(curf) - parameters.getAttr("baseline") [peaks, residual] = find_peaks.analyzeImage(image, bg_estimate = bg_estimate) else: [peaks, residual] = find_peaks.analyzeImage(image) # Save the peaks. if (type(peaks) == type(numpy.array([]))): # remove unconverged peaks peaks = find_peaks.getConvergedPeaks(peaks) # save results if(parameters.getAttr("orientation", "normal") == "inverted"): i3data.addMultiFitMolecules(peaks, movie_x, movie_y, curf+1, parameters.getAttr("pixel_size"), inverted = True) else: i3data.addMultiFitMolecules(peaks, movie_x, movie_y, curf+1, parameters.getAttr("pixel_size"), inverted = False) total_peaks += peaks.shape[0] print("Frame:", curf, peaks.shape[0], total_peaks) else: print("Frame:", curf, 0, total_peaks) curf += 1 print("") i3data.close() find_peaks.cleanUp() return 0 except KeyboardInterrupt: print("Analysis stopped.") i3data.close() find_peaks.cleanUp() return 1
import storm_analysis.sa_library.datareader as datareader import storm_analysis.sa_library.readinsight3 as readinsight3 if (len(sys.argv)!=4): print("usage: homotopy_psf <dax_file, input> <bin_file, input> <npy_file, output>") exit() # Minimum number of peaks to calculate the PSF from. min_peaks = 300 # Half width of the aoi size in pixels. aoi_size = 8 # Load dax file and corresponding molecule list file. dax_data = datareader.inferReader(sys.argv[1]) i3_data = readinsight3.loadI3File(sys.argv[2]) # Go through the frames identifying good peaks and adding them # to the average psf average_psf = numpy.zeros((4*aoi_size,4*aoi_size)) curf = 1 peaks_used = 0 total = 0.0 [dax_x, dax_y, dax_l] = dax_data.filmSize() while (curf < dax_l) and (peaks_used < min_peaks): # Select localizations in current frame & not near the edges. mask = (i3_data['fr'] == curf) & (i3_data['x'] > aoi_size) & (i3_data['x'] < (dax_y - aoi_size - 1)) & (i3_data['y'] > aoi_size) & (i3_data['y'] < (dax_x - aoi_size - 1)) xr = i3_data['x'][mask] yr = i3_data['y'][mask] ht = i3_data['h'][mask]
import numpy import storm_analysis.sa_library.i3dtype as i3dtype import storm_analysis.sa_library.readinsight3 as readinsight3 parser = argparse.ArgumentParser(description = "Check emitter photophysics.") parser.add_argument('--bin', dest='i3bin', type=str, required=True, help = "The name of Insight3 format file with the emitter positions.") parser.add_argument('--maxi', dest='maxi', type=int, required=False, help = "The maximum number of emitters to check.") args = parser.parse_args() # Initialization. i3_data = readinsight3.loadI3File(args.i3bin) if args.maxi is None: args.maxi = numpy.max(i3_data['i']) + 1 if (args.maxi > (numpy.max(i3_data['i']) + 1)): args.maxi = numpy.max(i3_data['i']) + 1 max_frame = numpy.max(i3_data['fr']) on_times = [] off_times = [] for i in range(int(args.maxi)): if ((i % 100) == 0): print("Analyzing emitter", i)
import storm_analysis.simulator.drift as drift import storm_analysis.simulator.photophysics as photophysics import storm_analysis.simulator.psf as psf import storm_analysis.simulator.simulate as simulate frames = 100 x_size = 300 y_size = 310 z_planes = [0.0] #z_planes = [-250.0, 250.0] #z_planes = [-250.0, 250.0] #z_planes = [-750.0, -250.0, 250.0, 750.0] z_value = -500.0 # Load emitter locations. i3_locs = readinsight3.loadI3File("emitters.bin") if False: i3_locs = i3_locs[0] # Make a bin file with emitter locations for each frame. with writeinsight3.I3Writer("test_olist.bin") as i3w: for i in range(frames): i3_temp = i3_locs.copy() i3dtype.setI3Field(i3_temp, "fr", i + 1) i3dtype.posSet(i3_temp, "z", z_value) i3w.addMolecules(i3_temp) # Load channel to channel mapping file. with open("map.map", 'rb') as fp: mappings = pickle.load(fp)
def measurePSF(movie_name, zfile_name, movie_mlist, psf_name, want2d = False, aoi_size = 12, z_range = 750.0, z_step = 50.0): """ The actual z range is 2x z_range (i.e. from -z_range to z_range). """ # Load dax file, z offset file and molecule list file. dax_data = datareader.inferReader(movie_name) z_offsets = None if os.path.exists(zfile_name): try: z_offsets = numpy.loadtxt(zfile_name, ndmin = 2)[:,1] except IndexError: z_offsets = None print("z offsets were not loaded.") i3_data = readinsight3.loadI3File(movie_mlist) if want2d: print("Measuring 2D PSF") else: print("Measuring 3D PSF") # # Go through the frames identifying good peaks and adding them # to the average psf. For 3D molecule z positions are rounded to # the nearest 50nm. # z_mid = int(z_range/z_step) max_z = 2 * z_mid + 1 average_psf = numpy.zeros((max_z,4*aoi_size,4*aoi_size)) peaks_used = 0 totals = numpy.zeros(max_z) [dax_x, dax_y, dax_l] = dax_data.filmSize() for curf in range(dax_l): # Select localizations in current frame & not near the edges. mask = (i3_data['fr'] == curf+1) & (i3_data['x'] > aoi_size) & (i3_data['x'] < (dax_x - aoi_size - 1)) & (i3_data['y'] > aoi_size) & (i3_data['y'] < (dax_y - aoi_size - 1)) xr = i3_data['x'][mask] yr = i3_data['y'][mask] # Use the z offset file if it was specified, otherwise use localization z positions. if z_offsets is None: if (curf == 0): print("Using fit z locations.") zr = i3_data['z'][mask] else: if (curf == 0): print("Using z offset file.") zr = numpy.ones(xr.size) * z_offsets[curf] ht = i3_data['h'][mask] # Remove localizations that are too close to each other. in_peaks = numpy.zeros((xr.size,util_c.getNPeakPar())) in_peaks[:,util_c.getXCenterIndex()] = xr in_peaks[:,util_c.getYCenterIndex()] = yr in_peaks[:,util_c.getZCenterIndex()] = zr in_peaks[:,util_c.getHeightIndex()] = ht out_peaks = util_c.removeNeighbors(in_peaks, 2*aoi_size) #out_peaks = util_c.removeNeighbors(in_peaks, aoi_size) print(curf, "peaks in", in_peaks.shape[0], ", peaks out", out_peaks.shape[0]) # Use remaining localizations to calculate spline. image = dax_data.loadAFrame(curf).astype(numpy.float64) xr = out_peaks[:,util_c.getXCenterIndex()] yr = out_peaks[:,util_c.getYCenterIndex()] zr = out_peaks[:,util_c.getZCenterIndex()] ht = out_peaks[:,util_c.getHeightIndex()] for i in range(xr.size): xf = xr[i] yf = yr[i] zf = zr[i] xi = int(xf) yi = int(yf) if want2d: zi = 0 else: zi = int(round(zf/z_step) + z_mid) # check the z is in range if (zi > -1) and (zi < max_z): # get localization image mat = image[xi-aoi_size:xi+aoi_size, yi-aoi_size:yi+aoi_size] # zoom in by 2x psf = scipy.ndimage.interpolation.zoom(mat, 2.0) # re-center image psf = scipy.ndimage.interpolation.shift(psf, (-2.0*(xf-xi), -2.0*(yf-yi)), mode='nearest') # add to average psf accumulator average_psf[zi,:,:] += psf totals[zi] += 1 # Force PSF to be zero (on average) at the boundaries. for i in range(max_z): edge = numpy.concatenate((average_psf[i,0,:], average_psf[i,-1,:], average_psf[i,:,0], average_psf[i,:,-1])) average_psf[i,:,:] -= numpy.mean(edge) # Normalize the PSF. if want2d: max_z = 1 for i in range(max_z): print(i, totals[i]) if (totals[i] > 0.0): average_psf[i,:,:] = average_psf[i,:,:]/numpy.sum(numpy.abs(average_psf[i,:,:])) average_psf = average_psf/numpy.max(average_psf) # Save PSF (in image form). if True: import storm_analysis.sa_library.daxwriter as daxwriter dxw = daxwriter.DaxWriter(os.path.join(os.path.dirname(psf_name), "psf.dax"), average_psf.shape[1], average_psf.shape[2]) for i in range(max_z): dxw.addFrame(1000.0 * average_psf[i,:,:] + 100) dxw.close() # Save PSF. if want2d: psf_dict = {"psf" : average_psf[0,:,:], "type" : "2D"} else: cur_z = -z_range z_vals = [] for i in range(max_z): z_vals.append(cur_z) cur_z += z_step psf_dict = {"psf" : average_psf, "pixel_size" : 0.080, # 1/2 the camera pixel size in nm. "type" : "3D", "zmin" : -z_range, "zmax" : z_range, "zvals" : z_vals} pickle.dump(psf_dict, open(psf_name, 'wb'))
parser.add_argument('--mm_map', dest='mm_map', type=str, required=True, help="The name of the micrometry map file.") parser.add_argument( '--results', dest='results', type=str, required=True, help="The name of the file to save the updated mapping in.") args = parser.parse_args() # Load locs1. i3_data1 = readinsight3.loadI3File(args.locs1) kd1 = scipy.spatial.KDTree( numpy.stack((i3_data1['xc'], i3_data1['yc']), axis=-1)) # Load locs2. i3_data2 = readinsight3.loadI3File(args.locs2) kd2 = scipy.spatial.KDTree( numpy.stack((i3_data2['xc'], i3_data2['yc']), axis=-1)) # Load 'first guess' transform. with open(args.mm_map, 'rb') as fp: mp_transform = pickle.load(fp) # Refine. [tr_1_to_0, tr_0_to_1 ] = refineTransform(kd1, kd2,
def peakFinding(find_peaks, movie_file, mlist_file, parameters): """ Does the peak finding. """ # open files for input & output movie_data = datareader.inferReader(movie_file) [movie_x, movie_y, movie_l] = movie_data.filmSize() # if the i3 file already exists, read it in, # write it out & start the analysis from the # end. total_peaks = 0 if (os.path.exists(mlist_file)): print("Found", mlist_file) i3data_in = readinsight3.loadI3File(mlist_file) try: curf = int(numpy.max(i3data_in['fr'])) except ValueError: curf = 0 print(" Starting analysis at frame:", curf) i3data = writeinsight3.I3Writer(mlist_file) if (curf > 0): i3data.addMolecules(i3data_in) total_peaks = i3data_in['x'].size else: curf = 0 i3data = writeinsight3.I3Writer(mlist_file) # process parameters if parameters.hasAttr("start_frame"): if (parameters.getAttr("start_frame") >= curf) and (parameters.getAttr("start_frame") < movie_l): curf = parameters.getAttr("start_frame") max_frame = movie_l if parameters.hasAttr("max_frame"): if (parameters.getAttr("max_frame") > 0) and (parameters.getAttr("max_frame") < movie_l): max_frame = parameters.getAttr("max_frame") static_bg_estimator = None if (parameters.getAttr("static_background_estimate", 0) > 0): print("Using static background estimator.") static_bg_estimator = static_background.StaticBGEstimator( movie_data, start_frame=curf, sample_size=parameters.getAttr("static_background_estimate")) # analyze the movie # catch keyboard interrupts & "gracefully" exit. try: while (curf < max_frame): #for j in range(l): # Set up the analysis. image = movie_data.loadAFrame(curf) - parameters.getAttr( "baseline") mask = (image < 1.0) if (numpy.sum(mask) > 0): print(" Removing negative values in frame", curf) image[mask] = 1.0 # Find and fit the peaks. if static_bg_estimator is not None: bg_estimate = static_bg_estimator.estimateBG( curf) - parameters.getAttr("baseline") [peaks, residual] = find_peaks.analyzeImage(image, bg_estimate=bg_estimate) else: [peaks, residual] = find_peaks.analyzeImage(image) # Save the peaks. if (type(peaks) == type(numpy.array([]))): # remove unconverged peaks peaks = find_peaks.getConvergedPeaks(peaks) # save results if (parameters.getAttr("orientation", "normal") == "inverted"): i3data.addMultiFitMolecules( peaks, movie_x, movie_y, curf + 1, parameters.getAttr("pixel_size"), inverted=True) else: i3data.addMultiFitMolecules( peaks, movie_x, movie_y, curf + 1, parameters.getAttr("pixel_size"), inverted=False) total_peaks += peaks.shape[0] print("Frame:", curf, peaks.shape[0], total_peaks) else: print("Frame:", curf, 0, total_peaks) curf += 1 print("") if parameters.getAttr("append_metadata", 0): etree = ElementTree.Element("xml") # Add analysis parameters. etree.append(parameters.toXMLElementTree()) # Add movie properties. movie_props = ElementTree.SubElement(etree, "movie") field = ElementTree.SubElement(movie_props, "hash_value") field.text = movie_data.hashID() for elt in [["movie_x", movie_x], ["movie_y", movie_y], ["movie_l", movie_l]]: field = ElementTree.SubElement(movie_props, elt[0]) field.text = str(elt[1]) i3data.closeWithMetadata(ElementTree.tostring(etree, 'ISO-8859-1')) else: i3data.close() find_peaks.cleanUp() return 0 except KeyboardInterrupt: print("Analysis stopped.") i3data.close() find_peaks.cleanUp() return 1
def simulate(self, dax_file, bin_file, n_frames): # # Initialization. # dax_data = daxwriter.DaxWriter(dax_file, self.x_size, self.y_size) i3_data_in = readinsight3.loadI3File(bin_file) out_fname_base = dax_file[:-4] i3_data_out = writeinsight3.I3Writer(out_fname_base + "_olist.bin") sim_settings = open(out_fname_base + "_sim_params.txt", "w") # # Create the user-specified class instances that will do # most of the actual work of the simulation. # bg = self.bg_factory(sim_settings, self.x_size, self.y_size, i3_data_in) cam = self.cam_factory(sim_settings, self.x_size, self.y_size, i3_data_in) drift = None if self.drift_factory is not None: drift = self.drift_factory(sim_settings, self.x_size, self.y_size, i3_data_in) pp = self.pphys_factory(sim_settings, self.x_size, self.y_size, i3_data_in) psf = self.psf_factory(sim_settings, self.x_size, self.y_size, i3_data_in) sim_settings.write( json.dumps({ "simulation": { "bin_file": bin_file, "x_size": str(self.x_size), "y_size": str(self.y_size) } }) + "\n") # # Generate the simulated movie. # for i in range(n_frames): # Generate the new image. image = numpy.zeros((self.x_size, self.y_size)) # Get the emitters that are on in the current frame. cur_i3 = pp.getEmitters(i).copy() print("Frame", i, cur_i3['x'].size, "emitters") # Add background to image. image += bg.getBackground(i) # Set 'bg' parameter of the emitters. cur_i3 = bg.getEmitterBackground(cur_i3) # Apply drift to the localizations. if drift is not None: drift.drift(i, cur_i3) # Foreground image += psf.getPSFs(cur_i3) # Camera image = cam.readImage(image) # Save the image. dax_data.addFrame(image) # Save the molecule locations. cur_i3['fr'] = i + 1 i3_data_out.addMolecules(cur_i3) dax_data.close() i3_data_out.close() sim_settings.close()
dest='channel', type=int, required=False, default=0, help="Channel that the localizations come from. Default is channel 0.") args = parser.parse_args() basename = os.path.splitext(args.i3bin)[0] # Load mappings. with open(args.mapping, 'rb') as fp: mappings = pickle.load(fp) # Load x,y locations. i3_in = readinsight3.loadI3File(args.i3bin) xi = i3_in["x"] yi = i3_in["y"] # Map to back channel 0 if necessary. if (args.channel != 0): xt_name = str(args.channel) + "_0_x" yt_name = str(args.channel) + "_0_y" [xi, yi] = mapXYLocations(xi, yi, mappings[xt_name], mappings[yt_name]) for i in range(8): if (i == 0): i3dtype.posSet(i3_in, "x", xi) i3dtype.posSet(i3_in, "y", yi) else:
def homotopyPSF(dax_file, bin_file, psf_file): # Minimum number of peaks to calculate the PSF from. min_peaks = 300 # Half width of the aoi size in pixels. aoi_size = 8 # Load dax file and corresponding molecule list file. dax_data = datareader.inferReader(dax_file) i3_data = readinsight3.loadI3File(bin_file) # Go through the frames identifying good peaks and adding them # to the average psf average_psf = numpy.zeros((4*aoi_size,4*aoi_size)) curf = 1 peaks_used = 0 total = 0.0 [dax_x, dax_y, dax_l] = dax_data.filmSize() while (curf < dax_l) and (peaks_used < min_peaks): # Select localizations in current frame & not near the edges. mask = (i3_data['fr'] == curf) & (i3_data['x'] > aoi_size) & (i3_data['x'] < (dax_y - aoi_size - 1)) & (i3_data['y'] > aoi_size) & (i3_data['y'] < (dax_x - aoi_size - 1)) xr = i3_data['x'][mask] yr = i3_data['y'][mask] ht = i3_data['h'][mask] # Remove localizations that are too close to each other. mask = iaUtilsC.removeNeighborsMask(xr, yr, aoi_size) print(curf, "peaks in", xr.size, ", peaks out", numpy.count_nonzero(mask)) xr = xr[mask] yr = yr[mask] ht = ht[mask] # Use remaining localizations to calculate spline. image = dax_data.loadAFrame(curf-1).astype(numpy.float64) for i in range(xr.size): xf = xr[i] yf = yr[i] xi = int(xf) yi = int(yf) # get localization image mat = image[xi-aoi_size:xi+aoi_size, yi-aoi_size:yi+aoi_size] # re-center image psf = scipy.ndimage.interpolation.shift(mat,(-(xf-xi),-(yf-yi)),mode='nearest') # zoom in by 2x psf = scipy.ndimage.interpolation.zoom(psf,2.0) # add to average psf accumulator average_psf += psf total += ht[i] peaks_used += 1 curf += 1 average_psf = average_psf/total average_psf = numpy.transpose(average_psf) # force psf to be zero (on average) at the boundaries. if True: edge = numpy.concatenate((average_psf[0,:], average_psf[-1,:], average_psf[:,0], average_psf[:,-1])) average_psf -= numpy.mean(edge) # save PSF (in numpy form). numpy.save(psf_file, average_psf) # save PSF (in image form). # # FIXME: This may be useful but it is annoying for automated testing as this file # is created in which ever directory the tests are run in. # if True: import tifffile tifffile.imsave("l1h_psf.tif", average_psf.astype(numpy.float32))
parser.add_argument( '--bin', dest='i3bin', type=str, required=True, help="The name of Insight3 format file with the emitter positions.") parser.add_argument('--maxi', dest='maxi', type=int, required=False, help="The maximum number of emitters to check.") args = parser.parse_args() # Initialization. i3_data = readinsight3.loadI3File(args.i3bin) if args.maxi is None: args.maxi = numpy.max(i3_data['i']) + 1 if (args.maxi > (numpy.max(i3_data['i']) + 1)): args.maxi = numpy.max(i3_data['i']) + 1 max_frame = numpy.max(i3_data['fr']) on_times = [] off_times = [] for i in range(int(args.maxi)): if ((i % 100) == 0): print("Analyzing emitter", i)
def mergeAnalysis(dir_name, bin_base_name, extensions=[".bin"]): # Create Insight3 file writers. i3_out = [] for ext in extensions: i3_out.append(writeinsight3.I3Writer(bin_base_name + ext)) # Find all the job*.xml files. job_xml_files = glob.glob(dir_name + "job*.xml") # Sort job files. job_xml_files = sorted( job_xml_files, key=lambda x: int( os.path.splitext(os.path.basename(x))[0].split("_")[1])) # Check for corresponding mlist.bin files. metadata = None last_frame = 0 for i in range(len(job_xml_files)): job_complete = True for j, ext in enumerate(extensions): mlist_name = dir_name + "p_" + str(i + 1) + "_mlist" + ext if os.path.exists(mlist_name) and readinsight3.checkStatus( mlist_name): # Load metadata from the first file. if (i == 0) and (j == 0): metadata = readinsight3.loadI3Metadata(mlist_name) # Read localizations. i3_data = readinsight3.loadI3File(mlist_name, verbose=False) # Check for empty file. if (i3_data.size == 0): print("No localizations found in", mlist_name) else: # Print frame range covered. if (j == 0): last_frame = i3_data["fr"][-1] print(i3_data["fr"][0], last_frame, mlist_name) # Add localizations to the output file. i3_out[j].addMolecules(i3_data) else: job_complete = False break if not job_complete: print("Merge failed because", job_xml_files[i], "is incomplete.") for j, ext in enumerate(extensions): i3_out[j].close() os.remove(bin_base_name + ext) assert (False) if metadata is None: print("No metadata found.") for i3w in i3_out: i3w.close() else: # Fix movie length node based on the last frame of the last molecule. metadata.find("movie").find("movie_l").text = str(last_frame) # Also need to fix analysis end points. We are assuming that the # entire movie was analyzed. metadata.find("settings").find("start_frame").text = "-1" metadata.find("settings").find("max_frame").text = "-1" for i3w in i3_out: i3w.closeWithMetadata(ElementTree.tostring(metadata, 'ISO-8859-1'))
def verifyNumberLocalizations(bin_fname): """ Return the number of localizations in a I3 file. """ return readinsight3.loadI3File(bin_fname, verbose = False).size
def peakFinding(find_peaks, movie_file, mlist_file, parameters): # open files for input & output movie_data = datareader.inferReader(movie_file) [movie_x, movie_y, movie_l] = movie_data.filmSize() # if the i3 file already exists, read it in, # write it out & start the analysis from the # end. total_peaks = 0 if (os.path.exists(mlist_file)): print("Found", mlist_file) i3data_in = readinsight3.loadI3File(mlist_file) try: curf = int(numpy.max(i3data_in['fr'])) except ValueError: curf = 0 print(" Starting analysis at frame:", curf) i3data = writeinsight3.I3Writer(mlist_file) if (curf > 0): i3data.addMolecules(i3data_in) total_peaks = i3data_in['x'].size else: curf = 0 i3data = writeinsight3.I3Writer(mlist_file) # process parameters if hasattr(parameters, "start_frame"): if (parameters.start_frame >= curf) and (parameters.start_frame < movie_l): curf = parameters.start_frame if hasattr(parameters, "max_frame"): if (parameters.max_frame > 0) and (parameters.max_frame < movie_l): movie_l = parameters.max_frame static_bg_estimator = None if hasattr(parameters, "static_background_estimate"): if (parameters.static_background_estimate > 0): print("Using static background estimator.") static_bg_estimator = static_background.StaticBGEstimator( movie_data, start_frame=curf, sample_size=parameters.static_background_estimate) # analyze the movie # catch keyboard interrupts & "gracefully" exit. try: while (curf < movie_l): #for j in range(l): # Set up the analysis. image = movie_data.loadAFrame(curf) - parameters.baseline mask = (image < 1.0) if (numpy.sum(mask) > 0): print(" Removing negative values in frame", curf) image[mask] = 1.0 # Find and fit the peaks. if static_bg_estimator is not None: bg_estimate = static_bg_estimator.estimateBG( curf) - parameters.baseline [peaks, residual] = find_peaks.analyzeImage(image, bg_estimate=bg_estimate) else: [peaks, residual] = find_peaks.analyzeImage(image) # Save the peaks. if (type(peaks) == type(numpy.array([]))): # remove unconverged peaks peaks = find_peaks.getConvergedPeaks(peaks) # save results if (parameters.orientation == "inverted"): i3data.addMultiFitMolecules(peaks, movie_x, movie_y, curf + 1, parameters.pixel_size, inverted=True) else: i3data.addMultiFitMolecules(peaks, movie_x, movie_y, curf + 1, parameters.pixel_size, inverted=False) total_peaks += peaks.shape[0] print("Frame:", curf, peaks.shape[0], total_peaks) else: print("Frame:", curf, 0, total_peaks) curf += 1 print("") i3data.close() find_peaks.cleanUp() return 0 except KeyboardInterrupt: print("Analysis stopped.") i3data.close() find_peaks.cleanUp() return 1
def __init__(self, bin_name=None, **kwds): super().__init__(**kwds) self.i3_data = readinsight3.loadI3File(bin_name) self.nm_per_pixel = self.parameters.getAttr("pixel_size")