def read(cls, cleftId, bound1Id, bound2Id, file, clean=True, byteOrder=None, dataType=None, arrayOrder=None, shape=None): """ Reads segmented image (label filed) from a file and sets required ids. Each of the id args (cleftId, bound1Id, bound2Id) can be a single int or a list (array) in which cases a (boundary or cleft) is formed form all specified ids. If file is in em or mrc format (file extension em or mrc) only the file argument is needed. If the file is in the raw data format (file extension raw) all arguments are required. Arguments: - bound1Id, bound2Id, cleftId: ids of boundaries 1, 2 and cleft - file: file name - clean: if true, only the segments corresponding to ids are kept - byteOrder: '<' (little-endian), '>' (big-endian) - dataType: any of the numpy types, e.g.: 'int8', 'int16', 'int32', 'float32', 'float64' - arrayOrder: 'C' (z-axis fastest), or 'F' (x-axis fastest) - shape: (x_dim, y_dim, z_dim) Returns: - instance of Segment """ # read file from pyto.io.image_io import ImageIO as ImageIO fi = ImageIO() fi.read(file=file, byteOrder=byteOrder, dataType=dataType, arrayOrder=arrayOrder, shape=shape) # make new instance cleft = cls(data=fi.data, cleftId=cleftId, bound1Id=bound1Id, bound2Id=bound2Id, copy=False, clean=clean) return cleft
def sortPaths(self, paths, mode='num', sequence=None): """ Sorts list of file path and returns it. If arg mode is 'num', paths are sorted by series number, which are expected to be between the rightmost '_' and '.' in the file names. If arg mode is 'tilt_angle', paths are sorted by tilt angle. If arg mode is 'sequence', paths are sorted by arg sequence that has to have the same length as paths. Arguments: - paths: list of file paths (with or without directory) - mode: sorting mode, 'num', 'tilt_angle', or 'sequence' - sequence: sequence of values corresponding to paths Returns sorted list of paths. """ if mode == 'tilt_angle': # get tilt angles from file headers unsorted = [] for file_ in paths: image = ImageIO(file=file_) image.readHeader() unsorted.append(image.tiltAngle) elif mode == 'num': # get file numbers unsorted = [int(self.splitBase(base=file_)[1]) for file_ in paths] elif mode == 'sequence': unsorted = sequence else: raise ValueError('Mode ', mode, " not understood. Allowed values ", "are 'num', 'tilt_angle' and 'sequence'.") # sort file numbers and paths unsorted = numpy.array(unsorted) sort_ind = unsorted.argsort() sorted_paths = [paths[ind] for ind in sort_ind] return sorted_paths
def setUp(self): """ Sets absolute path to this file directory and saves it as self.dir """ # set absolute path to current dir working_dir = os.getcwd() file_dir, name = os.path.split(__file__) self.dir = os.path.join(working_dir, file_dir) # make raw file self.raw_shape = (4, 3, 2) self.raw_dtype = 'int16' self.raw_data = numpy.arange(24, dtype=self.raw_dtype).reshape( self.raw_shape) raw = ImageIO() self.raw_file_name = 'data.raw' raw.write(file=self.raw_file_name, data=self.raw_data)
def modify(cls, old, new, fun, fun_kwargs={}, memmap=True): """ Reads an image (arg old), modifies old.data using function passed as arg fun and writes an image containing the modified data as a new image (arg new). The function passes (arg fun) has to have signature fun(Image, **fun_kwargs) and to return image data (ndarray). Meant for mrc files. The new image will have exactly the same header as the old image, except for the shape, length and min/max/mean values, which are set according to the new image data. Also works if old is an mrc and new is a raw file. Arguments: - old: old mrc image file name - new: new (subtomogram) mrc image file name - fun: function that takes old.data as an argument - memmap: if True, read memory map instead of the whole image Returns an instance of this class that holds the new image. This instance contains attribute image_io (pyto.io.ImageIO) that was used to write the new file. """ # read header from pyto.io.image_io import ImageIO as ImageIO image_io = ImageIO() image_io.readHeader(file=old) # read image image = cls.read(file=old, memmap=memmap) # modify data data = fun(image, **fun_kwargs) # write new (modified data) image_io.setData(data=data) image_io.write(file=new, pixel=image_io.pixel) #image_io.setFileFormat(file_=new) #if image_io.fileFormat == 'mrc': # image_io.write(file=new, pixel=image_io.pixel) #elif image_io.fileFormat == 'raw': # image_io.write(file=new, pixel=image_io.pixel) # image.image_io = image_io return image
def fix(self, mode=None, microscope=None, out=None, pad=0, start=1): """ Fixes wrong data in headers. Mode determines which values are fixed. Currently defined modes are: - 'polara_fei-tomo': images obtained on Polara (at MPI of Biochemistry) using FEI tomography package and saved in EM format. - 'krios_fei-tomo': images obtained on Krios (at MPI of Biochemistry) using FEI tomography package and saved in EM format. - 'cm300': images from cm300 in EM format - None: doesn't do anyhing If mode is 'polara_fei-tomo', then arg microscope has to be specified. The allowed values are specified in microscope_db.py. Currently (r564) these are: 'polara-1_01-07', 'polara-1_01-09' and 'polara-2_01-09'. Works for individual images only (not for stacks), because stacks are typically recorded by SerialEM and do not need fixing. Arguments: - out: directory where the fixed images are written. The fixed and the original images have the same base names. - mode: fix mode """ # parse arguments #if path is None: path = self.path #if mode is None: mode = self.mode # check for out, pad and start also? # make out directory if it doesn't exists if (out is not None) and (not os.path.isdir(out)): os.makedirs(out) # loop over images images_iter = self.images(out=out, pad=pad, start=start) for (in_path, out_path) in images_iter: # read image file image = ImageIO(file=in_path) image.read() # fix image.fix(mode=mode, microscope=microscope) # write fixed files image.write(file=out_path)
def read_pixel_size(cls, image_path): """ Reads pixel size from an image file. Raises ValueError if pixel size can not be read from the image Argument: - image_path: image path Returns: pixel size in A """ image_io = ImageIO() if image_path.endswith('.st'): image_io.readHeader(file=image_path, fileFormat='mrc') else: image_io.readHeader(file=image_path) if image_io.pixel is not None: if isinstance(image_io.pixel, (list, tuple)): pixel_a = 10 * image_io.pixel[0] else: pixel_a = 10 * image_io.pixel else: raise ValueError("Pixel size could not be found from image " + image_path + ". Please specify pixel_a as an argument.") # in case of 0 pix size if pixel_a == 0: raise ValueError("Pixel size could not be found from image " + image_path + ". Please specify pixel_a as an argument.") return pixel_a
def load_tomo(fname, mmap=False): """ Loads a tomogram in MRC, EM or VTI format and converts it into a numpy format. Args: fname (str): full path to the tomogram, has to end with '.mrc', '.em' or '.vti' mmap (boolean, optional): if True (default False) a numpy.memmap object is loaded instead of numpy.ndarray, which means that data are not loaded completely to memory, this is useful only for very large tomograms. Only valid with formats MRC and EM. VERY IMPORTANT: This subclass of ndarray has some unpleasant interaction with some operations, because it does not quite fit properly as a subclass of numpy.ndarray Returns: numpy.ndarray or numpy.memmap object """ # Input parsing stem, ext = os.path.splitext(fname) if mmap and (not (ext == '.mrc' or (ext == '.em'))): raise pexceptions.PySegInputError( expr='load_tomo', msg=('mmap option is only valid for MRC or EM formats, current ' + ext)) elif ext == '.mrc': image = ImageIO() image.readMRC(fname, memmap=mmap) im_data = image.data elif ext == '.em': image = ImageIO() image.readEM(fname, memmap=mmap) im_data = image.data elif ext == '.vti': reader = vtk.vtkXMLImageDataReader() reader.SetFileName(fname) reader.Update() im_data = vti_to_numpy(reader.GetOutput()) else: raise pexceptions.PySegInputError( expr='load_tomo', msg='{} is non valid format.'.format(ext)) # For avoiding 2D arrays if len(im_data.shape) == 2: im_data = np.reshape(im_data, (im_data.shape[0], im_data.shape[1], 1)) return im_data
def testPixelSize(self): """ Tests pixel size in read and write """ # arrays #ar_int8_2 = numpy.arange(24, dtype='int8').reshape((4,3,2)) ar_int16_2 = numpy.arange(24, dtype='int16').reshape((4, 3, 2)) # file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.mrc'), data=ar_int16_2, pixel=2.1) file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.mrc')) np_test.assert_almost_equal(file_in.pixel, 2.1)
def cut(cls, old, new, inset, memmap=True): """ Reads an image (arg old), cuts a subtomo defined by arg inset and writes the new image (arg new). Meant for mrc files. The new image will have exactly the same header as the old image, except for the shape, length and min/max/mean values, which are set according to the new image data. Arguments: - old: old mrc image file name - new: new (subtomogram) mrc image file name - inset: defines the subtomogram - memmap: if True, read memory map instead of the whole image Returns an instance of this class that holds the new image. This instance contains attribute image_io (pyto.io.ImageIO) that was used to write the new file. """ # read header from pyto.io.image_io import ImageIO as ImageIO image_io = ImageIO() image_io.readHeader(file=old) # read image and cut inset image = cls.read(file=old, memmap=memmap) image.useInset(inset=inset) # write new image_io.setData(data=image.data) image_io.write(file=new, pixel=image_io.pixel) # image.image_io = image_io return image
def find(cls, image_dir, image_prefix, ctf_dir, params, pixel_a=None, flatten='auto', tool='ctffind', executable=None, param_file='ctf_params.txt', fast=False, max_images=None, plot_ctf=True, plot_ps=True, b_plot=True, exp_f_plot=False, show_legend=True, plot_phases=True, plot_defoci=True, plot_resolution=True, print_results=True, print_validation=False): """ Determines and shows CTF fits for multiple images. All files located in (arg) image_dir whose namess start with (arg) image_prefix and that have extension mrc, em or st are selected for the ctf determination. If a selected file is 3D (image stack), and arg flatten is True or 'auto', all z-slices are summed up (saved in ctf_dir) and the ctf is detemined on the resulting (flattened. Alternatively, if arg flatten is False, z-slices are extracted, saved in ctf_dir and analyzed separately. All resulting files, as well as the extraced or flattened images (in case of 3D files) are saved or moved to directory ctf_dir. CTF is determined using external tools. Current options are: - CTFFIND - gCTF These tools have to be installed externally. Parameters for the ctf tools are specified as a dictionary (arg params). Parameters used for both ctffind and gctf are: - 'pixel_a', 'voltage', 'cs', 'amp', 'box', 'min_res', 'max_res', 'min_def', 'max_def', 'def_step', 'astig', 'phase', 'min_phase', 'max_phase', 'phase_step' Voltage ('voltage') should always be specified. The pixel size (pixel_a) has to be specified in case it can not be read from the image header. All other parameters are optional, if they are not specified the ctffind / gctg default values are used. The default values should be fine for single particle images. Parameter recommendations for phase plate images are given in the ctffind / gctf documentation. In case of ctffind, arg params can also be a list containing the parameter values in the same order as specified above, starting with voltage. Important for ctffind: Because the required arguments differ between versions 4.0 and 4.1, as well as depend on values specified, it is not guaranteed that the dictionary form of arg params will work. In case of problems, specify params as a list. In addition, all other gctf arguments can also be specified (without '--'). It is suggested to use: 'do_EPA':'', 'do_validation':'' Parameter units are the same as in the ctf deterimantion tools. Intended for use in an environment such as Jupyter notebook. Arguments: - image_dir: directory where images reside - image prefix: beginning of image file(s) - ctf_dir: directory where the ctf determination results and extracted images are saved - pixel_a: pixel size in A - params: ctf determination parameters - flatten: indicated whether 3D images should be flatten (True or 'auto') or not (False). - tool: name of the ctf detmination tool - executable: ctf tool executable - param_file: name of the temporary parameter file - fast: flag indicating whether ctffind --fast option is used - print_results: flag indicating if phase and defoci found are printed for each analyzed image - plot_ctf: flag indicating whether ctf is plotted for each analyzed image - show_legend: flag indicating whether a legend is shown on ctf graphs - plot_phases, plot_defoci: flags indicating whether a graph containing phases and defoci of all images respectivelly are plotted - max_images: max number if image analyzed, for testing Returns an instance of this class. The following attributes are all lists where elements correspond to individual images: - image_path_orig: image path of the input file - image_path: image path of the image that is actually used to deterime ctf. It differs from image_path_orig if the original (input) image is a stack that is flattened or used to extract slices - image_inds: index of a slice extracted for a stack - ctf_path: path of the ctf fit image - defocus_1, defocus_2, defocus: defoci along the two axes and the mean defocus in um - angle: defocus (astigmatism) angle - phase: phase shift in multiples of pi - resolution: resolution in nm - ccc: correlation coefficient - pixel_a: pixel size in A - b_factor: b-factor (gctf only) """ # initialize index = 0 new = cls() print_head = True if plot_ctf and fast: print("Warning: CTF will not be plotted because fast execution" + " was chosen") # check which ctf tool to use if tool == 'ctffind': if executable is None: executable = 'ctffind' elif tool == 'gctf': if executable is None: executable = 'gctf' else: raise ValueError("CTF determination tool " + str(tool) + " was not understood.") new.tool = tool # cftfind on all images file_list = np.sort(os.listdir(image_dir)) for image_name in file_list: # skip files that are not images if not image_name.startswith(image_prefix): continue if not (image_name.endswith('.mrc') or image_name.endswith('.st') or image_name.endswith('.em')): continue if image_name.endswith('ctf.mrc'): continue # set input image path image_path = os.path.join(image_dir, image_name) # figure out if to flatten or not (just once, assume all files # are the same) im_io = ImageIO(file=image_path) if image_name.endswith('.st'): im_io.readHeader(fileFormat='mrc') else: im_io.readHeader() z_dim = im_io.shape[2] n_digits = int(np.ceil(np.log10(z_dim))) if isinstance(flatten, bool): pass elif isinstance(flatten, basestring) and (flatten == 'auto'): if z_dim > 1: flatten = True else: flatten = False else: raise ValueError("Argument flatten: " + str(flatten) + " was not understood.") # load stack and prepare image name, if need to extract images if (z_dim > 1) and not flatten: image_dir, image_name = os.path.split(image_path) image_base, image_extension = image_name.rsplit('.', 1) image_name_new_tmplt = (image_base + '_%0' + str(n_digits) + 'd.mrc') if image_name.endswith('.st'): stack = Image.read(image_path, memmap=True, fileFormat='mrc') else: stack = Image.read(image_path, memmap=True) else: image_path_to_read = image_path # find ctf of the current image or stack for image_in_stack_ind in range(z_dim): # extract and save images if needed if (z_dim > 1) and not flatten: if not os.path.exists(ctf_dir): os.makedirs(ctf_dir) image_path_to_read = os.path.join( ctf_dir, (image_name_new_tmplt % image_in_stack_ind)) one_image = Image() one_image.data = stack.data[:, :, image_in_stack_ind] one_image.write(file=image_path_to_read, pixel=stack.pixelsize) # save image path retlated new.image_path_orig.append(image_path) new.image_inds.append(image_in_stack_ind) new.image_path.append(image_path_to_read) # find ctf if tool == 'ctffind': # ctffind res_one = cls.ctffind(image_path=image_path_to_read, flatten=flatten, ctf_dir=ctf_dir, executable=executable, pixel_a=pixel_a, params=params, param_file=param_file, fast=fast, print_head=print_head, print_results=print_results, plot_ctf=plot_ctf, show_legend=show_legend) elif tool == 'gctf': # gctf res_one = cls.gctf(image_path=image_path_to_read, params=params, pixel_a=pixel_a, flatten=flatten, ctf_dir=ctf_dir, executable=executable, plot_ctf=plot_ctf, plot_ps=plot_ps, b_plot=b_plot, exp_f_plot=exp_f_plot, show_legend=show_legend, print_results=print_results, print_head=print_head, print_validation=print_validation) # save gctf specific data try: new.b_factor.append(res_one['b_factor']) except AttributeError: new.b_factor = [res_one['b_factor']] for name, value in list(res_one.items()): if name.startswith(cls.validation_prefix): try: previous_val = getattr(new, name) previous_val.append(value) setattr(new, name, previous_val) except AttributeError: setattr(new, name, [value]) else: raise ValueError("Sorry tool: " + tool + " was not found.") # save data common for ctffind and gctf new.phases.append(res_one["phase"]) new.defoci.append(res_one["defocus"]) new.defoci_1.append(res_one['defocus_1']) new.defoci_2.append(res_one['defocus_2']) new.resolution.append(res_one['resolution']) new.pixel_a.append(res_one['pixel_a']) new.angle.append(res_one['angle']) new.ctf_path.append(res_one['ctf_path']) # keep track of n images processed so far print_head = False index = index + 1 if (max_images is not None) and (index > max_images): break if flatten: break # plot phases if plot_phases: plt.figure() plt.bar(list(range(index)), new.phases) plt.plot([0, index], [0.5, 0.5], 'r--') plt.ylabel('Phase shift [$\pi$]') plt.xlabel('Images') plt.title("Phase shift summary") # plot defocus if plot_defoci: plt.figure() plt.bar(list(range(index)), new.defoci) plt.ylabel('Defocus [$\mu m$]') plt.xlabel('Images') plt.title("Defocus summary") # plot resolution if plot_resolution: plt.figure() plt.bar(list(range(index)), new.resolution) plt.ylabel('Resolution [nm]') plt.xlabel('Images') plt.title("Resolution summary") return new
def gen_surface(tomo, lbl=1, mask=True, purge_ratio=1, field=False, mode_2d=False, verbose=False): """ Generates a VTK PolyData surface from a segmented tomogram. Args: tomo (numpy.ndarray or str): the input segmentation as numpy ndarray or the file name in MRC, EM or VTI format lbl (int, optional): label for the foreground, default 1 mask (boolean, optional): if True (default), the input segmentation is used as mask for the surface purge_ratio (int, optional): if greater than 1 (default), then 1 every purge_ratio points of the segmentation are randomly deleted field (boolean, optional): if True (default False), additionally returns the polarity distance scalar field mode_2d (boolean, optional): needed for polarity distance calculation (if field is True), if True (default False), ... verbose (boolean, optional): if True (default False), prints out messages for checking the progress Returns: - output surface (vtk.vtkPolyData) - polarity distance scalar field (np.ndarray), if field is True """ # Check input format if isinstance(tomo, str): fname, fext = os.path.splitext(tomo) # if fext == '.fits': # tomo = pyfits.getdata(tomo) if fext == '.mrc': hold = ImageIO() hold.readMRC(file=tomo) tomo = hold.data elif fext == '.em': hold = ImageIO() hold.readEM(file=tomo) tomo = hold.data elif fext == '.vti': reader = vtk.vtkXMLImageDataReader() reader.SetFileName(tomo) reader.Update() tomo = vti_to_numpy(reader.GetOutput()) else: error_msg = 'Format %s not readable.' % fext raise pexceptions.PySegInputError(expr='gen_surface', msg=error_msg) elif not isinstance(tomo, np.ndarray): error_msg = 'Input must be either a file name or a ndarray.' raise pexceptions.PySegInputError(expr='gen_surface', msg=error_msg) # Load file with the cloud of points nx, ny, nz = tomo.shape cloud = vtk.vtkPolyData() points = vtk.vtkPoints() cloud.SetPoints(points) if purge_ratio <= 1: for x in range(nx): for y in range(ny): for z in range(nz): if tomo[x, y, z] == lbl: points.InsertNextPoint(x, y, z) else: count = 0 mx_value = purge_ratio - 1 purge = np.random.randint(0, purge_ratio+1, nx*ny*nz) for x in range(nx): for y in range(ny): for z in range(nz): if purge[count] == mx_value: if tomo[x, y, z] == lbl: points.InsertNextPoint(x, y, z) count += 1 if verbose: print 'Cloud of points loaded...' # Creating the isosurface surf = vtk.vtkSurfaceReconstructionFilter() # surf.SetSampleSpacing(2) surf.SetSampleSpacing(purge_ratio) # surf.SetNeighborhoodSize(10) surf.SetInputData(cloud) contf = vtk.vtkContourFilter() contf.SetInputConnection(surf.GetOutputPort()) # if thick is None: contf.SetValue(0, 0) # else: # contf.SetValue(0, thick) # Sometimes the contouring algorithm can create a volume whose gradient # vector and ordering of polygon (using the right hand rule) are # inconsistent. vtkReverseSense cures this problem. reverse = vtk.vtkReverseSense() reverse.SetInputConnection(contf.GetOutputPort()) reverse.ReverseCellsOn() reverse.ReverseNormalsOn() reverse.Update() rsurf = reverse.GetOutput() if verbose: print 'Isosurfaces generated...' # Translate and scale to the proper positions cloud.ComputeBounds() rsurf.ComputeBounds() xmin, xmax, ymin, ymax, zmin, zmax = cloud.GetBounds() rxmin, rxmax, rymin, rymax, rzmin, rzmax = rsurf.GetBounds() scale_x = (xmax-xmin) / (rxmax-rxmin) scale_y = (ymax-ymin) / (rymax-rymin) denom = rzmax - rzmin num = zmax - xmin if (denom == 0) or (num == 0): scale_z = 1 else: scale_z = (zmax-zmin) / (rzmax-rzmin) transp = vtk.vtkTransform() transp.Translate(xmin, ymin, zmin) transp.Scale(scale_x, scale_y, scale_z) transp.Translate(-rxmin, -rymin, -rzmin) tpd = vtk.vtkTransformPolyDataFilter() tpd.SetInputData(rsurf) tpd.SetTransform(transp) tpd.Update() tsurf = tpd.GetOutput() if verbose: print 'Rescaled and translated...' # Masking according to distance to the original segmentation if mask: tomod = scipy.ndimage.morphology.distance_transform_edt( np.invert(tomo == lbl)) for i in range(tsurf.GetNumberOfCells()): # Check if all points which made up the polygon are in the mask points_cell = tsurf.GetCell(i).GetPoints() count = 0 for j in range(0, points_cell.GetNumberOfPoints()): x, y, z = points_cell.GetPoint(j) if (tomod[int(round(x)), int(round(y)), int(round(z))] > MAX_DIST_SURF): count += 1 if count > 0: tsurf.DeleteCell(i) # Release free memory tsurf.RemoveDeletedCells() if verbose: print 'Mask applied...' # Field distance if field: # Get normal attributes norm_flt = vtk.vtkPolyDataNormals() norm_flt.SetInputData(tsurf) norm_flt.ComputeCellNormalsOn() norm_flt.AutoOrientNormalsOn() norm_flt.ConsistencyOn() norm_flt.Update() tsurf = norm_flt.GetOutput() # for i in range(tsurf.GetPointData().GetNumberOfArrays()): # array = tsurf.GetPointData().GetArray(i) # if array.GetNumberOfComponents() == 3: # break array = tsurf.GetCellData().GetNormals() # Build membrane mask tomoh = np.ones(shape=tomo.shape, dtype=np.bool) tomon = np.ones(shape=(tomo.shape[0], tomo.shape[1], tomo.shape[2], 3), dtype=TypesConverter().vtk_to_numpy(array)) # for i in range(tsurf.GetNumberOfCells()): # points_cell = tsurf.GetCell(i).GetPoints() # for j in range(0, points_cell.GetNumberOfPoints()): # x, y, z = points_cell.GetPoint(j) # # print x, y, z, array.GetTuple(j) # x, y, z = int(round(x)), int(round(y)), int(round(z)) # tomo[x, y, z] = False # tomon[x, y, z, :] = array.GetTuple(j) for i in range(tsurf.GetNumberOfCells()): points_cell = tsurf.GetCell(i).GetPoints() for j in range(0, points_cell.GetNumberOfPoints()): x, y, z = points_cell.GetPoint(j) # print x, y, z, array.GetTuple(j) x, y, z = int(round(x)), int(round(y)), int(round(z)) if tomo[x, y, z] == lbl: tomoh[x, y, z] = False tomon[x, y, z, :] = array.GetTuple(i) # Distance transform tomod, ids = scipy.ndimage.morphology.distance_transform_edt( tomoh, return_indices=True) # Compute polarity if mode_2d: for x in range(nx): for y in range(ny): for z in range(nz): i_x, i_y, i_z = (ids[0, x, y, z], ids[1, x, y, z], ids[2, x, y, z]) norm = tomon[i_x, i_y, i_z] norm[2] = 0 pnorm = (i_x, i_y, 0) p = (x, y, 0) dprod = dot_norm(np.asarray(p, dtype=np.float), np.asarray(pnorm, dtype=np.float), np.asarray(norm, dtype=np.float)) tomod[x, y, z] = tomod[x, y, z] * np.sign(dprod) else: for x in range(nx): for y in range(ny): for z in range(nz): i_x, i_y, i_z = (ids[0, x, y, z], ids[1, x, y, z], ids[2, x, y, z]) hold_norm = tomon[i_x, i_y, i_z] norm = hold_norm # norm[0] = (-1) * hold_norm[1] # norm[1] = hold_norm[0] # norm[2] = hold_norm[2] pnorm = (i_x, i_y, i_z) p = (x, y, z) dprod = dot_norm(np.asarray(pnorm, dtype=np.float), np.asarray(p, dtype=np.float), np.asarray(norm, dtype=np.float)) tomod[x, y, z] = tomod[x, y, z] * np.sign(dprod) if verbose: print 'Distance field generated...' return tsurf, tomod if verbose: print 'Finished!' return tsurf
def save_numpy(array, fname): """ Saves a numpy array to a file in MRC, EM or VTI format. Args: array (numpy.ndarray): input array fname (str): full path to the tomogram, has to end with '.mrc', '.em' or '.vti' Returns: None """ _, ext = os.path.splitext(fname) # Parse input array for fulfilling format requirements if (ext == '.mrc') or (ext == '.em'): if ((array.dtype != 'ubyte') and (array.dtype != 'int16') and (array.dtype != 'float32')): array = array.astype('float32') # if (len(array.shape) == 3) and (array.shape[2] == 1): # array = np.reshape(array, (array.shape[0], array.shape[1])) if ext == '.vti': pname, fnameh = os.path.split(fname) save_vti(numpy_to_vti(array), fnameh, pname) # elif ext == '.fits': # warnings.resetwarnings() # warnings.filterwarnings('ignore', category=UserWarning, append=True) # pyfits.writeto(fname, array, clobber=True, output_verify='silentfix') # warnings.resetwarnings() # warnings.filterwarnings('always', category=UserWarning, append=True) elif ext == '.mrc': img = ImageIO() # img.setData(np.transpose(array, (1, 0, 2))) img.setData(array) img.writeMRC(fname) elif ext == '.em': img = ImageIO() # img.setData(np.transpose(array, (1, 0, 2))) img.setData(array) img.writeEM(fname) else: error_msg = 'Format not valid %s.' % ext raise pexceptions.PySegInputError(expr='save_numpy', msg=error_msg)
def sort(self, seq=None, out=None, pad=0, start=1, fix_mode=None, microscope=None, limit=None, limit_mode='std', size=5, byte_order=None, test=False): """ Sorts series, fixes image headers, corrects the data and writes the sorted and corrected images. A series is sorted by tilt angle that is read from each image, or by the elements of seq (not necessarily angles) if given. Sorted images names have the form: directory/name + number + extension. where out is: directory/name_.extension and numbers start with start argument and are padded with pad (argument) number of zeros (see self.images). Argument fix_mode detemines how the headers are fixed: - None: no fixing - 'polara_fei-tomo': for series obtained on polara with the FEI tomo software - 'krios_fei-tomo': for series obtained on krios with the FEI tomo software - 'cm300': for series from CM300 (see pyto.io.image_io). If fix_mode is 'polara_fei-tomo', then arg microscope has to be specified. The allowed values are specified in microscope_db.py. Currently (r564) these are: 'polara-1_01-07', 'polara-1_01-09' and 'polara-2_01-09'. If fix_mode is None, microscope does nor need to be specified. Series recorded by SerialEM typically do not need fixing. Works for individual images only (not for stacks). Arguments: - seq: if specified this sequence used for sorting projection, otherwise tilt angles are used - out: sorted series path - pad: number of digits of a projection number in the sorted series file name - start: start number for sorted series file names - fix_mode: determined if and how headers are fixed - microscope: microscope type, see pyto.io.microscope_db - limit_mode: 'abs' or 'std' - limit: absolute greyscale limit if limit_mode is 'abs', or the number of stds above and below the mean - size: size of the subarray used to find the replacement value for a voxel that's out of the limits - byte_order: '<' or '>', default is the byte order of the machine - test: if True all steps are done except writing corrected images """ # shortcuts path = self.path match_mode = self.mode # make tilt angles - file names dictionary # ToDo: use self.sortPath() instead in_paths = [] images_iter = self.images(mode=match_mode) if seq is None: seq = [] # get tilt angles from file headers for in_path in images_iter: image = ImageIO(file=in_path) image.readHeader() seq.append(image.tiltAngle) in_paths.append(in_path) image.file_.close() else: # use seq to sort for in_path in images_iter: in_paths.append(in_path) # sort (note: dictionary angle:in_path fails for multiple files with # same angles) seq_arr = numpy.array(seq) in_paths_arr = numpy.array(in_paths) sort_ind = seq_arr.argsort() sorted_seq = seq_arr[sort_ind] sorted_in_paths = in_paths_arr[sort_ind] # parse out and make out directory if it doesn't exists if out is not None: out_dir, out_base_pat = os.path.split(out) if out_dir == '': out_dir = '.' if (not test) and (not os.path.isdir(out_dir)): os.makedirs(out_dir) else: out_path = None # initialize file counter if start is None: ind = None else: ind = start - 1 # loop over sorted in files for (item, in_path) in zip(sorted_seq, sorted_in_paths): if ind is not None: ind += 1 # parse in file path in_dir, in_base = os.path.split(in_path) if out is not None: # make out file path out_base = self.convertBase(in_base=in_base, out_base=out_base_pat, pad=pad, index=ind) out_path = os.path.join(out_dir, out_base) # read files im_io = ImageIO(file=in_path) im_io.read() # log logging.info("%5.1f: %s -> %s %6.1f %6.1f" \ % (item, in_path, out_path, \ im_io.data.mean(), im_io.data.std())) # fix if fix_mode is not None: im_io.fix(mode=fix_mode, microscope=microscope) # limit if limit is not None: image = Image(im_io.data) image.limit(limit=limit, mode=limit_mode, size=size) # write fixed file if not test: if byte_order is None: byte_order = im_io.machineByteOrder im_io.write(file=out_path, byteOrder=byte_order, data=image.data) else: logging.info(" " + str(item) + ": " + in_path + " -> "\ + str(out_path))
def write( self, file, byteOrder=None, dataType=None, fileFormat=None, arrayOrder=None, shape=None, length=None, pixel=1, casting=u'unsafe', header=False, existing=False): """ Writes image to a file in em, mrc or raw format. If fileFormat is not given it is determined from the file extension. Data (image) has to be specified by arg data or previously set self.data attribute. Data type and shape are determined by args dataType and shape, or by the data type and shape of the data, in this order.. If data type (determined as described above) is not one of the data types used for the specified file format (ubyte, int16, float32, complex64 for mrc and uint8, uint16, int32, float32, float64, complex64 for em), then the value of arg dataType has to be one of the appropriate data types. Otherwise an exception is raised. If data type (determined as described above) is different from the type of actual data, the data is converted to the data type. Note that if these two types are incompatible according to arg casting, an exception is raised. The data is converted to the (prevously determined) shape and array order and then written. That means that the shape and the array order may be changed in the original array (argument data or self.data). However, array order is not written the header. Additional header parameters are determined for mrc format. Nxstart, nystart and nzstart are set to 0, while mx, my and mz to the corresponding data size (grid size). xlen, ylen and zlen are taken from arg length if given, or obtained by multiplying data size with pixel size (in nm). If arg header is True and if the file type corresponding to arg file is the same as self.fileFormat, a header consisting of self.header (list) and self.extendedHeaderString (str) are written as file header. The extended header should be used only for mrc format. Arguments: - file: file name - fileFormat: 'em', 'mrc', or 'raw' - byteOrder: '<' (little-endian), '>' (big-endian) - dataType: any of the numpy types, e.g.: 'int8', 'int16', 'int32', 'float32', 'float64' - arrayOrder: 'C' (z-axis fastest), or 'F' (x-axis fastest) - shape: (x_dim, y_dim, z_dim) - length: (list aor ndarray) length in each dimension in nm (used only for mrc format) - pixel: pixel size in nm (used only for mrc format if length is None) - casting: Controls what kind of data casting may occur: 'no', 'equiv', 'safe', 'same_kind', 'unsafe'. Identical to numpy.astype() method. - header: flag indicating if self.header is written as file header - existing: flag indicating whether the already existing file_io attribute is used for writting Returns file instance. """ # write file from pyto.io.image_io import ImageIO as ImageIO if existing and (self.file_io is not None): fi = self.file_io fi = ImageIO() # get file format of the file to be written fi.setFileFormat(file_=file, fileFormat=fileFormat) new_file_format = fi.fileFormat # set header to pass as argument, if new and old file types are the same header_arg = None extended = None try: if header and (new_file_format == self.fileFormat): header_arg = self.header extended = self.extendedHeaderString except AttributeError: pass # write fi.write( file=file, data=self.data, fileFormat=fileFormat, byteOrder=byteOrder, dataType=dataType, arrayOrder=arrayOrder, shape=shape, length=length, pixel=pixel, casting=casting, header=header_arg, extended=extended) return fi.file_
def read(cls, file, fileFormat=None, byteOrder=None, dataType=None, arrayOrder=None, shape=None, header=False, memmap=False): """ Reads image from a file and saves the data in numpy.array format. Images can be in em, mrc and raw format. Unless arg fileFormat is specified, file format is determined from the extension: - 'em' and 'EM' for em format - 'mrc', 'rec' and 'mrcs' for mrc format - 'raw', 'dat' and 'RAW' for raw format If file is in em or mrc format (file extension em or mrc) only the file argument is needed. If the file is in the raw data format (file extension raw) all arguments (except fileFormat) are required. If arg header is True, file header (em, mrc or raw) is saved as attribute header. In any case attribute fileFormat is set. For mrc and em files, array order is determined from arg arrayOrder, from self.arrayOrder, or it is set to the default ("F") in this order. Data is read according the determined array order. That is, array order is not read from the file header. If arg memmap is True, instead into a nparray, the data is read to a memory map. That means that the complete data is not read into the memory, but the required parts are read on demand. This is useful when working with large images, but might not always work properly because the memory map is not quite properly a subclass of numpy.ndarray (from Numpy doc). Data from mrc files is always read as 3D because mrc header always contains lengths for all three dimensions (the length of the last dimeension is 1 for 2D images). In such cases one can obtain the 2D ndarray using: self.data.reshape(self.data.shape[0:2]) Sets attributes: - data: (ndarray) image data - pixelsize: (float or list of floats) pixel size in nm, 1 if pixel size not known - length: (list or ndarray) length in each dimension in nm - fileFormat: file format ('em', 'mrc', or 'raw') - memmap: from the argument - header: header Arguments: - file: file name - fileFormat: 'em', 'mrc', or 'raw' - byteOrder: '<' (little-endian), '>' (big-endian) - dataType: any of the numpy types, e.g.: 'int8', 'int16', 'int32', 'float32', 'float64' - arrayOrder: 'C' (z-axis fastest), or 'F' (x-axis fastest) - shape: (x_dim, y_dim, z_dim) - header: flag indicating if file header is read and saved - memmap: Flag indicating if the data is read to a memory map, instead of reading it into a ndarray Returns: - instance of this class """ # read file from pyto.io.image_io import ImageIO as ImageIO fi = ImageIO() fi.read( file=file, fileFormat=fileFormat, byteOrder=byteOrder, dataType=dataType, arrayOrder=arrayOrder, shape=shape, memmap=memmap) # instantiate object and set fullInset object = cls(data=fi.data) # is this needed (ugly) object.saveFull() # save file header object.header = None if header: object.header = fi.header try: object.extendedHeaderString = fi.extendedHeaderString except AttributeError: object.extendedHeaderString = None object.fileFormat = fi.fileFormat # save pixel size try: object.pixelsize = fi.pixelsize except (AttributeError, ValueError): object.pixelsize = 1 # save length try: object.length = fi.length except (AttributeError, ValueError): object.length = numpy.asarray(fi.data.shape) \ * numpy.asarray(object.pixelsize) # save memmap object.memmap = fi.memmap # save file io object # removed because if object.data is changed, object.file_io.data will # still hold a copy of data #object.file_io = fi # save file type #try: # object.fileType = fi.fileType #except (AttributeError, ValueError): # object.fileType = None return object
def getDose(self, conversion=1, mode='pixel_size', projection_dose=False): """ Calculates dose for the whole series (in e/A^2) and the average counts per pixel.. Logs average, std, min and max counts for each image to INFO. Mode determines how the pixels are converted to A^2: - 'pixel_size': use pixelsize from the image header (default) - 'cm300_fix': use only for cm300 series with headers that were not corrected Arguments: - conversion: number of electrons per pixel (if known microscope use pyto.io.microscope_db.conversion[microscope]). - mode: 'pixel_size' should be used, other moders were used in special cases - projection_dose: flag indicating if dose [e/A^2] for each projection is returned Returns total_dose, mean_count: - total dose in e/A^2 - mean count in counts/pixel - projection_dose (if projection_dose=True) 2D ndarray containing tilt angle and projection dose for all projections (shape n_projections x 2) """ # set images to z positions if projections are in a stack, or # to projection file names if individual projection files if self.stack: image_stack = Image.read(file=self.path, memmap=True) images = numpy.array(list(range(image_stack.data.shape[2]))) images = images[numpy.argsort(self.tiltAngles)] else: images = list(self.images()) images = self.sortPaths(paths=images, mode='tilt_angle') tot_dose = 0 tot_count = 0 num = 0. proj_dose = [] for fi in images: # read single projection file if self.stack: image = Image(data=image_stack.data[:, :, fi]) if isinstance(image_stack.pixelsize, (numpy.ndarray, list)): image.pixelsize = image_stack.pixelsize[0] else: image.pixelsize = image_stack.pixelsize image.tiltAngle = self.tiltAngles[fi] else: image = ImageIO(file=fi) image.read() # calculate mean = image.data.mean() tot_count += mean num += 1 if mode == 'pixel_size': loc_dose = mean / (conversion * image.pixelsize**2) elif (mode == 'cm300_fix') or (image.pixelsize == 2.048): ccd_pixelsize = 30000 pixelsize = ccd_pixelsize / image.magnification loc_dose = mean / (conversion * pixelsize**2) else: raise ValueError("Sorry, dose calculation mode: " + mode + " is not recognized.") tot_dose += loc_dose # save and print stats for an individual image proj_dose.append([image.tiltAngle, loc_dose / 100.]) logging.info('%s, (%5.1f): mean=%6.1f, std=%6.1f, min=%d, max=%d' % (fi, image.tiltAngle, mean, image.data.std(), image.data.min(), image.data.max())) # mean counts mean_count = tot_count / num # total dose per A^2 tot_dose = tot_dose / 100 # print total values logging.info("Mean count per pixel: %6.1f" % mean_count) logging.info("Total electron dose per A^2: %6.1f" % tot_dose) if projection_dose: return tot_dose, mean_count, numpy.asarray(proj_dose) else: return tot_dose, mean_count
def save_numpy(array, fname): """ Saves a numpy array to a file in MRC, EM or VTI format. Args: array (numpy.ndarray): input array fname (str): full path to the tomogram, has to end with '.mrc', '.em' or '.vti' Returns: None """ _, ext = os.path.splitext(fname) # Parse input array for fulfilling format requirements if (ext == '.mrc') or (ext == '.em'): if ((array.dtype != 'ubyte') and (array.dtype != 'int16') and (array.dtype != 'float32')): array = array.astype('float32') # if (len(array.shape) == 3) and (array.shape[2] == 1): # array = np.reshape(array, (array.shape[0], array.shape[1])) if ext == '.vti': pname, fnameh = os.path.split(fname) save_vti(numpy_to_vti(array), fnameh, pname) elif ext == '.mrc': img = ImageIO() # img.setData(np.transpose(array, (1, 0, 2))) img.setData(array) img.writeMRC(fname) elif ext == '.em': img = ImageIO() # img.setData(np.transpose(array, (1, 0, 2))) img.setData(array) img.writeEM(fname) else: raise pexceptions.PySegInputError( expr='save_numpy', msg='Format not valid {}.'.format(ext))
def testRead(self): """ Tests reading EM and MRC files """ # EM tomo em = ImageIO() em.read(file=os.path.join(self.dir, "bin-2.em")) expected = numpy.array([[-0.0242, -0.0250, 0.0883], [0.0640, 0.0071, -0.1300], [-0.0421, -0.0392, -0.0312]]) np_test.assert_almost_equal(em.data[50:53, 120:123, 40], expected, decimal=4) expected = numpy.array([[-0.0573, 0.0569, 0.0386], [0.1309, 0.1211, -0.0881], [-0.0110, -0.0240, 0.0347]]) np_test.assert_almost_equal(em.data[150:153, 20:23, 10], expected, decimal=4) np_test.assert_equal(em.byteOrder, '<') np_test.assert_equal(em.arrayOrder, 'F') np_test.assert_equal(em.dataType, 'float32') np_test.assert_equal(em.data.dtype, numpy.dtype('float32')) np_test.assert_equal(em.memmap, False) # EM tomo with memory map em.read(file=os.path.join(self.dir, "bin-2.em"), memmap=True) expected = numpy.array([[-0.0242, -0.0250, 0.0883], [0.0640, 0.0071, -0.1300], [-0.0421, -0.0392, -0.0312]]) np_test.assert_almost_equal(em.data[50:53, 120:123, 40], expected, decimal=4) expected = numpy.array([[-0.0573, 0.0569, 0.0386], [0.1309, 0.1211, -0.0881], [-0.0110, -0.0240, 0.0347]]) np_test.assert_almost_equal(em.data[150:153, 20:23, 10], expected, decimal=4) np_test.assert_equal(em.byteOrder, '<') np_test.assert_equal(em.arrayOrder, 'F') np_test.assert_equal(em.dataType, 'float32') np_test.assert_equal(em.data.dtype, numpy.dtype('float32')) np_test.assert_equal(em.memmap, True) # EM, big-endian em = ImageIO() em.read(file=os.path.join(self.dir, "mac-file.em")) np_test.assert_equal(em.byteOrder, '>') # EM, little-endian em = ImageIO() em.read(file=os.path.join(self.dir, "pc-file.em")) np_test.assert_equal(em.byteOrder, '<') em.read(file=os.path.join(self.dir, "pc-file.em"), memmap=True) np_test.assert_equal(em.byteOrder, '<') # MRC tomo mrc = ImageIO() mrc.read(file=os.path.join(self.dir, "bin-2.mrc")) expected = numpy.array([[-0.0242, -0.0250, 0.0883], [0.0640, 0.0071, -0.1300], [-0.0421, -0.0392, -0.0312]]) np_test.assert_almost_equal(mrc.data[50:53, 120:123, 40], expected, decimal=4) expected = numpy.array([[-0.0573, 0.0569, 0.0386], [0.1309, 0.1211, -0.0881], [-0.0110, -0.0240, 0.0347]]) np_test.assert_almost_equal(mrc.data[150:153, 20:23, 10], expected, decimal=4) np_test.assert_equal(mrc.byteOrder, '<') np_test.assert_equal(mrc.arrayOrder, 'F') np_test.assert_equal(mrc.dataType, 'float32') np_test.assert_equal(mrc.data.dtype, numpy.dtype('float32')) np_test.assert_equal(mrc.memmap, False) # MRC tomo with memmap mrc = ImageIO() mrc.read(file=os.path.join(self.dir, "bin-2.mrc"), memmap=True) expected = numpy.array([[-0.0242, -0.0250, 0.0883], [0.0640, 0.0071, -0.1300], [-0.0421, -0.0392, -0.0312]]) np_test.assert_almost_equal(mrc.data[50:53, 120:123, 40], expected, decimal=4) expected = numpy.array([[-0.0573, 0.0569, 0.0386], [0.1309, 0.1211, -0.0881], [-0.0110, -0.0240, 0.0347]]) np_test.assert_almost_equal(mrc.data[150:153, 20:23, 10], expected, decimal=4) np_test.assert_equal(mrc.byteOrder, '<') np_test.assert_equal(mrc.arrayOrder, 'F') np_test.assert_equal(mrc.dataType, 'float32') np_test.assert_equal(mrc.data.dtype, numpy.dtype('float32')) np_test.assert_equal(mrc.memmap, True) # MRC tomo with extended header mrc = ImageIO() mrc.read(file=os.path.join(self.dir, "bin-2_ext.mrc"), memmap=False) expected = numpy.array([[-0.0242, -0.0250, 0.0883], [0.0640, 0.0071, -0.1300], [-0.0421, -0.0392, -0.0312]]) np_test.assert_almost_equal(mrc.data[50:53, 120:123, 40], expected, decimal=4) expected = numpy.array([[-0.0573, 0.0569, 0.0386], [0.1309, 0.1211, -0.0881], [-0.0110, -0.0240, 0.0347]]) np_test.assert_almost_equal(mrc.data[150:153, 20:23, 10], expected, decimal=4) np_test.assert_equal(mrc.byteOrder, '<') np_test.assert_equal(mrc.arrayOrder, 'F') np_test.assert_equal(mrc.dataType, 'float32') np_test.assert_equal(mrc.data.dtype, numpy.dtype('float32')) np_test.assert_equal(mrc.memmap, False) np_test.assert_equal(mrc.extendedHeaderLength, 5120) # MRC tomo with extended header and with memmap mrc = ImageIO() mrc.read(file=os.path.join(self.dir, "bin-2_ext.mrc"), memmap=True) expected = numpy.array([[-0.0242, -0.0250, 0.0883], [0.0640, 0.0071, -0.1300], [-0.0421, -0.0392, -0.0312]]) np_test.assert_almost_equal(mrc.data[50:53, 120:123, 40], expected, decimal=4) expected = numpy.array([[-0.0573, 0.0569, 0.0386], [0.1309, 0.1211, -0.0881], [-0.0110, -0.0240, 0.0347]]) np_test.assert_almost_equal(mrc.data[150:153, 20:23, 10], expected, decimal=4) np_test.assert_equal(mrc.byteOrder, '<') np_test.assert_equal(mrc.arrayOrder, 'F') np_test.assert_equal(mrc.dataType, 'float32') np_test.assert_equal(mrc.data.dtype, numpy.dtype('float32')) np_test.assert_equal(mrc.memmap, True) np_test.assert_equal(mrc.extendedHeaderLength, 5120) # another MRC tomo (generated by and) mrc = ImageIO() mrc.read(file=os.path.join(self.dir, "and-tomo.mrc")) expected = numpy.array([[-0.0329, -0.0006, -0.0698], [-0.0101, -0.1196, -0.1295], [0.0844, -0.0400, -0.0716]]) np_test.assert_almost_equal(mrc.data[50:53, 120:123, 40], expected, decimal=4) expected = numpy.array([[-0.0019, -0.0085, 0.0036], [0.0781, 0.0279, -0.0365], [0.0210, -0.0193, -0.0355]]) np_test.assert_almost_equal(mrc.data[150:153, 20:23, 60], expected, decimal=4) np_test.assert_equal(mrc.dataType, 'float32') np_test.assert_equal(mrc.data.dtype, numpy.dtype('float32')) np_test.assert_equal(mrc.memmap, False) # another MRC tomo (generated by and) with memmap mrc = ImageIO() mrc.read(file=os.path.join(self.dir, "and-tomo.mrc"), memmap=True) expected = numpy.array([[-0.0329, -0.0006, -0.0698], [-0.0101, -0.1196, -0.1295], [0.0844, -0.0400, -0.0716]]) np_test.assert_almost_equal(mrc.data[50:53, 120:123, 40], expected, decimal=4) expected = numpy.array([[-0.0019, -0.0085, 0.0036], [0.0781, 0.0279, -0.0365], [0.0210, -0.0193, -0.0355]]) np_test.assert_almost_equal(mrc.data[150:153, 20:23, 60], expected, decimal=4) np_test.assert_equal(mrc.dataType, 'float32') np_test.assert_equal(mrc.data.dtype, numpy.dtype('float32')) np_test.assert_equal(mrc.memmap, True) # mrc with the opposite byte order mrc2 = ImageIO() mrc2.read(file=os.path.join(self.dir, "swapped_byte_order.mrc")) expected = numpy.array([[0.000, 0.000], [-0.341, -6.702], [0.782, -11.780], [0.327, -14.298], [-0.691, -17.411], [-0.337, -18.076], [-0.669, -19.157], [-0.799, -20.400], [-0.793, -21.286], [-1.008, -21.386]]) np_test.assert_almost_equal(mrc2.data[:, :, 0], expected, decimal=3) np_test.assert_equal(mrc2.memmap, False) raised = False try: mrc2.read(file=os.path.join(self.dir, "swapped_byte_order.mrc"), memmap=True) except ValueError: raised = True np_test.assert_equal(raised, True) np_test.assert_equal(mrc2.memmap, True) # new style header mrc mrc_new = ImageIO() mrc_new.read(file=os.path.join(self.dir, 'new-head_int16.mrc')) np_test.assert_equal(mrc_new.dataType, 'int16') np_test.assert_equal(mrc_new.data.dtype, numpy.dtype('int16')) np_test.assert_equal(mrc_new.byteOrder, '<') np_test.assert_equal(mrc_new.arrayOrder, 'F') np_test.assert_equal(mrc_new.shape, (40, 30, 20)) np_test.assert_equal(mrc_new.pixel, [0.4, 0.4, 0.4]) np_test.assert_equal(mrc_new.pixelsize, 0.4) np_test.assert_equal(mrc_new.data[14, 8, 10], -14) np_test.assert_equal(mrc_new.data[15, 23, 12], 10) np_test.assert_equal(mrc_new.data[23, 29, 16], 2) np_test.assert_equal(mrc_new.memmap, False) # new style header mrc mrc_new = ImageIO() mrc_new.read(file=os.path.join(self.dir, 'new-head_int16.mrc'), memmap=True) np_test.assert_equal(mrc_new.dataType, 'int16') np_test.assert_equal(mrc_new.data.dtype, numpy.dtype('int16')) np_test.assert_equal(mrc_new.byteOrder, '<') np_test.assert_equal(mrc_new.arrayOrder, 'F') np_test.assert_equal(mrc_new.shape, (40, 30, 20)) np_test.assert_equal(mrc_new.pixel, [0.4, 0.4, 0.4]) np_test.assert_equal(mrc_new.pixelsize, 0.4) np_test.assert_equal(mrc_new.data[14, 8, 10], -14) np_test.assert_equal(mrc_new.data[15, 23, 12], 10) np_test.assert_equal(mrc_new.data[23, 29, 16], 2) np_test.assert_equal(mrc_new.memmap, True) np_test.assert_equal(mrc_new.n_labels, 9) np_test.assert_equal(len(mrc_new.labels), 9) desired = ( b"COMBINEFFT: Combined FFT from two tomograms " + b"07-Oct-13 17:15:24") np_test.assert_equal(len(mrc_new.labels[3]), 80) np_test.assert_equal(mrc_new.labels[3][:len(desired)], desired) desired = ( b"NEWSTACK: Images copied 10-Oct-13 18:00:03" ) np_test.assert_equal(len(mrc_new.labels[6]), 80) np_test.assert_equal(mrc_new.labels[6][:len(desired)], desired) # test raw file raw = ImageIO() raw.read(file=self.raw_file_name, dataType=self.raw_dtype, shape=self.raw_shape) np_test.assert_equal(raw.data, self.raw_data) np_test.assert_equal(raw.memmap, False) # test raw file with memmap raw = ImageIO() raw.read(file=self.raw_file_name, dataType=self.raw_dtype, shape=self.raw_shape, memmap=True) np_test.assert_equal(raw.data, self.raw_data) np_test.assert_equal(raw.memmap, True)
def testWrite(self): """ Tests write (and implicitly read), for em, mrc and raw format. """ # arrays ar_uint8 = numpy.array([54, 200, 5, 7, 45, 123], dtype='uint8').reshape((3, 1, 2)) ar_int8 = numpy.array([54, 2, -5, 7, 45, 123], dtype='uint8').reshape( (3, 1, 2)) ar_uint16 = numpy.array([1034, 546, 248, 40000, 2345, 365, 4876, 563], dtype='uint16').reshape((2, 2, 2)) ar_int16 = numpy.array([1034, 546, -248, 156, 2345, 365, -4876, 563], dtype='int16').reshape((2, 2, 2)) ar_int32 = numpy.array( [1034, 56546, -223448, 156, 2345, 2**31 - 10, -884876, 563], dtype='int32').reshape((2, 2, 2)) ar_uint32 = numpy.array( [1034, 56546, 223448, 156, 2345, 365, 884876, 2**32 - 10], dtype='uint32').reshape((2, 2, 2)) ar_int8_2 = numpy.arange(24, dtype='int8').reshape((4, 3, 2)) ar_int16_2 = numpy.arange(24, dtype='int16').reshape((4, 3, 2)) ar2_int16 = numpy.array([1034, 546, -248, 156, 2345, 365, -4876, 563], dtype='int16').reshape((2, 4)) ar_int16_f = numpy.array([1034, 546, -248, 156, 2345, 365, -4876, 563], dtype='int16', order='F').reshape((2, 2, 2)) ar_int16_c = numpy.array([1034, 546, -248, 156, 2345, 365, -4876, 563], dtype='int16', order='C').reshape((2, 2, 2)) # em uint8 file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.em'), data=ar_uint8) file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.em')) np_test.assert_equal(file_in.dataType, 'uint8') np_test.assert_equal(file_in.data, ar_uint8) # em uint16 file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.em'), data=ar_uint16) file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.em')) np_test.assert_equal(file_in.dataType, 'uint16') np_test.assert_equal(file_in.data, ar_uint16) # em int16 converted to int32, safe casting file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.em'), data=ar_int16, dataType='int32', casting='safe') file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.em')) np_test.assert_equal(file_in.dataType, 'int32') np_test.assert_equal(file_in.data, ar_int16) # em int16, safe casting file_out = ImageIO() np_test.assert_raises( TypeError, file_out.write, **{ 'file': os.path.join(self.dir, '_test.em'), 'data': ar_int16, 'casting': 'safe' }) # em int16 converted to uint16, unsafe casting file_out = ImageIO() print("int16 to uint16") file_out.write(file=os.path.join(self.dir, '_test.em'), data=ar_int16, dataType='uint16', casting='unsafe') print("int16 to uint16 end") file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.em')) np_test.assert_equal(file_in.dataType, 'uint16') np_test.assert_equal(file_in.data.dtype, numpy.dtype('uint16')) np_test.assert_equal(file_in.data[0, 1, 0] == ar_int16[0, 1, 0], False) # em int16 to uint16, safe casting file_out = ImageIO() np_test.assert_raises( TypeError, file_out.write, **{ 'file': os.path.join(self.dir, '_test.em'), 'data': ar_int16, 'dataType': 'uint16', 'casting': 'safe' }) # em uint16 to int16, unsafe casting file_out = ImageIO() np_test.assert_raises( TypeError, file_out.write, **{ 'file': os.path.join(self.dir, '_test.em'), 'data': ar_uint16, 'dataType': 'int16', 'casting': 'unsafe' }) # em uint32 to int32, safe casting print("uint32 to int32 safe") file_out = ImageIO() np_test.assert_raises( TypeError, file_out.write, **{ 'file': os.path.join(self.dir, '_test.em'), 'data': ar_uint32, 'dataType': 'int32', 'casting': 'safe' }) # em uint32 converted to int32, unsafe casting print("uint32 to int32") file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.em'), data=ar_uint32, dataType='int32', casting='unsafe') file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.em')) np_test.assert_equal(file_in.dataType, 'int32') #np_test.assert_equal(file_in.data, ar_uint32) should fail np_test.assert_equal(file_in.data[0, 0, 0] == ar_uint32[0, 0, 0], True) np_test.assert_equal(file_in.data[1, 1, 1] == ar_uint32[1, 1, 1], False) # em uint32 to float32, safe casting file_out = ImageIO() np_test.assert_raises( TypeError, file_out.write, **{ 'file': os.path.join(self.dir, '_test.em'), 'data': ar_uint32, 'dataType': 'float32', 'casting': 'safe' }) # em uint32 to float32, unsafe casting file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.em'), data=ar_uint32, dataType='float32', casting='unsafe') file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.em')) np_test.assert_equal(file_in.dataType, 'float32') #np_test.assert_almost_equal(file_in.data, ar_uint32) should fail np_test.assert_equal(file_in.data[0, 0, 0] == ar_uint32[0, 0, 0], True) np_test.assert_equal(file_in.data[1, 1, 1] == ar_uint32[1, 1, 1], False) # em int32 to float32, unsafe casting file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.em'), data=ar_int32, dataType='float32', casting='unsafe') file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.em')) np_test.assert_equal(file_in.dataType, 'float32') #np_test.assert_almost_equal(file_in.data, ar_int32) should fail np_test.assert_equal(file_in.data[0, 0, 0] == ar_int32[0, 0, 0], True) np_test.assert_equal(file_in.data[1, 0, 1] == ar_int32[1, 0, 1], False) # em int32 to float64, safe casting file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.em'), data=ar_int32, dataType='float64', casting='safe') file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.em')) np_test.assert_equal(file_in.dataType, 'float64') np_test.assert_almost_equal(file_in.data, ar_int32) # mrc data type and shape from args file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.mrc'), data=ar_int8_2, shape=(2, 3, 4), dataType='int16') file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.mrc')) np_test.assert_equal(file_in.dataType, 'int16') np_test.assert_equal(file_in.shape, (2, 3, 4)) # mrc data type and shape from previously given data file_out = ImageIO() file_out.setData(ar_int16_2) file_out.write(file=os.path.join(self.dir, '_test.mrc')) file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.mrc')) np_test.assert_equal(file_in.dataType, 'int16') np_test.assert_equal(file_in.shape, (4, 3, 2)) # mrc data type and shape from attributes file_out = ImageIO() file_out.data = ar_int8_2 file_out.shape = (2, 3, 4) file_out.dataType = 'int16' file_out.write(file=os.path.join(self.dir, '_test.mrc')) file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.mrc')) np_test.assert_equal(file_in.dataType, 'int16') np_test.assert_equal(file_in.shape, (2, 3, 4)) # mrc data type and shape from data file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.mrc'), data=ar_int16_2) file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.mrc')) np_test.assert_equal(file_in.dataType, 'int16') np_test.assert_equal(file_in.shape, (4, 3, 2)) # mrc uint8, same as ubyte file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.mrc'), data=ar_uint8) file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.mrc')) np_test.assert_equal(file_in.dataType, 'ubyte') np_test.assert_almost_equal(file_in.data, ar_uint8) # mrc uint16 file_out = ImageIO() np_test.assert_raises((KeyError, TypeError), file_out.write, **{ 'file': os.path.join(self.dir, '_test.mrc'), 'data': ar_uint16 }) # mrc uint16 to int16, safe casting file_out = ImageIO() np_test.assert_raises( TypeError, file_out.write, **{ 'file': os.path.join(self.dir, '_test.mrc'), 'data': ar_uint16, 'dataType': 'ubyte', 'casting': 'safe' }) # mrc uint16 to int16, unsafe casting file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.mrc'), data=ar_uint16, dataType='int16', casting='unsafe') file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.mrc')) np_test.assert_equal(file_in.dataType, 'int16') #np_test.assert_almost_equal(file_in.data, ar_uint16) should fail np_test.assert_equal(file_in.data[0, 0, 0] == ar_uint16[0, 0, 0], True) np_test.assert_equal(file_in.data[0, 1, 1] == ar_uint16[0, 1, 1], False) # mrc int16 file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.mrc'), data=ar_int16, pixel=2.3) file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.mrc')) np_test.assert_equal(file_in.dataType, 'int16') np_test.assert_equal(file_in.data, ar_int16) np_test.assert_equal(file_in.pixel, [2.3, 2.3, 2.3]) np_test.assert_equal(file_in.pixelsize, 2.3) # mrc int16 2D file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.mrc'), data=ar2_int16, pixel=3.4) file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.mrc')) np_test.assert_equal(file_in.dataType, 'int16') np_test.assert_equal(file_in.data[:, :, 0], ar2_int16) np_test.assert_equal(file_in.pixelsize, 3.4) # mrc int8 to int16 file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.mrc'), data=ar_int8, dataType='int16', casting='safe') file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.mrc')) np_test.assert_equal(file_in.dataType, 'int16') np_test.assert_equal(file_in.data, ar_int8) # mrc int32 file_out = ImageIO() np_test.assert_raises((KeyError, TypeError), file_out.write, **{ 'file': os.path.join(self.dir, '_test.mrc'), 'data': ar_int32 }) # mrc int32 to int16 file_out = ImageIO() np_test.assert_raises( TypeError, file_out.write, **{ 'file': os.path.join(self.dir, '_test.mrc'), 'data': ar_int32, 'dataType': 'int16', 'casting': 'safe' }) # mrc int32 to float32 file_out = ImageIO() np_test.assert_raises( TypeError, file_out.write, **{ 'file': os.path.join(self.dir, '_test.mrc'), 'data': ar_int32, 'dataType': 'float32', 'casting': 'safe' }) # mrc int32 to complex64 file_out = ImageIO() np_test.assert_raises( TypeError, file_out.write, **{ 'file': os.path.join(self.dir, '_test.mrc'), 'data': ar_int32, 'dataType': 'complex64', 'casting': 'safe' }) # raw int16 file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.raw'), data=ar_int16) file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.raw'), dataType='int16', shape=(2, 2, 2)) np_test.assert_equal(file_in.dataType, 'int16') np_test.assert_equal(file_in.data, ar_int16) # raw int8 to int16 file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.raw'), data=ar_int8, dataType='int16') file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.raw'), dataType='int16', shape=(3, 1, 2)) np_test.assert_equal(file_in.dataType, 'int16') np_test.assert_equal(file_in.data, ar_int8) # raw int16 to int8 file_out = ImageIO() np_test.assert_raises( TypeError, file_out.write, **{ 'file': os.path.join(self.dir, '_test.raw'), 'data': ar_int16, 'dataType': 'int8', 'casting': 'safe' }) # explain error messages printed before print("It's fine if few error messages were printed just before " + "this line, because they have been caught.") # shape param file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.mrc'), data=ar_int16, dataType='int16') file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.mrc'), dataType='int16') np_test.assert_equal(file_in.data.shape, (2, 2, 2)) file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.mrc'), data=ar_int16, dataType='int16', shape=(1, 4, 2)) file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.mrc'), dataType='int16') np_test.assert_equal(file_in.data.shape, (1, 4, 2)) file_out.write(file=os.path.join(self.dir, '_test.mrc'), data=ar_int16, dataType='int16', shape=(4, 2)) file_in.readHeader(file=os.path.join(self.dir, '_test.mrc')) file_in.read(file=os.path.join(self.dir, '_test.mrc'), dataType='int16') np_test.assert_equal(file_in.data.shape, (4, 2, 1)) file_in.read(file=os.path.join(self.dir, '_test.mrc'), dataType='int16', shape=(2, 2, 2)) np_test.assert_equal(file_in.data.shape, (2, 2, 2)) # array order C, read write default (F) file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.mrc'), data=ar_int16_c) file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.mrc')) np_test.assert_equal(file_in.data, ar_int16_c) # array order C, read write C file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.mrc'), data=ar_int16_c, arrayOrder='C') file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.mrc'), arrayOrder='C') np_test.assert_equal(file_in.data, ar_int16_c) # array order F, read write default (F) file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.mrc'), data=ar_int16_f) file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.mrc')) np_test.assert_equal(file_in.data, ar_int16_f) # array order F, read write F file_out = ImageIO() file_out.write(file=os.path.join(self.dir, '_test.mrc'), data=ar_int16_f, arrayOrder='F') file_in = ImageIO() file_in.read(file=os.path.join(self.dir, '_test.mrc'), arrayOrder='F') np_test.assert_equal(file_in.data, ar_int16_f)