def _trigger(self): V = model.get_volume() # with tf.get_default_session() as sess: V_py = tf.get_default_session().run(V) V_py = np.fft.fftshift(V_py) vname = os.path.join(logger.get_logger_dir(), 'volume.mrc') mrcfile.new(vname, data=V_py, overwrite=True)
def threshold_binarize_mrc(inmrc, thresholded, thresholdedbinarized, FSCCutoff, ThresholdForSphericity, highpassfilter, apix,gpu=False): # inmrc, thresholded, and thresholdedbinarized are paths to .mrc files # Read MRC inputmrc = (mrcfile.open(inmrc)).data # binarize array if gpu: outarraythresholded, outarraythresholdedbinarized = cuda_functions.threshold_binarize_array_cuda(inputmrc, FSCCutoff, ThresholdForSphericity, highpassfilter, apix) else: outarraythresholded, outarraythresholdedbinarized = threshold_binarize_array(inputmrc, FSCCutoff, ThresholdForSphericity, highpassfilter, apix) # write mrc file mrc_write = mrcfile.new(thresholded,overwrite=True) mrc_write.set_data(outarraythresholded.astype('<f4')) mrc_write.voxel_size = (float(apix),float(apix),float(apix)) mrc_write.update_header_from_data() mrc_write.close() mrc_write = mrcfile.new(thresholdedbinarized,overwrite=True) mrc_write.set_data(outarraythresholdedbinarized.astype('<f4')) mrc_write.voxel_size = (float(apix),float(apix),float(apix)) mrc_write.update_header_from_data() mrc_write.close()
def write_mrc(imageblock, file_path, overwrite=False): """ write an image block to disk as an .mrc file """ if not file_path.endswith('.mrc'): file_path = file_path + '.mrc' mrcfile.new(file_path, imageblock.data, overwrite=overwrite)
def test_read(tmp_path): # mrc file mrc_path1 = tmp_path / 'test1.mrc' mrc_path2 = tmp_path / 'test2.mrc' mrcfile.new(str(mrc_path1), np.ones((3, 3), dtype=np.float32)) mrcfile.new(str(mrc_path2), np.ones((3, 3), dtype=np.float32)) # star file df = pd.DataFrame({ 'rlnCoordinateX': [1, 2], 'rlnCoordinateY': [1, 2], 'rlnCoordinateZ': [1, 2], 'rlnOriginX': [1, 2], 'rlnOriginY': [1, 2], 'rlnOriginZ': [1, 2], 'rlnAngleRot': [1, 2], 'rlnAngleTilt': [1, 2], 'rlnAnglePsi': [1, 2], 'rlnMicrographName': ['test1', 'test2'], }) star_path = tmp_path / 'test.star' starfile.new(df, star_path) p = read(tmp_path / '*', name_regex=r'test\d') assert len(p) == 4 assert len(p.volumes) == 2
def gen_mrc(dir_path, classes=None, centre_crop=True, save_to_local=False): classes = [4, 8, 16, 32, 64, 128, 256] if classes is None else classes clean_data = mrcfile.open(os.path.join(dir_path, 'exper_20000_clean.mrc')).data noisy_data = mrcfile.open(os.path.join(dir_path, 'exper_20000_noisy.mrc')).data for j in classes: with mrcfile.new(dir_path + '/exper_20000_noisy_size' + str(j) + '_mrc', overwrite=True) as mrc: with mrcfile.new(dir_path + '/exper_20000_clean_size' + str(j) + '_mrc', overwrite=True) as mrc1: mrc1.set_data(np.zeros((20000, j, j), dtype=np.float32)) mrc.set_data(np.zeros((20000, j, j), dtype=np.float32)) for i in range(20000): im_noisy = noisy_data[i] im_clean = clean_data[i] im_noisy = Image.fromarray(im_noisy) im_clean = Image.fromarray(im_clean) mrc.data[i, :, :] = np.array( _resize_image(im_noisy, (j, j))) mrc1.data[i, :, :] = np.array( _resize_image(im_clean, (j, j))) print('ok')
def extractCandidates(jobFilename='', resultFilename='', orientFilename='', sizeParticle=None, maxNumParticle=0, minScore=-1, write2disk=0, margin=None,mask=None, structuredMask=None): # construct the original job from the xml file if jobFilename=='': jobFilename='JobInfo.xml' from pytom.localization.peak_job import PeakJob job = PeakJob() job.fromXMLFile(jobFilename) from pytom.localization.extraction_peak_result import ExPeakResult res = ExPeakResult() res.volFilename = job.volume.getFilename() from pytom.basic.files import read from pytom_numpy import vol2npy from copy import deepcopy if resultFilename=='': res.resultFilename='scores.em' else: res.resultFilename = resultFilename if orientFilename=='': res.orientFilename='angles.em' else: res.orientFilename = orientFilename if mask: resultfile, maskfile = read(res.resultFilename), read(mask) resultdata, maskdata = deepcopy(vol2npy(resultfile)), deepcopy(vol2npy(maskfile)) import mrcfile x,y,z,sx,sy,sz = job.volume.subregion masked_data = (resultdata*maskdata[x:x+sx,y:y+sy,z:z+sz]) #masked_data[masked_data < 0.00001] = resultdata.median() mrcfile.new(res.resultFilename.replace('.em', '_masked.em'),masked_data.T, overwrite=True) res.resultFilename = res.resultFilename.replace('.em', '_masked.em') res.angleList = job.rotations[:] res.score = job.score.__class__ if maxNumParticle <= 0: return None res.readAll() if sizeParticle==None: ref = job.reference.getVolume() sizeParticle = [ref.sizeX(),ref.sizeY(),ref.sizeZ()] print(job.volume.subregion[:3]) particleList = res.findParticles(sizeParticle,maxNumParticle,minScore,write2disk,margin, offset=job.volume.subregion[:3], structured_mask=structuredMask) return particleList
def chained_save_stack(ctx_obj, o): """ Save MRC stack to output file """ if os.path.exists(o): # TODO move this check before anything starts running logger.error("output file {} already exists! " "please rename/delete or use flag -o with different output name") sys.exit(1) logger.info("saving stack {}..".format(o)) mrcfile.new(o, ctx_obj.stack)
def extract_single_image(dataSlice, sliceId, out_name, tiltangle, origdir, outdir, prefix): if origdir: outname = os.path.join(outdir, out_name.replace('sorted_', prefix)) else: outname = os.path.join(outdir, '{}{:02d}.mrc'.format(prefix, sliceId)) print(f'extracted {os.path.basename(outname)} into {outdir}') # write(outname, data[:,:,sliceId], tilt_angle=tiltangle) mrcfile.new(outname, dataSlice.astype('float32'), overwrite=True)
def setUp(self): os.makedirs(self.tmp_dir) shape = (64, 64, 64) self.data = np.random.rand(*shape).astype('float32') with mrcfile.new(self.out) as f: f.set_data(self.data) with mrcfile.new(self.out_compressed, compression='gzip') as f: f.set_data(self.data)
def create_good_files(self): """Create some files known to be valid and return their names.""" good_mrc_name_1 = os.path.join(self.test_output, "good_file_1.mrc") good_mrc_name_2 = os.path.join(self.test_output, "good_file_2.mrc") # Make good files which will pass validation with mrcfile.new(good_mrc_name_1) as mrc: mrc.set_data(np.arange(36, dtype=np.float32).reshape(3, 3, 4)) with mrcfile.new(good_mrc_name_2) as mrc: mrc.set_data(np.arange(36, dtype=np.uint16).reshape(3, 3, 4)) return [good_mrc_name_1, good_mrc_name_2]
def write_mrc(map, pixel_size, filename): # based on standard here: https://bio3d.colorado.edu/imod/doc/mrc_format.txt # # mrcfile initializes most things to zero, so we don't need to set everything. # alpha, beta, and gamma are initialized to 90., and mapc, mapr, and maps are # initialized to 1, 2, 3 nx, ny, nz = np.shape(map) xlen, ylen, zlen = [x * pixel_size for x in [nx, ny, nz]] # MRC files require the mean, max, and min for scaling. flat_map = map.flatten() amin = min(flat_map) amax = max(flat_map) amean = mean(flat_map) try: with mrcfile.new(filename) as mrc: mrc.header.nx, mrc.header.ny, mrc.header.nz = nx, ny, nz mrc.header.mode = 2 mrc.header.mx, mrc.header.my, mrc.header.mz = nx, ny, nz mrc.header.xlen, mrc.header.ylen, mrc.header.zlen = xlen, ylen, zlen mrc.header.amin = amin mrc.header.amax = amax mrc.header.amean = amean mrc.set_data(map.astype(np.float32)) except ValueError: filename = input(f'{filename} already exists, input new filename:\n') if filename[-4:] != '.mrc': filename = filename + '.mrc' with mrcfile.new(filename, overwrite=True) as mrc: mrc.header.nx, mrc.header.ny, mrc.header.nz = nx, ny, nz mrc.header.mode = 2 mrc.header.mx, mrc.header.my, mrc.header.mz = nx, ny, nz mrc.header.xlen, mrc.header.ylen, mrc.header.zlen = xlen, ylen, zlen mrc.header.amin = amin mrc.header.amax = amax mrc.header.amean = amean mrc.set_data(map.astype(np.float32))
def setUp(self): os.makedirs(self.tmp_dir) shape = (16, 32, 64) self.data = np.random.rand(*shape).astype('float32') # change axes order to fit MRC convention data0 = np.swapaxes(self.data, 0, -1) data1 = np.fliplr(data0) out_data = np.swapaxes(data1, 0, -1) with mrcfile.new(self.out) as f: f.set_data(out_data) with mrcfile.new(self.out_compressed, compression='gzip') as f: f.set_data(out_data)
def generateMask(self): ''' Generate the outside shell mask of the given virus. First, initialize data as ones where voxel threshold is larger than the given value, and zero where voxel threshold is smaller than the given value Second, scatter those 'ones' voxel in a given radius, i.e. use a cube or sphere among which voxels are set to ones Output: the mask of input filename ''' tmp_data = (self.model_data >= self.threshold ) * 1 # numpy matrix operation, simple and fast data = np.array(tmp_data, dtype=np.int8) print("Dealing with scattering voxels") for pos in np.argwhere(tmp_data == 1): for r in range(1, self.r + 1): loc = self.scatter(pos, r) data[loc[:, 0], loc[:, 1], loc[:, 2]] = 1 # avoid frequent for-loop with mf.new(self.filename + '.mask.mrc', overwrite=True) as nmask: print("Generating mask") nmask.set_data(data) nmask.voxel_size = self.voxel_size print("Mask Generated -- Done")
def export_spectra(filename, simObj): """ Write out spectral data from simulations Args: filename (str): name of the MRC file containing the binned intensities simObj (Simulation): the simulation object from simulations """ max_twotheta = simObj.screen.max_twotheta bins = np.linspace(0, max_twotheta, simObj.screen.npix // 2) twotheta_bins = np.digitize(simObj.screen.two_theta, bins) binned_intensities = np.zeros((simObj.num_images + 1, len(bins))) binned_intensities[0] = bins for i in range(simObj.screen.npix): for j in range(simObj.screen.npix): if simObj.screen.two_theta[i, j] > max_twotheta: continue else: binned_intensities[1:, twotheta_bins[i, j] - 1] += simObj.all_intensities[i, j] # Check if file already exists assert (not os.path.isfile(filename)), \ "Error in simulation.export_spectra: File already exists." with mrcfile.new(filename) as mrc: mrc.set_data(binned_intensities.astype(np.float32))
def main(): parser = get_cli() args = parser.parse_args() pythonpath = args.pythonpath sys.path.append(pythonpath) # gpu = args.gpu # if gpu is None: # print("No CUDA_VISIBLE_DEVICES passed...") # if torch.cuda.is_available(): # os.environ["CUDA_VISIBLE_DEVICES"] = "0" # else: # os.environ["CUDA_VISIBLE_DEVICES"] = gpu config_file = args.config_file config = Config(user_config_file=config_file) df = pd.read_csv(config.dataset_table, dtype={"tomo_name": str}) df.set_index('tomo_name', inplace=True) tomo_name = args.tomo_name input_tomo = df[config.processing_tomo][tomo_name] target_tomo = os.path.join(config.work_dir, tomo_name) target_tomo = os.path.join(target_tomo, "match_spectrum_filtered.mrc") with mrcfile.open(input_tomo, permissive=True) as m: tomo = m.data.astype("f4") tomo_h = m.header target_spectrum = pd.read_csv(target_tomo, sep="\t")["intensity"].values filtered_tomo = match_spectrum(tomo, target_spectrum, config.cutoff, config.smoothen) m = mrcfile.new(args.output, overwrite=True) m.set_data(filtered_tomo) m.set_extended_header(tomo_h) m.close()
def test_file_too_large(self): with mrcfile.new(self.temp_mrc_name) as mrc: mrc.set_data(np.arange(36, dtype=np.float32).reshape(3, 3, 4)) mrc.header.nz = 2 self.check_temp_mrc_invalid_with_warning("larger than expected") assert len(sys.stdout.getvalue()) == 0 assert len(sys.stderr.getvalue()) == 0
def test_many_problems_simultaneously(self): data = np.arange(-10, 20, dtype=np.float32).reshape(3, 2, 5) with mrcfile.new(self.temp_mrc_name) as mrc: mrc.set_data(data) mrc.set_extended_header(data) mrc.header.nz = 2 mrc.header.my = -1000 mrc.header.mz = -5 mrc.header.cella.y = -12.1 mrc.header.mapc = 5 mrc.header.dmin = 10 mrc.header.dmax = 11 mrc.header.dmean = 19.0 mrc.header.ispg = -20 mrc.header.exttyp = "fake" mrc.header.nversion = 0 mrc.header.rms = 0.0 mrc.header.nlabl = 4 mrc.header.label[9] = "test label" with warnings.catch_warnings(record=True): result = mrcfile.validate(self.temp_mrc_name, print_file=self.print_stream) assert result == False print_output = self.print_stream.getvalue() assert len(print_output.split("\n")) == 15 assert len(sys.stdout.getvalue()) == 0 assert len(sys.stderr.getvalue()) == 0
def test_invalid_mode(self): with mrcfile.new(self.temp_mrc_name) as mrc: mrc.set_data(np.arange(12, dtype=np.float32).reshape(1, 3, 4)) mrc.header.mode = 8 self.check_temp_mrc_invalid_with_warning("mode") assert len(sys.stdout.getvalue()) == 0 assert len(sys.stderr.getvalue()) == 0
def test_file_too_small(self): with mrcfile.new(self.temp_mrc_name) as mrc: mrc.set_data(np.arange(12, dtype=np.float32).reshape(1, 3, 4)) mrc.header.nz = 2 self.check_temp_mrc_invalid_with_warning("data block") assert len(sys.stdout.getvalue()) == 0 assert len(sys.stderr.getvalue()) == 0
def saveMRC( ndimage, filename ): """ Simple function to save as mrc. WILL OVERWRITE EXISTING FILES """ with mrcfile.new( filename, overwrite=True) as mrc: mrc.set_data( ndimage.astype(np.float32)) return
def save(self, filename): print("Saving MRC file: {}".format(filename)) new_mrcfile = mrc.new(filename) new_mrcfile.set_data(self.data) new_mrcfile.update_header_from_data() new_mrcfile.update_header_stats() new_mrcfile.close()
def write_mrc_from_atoms( path: Path, atoms: mda.AtomGroup, path_out: Path, context: float = 4.0, cut_box=True, keep_data=False, ): """mask a mrc map using an group of atoms""" if not atoms: logger.warning("Cannot crop empty atom selection. No file written") return with mrcfile.open(path) as mrc: voxel, grid, origin, full_data = _get_mrc_properties(mrc) data_mask = _create_voxel_mask(atoms, grid, origin, voxel, context, symmetric=keep_data) data = full_data * data_mask if cut_box: data, origin, voxel = _mrc_cutbox(data, full_data, origin, voxel, keep_full=keep_data) with mrcfile.new(path_out, overwrite=True) as mrc_out: mrc_out.set_data(data.transpose()) mrc_out.voxel_size = tuple(voxel.tolist()) mrc_out.header["origin"] = tuple(origin)
def write_data(data, path): assert data.ndim == 3 # only for 3D array data = data.astype(N.float32) data = data.transpose([2,1,0]) # this is according to tomominer.image.vol.eman2_util.numpy2em with mrcfile.new(path) as m: m.set_data(data)
def abinitio_cmd(stack_file, output): """\b ############################ Abinitio ############################ Abinitio accepts a stack file, calculates Abinitio algorithm on it and saves the results into output file (default adds '_abinitio' to stack name) """ if output is None: output = set_output_name(stack_file, 'abinitio') if os.path.exists(output): logger.error(f"file {yellow(output)} already exsits! remove first " "or use another name with '-o NAME'") return stack = load_stack_from_file(stack_file) logger.info(f'running abinitio on stack file {stack_file}..') output_stack = Abinitio.cryo_abinitio_c1_worker(stack) with mrcfile.new(output) as mrc_fh: mrc_fh.set_data(output_stack.astype('float32')) logger.info(f"saved to {yellow(output)}.")
def saveMRC(selectedData, voxelSize, cellVec, cellA_XYZ, filename, zip=True): with mrcfile.new(filename, overwrite=True, compression='gzip' if zip else None) as outmrc: outmrc.set_data(selectedData) outmrc.set_volume() outmrc.voxel_size = (voxelSize[0], voxelSize[1], voxelSize[2]) outmrc.header.nxstart, outmrc.header.nystart, outmrc.header.nzstart = cellVec[0], cellVec[1], cellVec[2] outmrc.header.cella.x, outmrc.header.cella.y, outmrc.header.cella.z = cellA_XYZ
def normalize_cmd(stack_file, output=None): """ \b ############################ Normalize Stack ############################ Normalize stack of projections. \b Example: $ python aspire.py normalize projections.mrc will produce file projections_normalized.mrc """ if output is None: output = set_output_name(stack_file, 'normalized') if os.path.exists(output): logger.error(f"output file {yellow(output)} already exsits! " f"remove first or use another name with '-o NAME' flag") return logger.info("normalizing projections..") stack = load_stack_from_file(stack_file, c_contiguous=True) normalized_stack = PreProcessor.normalize_background(stack.astype('float64')) with mrcfile.new(output) as fh: fh.set_data(normalized_stack.astype('float32')) logger.info(f"saved to {yellow(output)}.")
def downsample_stack_file(cls, stack_file, side, output_stack_file=None, mask_file=None): if output_stack_file is None: output_stack_file = set_output_name(stack_file, 'downsampled') if os.path.exists(output_stack_file): raise FileExistsError( f"output file '{yellow(output_stack_file)}' already exists!") if mask_file: if not os.path.exists(mask_file): logger.error(f"mask file {yellow(mask_file)} doesn't exist!") mask = load_stack_from_file(mask_file) else: mask = None stack = load_stack_from_file(stack_file) downsampled_stack = cls.downsample(stack, side, compute_fx=False, stack=True, mask=mask) logger.info( f"downsampled stack from size {stack.shape} to {downsampled_stack.shape}." f" saving to {yellow(output_stack_file)}..") with mrcfile.new(output_stack_file) as mrc_fh: mrc_fh.set_data(downsampled_stack) logger.debug(f"saved to {output_stack_file}")
def crop_stack_file(cls, stack_file, size, output_stack_file=None, fill_value=None): if output_stack_file is None: output_stack_file = set_output_name(stack_file, 'cropped') if os.path.exists(output_stack_file): raise FileExistsError( f"output file '{yellow(output_stack_file)}' already exists!") stack = load_stack_from_file(stack_file) fill_value = fill_value or PreProcessorConfig.crop_stack_fill_value cropped_stack = cls.crop_stack(stack, size, fill_value=fill_value) action = 'cropped' if size < stack.shape[1] else 'padded' logger.info( f"{action} stack from size {stack.shape} to size {cropped_stack.shape}." f" saving to {yellow(output_stack_file)}..") with mrcfile.new(output_stack_file) as mrc: mrc.set_data(cropped_stack) logger.debug(f"saved to {output_stack_file}")
def execute(paths): threshold = get_threshold(paths) experimental_map = mrcfile.open(paths['cleaned_map'], mode='r') experimental_data = deepcopy(experimental_map.data) # Remove low valued data and translate the higher values down to zero. experimental_data[experimental_data < 0] = 0 # experimental_data = percentile_filter(experimental_data, numpy.shape(experimental_data), 5) # Change all values < threshold to 0 experimental_data[experimental_data < threshold] = 0 # translate data to have min = 0 experimental_data[experimental_data > 0] -= threshold # normalize data with percentile value percentile = numpy.percentile( experimental_data[numpy.nonzero(experimental_data)], 60) experimental_data /= percentile # Get rid of the very high-intensity voxels by setting them to 95-percentile percentile_98 = numpy.percentile( experimental_data[numpy.nonzero(experimental_data)], 98) experimental_data[experimental_data > percentile_98] = percentile_98 # Print the normalized file to disk. with mrcfile.new(paths['normalized_map'], overwrite=True) as mrc: mrc.set_data(experimental_data) mrc.header.origin = experimental_map.header.origin mrc.close()
def generate_projections(outputFolder, modelID, SIZE, angles, grandcell): from voltools.volume import Volume volume = zeros((1200, SIZE, SIZE), dtype=float32) volume[225:-225, :, :] = grandcell[:, :, :] d_vol = Volume(volume, cuda_warmup=False) dx, dy, dz = grandcell.shape angles = numpy.arange(angles[0], int(angles[1]) + angleIncrement / 2, angleIncrement) noisefree_projections = zeros((len(angles), SIZE // 2, SIZE // 2), dtype=float32) for n, angle in enumerate(angles): gpu_proj = d_vol.transform(rotation=(0, angle, 0), rotation_order='szyx', rotation_units='deg', around_center=True).project(cpu=True) # noisefree_projections[n] = gpu_proj[300:-300,300:-300] noisefree_projections[n] = gpu_proj[256:-256, 256:-256] # = create_projections(dens, range(-60, 61, 3)) # noisefree_projections[n] /= noisefree_projections[n][:,140:160].mean() if not os.path.exists('{}/model_{}'.format(outputFolder, modelID)): os.mkdir('{}/model_{}'.format(outputFolder, modelID)) a = mrcfile.new('{}/model_{}/projections_grandmodel_{}.mrc'.format( outputFolder, modelID, modelID), data=noisefree_projections, overwrite=True) a.close() del d_vol return noisefree_projections