def _trigger(self):
V = model.get_volume()
# with tf.get_default_session() as sess:
V_py = tf.get_default_session().run(V)
V_py = np.fft.fftshift(V_py)
vname = os.path.join(logger.get_logger_dir(), 'volume.mrc')
mrcfile.new(vname, data=V_py, overwrite=True)
def threshold_binarize_mrc(inmrc, thresholded, thresholdedbinarized, FSCCutoff, ThresholdForSphericity, highpassfilter, apix,gpu=False):
# inmrc, thresholded, and thresholdedbinarized are paths to .mrc files
# Read MRC
inputmrc = (mrcfile.open(inmrc)).data
# binarize array
if gpu:
outarraythresholded, outarraythresholdedbinarized = cuda_functions.threshold_binarize_array_cuda(inputmrc, FSCCutoff, ThresholdForSphericity, highpassfilter, apix)
else:
outarraythresholded, outarraythresholdedbinarized = threshold_binarize_array(inputmrc, FSCCutoff, ThresholdForSphericity, highpassfilter, apix)
# write mrc file
mrc_write = mrcfile.new(thresholded,overwrite=True)
mrc_write.set_data(outarraythresholded.astype('<f4'))
mrc_write.voxel_size = (float(apix),float(apix),float(apix))
mrc_write.update_header_from_data()
mrc_write.close()
mrc_write = mrcfile.new(thresholdedbinarized,overwrite=True)
mrc_write.set_data(outarraythresholdedbinarized.astype('<f4'))
mrc_write.voxel_size = (float(apix),float(apix),float(apix))
mrc_write.update_header_from_data()
mrc_write.close()
def write_mrc(imageblock, file_path, overwrite=False):
"""
write an image block to disk as an .mrc file
"""
if not file_path.endswith('.mrc'):
file_path = file_path + '.mrc'
mrcfile.new(file_path, imageblock.data, overwrite=overwrite)
def test_read(tmp_path):
# mrc file
mrc_path1 = tmp_path / 'test1.mrc'
mrc_path2 = tmp_path / 'test2.mrc'
mrcfile.new(str(mrc_path1), np.ones((3, 3), dtype=np.float32))
mrcfile.new(str(mrc_path2), np.ones((3, 3), dtype=np.float32))
# star file
df = pd.DataFrame({
'rlnCoordinateX': [1, 2],
'rlnCoordinateY': [1, 2],
'rlnCoordinateZ': [1, 2],
'rlnOriginX': [1, 2],
'rlnOriginY': [1, 2],
'rlnOriginZ': [1, 2],
'rlnAngleRot': [1, 2],
'rlnAngleTilt': [1, 2],
'rlnAnglePsi': [1, 2],
'rlnMicrographName': ['test1', 'test2'],
})
star_path = tmp_path / 'test.star'
starfile.new(df, star_path)
p = read(tmp_path / '*', name_regex=r'test\d')
assert len(p) == 4
assert len(p.volumes) == 2
def gen_mrc(dir_path, classes=None, centre_crop=True, save_to_local=False):
classes = [4, 8, 16, 32, 64, 128, 256] if classes is None else classes
clean_data = mrcfile.open(os.path.join(dir_path,
'exper_20000_clean.mrc')).data
noisy_data = mrcfile.open(os.path.join(dir_path,
'exper_20000_noisy.mrc')).data
for j in classes:
with mrcfile.new(dir_path + '/exper_20000_noisy_size' + str(j) +
'_mrc',
overwrite=True) as mrc:
with mrcfile.new(dir_path + '/exper_20000_clean_size' + str(j) +
'_mrc',
overwrite=True) as mrc1:
mrc1.set_data(np.zeros((20000, j, j), dtype=np.float32))
mrc.set_data(np.zeros((20000, j, j), dtype=np.float32))
for i in range(20000):
im_noisy = noisy_data[i]
im_clean = clean_data[i]
im_noisy = Image.fromarray(im_noisy)
im_clean = Image.fromarray(im_clean)
mrc.data[i, :, :] = np.array(
_resize_image(im_noisy, (j, j)))
mrc1.data[i, :, :] = np.array(
_resize_image(im_clean, (j, j)))
print('ok')
def extractCandidates(jobFilename='', resultFilename='', orientFilename='', sizeParticle=None, maxNumParticle=0,
minScore=-1, write2disk=0, margin=None,mask=None, structuredMask=None):
# construct the original job from the xml file
if jobFilename=='':
jobFilename='JobInfo.xml'
from pytom.localization.peak_job import PeakJob
job = PeakJob()
job.fromXMLFile(jobFilename)
from pytom.localization.extraction_peak_result import ExPeakResult
res = ExPeakResult()
res.volFilename = job.volume.getFilename()
from pytom.basic.files import read
from pytom_numpy import vol2npy
from copy import deepcopy
if resultFilename=='':
res.resultFilename='scores.em'
else:
res.resultFilename = resultFilename
if orientFilename=='':
res.orientFilename='angles.em'
else:
res.orientFilename = orientFilename
if mask:
resultfile, maskfile = read(res.resultFilename), read(mask)
resultdata, maskdata = deepcopy(vol2npy(resultfile)), deepcopy(vol2npy(maskfile))
import mrcfile
x,y,z,sx,sy,sz = job.volume.subregion
masked_data = (resultdata*maskdata[x:x+sx,y:y+sy,z:z+sz])
#masked_data[masked_data < 0.00001] = resultdata.median()
mrcfile.new(res.resultFilename.replace('.em', '_masked.em'),masked_data.T, overwrite=True)
res.resultFilename = res.resultFilename.replace('.em', '_masked.em')
res.angleList = job.rotations[:]
res.score = job.score.__class__
if maxNumParticle <= 0:
return None
res.readAll()
if sizeParticle==None:
ref = job.reference.getVolume()
sizeParticle = [ref.sizeX(),ref.sizeY(),ref.sizeZ()]
print(job.volume.subregion[:3])
particleList = res.findParticles(sizeParticle,maxNumParticle,minScore,write2disk,margin, offset=job.volume.subregion[:3], structured_mask=structuredMask)
return particleList
def chained_save_stack(ctx_obj, o):
""" Save MRC stack to output file """
if os.path.exists(o): # TODO move this check before anything starts running
logger.error("output file {} already exists! "
"please rename/delete or use flag -o with different output name")
sys.exit(1)
logger.info("saving stack {}..".format(o))
mrcfile.new(o, ctx_obj.stack)
def extract_single_image(dataSlice, sliceId, out_name, tiltangle, origdir,
outdir, prefix):
if origdir:
outname = os.path.join(outdir, out_name.replace('sorted_', prefix))
else:
outname = os.path.join(outdir, '{}{:02d}.mrc'.format(prefix, sliceId))
print(f'extracted {os.path.basename(outname)} into {outdir}')
# write(outname, data[:,:,sliceId], tilt_angle=tiltangle)
mrcfile.new(outname, dataSlice.astype('float32'), overwrite=True)
def setUp(self):
os.makedirs(self.tmp_dir)
shape = (64, 64, 64)
self.data = np.random.rand(*shape).astype('float32')
with mrcfile.new(self.out) as f:
f.set_data(self.data)
with mrcfile.new(self.out_compressed, compression='gzip') as f:
f.set_data(self.data)
def create_good_files(self):
"""Create some files known to be valid and return their names."""
good_mrc_name_1 = os.path.join(self.test_output, "good_file_1.mrc")
good_mrc_name_2 = os.path.join(self.test_output, "good_file_2.mrc")
# Make good files which will pass validation
with mrcfile.new(good_mrc_name_1) as mrc:
mrc.set_data(np.arange(36, dtype=np.float32).reshape(3, 3, 4))
with mrcfile.new(good_mrc_name_2) as mrc:
mrc.set_data(np.arange(36, dtype=np.uint16).reshape(3, 3, 4))
return [good_mrc_name_1, good_mrc_name_2]
def write_mrc(map, pixel_size, filename):
# based on standard here: https://bio3d.colorado.edu/imod/doc/mrc_format.txt
#
# mrcfile initializes most things to zero, so we don't need to set everything.
# alpha, beta, and gamma are initialized to 90., and mapc, mapr, and maps are
# initialized to 1, 2, 3
nx, ny, nz = np.shape(map)
xlen, ylen, zlen = [x * pixel_size for x in [nx, ny, nz]]
# MRC files require the mean, max, and min for scaling.
flat_map = map.flatten()
amin = min(flat_map)
amax = max(flat_map)
amean = mean(flat_map)
try:
with mrcfile.new(filename) as mrc:
mrc.header.nx, mrc.header.ny, mrc.header.nz = nx, ny, nz
mrc.header.mode = 2
mrc.header.mx, mrc.header.my, mrc.header.mz = nx, ny, nz
mrc.header.xlen, mrc.header.ylen, mrc.header.zlen = xlen, ylen, zlen
mrc.header.amin = amin
mrc.header.amax = amax
mrc.header.amean = amean
mrc.set_data(map.astype(np.float32))
except ValueError:
filename = input(f'{filename} already exists, input new filename:\n')
if filename[-4:] != '.mrc':
filename = filename + '.mrc'
with mrcfile.new(filename, overwrite=True) as mrc:
mrc.header.nx, mrc.header.ny, mrc.header.nz = nx, ny, nz
mrc.header.mode = 2
mrc.header.mx, mrc.header.my, mrc.header.mz = nx, ny, nz
mrc.header.xlen, mrc.header.ylen, mrc.header.zlen = xlen, ylen, zlen
mrc.header.amin = amin
mrc.header.amax = amax
mrc.header.amean = amean
mrc.set_data(map.astype(np.float32))
def setUp(self):
os.makedirs(self.tmp_dir)
shape = (16, 32, 64)
self.data = np.random.rand(*shape).astype('float32')
# change axes order to fit MRC convention
data0 = np.swapaxes(self.data, 0, -1)
data1 = np.fliplr(data0)
out_data = np.swapaxes(data1, 0, -1)
with mrcfile.new(self.out) as f:
f.set_data(out_data)
with mrcfile.new(self.out_compressed, compression='gzip') as f:
f.set_data(out_data)
def generateMask(self):
'''
Generate the outside shell mask of the given virus.
First, initialize data as ones where voxel threshold is larger than the given value, and zero where voxel threshold is smaller than the given value
Second, scatter those 'ones' voxel in a given radius, i.e. use a cube or sphere among which voxels are set to ones
Output: the mask of input filename
'''
tmp_data = (self.model_data >= self.threshold
) * 1 # numpy matrix operation, simple and fast
data = np.array(tmp_data, dtype=np.int8)
print("Dealing with scattering voxels")
for pos in np.argwhere(tmp_data == 1):
for r in range(1, self.r + 1):
loc = self.scatter(pos, r)
data[loc[:, 0], loc[:, 1],
loc[:, 2]] = 1 # avoid frequent for-loop
with mf.new(self.filename + '.mask.mrc', overwrite=True) as nmask:
print("Generating mask")
nmask.set_data(data)
nmask.voxel_size = self.voxel_size
print("Mask Generated -- Done")
def export_spectra(filename, simObj):
"""
Write out spectral data from simulations
Args:
filename (str): name of the MRC file containing the binned intensities
simObj (Simulation): the simulation object from simulations
"""
max_twotheta = simObj.screen.max_twotheta
bins = np.linspace(0, max_twotheta, simObj.screen.npix // 2)
twotheta_bins = np.digitize(simObj.screen.two_theta, bins)
binned_intensities = np.zeros((simObj.num_images + 1, len(bins)))
binned_intensities[0] = bins
for i in range(simObj.screen.npix):
for j in range(simObj.screen.npix):
if simObj.screen.two_theta[i, j] > max_twotheta:
continue
else:
binned_intensities[1:, twotheta_bins[i, j] -
1] += simObj.all_intensities[i, j]
# Check if file already exists
assert (not os.path.isfile(filename)), \
"Error in simulation.export_spectra: File already exists."
with mrcfile.new(filename) as mrc:
mrc.set_data(binned_intensities.astype(np.float32))
def main():
parser = get_cli()
args = parser.parse_args()
pythonpath = args.pythonpath
sys.path.append(pythonpath)
# gpu = args.gpu
# if gpu is None:
# print("No CUDA_VISIBLE_DEVICES passed...")
# if torch.cuda.is_available():
# os.environ["CUDA_VISIBLE_DEVICES"] = "0"
# else:
# os.environ["CUDA_VISIBLE_DEVICES"] = gpu
config_file = args.config_file
config = Config(user_config_file=config_file)
df = pd.read_csv(config.dataset_table, dtype={"tomo_name": str})
df.set_index('tomo_name', inplace=True)
tomo_name = args.tomo_name
input_tomo = df[config.processing_tomo][tomo_name]
target_tomo = os.path.join(config.work_dir, tomo_name)
target_tomo = os.path.join(target_tomo, "match_spectrum_filtered.mrc")
with mrcfile.open(input_tomo, permissive=True) as m:
tomo = m.data.astype("f4")
tomo_h = m.header
target_spectrum = pd.read_csv(target_tomo, sep="\t")["intensity"].values
filtered_tomo = match_spectrum(tomo, target_spectrum, config.cutoff,
config.smoothen)
m = mrcfile.new(args.output, overwrite=True)
m.set_data(filtered_tomo)
m.set_extended_header(tomo_h)
m.close()
def test_file_too_large(self):
with mrcfile.new(self.temp_mrc_name) as mrc:
mrc.set_data(np.arange(36, dtype=np.float32).reshape(3, 3, 4))
mrc.header.nz = 2
self.check_temp_mrc_invalid_with_warning("larger than expected")
assert len(sys.stdout.getvalue()) == 0
assert len(sys.stderr.getvalue()) == 0
def test_many_problems_simultaneously(self):
data = np.arange(-10, 20, dtype=np.float32).reshape(3, 2, 5)
with mrcfile.new(self.temp_mrc_name) as mrc:
mrc.set_data(data)
mrc.set_extended_header(data)
mrc.header.nz = 2
mrc.header.my = -1000
mrc.header.mz = -5
mrc.header.cella.y = -12.1
mrc.header.mapc = 5
mrc.header.dmin = 10
mrc.header.dmax = 11
mrc.header.dmean = 19.0
mrc.header.ispg = -20
mrc.header.exttyp = "fake"
mrc.header.nversion = 0
mrc.header.rms = 0.0
mrc.header.nlabl = 4
mrc.header.label[9] = "test label"
with warnings.catch_warnings(record=True):
result = mrcfile.validate(self.temp_mrc_name,
print_file=self.print_stream)
assert result == False
print_output = self.print_stream.getvalue()
assert len(print_output.split("\n")) == 15
assert len(sys.stdout.getvalue()) == 0
assert len(sys.stderr.getvalue()) == 0
def test_invalid_mode(self):
with mrcfile.new(self.temp_mrc_name) as mrc:
mrc.set_data(np.arange(12, dtype=np.float32).reshape(1, 3, 4))
mrc.header.mode = 8
self.check_temp_mrc_invalid_with_warning("mode")
assert len(sys.stdout.getvalue()) == 0
assert len(sys.stderr.getvalue()) == 0
def test_file_too_small(self):
with mrcfile.new(self.temp_mrc_name) as mrc:
mrc.set_data(np.arange(12, dtype=np.float32).reshape(1, 3, 4))
mrc.header.nz = 2
self.check_temp_mrc_invalid_with_warning("data block")
assert len(sys.stdout.getvalue()) == 0
assert len(sys.stderr.getvalue()) == 0
def saveMRC( ndimage, filename ):
""" Simple function to save as mrc. WILL OVERWRITE EXISTING FILES """
with mrcfile.new( filename, overwrite=True) as mrc:
mrc.set_data( ndimage.astype(np.float32))
return
def save(self, filename):
print("Saving MRC file: {}".format(filename))
new_mrcfile = mrc.new(filename)
new_mrcfile.set_data(self.data)
new_mrcfile.update_header_from_data()
new_mrcfile.update_header_stats()
new_mrcfile.close()
def write_mrc_from_atoms(
path: Path,
atoms: mda.AtomGroup,
path_out: Path,
context: float = 4.0,
cut_box=True,
keep_data=False,
):
"""mask a mrc map using an group of atoms"""
if not atoms:
logger.warning("Cannot crop empty atom selection. No file written")
return
with mrcfile.open(path) as mrc:
voxel, grid, origin, full_data = _get_mrc_properties(mrc)
data_mask = _create_voxel_mask(atoms,
grid,
origin,
voxel,
context,
symmetric=keep_data)
data = full_data * data_mask
if cut_box:
data, origin, voxel = _mrc_cutbox(data,
full_data,
origin,
voxel,
keep_full=keep_data)
with mrcfile.new(path_out, overwrite=True) as mrc_out:
mrc_out.set_data(data.transpose())
mrc_out.voxel_size = tuple(voxel.tolist())
mrc_out.header["origin"] = tuple(origin)
def write_data(data, path):
assert data.ndim == 3 # only for 3D array
data = data.astype(N.float32)
data = data.transpose([2,1,0]) # this is according to tomominer.image.vol.eman2_util.numpy2em
with mrcfile.new(path) as m:
m.set_data(data)
def abinitio_cmd(stack_file, output):
"""\b
############################
Abinitio
############################
Abinitio accepts a stack file, calculates Abinitio algorithm on it and saves
the results into output file (default adds '_abinitio' to stack name)
"""
if output is None:
output = set_output_name(stack_file, 'abinitio')
if os.path.exists(output):
logger.error(f"file {yellow(output)} already exsits! remove first "
"or use another name with '-o NAME'")
return
stack = load_stack_from_file(stack_file)
logger.info(f'running abinitio on stack file {stack_file}..')
output_stack = Abinitio.cryo_abinitio_c1_worker(stack)
with mrcfile.new(output) as mrc_fh:
mrc_fh.set_data(output_stack.astype('float32'))
logger.info(f"saved to {yellow(output)}.")
def saveMRC(selectedData, voxelSize, cellVec, cellA_XYZ, filename, zip=True):
with mrcfile.new(filename, overwrite=True, compression='gzip' if zip else None) as outmrc:
outmrc.set_data(selectedData)
outmrc.set_volume()
outmrc.voxel_size = (voxelSize[0], voxelSize[1], voxelSize[2])
outmrc.header.nxstart, outmrc.header.nystart, outmrc.header.nzstart = cellVec[0], cellVec[1], cellVec[2]
outmrc.header.cella.x, outmrc.header.cella.y, outmrc.header.cella.z = cellA_XYZ
def normalize_cmd(stack_file, output=None):
""" \b
############################
Normalize Stack
############################
Normalize stack of projections.
\b
Example:
$ python aspire.py normalize projections.mrc
will produce file projections_normalized.mrc
"""
if output is None:
output = set_output_name(stack_file, 'normalized')
if os.path.exists(output):
logger.error(f"output file {yellow(output)} already exsits! "
f"remove first or use another name with '-o NAME' flag")
return
logger.info("normalizing projections..")
stack = load_stack_from_file(stack_file, c_contiguous=True)
normalized_stack = PreProcessor.normalize_background(stack.astype('float64'))
with mrcfile.new(output) as fh:
fh.set_data(normalized_stack.astype('float32'))
logger.info(f"saved to {yellow(output)}.")
def downsample_stack_file(cls,
stack_file,
side,
output_stack_file=None,
mask_file=None):
if output_stack_file is None:
output_stack_file = set_output_name(stack_file, 'downsampled')
if os.path.exists(output_stack_file):
raise FileExistsError(
f"output file '{yellow(output_stack_file)}' already exists!")
if mask_file:
if not os.path.exists(mask_file):
logger.error(f"mask file {yellow(mask_file)} doesn't exist!")
mask = load_stack_from_file(mask_file)
else:
mask = None
stack = load_stack_from_file(stack_file)
downsampled_stack = cls.downsample(stack,
side,
compute_fx=False,
stack=True,
mask=mask)
logger.info(
f"downsampled stack from size {stack.shape} to {downsampled_stack.shape}."
f" saving to {yellow(output_stack_file)}..")
with mrcfile.new(output_stack_file) as mrc_fh:
mrc_fh.set_data(downsampled_stack)
logger.debug(f"saved to {output_stack_file}")
def crop_stack_file(cls,
stack_file,
size,
output_stack_file=None,
fill_value=None):
if output_stack_file is None:
output_stack_file = set_output_name(stack_file, 'cropped')
if os.path.exists(output_stack_file):
raise FileExistsError(
f"output file '{yellow(output_stack_file)}' already exists!")
stack = load_stack_from_file(stack_file)
fill_value = fill_value or PreProcessorConfig.crop_stack_fill_value
cropped_stack = cls.crop_stack(stack, size, fill_value=fill_value)
action = 'cropped' if size < stack.shape[1] else 'padded'
logger.info(
f"{action} stack from size {stack.shape} to size {cropped_stack.shape}."
f" saving to {yellow(output_stack_file)}..")
with mrcfile.new(output_stack_file) as mrc:
mrc.set_data(cropped_stack)
logger.debug(f"saved to {output_stack_file}")
def execute(paths):
threshold = get_threshold(paths)
experimental_map = mrcfile.open(paths['cleaned_map'], mode='r')
experimental_data = deepcopy(experimental_map.data)
# Remove low valued data and translate the higher values down to zero.
experimental_data[experimental_data < 0] = 0
# experimental_data = percentile_filter(experimental_data, numpy.shape(experimental_data), 5)
# Change all values < threshold to 0
experimental_data[experimental_data < threshold] = 0
# translate data to have min = 0
experimental_data[experimental_data > 0] -= threshold
# normalize data with percentile value
percentile = numpy.percentile(
experimental_data[numpy.nonzero(experimental_data)], 60)
experimental_data /= percentile
# Get rid of the very high-intensity voxels by setting them to 95-percentile
percentile_98 = numpy.percentile(
experimental_data[numpy.nonzero(experimental_data)], 98)
experimental_data[experimental_data > percentile_98] = percentile_98
# Print the normalized file to disk.
with mrcfile.new(paths['normalized_map'], overwrite=True) as mrc:
mrc.set_data(experimental_data)
mrc.header.origin = experimental_map.header.origin
mrc.close()
def generate_projections(outputFolder, modelID, SIZE, angles, grandcell):
from voltools.volume import Volume
volume = zeros((1200, SIZE, SIZE), dtype=float32)
volume[225:-225, :, :] = grandcell[:, :, :]
d_vol = Volume(volume, cuda_warmup=False)
dx, dy, dz = grandcell.shape
angles = numpy.arange(angles[0],
int(angles[1]) + angleIncrement / 2, angleIncrement)
noisefree_projections = zeros((len(angles), SIZE // 2, SIZE // 2),
dtype=float32)
for n, angle in enumerate(angles):
gpu_proj = d_vol.transform(rotation=(0, angle, 0),
rotation_order='szyx',
rotation_units='deg',
around_center=True).project(cpu=True)
# noisefree_projections[n] = gpu_proj[300:-300,300:-300]
noisefree_projections[n] = gpu_proj[256:-256, 256:-256]
# = create_projections(dens, range(-60, 61, 3))
# noisefree_projections[n] /= noisefree_projections[n][:,140:160].mean()
if not os.path.exists('{}/model_{}'.format(outputFolder, modelID)):
os.mkdir('{}/model_{}'.format(outputFolder, modelID))
a = mrcfile.new('{}/model_{}/projections_grandmodel_{}.mrc'.format(
outputFolder, modelID, modelID),
data=noisefree_projections,
overwrite=True)
a.close()
del d_vol
return noisefree_projections
