def main(): inputs = sys.argv[1:-1] output = sys.argv[-1] idx = 0 m = 1 M = len(inputs) for stack in inputs: N = spx.EMUtil.get_image_count(stack) for n in np.arange(N): img = spx.EMData() img.read_image(stack, n) # Write image to the particle stack: if idx == 0: # If this is the first image, we initiate as a normal .mrcs stack. img.write_image(output, idx) else: # Subsequent images are directly appended to the file: with open(output, 'ab') as mrcf: spx.EMNumPy.em2numpy(img).tofile(mrcf) idx += 1 print 'Appended image %d/%d from stack %d/%d. Total images appended: %d.' % (n+1, N, m, M, idx) m += 1 # Maybe it is better to update the NZ in header with the MRCheaderUpdate.py script? header = ioMRC.readMRCHeader( output ) header['dimensions'][0] = idx # Now we write the header back: with open( output, 'rb+' ) as mrcf: ioMRC.writeMRCHeader( mrcf, header, endchar = '<' ) print 'Done!'
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + """ [options] <.mrc(s) file>
Given an existing .mrc(s) file, will evaluate its size and number of images/slices (NZ) and UPDATE IN-PLACE (CAUTION!) corresponding header information.
Reference for MRC format description:
http://bio3d.colorado.edu/imod/doc/mrc_format.txt
This script is based on Robb McLeod's ioMRC.py available at:
https://bitbucket.org/emmrcz/python-mrcz
NOTE: Only vanilla MRC(S) files are supported (i.e. compressed MRC formats not supported)
"""
parser = OptionParser(usage)
parser.add_option( "--stats", action="store_true", help="Will update the statistics of the file (minval, maxval, avgval, std). May take some time depending on file size.", default=False )
parser.add_option( "--endian", metavar="le", type="string", default='le', help="MRC file endianness (default is little-endian)" )
(options, args) = parser.parse_args()
command = ' '.join(sys.argv)
if options.endian == 'le':
endchar = '<'
else:
endchar = '>'
fpath = args[0] # The file to be updated
hsize = 1024 # Standard MRC header is 1024 bytes long
S = os.path.getsize( fpath ) # Get the current size of the file in bytes
# print S-hsize
header = ioMRC.readMRCHeader( fpath, slices=0 )[0] # Read in the current header of the file
if 'meta' not in header.keys():
header['meta'] = None
sizeof = np.dtype(header['dtype']).itemsize # Get the size in bytes of each value in the current MRC format
# print sizeof
with open(fpath, 'rb+') as f:
NX = header['dimensions'][2]
NY = header['dimensions'][1]
NZ = header['dimensions'][0]
print('Current dimensions of MRC file: NX=%d,NY=%d,NZ=%d' % (NX,NY,NZ))
NZnew = ( S - hsize ) / ( NX*NY*sizeof ) # This will yield the actual number of Z slices
print('New dimensions of MRC file: NX=%d,NY=%d,NZ=%d' % (NX,NY,NZnew))
header['dimensions'][0] = NZnew
if options.stats:
# Let's update the statistics of the stack/volume:
header['meanImage'] = 0.0
f.seek(1024)
for i in np.arange(NZnew):
# We will read one image or slice at a time in order to save memory:
image = np.fromfile( f, dtype=header['dtype'], count=NX*NY )
minImage = np.min(image)
maxImage = np.max(image)
meanImage = np.mean(image)
if minImage < header['minImage']:
header['minImage'] = minImage
if maxImage > header['maxImage']:
header['maxImage'] = maxImage
header['meanImage'] += meanImage
header['meanImage'] /= NZnew
# Now we write the header back:
ioMRC.writeMRCHeader( f, header, slices=0, endchar=endchar )
def main():
# Get arguments:
folders = sys.argv[1]
merge_dirfile = sys.argv[2]
png_path = sys.argv[3]
stack_path = sys.argv[4]
stack_rootname = sys.argv[5]
box_size = int(sys.argv[6])
phaori_shift = np.array([float(sys.argv[7].split(',')[0]), float(sys.argv[7].split(',')[1])]) # How many degrees to offset the phase origins in order to get a protein in the center of the particle for a zero-tilt image
apix = float(sys.argv[8]) # pixel size in Angstroems
microscope_voltage = float(sys.argv[9]) # microscope voltage in kV
microscope_cs = float(sys.argv[10]) # microscope spherical aberration in mm
ampcontrast = 1-float(sys.argv[11])**2 # amplitude contrast of CTF (obtained here from phase contrast)
magnification = float(sys.argv[12])
sigcc = float(sys.argv[13])
if sys.argv[14] == 'y':
invert_micrograph = True
else:
invert_micrograph = False
if sys.argv[15] == 'y':
normalize_box = True
else:
normalize_box = False
if sys.argv[16] == 'y':
calculate_defocus_tilted = True
else:
calculate_defocus_tilted = False
if sys.argv[17] == 'y':
save_phase_flipped = True
else:
save_phase_flipped = False
if sys.argv[18] == 'y':
save_wiener_filtered = True
else:
save_wiener_filtered = False
sigma = float(sys.argv[19]) # Sigma for normalization of the windowed images (if normalize_box == True)
if sys.argv[20] == 'Defocus/Lattice':
tiltgeom = ''
elif sys.argv[20] == 'Defocus':
tiltgeom = 'DEFOCUS_'
elif sys.argv[20] == 'Lattice':
tiltgeom = 'LATTICE_'
elif sys.argv[20] == 'Merge':
tiltgeom = 'MERGE_'
if sys.argv[21] == 'Micrograph':
ctfcor = True
stack_rootname = stack_rootname + '_ctfcor'
else:
ctfcor = False
if sys.argv[22] == 'y':
save_pick_fig = True
else:
save_pick_fig = False
if sys.argv[23] == 'y':
use_masked_image = True
else:
use_masked_image = False
n_threads = int(sys.argv[24])
if n_threads < 1:
n_threads = 1
this_thread = int(sys.argv[25])
# End arguments
f = open(merge_dirfile,'r')
img_dirs = f.readlines()
f.close()
# Constant parameters to be written on the .par file of this dataset:
shx = 0.0
shy = 0.0
occ = 100.0
logp = 0
sig = 0.5 # This has nothing to do with the normalization SIGMA!
score = 0.0
chg = 0.0
idx = 0
# box_fail = 0
phaori_err = 0
prog = 0.0
N = len(img_dirs)
batch_size = round(float(N)/n_threads)
first_img = int((this_thread-1) * batch_size)
if this_thread < n_threads:
last_img = int(first_img + batch_size)
else:
last_img = N
img_dirs = img_dirs[first_img:last_img]
n = first_img + 1
print '\nJob %d/%d picking particles from micrographs %d to %d...\n' % (this_thread, n_threads, n, last_img)
f = open(stack_path+stack_rootname+'_1_r1-%.4d.par' % this_thread, 'w+')
for d in img_dirs:
d = d.strip()
imname = d.split('/')[-1]
try:
# Read in all relevant image parameters:
params = Read2dxCfgFile(folders+d+'/2dx_image.cfg')
except:
print '\nProblem with image %s!\n' % d
continue
if ctfcor:
try:
mrc = glob.glob(folders+d+'/image_ctfcor.mrc')[0]
img = spx.get_image(mrc) # Read image
bf = open(folders+d+'/image_ctfcor.box', 'w+')
except:
print '\nCTF-corrected micrograph not found for image %s!\n' % d
continue
else:
if use_masked_image:
# First we look for the masked, zero-padded, normalized micrograph:
try:
# # There might be some funny numbers appended to the file name so we have to look for the shortest one to get the right file:
# mrclist = glob.glob(folders+d+'/m'+imname+'*.mrc')
# lenlist = []
# for m in mrclist:
# lenlist.append(len(m))
# shortest_idx = np.argsort(lenlist)[0]
# # print mrclist[shortest_idx]
# mrc = mrclist[shortest_idx]
# bf = open(os.path.splitext(mrc)[0]+'.box', 'w+')
# # mrc = sorted(glob.glob(folders+d+'/m'+imname+'*.mrc'))[0]
# # bf = open(folders+d+'/m'+imname+'.box', 'w+')
mrc = folders + d + '/' + params['imagename'] + '.mrc'
img = spx.get_image(mrc) # Read image
bf = open(folders + d + '/' + params['imagename'] + '.box', 'w+')
except:
# If it doesn't exist, we try the unmasked, zero-padded, normalized micrograph:
try:
# mrclist = glob.glob(folders+d+'/'+imname+'*.mrc')
# lenlist = []
# for m in mrclist:
# lenlist.append(len(m))
# shortest_idx = np.argsort(lenlist)[0]
# # print mrclist[shortest_idx]
# mrc = mrclist[shortest_idx]
# bf = open(os.path.splitext(mrc)[0]+'.box', 'w+')
# # mrc = sorted(glob.glob(folders+d+'/m'+imname+'*.mrc'))[0]
# # bf = open(folders+d+'/m'+imname+'.box', 'w+')
mrc = folders + d + '/' + params['nonmaskimagename'] + '.mrc'
img = spx.get_image(mrc) # Read image
bf = open(folders + d + '/' + params['nonmaskimagename'] + '.box', 'w+')
except:
# If neither exist we skip this image
print '::\nProblem with image %s!\n' % d
continue
else:
# If the user requires, we go directly to the unmasked, zero-padded, normalized micrograph:
try:
# mrclist = glob.glob(folders+d+'/'+imname+'*.mrc')
# lenlist = []
# for m in mrclist:
# lenlist.append(len(m))
# shortest_idx = np.argsort(lenlist)[0]
# # print mrclist[shortest_idx]
# mrc = mrclist[shortest_idx]
# bf = open(os.path.splitext(mrc)[0]+'.box', 'w+')
# # mrc = sorted(glob.glob(folders+d+'/m'+imname+'*.mrc'))[0]
# # bf = open(folders+d+'/m'+imname+'.box', 'w+')
mrc = folders + d + '/' + params['nonmaskimagename'] + '.mrc'
img = spx.get_image(mrc) # Read image
bf = open(folders + d + '/' + params['nonmaskimagename'] + '.box', 'w+')
except:
# If neither exist we skip this image
print '::\nProblem with image %s!' % d
print '::'
continue
print '::\nNow boxing unit cells of micrograph %d/%d.\n' % (n, N)
print mrc
try:
if invert_micrograph:
img = -1.0 * img
imgarr = spx.EMNumPy.em2numpy(img)
profile = glob.glob(folders+d+'/*profile.dat')[0]
dat = ReadProfileDat(profile)
ccmean = np.mean(dat[:,4])
ccstd = np.std(dat[:,4])
cc_thr = ccmean + sigcc * ccstd
print ':\nImage average value:%.2f' % img['mean']
print ':Image standard deviation:%.2f' % img['sigma']
print ':'
print ':CC scores average value:%.2f' % ccmean
print ':CC scores standard deviation:%.2f' % ccstd
print ':Only particles with CC score above %.2f will be picked.\n' % cc_thr
# # Get several values related to defocus, astigmatism and tilting:
# params = Read2dxCfgFile(folders+d+'/2dx_image.cfg')
w = img.get_xsize()
# Get the unit-cell vectors:
if sum(params['PHAORI']) == 0:
PhaOriX = 0
PhaOriY = 0
a = [0,0]
b = [0,0]
else:
a,b = LatticeReciprocal2Real(params['u'], params['v'], w)
# Convert from Numpy-array to list:
a = [a[0], a[1]]
b = [b[0], b[1]]
x,y = CalculatePickingPositions(dat, a, b, w, params['PHAORI'], phaori_shift, params[tiltgeom+'TLTANG'])
# Plot the picking profile:
if save_pick_fig:
meanimg = np.mean(imgarr)
stdimg = np.std(imgarr)
climimg = [meanimg - 2 * stdimg, meanimg + 2 * stdimg]
fig1 = plt.figure()
plt.imshow(imgarr, cmap=cm.gray, vmin=climimg[0], vmax=climimg[1])
Axes1 = fig1.gca()
# The following values are constant within each crystal:
phi = 90.0 - params[tiltgeom+'TAXA']
theta = params[tiltgeom+'TLTANG']
psi = 270.0 - params[tiltgeom+'TLTAXIS']
ang = params['AST_ANGLE']
m = 0
for i in np.arange(dat.shape[0]):
try:
# Adjust the picking coordinates for the .box file and picking plots:
xbox = x[i] + w/2 - box_size/2
ybox = y[i] + w/2 - box_size/2
# if dat[i,4] < cc_thr or np.isnan(x[i]) or np.isnan(y[i]):
if dat[i,4] < cc_thr:
if save_pick_fig:
# Write red patch on image to be saved as .png describing the picking positions:
# Axes1.add_patch(patches.Circle((dat[i,2], dat[i,3]), edgecolor='red', facecolor='none', linewidth=0.2, radius=20))
Axes1.add_patch(patches.Rectangle(xy=(xbox, ybox), width=box_size, height=box_size, edgecolor='red', facecolor='none', linewidth=0.2))
else:
box = spx.Util.window(img,int(box_size),int(box_size),1,int(round(x[i])),int(round(y[i])))
# Normalize box to zero mean and constant pre-defined sigma:
if normalize_box:
box = NormalizeStack([box], sigma)[0]
if calculate_defocus_tilted and not ctfcor:
RLDEF1,RLDEF2 = CalculateDefocusTilted(x[i], y[i], apix, params[tiltgeom+'TLTAXIS'], params[tiltgeom+'TLTANG'], params['DEFOCUS1'], params['DEFOCUS2'])
else:
RLDEF1 = params['DEFOCUS1']
RLDEF2 = params['DEFOCUS2']
# Write .par file with the parameters for each particle in the dataset:
print >>f, ' %d' % (idx+1),' %.2f' % psi,' %.2f' % theta,' %.2f' % phi,' %.2f' % shx,' %.2f' % shy,' %d' % magnification,' %d' % n,' %.2f' % RLDEF1,' %.2f' % RLDEF2,' %.2f' % ang,' %.2f' % occ,' %d' % logp,' %.4f' % sig,' %.2f' % score,' %.2f' % chg
# Write the picking information to the .box file:
print >>bf, '%d' % xbox, '\t%d' % ybox, '\t%d' % box_size, '\t%d' % box_size
# Write image to the particle stack:
if idx == 0:
# If this is the first image, we initiate as a normal .mrcs stack.
box.write_image(stack_path+stack_rootname+'-%.4d.mrcs' % this_thread, idx)
else:
# Subsequent images are directly appended to the file:
with open(stack_path+stack_rootname+'-%.4d.mrcs' % this_thread, 'ab') as mrcf:
spx.EMNumPy.em2numpy(box).tofile(mrcf)
if (save_phase_flipped or save_wiener_filtered) and not ctfcor:
# Convert CTF parameters to SPARX convention:
defocus = (RLDEF1+RLDEF2)/2
ast = RLDEF1-RLDEF2
if params['AST_ANGLE'] < 0.0:
astang = 360.0 + params['AST_ANGLE']
else:
astang = params['AST_ANGLE']
# Generate SPARX CTF object:
p = [defocus * 1e-4, microscope_cs, microscope_voltage, apix, 0, ampcontrast * 100, ast * 1e-4, astang]
spx.set_ctf(box, p)
ctf = spx.generate_ctf(p)
# Phase-flip the image:
if save_phase_flipped and not ctfcor:
boxctfcor = spx.filt_ctf(box, ctf,binary=1)
if normalize_box:
boxctfcor = NormalizeStack([boxctfcor], sigma)[0]
# Write image to the particle stack:
if idx == 0:
# If this is the first image, we initiate as a normal .mrcs stack.
boxctfcor.write_image( stack_path+stack_rootname+'_phase-flipped-%.4d.mrcs' % this_thread, idx )
else:
# Subsequent images are directly appended to the file:
with open( stack_path+stack_rootname+'_phase-flipped-%.4d.mrcs' % this_thread, 'ab' ) as mrcf:
spx.EMNumPy.em2numpy(boxctfcor).tofile( mrcf )
# Wiener-filter the image:
if save_wiener_filtered and not ctfcor:
boxctfcor = spx.filt_ctf(box, ctf,binary=0)
if normalize_box:
boxctfcor = NormalizeStack([boxctfcor], sigma)[0]
# Write image to the particle stack:
if idx == 0:
# If this is the first image, we initiate as a normal .mrcs stack.
boxctfcor.write_image( stack_path+stack_rootname+'_wiener-filtered-%.4d.mrcs' % this_thread, idx )
else:
# Subsequent images are directly appended to the file:
with open( stack_path+stack_rootname+'_wiener-filtered-%.4d.mrcs' % this_thread, 'ab' ) as mrcf:
spx.EMNumPy.em2numpy(boxctfcor).tofile( mrcf )
if save_pick_fig:
# Write green patch on image to be saved as .png describing the picking positions:
# Axes1.add_patch(patches.Circle((dat[i,2], dat[i,3]), edgecolor='lime', facecolor='none', linewidth=0.2, radius=20))
Axes1.add_patch(patches.Rectangle(xy=(xbox, ybox), width=box_size, height=box_size, edgecolor='lime', facecolor='none', linewidth=0.2))
m += 1
idx += 1
except RuntimeError:
if save_pick_fig:
# Write red patch on image to be saved as .png describing the picking positions:
# Axes1.add_patch(patches.Circle((dat[i,2], dat[i,3]), edgecolor='red', facecolor='none', linewidth=0.2, radius=20))
Axes1.add_patch(patches.Rectangle(xy=(xbox, ybox), width=box_size, height=box_size, edgecolor='red', facecolor='none', linewidth=0.2))
print 'Failed to box CC peak (%d,%d) at position (%d,%d) in micrograph %d/%d!' % (dat[i,0], dat[i,1], int(round(x[i])), int(round(y[i])), n, N)
print '\nBoxed %d/%d CC peaks from micrograph %d/%d.\n' % (m, i+1, n, N)
# Update the counts in the MRC headers:
# First the normal particle stack:
header = ioMRC.readMRCHeader( stack_path+stack_rootname+'-%.4d.mrcs' % this_thread )
header['dimensions'][0] = idx
# Now we write the header back:
with open( stack_path+stack_rootname+'-%.4d.mrcs' % this_thread, 'rb+' ) as mrcf:
ioMRC.writeMRCHeader( mrcf, header, endchar = '<' )
# Then the phase-flipped:
if save_phase_flipped and not ctfcor:
header = ioMRC.readMRCHeader( stack_path+stack_rootname+'_phase-flipped-%.4d.mrcs' % this_thread )
header['dimensions'][0] = idx
# Now we write the header back:
with open( stack_path+stack_rootname+'_phase-flipped-%.4d.mrcs' % this_thread, 'rb+' ) as mrcf:
ioMRC.writeMRCHeader( mrcf, header, endchar = '<' )
# Then the Wiener-filtered:
if save_wiener_filtered and not ctfcor:
header = ioMRC.readMRCHeader( stack_path+stack_rootname+'_wiener-filtered-%.4d.mrcs' % this_thread )
header['dimensions'][0] = idx
# Now we write the header back:
with open( stack_path+stack_rootname+'_wiener-filtered-%.4d.mrcs' % this_thread, 'rb+' ) as mrcf:
ioMRC.writeMRCHeader( mrcf, header, endchar = '<' )
# print '<<@progress: %d>>' % round(n*100.0/N)
# Report progress to the GUI:
prog += 75.0/N
if prog >= 1.0:
print '<<@progress: +%d>>' % round(prog)
prog -= np.floor(prog)
if save_pick_fig:
fig1.savefig(png_path+'mic_%.3d_' % n+imname+'_picking.png', dpi=300)
plt.close(fig1)
n += 1
except RuntimeError:
# print '::PROBLEM WITH MICROGRAPH %d/%d!!! Maybe it was not found?' % (n, N)
# print '::'
# print mrc
print '\nPROBLEM WITH MICROGRAPH:'
print '%s' % mrc
print 'Maybe it was not found?'
except ValueError:
# print '::PROBLEM WITH CC PROFILE FOR IMAGE %d/%d!!!' % (n, N)
# print '::'
# print mrc
print '\nPROBLEM WITH CC PROFILE FOR IMAGE:'
print '%s' % mrc
bf.close()
# n += 1
# print '::Total boxed unit cells:%d' % idx
# print '::Failed to box %d unit cells.' % box_fail
# print '<<@progress: +%d>>' % round(prog/0.01)
print '\nJob %d/%d finished picking particles.\n' % (this_thread, n_threads)
f.close()
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + """ [options] <.mrc(s) file> Given an existing .mrc(s) file, will evaluate its size and number of images/slices (NZ) and UPDATE IN-PLACE (CAUTION!) corresponding header information. Reference for MRC format description: http://bio3d.colorado.edu/imod/doc/mrc_format.txt This script is based on Robb McLeod's ioMRC.py available at: https://bitbucket.org/emmrcz/python-mrcz NOTE: Only vanilla MRC(S) files are supported (i.e. compressed MRC formats not supported) """ parser = OptionParser(usage) parser.add_option( "--stats", action="store_true", help="Will update the statistics of the file (minval, maxval, avgval, std). May take some time depending on file size.", default=False ) parser.add_option( "--endian", metavar="le", type="string", default='le', help="MRC file endianness (default is little-endian)" ) (options, args) = parser.parse_args() command = ' '.join(sys.argv) if options.endian == 'le': endchar = '<' else: endchar = '>' fpath = args[0] # The file to be updated hsize = 1024 # Standard MRC header is 1024 bytes long S = os.path.getsize( fpath ) # Get the current size of the file in bytes # print S-hsize header = ioMRC.readMRCHeader( fpath ) # Read in the current header of the file sizeof = np.dtype(header['dtype']).itemsize # Get the size in bytes of each value in the current MRC format # print sizeof with open(fpath, 'rb+') as f: NX = header['dimensions'][2] NY = header['dimensions'][1] NZ = header['dimensions'][0] print 'Current dimensions of MRC file: NX=%d,NY=%d,NZ=%d' % (NX,NY,NZ) NZnew = ( S - hsize ) / ( NX*NY*sizeof ) # This will yield the actual number of Z slices print 'New dimensions of MRC file: NX=%d,NY=%d,NZ=%d' % (NX,NY,NZnew) header['dimensions'][0] = NZnew if options.stats: # Let's update the statistics of the stack/volume: header['meanImage'] = 0.0 f.seek(1024) for i in np.arange(NZnew): # We will read one image or slice at a time in order to save memory: image = np.fromfile( f, dtype=header['dtype'], count=NX*NY ) minImage = np.min(image) maxImage = np.max(image) meanImage = np.mean(image) if minImage < header['minImage']: header['minImage'] = minImage if maxImage > header['maxImage']: header['maxImage'] = maxImage header['meanImage'] += meanImage header['meanImage'] /= NZnew # Now we write the header back: ioMRC.writeMRCHeader( f, header, endchar )