def main():
progname = os.path.basename(sys.argv[0])
usage = progname + """ <half-map1> <half-map2> [options]
Given two unmasked and unfiltered reconstructions from random halves of your dataset, calculates the FSC between them and applies additional postprocessing filters.
Output:
-FSC plot(s) in PNG format
-Text file containing description of FSC and filters applied (_data.fsc)
-Masked and unmasked postprocessed maps
"""
parser = OptionParser(usage)
parser.add_option("--out",
metavar="postprocess",
type="string",
default='postprocess',
help="Output rootname.")
parser.add_option("--angpix",
metavar=1.0,
type="float",
help="Pixel size in Angstroems")
parser.add_option(
"--fsc_threshold",
metavar=0.143,
default=0.143,
type="float",
help="Display the resolution at which the FSC curve crosses this value."
)
parser.add_option(
"--lowpass",
metavar='"auto"',
default='auto',
help=
"Resolution (in Angstroems) at which to low-pass filter the final map. A negative value will skip low-pass filtering. Default is to low-pass at resolution determined from FSC threshold."
)
parser.add_option(
"--mask",
metavar="MyMask.mrc",
type="string",
help=
"A file containing the mask to be applied to the half-maps before calculating the FSC."
)
parser.add_option(
"--force_mask",
action="store_true",
help=
"Force using this mask even if it has strange properties such as values outside the range [0,1].",
default=False)
parser.add_option(
"--mask_radius",
metavar=0.5,
default=None,
type="float",
help=
"If not specifying a mask file, a soft spherical mask can be created. This is the radius of this mask, in pixels or fraction of the box size."
)
parser.add_option(
"--mask_edge_width",
metavar=6.0,
default=None,
type="float",
help=
"This is the width of the cosine edge for the mask to be created, in pixels or fraction of the box size."
)
parser.add_option(
"--mask_center",
metavar="0,0,0",
default=None,
type="string",
help=
"Three numbers describing where the center of spherical mask should be placed within the 3D box (in pixels). Default is the middle of the box: (0,0,0). Can be positive or negative."
)
parser.add_option(
"--mw",
metavar=1000.0,
type="float",
help=
"Molecular mass in kDa of particle or helical segment comprised within the mask. Needed to calculate volume-normalized Single-Particle Wiener filter (Sindelar & Grigorieff, JSB 2012). If not specified, will do conventional FSC weighting on the final map (Rosenthal & Henderson, JMB 2003)."
)
parser.add_option(
"--mw_ignore",
metavar=0.0,
type="float",
default=0.0,
help=
"EXPERIMENTAL OPTION: Molecular mass in kDa present within the mask that needs to be ignored. Needed to calculate an adaptation of the volume-normalized Single-Particle Wiener filter (Sindelar & Grigorieff, JSB 2012). Only used if --mw is also specified. May be useful if your particles are extracted from 2D crystals."
)
parser.add_option(
"--skip_fsc_weighting",
action="store_true",
help=
"Do NOT apply FSC weighting (Rosenthal & Henderson, JMB 2003) to the final map, nor the Single-Particle Wiener filter (Sindelar & Grigorieff, JSB 2012).",
default=False)
parser.add_option(
"--apply_fsc2",
action="store_true",
help=
"Apply the square of the FSC curve as a filter. Generally should be used together with the --skip_fsc_weighting option.",
default=False)
parser.add_option(
"--gaussian",
action="store_true",
help=
"Apply a Gaussian low-pass filter to the map, with cutoff defined by --lowpass. (default)",
default=True)
parser.add_option(
"--cosine",
action="store_true",
help=
"Apply a cosine-edge instead of Gaussian low-pass filter to the map, with cutoff defined by --lowpass. The width of the cosine edge can be specified with the option --edge_width.",
default=False)
parser.add_option(
"--cosine_edge_width",
metavar=2.0,
type="float",
help=
"Width of the cosine-edge filter (in Fourier pixels). If set to zero, becomes a top-hat filter.",
default=2.0)
parser.add_option(
"--tophat",
action="store_true",
help=
"Apply a top-hat low-pass filter to the final map. Equivalent to specifying --cosine with --edge_width=0.",
default=False)
parser.add_option(
"--mtf",
type="string",
help="File containing the detector MTF for sharpening of the final map."
)
parser.add_option(
"--auto_bfac",
metavar="10.0,0.0",
default=None,
type="string",
help=
"Estimate B-factor automatically using information in this resolution range, in Angstroems (lowres,highres). This works based on the Guinier plot, which should ideally be a straight line from ~10.0 A and beyond (Rosenthal & Henderson, JMB 2003).'If you set lowres and/or maxres to -1, these values will be calculated automatically. MTF and FSC weighting information are employed, if not ommitted."
)
parser.add_option(
"--adhoc_bfac",
metavar=0.0,
type="float",
help=
"Apply an ad-hoc B-factor to the map (in Angstroems^2). Can be positive (smoothing) or negative (sharpening).",
default=0.0)
parser.add_option(
"--randomize_below_fsc",
metavar=0.8,
type="float",
help=
"If provided, will randomize phases for all Fourier shells beyond where the FSC drops below this value, to assess correlations introduced by the mask by High-Resolution Noise Substitution (Chen et al, Ultramicroscopy 2013). Be aware that this procedure may introduce a 'dip' in the FSC curves at the corresponding resolution value."
)
# parser.add_option("--evaluate_spw_random", action="store_true", default=False, help="If both --mw and --randomize_below_fsc are provided, will evalute the performance of the Single-Particle Wiener filter (Sindelar & Grigorieff, JSB 2012) on the phase-randomized maps. Useful to assess how much artifacts (e.g. ringing) are being amplified by this filter.")
# parser.add_option("--cone_aperture", metavar=90.0, type="float", help="Instead of the FSC, calculate the Fourier Conical Correlation, within a cone with this aperture, in degrees.")
parser.add_option(
"--cone_aperture",
metavar=90.0,
type="float",
help=
"If data contains a missing cone, use this option to exclude it from FSC calculations. A missing cone may introduce artificially high correlations in the FSC. The cone aperture (2*Theta) is given in degrees."
)
parser.add_option(
"--xy_only",
action="store_true",
default=False,
help=
"CAUTION! EXPERIMENTAL OPTION: Evaluate average resolution along X,Y planes only."
)
parser.add_option(
"--z_only",
action="store_true",
default=False,
help=
"CAUTION! EXPERIMENTAL OPTION: Evaluate average resolution along the Z direction only."
)
parser.add_option(
"--refine_res_lim",
metavar=10.0,
type="float",
help=
"Resolution limit in Angstroems used during the refinement, to be displayed on the FSC plots."
)
parser.add_option(
"--resample",
metavar=1.0,
type="float",
help=
"Resample the final result in Fourier space to this pixel size (in Angstroems) in order to make the volume larger or smaller."
)
# parser.add_option("--three_sigma", action="store_true", help="Show the 3-Sigma criterion curve on the FSC plots (van Heel, Ultramicroscopy 1987).", default=False)
parser.add_option(
"--dpi",
metavar=300,
type="int",
default=300,
help=
"Resolution of the PNG files to be saved with the FSC curves (dots per inch)."
)
(options, args) = parser.parse_args()
command = ' '.join(sys.argv)
# Do some sanity checks:
if len(sys.argv) < 3:
# print len(sys.argv), args
print 'You must specify at least two map files to compute an FSC:'
print usage
sys.exit(1)
if options.lowpass != 'auto':
options.lowpass = float(options.lowpass)
if options.mw != None and options.mw < 0.0:
print 'Molecular mass cannot be negative!'
sys.exit(1)
if options.mw_ignore != None and options.mw_ignore < 0.0:
print 'Molecular mass to be ignored cannot be negative!'
sys.exit(1)
if options.angpix == None:
print '\nWARNING: Pixel size was not specified. Assuming 1.0 A/pixel.'
options.angpix = 1.0
elif options.angpix <= 0.0:
print 'Pixel size must be greater than zero!'
sys.exit(1)
if options.cosine_edge_width != None and options.cosine_edge_width < 0.0:
print '\nCosine edge width cannot be negative!'
sys.exit(1)
if options.refine_res_lim != None and options.refine_res_lim <= 0.0:
print 'Refinement resolution limit must be greater than zero!'
sys.exit(1)
if options.randomize_below_fsc != None and (
options.randomize_below_fsc < -1.0
or options.randomize_below_fsc > 1.0):
print 'FSC values for phase-randomization must be in the range [-1,1]!'
sys.exit(1)
if options.cone_aperture != None:
options.cone_aperture = float(options.cone_aperture) / 2
if options.tophat:
options.gaussian = False
options.cosine = True
options.cosine_edge_width = 0.0
if options.mask_center == None:
mask_center = [0, 0, 0]
else:
mask_center = np.array(map(float, options.mask_center.split(',')))
if options.resample != None and options.resample <= 0.0:
print(
'Resampling pixel size must be greater than zero! options.resample = %f A'
% options.resample)
sys.exit(1)
# Read in the two maps:
sys.stdout = open(os.devnull, "w") # Suppress output
map1 = ioMRC.readMRC(args[0])[0]
map2 = ioMRC.readMRC(args[1])[0]
map3 = ioMRC.readMRC(args[2])[0]
map4 = ioMRC.readMRC(args[3])[0]
sys.stdout = sys.__stdout__
# We check if there is a mask file and if not, should we create one?
if options.mask != None:
sys.stdout = open(os.devnull, "w") # Suppress output
mask = ioMRC.readMRC(options.mask)[0]
sys.stdout = sys.__stdout__
elif options.mask_radius != None or options.mask_edge_width != None:
if options.mask_radius == None:
options.mask_radius = 0.5
if options.mask_edge_width == None:
options.mask_edge_width = 6.0
mask = util.SoftMask(map1.shape,
radius=options.mask_radius,
width=options.mask_edge_width,
xyz=mask_center)
sys.stdout = open(os.devnull, "w") # Suppress output
ioMRC.writeMRC(mask,
options.out + '-mask.mrc',
dtype='float32',
pixelsize=options.angpix,
quickStats=True)
sys.stdout = sys.__stdout__
options.mask = options.out + '-mask.mrc'
# Resolution range to estimate B-factor:
if options.auto_bfac != None:
resrange = options.auto_bfac.split(',')
minres = float(resrange[0])
maxres = float(resrange[1])
NSAM = np.round(
np.sqrt(np.sum(np.power(map1.shape, 2))) / 2.0 /
np.sqrt(len(map1.shape))).astype(
'int') + 1 # For cubic volumes this is just half the box size + 1.
freq = (np.arange(NSAM) / (2.0 * (NSAM - 1) * options.angpix)).reshape(
NSAM, 1)
freq[0] = 1.0 / 999 # Just to avoid dividing by zero later
freq2 = freq * freq
if np.any(map1.shape != map2.shape):
print 'Input maps must be the same size!'
sys.exit(1)
print '\nCalculating unmasked FSC...'
if options.cone_aperture == None:
fsc12 = util.FCC(map1,
map2,
xy_only=options.xy_only,
z_only=options.z_only)
fsc34 = util.FCC(map3,
map4,
xy_only=options.xy_only,
z_only=options.z_only)
else:
# l = NSAM/2 + 1
# if np.mod(NSAM, 2):
# freq[1:] = np.arange( float(l) ) / (NSAM - 1)
# else:
# freq[1:] = np.arange( float(l) ) / NSAM
fsc = util.FCC(map1,
map2, [options.cone_aperture],
invertCone=True,
xy_only=options.xy_only,
z_only=options.z_only)
# if options.three_sigma:
# three_sigma_curve = 3.0 / np.sqrt( np.reshape(fscmat[:,2], (l, 1)) )
dat = np.append(
1.0 / freq, freq, axis=1
) # Start creating the matrix that will be written to an output file
head = 'Res \t1/Res \t' # Header of the output file describing the data columns
res12 = ResolutionAtThreshold(freq[1:], fsc12[1:NSAM],
options.fsc_threshold)
res34 = ResolutionAtThreshold(freq[1:], fsc34[1:NSAM],
options.fsc_threshold)
print 'FSC 1-2 >= %.3f up to %.3f A (unmasked)' % (options.fsc_threshold,
res12)
print 'FSC 3-4 >= %.3f up to %.3f A (unmasked)' % (options.fsc_threshold,
res34)
# Plot
plt.figure()
# if options.three_sigma:
# plt.plot(freq[1:], fsc, freq[1:], three_sigma_curve)
# else:
# plt.plot(freq[1:], fsc)
plt.plot(freq[1:], fsc12[1:NSAM], freq[1:], fsc34[1:NSAM])
plt.title('Fourier Ring Correlation')
plt.ylabel('FRC')
plt.xlabel('Spatial frequency (1/A)')
plt.legend(['4k4k', '8k4k'])
plt.ylim([0.0, 1.0])
plt.minorticks_on()
plt.grid(b=True, which='both')
plt.savefig(options.out + '_frc-intra.png', dpi=options.dpi)
plt.close()
# # plt.plot(freq[1:], fsc)
# map1amps = np.abs( np.fft.fftshift( np.fft.fftn( map1 ) ) )
# map2amps = np.abs( np.fft.fftshift( np.fft.fftn( map2 ) ) )
# map1ps = util.RadialProfile( map1amps )
# map2ps = util.RadialProfile( map2amps )
# plt.plot(freq[1:], map1ps[1:NSAM], freq[1:], map2ps[1:NSAM] )
# plt.title('1D Amplitude Spectrum')
# plt.ylabel('Amplitudes (a.u.)')
# plt.xlabel('Spatial frequency (1/A)')
# # plt.ylim([0.0,1.0])
# plt.minorticks_on()
# # ax = plt.gca()
# # ax.set_yticks([options.fsc_threshold], minor=True)
# # ax.set_yticklabels([str(options.fsc_threshold)], minor=True)
# # if options.refine_res_lim != None:
# # ax.axvline(1.0/options.refine_res_lim, linestyle='dashed', linewidth=0.75, color='m')
# plt.legend([args[0], args[1]])
# plt.grid(b=True, which='both')
# plt.savefig(options.out+'_amps.png', dpi=options.dpi)
# plt.close()
# plt.plot(freq[1:], map1ps[1:NSAM]**2, freq[1:], map2ps[1:NSAM]**2 )
# plt.title('1D Power Spectrum [Amps^2]')
# plt.ylabel('Power (a.u.)')
# plt.xlabel('Spatial frequency (1/A)')
# # plt.ylim([0.0,1.0])
# plt.minorticks_on()
# # ax = plt.gca()
# # ax.set_yticks([options.fsc_threshold], minor=True)
# # ax.set_yticklabels([str(options.fsc_threshold)], minor=True)
# # if options.refine_res_lim != None:
# # ax.axvline(1.0/options.refine_res_lim, linestyle='dashed', linewidth=0.75, color='m')
# plt.legend([args[0], args[1]])
# plt.grid(b=True, which='both')
# plt.savefig(options.out+'_ps.png', dpi=options.dpi)
# plt.close()
# plt.plot(freq[1:], np.log( map1ps[1:NSAM] ), freq[1:], np.log( map2ps[1:NSAM] ) )
# plt.title('Guinier plot [ log(Amps) ]')
# plt.ylabel('log(Amps)')
# plt.xlabel('Spatial frequency (1/A)')
# # plt.ylim([0.0,1.0])
# plt.minorticks_on()
# # ax = plt.gca()
# # ax.set_yticks([options.fsc_threshold], minor=True)
# # ax.set_yticklabels([str(options.fsc_threshold)], minor=True)
# # if options.refine_res_lim != None:
# # ax.axvline(1.0/options.refine_res_lim, linestyle='dashed', linewidth=0.75, color='m')
# plt.legend([args[0], args[1]])
# plt.grid(b=True, which='both')
# plt.savefig(options.out+'_guinier.png', dpi=options.dpi)
# plt.close()
dat = np.append(dat, fsc12[:NSAM], axis=1) # Append the unmasked FSC
head += 'FSC12-unmasked\t'
dat = np.append(dat, fsc34[:NSAM], axis=1) # Append the unmasked FSC
head += 'FSC34-unmasked\t'
# Now we go to the mask-related operations which are activated if a mask or MW are specified. If only
if options.mask != None or options.mw != None:
if options.mask == None: # If MW is specified but no mask, we issue a warning:
print '\nWARNING: You specified MW without a mask. This may produce inaccurate results!'
# rmin = np.float( np.min( map1.shape ) ) / 2.0
# mask = util.SoftMask( map1.shape, radius = rmin - 4.0, width = 6.0 )
mask = np.ones(map1.shape, dtype='float')
sys.stdout = open(os.devnull, "w") # Suppress output
ioMRC.writeMRC(mask,
options.out + '-mask.mrc',
dtype='float32',
pixelsize=options.angpix,
quickStats=True)
sys.stdout = sys.__stdout__
options.mask = options.out + '-mask.mrc'
if options.force_mask == False:
if (mask.min() < -0.001 or mask.max() > 1.001):
print '\nMask values not in range [0,1]! Min: %.6f, Max: %.6f' % (
mask.min(), mask.max())
sys.exit(1)
else:
print '\nWARNING: You are forcing a mask that may have strange properties. Use at your own risk!!!'
map1masked = map1 * mask
map2masked = map2 * mask
print '\nCalculating masked FSC...'
if options.cone_aperture == None:
fsc_mask = util.FCC(map1masked,
map2masked,
xy_only=options.xy_only,
z_only=options.z_only)
else:
fsc_mask = util.FCC(map1masked,
map2masked, [options.cone_aperture],
invertCone=True,
xy_only=options.xy_only,
z_only=options.z_only)
res_mask = ResolutionAtThreshold(freq[1:], fsc_mask[1:NSAM],
options.fsc_threshold)
print 'FSC >= %.3f up to %.3f A (masked)' % (options.fsc_threshold,
res_mask)
dat = np.append(dat, fsc_mask[:NSAM], axis=1) # Append the masked FSC
head += 'FSC-masked\t'
if options.randomize_below_fsc == None:
# Plot
plt.figure()
plt.plot(freq[1:], fsc_mask[1:NSAM])
plt.title('Fourier Shell Correlation - masked')
plt.ylabel('FSC')
plt.xlabel('Spatial frequency (1/A)')
plt.minorticks_on()
ax = plt.gca()
ax.set_yticks([options.fsc_threshold], minor=True)
ax.set_yticklabels([str(options.fsc_threshold)], minor=True)
if options.refine_res_lim != None:
ax.axvline(1.0 / options.refine_res_lim,
linestyle='dashed',
linewidth=0.75,
color='m')
plt.grid(b=True, which='both')
plt.savefig(options.out + '_fsc-masked.png', dpi=options.dpi)
plt.close()
else:
rand_res = ResolutionAtThreshold(freq[1:], fsc[1:NSAM],
options.randomize_below_fsc)
print '\nRandomizing phases beyond %.2f A...\n' % rand_res
rand_freq = 1.0 / rand_res
np.random.seed(
seed=123
) # We have to enforce the random seed otherwise different runs would not be comparable
map1randphase = util.HighResolutionNoiseSubstitution(
map1, lp=rand_res, apix=options.angpix)
np.random.seed(
seed=1234) # Cannot use same random seed for both maps!!!
map2randphase = util.HighResolutionNoiseSubstitution(
map2, lp=rand_res, apix=options.angpix)
# We mask the phase-randomized maps:
map1randphasemasked = map1randphase * mask
map2randphasemasked = map2randphase * mask
print '\nCalculating masked FSC for phase-randomized maps...'
if options.cone_aperture == None:
fsc_mask_rnd = util.FCC(map1randphasemasked,
map2randphasemasked,
xy_only=options.xy_only,
z_only=options.z_only)
else:
fsc_mask_rnd = util.FCC(map1randphasemasked,
map2randphasemasked,
[options.cone_aperture],
invertCone=True,
xy_only=options.xy_only,
z_only=options.z_only)
# We compute FSCtrue following (Chen et al, Ultramicroscopy 2013). For masked maps this will correct the FSC for eventual refinement overfitting, including from the mask:
# fsc_mask_true[freq >= rand_freq] = (fsc_mask[freq >= rand_freq] - fsc_mask_rnd[freq >= rand_freq]) / (1 - fsc_mask_rnd[freq >= rand_freq])
fsc_mask_true = ((fsc_mask - fsc_mask_rnd) / (1.0 - fsc_mask_rnd))
fsc_mask_true[:NSAM][freq < rand_freq] = fsc_mask[:NSAM][
freq < rand_freq]
fsc_mask_true = np.nan_to_num(fsc_mask_true)
res_mask_true = ResolutionAtThreshold(freq[1:],
fsc_mask_true[1:NSAM],
options.fsc_threshold)
print 'FSC >= %.3f up to %.3f A (masked - true)' % (
options.fsc_threshold, res_mask_true)
dat = np.append(dat, fsc_mask_true[:NSAM],
axis=1) # Append the true masked FSC
head += 'FSC-masked_true\t'
# Plot
plt.figure()
plt.plot(freq[1:], fsc_mask[1:NSAM], freq[1:],
fsc_mask_rnd[1:NSAM], freq[1:], fsc_mask_true[1:NSAM])
plt.title('Fourier Shell Correlation - masked')
plt.ylabel('FSC')
plt.xlabel('Spatial frequency (1/A)')
plt.legend(['FSC', 'FSC - phase randomized', 'FSC - true'])
plt.minorticks_on()
ax = plt.gca()
ax.set_yticks([options.fsc_threshold], minor=True)
ax.set_yticklabels([str(options.fsc_threshold)], minor=True)
if options.refine_res_lim != None:
ax.axvline(1.0 / options.refine_res_lim,
linestyle='dashed',
linewidth=0.75,
color='m')
plt.grid(b=True, which='both')
plt.savefig(options.out + '_fsc-masked_true.png', dpi=options.dpi)
plt.close()
res_mask = res_mask_true
if options.mw == None and options.randomize_below_fsc == None:
# Plot
plt.figure()
plt.plot(freq[1:], fsc[1:NSAM], freq[1:], fsc_mask[1:NSAM])
plt.title('Fourier Shell Correlation')
plt.ylabel('FSC')
plt.xlabel('Spatial frequency (1/A)')
plt.legend(['unmasked', 'masked'])
plt.minorticks_on()
ax = plt.gca()
ax.set_yticks([options.fsc_threshold], minor=True)
ax.set_yticklabels([str(options.fsc_threshold)], minor=True)
if options.refine_res_lim != None:
ax.axvline(1.0 / options.refine_res_lim,
linestyle='dashed',
linewidth=0.75,
color='m')
plt.grid(b=True, which='both')
plt.savefig(options.out + '_fsc.png', dpi=options.dpi)
plt.close()
elif options.mw == None and options.randomize_below_fsc != None:
# Plot
plt.figure()
plt.plot(freq[1:], fsc[1:NSAM], freq[1:], fsc_mask_true[1:NSAM])
plt.title('Fourier Shell Correlation')
plt.ylabel('FSC')
plt.xlabel('Spatial frequency (1/A)')
plt.legend(['unmasked', 'masked - true'])
plt.minorticks_on()
ax = plt.gca()
ax.set_yticks([options.fsc_threshold], minor=True)
ax.set_yticklabels([str(options.fsc_threshold)], minor=True)
if options.refine_res_lim != None:
ax.axvline(1.0 / options.refine_res_lim,
linestyle='dashed',
linewidth=0.75,
color='m')
plt.grid(b=True, which='both')
plt.savefig(options.out + '_fsc.png', dpi=options.dpi)
plt.close()
if options.mw != None:
DALT = 0.81 # Da/A^3
# Estimate fraction of volume occupied by the molecule:
fpart = 1000.0 * options.mw / DALT / (options.angpix * 2 * NSAM)**3
fignore = 1000.0 * options.mw_ignore / DALT / (options.angpix * 2 *
NSAM)**3
# Fraction of the volume occupied by the mask:
maskvoxsum = np.sum(mask)
fmask = maskvoxsum / (2 * NSAM)**3
print '\nCalculating Single-Particle Wiener filter...'
print '\nFraction of particle within the volume (Fpart): %.6f' % fpart
print 'Fraction of mask within the volume (Fmask): %.6f' % fmask
if options.mw_ignore > 0.0:
print 'Fraction of densities to be ignored within the volume (Fignore): %.6f' % fignore
print 'Fpart/(Fmask-Fignore) ratio: %.6f' % (fpart /
(fmask - fignore))
if (fpart / (fmask - fignore)) >= 1.0:
print '\nWARNING: Your particle occupies a volume bigger than the mask. Mask is probably too tight or even too small!'
else:
print 'Fpart/Fmask ratio: %.6f' % (fpart / fmask)
if (fpart / fmask) >= 1.0:
print '\nWARNING: Your particle occupies a volume bigger than the mask. Mask is probably too tight or even too small!'
# Let's do Single-Particle Wiener filtering following (Sindelar & Grigorieff, 2012):
if options.randomize_below_fsc == None:
fsc_spw = fsc_mask / (fsc_mask + (fpart / (fmask - fignore)) *
(1.0 - fsc_mask))
else:
fsc_spw = fsc_mask_true / (fsc_mask_true +
(fpart / (fmask - fignore)) *
(1.0 - fsc_mask_true))
res_spw = ResolutionAtThreshold(freq[1:], fsc_spw[1:NSAM],
options.fsc_threshold)
print '\nFSC >= %.3f up to %.3f A (volume-normalized)' % (
options.fsc_threshold, res_spw)
dat = np.append(dat, fsc_spw[:NSAM], axis=1) # Append the FSC-SPW
head += 'FSC-SPW \t'
# Plot
plt.figure()
plt.plot(freq[1:], fsc_spw[1:NSAM])
plt.title(
'Fourier Shell Correlation - Single-Particle Wiener filter')
plt.ylabel('FSC')
plt.xlabel('Spatial frequency (1/A)')
plt.minorticks_on()
ax = plt.gca()
ax.set_yticks([options.fsc_threshold], minor=True)
ax.set_yticklabels([str(options.fsc_threshold)], minor=True)
if options.refine_res_lim != None:
ax.axvline(1.0 / options.refine_res_lim,
linestyle='dashed',
linewidth=0.75,
color='m')
plt.grid(b=True, which='both')
plt.savefig(options.out + '_fsc-spw.png', dpi=options.dpi)
plt.close()
if options.randomize_below_fsc != None:
# Plot
plt.figure()
plt.plot(freq[1:], fsc[1:NSAM], freq[1:], fsc_mask[1:NSAM],
freq[1:], fsc_spw[1:NSAM], freq[1:],
fsc_mask_true[1:NSAM])
plt.title('Fourier Shell Correlation')
plt.ylabel('FSC')
plt.xlabel('Spatial frequency (1/A)')
plt.legend(
['unmasked', 'masked', 'masked - SPW', 'masked - true'])
plt.minorticks_on()
ax = plt.gca()
ax.set_yticks([options.fsc_threshold], minor=True)
ax.set_yticklabels([str(options.fsc_threshold)], minor=True)
if options.refine_res_lim != None:
ax.axvline(1.0 / options.refine_res_lim,
linestyle='dashed',
linewidth=0.75,
color='m')
plt.grid(b=True, which='both')
plt.savefig(options.out + '_fsc.png', dpi=options.dpi)
plt.close()
else:
# Plot
plt.figure()
plt.plot(freq[1:], fsc[1:NSAM], freq[1:], fsc_mask[1:NSAM],
freq[1:], fsc_spw[1:NSAM])
plt.title('Fourier Shell Correlation')
plt.ylabel('FSC')
plt.xlabel('Spatial frequency (1/A)')
plt.legend(['unmasked', 'masked', 'masked - SPW'])
plt.minorticks_on()
ax = plt.gca()
ax.set_yticks([options.fsc_threshold], minor=True)
ax.set_yticklabels([str(options.fsc_threshold)], minor=True)
if options.refine_res_lim != None:
ax.axvline(1.0 / options.refine_res_lim,
linestyle='dashed',
linewidth=0.75,
color='m')
plt.grid(b=True, which='both')
plt.savefig(options.out + '_fsc.png', dpi=options.dpi)
plt.close()
#### MAP FILTERING STEPS:
# 1. Sum the two half-reconstructions:
print '\nAveraging the two half-maps...'
fullmap = 0.5 * (map1 + map2)
# 2. Apply FSC weighting or SPW filter to the final map, accordingly:
if options.skip_fsc_weighting == False:
print 'Applying FSC weighting (Cref) to the map...'
if options.mask == None and options.mw == None:
# Derive weights from unmasked FSC
fsc_weights = np.sqrt(2 * np.abs(fsc) / (1 + np.abs(fsc)))
elif options.mw == None:
# Derive weights from masked FSC
if options.randomize_below_fsc != None:
fsc_weights = np.sqrt(2 * np.abs(fsc_mask_true) /
(1 + np.abs(fsc_mask_true)))
else:
fsc_weights = np.sqrt(2 * np.abs(fsc_mask) /
(1 + np.abs(fsc_mask)))
else:
fsc_weights = np.sqrt(2 * np.abs(fsc_spw) / (1 + np.abs(fsc_spw)))
fullmap = util.RadialFilter(fullmap, fsc_weights, return_filter=False)
dat = np.append(dat, fsc_weights[:NSAM],
axis=1) # Append the FSC weighting
head += 'Cref_Weights\t'
# 3. Sharpen map by recovering amplitudes from detector's MTF:
if options.mtf != None:
print 'Dividing map by the detector MTF...'
try:
mtf = np.loadtxt(options.mtf)
ignore_mtf = False
except ValueError:
if options.mtf[-5:] == '.star':
mtf = np.loadtxt(options.mtf, skiprows=4)
ignore_mtf = False
else:
print 'Could not read MTF file! Ignoring MTF...'
ignore_mtf = True
if ignore_mtf == False:
# We need to know the MTF values at the Fourier bins of our map. So we interpolate from the MTF description available:
NSAMfull = np.ceil(
np.sqrt(np.sum(np.power(map1.shape, 2))) / 2.0 + 1
).astype(
'int'
) # For cubic volumes this is just half the box size multiplied by sqrt(2).
freqfull = (np.arange(NSAMfull) /
(2.0 * NSAM * options.angpix)).reshape(NSAMfull, 1)
freqfull[0] = 1.0 / 999 # Just to avoid dividing by zero later
interp_mtf = np.interp(freqfull, mtf[:, 0], mtf[:, 1])
# print len(interp_mtf),len(freq[1:]full)
# Divide Fourier components by the detector MTF:
inv_mtf = 1.0 / interp_mtf
fullmap = util.RadialFilter(fullmap, inv_mtf, return_filter=False)
dat = np.append(dat, inv_mtf[:NSAM],
axis=1) # Append the inverse MTF applied
head += 'InverseMTF\t'
# 4. Perform automatic sharpening based on the Guinier plot:
##### GUINIER PLOT #####
if options.auto_bfac != None:
# Here we use the same method as relion_postprocess. Note there is a difference in the normalization of the FFT, but that doesn't affect the results (only the intercept of fit).
# NOTE: the bfactor.exe and EM-BFACTOR programs use a different fitting method.
radamp = util.RadialProfile(
np.abs(np.fft.fftshift(np.fft.fftn(fullmap))))[:NSAM]
lnF = np.log(radamp)
if minres == -1.0:
minres = 10.0
if maxres == -1.0:
if options.mw != None:
maxres = res_spw
elif options.mask != None:
maxres = res_mask
else:
maxres = res
print '\nEstimating contrast decay (B-factor) from Guinier plot between %.2f A and %.2f A...\n' % (
minres, maxres)
hirange = 1. / freq <= minres
lorange = 1. / freq >= maxres
resrange = hirange * lorange
resrange = resrange[:, 0]
fit = np.polyfit(freq2[resrange, 0], lnF[resrange], deg=1)
fitline = fit[0] * freq2 + fit[1]
print 'Slope of fit: %.4f' % (fit[0])
print 'Intercept of fit: %.4f' % (fit[1])
print 'Correlation of fit: %.5f' % (np.corrcoef(
lnF[resrange], fitline[resrange, 0])[0, 1])
print 'B-factor for contrast restoration: %.4f A^2\n' % (4.0 * fit[0])
fullmap = util.FilterBfactor(fullmap,
apix=options.angpix,
B=4.0 * fit[0],
return_filter=False)
guinierfilt = np.exp(
-fit[0] * freq2) # Just for appending to the output data file
dat = np.append(
dat, guinierfilt[:NSAM],
axis=1) # Append the B-factor filter derived from the Guinier plot
head += 'Auto_B-factor\t'
radampnew = util.RadialProfile(
np.abs(np.fft.fftshift(np.fft.fftn(fullmap))))[:NSAM]
lnFnew = np.log(radampnew)
# Plot
plt.figure()
plt.plot(freq2, lnF, freq2[lorange[:, 0], 0], lnFnew[lorange[:, 0]],
freq2[resrange, 0], fitline[resrange])
plt.title('Guinier Plot')
plt.ylabel('ln(F)')
plt.xlabel('Spatial frequency^2 (1/A^2)')
plt.legend(['Exp.', 'Exp. sharpened', 'Fit'])
ax = plt.gca()
ax.axvline(1.0 / minres**2,
linestyle='dashed',
linewidth=0.75,
color='m')
ax.axvline(1.0 / maxres**2,
linestyle='dashed',
linewidth=0.75,
color='m')
plt.grid(b=True, which='both')
plt.savefig(options.out + '_guinier.png', dpi=options.dpi)
plt.close()
if options.cosine == False:
print '\nWARNING: You should probably specify --cosine option to low-pass filter your map after sharpening!\n'
# 5. Apply an ad-hoc B-factor for smoothing or sharpening the map, if provided:
if options.adhoc_bfac != 0.0:
print 'Applying ad-hoc B-factor to the map...'
fullmap = util.FilterBfactor(fullmap,
apix=options.angpix,
B=options.adhoc_bfac,
return_filter=False)
freq2 = freq * freq
bfacfilt = np.exp(-options.adhoc_bfac * freq2 /
4.0) # Just for appending to the output data file
dat = np.append(dat, bfacfilt[:NSAM],
axis=1) # Append the ad-hoc B-factor filter applied
head += 'Adhoc_B-factor\t'
if options.cosine == False:
print '\nWARNING: You should probably specify --cosine option to low-pass filter your map after sharpening!\n'
# 6. Apply FSC^2 weighting to the final map:
if options.apply_fsc2:
print 'Applying FSC^2 weighting to the map...'
if options.mask == None and options.mw == None:
# Derive weights from unmasked FSC
fsc2_weights = fsc**2
elif options.mw == None:
# Derive weights from masked FSC
if options.randomize_below_fsc != None:
fsc2_weights = fsc_mask_true**2
else:
fsc2_weights = fsc_mask**2
else:
fsc2_weights = fsc_spw**2
fullmap = util.RadialFilter(fullmap, fsc2_weights, return_filter=False)
dat = np.append(dat, fsc2_weights[:NSAM],
axis=1) # Append the FSC weighting
head += 'FSC^2_Weights\t'
# 7. Impose a Gaussian or Cosine or Top-hat low-pass filter with cutoff at given resolution, or resolution determined from FSC threshold:
if options.lowpass == 'auto':
print 'Low-pass filtering the map at resolution cutoff...'
if options.mw != None:
res_cutoff = res_spw
elif options.mask != None:
res_cutoff = res_mask
else:
res_cutoff = res
if options.tophat == False and options.cosine == False:
fullmap = util.FilterGauss(fullmap,
apix=options.angpix,
lp=res_cutoff,
return_filter=False)
lp = np.exp(-res_cutoff**2 * freq2[:, 0] / 2)
else:
fullmap = util.FilterCosine(fullmap,
apix=options.angpix,
lp=res_cutoff,
return_filter=False,
width=options.cosine_edge_width)
cosrad = np.argmin(np.abs(1. / freq - res_cutoff))
rii = cosrad + options.cosine_edge_width / 2
rih = cosrad - options.cosine_edge_width / 2
lp = np.zeros(freq[:, 0].shape)
r = np.arange(len(freq))
fill_idx = r <= rih
lp[fill_idx] = 1.0
rih_idx = r > rih
rii_idx = r <= rii
edge_idx = rih_idx * rii_idx
lp[edge_idx] = (1.0 + np.cos(
np.pi * (r[edge_idx] - rih) / options.cosine_edge_width)) / 2.0
dat = np.append(dat, lp.reshape(NSAM, 1),
axis=1) # Append the low-pass filter applied
head += 'Low-pass \t'
elif options.lowpass >= 0.0:
print 'Low-pass filtering the map at resolution cutoff...'
res_cutoff = options.lowpass
if options.tophat == False and options.cosine == False:
fullmap = util.FilterGauss(fullmap,
apix=options.angpix,
lp=res_cutoff,
return_filter=False)
lp = np.exp(-res_cutoff**2 * freq2[:, 0] / 2)
else:
fullmap = util.FilterCosine(fullmap,
apix=options.angpix,
lp=res_cutoff,
return_filter=False,
width=options.cosine_edge_width)
cosrad = np.where(freq <= 1. / res_cutoff)[0][0]
rii = cosrad + options.cosine_edge_width / 2
rih = cosrad - options.cosine_edge_width / 2
lp = np.zeros(freq[:, 0].shape)
r = np.arange(len(freq))
fill_idx = r <= rih
lp[fill_idx] = 1.0
rih_idx = r > rih
rii_idx = r <= rii
edge_idx = rih_idx * rii_idx
lp[edge_idx] = (1.0 + np.cos(
np.pi * (r[edge_idx] - rih) / options.cosine_edge_width)) / 2.0
dat = np.append(dat, lp.reshape(NSAM, 1),
axis=1) # Append the low-pass filter applied
head += 'Low-pass \t'
# 8. Apply mask, if provided:
if options.mask != None or options.mw != None:
print 'Masking the map...'
masked = fullmap * mask
if options.resample == None:
sys.stdout = open(os.devnull, "w") # Suppress output
ioMRC.writeMRC(masked,
options.out + '-masked.mrc',
dtype='float32',
pixelsize=options.angpix,
quickStats=True)
sys.stdout = sys.__stdout__
else:
masked = util.Resample(masked,
apix=options.angpix,
newapix=options.resample)
sys.stdout = open(os.devnull, "w") # Suppress output
ioMRC.writeMRC(masked,
options.out + '-masked.mrc',
dtype='float32',
pixelsize=options.resample,
quickStats=True)
sys.stdout = sys.__stdout__
mask = util.Resample(mask,
apix=options.angpix,
newapix=options.resample)
sys.stdout = open(os.devnull, "w") # Suppress output
ioMRC.writeMRC(mask,
options.out + '-mask.mrc',
dtype='float32',
pixelsize=options.resample,
quickStats=True)
sys.stdout = sys.__stdout__
# Write filtered, unmasked map
if options.resample == None:
sys.stdout = open(os.devnull, "w") # Suppress output
ioMRC.writeMRC(fullmap,
options.out + '-unmasked.mrc',
dtype='float32',
pixelsize=options.angpix,
quickStats=True)
sys.stdout = sys.__stdout__
else:
fullmap = util.Resample(fullmap,
apix=options.angpix,
newapix=options.resample)
sys.stdout = open(os.devnull, "w") # Suppress output
ioMRC.writeMRC(fullmap,
options.out + '-unmasked.mrc',
dtype='float32',
pixelsize=options.resample,
quickStats=True)
sys.stdout = sys.__stdout__
# Save output file with all relevant FSC data
np.savetxt(options.out + '_data.fsc',
np.matrix(dat),
header=command + '\n' + head,
delimiter='\t',
fmt='%.6f')
print '\nDone!'
def main():
# Get arguments:
folders = sys.argv[1]
merge_dirfile = sys.argv[2]
png_path = sys.argv[3]
stack_path = sys.argv[4]
stack_rootname = sys.argv[5]
box_size = int(sys.argv[6])
phaori_shift = np.array([float(sys.argv[7].split(',')[0]), float(sys.argv[7].split(',')[1])]) # How many degrees to offset the phase origins in order to get a protein in the center of the particle for a zero-tilt image
apix = float(sys.argv[8]) # pixel size in Angstroems
microscope_voltage = float(sys.argv[9]) # microscope voltage in kV
microscope_cs = float(sys.argv[10]) # microscope spherical aberration in mm
ampcontrast = 1.0-float(sys.argv[11])**2 # amplitude contrast of CTF (obtained here from phase contrast)
magnification = float(sys.argv[12])
sigcc = float(sys.argv[13])
if sys.argv[14] == 'y':
invert_micrograph = True
else:
invert_micrograph = False
if sys.argv[15] == 'y':
normalize_box = True
else:
normalize_box = False
if sys.argv[16] == 'y':
calculate_defocus_tilted = True
else:
calculate_defocus_tilted = False
if sys.argv[17] == 'y':
save_phase_flipped = True
else:
save_phase_flipped = False
if sys.argv[18] == 'y':
save_ctf_multiplied = True
else:
save_ctf_multiplied = False
if sys.argv[19] == 'y':
save_wiener_filtered = True
else:
save_wiener_filtered = False
wiener_constant = float(sys.argv[20])
sigma = float(sys.argv[21]) # Sigma for normalization of the windowed images (if normalize_box == True)
sigma_rad = float(sys.argv[22]) # Radius for normalization of the windowed images (if normalize_box == True), for estimating AVG and STD
if sys.argv[23] == 'Defocus/Lattice':
tiltgeom = ''
elif sys.argv[23] == 'Defocus':
tiltgeom = 'DEFOCUS_'
elif sys.argv[23] == 'Lattice':
tiltgeom = 'LATTICE_'
elif sys.argv[23] == 'Merge':
tiltgeom = 'MERGE_'
if sys.argv[24] == 'Micrograph':
ctfcor = True
# stack_rootname = stack_rootname + '_ctfcor'
else:
ctfcor = False
if sys.argv[25] == 'y':
save_pick_fig = True
else:
save_pick_fig = False
if sys.argv[26] == 'y':
use_masked_image = True
else:
use_masked_image = False
if sys.argv[27] == 'y':
do_resample = True
else:
do_resample = False
n_threads = int(sys.argv[28])
if n_threads < 1:
n_threads = 1
this_thread = int(sys.argv[29])
# End arguments
f = open(merge_dirfile,'r')
img_dirs = f.readlines()
f.close()
# Constant parameters to be written on the .par file of this dataset:
shx = 0.0
shy = 0.0
occ = 100.0
logp = 0
sig = 0.5 # This has nothing to do with the normalization SIGMA!
score = 0.0
chg = 0.0
idx = 0
# box_fail = 0
phaori_err = 0
prog = 0.0
N = len(img_dirs)
# batch_size = round(float(N)/n_threads)
batch_size = int( round( float( N ) / n_threads ) )
first_img = ( this_thread - 1 ) * batch_size
if this_thread < n_threads:
last_img = first_img + batch_size
else:
last_img = N
img_dirs = img_dirs[first_img:last_img]
n = first_img + 1
print( '\nJob %d/%d picking particles from micrographs %d to %d...\n' % (this_thread, n_threads, n, last_img) )
# Open the .par file to store all particle parameters:
f = open(stack_path+stack_rootname+'_1_r1-%.4d.par' % this_thread, 'w+')
# Open the master coordinates file:
mcf = open(stack_path+stack_rootname+'_coordinates_master-%.4d.txt' % this_thread, 'w+')
for d in img_dirs:
d = d.strip()
imname = d.split('/')[-1]
try:
# Read in all relevant image parameters:
params = Read2dxCfgFile(folders+d+'/2dx_image.cfg')
except:
print( '\nProblem with image %s!\n' % d )
continue
# if ctfcor:
# try:
# mrc = glob.glob(folders+d+'/image_ctfcor.mrc')[0]
# img = ioMRC.readMRC(mrc)[0] # Read image
# bf = open(folders+d+'/image_ctfcor.box', 'w+')
# except:
# print '\nCTF-corrected micrograph not found for image %s!\n' % d
# continue
# else:
if use_masked_image:
# First we look for the masked, zero-padded, normalized micrograph:
try:
# # There might be some funny numbers appended to the file name so we have to look for the shortest one to get the right file:
# mrclist = glob.glob(folders+d+'/m'+imname+'*.mrc')
# lenlist = []
# for m in mrclist:
# lenlist.append(len(m))
# shortest_idx = np.argsort(lenlist)[0]
# # print mrclist[shortest_idx]
# mrc = mrclist[shortest_idx]
# bf = open(os.path.splitext(mrc)[0]+'.box', 'w+')
# # mrc = sorted(glob.glob(folders+d+'/m'+imname+'*.mrc'))[0]
# # bf = open(folders+d+'/m'+imname+'.box', 'w+')
mrc = folders + d + '/' + params['imagename'] + '.mrc'
sys.stdout = open(os.devnull, "w") # Suppress output
img = ioMRC.readMRC(mrc)[0][0] # Read image
sys.stdout = sys.__stdout__
bf = open(folders + d + '/' + params['imagename'] + '.box', 'w+')
except:
# If it doesn't exist, we try the unmasked, zero-padded, normalized micrograph:
try:
# mrclist = glob.glob(folders+d+'/'+imname+'*.mrc')
# lenlist = []
# for m in mrclist:
# lenlist.append(len(m))
# shortest_idx = np.argsort(lenlist)[0]
# # print mrclist[shortest_idx]
# mrc = mrclist[shortest_idx]
# bf = open(os.path.splitext(mrc)[0]+'.box', 'w+')
# # mrc = sorted(glob.glob(folders+d+'/m'+imname+'*.mrc'))[0]
# # bf = open(folders+d+'/m'+imname+'.box', 'w+')
mrc = folders + d + '/' + params['nonmaskimagename'] + '.mrc'
sys.stdout = open(os.devnull, "w") # Suppress output
img = ioMRC.readMRC(mrc)[0][0] # Read image
sys.stdout = sys.__stdout__
bf = open(folders + d + '/' + params['nonmaskimagename'] + '.box', 'w+')
except:
# If neither exist we skip this image
print( '::\nProblem with image %s!\n' % d )
continue
else:
# If the user requires, we go directly to the unmasked, zero-padded, normalized micrograph:
try:
# mrclist = glob.glob(folders+d+'/'+imname+'*.mrc')
# lenlist = []
# for m in mrclist:
# lenlist.append(len(m))
# shortest_idx = np.argsort(lenlist)[0]
# # print mrclist[shortest_idx]
# mrc = mrclist[shortest_idx]
# bf = open(os.path.splitext(mrc)[0]+'.box', 'w+')
# # mrc = sorted(glob.glob(folders+d+'/m'+imname+'*.mrc'))[0]
# # bf = open(folders+d+'/m'+imname+'.box', 'w+')
mrc = folders + d + '/' + params['nonmaskimagename'] + '.mrc'
sys.stdout = open(os.devnull, "w") # Suppress output
img = ioMRC.readMRC(mrc)[0][0] # Read image
sys.stdout = sys.__stdout__
bf = open(folders + d + '/' + params['nonmaskimagename'] + '.box', 'w+')
except:
# If neither exist we skip this image
print( '::\nProblem with image %s!' % d )
print( '::' )
continue
# Here we check whether the pixel size defined in the image cfg file agrees with the desired one:
# print params.keys()
apixold = apix
if ( params['sample_pixel'] < 0.99 * apix ) or ( params['sample_pixel'] > 1.01 * apix ): # Should not differ by more than 1% !
try:
apixold = params['stepdigitizer'] * 1e4 / params['magnification'] # We give it a second chance by calculating from stepdigitizer and magnification
# print params['stepdigitizer'], params['magnification']
except KeyError:
params_master = Read2dxCfgFile(folders+d+'/../2dx_master.cfg') # Try to fetch stepdigitizer information from the group's 2dx_master.cfg file
apixold = params_master['stepdigitizer'] * 1e4 / params['magnification'] # We give it a second chance by calculating from stepdigitizer and magnification
if ( apixold < 0.99 * apix ) or ( apixold > 1.01 * apix ): # Should not differ by more than 1% !
if do_resample:
# We resample the micrograph in Fourier space to bring it to the desired pixel size:
print( apixold, apix )
img = util.Resample( img, apix=apixold, newapix=apix )
else:
print( '::\nSkipping image %s: pixel size of this image seems to be different from the one defined (%f A).' % (d, apix) )
print( '::\nPlease check it if you would like to include this image.' )
print( '::' )
continue
# TO DO: if magnification differs (by failing the tests above), should we resample the micrograph to desired mag?
print( '::\nNow boxing unit cells of micrograph %d/%d.\n' % (n, N) )
print( mrc )
try:
if invert_micrograph:
img = -1.0 * img
# img = spx.EMNumPy.em2numpy(img)
profile = glob.glob(folders+d+'/*profile.dat')[0]
dat = ReadProfileDat(profile)
ccmean = np.mean(dat[:,4])
ccstd = np.std(dat[:,4])
cc_thr = ccmean + sigcc * ccstd
print( ':\nImage average value:%.2f' % img.mean() )
print( ':Image standard deviation:%.2f' % img.std() )
print( ':' )
print( ':CC scores average value:%.2f' % ccmean )
print( ':CC scores standard deviation:%.2f' % ccstd )
print( ':Only particles with CC score above %.2f will be picked.\n' % cc_thr )
# # Get several values related to defocus, astigmatism and tilting:
# params = Read2dxCfgFile(folders+d+'/2dx_image.cfg')
# w = img.get_xsize()
w = img.shape[0]
# Get the unit-cell vectors:
if sum(params['PHAORI']) == 0:
PhaOriX = 0
PhaOriY = 0
a = [0,0]
b = [0,0]
else:
a,b = LatticeReciprocal2Real(params['u'], params['v'], w * apix / apixold ) # These parameters are w.r.t to the original image size
# Convert from Numpy-array to list:
a = [a[0], a[1]]
b = [b[0], b[1]]
x,y = CalculatePickingPositions(dat, a, b, w * apix / apixold, params['PHAORI'], phaori_shift, params[tiltgeom+'TLTANG']) # These parameters are w.r.t to the original image size
x *= apixold / apix
y *= apixold / apix
# Plot the picking profile:
if save_pick_fig:
meanimg = img.mean()
stdimg = img.std()
climimg = [meanimg - 2 * stdimg, meanimg + 2 * stdimg]
fig1 = plt.figure()
plt.imshow(img, cmap=cm.gray, vmin=climimg[0], vmax=climimg[1])
Axes1 = fig1.gca()
# The following values are constant within each crystal:
phi = 90.0 - params[tiltgeom+'TAXA']
theta = params[tiltgeom+'TLTANG']
psi = 270.0 - params[tiltgeom+'TLTAXIS']
ang = params['AST_ANGLE']
max_good = np.sum( dat[:,4] < cc_thr ) # The maximum number of particles we may extract from this micrograph
boxes = np.zeros( (max_good, box_size, box_size), dtype='float32' )
if save_phase_flipped:
boxespf = np.zeros( (max_good, box_size, box_size), dtype='float32' )
if save_ctf_multiplied:
boxescm = np.zeros( (max_good, box_size, box_size), dtype='float32' )
if save_wiener_filtered:
boxeswf = np.zeros( (max_good, box_size, box_size), dtype='float32' )
idx_start = idx # Absolute index in the beginning of this crystal
m = 0
ptcl_idx = np.arange( dat.shape[0] )
# print 'ptcl_idx[-1] is ',ptcl_idx[-1]
# if shuffle_order:
# np.random.seed( seed=n ) # Fix random seed to get reproducible results
# np.random.shuffle( ptcl_idx )
for i in ptcl_idx:
# for i in np.arange(dat.shape[0]):
# print i,m
try:
# Adjust the picking coordinates for the .box file and picking plots:
xbox_plot = x[i] + w/2
ybox_plot = y[i] + w/2
xbox = xbox_plot - box_size/2
ybox = ybox_plot - box_size/2
# if dat[i,4] < cc_thr or np.isnan(x[i]) or np.isnan(y[i]):
if dat[i,4] < cc_thr:
if save_pick_fig:
# Write red patch on image to be saved as .png describing the picking positions:
Axes1.add_patch(patches.Circle((xbox_plot, ybox_plot), edgecolor='magenta', facecolor='none', linewidth=0.8, radius=box_size/8))
# Axes1.add_patch(patches.Rectangle(xy=(xbox, ybox), width=box_size, height=box_size, edgecolor='red', facecolor='none', linewidth=0.2))
else:
# Extract the particle from the micrograph at specified position:
# NOTE THAT X,Y CONVENTIONS FOR ioMRC/NUMPY ARE INVERTED IN RELATION TO SPARX/EMAN!!!
# box = spx.Util.window(img,int(box_size),int(box_size),1,int(round(x[i])),int(round(y[i])))
xi = int(round(x[i]))
yi = int(round(y[i]))
# print xi-w/2-box_size/2, xi-w/2+box_size/2
box_ext = img[yi-w//2-box_size//2:yi-w//2+box_size//2, xi-w//2-box_size//2:xi-w//2+box_size//2]
# Normalize box to zero mean and constant pre-defined sigma:
if normalize_box:
# box = NormalizeStack([box], sigma)[0]
# try:
box = util.NormalizeImg( box_ext, std=sigma, radius=sigma_rad )
else:
box = box_ext
# except Warning:
# raise ZeroDivisionError( "Standard deviation of image is zero!" )
# Sometimes the box contains weird values that may be tricky to detect. Testing the mean of the pixels for NaN and Inf seems to work:
# tmpmean = box.mean()
# if np.isnan( tmpmean ) or np.isinf( tmpmean ):
# print tmpmean
tmpstd = box.std()
if np.isnan( tmpstd ) or np.isinf( tmpstd ):
# print tmpstd
print( "The box of CC peak (%d,%d) at position (%d,%d) in micrograph %d/%d contains strange values (NaN or Inf)! Will be discarded." % (dat[i,0], dat[i,1], int(round(x[i])), int(round(y[i])), n, N) )
raise ValueError
boxes[m,:,:] = box
# if m == 0:
# boxes[0,:,:] = box
# else:
# # print box.shape
# boxes = np.append( boxes, box.reshape( ( 1, box_size, box_size) ), axis=0 )
if calculate_defocus_tilted or save_phase_flipped or save_ctf_multiplied or save_wiener_filtered:
RLDEF1,RLDEF2 = CalculateDefocusTilted(x[i], y[i], apix, params[tiltgeom+'TLTAXIS'], params[tiltgeom+'TLTANG'], params['DEFOCUS1'], params['DEFOCUS2'])
else:
RLDEF1 = params['DEFOCUS1']
RLDEF2 = params['DEFOCUS2']
if RLDEF1 <= 0.0 or RLDEF2 <= 0.0:
print( "The box of CC peak (%d,%d) at position (%d,%d) in micrograph %d/%d has a negative defocus value! Tilt geometry for this crystal is probably wrong. Particle will be discarded." % (dat[i,0], dat[i,1], int(round(x[i])), int(round(y[i])), n, N) )
raise ValueError
# Write .par file with the parameters for each particle in the dataset:
print( ' %d' % (idx+1),' %.2f' % psi,' %.2f' % theta,' %.2f' % phi,' %.2f' % shx,' %.2f' % shy,' %d' % magnification,' %d' % n,' %.2f' % RLDEF1,' %.2f' % RLDEF2,' %.2f' % ang,' %.2f' % occ,' %d' % logp,' %.4f' % sig,' %.2f' % score,' %.2f' % chg, file=f )
# Write the picking information to the .box file:
print( '%d' % xbox, '\t%d' % ybox, '\t%d' % box_size, '\t%d' % box_size, file=bf )
# Write the picking and defocus information to the master coordinates file:
print( '%d' % (idx+1),'\t%d' % n,'\t%s' % folders+d+'/'+imname,'\t%d' % xbox, '\t%d' % ybox, '\t%d' % box_size, '\t%d' % box_size,'\t%.2f' % RLDEF1,'\t%.2f' % RLDEF2, file=mcf )
# Write image to the particle stack:
# if idx == 0:
# # If this is the first image, we initiate as a normal .mrcs stack.
# box.write_image(stack_path+stack_rootname+'-%.4d.mrcs' % this_thread, idx)
# else:
# # Subsequent images are directly appended to the file:
# with open(stack_path+stack_rootname+'-%.4d.mrcs' % this_thread, 'ab') as mrcf:
# spx.EMNumPy.em2numpy(box).tofile(mrcf)
# if (save_phase_flipped or save_wiener_filtered) and not ctfcor:
# # Convert CTF parameters to SPARX convention:
# defocus = (RLDEF1+RLDEF2)/2
# ast = RLDEF1-RLDEF2
# if params['AST_ANGLE'] < 0.0:
# astang = 360.0 + params['AST_ANGLE']
# else:
# astang = params['AST_ANGLE']
# # Generate SPARX CTF object:
# p = [defocus * 1e-4, microscope_cs, microscope_voltage, apix, 0, ampcontrast * 100, ast * 1e-4, astang]
# spx.set_ctf(box, p)
# ctf = spx.generate_ctf(p)
# Phase-flip the image:
if save_phase_flipped:
# Apply CTF correction on whole micrograph to reduce delocalization effects:
# imgctfcor = spx.filt_ctf( img, ctf, binary=1 )
# boxctfcor = spx.Util.window( imgctfcor, int( box_size ), int( box_size ), 1, int( round( x[i] ) ), int( round( y[i] ) ) )
if ctfcor:
imgctfcor = CTF.CorrectCTF( img, DF1=RLDEF1, DF2=RLDEF2, AST=params['AST_ANGLE'], WGH=ampcontrast, apix=apix, Cs=microscope_cs, kV=microscope_voltage, phase_flip=True, return_ctf=False, invert_contrast=False )[0]
boxctfcor = imgctfcor[yi-w//2-box_size//2:yi-w//2+box_size//2, xi-w//2-box_size//2:xi-w//2+box_size//2]
else:
boxctfcor = CTF.CorrectCTF( box_ext, DF1=RLDEF1, DF2=RLDEF2, AST=params['AST_ANGLE'], WGH=ampcontrast, apix=apix, Cs=microscope_cs, kV=microscope_voltage, phase_flip=True, return_ctf=False, invert_contrast=False )[0]
if normalize_box:
# try:
boxctfcor = util.NormalizeImg( boxctfcor, std=sigma, radius=sigma_rad )
# except Warning:
# raise ZeroDivisionError( "Standard deviation of image is zero!" )
boxespf[m,:,:] = boxctfcor
# if m == 0:
# boxespf[0,:,:] = boxctfcor
# else:
# boxespf = np.append( boxespf, boxctfcor.reshape( ( 1, box_size, box_size) ), axis=0 )
# Write image to the particle stack:
# if idx == 0:
# # If this is the first image, we initiate as a normal .mrcs stack.
# boxctfcor.write_image( stack_path+stack_rootname+'_phase-flipped-%.4d.mrcs' % this_thread, idx )
# else:
# # Subsequent images are directly appended to the file:
# with open( stack_path+stack_rootname+'_phase-flipped-%.4d.mrcs' % this_thread, 'ab' ) as mrcf:
# spx.EMNumPy.em2numpy(boxctfcor).tofile( mrcf )
# CTF-multiply the image:
if save_ctf_multiplied:
# # Apply CTF correction on whole micrograph to reduce delocalization effects:
# imgctfcor = spx.filt_ctf( img, ctf, binary=0 )
# boxctfcor = spx.Util.window( imgctfcor, int( box_size ), int( box_size ), 1, int( round( x[i] ) ), int( round( y[i] ) ) )
if ctfcor:
imgctfcor = CTF.CorrectCTF( img, DF1=RLDEF1, DF2=RLDEF2, AST=params['AST_ANGLE'], WGH=ampcontrast, apix=apix, Cs=microscope_cs, kV=microscope_voltage, ctf_multiply=True, return_ctf=False, invert_contrast=False )[0]
boxctfcor = imgctfcor[yi-w//2-box_size//2:yi-w//2+box_size//2, xi-w//2-box_size//2:xi-w//2+box_size//2]
else:
boxctfcor = CTF.CorrectCTF( box_ext, DF1=RLDEF1, DF2=RLDEF2, AST=params['AST_ANGLE'], WGH=ampcontrast, apix=apix, Cs=microscope_cs, kV=microscope_voltage, ctf_multiply=True, return_ctf=False, invert_contrast=False )[0]
if normalize_box:
# try:
boxctfcor = util.NormalizeImg( boxctfcor, std=sigma, radius=sigma_rad )
# except Warning:
# raise ZeroDivisionError( "Standard deviation of image is zero!" )
boxescm[m,:,:] = boxctfcor
# if m == 0:
# boxescm[0,:,:] = boxctfcor
# else:
# boxescm = np.append( boxescm, boxctfcor.reshape( ( 1, box_size, box_size) ), axis=0 )
# Write image to the particle stack:
# if idx == 0:
# # If this is the first image, we initiate as a normal .mrcs stack.
# boxctfcor.write_image( stack_path+stack_rootname+'_wiener-filtered-%.4d.mrcs' % this_thread, idx )
# else:
# # Subsequent images are directly appended to the file:
# with open( stack_path+stack_rootname+'_wiener-filtered-%.4d.mrcs' % this_thread, 'ab' ) as mrcf:
# spx.EMNumPy.em2numpy(boxctfcor).tofile( mrcf )
# Wiener-filter the image:
if save_wiener_filtered:
# # Apply CTF correction on whole micrograph to reduce delocalization effects:
# imgctfcor = spx.filt_ctf( img, ctf, binary=0 )
# boxctfcor = spx.Util.window( imgctfcor, int( box_size ), int( box_size ), 1, int( round( x[i] ) ), int( round( y[i] ) ) )
if ctfcor:
imgctfcor = CTF.CorrectCTF( img, DF1=RLDEF1, DF2=RLDEF2, AST=params['AST_ANGLE'], WGH=ampcontrast, apix=apix, Cs=microscope_cs, kV=microscope_voltage, wiener_filter=True, return_ctf=False, invert_contrast=False, C=wiener_constant )[0]
boxctfcor = imgctfcor[yi-w//2-box_size//2:yi-w//2+box_size//2, xi-w//2-box_size//2:xi-w//2+box_size//2]
else:
boxctfcor = CTF.CorrectCTF( box_ext, DF1=RLDEF1, DF2=RLDEF2, AST=params['AST_ANGLE'], WGH=ampcontrast, apix=apix, Cs=microscope_cs, kV=microscope_voltage, wiener_filter=True, return_ctf=False, invert_contrast=False, C=wiener_constant )[0]
if normalize_box:
# try:
boxctfcor = util.NormalizeImg( boxctfcor, std=sigma, radius=sigma_rad )
# except Warning:
# raise ZeroDivisionError( "Standard deviation of image is zero!" )
boxeswf[m,:,:] = boxctfcor
# if m == 0:
# boxeswf[0,:,:] = boxctfcor
# else:
# boxeswf = np.append( boxeswf, boxctfcor.reshape( ( 1, box_size, box_size) ), axis=0 )
# Write image to the particle stack:
# if idx == 0:
# # If this is the first image, we initiate as a normal .mrcs stack.
# boxctfcor.write_image( stack_path+stack_rootname+'_wiener-filtered-%.4d.mrcs' % this_thread, idx )
# else:
# # Subsequent images are directly appended to the file:
# with open( stack_path+stack_rootname+'_wiener-filtered-%.4d.mrcs' % this_thread, 'ab' ) as mrcf:
# spx.EMNumPy.em2numpy(boxctfcor).tofile( mrcf )
if save_pick_fig:
# Write green patch on image to be saved as .png describing the picking positions:
Axes1.add_patch(patches.Circle((xbox_plot, ybox_plot), edgecolor='lime', facecolor='none', linewidth=0.8, radius=box_size/8))
# Axes1.add_patch(patches.Rectangle(xy=(xbox, ybox), width=box_size, height=box_size, edgecolor='lime', facecolor='none', linewidth=0.2))
m += 1
idx += 1
except ( RuntimeError, ValueError, ZeroDivisionError, IndexError ) as e:
if save_pick_fig:
# Write red patch on image to be saved as .png describing the picking positions:
Axes1.add_patch(patches.Circle((xbox_plot, ybox_plot), edgecolor='magenta', facecolor='none', linewidth=0.8, radius=box_size/8))
# Axes1.add_patch(patches.Rectangle(xy=(xbox, ybox), width=box_size, height=box_size, edgecolor='red', facecolor='none', linewidth=0.2))
print( 'Failed to box CC peak (%d,%d) at position (%d,%d) in micrograph %d/%d!' % (dat[i,0], dat[i,1], int(round(x[i])), int(round(y[i])), n, N) )
# except ValueError:
# if save_pick_fig:
# # Write red patch on image to be saved as .png describing the picking positions:
# # Axes1.add_patch(patches.Circle((dat[i,2], dat[i,3]), edgecolor='red', facecolor='none', linewidth=0.2, radius=20))
# Axes1.add_patch(patches.Rectangle(xy=(xbox, ybox), width=box_size, height=box_size, edgecolor='red', facecolor='none', linewidth=0.2))
# print 'Failed to box CC peak (%d,%d) at position (%d,%d) in micrograph %d/%d!' % (dat[i,0], dat[i,1], int(round(x[i])), int(round(y[i])), n, N)
# except ZeroDivisionError:
# if save_pick_fig:
# # Write red patch on image to be saved as .png describing the picking positions:
# # Axes1.add_patch(patches.Circle((dat[i,2], dat[i,3]), edgecolor='red', facecolor='none', linewidth=0.2, radius=20))
# Axes1.add_patch(patches.Rectangle(xy=(xbox, ybox), width=box_size, height=box_size, edgecolor='red', facecolor='none', linewidth=0.2))
# print 'Failed to box CC peak (%d,%d) at position (%d,%d) in micrograph %d/%d!' % (dat[i,0], dat[i,1], int(round(x[i])), int(round(y[i])), n, N)
# Particles are written to stacks in crystal batches, thus saving fopen() calls:
sys.stdout = open(os.devnull, "w") # Suppress output
ioMRC.writeMRC( boxes[:m,:,:], stack_path+stack_rootname+'-%.4d.mrcs' % this_thread, dtype='float32', idx=idx_start )
if save_phase_flipped:
ioMRC.writeMRC( boxespf[:m,:,:], stack_path+stack_rootname+'_phase-flipped-%.4d.mrcs' % this_thread, dtype='float32', idx=idx_start )
if save_ctf_multiplied:
ioMRC.writeMRC( boxescm[:m,:,:], stack_path+stack_rootname+'_ctf-multiplied-%.4d.mrcs' % this_thread, dtype='float32', idx=idx_start )
if save_wiener_filtered:
ioMRC.writeMRC( boxeswf[:m,:,:], stack_path+stack_rootname+'_wiener-filtered-%.4d.mrcs' % this_thread, dtype='float32', idx=idx_start )
sys.stdout = sys.__stdout__
print( '\nBoxed %d/%d CC peaks from micrograph %d/%d.\n' % (m, dat.shape[0], n, N) )
# # Update the counts in the MRC headers:
# # First the normal particle stack:
# header = ioMRC.readMRCHeader( stack_path+stack_rootname+'-%.4d.mrcs' % this_thread )
# header['dimensions'][0] = idx
# # Now we write the header back:
# with open( stack_path+stack_rootname+'-%.4d.mrcs' % this_thread, 'rb+' ) as mrcf:
# ioMRC.writeMRCHeader( mrcf, header, endchar = '<' )
# # Then the phase-flipped:
# if save_phase_flipped and not ctfcor:
# header = ioMRC.readMRCHeader( stack_path+stack_rootname+'_phase-flipped-%.4d.mrcs' % this_thread )
# header['dimensions'][0] = idx
# # Now we write the header back:
# with open( stack_path+stack_rootname+'_phase-flipped-%.4d.mrcs' % this_thread, 'rb+' ) as mrcf:
# ioMRC.writeMRCHeader( mrcf, header, endchar = '<' )
# # Then the Wiener-filtered:
# if save_wiener_filtered and not ctfcor:
# header = ioMRC.readMRCHeader( stack_path+stack_rootname+'_wiener-filtered-%.4d.mrcs' % this_thread )
# header['dimensions'][0] = idx
# # Now we write the header back:
# with open( stack_path+stack_rootname+'_wiener-filtered-%.4d.mrcs' % this_thread, 'rb+' ) as mrcf:
# ioMRC.writeMRCHeader( mrcf, header, endchar = '<' )
# print '<<@progress: %d>>' % round(n*100.0/N)
# Report progress to the GUI:
prog += 75.0/N
if prog >= 1.0:
print( '<<@progress: +%d>>' % round(prog) )
prog -= np.floor(prog)
if save_pick_fig:
fig1.savefig(png_path+'mic_%.3d_' % n+imname+'_picking.png', dpi=300)
plt.close(fig1)
n += 1
except RuntimeError:
# print '::PROBLEM WITH MICROGRAPH %d/%d!!! Maybe it was not found?' % (n, N)
# print '::'
# print mrc
print( '\nPROBLEM WITH MICROGRAPH:' )
print( '%s' % mrc )
print( 'Maybe it was not found?' )
except ValueError:
# print '::PROBLEM WITH CC PROFILE FOR IMAGE %d/%d!!!' % (n, N)
# print '::'
# print mrc
print( '\nPROBLEM WITH CC PROFILE FOR IMAGE:' )
print( '%s' % mrc )
bf.close()
# n += 1
# print '::Total boxed unit cells:%d' % idx
# print '::Failed to box %d unit cells.' % box_fail
# print '<<@progress: +%d>>' % round(prog/0.01)
print( '\nJob %d/%d finished picking particles.\n' % (this_thread, n_threads) )
f.close()
mcf.close()
def ApplySymmetry(img, sym='c1', pad=1):
# Padding reduces interpolation artefacts at the expense of RAM and computation time
imgshape_ori = img.shape
if len(imgshape_ori) == 2:
img = img.reshape((1, imgshape_ori[0], imgshape_ori[1]))
imsize = np.array(img.shape)
# print imsize
if pad != 1:
img = util.Resample(img, newsize=imsize * pad)
[Z, Y, X] = np.mgrid[0:img.shape[0], 0:img.shape[1], 0:img.shape[2]]
# Begin symmetry operations:
# Symmetry operations that can be trivially obtained by coordinate manipulations are explicitly defined for speedup:
if sym.lower() == 'c1':
imgsym = img
elif sym.lower() == 'c2':
imgsym = copy.copy(img)
imgsym += img[Z, -Y, -X]
imgsym /= 2.0
elif sym.lower() == 'c4':
imgsym = copy.copy(img)
imgsym += img[Z, -Y, -X]
imgsym += img[Z, -X, Y]
imgsym += img[Z, X, -Y]
imgsym /= 4.0
elif sym.lower() == 'd1':
imgsym = copy.copy(img)
imgsym += img[-Z, -Y, X]
imgsym /= 2.0
elif sym.lower() == 'd2':
imgsym = copy.copy(img)
imgsym += img[Z, -Y, -X]
imgsym += imgsym[-Z, -Y, X]
imgsym /= 4.0
elif sym.lower() == 'd4':
imgsym = copy.copy(img)
imgsym += img[Z, -Y, -X]
imgsym += img[Z, -X, Y]
imgsym += img[Z, X, -Y]
imgsym += imgsym[-Z, -Y, X]
imgsym /= 8.0
elif sym.lower() == 'p4212':
imgsym = copy.copy(img)
imgsym += img[Z, -Y, -X]
Ynew_a = -Y + img.shape[1] // 2
Ynew_b = (Ynew_a + 2 * Y) % img.shape[1]
Xnew_a = -X + img.shape[2] // 2
Xnew_b = (Xnew_a + 2 * X) % img.shape[2]
imgsym += img[Z, Xnew_b, Ynew_a]
imgsym += img[Z, Xnew_a, Ynew_b]
imgsym += img[Z, Ynew_b, Xnew_a]
imgsym += img[Z, Ynew_a, Xnew_b]
imgsym += img[-Z, X, Y]
imgsym += img[-Z, -X, -Y]
imgsym /= 8.0
elif sym[0].lower() == 'c':
n = np.int(sym[1:])
# print n
imgsym = copy.copy(img)
ang = 360.0 / n
for k in np.arange(1, n):
# print k * ang
imgrot = util.Rotate(img,
rot=[k * ang, 0, 0],
interpolation='trilinear',
pad=1)
# print np.allclose(imgrot, 0.0)
imgsym += imgrot
# print np.allclose(imgsym, img)
imgsym /= n
elif sym[0].lower() == 'd':
n = np.int(sym[1:])
# print n
imgsym = copy.copy(img)
imgsym += img[-Z, Y, X]
ang = 360.0 / n
for k in np.arange(1, n + 1):
# print k * ang
imgrot = util.Rotate(img,
rot=[k * ang, 0, 0],
interpolation='trilinear',
pad=1)
imgsym += imgrot
imgsym += imgsym[-Z, -Y, X]
imgsym /= 2 * n
else:
raise Exception("Symmetry operator not supported! sym = %s" % sym)
if pad != 1:
imgsym = util.Resample(imgsym, newsize=imgshape_ori)
if len(imgshape_ori) == 2:
imgsym = imgsym.reshape((imgshape_ori[0], imgshape_ori[1]))
return imgsym
#!/usr/bin/env python
import focus_utilities as util
from mrcz import ioMRC
import sys
import numpy as np
stackin = ioMRC.readMRC( sys.argv[1] )[0]
stackout = sys.argv[2]
factor = np.float( sys.argv[3] )
newsize = np.round( np.array( stackin.shape[1:] ) * factor )
for i in range( stackin.shape[0] ):
print(' Resampling slice %d/%d ' % ( i+1, stackin.shape[0] ) )
ioMRC.writeMRC( util.Resample( stackin[i,:,:], newsize=newsize ), stackout, idx=i )
print( 'Done!' )