def sanity_checks(args, myid):
"""
Check if the command line arguments are valid.
Arguments:
args - Command line arguments as dictionary
myid - MPI id
Returns:
None
"""
if args['gain_file'] is not None:
if not os.path.isfile(args['gain_file']):
sp_global_def.ERROR('If the gain_file option is provided, the gain file must exist!', myid=myid)
if args['additional_dose_unadjusted'] and args['skip_dose_adjustment']:
sp_global_def.ERROR('If the additional_dose_unadjusted option is provided, the skip_dose_adjustment cannot be provided as well!', myid=myid)
if args['selection_file'] is not None and not os.path.isfile(args.selection_file):
sp_global_def.ERROR('If the selection_file option is provided, the specified file needs to exist!', myid=myid)
if not glob.glob(args['input_micrograph_pattern']):
sp_global_def.ERROR('No files found with the specified pattern!: {0}'.format(args['input_micrograph_pattern']), myid=myid)
if os.path.exists(args['output_directory']) and not args['overwrite']:
sp_global_def.ERROR('Output directory is not allowed to exist!', myid=myid)
def angle_diff_sym(angle1, angle2, simi=1):
'''
This function determines the relative angle around Z axis (phi) between two sets of angles
taking into account point group symmetry with multiplicity simi.
The input has to be in the form [[phi0,theta0], [phi1,theta1], ...]
Only sets that have theta in the same range (0,90), or (90,180) are included in calculation.
The resulting angle has to be added (modulo 360/simi) to the first set.
'''
pass #IMPORTIMPORTIMPORT from math import cos, sin, pi, atan2, degrees, radians
nima = len(angle1)
if len(angle2) != nima:
sp_global_def.ERROR("List lengths do not agree!", "angle_diff_sym", 1)
cosi = 0.0
sini = 0.0
agree = 0
for i in range(nima):
if (((angle2[i][1] < 90.0) and (angle1[i][1] < 90.0))
or ((angle2[i][1] > 90.0) and (angle1[i][1] > 90.0))):
qt = numpy.radians((angle2[i][0] - angle1[i][0]) * simi)
cosi += numpy.cos(qt)
sini += numpy.sin(qt)
agree += 1
if (agree == 0): return 0.0
else: return numpy.degrees(math.atan2(sini, cosi) / simi) % (360.0 / simi)
def sanity_checks(command_args, input_vol):
if os.path.isfile(
os.path.join(command_args.output_dir,
command_args.prefix + "_mask.hdf")):
if not command_args.overwrite:
sp_global_def.ERROR(
"Output mask already exists! Please provide the overwrite option if you want to overwrite the existing mask."
)
if (command_args.s_nx is not None and command_args.s_ny is None
and command_args.s_ny is None and command_args.second_mask_shape):
command_args.s_ny = command_args.s_nx
command_args.s_nz = command_args.s_nx
elif (command_args.s_nx is not None and command_args.s_ny is not None
and command_args.s_ny is not None
and command_args.second_mask_shape):
pass
elif command_args.second_mask_shape:
sp_global_def.ERROR(
"You need to specify s_nx only or s_nx and s_ny and s_nz")
if command_args.second_mask_shape in ("cylinder", "sphere"):
nx = input_vol.get_xsize()
ny = input_vol.get_ysize()
nz = input_vol.get_zsize()
if command_args.s_radius > old_div(
nx, 2) or command_args.s_radius > old_div(ny, 2):
sp_global_def.ERROR(
"Provided radius is larger than input image dimensions!")
if command_args.second_mask_shape is not None:
nx = input_vol.get_xsize()
ny = input_vol.get_ysize()
nz = input_vol.get_zsize()
if command_args.s_nx > nx or command_args.s_ny > ny or command_args.s_nz > nz:
sp_global_def.ERROR(
"Provided s_nx, s_ny, s_nz mask dimension is larger than input image dimensions!"
)
def main():
progname = os.path.basename(sys.argv[0])
# # horatio active_refactoring Jy51i1EwmLD4tWZ9_00003_1
# usage = progname + " stack <name_output> --ali --active --set_size=param_name_with_size --set_members=param_name_with_id"
# # horatio active_refactoring Jy51i1EwmLD4tWZ9_00003_2
usage = progname + " stack <name_output> --ali --set_size=param_name_with_size --set_members=param_name_with_id"
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--ali" , action = "store_true", default=False, help="Perform average using alignment parameters")
# # horatio active_refactoring Jy51i1EwmLD4tWZ9_00004_1
# parser.add_option("--active" , action = "store_true", default=False, help="Perform average only for active images")
parser.add_option("--set_size" , type = "string" , default=None , help="Save number of input images to parameter with given name")
parser.add_option("--set_members", type = "string" , default=None , help="Save list of id of input images to parameter named \"members\", id of input images are taken from parameter with given name")
parser.add_option("--filament" , action = "store_true", default=False, help="Calculate stack of averages according to filament membership")
(options, args) = parser.parse_args()
if len(args) < 1 or len(args) > 2:
sxprint( "Usage: " + usage )
sxprint( "Please run \'" + progname + " -h\' for detailed options" )
sp_global_def.ERROR( "Invalid number of parameters used. Please see usage information above." )
return
else:
if len(args) == 1:
name_output = None
else:
name_output = args[1]
if options.filament:
from sp_development import ave_ali_filament
if sp_global_def.CACHE_DISABLE:
from sp_utilities import disable_bdb_cache
disable_bdb_cache()
sp_global_def.BATCH = True
ave_ali_filament(args[0], name_output, options.ali)
sp_global_def.BATCH = False
else:
from sp_applications import ave_ali
if sp_global_def.CACHE_DISABLE:
from sp_utilities import disable_bdb_cache
disable_bdb_cache()
sp_global_def.BATCH = True
ave_ali(args[0], name_output, options.ali, options.set_size, options.set_members)
sp_global_def.BATCH = False
def prgl(volft, params, interpolation_method=0, return_real=True):
"""
Name
prgl - calculate 2-D projection of a 3-D volume using either NN Fourier or or trilinear Fourier
Input
vol: input volume, the volume has to be cubic
params: input parameters given as a list [phi, theta, psi, s2x, s2y], projection in calculated using the three Eulerian angles and then shifted by sx,sy
interpolation_method = 0 NN
interpolation_method = 1 trilinear
return_real: True - return real; False - return FT of a projection.
Output
proj: generated 2-D projection
"""
# params: phi, theta, psi, sx, sy
pass #IMPORTIMPORTIMPORT from sp_fundamentals import fft
pass #IMPORTIMPORTIMPORT from sp_utilities import set_params_proj, info
pass #IMPORTIMPORTIMPORT from EMAN2 import Processor
if (interpolation_method < 0 or interpolation_method > 1):
sp_global_def.ERROR('Unsupported interpolation method',
"interpolation_method", 1, 0)
npad = volft.get_attr_default("npad", 1)
R = EMAN2_cppwrap.Transform({
"type": "spider",
"phi": params[0],
"theta": params[1],
"psi": params[2]
})
if (npad == 1): temp = volft.extract_section(R, interpolation_method)
elif (npad == 2): temp = volft.extract_section2(R, interpolation_method)
temp.fft_shuffle()
temp.center_origin_fft()
if (params[3] != 0. or params[4] != 0.):
filt_params = {
"filter_type": EMAN2_cppwrap.Processor.fourier_filter_types.SHIFT,
"x_shift": params[3],
"y_shift": params[4],
"z_shift": 0.0
}
temp = EMAN2_cppwrap.Processor.EMFourierFilter(temp, filt_params)
if return_real:
temp.do_ift_inplace()
temp.set_attr_dict({'ctf_applied': 0, 'npad': 1})
temp.depad()
else:
temp.set_attr_dict({'ctf_applied': 0, 'npad': 1})
sp_utilities.set_params_proj(
temp, [params[0], params[1], params[2], -params[3], -params[4]])
return temp
def ormq_fast(dimage, crefim, xrng, yrng, step, numr, mode, delta=0.0):
"""Determine shift and rotation between image and reference image (crefim)
crefim should be as FT of polar coords with applied weights
consider mirror
cnx, cny in FORTRAN convention
"""
# from math import pi, cos, sin, radians
# print "ORMQ_FAST"
maxrange = old_div(len(dimage), 2)
# istep = int(2*step)
istep = int(step)
"""Multiline Comment5"""
lkx = rkx = int(xrng * istep)
lky = rky = int(yrng * istep)
peak = -1.0e23
for j in range(-lky, rky + 1, istep):
for i in range(-lkx, rkx + 1, istep):
if delta == 0.0:
retvals = EMAN2_cppwrap.Util.Crosrng_ms(
crefim, dimage[i + maxrange][j + maxrange], numr, 0.0)
else:
retvals = EMAN2_cppwrap.Util.Crosrng_ms_delta(
crefim, dimage[i + maxrange][j + maxrange], numr, delta)
qn = retvals["qn"]
qm = retvals["qm"]
if qn >= peak or qm >= peak:
sx = i
sy = j
if qn >= qm:
ang = ang_n(retvals["tot"], mode, numr[-1])
peak = qn
mirror = 0
else:
ang = ang_n(retvals["tmt"], mode, numr[-1])
peak = qm
mirror = 1
"""Multiline Comment6"""
if peak < -1.0e20:
sp_global_def.ERROR("ormq_fast",
"failed, most likely due to search ranges", 1)
# return ang, sx/2.0, sy/2.0, mirror, peak
return ang, sx, sy, mirror, peak
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + " configure_file.cfg"
parser = OptionParser(usage, version=SPARXVERSION)
(options, args) = parser.parse_args()
if len(args) != 1:
sxprint("Usage: " + usage)
sxprint("Please run \'" + progname + " -h\' for detailed options")
sp_global_def.ERROR(
"Invalid number of parameters used. Please see usage information above."
)
return
if sp_global_def.CACHE_DISABLE:
from sp_utilities import disable_bdb_cache
disable_bdb_cache()
from sp_development import ali2d_mref
sp_global_def.BATCH = True
ali2d_mref(args[0])
sp_global_def.BATCH = False
def angle_diff(angle1, angle2):
"""
This function determines the relative angle between two sets of angles.
The resulting angle has to be added (modulo 360) to the first set.
"""
nima = len(angle1)
nima2 = len(angle2)
if nima2 != nima:
sp_global_def.ERROR("Error: List lengths do not agree!", "angle_diff",
1)
else:
del nima2
cosi = 0.0
sini = 0.0
for i in range(nima):
qt = numpy.radians(angle2[i] - angle1[i])
cosi += numpy.cos(qt)
sini += numpy.sin(qt)
alphai = numpy.degrees(math.atan2(sini, cosi)) % 360.0
return alphai
def search_range(n, radius, shift, range, location=""):
"""
Find permissible ranges for translational searches by resampling into polar coordinates
n - image size; radius - particle radius, the circle has to fit into the square image;
shift - current particle shift; range - desired maximum range search
Output: a list of two elements:
left range (positive)
right range
NOTE - ranges are with respect to the point n//2+1-shift within image (in 3D)
"""
cn = old_div(n, 2) + 1
ql = cn + shift - radius - 2 # lower end is positive
qe = n - cn - shift - radius # upper end
if ql < 0 or qe < 0:
sp_global_def.ERROR(
"Shift of particle too large, results may be incorrect: %4d %3d %f %f %f %f %f"
% (n, cn, radius, shift, range, ql, qe),
"search_range " + location,
0,
)
ql = max(ql, 0)
qe = max(qe, 0)
# ???for mysterious reasons it has to be this way as C code changes the order of searches.
return [min(qe, range), min(ql, range)]
def main():
# Parse the Options
progname = os.path.basename(sys.argv[0])
usage = (progname +
""" unblur_path input_micrograph_pattern output_directory
--summovie_path
--selection_list
--nr_frames=nr_frames
--pixel_size=pixel_size
--voltage=voltage
--exposure_per_frame=exposure_per_frame
--pre_exposure=pre_exposure
--nr_threads
--save_frames
--skip_dose_filter
--expert_mode
--shift_initial=shift_initial
--shift_radius=shift_radius
--b_factor=b_factor
--fourier_vertical=fourier_vertical
--fourier_horizontal=fourier_horizontal
--shift_threshold=shift_threshold
--iterations=iterations
--dont_restore_noise
--verbose
sxunblur exists only in non-MPI version.
Perform unblur and with dose filtering and summovie without dose filtering.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--summovie_path=~/my_app/summovie
--nr_frames=25 --pixel_size=1.19 --exposure_per_frame=1.0
--voltage=300.0 --pre_exposure=0.0 --nr_threads=1
Perform unblur with dose filtering and summovie without dose filtering with selection list.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--summovie_path=~/my_app/summovie
--selection_list=selected_micrograph_file
--nr_frames=25 --pixel_size=1.19 --exposure_per_frame=1.0
--voltage=300.0 --pre_exposure=0.0 --nr_threads=1
Perform unblur without dose filtering.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--nr_frames=25 --pixel_size=1.19 --skip_dose_filter --nr_threads=1
Perform unblur without dose filtering and save the frames.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--nr_frames=25 --pixel_size=1.19 --skip_dose_filter --save_frames --nr_threads=1
Perform unblur with dose filtering and summovie without dose filtering with all options.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--summovie_path=~/my_app/summovie
--nr_frames=25 --pixel_size=1.19 --exposure_per_frame=1.0
--voltage=300.0 --pre_exposure=0.0 --save_frames --expert_mode
--shift_initial=2.0 --shift_radius=200.0 --b_factor=1500.0
--fourier_vertical=1 --fourier_horizontal=1 --shift_threshold=0.1
--iterations=10 --verbose --nr_threads=1
""")
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option(
"--summovie_path",
type="str",
default="",
help=
"summovie executable path (SPHIRE specific): Specify the file path of summovie executable. (default none)",
)
parser.add_option(
"--selection_list",
type="str",
default="",
help=
"Micrograph selecting list (SPHIRE specific): Specify a name of micrograph selection list text file. The file extension must be '.txt'. If this is not provided, all files matched with the micrograph name pattern will be processed. (default none)",
)
parser.add_option(
"--nr_frames",
type="int",
default=3,
help=
"Number of movie frames: The number of movie frames in each input micrograph. (default 3)",
)
parser.add_option("--sum_suffix",
type="str",
default="_sum",
help=optparse.SUPPRESS_HELP)
parser.add_option("--shift_suffix",
type="str",
default="_shift",
help=optparse.SUPPRESS_HELP)
parser.add_option(
"--pixel_size",
type="float",
default=-1.0,
help=
"Pixel size [A]: The pixel size of input micrographs. (default required float)",
)
parser.add_option(
"--voltage",
type="float",
default=300.0,
help=
"Microscope voltage [kV]: The acceleration voltage of microscope used for imaging. (default 300.0)",
)
parser.add_option(
"--exposure_per_frame",
type="float",
default=2.0,
help=
"Per frame exposure [e/A^2]: The electron dose per frame in e/A^2. (default 2.0)",
)
parser.add_option(
"--pre_exposure",
type="float",
default=0.0,
help=
"Pre-exposure [e/A^2]: The electron does in e/A^2 used for exposure prior to imaging .(default 0.0)",
)
parser.add_option(
"--nr_threads",
type="int",
default=1,
help=
"Number of threads: The number of threads unblur can use. The higher the faster, but it requires larger memory. (default 1)",
)
parser.add_option(
"--save_frames",
action="store_true",
default=False,
help=
"Save aligned movie frames: Save aligned movie frames. This option slows down the process. (default False)",
)
parser.add_option("--frames_suffix",
type="string",
default="_frames",
help=optparse.SUPPRESS_HELP)
parser.add_option(
"--skip_dose_filter",
action="store_true",
default=False,
help=
"Skip dose filter step: With this option, voltage, exposure per frame, and pre exposure will be ignored. (default False)",
)
parser.add_option(
"--expert_mode",
action="store_true",
default=False,
help=
"Use expert mode: Requires initial shift, shift radius, b-factor, fourier_vertical, fourier_horizontal, shift threshold, iterations, restore noise, and verbosity options. (default False)",
)
parser.add_option("--frc_suffix",
type="string",
default="_frc",
help=optparse.SUPPRESS_HELP)
parser.add_option(
"--shift_initial",
type="float",
default=2.0,
help=
"Minimum shift for initial search [A] (expert mode): Effective with unblur expert mode. (default 2.0)",
)
parser.add_option(
"--shift_radius",
type="float",
default=200.0,
help=
"Outer radius shift limit [A] (expert mode): Effective with unblur expert mode. (default 200.0)",
)
parser.add_option(
"--b_factor",
type="float",
default=1500.0,
help=
"Apply B-factor to images [A^2] (expert mode): Effective with unblur expert mode. (default 1500.0)",
)
parser.add_option(
"--fourier_vertical",
type="int",
default=1,
help=
"Vertical Fourier central mask size (expert mode): The half-width of central vertical line of Fourier mask. Effective with unblur expert mode. (default 1)",
)
parser.add_option(
"--fourier_horizontal",
type="int",
default=1,
help=
"Horizontal Fourier central mask size (expert mode): The half-width of central horizontal line of Fourier mask. Effective with unblur expert mode. (default 1)",
)
parser.add_option(
"--shift_threshold",
type="float",
default=0.1,
help=
"Termination shift threshold (expert mode): Effective with unblur expert mode. (default 0.1)",
)
parser.add_option(
"--iterations",
type="int",
default=10,
help=
"Maximum iterations (expert mode): Effective with unblur expert mode. (default 10)",
)
parser.add_option(
"--dont_restore_noise",
action="store_true",
default=False,
help=
"Do not restore noise power (expert mode): Effective with unblur expert mode. (default False)",
)
parser.add_option(
"--verbose",
action="store_true",
default=False,
help=
"Verbose (expert mode): Effective with unblur expert mode. (default False)",
)
parser.add_option(
"--unblur_ready",
action="store_true",
default=False,
help=optparse.SUPPRESS_HELP,
)
# list of the options and the arguments
(options, args) = parser.parse_args(sys.argv[1:])
sp_global_def.BATCH = True
# If there arent enough arguments, stop the script
if len(args) != 3:
sp_global_def.sxprint("Usage: " + usage)
sp_global_def.ERROR(
"Invalid number of parameters used. Please see usage information above."
)
return
# Convert the realtive parts to absolute ones
unblur_path = os.path.realpath(args[0]) # unblur_path
input_image = os.path.realpath(args[1]) # input_micrograph_pattern
output_dir = os.path.realpath(args[2]) # output_directory
# If the unblur executable file does not exists, stop the script
if not os.path.exists(unblur_path):
sp_global_def.ERROR(
"Unblur directory does not exist, please change the name and restart the program"
)
return
# If the output directory exists, stop the script
if os.path.exists(output_dir):
sp_global_def.ERROR(
"Output directory exists, please change the name and restart the program"
)
return
# If the input file does not exists, stop the script
file_list = glob.glob(input_image)
if not file_list:
sp_global_def.ERROR(
"Input file does not exist, please change the name and restart the program"
)
return
# If the skip_dose_filter option is false, the summovie path is necessary
if not options.skip_dose_filter and not os.path.exists(
options.summovie_path):
sp_global_def.ERROR(
"Path to the SumMovie executable is necessary when dose weighting is performed"
)
return
# Output paths
corrected_path = "{:s}/corrsum_dose_filtered".format(output_dir)
uncorrected_path = "{:s}/corrsum".format(output_dir)
shift_path = "{:s}/shift".format(output_dir)
frc_path = "{:s}/frc".format(output_dir)
log_path = "{:s}/logfiles".format(output_dir)
temp_path = "{0}/temp".format(output_dir)
# Split the path of the image name at the "/" Characters.
# The last entry contains the micrograph name.
# Split the micrograph name at the wildcard character for the
# prefix and suffix.
input_split = input_image.split("/")
input_name = input_split[-1].split("*")
# Get the input directory
if len(input_split) != 1:
input_dir = input_image[:-len(input_split[-1])]
else:
input_dir = ""
# Create output directorys
if not os.path.exists(output_dir):
os.makedirs(output_dir)
sp_global_def.write_command(output_dir)
if not os.path.exists(uncorrected_path):
os.mkdir(uncorrected_path)
if not os.path.exists(shift_path):
os.mkdir(shift_path)
if not os.path.exists(corrected_path):
if not options.skip_dose_filter:
os.mkdir(corrected_path)
if not os.path.exists(frc_path):
if options.expert_mode or not options.skip_dose_filter:
os.mkdir(frc_path)
if not os.path.exists(temp_path) and not options.unblur_ready:
os.mkdir(temp_path)
if not os.path.exists(log_path):
os.mkdir(log_path)
# Run unblur
run_unblur(
unblur_path=unblur_path,
input_image=input_image,
input_dir=input_dir,
output_dir=output_dir,
corrected_path=corrected_path,
uncorrected_path=uncorrected_path,
shift_path=shift_path,
frc_path=frc_path,
temp_path=temp_path,
log_path=log_path,
file_list=file_list,
options=options,
)
if not options.unblur_ready:
# Remove temp folder
for entry in glob.glob("{0}/*".format(temp_path)):
os.remove(entry)
os.rmdir(temp_path)
sp_global_def.sxprint("All Done!")
sp_global_def.BATCH = False
def run_unblur(
unblur_path,
input_image,
input_dir,
output_dir,
corrected_path,
uncorrected_path,
shift_path,
frc_path,
temp_path,
log_path,
file_list,
options,
):
# Lists to write the text files later
micrograph_list = []
shift_list = []
if options.save_frames:
frames_list = []
# If micrograph list is provided just process the images in the list
mic_list = options.selection_list
if mic_list:
# Import list file
try:
set_selection = numpy.genfromtxt(mic_list, dtype=None)
except TypeError:
sp_global_def.ERROR("no entrys in list file {0}".format(mic_list))
# List of files which are in pattern and list
file_list = [
entry for entry in file_list if
entry[len(input_dir):] in set_selection and os.path.exists(entry)
]
# If no match is there abort
if len(file_list) == 0:
sp_global_def.ERROR(
"no files in {0} matched the file pattern:\n".format(mic_list),
1)
# Get the number of files
nr_files = len(file_list)
# Timeing stuff
time_start = time.time()
time_list = []
# Loop over all files
for index, inputfile in enumerate(sorted(file_list)):
# Check, if there is an prefix and suffix.
# If there is more then one entry: the suffix is the last one.
# Otherwhise its just the one after the dot.
input_suffix = inputfile.split("/")[-1].split(".")[-1]
# First output to introduce the programm
if index == 0:
sp_global_def.sxprint(
"Progress: 0.0%; Time: --h:--m:--s/--h:--m:--s; Unblur started!"
)
# Time begin
t1 = time.time()
# Get the output names
file_name = inputfile[len(input_dir):-len(input_suffix) - 1]
if options.skip_dose_filter:
micrograph_name = "{0}/{1}{2}.mrc".format(uncorrected_path,
file_name,
options.sum_suffix)
frames_name = "{0}/{1}{2}.mrc".format(uncorrected_path, file_name,
options.frames_suffix)
frc_name = "{0}/{1}{2}.txt".format(frc_path, file_name,
options.frc_suffix)
else:
micrograph_name = "{0}/{1}{2}.mrc".format(corrected_path,
file_name,
options.sum_suffix)
frames_name = "{0}/{1}{2}.mrc".format(corrected_path, file_name,
options.frames_suffix)
micrograph_name_skip = "{0}/{1}{2}.mrc".format(
uncorrected_path, file_name, options.sum_suffix)
frames_name_skip = "{0}/{1}{2}.mrc".format(uncorrected_path,
file_name,
options.frames_suffix)
frc_name = "{0}/{1}{2}.txt".format(frc_path, file_name,
options.frc_suffix)
frc_summovie_name = "{0}/{1}_summovie{2}.txt".format(
frc_path, file_name, options.frc_suffix)
shift_name = "{0}/{1}{2}.txt".format(shift_path, file_name,
options.shift_suffix)
if not options.unblur_ready:
temp_name = "{0}/{1}{2}.mrc".format(temp_path, file_name,
options.sum_suffix)
else:
temp_name = inputfile
log_name = "{0}/{1}.log".format(log_path, file_name)
error_name = "{0}/{1}.err".format(log_path, file_name)
# Append the names to the lists
micrograph_list.append("{0}{1}.mrc".format(file_name,
options.sum_suffix))
shift_list.append(shift_name)
if options.save_frames:
frames_list.append("{0}{1}.mrc".format(file_name,
options.frames_suffix))
# First build the unblur/summovie command
if not options.skip_dose_filter:
unblur_command = create_unblur_command(temp_name, micrograph_name,
shift_name, frames_name,
frc_name, options)
# Options for the summovie command
options_summovie = {
"first": 1,
"last": -1,
"nr_frames": options.nr_frames,
"pixel_size": options.pixel_size,
"exposure_per_frame": options.exposure_per_frame,
"voltage": options.voltage,
"pre_exposure": options.pre_exposure,
"dont_restore_noise": options.dont_restore_noise,
"apply_dose_filter": False,
}
summovie_command = sp_utilities.create_summovie_command(
temp_name,
micrograph_name_skip,
shift_name,
frc_summovie_name,
options_summovie,
)
else:
unblur_command = create_unblur_command(temp_name, micrograph_name,
shift_name, frc_name,
frames_name, options)
# Export the number of threads
export_threads_command = []
# Export
export_threads_command.append("export")
# Nr of threads
export_threads_command.append("OMP_NUM_THREADS={0}".format(
options.nr_threads))
if not options.unblur_ready:
# Do a e2proc3d.py
e2proc3d_command = []
# e2proc3d
e2proc3d_command.append("e2proc3d.py")
# inputfile
e2proc3d_command.append("{0}".format(inputfile))
# outputfile
e2proc3d_command.append("{0}".format(temp_name))
# Translate the command to single strings
if not options.unblur_ready:
e2proc3d_command = r" ".join(e2proc3d_command)
export_threads_command = r" ".join(export_threads_command)
unblur_command = "\n".join(unblur_command)
if not options.skip_dose_filter:
summovie_command = "\n".join(summovie_command)
# Build full command
if not options.unblur_ready:
if not options.skip_dose_filter:
full_command = r'{0}; {1}; echo "{2}" | {3}'.format(
export_threads_command,
e2proc3d_command,
unblur_command,
unblur_path,
)
full_command_summovie = r'{0}; echo "{1}" | {2}'.format(
export_threads_command, summovie_command,
options.summovie_path)
else:
full_command = r'{0}; {1}; echo "{2}" | {3}'.format(
export_threads_command,
e2proc3d_command,
unblur_command,
unblur_path,
)
else:
if not options.skip_dose_filter:
full_command = r'{0}; echo "{1}" | {2}'.format(
export_threads_command, unblur_command, unblur_path)
full_command_summovie = r'{0}; echo "{1}" | {2}'.format(
export_threads_command, summovie_command,
options.summovie_path)
else:
full_command = r'{0}; echo "{1}" | {2}'.format(
export_threads_command, unblur_command, unblur_path)
# Remove temp unblur files
temp_unblur_files = glob.glob(".UnBlur*")
for entry in temp_unblur_files:
os.remove(entry)
# Remove temp summovie files
temp_summovie_files = glob.glob(".SumMovie*")
for entry in temp_summovie_files:
os.remove(entry)
with open(log_name, "w") as f:
with open(error_name, "w") as e:
# Execute Command
if not options.skip_dose_filter:
subprocess.Popen([full_command],
shell=True,
stdout=f,
stderr=e).wait()
# Remove temp unblur files
temp_unblur_files = glob.glob(".UnBlur*")
for entry in temp_unblur_files:
os.remove(entry)
# Remove temp summovie files
temp_summovie_files = glob.glob(".SumMovie*")
for entry in temp_summovie_files:
os.remove(entry)
subprocess.Popen([full_command_summovie],
shell=True,
stdout=f,
stderr=e).wait()
else:
subprocess.Popen([full_command],
shell=True,
stdout=f,
stderr=e).wait()
# Remove temp unblur files
temp_unblur_files = glob.glob(".UnBlur*")
for entry in temp_unblur_files:
os.remove(entry)
# Remove temp summovie files
temp_summovie_files = glob.glob(".SumMovie*")
for entry in temp_summovie_files:
os.remove(entry)
if not options.unblur_ready:
if os.path.exists(temp_name):
# Remove temp file
os.remove(temp_name)
else:
sp_global_def.sxprint(
("Error with file:\n{0}".format(inputfile)))
# Check if SumMovie and UnBlur finished cleanly
with open(log_name, "r") as r:
clean_summovie = False
clean_unblur = False
for line in r:
if "SumMovie finished cleanly." in line:
clean_summovie = True
if "UnBlur finished cleanly." in line:
clean_unblur = True
if clean_unblur:
sp_global_def.sxprint("UnBlur finished cleanly.")
else:
sp_global_def.ERROR(
"unblur error. check the logfile for more information: {0}".
format(log_name),
action=0,
)
if clean_summovie:
sp_global_def.sxprint("SumMovie finished cleanly.")
else:
sp_global_def.ERROR(
"summovie error. check the logfile for more information: {0}".
format(log_name),
action=0,
)
time_list.append(time.time() - t1)
# Do progress output
percent = round(100 * (index + 1) / float(nr_files), 2)
estimated_time = nr_files * sum(time_list) / float(len(time_list))
estimated_time_h = estimated_time // 3600
estimated_time_m = (estimated_time - estimated_time_h * 3600) // 60
estimated_time_s = (estimated_time - estimated_time_h * 3600 -
estimated_time_m * 60)
current_time = time.time() - time_start
current_time_h = current_time // 3600
current_time_m = (current_time - current_time_h * 3600) // 60
current_time_s = current_time - current_time_h * 3600 - current_time_m * 60
sp_global_def.sxprint((
"Progress: {0:.2f}%; Time: {1:.0f}h:{2:.0f}m:{3:.0f}s/{4:.0f}h:{5:.0f}m:{6:.0f}s; Micrograph done:{7}"
.format(
percent,
current_time_h,
current_time_m,
current_time_s,
estimated_time_h,
estimated_time_m,
estimated_time_s,
file_name,
)))
# Write micrograph and shift list
with open("{0}/unblur_micrographs.txt".format(output_dir), "w") as f:
for entry in sorted(micrograph_list):
f.write("{0}\n".format(entry))
with open("{0}/unblur_shiftfiles.txt".format(output_dir), "w") as f:
for entry in sorted(shift_list):
f.write("{0}\n".format(entry))
if options.save_frames:
with open("{0}/unblur_frames.txt".format(output_dir), "w") as f:
for entry in sorted(frames_list):
f.write("{0}\n".format(entry))
def main_proj_compare(classavgstack,
reconfile,
outdir,
options,
mode='viper',
prjmethod='trilinear',
classangles=None,
partangles=None,
selectdoc=None,
verbose=False,
displayYN=False):
"""
Main function overseeing various projection-comparison modes.
Arguments:
classavgstack : Input image stack
reconfile : Map of which to generate projections (an optionally perform alignment)
outdir : Output directory
mode : Mode, viper (pre-existing angles for each input image), projmatch (angles from internal projection-matching)
verbose : (boolean) Whether to write additional information to screen
options : (list) Command-line options, run 'sxproj_compare.py -h' for an exhaustive list
classangles : Angles and shifts for each input class average
partangles : Angles and shifts for each particle (mode meridien)
selectdoc : Selection file for included images
prjmethod : Interpolation method to use
displayYN : (boolean) Whether to automatically open montage
"""
# Expand path for outputs
refprojstack = os.path.join(outdir, 'refproj.hdf')
refanglesdoc = os.path.join(outdir, 'refangles.txt')
outaligndoc = os.path.join(outdir, 'docalign2d.txt')
# If not an input, will create an output, in modes projmatch
if classangles == None:
classangles = os.path.join(outdir, 'docangles.txt')
# You need either input angles (mode viper) or to calculate them on the fly (mode projmatch)
if mode == 'viper':
sp_global_def.ERROR(
"\nERROR!! Input alignment parameters not specified.",
__file__, 1)
sxprint('Type %s --help to see available options\n' %
os.path.basename(__file__))
exit()
# Check if inputs exist
check(classavgstack, verbose=verbose)
check(reconfile, verbose=verbose)
if verbose: sxprint('')
# Check that dimensions of images and volume agree (maybe rescale volume)
voldim = EMAN2.EMData(reconfile).get_xsize()
imgdim = EMAN2.EMData(classavgstack, 0).get_xsize()
if voldim != imgdim:
sp_global_def.ERROR(
"\nERROR!! Dimension of input volume doesn't match that of image stack: %s vs. %s"
% (voldim, imgdim), __file__, 1)
scale = float(
imgdim
) / voldim # only approximate, since full-sized particle radius is arbitrary
msg = 'The command to resize the volume will be of the form:\n'
msg += 'e2proc3d.py %s resized_vol.hdf --scale=%1.5f --clip=%s,%s,%s\n' % (
reconfile, scale, imgdim, imgdim, imgdim)
msg += 'Check the file in the ISAC directory named "README_shrink_ratio.txt" for confirmation.\n'
sxprint(msg)
exit()
# Here if you want to be fancy, there should be an option to chose the projection method,
# the mechanism can be copied from sxproject3d.py PAP
if prjmethod == 'trilinear':
method_num = 1
elif prjmethod == 'gridding':
method_num = -1
elif prjmethod == 'nn':
method_num = 0
else:
sp_global_def.ERROR(
"\nERROR!! Valid projection methods are: trilinear (default), gridding, and nn (nearest neighbor).",
__file__, 1)
sxprint('Usage:\n%s' % USAGE)
exit()
# Set output directory and log file name
log, verbose = prepare_outdir_log(outdir, verbose)
# In case class averages include discarded images, apply selection file
if mode == 'viper':
if selectdoc:
goodavgs, extension = os.path.splitext(
os.path.basename(classavgstack))
newclasses = os.path.join(outdir, goodavgs + "_kept" + extension)
# e2proc2d appends to existing files, so rename existing output
if os.path.exists(newclasses):
renamefile = newclasses + '.bak'
print_log_msg(
"Selected-classes stack %s exists, renaming to %s" %
(newclasses, renamefile), log, verbose)
print_log_msg("mv %s %s\n" % (newclasses, renamefile), log,
verbose)
os.rename(newclasses, renamefile)
print_log_msg(
'Creating subset of %s to %s based on selection list %s' %
(classavgstack, newclasses, selectdoc), log, verbose)
cmd = "e2proc2d.py %s %s --list=%s" % (classavgstack, newclasses,
selectdoc)
print_log_msg(cmd, log, verbose)
os.system(cmd)
sxprint('')
# Update class-averages
classavgstack = newclasses
# align de novo to reference map
if mode == 'projmatch':
# Generate reference projections
print_log_msg(
'Projecting %s to output %s using an increment of %s degrees using %s symmetry'
% (reconfile, refprojstack, options.delta, options.symmetry), log,
verbose)
cmd = 'sxproject3d.py %s %s --delta=%s --method=S --phiEqpsi=Minus --symmetry=%s' % (
reconfile, refprojstack, options.delta, options.symmetry)
if options.prjmethod == 'trilinear': cmd += ' --trilinear'
cmd += '\n'
print_log_msg(cmd, log, verbose)
project3d(reconfile,
refprojstack,
delta=options.delta,
symmetry=options.symmetry)
# Export projection angles
print_log_msg(
"Exporting projection angles from %s to %s" %
(refprojstack, refanglesdoc), log, verbose)
cmd = "sp_header.py %s --params=xform.projection --import=%s\n" % (
refprojstack, refanglesdoc)
print_log_msg(cmd, log, verbose)
header(refprojstack, 'xform.projection', fexport=refanglesdoc)
# Perform multi-reference alignment
if options.align == 'ali2d':
projdir = os.path.join(
outdir, 'Projdir') # used if input angles no provided
if os.path.isdir(projdir):
print_log_msg('Removing pre-existing directory %s' % projdir,
log, verbose)
print_log_msg('rm -r %s\n' % projdir, log, verbose)
shutil.rmtree(
projdir) # os.rmdir only removes empty directories
# Zero out alignment parameters in header
print_log_msg(
'Zeroing out alignment parameters in header of %s' %
classavgstack, log, verbose)
cmd = 'sxheader.py %s --params xform.align2d --zero\n' % classavgstack
print_log_msg(cmd, log, verbose)
header(classavgstack, 'xform.align2d', zero=True)
# Perform multi-reference alignment
msg = 'Aligning images in %s to projections %s with a radius of %s and a maximum allowed shift of %s' % (
classavgstack, refprojstack, options.matchrad,
options.matchshift)
print_log_msg(msg, log, verbose)
cmd = 'sxmref_ali2d.py %s %s %s --ou=%s --xr=%s --yr=%s\n' % (
classavgstack, refprojstack, projdir, options.matchrad,
options.matchshift, options.matchshift)
print_log_msg(cmd, log, verbose)
mref_ali2d(classavgstack,
refprojstack,
projdir,
ou=options.matchrad,
xrng=options.matchshift,
yrng=options.matchshift)
# Export alignment parameters
print_log_msg(
'Exporting angles from %s into %s' %
(classavgstack, classangles), log, verbose)
cmd = "sp_header.py %s --params=xform.align2d --export=%s\n" % (
classavgstack, classangles)
print_log_msg(cmd, log, verbose)
header(classavgstack, 'xform.align2d', fexport=classangles)
# By default, use AP SH
else:
apsh(refprojstack,
classavgstack,
outangles=classangles,
refanglesdoc=refanglesdoc,
outaligndoc=outaligndoc,
outerradius=options.matchrad,
maxshift=options.matchshift,
ringstep=options.matchstep,
log=log,
verbose=verbose)
# Diagnostic
alignlist = read_text_row(
classangles) # contain 2D alignment parameters
nimg1 = EMAN2.EMUtil.get_image_count(classavgstack)
assert len(alignlist) == nimg1, "MRK_DEBUG"
# Get alignment parameters from MERIDIEN
if mode == 'meridien':
continueTF = True # Will proceed unless some information is missing
if not partangles:
sp_global_def.ERROR(
"\nERROR!! Input alignment parameters not provided.", __file__,
1)
continueTF = False
if not continueTF:
sxprint('Type %s --help to see available options\n' %
os.path.basename(__file__))
exit()
if not options.classdocs or options.outliers:
classdir = os.path.join(outdir, 'Byclass')
if not os.path.isdir(classdir): os.makedirs(classdir)
if options.outliers:
goodclassparttemplate = os.path.join(
classdir, 'goodpartsclass{0:03d}.txt')
else:
goodclassparttemplate = None
if not options.classdocs:
classmap = os.path.join(classdir, 'classmap.txt')
classdoc = os.path.join(classdir, 'docclass{0:03d}.txt')
options.classdocs = os.path.join(classdir, 'docclass*.txt')
# Separate particles by class
vomq(classavgstack,
classmap,
classdoc,
log=log,
verbose=verbose)
mode_meridien(reconfile,
classavgstack,
options.classdocs,
partangles,
selectdoc,
options.refineshift,
options.refinerad,
classangles,
outaligndoc,
interpolation_method=method_num,
outliers=options.outliers,
goodclassparttemplate=goodclassparttemplate,
alignopt=options.align,
ringstep=options.refinestep,
log=log,
verbose=verbose)
# Import Euler angles
print_log_msg(
"Importing parameter information into %s from %s" %
(classavgstack, classangles), log, verbose)
cmd = "sp_header.py %s --params=xform.projection --import=%s\n" % (
classavgstack, classangles)
print_log_msg(cmd, log, verbose)
header(classavgstack, 'xform.projection', fimport=classangles)
# Make comparison stack between class averages (images 0,2,4,...) and re-projections (images 1,3,5,...)
compstack = compare_projs(reconfile,
classavgstack,
classangles,
outdir,
interpolation_method=method_num,
log=log,
verbose=verbose)
# Optionally pop up e2display
if displayYN:
sxprint('Opening montage')
cmd = "e2display.py %s\n" % compstack
sxprint(cmd)
os.system(cmd)
sxprint("Done!")
def average_angles(alignlist,
partdoc,
selectdoc=None,
init_angles=None,
threshold=None,
goodpartdoc=None,
log=None,
verbose=False):
"""
Computes a vector average of a set of particles' Euler angles phi and theta.
Arguments:
alignlist : Alignment parameter doc file, i.e., from MERIDIEN refinment
partdoc : List of particle indices whose angles should be averaged
selectdoc : Input substack selection file if particles removed before refinement (e.g., Substack/isac_substack_particle_id_list.txt)
init_angles : List (2 elements) with initial phi and theta angles, for excluding outliers
threshold : Angular threshold (degrees) beyond which particles exceeding this angular difference from init_angles will be excluded
goodpartdoc : Output list of retained particles if a threshold was specified
log : Logger object
verbose : (boolean) Whether to write additional information to screen
Returns:
list of 2 elements:
avg_phi
avg_theta
"""
# Read alignment parameters
if isinstance(alignlist, str): alignlist = read_text_row(outaligndoc)
# (If loading the same parameters repeatedly, better to read the file once externally and pass only the list.)
# Read class list
partlist = read_text_row(partdoc)
if selectdoc:
selectlist = read_text_row(selectdoc)
else:
selectlist = None
sum_phi = np.array([0.0, 0.0])
sum_theta = np.array([0.0, 0.0])
totparts = 0
num_outliers = 0
goodpartlist = []
goodpartcounter = 0
# Loop through particles
for totpartnum in partlist:
if selectlist:
goodpartnum = selectlist.index(totpartnum)
else:
goodpartnum = totpartnum[0]
try:
phi_deg = alignlist[goodpartnum][0]
theta_deg = alignlist[goodpartnum][1]
phi_rad = np.deg2rad(phi_deg)
theta_rad = np.deg2rad(theta_deg)
except IndexError:
msg = "\nERROR!! %s tries to access particle #%s" % (partdoc,
goodpartnum)
numalignparts = len(alignlist)
msg += "\nAlignment doc file has only %s entries" % (numalignparts)
msg += "\nMaybe try substack selection file with flag '--select <substack_select>'?"
sp_global_def.ERROR(msg, __file__, 1)
exit()
if init_angles:
angdif = angle_diff(init_angles, [phi_deg, theta_deg])
if angdif > 180: angdif = 360.0 - angdif
totparts += 1
# Exclude particles exceeding optional threshold
if threshold == None or angdif < threshold:
sum_phi += (np.cos(phi_rad), np.sin(phi_rad))
sum_theta += (np.cos(theta_rad), np.sin(theta_rad))
goodpartlist.append(goodpartnum)
goodpartcounter += 1
else:
num_outliers += 1
# Compute final average
avg_phi = degrees(atan2(sum_phi[1], sum_phi[0]))
avg_theta = degrees(atan2(sum_theta[1], sum_theta[0]))
# Clean up, might reuse
del alignlist
del partlist
del selectlist
msg = "Particle list %s: average angles (%s, %s)" % (partdoc, avg_phi,
avg_theta)
print_log_msg(msg, log, verbose)
if threshold:
msg = "Found %s out of %s outliers exceeding an angle difference of %s degrees from initial estimate" % (
num_outliers, totparts, threshold)
print_log_msg(msg, log, verbose)
if goodpartdoc:
if goodpartcounter > 0:
write_text_row(goodpartlist, goodpartdoc)
msg = "Wrote %s particles to %s" % (goodpartcounter,
goodpartdoc)
print_log_msg(msg, log, verbose)
else:
msg = "WARNING!! Kept 0 particles from class %s" % partdoc
print_log_msg(msg, log, verbose)
[avg_phi, avg_theta] = init_angles
return [avg_phi, avg_theta]
"--hold_flag",
type=str,
default=None,
help="Hold flag for the submission command, e.g. -hold_jid",
)
parser.add_argument(
"--first_hold_number",
type=str,
default=None,
help="Wait number of an already running job",
)
args = parser.parse_args()
sp_global_def.BATCH = True
if not os.path.exists(args.input_run_dir):
sp_global_def.ERROR("Input directory does not exist!", "sxbatch.py", 1)
qsub_dict = {
"qsub": glob.re.compile("Your job (\w+)"),
"sbatch": glob.re.compile("Submitted batch job (\w+)"),
}
if args.submission_command.split()[0] not in qsub_dict and args.hold_flag:
sp_global_def.ERROR(
"Qsub return output not known! Please contact the SPHIRE authors!",
"sxbatch.py",
1,
)
sp_global_def.write_command(".")
if args.first_hold_number:
def recons3d_4nn_MPI(
myid,
prjlist,
symmetry="c1",
finfo=None,
snr=1.0,
npad=2,
xysize=-1,
zsize=-1,
mpi_comm=None,
):
if mpi_comm == None:
mpi_comm = mpi.MPI_COMM_WORLD
if type(prjlist) == list:
prjlist = sp_utilities.iterImagesList(prjlist)
if not prjlist.goToNext():
sp_global_def.ERROR("empty input list", "recons3d_4nn_MPI", 1)
imgsize = prjlist.image().get_xsize()
if prjlist.image().get_ysize() != imgsize:
imgsize = max(imgsize, prjlist.image().get_ysize())
dopad = True
else:
dopad = False
prjlist.goToPrev()
fftvol = EMAN2_cppwrap.EMData()
weight = EMAN2_cppwrap.EMData()
if xysize == -1 and zsize == -1:
params = {
"size": imgsize,
"npad": npad,
"symmetry": symmetry,
"fftvol": fftvol,
"weight": weight,
"snr": snr,
}
r = EMAN2_cppwrap.Reconstructors.get("nn4", params)
else:
if xysize != -1 and zsize != -1:
rx = old_div(float(xysize), imgsize)
ry = old_div(float(xysize), imgsize)
rz = old_div(float(zsize), imgsize)
elif xysize != -1:
rx = old_div(float(xysize), imgsize)
ry = old_div(float(xysize), imgsize)
rz = 1.0
else:
rx = 1.0
ry = 1.0
rz = old_div(float(zsize), imgsize)
params = {
"sizeprojection": imgsize,
"npad": npad,
"symmetry": symmetry,
"fftvol": fftvol,
"weight": weight,
"xratio": rx,
"yratio": ry,
"zratio": rz,
}
r = EMAN2_cppwrap.Reconstructors.get("nn4_rect", params)
r.setup()
if not (finfo is None):
nimg = 0
while prjlist.goToNext():
prj = prjlist.image()
if dopad:
prj = sp_utilities.pad(prj, imgsize, imgsize, 1, "circumference")
insert_slices(r, prj)
if not (finfo is None):
nimg += 1
finfo.write("Image %4d inserted.\n" % (nimg))
finfo.flush()
if not (finfo is None):
finfo.write("Begin reducing ...\n")
finfo.flush()
sp_utilities.reduce_EMData_to_root(fftvol, myid, comm=mpi_comm)
sp_utilities.reduce_EMData_to_root(weight, myid, comm=mpi_comm)
if myid == 0:
dummy = r.finish(True)
else:
if xysize == -1 and zsize == -1:
fftvol = sp_utilities.model_blank(imgsize, imgsize, imgsize)
else:
if zsize == -1:
fftvol = sp_utilities.model_blank(xysize, xysize, imgsize)
elif xysize == -1:
fftvol = sp_utilities.model_blank(imgsize, imgsize, zsize)
else:
fftvol = sp_utilities.model_blank(xysize, xysize, zsize)
return fftvol
def main():
arglist = []
for arg in sys.argv:
arglist.append(arg)
progname = optparse.os.path.basename(arglist[0])
usage = (
progname
+ " prj_stack volume [begin end step] --CTF --npad=ntimes_padding --list=file --group=ID --snr=SNR --sym=symmetry --verbose=(0|1) --xysize --MPI"
)
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option(
"--CTF", action="store_true", default=False, help="apply CTF correction"
)
parser.add_option("--snr", type="float", default=1.0, help="Signal-to-Noise Ratio")
parser.add_option("--sym", type="string", default="c1", help="symmetry")
parser.add_option(
"--list",
type="string",
help="file with list of images to be used in the first column",
)
parser.add_option(
"--group",
type="int",
default=-1,
help="perform reconstruction using images for a given group number (group is attribute in the header)",
)
parser.add_option(
"--MPI", action="store_true", default=False, help="use MPI version "
)
parser.add_option(
"--npad", type="int", default=2, help="number of times padding (default 2)"
)
parser.add_option(
"--verbose",
type="int",
default=0,
help="verbose level: 0 no verbose, 1 verbose",
)
parser.add_option(
"--xysize", type="int", default=-1, help="user expected size at xy direction"
)
parser.add_option(
"--zsize", type="int", default=-1, help="user expected size at z direction"
)
parser.add_option(
"--smearstep",
type="float",
default=0.0,
help="Rotational smear step (default 0.0, no smear)",
)
parser.add_option(
"--interpolation_method",
type="string",
default="4nn",
help="4nn, or tril: nearest neighbor, or trilinear interpolation",
)
parser.add_option(
"--niter",
type="int",
default=10,
help="number of iterations for iterative reconstruction",
)
parser.add_option(
"--upweighted",
action="store_true",
default=False,
help="apply background noise",
)
parser.add_option(
"--compensate",
action="store_true",
default=False,
help="compensate in reconstruction",
)
parser.add_option(
"--chunk_id",
type="int",
default=-1,
help="reconstruct both odd and even groups of particles",
)
parser.add_option(
"--target_window_size",
type="int",
default=-1,
help=" size of the targeted reconstruction ",
)
(options, args) = parser.parse_args(arglist[1:])
if sp_global_def.CACHE_DISABLE:
sp_utilities.disable_bdb_cache()
if len(args) == 2:
prj_stack = args[0]
vol_stack = args[1]
nimage = EMAN2_cppwrap.EMUtil.get_image_count(prj_stack)
pid_list = list(range(0, nimage))
elif len(args) == 5:
prj_stack = args[0]
vol_stack = args[1]
begin = string.atoi(args[2])
end = string.atoi(args[3])
step = string.atoi(args[4])
pid_list = list(range(begin, end, step))
else:
sp_global_def.ERROR("Incomplete list of arguments")
return
if options.list and options.group > -1:
sp_global_def.ERROR("options group and list cannot be used together")
return
sp_global_def.BATCH = True
if options.interpolation_method == "4nn":
sp_applications.recons3d_n(
prj_stack,
pid_list,
vol_stack,
options.CTF,
options.snr,
1,
options.npad,
options.sym,
options.list,
options.group,
options.verbose,
options.MPI,
options.xysize,
options.zsize,
options.smearstep,
options.upweighted,
options.compensate,
options.chunk_id,
)
elif options.interpolation_method == "tril":
if options.MPI is False:
sp_global_def.ERROR(
"Trilinear interpolation reconstruction has MPI version only!"
)
return
sp_applications.recons3d_trl_MPI(
prj_stack,
pid_list,
vol_stack,
options.CTF,
options.snr,
1,
options.npad,
options.sym,
options.verbose,
options.niter,
options.compensate,
options.target_window_size,
)
else:
sp_global_def.ERROR(
"Wrong interpolation method. The current options are 4nn, and tril. 4nn is the default one."
)
return
sp_global_def.write_command(optparse.os.path.dirname(vol_stack))
sp_global_def.BATCH = False
def import_params(params_file, dim):
"""
Import 2D or 3D parameters.
The 2d params file has the format:
angle, shift x, shift y, mirror
The 3d params file has the format:
angle_phi, angle_theta, angle_psi, tx, ty
Arguments:
params_2d_file - File name of the 2d parameter files
Returns:
parameter array
"""
if dim == "2d":
dtype_import_list = [
("angle_psi", float),
("shift_x", float),
("shift_y", float),
("mirror", int),
]
sxprint("What happens with mirror?")
elif dim == "3d":
dtype_import_list = [
("angle_rot", float),
("angle_theta", float),
("angle_psi", float),
("shift_x", float),
("shift_y", float),
]
else:
sp_global_def.ERROR(
"Dimension {0} not supported. Only '2d' and '3d' are supported.".
format(dim),
"sp_sphire2relion",
)
input_data = np.genfromtxt(params_file, dtype=dtype_import_list)
dtype_output_list = [
("_rlnOriginX", float),
("_rlnOriginY", float),
("_rlnAngleRot", float),
("_rlnAngleTilt", float),
("_rlnAnglePsi", float),
]
params_array = np.empty(len(input_data), dtype=sorted(dtype_output_list))
params_array["_rlnOriginX"] = input_data["shift_x"]
params_array["_rlnOriginY"] = input_data["shift_y"]
params_array["_rlnAnglePsi"] = input_data["angle_psi"]
if dim == "2d":
params_array["_rlnAngleTilt"] = 0
params_array["_rlnAngleRot"] = 0
elif dim == "3d":
params_array["_rlnAngleTilt"] = input_data["angle_theta"]
params_array["_rlnAngleRot"] = input_data["angle_rot"]
return params_array
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + " stack outdir <maskfile> --ir=inner_radius --ou=outer_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translation_step --dst=delta --center=center --maxit=max_iteration --CTF --snr=SNR --Fourvar=Fourier_variance --Ng=group_number --Function=user_function_name --CUDA --GPUID --MPI"
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--ir", type="float", default=1, help="inner radius for rotational correlation > 0 (set to 1)")
parser.add_option("--ou", type="float", default=-1, help="outer radius for rotational correlation < nx/2-1 (set to the radius of the particle)")
parser.add_option("--rs", type="float", default=1, help="step between rings in rotational correlation > 0 (set to 1)" )
parser.add_option("--xr", type="string", default="4 2 1 1", help="range for translation search in x direction, search is +/xr ")
parser.add_option("--yr", type="string", default="-1", help="range for translation search in y direction, search is +/yr ")
parser.add_option("--ts", type="string", default="2 1 0.5 0.25",help="step of translation search in both directions")
parser.add_option("--nomirror", action="store_true", default=False, help="Disable checking mirror orientations of images (default False)")
parser.add_option("--dst", type="float", default=0.0, help="delta")
parser.add_option("--center", type="float", default=-1, help="-1.average center method; 0.not centered; 1.phase approximation; 2.cc with Gaussian function; 3.cc with donut-shaped image 4.cc with user-defined reference 5.cc with self-rotated average")
parser.add_option("--maxit", type="float", default=0, help="maximum number of iterations (0 means the maximum iterations is 10, but it will automatically stop should the criterion falls")
parser.add_option("--CTF", action="store_true", default=False, help="use CTF correction during alignment")
parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio of the data (set to 1.0)")
parser.add_option("--Fourvar", action="store_true", default=False, help="compute Fourier variance")
#parser.add_option("--Ng", type="int", default=-1, help="number of groups in the new CTF filteration")
parser.add_option("--function", type="string", default="ref_ali2d", help="name of the reference preparation function (default ref_ali2d)")
#parser.add_option("--CUDA", action="store_true", default=False, help="use CUDA program")
#parser.add_option("--GPUID", type="string", default="", help="ID of GPUs available")
parser.add_option("--MPI", action="store_true", default=False, help="use MPI version ")
parser.add_option("--rotational", action="store_true", default=False, help="rotational alignment with optional limited in-plane angle, the parameters are: ir, ou, rs, psi_max, mode(F or H), maxit, orient, randomize")
parser.add_option("--psi_max", type="float", default=180.0, help="psi_max")
parser.add_option("--mode", type="string", default="F", help="Full or Half rings, default F")
parser.add_option("--randomize",action="store_true", default=False, help="randomize initial rotations (suboption of friedel, default False)")
parser.add_option("--orient", action="store_true", default=False, help="orient images such that the average is symmetric about x-axis, for layer lines (suboption of friedel, default False)")
parser.add_option("--template", type="string", default=None, help="2D alignment will be initialized using the template provided (only non-MPI version, default None)")
parser.add_option("--random_method", type="string", default="", help="use SHC or SCF (default standard method)")
(options, args) = parser.parse_args()
if len(args) < 2 or len(args) > 3:
sxprint( "Usage: " + usage )
sxprint( "Please run \'" + progname + " -h\' for detailed options" )
sp_global_def.ERROR( "Invalid number of parameters used. Please see usage information above." )
return
elif(options.rotational):
from sp_applications import ali2d_rotationaltop
sp_global_def.BATCH = True
ali2d_rotationaltop(args[1], args[0], options.randomize, options.orient, options.ir, options.ou, options.rs, options.psi_max, options.mode, options.maxit)
else:
if args[1] == 'None':
outdir = None
else:
outdir = args[1]
if len(args) == 2:
mask = None
else:
mask = args[2]
if sp_global_def.CACHE_DISABLE:
from sp_utilities import disable_bdb_cache
disable_bdb_cache()
sp_global_def.BATCH = True
if options.MPI:
from sp_applications import ali2d_base
from mpi import mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
if(myid == main_node):
import subprocess
from sp_logger import Logger, BaseLogger_Files
# Create output directory
log = Logger(BaseLogger_Files())
log.prefix = os.path.join(outdir)
cmd = "mkdir "+log.prefix
outcome = subprocess.call(cmd, shell=True)
log.prefix += "/"
else:
outcome = 0
log = None
from sp_utilities import bcast_number_to_all
outcome = bcast_number_to_all(outcome, source_node = main_node)
if(outcome == 1):
sp_global_def.ERROR( "Output directory exists, please change the name and restart the program", myid=myid )
dummy = ali2d_base(args[0], outdir, mask, options.ir, options.ou, options.rs, options.xr, options.yr, \
options.ts, options.nomirror, options.dst, \
options.center, options.maxit, options.CTF, options.snr, options.Fourvar, \
options.function, random_method = options.random_method, log = log, \
number_of_proc = number_of_proc, myid = myid, main_node = main_node, mpi_comm = MPI_COMM_WORLD,\
write_headers = True)
else:
sxprint( " Non-MPI is no more in use, try MPI option, please." )
"""
from sp_applications import ali2d
ali2d(args[0], outdir, mask, options.ir, options.ou, options.rs, options.xr, options.yr, \
options.ts, options.nomirror, options.dst, \
options.center, options.maxit, options.CTF, options.snr, options.Fourvar, \
-1, options.function, False, "", options.MPI, \
options.template, random_method = options.random_method)
"""
sp_global_def.BATCH = False
def main():
arglist = []
for arg in sys.argv:
arglist.append(arg)
progname = os.path.basename(arglist[0])
usage = progname + """ firstvolume secondvolume maskfile directory --prefix --wn --step --cutoff --radius --fsc --res_overall --out_ang_res --apix --MPI
Compute local resolution in real space within area outlined by the maskfile and within regions wn x wn x wn
"""
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option("--prefix",
type="str",
default='localres',
help="Prefix for the output files. (default localres)")
parser.add_option(
"--wn",
type="int",
default=7,
help=
"Size of window within which local real-space FSC is computed. (default 7)"
)
parser.add_option(
"--step",
type="float",
default=1.0,
help="Shell step in Fourier size in pixels. (default 1.0)")
parser.add_option("--cutoff",
type="float",
default=0.143,
help="Resolution cut-off for FSC. (default 0.143)")
parser.add_option(
"--radius",
type="int",
default=-1,
help=
"If there is no maskfile, sphere with r=radius will be used. By default, the radius is nx/2-wn (default -1)"
)
parser.add_option(
"--fsc",
type="string",
default=None,
help=
"Save overall FSC curve (might be truncated). By default, the program does not save the FSC curve. (default none)"
)
parser.add_option(
"--res_overall",
type="float",
default=-1.0,
help=
"Overall resolution at the cutoff level estimated by the user [abs units]. (default None)"
)
parser.add_option(
"--out_ang_res",
action="store_true",
default=False,
help=
"Additionally creates a local resolution file in Angstroms. (default False)"
)
parser.add_option(
"--apix",
type="float",
default=1.0,
help=
"Pixel size in Angstrom. Effective only with --out_ang_res options. (default 1.0)"
)
parser.add_option("--MPI",
action="store_true",
default=False,
help="Use MPI version.")
(options, args) = parser.parse_args(arglist[1:])
if len(args) < 3 or len(args) > 4:
sxprint("Usage: " + usage)
ERROR(
"Invalid number of parameters used. Please see usage information above."
)
return
if sp_global_def.CACHE_DISABLE:
sp_utilities.disable_bdb_cache()
res_overall = options.res_overall
if options.MPI:
number_of_proc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
main_node = 0
sp_global_def.MPI = True
cutoff = options.cutoff
nk = int(options.wn)
if (myid == main_node):
#print sys.argv
vi = sp_utilities.get_im(sys.argv[1])
ui = sp_utilities.get_im(sys.argv[2])
nx = vi.get_xsize()
ny = vi.get_ysize()
nz = vi.get_zsize()
dis = [nx, ny, nz]
else:
dis = [0, 0, 0, 0]
sp_global_def.BATCH = True
dis = sp_utilities.bcast_list_to_all(dis, myid, source_node=main_node)
if (myid != main_node):
nx = int(dis[0])
ny = int(dis[1])
nz = int(dis[2])
vi = sp_utilities.model_blank(nx, ny, nz)
ui = sp_utilities.model_blank(nx, ny, nz)
if len(args) == 3:
m = sp_utilities.model_circle((min(nx, ny, nz) - nk) // 2, nx, ny,
nz)
outdir = args[2]
elif len(args) == 4:
if (myid == main_node):
m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5)
else:
m = sp_utilities.model_blank(nx, ny, nz)
outdir = args[3]
if os.path.exists(outdir) and myid == 0:
sp_global_def.ERROR('Output directory already exists!')
elif myid == 0:
os.makedirs(outdir)
sp_global_def.write_command(outdir)
sp_utilities.bcast_EMData_to_all(m, myid, main_node)
"""Multiline Comment0"""
freqvol, resolut = sp_statistics.locres(vi, ui, m, nk, cutoff,
options.step, myid, main_node,
number_of_proc)
if (myid == 0):
# Remove outliers based on the Interquartile range
output_volume(freqvol, resolut, options.apix, outdir,
options.prefix, options.fsc, options.out_ang_res, nx,
ny, nz, res_overall)
else:
cutoff = options.cutoff
vi = sp_utilities.get_im(args[0])
ui = sp_utilities.get_im(args[1])
nn = vi.get_xsize()
nx = nn
ny = nn
nz = nn
nk = int(options.wn)
if len(args) == 3:
m = sp_utilities.model_circle((nn - nk) // 2, nn, nn, nn)
outdir = args[2]
elif len(args) == 4:
m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5)
outdir = args[3]
if os.path.exists(outdir):
sp_global_def.ERROR('Output directory already exists!')
else:
os.makedirs(outdir)
sp_global_def.write_command(outdir)
mc = sp_utilities.model_blank(nn, nn, nn, 1.0) - m
vf = sp_fundamentals.fft(vi)
uf = sp_fundamentals.fft(ui)
"""Multiline Comment1"""
lp = int(nn / 2 / options.step + 0.5)
step = 0.5 / lp
freqvol = sp_utilities.model_blank(nn, nn, nn)
resolut = []
for i in range(1, lp):
fl = step * i
fh = fl + step
#print(lp,i,step,fl,fh)
v = sp_fundamentals.fft(sp_filter.filt_tophatb(vf, fl, fh))
u = sp_fundamentals.fft(sp_filter.filt_tophatb(uf, fl, fh))
tmp1 = EMAN2_cppwrap.Util.muln_img(v, v)
tmp2 = EMAN2_cppwrap.Util.muln_img(u, u)
do = EMAN2_cppwrap.Util.infomask(
sp_morphology.square_root(
sp_morphology.threshold(
EMAN2_cppwrap.Util.muln_img(tmp1, tmp2))), m, True)[0]
tmp3 = EMAN2_cppwrap.Util.muln_img(u, v)
dp = EMAN2_cppwrap.Util.infomask(tmp3, m, True)[0]
resolut.append([i, (fl + fh) / 2.0, dp / do])
tmp1 = EMAN2_cppwrap.Util.box_convolution(tmp1, nk)
tmp2 = EMAN2_cppwrap.Util.box_convolution(tmp2, nk)
tmp3 = EMAN2_cppwrap.Util.box_convolution(tmp3, nk)
EMAN2_cppwrap.Util.mul_img(tmp1, tmp2)
tmp1 = sp_morphology.square_root(sp_morphology.threshold(tmp1))
EMAN2_cppwrap.Util.mul_img(tmp1, m)
EMAN2_cppwrap.Util.add_img(tmp1, mc)
EMAN2_cppwrap.Util.mul_img(tmp3, m)
EMAN2_cppwrap.Util.add_img(tmp3, mc)
EMAN2_cppwrap.Util.div_img(tmp3, tmp1)
EMAN2_cppwrap.Util.mul_img(tmp3, m)
freq = (fl + fh) / 2.0
bailout = True
for x in range(nn):
for y in range(nn):
for z in range(nn):
if (m.get_value_at(x, y, z) > 0.5):
if (freqvol.get_value_at(x, y, z) == 0.0):
if (tmp3.get_value_at(x, y, z) < cutoff):
freqvol.set_value_at(x, y, z, freq)
bailout = False
else:
bailout = False
if (bailout): break
#print(len(resolut))
# remove outliers
output_volume(freqvol, resolut, options.apix, outdir, options.prefix,
options.fsc, options.out_ang_res, nx, ny, nz,
res_overall)
def recons3d_4nn_ctf_MPI(
myid,
prjlist,
snr=1.0,
sign=1,
symmetry="c1",
finfo=None,
npad=2,
xysize=-1,
zsize=-1,
mpi_comm=None,
smearstep=0.0,
):
"""
recons3d_4nn_ctf - calculate CTF-corrected 3-D reconstruction from a set of projections using three Eulerian angles, two shifts, and CTF settings for each projeciton image
Input
stack: name of the stack file containing projection data, projections have to be squares
list_proj: list of projections to be included in the reconstruction or image iterator
snr: Signal-to-Noise Ratio of the data
sign: sign of the CTF
symmetry: point-group symmetry to be enforced, each projection will enter the reconstruction in all symmetry-related directions.
"""
if mpi_comm == None:
mpi_comm = mpi.MPI_COMM_WORLD
if type(prjlist) == list:
prjlist = sp_utilities.iterImagesList(prjlist)
if not prjlist.goToNext():
sp_global_def.ERROR("empty input list", "recons3d_4nn_ctf_MPI", 1)
imgsize = prjlist.image().get_xsize()
if prjlist.image().get_ysize() != imgsize:
imgsize = max(imgsize, prjlist.image().get_ysize())
dopad = True
else:
dopad = False
prjlist.goToPrev()
fftvol = EMAN2_cppwrap.EMData()
if smearstep > 0.0:
# if myid == 0: print " Setting smear in prepare_recons_ctf"
ns = 1
smear = []
for j in range(-ns, ns + 1):
if j != 0:
for i in range(-ns, ns + 1):
for k in range(-ns, ns + 1):
smear += [
i * smearstep, j * smearstep, k * smearstep, 1.0
]
# Deal with theta = 0.0 cases
prj = []
for i in range(-ns, ns + 1):
for k in range(-ns, ns + 1):
prj.append(i + k)
for i in range(-2 * ns, 2 * ns + 1, 1):
smear += [i * smearstep, 0.0, 0.0, float(prj.count(i))]
# if myid == 0: print " Smear ",smear
fftvol.set_attr("smear", smear)
weight = EMAN2_cppwrap.EMData()
if xysize == -1 and zsize == -1:
params = {
"size": imgsize,
"npad": npad,
"snr": snr,
"sign": sign,
"symmetry": symmetry,
"fftvol": fftvol,
"weight": weight,
}
r = EMAN2_cppwrap.Reconstructors.get("nn4_ctf", params)
else:
if xysize != -1 and zsize != -1:
rx = old_div(float(xysize), imgsize)
ry = old_div(float(xysize), imgsize)
rz = old_div(float(zsize), imgsize)
elif xysize != -1:
rx = old_div(float(xysize), imgsize)
ry = old_div(float(xysize), imgsize)
rz = 1.0
else:
rx = 1.0
ry = 1.0
rz = old_div(float(zsize), imgsize)
# There is an error here with sizeprojection PAP 10/22/2014
params = {
"size": sizeprojection,
"npad": npad,
"snr": snr,
"sign": sign,
"symmetry": symmetry,
"fftvol": fftvol,
"weight": weight,
"xratio": rx,
"yratio": ry,
"zratio": rz,
}
r = EMAN2_cppwrap.Reconstructors.get("nn4_ctf_rect", params)
r.setup()
# if not (finfo is None):
nimg = 0
while prjlist.goToNext():
prj = prjlist.image()
if dopad:
prj = sp_utilities.pad(prj, imgsize, imgsize, 1, "circumference")
# if params:
insert_slices(r, prj)
if not (finfo is None):
nimg += 1
finfo.write(" %4d inserted\n" % (nimg))
finfo.flush()
del sp_utilities.pad
if not (finfo is None):
finfo.write("begin reduce\n")
finfo.flush()
sp_utilities.reduce_EMData_to_root(fftvol, myid, comm=mpi_comm)
sp_utilities.reduce_EMData_to_root(weight, myid, comm=mpi_comm)
if not (finfo is None):
finfo.write("after reduce\n")
finfo.flush()
if myid == 0:
dummy = r.finish(True)
else:
if xysize == -1 and zsize == -1:
fftvol = sp_utilities.model_blank(imgsize, imgsize, imgsize)
else:
if zsize == -1:
fftvol = sp_utilities.model_blank(xysize, xysize, imgsize)
elif xysize == -1:
fftvol = sp_utilities.model_blank(imgsize, imgsize, zsize)
else:
fftvol = sp_utilities.model_blank(xysize, xysize, zsize)
return fftvol
def recons3d_4nn(
stack_name,
list_proj=[],
symmetry="c1",
npad=4,
snr=None,
weighting=1,
varsnr=False,
xysize=-1,
zsize=-1,
):
"""
Perform a 3-D reconstruction using Pawel's FFT Back Projection algorithm.
Input:
stack_name - name of the file with projection data.
list_proj - list of projections to be used in the reconstruction
symmetry - Point group of the target molecule (defaults to "C1")
npad -
Angles and shifts are passed in the file header as set_attr. Keywords are phi, theta, psi, sx, sy
Return: 3D reconstructed volume image
Usage:
vol = recons3d_4nn(filepattern, list_proj, symmetry)
"""
if list_proj == []:
if type(stack_name) == bytes:
nima = EMAN2_cppwrap.EMUtil.get_image_count(stack_name)
else:
nima = len(stack_name)
list_proj = list(range(nima))
# read first image to determine the size to use
if type(stack_name) == bytes:
proj = EMAN2_cppwrap.EMData()
proj.read_image(stack_name, list_proj[0])
else:
proj = stack_name[list_proj[0]].copy()
size = proj.get_xsize()
# sanity check -- image must be square
if size != proj.get_ysize():
sp_global_def.ERROR("input data has to be square", "recons3d_4nn", 1)
# reconstructor
fftvol = EMAN2_cppwrap.EMData()
weight = EMAN2_cppwrap.EMData()
params = {
"npad": npad,
"symmetry": symmetry,
"weighting": weighting,
"fftvol": fftvol,
"weight": weight,
}
if xysize == -1 and zsize == -1:
params["size"] = size
if snr != None:
params["snr"] = snr
# params["varsnr"] = int(varsnr)
r = EMAN2_cppwrap.Reconstructors.get("nn4", params)
else:
if xysize != -1 and zsize != -1:
rx = old_div(float(xysize), size)
ry = old_div(float(xysize), size)
rz = old_div(float(zsize), size)
elif xysize != -1:
rx = old_div(float(xysize), size)
ry = old_div(float(xysize), size)
rz = 1.0
else:
rx = 1.0
ry = 1.0
rz = old_div(float(zsize), size)
if snr is None:
params["sizeprojection"] = size
params["xratio"] = rx
params["yratio"] = ry
params["zratio"] = rz
else:
params["sizeprojection"] = size
params["snr"] = snr
params["varsnr"] = int(varsnr)
params["xratio"] = rx
params["yratio"] = ry
params["zratio"] = rz
r = EMAN2_cppwrap.Reconstructors.get("nn4_rect", params)
r.setup()
if type(stack_name) == bytes:
for i in range(len(list_proj)):
proj.read_image(stack_name, list_proj[i])
# horatio active_refactoring Jy51i1EwmLD4tWZ9_00000_1
# active = proj.get_attr_default('active', 1)
# if active == 1:
# insert_slices(r, proj)
insert_slices(r, proj)
else:
for i in list_proj:
# horatio active_refactoring Jy51i1EwmLD4tWZ9_00000_1
# active = stack_name[i].get_attr_default('active', 1)
# if active == 1:
# insert_slices(r, stack_name[i])
insert_slices(r, stack_name[i])
dummy = r.finish(True)
return fftvol
def recons3d_4nnw_MPI(
myid,
prjlist,
bckgdata,
snr=1.0,
sign=1,
symmetry="c1",
finfo=None,
npad=2,
xysize=-1,
zsize=-1,
mpi_comm=None,
smearstep=0.0,
fsc=None,
):
"""
recons3d_4nn_ctf - calculate CTF-corrected 3-D reconstruction from a set of projections using three Eulerian angles, two shifts, and CTF settings for each projeciton image
Input
stack: name of the stack file containing projection data, projections have to be squares
prjlist: list of projections to be included in the reconstruction or image iterator
bckgdata = [get_im("tsd.hdf"),read_text_file("data_stamp.txt")]
snr: Signal-to-Noise Ratio of the data
sign: sign of the CTF
symmetry: point-group symmetry to be enforced, each projection will enter the reconstruction in all symmetry-related directions.
"""
pass # IMPORTIMPORTIMPORT from sp_utilities import reduce_EMData_to_root, pad
pass # IMPORTIMPORTIMPORT from EMAN2 import Reconstructors
pass # IMPORTIMPORTIMPORT from sp_utilities import iterImagesList, set_params_proj, model_blank
pass # IMPORTIMPORTIMPORT from mpi import MPI_COMM_WORLD
pass # IMPORTIMPORTIMPORT import types
if mpi_comm == None:
mpi_comm = mpi.MPI_COMM_WORLD
if type(prjlist) == list:
prjlist = sp_utilities.iterImagesList(prjlist)
if not prjlist.goToNext():
sp_global_def.ERROR("empty input list", "recons3d_4nnw_MPI", 1)
imgsize = prjlist.image().get_xsize()
if prjlist.image().get_ysize() != imgsize:
imgsize = max(imgsize, prjlist.image().get_ysize())
dopad = True
else:
dopad = False
prjlist.goToPrev()
# Do the FSC shtick.
bnx = old_div(imgsize * npad, 2) + 1
if fsc:
pass # IMPORTIMPORTIMPORT from math import sqrt
pass # IMPORTIMPORTIMPORT from sp_utilities import reshape_1d
t = [0.0] * len(fsc)
for i in range(len(fsc)):
t[i] = min(max(fsc[i], 0.0), 0.999)
t = sp_utilities.reshape_1d(t, len(t), npad * len(t))
refvol = sp_utilities.model_blank(bnx, 1, 1, 0.0)
for i in range(len(fsc)):
refvol.set_value_at(i, t[i])
else:
refvol = sp_utilities.model_blank(bnx, 1, 1, 1.0)
refvol.set_attr("fudge", 1.0)
fftvol = EMAN2_cppwrap.EMData()
weight = EMAN2_cppwrap.EMData()
if smearstep > 0.0:
# if myid == 0: print " Setting smear in prepare_recons_ctf"
ns = 1
smear = []
for j in range(-ns, ns + 1):
if j != 0:
for i in range(-ns, ns + 1):
for k in range(-ns, ns + 1):
smear += [
i * smearstep, j * smearstep, k * smearstep, 1.0
]
# Deal with theta = 0.0 cases
prj = []
for i in range(-ns, ns + 1):
for k in range(-ns, ns + 1):
prj.append(i + k)
for i in range(-2 * ns, 2 * ns + 1, 1):
smear += [i * smearstep, 0.0, 0.0, float(prj.count(i))]
# if myid == 0: print " Smear ",smear
fftvol.set_attr("smear", smear)
if xysize == -1 and zsize == -1:
params = {
"size": imgsize,
"npad": npad,
"snr": snr,
"sign": sign,
"symmetry": symmetry,
"refvol": refvol,
"fftvol": fftvol,
"weight": weight,
}
r = EMAN2_cppwrap.Reconstructors.get("nn4_ctfw", params)
else:
if xysize != -1 and zsize != -1:
rx = old_div(float(xysize), imgsize)
ry = old_div(float(xysize), imgsize)
rz = old_div(float(zsize), imgsize)
elif xysize != -1:
rx = old_div(float(xysize), imgsize)
ry = old_div(float(xysize), imgsize)
rz = 1.0
else:
rx = 1.0
ry = 1.0
rz = old_div(float(zsize), imgsize)
# There is an error here with sizeprojection PAP 10/22/2014
params = {
"size": sizeprojection,
"npad": npad,
"snr": snr,
"sign": sign,
"symmetry": symmetry,
"fftvol": fftvol,
"weight": weight,
"xratio": rx,
"yratio": ry,
"zratio": rz,
}
r = EMAN2_cppwrap.Reconstructors.get("nn4_ctf_rect", params)
r.setup()
# from utilities import model_blank, get_im, read_text_file
# bckgdata = [get_im("tsd.hdf"),read_text_file("data_stamp.txt")]
nnx = bckgdata[0].get_xsize()
nny = bckgdata[0].get_ysize()
bckgnoise = []
for i in range(nny):
prj = sp_utilities.model_blank(nnx)
for k in range(nnx):
prj[k] = bckgdata[0].get_value_at(k, i)
bckgnoise.append(prj)
datastamp = bckgdata[1]
if not (finfo is None):
nimg = 0
while prjlist.goToNext():
prj = prjlist.image()
try:
stmp = old_div(nnx, 0)
stmp = prj.get_attr("ptcl_source_image")
except:
try:
stmp = prj.get_attr("ctf")
stmp = round(stmp.defocus, 4)
except:
sp_global_def.ERROR(
"Either ptcl_source_image or ctf has to be present in the header.",
"recons3d_4nnw_MPI",
1,
myid,
)
try:
indx = datastamp.index(stmp)
except:
sp_global_def.ERROR("Problem with indexing ptcl_source_image.",
"recons3d_4nnw_MPI", 1, myid)
if dopad:
prj = sp_utilities.pad(prj, imgsize, imgsize, 1, "circumference")
prj.set_attr("bckgnoise", bckgnoise[indx])
insert_slices(r, prj)
if not (finfo is None):
nimg += 1
finfo.write(" %4d inserted\n" % (nimg))
finfo.flush()
del sp_utilities.pad
if not (finfo is None):
finfo.write("begin reduce\n")
finfo.flush()
sp_utilities.reduce_EMData_to_root(fftvol, myid, comm=mpi_comm)
sp_utilities.reduce_EMData_to_root(weight, myid, comm=mpi_comm)
if not (finfo is None):
finfo.write("after reduce\n")
finfo.flush()
if myid == 0:
dummy = r.finish(True)
else:
pass # IMPORTIMPORTIMPORT from sp_utilities import model_blank
if xysize == -1 and zsize == -1:
fftvol = sp_utilities.model_blank(imgsize, imgsize, imgsize)
else:
if zsize == -1:
fftvol = sp_utilities.model_blank(xysize, xysize, imgsize)
elif xysize == -1:
fftvol = sp_utilities.model_blank(imgsize, imgsize, zsize)
else:
fftvol = sp_utilities.model_blank(xysize, xysize, zsize)
return fftvol
def _main_():
_, child_argparser = setup_argparser()
# These global variables are kind of ugly, but are necessary in python2. Will be removed in
# python 3.
global STACK_FILE_PATH
global PROJECTION_PARAMETERS
global VOLUME1
global VOLUME2
global DEFOCUS_SEARCH_RANGE
global DEFOCUS_STEP_SIZE
global MASK_VOLUME
global RESOLUTION
global PIXEL_SIZE
global HALF_MAP_ASSIGNEMENTS
args = child_argparser.parse_args()
if "meridien" in sys.argv[1]:
meridien_path = args.meridien_path
files = sp_ctf_refine_io.read_meridien_data(meridien_path)
volume1_file_path = files["first_halfmap"]
volume2_file_path = files["second_halfmap"]
chunk_file_path = files["chunk1"]
params_file_path = files["final_params"]
else:
volume1_file_path = args.volume
volume2_file_path = args.volume2
params_file_path = args.params_path
chunk_file_path = args.chunk
if volume2_file_path is None and chunk_file_path is not None:
sp_global_def.ERROR(
"If chunk file is specified, you need to specify a second volume (-v2)"
)
STACK_FILE_PATH = args.inputstack # "bdb:name"
mask_file_path = args.mask
DEFOCUS_SEARCH_RANGE = args.range
DEFOCUS_STEP_SIZE = args.delta
output_folder = args.outputdir
if os.path.exists(output_folder):
sp_global_def.ERROR("Output folder already exists. Stop execution.")
else:
os.makedirs(output_folder)
sp_global_def.write_command(output_folder)
output_virtual_stack_path = "bdb:" + os.path.join(output_folder, "ctf_refined")
output_stats_path = os.path.join(output_folder, "statistics")
number_of_particles_to_read = args.number_particles
volume_nominal_resolution = args.resolution
RESOLUTION = volume_nominal_resolution
PIXEL_SIZE = args.apix
PROJECTION_PARAMETERS = sp_ctf_refine_io.read_meridien_params(params_file_path)
num_cpu = multiprocessing.cpu_count() - 1
volume1, volume2, MASK_VOLUME = sp_ctf_refine_io.read_volume(
path_vol_1=volume1_file_path,
path_vol_2=volume2_file_path,
path_mask=mask_file_path,
resolution=volume_nominal_resolution,
pixel_size=PIXEL_SIZE,
)
VOLUME1 = volume1
VOLUME2 = volume2
particle_chunks, number_of_particles = indices_to_chunks(
STACK_FILE_PATH,
chunk_size=100,
number_of_particles_to_read=number_of_particles_to_read,
)
if chunk_file_path:
HALF_MAP_ASSIGNEMENTS = get_half_map_assigments_per_particle(
stack_path=STACK_FILE_PATH,
chunk_path=chunk_file_path,
number_of_particles_to_read=number_of_particles_to_read,
)
sp_global_def.sxprint("####Start refinement####")
start = time.time()
# for chunk in particle_chunks:
# refine_set(chunk)
#####################################################################
# Python 2 workaround to get rid of a memory leak.
# Pool eats up memory and only give it back after it returns.
# So let it return more often by deviding chunks into chunks :-)
#####################################################################
num_chunks = len(particle_chunks)
refinement_results = []
with tqdm(total=num_chunks, file=sys.stdout) as pbar:
for i in range(0, num_chunks, num_cpu):
subset_chunk = particle_chunks[i: (i + num_cpu)]
pool = multiprocessing.Pool(num_cpu)
refinement_result = pool.map_async(refine_set, subset_chunk, chunksize=1)
pool.close()
# print_progress(refinement_result)
pool.join()
for res in refinement_result.get():
refinement_results.append(res)
pbar.update(len(subset_chunk))
#####################################################################
end = time.time()
sp_global_def.sxprint("Time for ", number_of_particles, " Particles:", end - start)
# Ouput results
refined_ctfs_as_list, refinement_results_per_micrograph = merge_ctf_refinement_results(
refinement_results=refinement_results
)
sp_ctf_refine_io.write_virtual_bdb_stack(
output_stack_path=output_virtual_stack_path,
origin_stack_path=STACK_FILE_PATH,
refined_ctfs=refined_ctfs_as_list,
number_of_particles=number_of_particles,
)
sp_global_def.sxprint("Write statistics...")
# WRITE STATISTICS
if not os.path.exists(output_stats_path):
os.makedirs(output_stats_path)
refinement_stats_per_micrograh = calc_statistics(refinement_results_per_micrograph)
sp_ctf_refine_io.write_statistics(output_stats_path, refinement_stats_per_micrograh)
# Estimate error Range
min_max_error, min_max_ratio = calculate_result_ranges(
refinement_results_per_micrograph
)
# Save particle plots
sp_global_def.sxprint("Write images...")
path_output_img = os.path.join(output_stats_path, "img/")
if not os.path.exists(path_output_img):
os.makedirs(path_output_img)
sp_ctf_refine_plotting.create_and_save_particle_plots(
path_output_img=path_output_img,
stack_file_path=STACK_FILE_PATH,
refinement_results_per_micrograph=refinement_results_per_micrograph,
min_max_error=min_max_error,
min_max_ratio=min_max_ratio,
)
sp_global_def.sxprint("Write other...")
refinement_results_matrix = get_refinement_results_matrix(
refinement_results_per_micrograph
)
particle_error_path = os.path.join(output_stats_path, "particle_results.txt")
refinement_results_matrix = refinement_results_matrix[
refinement_results_matrix[:, 0].argsort()
]
np.savetxt(
particle_error_path,
refinement_results_matrix,
delimiter=",",
fmt=["%d", "%f", "%f", "%f"],
)
sp_global_def.sxprint("Done")
def main():
# Read arguments
args = argparser.parse_args()
architecture = args.architecture
particle_diameter = args.particle_diameter
trainging_dir = args.training_dir
annot_dir = args.annot_dir
input_size = args.input_size
output_dir = args.output_directory
if os.path.exists(output_dir):
sp_global_def.ERROR("Output folder already exists. Stop execution.")
else:
os.makedirs(output_dir)
sp_global_def.write_command(output_dir)
num_patches = args.num_patches
overlap_patches = args.overlap_patches
train_times = args.train_times
saved_weights_name = args.saved_weights_name
saved_weights_name = os.path.join(output_dir, saved_weights_name)
pretrained_weights_name = args.pretrained_weights_name
batch_size = args.batch_size
learning_rate = args.learning_rate
np_epoch = args.np_epoch
object_scale = args.object_scale
no_object_scale = args.no_object_scale
coord_scale = args.coord_scale
valid_image_dir = args.valid_image_dir
valid_annot_dir = args.valid_annot_dir
fine_tune = args.fine_tune
gpu_fraction = args.gpu_fraction
num_cpu = args.num_cpu
cryolo_train_path = args.cryolo_train_path
skiplowpass = args.skiplowpass
cutoff = args.cutoff
filtered_dir = args.filtered_dir
warmup = args.warmup
early_stop = int(args.early)
str_gpus = [str(entry).strip() for entry in args.gpu.split(",")]
arg_gpu = ' '.join(str_gpus)
# Create config file
model_dict = {'architecture': architecture,
'input_size': input_size,
'anchors': [particle_diameter, particle_diameter],
'overlap_patches': overlap_patches,
'max_box_per_image': 1000,
'num_patches': num_patches}
if not skiplowpass:
model_dict['filter'] = [cutoff,filtered_dir]
else:
if args.usejanni:
model_dict['filter'] = [args.janni_model, args.janni_overlap, args.janni_batches, filtered_dir]
train_dict = {'train_image_folder': trainging_dir,
'train_annot_folder': annot_dir,
'train_times': train_times,
'pretrained_weights': pretrained_weights_name,
'batch_size': batch_size,
'learning_rate': learning_rate,
'nb_epoch': np_epoch,
'warmup_epochs': 0,
'object_scale': object_scale,
'no_object_scale': no_object_scale,
'coord_scale': coord_scale,
'class_scale': 1.0,
"saved_weights_name": saved_weights_name,
"debug": True,
"log_path": "cryolo_logs/"
}
valid_dict = {'valid_image_folder': valid_image_dir,
'valid_annot_folder': valid_annot_dir,
'valid_times': 1
}
dict = {"model": model_dict, "train": train_dict, "valid": valid_dict}
path = os.path.join(output_dir,"config_yolo.json")
with open(path, 'w') as f:
dump(dict, f, ensure_ascii=False, indent=4)
# Run the training
config_argument = "-c=" + path
warmup_argument = "-w=" + str(warmup)
gpu_argument = "-g " + arg_gpu
early_stop = "-e=" + str(early_stop)
fine_tune_argument = ""
if fine_tune:
fine_tune_argument = "--fine_tune"
gpu_fraction_arg = "--gpu_fraction=1.0"
if gpu_fraction < 1.0 and gpu_fraction > 0.0:
gpu_fraction_arg = "--gpu_fraction=" + str(gpu_fraction)
num_cpu_arg = ""
if num_cpu != -1:
num_cpu_arg = "--num_cpu="+str(num_cpu)
cryolo_ex_pth = "cryolo_train.py"
if cryolo_train_path:
cryolo_ex_pth = cryolo_train_path
'''
if not fine_tune:
command = [cryolo_ex_pth, "-c=config_yolo.json", warmup_argument, gpu_argument, early_stop, gpu_fraction_arg]
if num_cpu != -1:
command.append(num_cpu_arg)
subprocess.check_call(
command)
'''
if fine_tune:
warmup_argument = "-w=0"
command = [cryolo_ex_pth, config_argument, warmup_argument, gpu_argument, early_stop, gpu_fraction_arg]
if fine_tune:
command.append(fine_tune_argument)
if num_cpu != -1:
command.append(num_cpu_arg)
subprocess.check_call(
command)
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + " proj_stack output_averages --MPI"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--img_per_group",
type="int",
default=100,
help="number of images per group")
parser.add_option("--radius",
type="int",
default=-1,
help="radius for alignment")
parser.add_option(
"--xr",
type="string",
default="2 1",
help="range for translation search in x direction, search is +/xr")
parser.add_option(
"--yr",
type="string",
default="-1",
help=
"range for translation search in y direction, search is +/yr (default = same as xr)"
)
parser.add_option(
"--ts",
type="string",
default="1 0.5",
help=
"step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional"
)
parser.add_option(
"--iter",
type="int",
default=30,
help="number of iterations within alignment (default = 30)")
parser.add_option(
"--num_ali",
type="int",
default=5,
help="number of alignments performed for stability (default = 5)")
parser.add_option("--thld_err",
type="float",
default=1.0,
help="threshold of pixel error (default = 1.732)")
parser.add_option(
"--grouping",
type="string",
default="GRP",
help=
"do grouping of projections: PPR - per projection, GRP - different size groups, exclusive (default), GEV - grouping equal size"
)
parser.add_option(
"--delta",
type="float",
default=-1.0,
help="angular step for reference projections (required for GEV method)"
)
parser.add_option(
"--fl",
type="float",
default=0.3,
help="cut-off frequency of hyperbolic tangent low-pass Fourier filter")
parser.add_option(
"--aa",
type="float",
default=0.2,
help="fall-off of hyperbolic tangent low-pass Fourier filter")
parser.add_option("--CTF",
action="store_true",
default=False,
help="Consider CTF correction during the alignment ")
parser.add_option("--MPI",
action="store_true",
default=False,
help="use MPI version")
(options, args) = parser.parse_args()
myid = mpi.mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi.mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
if len(args) == 2:
stack = args[0]
outdir = args[1]
else:
sp_global_def.ERROR("Incomplete list of arguments",
"sxproj_stability.main",
1,
myid=myid)
return
if not options.MPI:
sp_global_def.ERROR("Non-MPI not supported!",
"sxproj_stability.main",
1,
myid=myid)
return
if sp_global_def.CACHE_DISABLE:
from sp_utilities import disable_bdb_cache
disable_bdb_cache()
sp_global_def.BATCH = True
img_per_grp = options.img_per_group
radius = options.radius
ite = options.iter
num_ali = options.num_ali
thld_err = options.thld_err
xrng = get_input_from_string(options.xr)
if options.yr == "-1":
yrng = xrng
else:
yrng = get_input_from_string(options.yr)
step = get_input_from_string(options.ts)
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
else:
nima = 0
nx = 0
ny = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
if radius == -1: radius = nx / 2 - 2
mask = model_circle(radius, nx, nx)
st = time()
if options.grouping == "GRP":
if myid == main_node:
sxprint(" A ", myid, " ", time() - st)
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
proj_params = []
for i in range(nima):
dp = proj_attr[i].get_params("spider")
phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[
"psi"], -dp["tx"], -dp["ty"]
proj_params.append([phi, theta, psi, s2x, s2y])
# Here is where the grouping is done, I didn't put enough annotation in the group_proj_by_phitheta,
# So I will briefly explain it here
# proj_list : Returns a list of list of particle numbers, each list contains img_per_grp particle numbers
# except for the last one. Depending on the number of particles left, they will either form a
# group or append themselves to the last group
# angle_list : Also returns a list of list, each list contains three numbers (phi, theta, delta), (phi,
# theta) is the projection angle of the center of the group, delta is the range of this group
# mirror_list: Also returns a list of list, each list contains img_per_grp True or False, which indicates
# whether it should take mirror position.
# In this program angle_list and mirror list are not of interest.
proj_list_all, angle_list, mirror_list = group_proj_by_phitheta(
proj_params, img_per_grp=img_per_grp)
del proj_params
sxprint(" B number of groups ", myid, " ", len(proj_list_all),
time() - st)
mpi_barrier(MPI_COMM_WORLD)
# Number of groups, actually there could be one or two more groups, since the size of the remaining group varies
# we will simply assign them to main node.
n_grp = nima / img_per_grp - 1
# Divide proj_list_all equally to all nodes, and becomes proj_list
proj_list = []
for i in range(n_grp):
proc_to_stay = i % number_of_proc
if proc_to_stay == main_node:
if myid == main_node: proj_list.append(proj_list_all[i])
elif myid == main_node:
mpi_send(len(proj_list_all[i]), 1, MPI_INT, proc_to_stay,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(proj_list_all[i], len(proj_list_all[i]), MPI_INT,
proc_to_stay, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
elif myid == proc_to_stay:
img_per_grp = mpi_recv(1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
img_per_grp = int(img_per_grp[0])
temp = mpi_recv(img_per_grp, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
proj_list.append(list(map(int, temp)))
del temp
mpi_barrier(MPI_COMM_WORLD)
sxprint(" C ", myid, " ", time() - st)
if myid == main_node:
# Assign the remaining groups to main_node
for i in range(n_grp, len(proj_list_all)):
proj_list.append(proj_list_all[i])
del proj_list_all, angle_list, mirror_list
# Compute stability per projection projection direction, equal number assigned, thus overlaps
elif options.grouping == "GEV":
if options.delta == -1.0:
ERROR(
"Angular step for reference projections is required for GEV method"
)
return
from sp_utilities import even_angles, nearestk_to_refdir, getvec
refproj = even_angles(options.delta)
img_begin, img_end = MPI_start_end(len(refproj), number_of_proc, myid)
# Now each processor keeps its own share of reference projections
refprojdir = refproj[img_begin:img_end]
del refproj
ref_ang = [0.0] * (len(refprojdir) * 2)
for i in range(len(refprojdir)):
ref_ang[i * 2] = refprojdir[0][0]
ref_ang[i * 2 + 1] = refprojdir[0][1] + i * 0.1
sxprint(" A ", myid, " ", time() - st)
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
# the solution below is very slow, do not use it unless there is a problem with the i/O
"""
for i in xrange(number_of_proc):
if myid == i:
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
mpi_barrier(MPI_COMM_WORLD)
"""
sxprint(" B ", myid, " ", time() - st)
proj_ang = [0.0] * (nima * 2)
for i in range(nima):
dp = proj_attr[i].get_params("spider")
proj_ang[i * 2] = dp["phi"]
proj_ang[i * 2 + 1] = dp["theta"]
sxprint(" C ", myid, " ", time() - st)
asi = Util.nearestk_to_refdir(proj_ang, ref_ang, img_per_grp)
del proj_ang, ref_ang
proj_list = []
for i in range(len(refprojdir)):
proj_list.append(asi[i * img_per_grp:(i + 1) * img_per_grp])
del asi
sxprint(" D ", myid, " ", time() - st)
#from sys import exit
#exit()
# Compute stability per projection
elif options.grouping == "PPR":
sxprint(" A ", myid, " ", time() - st)
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
sxprint(" B ", myid, " ", time() - st)
proj_params = []
for i in range(nima):
dp = proj_attr[i].get_params("spider")
phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[
"psi"], -dp["tx"], -dp["ty"]
proj_params.append([phi, theta, psi, s2x, s2y])
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
sxprint(" C ", myid, " ", time() - st)
from sp_utilities import nearest_proj
proj_list, mirror_list = nearest_proj(
proj_params, img_per_grp,
list(range(img_begin, img_begin + 1))) #range(img_begin, img_end))
refprojdir = proj_params[img_begin:img_end]
del proj_params, mirror_list
sxprint(" D ", myid, " ", time() - st)
else:
ERROR("Incorrect projection grouping option")
return
###########################################################################################################
# Begin stability test
from sp_utilities import get_params_proj, read_text_file
#if myid == 0:
# from utilities import read_text_file
# proj_list[0] = map(int, read_text_file("lggrpp0.txt"))
from sp_utilities import model_blank
aveList = [model_blank(nx, ny)] * len(proj_list)
if options.grouping == "GRP":
refprojdir = [[0.0, 0.0, -1.0]] * len(proj_list)
for i in range(len(proj_list)):
sxprint(" E ", myid, " ", time() - st)
class_data = EMData.read_images(stack, proj_list[i])
#print " R ",myid," ",time()-st
if options.CTF:
from sp_filter import filt_ctf
for im in range(len(class_data)): # MEM LEAK!!
atemp = class_data[im].copy()
btemp = filt_ctf(atemp, atemp.get_attr("ctf"), binary=1)
class_data[im] = btemp
#class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1)
for im in class_data:
try:
t = im.get_attr(
"xform.align2d") # if they are there, no need to set them!
except:
try:
t = im.get_attr("xform.projection")
d = t.get_params("spider")
set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0])
except:
set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0])
#print " F ",myid," ",time()-st
# Here, we perform realignment num_ali times
all_ali_params = []
for j in range(num_ali):
if (xrng[0] == 0.0 and yrng[0] == 0.0):
avet = ali2d_ras(class_data,
randomize=True,
ir=1,
ou=radius,
rs=1,
step=1.0,
dst=90.0,
maxit=ite,
check_mirror=True,
FH=options.fl,
FF=options.aa)
else:
avet = within_group_refinement(class_data, mask, True, 1,
radius, 1, xrng, yrng, step,
90.0, ite, options.fl,
options.aa)
ali_params = []
for im in range(len(class_data)):
alpha, sx, sy, mirror, scale = get_params2D(class_data[im])
ali_params.extend([alpha, sx, sy, mirror])
all_ali_params.append(ali_params)
#aveList[i] = avet
#print " G ",myid," ",time()-st
del ali_params
# We determine the stability of this group here.
# stable_set contains all particles deemed stable, it is a list of list
# each list has two elements, the first is the pixel error, the second is the image number
# stable_set is sorted based on pixel error
#from utilities import write_text_file
#write_text_file(all_ali_params, "all_ali_params%03d.txt"%myid)
stable_set, mir_stab_rate, average_pix_err = multi_align_stability(
all_ali_params, 0.0, 10000.0, thld_err, False, 2 * radius + 1)
#print " H ",myid," ",time()-st
if (len(stable_set) > 5):
stable_set_id = []
members = []
pix_err = []
# First put the stable members into attr 'members' and 'pix_err'
for s in stable_set:
# s[1] - number in this subset
stable_set_id.append(s[1])
# the original image number
members.append(proj_list[i][s[1]])
pix_err.append(s[0])
# Then put the unstable members into attr 'members' and 'pix_err'
from sp_fundamentals import rot_shift2D
avet.to_zero()
if options.grouping == "GRP":
aphi = 0.0
atht = 0.0
vphi = 0.0
vtht = 0.0
l = -1
for j in range(len(proj_list[i])):
# Here it will only work if stable_set_id is sorted in the increasing number, see how l progresses
if j in stable_set_id:
l += 1
avet += rot_shift2D(class_data[j], stable_set[l][2][0],
stable_set[l][2][1],
stable_set[l][2][2],
stable_set[l][2][3])
if options.grouping == "GRP":
phi, theta, psi, sxs, sy_s = get_params_proj(
class_data[j])
if (theta > 90.0):
phi = (phi + 540.0) % 360.0
theta = 180.0 - theta
aphi += phi
atht += theta
vphi += phi * phi
vtht += theta * theta
else:
members.append(proj_list[i][j])
pix_err.append(99999.99)
aveList[i] = avet.copy()
if l > 1:
l += 1
aveList[i] /= l
if options.grouping == "GRP":
aphi /= l
atht /= l
vphi = (vphi - l * aphi * aphi) / l
vtht = (vtht - l * atht * atht) / l
from math import sqrt
refprojdir[i] = [
aphi, atht,
(sqrt(max(vphi, 0.0)) + sqrt(max(vtht, 0.0))) / 2.0
]
# Here more information has to be stored, PARTICULARLY WHAT IS THE REFERENCE DIRECTION
aveList[i].set_attr('members', members)
aveList[i].set_attr('refprojdir', refprojdir[i])
aveList[i].set_attr('pixerr', pix_err)
else:
sxprint(" empty group ", i, refprojdir[i])
aveList[i].set_attr('members', [-1])
aveList[i].set_attr('refprojdir', refprojdir[i])
aveList[i].set_attr('pixerr', [99999.])
del class_data
if myid == main_node:
km = 0
for i in range(number_of_proc):
if i == main_node:
for im in range(len(aveList)):
aveList[im].write_image(args[1], km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in range(nl):
ave = recv_EMData(i, im + i + 70000)
nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
ave.set_attr('members', list(map(int, members)))
members = mpi_recv(nm, MPI_FLOAT, i,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('pixerr', list(map(float, members)))
members = mpi_recv(3, MPI_FLOAT, i,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('refprojdir', list(map(float, members)))
ave.write_image(args[1], km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
for im in range(len(aveList)):
send_EMData(aveList[im], main_node, im + myid + 70000)
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
members = aveList[im].get_attr('pixerr')
mpi_send(members, len(members), MPI_FLOAT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
except:
mpi_send([-999.0, -999.0, -999.0], 3, MPI_FLOAT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
sp_global_def.BATCH = False
mpi_barrier(MPI_COMM_WORLD)
def main(args):
progname = optparse.os.path.basename(sys.argv[0])
usage = (
progname +
" stack [output_directory] --ir=inner_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translational_search_step --delta=angular_step --center=center_type --maxit1=max_iter1 --maxit2=max_iter2 --L2threshold=0.1 --ref_a=S --sym=c1"
)
usage += """
stack 2D images in a stack file: (default required string)
directory output directory name: into which the results will be written (if it does not exist, it will be created, if it does exist, the results will be written possibly overwriting previous results) (default required string)
"""
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option(
"--radius",
type="int",
default=29,
help=
"radius of the particle: has to be less than < int(nx/2)-1 (default 29)",
)
parser.add_option(
"--xr",
type="string",
default="0",
help=
"range for translation search in x direction: search is +/xr in pixels (default '0')",
)
parser.add_option(
"--yr",
type="string",
default="0",
help=
"range for translation search in y direction: if omitted will be set to xr, search is +/yr in pixels (default '0')",
)
parser.add_option("--mask3D",
type="string",
default=None,
help="3D mask file: (default sphere)")
parser.add_option(
"--moon_elimination",
type="string",
default="",
help=
"elimination of disconnected pieces: two arguments: mass in KDa and pixel size in px/A separated by comma, no space (default none)",
)
parser.add_option(
"--ir",
type="int",
default=1,
help="inner radius for rotational search: > 0 (default 1)",
)
# 'radius' and 'ou' are the same as per Pawel's request; 'ou' is hidden from the user
# the 'ou' variable is not changed to 'radius' in the 'sparx' program. This change is at interface level only for sxviper.
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--ou",
type="int",
default=-1,
help=optparse.SUPPRESS_HELP)
parser.add_option(
"--rs",
type="int",
default=1,
help="step between rings in rotational search: >0 (default 1)",
)
parser.add_option(
"--ts",
type="string",
default="1.0",
help=
"step size of the translation search in x-y directions: search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional (default '1.0')",
)
parser.add_option(
"--delta",
type="string",
default="2.0",
help="angular step of reference projections: (default '2.0')",
)
parser.add_option(
"--center",
type="float",
default=-1.0,
help=
"centering of 3D template: average shift method; 0: no centering; 1: center of gravity (default -1.0)",
)
parser.add_option(
"--maxit1",
type="int",
default=400,
help=
"maximum number of iterations performed for the GA part: (default 400)",
)
parser.add_option(
"--maxit2",
type="int",
default=50,
help=
"maximum number of iterations performed for the finishing up part: (default 50)",
)
parser.add_option(
"--L2threshold",
type="float",
default=0.03,
help=
"stopping criterion of GA: given as a maximum relative dispersion of volumes' L2 norms: (default 0.03)",
)
parser.add_option(
"--ref_a",
type="string",
default="S",
help=
"method for generating the quasi-uniformly distributed projection directions: (default S)",
)
parser.add_option(
"--sym",
type="string",
default="c1",
help="point-group symmetry of the structure: (default c1)",
)
# parser.add_option("--function", type="string", default="ref_ali3d", help="name of the reference preparation function (ref_ali3d by default)")
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--function",
type="string",
default="ref_ali3d",
help=optparse.SUPPRESS_HELP)
parser.add_option(
"--nruns",
type="int",
default=6,
help=
"GA population: aka number of quasi-independent volumes (default 6)",
)
parser.add_option(
"--doga",
type="float",
default=0.1,
help=
"do GA when fraction of orientation changes less than 1.0 degrees is at least doga: (default 0.1)",
)
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option(
"--npad",
type="int",
default=2,
help="padding size for 3D reconstruction (default=2)",
)
parser.add_option(
"--fl",
type="float",
default=0.25,
help=
"cut-off frequency applied to the template volume: using a hyperbolic tangent low-pass filter (default 0.25)",
)
parser.add_option(
"--aa",
type="float",
default=0.1,
help="fall-off of hyperbolic tangent low-pass filter: (default 0.1)",
)
parser.add_option(
"--pwreference",
type="string",
default="",
help="text file with a reference power spectrum: (default none)",
)
parser.add_option(
"--debug",
action="store_true",
default=False,
help="debug info printout: (default False)",
)
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option(
"--return_options",
action="store_true",
dest="return_options",
default=False,
help=optparse.SUPPRESS_HELP,
)
# parser.add_option("--an", type="string", default= "-1", help="NOT USED angular neighborhood for local searches (phi and theta)")
# parser.add_option("--CTF", action="store_true", default=False, help="NOT USED Consider CTF correction during the alignment ")
# parser.add_option("--snr", type="float", default= 1.0, help="NOT USED Signal-to-Noise Ratio of the data (default 1.0)")
# (options, args) = parser.parse_args(sys.argv[1:])
required_option_list = ["radius"]
(options, args) = parser.parse_args(args)
# option_dict = vars(options)
# print parser
if options.return_options:
return parser
if options.moon_elimination == "":
options.moon_elimination = []
else:
options.moon_elimination = list(
map(float, options.moon_elimination.split(",")))
# Making sure all required options appeared.
for required_option in required_option_list:
if not options.__dict__[required_option]:
sp_global_def.sxprint("\n ==%s== mandatory option is missing.\n" %
required_option)
sp_global_def.sxprint("Please run '" + progname +
" -h' for detailed options")
sp_global_def.ERROR("Missing parameter. Please see above")
return
if len(args) < 2 or len(args) > 3:
sp_global_def.sxprint("Usage: " + usage)
sp_global_def.sxprint("Please run '" + progname +
" -h' for detailed options")
sp_global_def.ERROR(
"Invalid number of parameters used. Please see usage information above."
)
return
log = sp_logger.Logger(sp_logger.BaseLogger_Files())
# 'radius' and 'ou' are the same as per Pawel's request; 'ou' is hidden from the user
# the 'ou' variable is not changed to 'radius' in the 'sparx' program. This change is at interface level only for sxviper.
options.ou = options.radius
runs_count = options.nruns
mpi_rank = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
mpi_size = mpi.mpi_comm_size(
mpi.MPI_COMM_WORLD
) # Total number of processes, passed by --np option.
if mpi_rank == 0:
all_projs = EMAN2_cppwrap.EMData.read_images(args[0])
subset = list(range(len(all_projs)))
# if mpi_size > len(all_projs):
# ERROR('Number of processes supplied by --np needs to be less than or equal to %d (total number of images) ' % len(all_projs), 'sxviper', 1)
# mpi.mpi_finalize()
# return
else:
all_projs = None
subset = None
outdir = args[1]
error = 0
if mpi_rank == 0:
if mpi_size % options.nruns != 0:
sp_global_def.ERROR(
"Number of processes needs to be a multiple of total number of runs. Total runs by default are 3, you can change it by specifying --nruns option.",
action=0,
)
error = 1
if optparse.os.path.exists(outdir):
sp_global_def.ERROR(
"Output directory '%s' exists, please change the name and restart the program"
% outdir,
action=0,
)
error = 1
sp_global_def.LOGFILE = optparse.os.path.join(outdir,
sp_global_def.LOGFILE)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
error = sp_utilities.bcast_number_to_all(error,
source_node=0,
mpi_comm=mpi.MPI_COMM_WORLD)
if error == 1:
return
if mpi_rank == 0:
optparse.os.makedirs(outdir)
sp_global_def.write_command(outdir)
if outdir[-1] != "/":
outdir += "/"
log.prefix = outdir
# if len(args) > 2:
# ref_vol = get_im(args[2])
# else:
# ref_vol = None
options.user_func = sp_user_functions.factory[options.function]
options.CTF = False
options.snr = 1.0
options.an = -1.0
out_params, out_vol, out_peaks = sp_multi_shc.multi_shc(
all_projs,
subset,
runs_count,
options,
mpi_comm=mpi.MPI_COMM_WORLD,
log=log)
def main():
arglist = []
for arg in sys.argv:
arglist.append(arg)
progname = optparse.os.path.basename(arglist[0])
usage = progname + """ inputvolume locresvolume maskfile outputfile --radius --falloff --MPI
Locally filer a volume based on local resolution volume (sxlocres.py) within area outlined by the maskfile
"""
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option(
"--radius",
type="int",
default=-1,
help=
"if there is no maskfile, sphere with r=radius will be used, by default the radius is nx/2-1"
)
parser.add_option("--falloff",
type="float",
default=0.1,
help="falloff of tanl filter (default 0.1)")
parser.add_option("--MPI",
action="store_true",
default=False,
help="use MPI version")
(options, args) = parser.parse_args(arglist[1:])
if len(args) < 3 or len(args) > 4:
sp_global_def.sxprint("See usage " + usage)
sp_global_def.ERROR(
"Wrong number of parameters. Please see usage information above.")
return
if sp_global_def.CACHE_DISABLE:
pass #IMPORTIMPORTIMPORT from sp_utilities import disable_bdb_cache
sp_utilities.disable_bdb_cache()
if options.MPI:
number_of_proc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
main_node = 0
if (myid == main_node):
#print sys.argv
vi = sp_utilities.get_im(sys.argv[1])
ui = sp_utilities.get_im(sys.argv[2])
#print Util.infomask(ui, None, True)
radius = options.radius
nx = vi.get_xsize()
ny = vi.get_ysize()
nz = vi.get_zsize()
dis = [nx, ny, nz]
else:
falloff = 0.0
radius = 0
dis = [0, 0, 0]
vi = None
ui = None
dis = sp_utilities.bcast_list_to_all(dis, myid, source_node=main_node)
if (myid != main_node):
nx = int(dis[0])
ny = int(dis[1])
nz = int(dis[2])
radius = sp_utilities.bcast_number_to_all(radius, main_node)
if len(args) == 3:
if (radius == -1):
radius = min(nx, ny, nz) // 2 - 1
m = sp_utilities.model_circle(radius, nx, ny, nz)
outvol = args[2]
elif len(args) == 4:
if (myid == main_node):
m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5)
else:
m = sp_utilities.model_blank(nx, ny, nz)
outvol = args[3]
sp_utilities.bcast_EMData_to_all(m, myid, main_node)
pass #IMPORTIMPORTIMPORT from sp_filter import filterlocal
filteredvol = sp_filter.filterlocal(ui, vi, m, options.falloff, myid,
main_node, number_of_proc)
if (myid == 0):
filteredvol.write_image(outvol)
else:
vi = sp_utilities.get_im(args[0])
ui = sp_utilities.get_im(
args[1]
) # resolution volume, values are assumed to be from 0 to 0.5
nn = vi.get_xsize()
falloff = options.falloff
if len(args) == 3:
radius = options.radius
if (radius == -1):
radius = nn // 2 - 1
m = sp_utilities.model_circle(radius, nn, nn, nn)
outvol = args[2]
elif len(args) == 4:
m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5)
outvol = args[3]
sp_fundamentals.fftip(vi) # this is the volume to be filtered
# Round all resolution numbers to two digits
for x in range(nn):
for y in range(nn):
for z in range(nn):
ui.set_value_at_fast(x, y, z,
round(ui.get_value_at(x, y, z), 2))
st = EMAN2_cppwrap.Util.infomask(ui, m, True)
filteredvol = sp_utilities.model_blank(nn, nn, nn)
cutoff = max(st[2] - 0.01, 0.0)
while (cutoff < st[3]):
cutoff = round(cutoff + 0.01, 2)
pt = EMAN2_cppwrap.Util.infomask(
sp_morphology.threshold_outside(ui, cutoff - 0.00501,
cutoff + 0.005), m, True)
if (pt[0] != 0.0):
vovo = sp_fundamentals.fft(
sp_filter.filt_tanl(vi, cutoff, falloff))
for x in range(nn):
for y in range(nn):
for z in range(nn):
if (m.get_value_at(x, y, z) > 0.5):
if (round(ui.get_value_at(x, y, z),
2) == cutoff):
filteredvol.set_value_at_fast(
x, y, z, vovo.get_value_at(x, y, z))
sp_global_def.write_command(optparse.os.path.dirname(outvol))
filteredvol.write_image(outvol)
def sanity_checks(args):
"""
Check if the settings are valid.
Arguments:
args - Command line arguments parsed via argparse
Returns:
Output file name
"""
file_exists_check = [
args.particle_stack,
args.partres_file,
args.params_2d_file,
args.params_3d_file,
args.params_3d_index_file,
args.list,
args.exlist,
]
stack_dependency_check = [
args.params_2d_file,
args.params_3d_file,
args.params_3d_index_file,
args.list,
args.exlist,
]
if not args.particle_stack and not args.partres_file:
sp_global_def.ERROR(
"Particle_stack or partres_file option needs to be present!",
"sp_sphire2relion",
1,
)
for option in stack_dependency_check:
if option and not args.particle_stack:
sp_global_def.ERROR(
"{0} requires particle stack option!".format(option),
"sp_sphire2relion",
1,
)
for option in file_exists_check:
if option:
if option.startswith("bdb:"):
if "#" in option:
raw_dirnames, basename = option.split("#")
dirnames = raw_dirnames[4:]
else:
dirnames = os.path.dirname(option[4:])
if not dirnames:
dirnames = "."
basename = os.path.basename(option[4:])
option = "{0}/EMAN2DB/{1}.bdb".format(dirnames, basename)
if not os.path.isfile(option):
sp_global_def.ERROR("{0} stack must exist!".format(option),
"sp_sphire2relion", 1)
if args.list and args.exlist:
sp_global_def.ERROR(
"Arguments list and exlist cannot be used at the same time.",
"sp_sphire2relion",
1,
)
if args.params_2d_file and args.params_3d_file:
sp_global_def.ERROR(
"Arguments params_2d_file and params_3d_file cannot be used at the same time.",
"sp_sphire2relion",
1,
)
if args.params_3d_index_file and not args.params_3d_file:
sp_global_def.ERROR(
"Arguments params_3d_index_file requires params_3d_file to be set.",
"sp_sphire2relion",
1,
)
if args.params_3d_chunk_file_0 and not args.params_3d_file:
sp_global_def.ERROR(
"Arguments params_3d_chunk_files requires params_3d_file to be set.",
"sp_sphire2relion",
1,
)
if args.params_3d_chunk_file_1 and not args.params_3d_file:
sp_global_def.ERROR(
"Arguments params_3d_chunk_files requires params_3d_file to be set.",
"sp_sphire2relion",
1,
)
try:
os.makedirs(args.output_directory)
except OSError:
pass
sp_global_def.write_command(args.output_directory)
output_path = os.path.join(args.output_directory, args.output_name)
if os.path.exists(output_path) and args.force:
pass
elif os.path.exists(output_path) and not args.force:
sp_global_def.ERROR(
"Output file {0} must not exist! Use the --force flag to overwrite existing files"
.format(output_path),
"sp_sphire2relion",
1,
)
else:
pass
return output_path
def import_partres_file(partres_file):
"""
Import the information from a SPHIRE partres file.
Arguments:
partres_file - Partres file name
Returns:
Array containing the ctf information.
"""
with open(partres_file, "r") as partres_reader:
number_of_columns = len(partres_reader.readline().split())
if number_of_columns == 22:
columns = [0, 1, 2, 3, 6, 7, 17, 19, 20, 21]
dtype_import_list = [
("defocus", float),
("cs", float),
("voltage", float),
("pixel_size", float),
("astig_amp", float),
("astig_angle", float),
("max_resolution", float),
("amplitude_contrast", float),
("phase_shift", float),
("micrograph_name", "|S1000"),
]
dtype_output_list = [
("_rlnDefocusU", float),
("_rlnDefocusV", float),
("_rlnDefocusAngle", float),
("_rlnMicrographName", "|S1000"),
("_rlnDetectorPixelSize", float),
("_rlnMagnification", float),
("_rlnCtfMaxResolution", float),
("_rlnPhaseShift", float),
("_rlnAmplitudeContrast", float),
("_rlnSphericalAberration", float),
("_rlnVoltage", float),
]
else:
sp_global_def.ERROR(
"Number of columns in partres file not known: {0}".format(
number_of_columns),
"sp_sphire2relion",
)
assert len(columns) == len(dtype_import_list)
partres_import_array = np.genfromtxt(partres_file,
dtype=dtype_import_list,
usecols=columns)
partres_array = np.empty(partres_import_array.shape[0],
sorted(dtype_output_list))
partres_array["_rlnDefocusU"] = (
20000 * partres_import_array["defocus"] -
10000 * partres_import_array["astig_amp"]) / 2
partres_array["_rlnDefocusV"] = (20000 * partres_import_array["defocus"] -
partres_array["_rlnDefocusU"])
partres_array[
"_rlnDefocusAngle"] = 45 - partres_import_array["astig_angle"]
partres_array["_rlnMicrographName"] = partres_import_array[
"micrograph_name"]
partres_array["_rlnAmplitudeContrast"] = (
partres_import_array["amplitude_contrast"] / 100)
partres_array["_rlnVoltage"] = partres_import_array["voltage"]
partres_array["_rlnSphericalAberration"] = partres_import_array["cs"]
partres_array["_rlnPhaseShift"] = partres_import_array["phase_shift"]
partres_array["_rlnDetectorPixelSize"] = partres_import_array["pixel_size"]
partres_array["_rlnMagnification"] = 10000
partres_array[
"_rlnCtfMaxResolution"] = 1 / partres_import_array["max_resolution"]
return partres_array
# If output directory not specified, write to same directory as class averages
if not options.outdir:
outdir = os.path.dirname(os.path.realpath(options.classavgs))
else:
outdir = options.outdir
if options.mode == 'viper':
selectdoc = options.classselect
elif options.mode == 'projmatch':
selectdoc = None
elif options.mode == 'meridien':
selectdoc = options.partselect
else:
sp_global_def.ERROR(
"\nERROR!! Valid mode not specified. Valid modes are: viper, projmatch, and meridien.",
__file__, 1)
sxprint('Type %s --help to see available options\n' %
os.path.basename(__file__))
exit()
main_proj_compare(options.classavgs,
options.vol3d,
outdir,
options,
mode=options.mode,
prjmethod=options.prjmethod,
classangles=options.classangles,
partangles=options.partangles,
selectdoc=selectdoc,
verbose=options.verbose,
