def main():
progname = os.path.basename(sys.argv[0])
usage = progname + " input_filename output_filename --sym=Symmetry group --phi --theta --psi=The 3 Eulerian angles in degrees --r=Radius of mask --phirange --thetarange --psirange=A search scale for each angle --ftol --xtol = convergence criterion the function and angles values"
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--sym", type="string", default="c1", help=" String that specifies the point group symmetry. default = 'c1'")
parser.add_option("--phi", type='float',default=0.0, help=" phi angle, default = 0")
parser.add_option("--theta", type='float',default=0.0, help=" theta angle, default=0")
parser.add_option("--psi", type='float',default=0.0, help=" phi angle, default=0")
parser.add_option("--r", type='float',default=None,help=" Input the radius of the mask. default=None")
parser.add_option("--phirange", type='float',default=20.0,help=" The search scale for phi angle...default=20")
parser.add_option("--thetarange", type='float',default=20.0,help=" The search scale for theta angle...default=20")
parser.add_option("--psirange", type='float',default=20.0,help=" The search scale for psi angle...default=20")
parser.add_option("--ftol", type='float',default=1.e-4,help=" convergence criterion on the function values...default = 1.e-4")
parser.add_option("--xtol", type='float',default=1.e-4,help=" convergence criterion on the variable values...default = 1.e-4")
(options, args) = parser.parse_args()
if len(args) != 2:
sxprint("Usage: " + usage)
sxprint("Please run \'" + progname + " -h\' for detailed options")
ERROR( "Invalid number of parameters used. Please see usage information above." )
return
else:
if sp_global_def.CACHE_DISABLE:
from sp_utilities import disable_bdb_cache
disable_bdb_cache()
from sp_applications import rot_sym
sp_global_def.BATCH = True
rot_sym(args[0],args[1],options.sym,options.r,options.phi,options.theta,options.psi,options.phirange,options.thetarange,options.psirange,options.ftol,options.xtol)
sp_global_def.write_command('.')
sp_global_def.BATCH = False
def main():
args = argparser.parse_args()
cindy_path = args.cinderella_path
input_stack = args.inputstack
out_dir = args.output_dir
model_path = args.model_path
conf_thresh = args.confidence_threshold
batch_size = args.batch_size
gpu = args.gpu
complete_command = [cindy_path]
arg_input_stack = "-i=" + str(input_stack)
complete_command.append(arg_input_stack)
arg_out_dir = "-o=" + str(out_dir)
complete_command.append(arg_out_dir)
arg_model_path = "-w=" + str(model_path)
complete_command.append(arg_model_path)
arg_conf_thresh = "-t=" + str(conf_thresh)
complete_command.append(arg_conf_thresh)
arg_batch_size = "-b=" + str(batch_size)
complete_command.append(arg_batch_size)
if gpu != -1:
arg_gpu = "--gpu=" + str(gpu)
complete_command.append(arg_gpu)
sp_global_def.write_command()
subprocess.check_call(complete_command)
sp_global_def.write_command(out_dir)
def prepare_outdir_log(outdir='.', verbose=False, is_main=True):
"""
Prepares output directory and sets up log file.
Arguments:
outdir : Output directory
verbose : (boolean) Whether to write to screen
is_main : (boolean) If using multiple cores, some tasks only need to be performed once, not by all cores
Returns:
log : instance of Logger class
verbose : (boolean) New version of Logger can write to screen simultaneously
"""
# Create directory if it doesn't exist
if is_main:
if os.path.isdir(outdir):
sxprint("Writing to output directory: %s" % outdir)
else:
sxprint("Created output directory: %s" % outdir)
os.makedirs(
outdir) # os.mkdir() can only operate one directory deep
sp_global_def.write_command(outdir)
logname = "log_" + datetime.now().strftime("%Y%m%d_%H%M%S") + ".txt"
logname = os.path.join(outdir, logname)
#if global_def.LOGFILE:
#global_def.LOGFILE = logname
#print('LOGFILE', global_def.LOGFILE)
#exit()
# May be using old version of logger.py
try:
if verbose:
log = Logger(base_logger=BaseLogger_Files(),
base_logger2=BaseLogger_Print(),
file_name=logname)
verbose = False # logger output will be echoed to screen
else:
log = Logger(base_logger=BaseLogger_Files(), file_name=logname)
except TypeError:
if is_main: sxprint("WARNING: Using old sp_logger.py library")
log = Logger(base_logger=BaseLogger_Files()) #, file_name=logname)
logname = 'log.txt'
if is_main: sxprint("Writing log file to %s\n" % logname)
if is_main:
progbase = os.path.basename(__file__).split('.')[0].upper()
#print('progbase', progbase)
#exit()
length = len(progbase) + 4
log.add("\n" + " " * TIMESTAMP_LENGTH + "*" * length + "\n" +
" " * TIMESTAMP_LENGTH + "* " + progbase + " *\n" +
" " * TIMESTAMP_LENGTH + "*" * length)
return log, verbose
def _main_():
argparser = setup_argparser()
args = argparser.parse_args()
path_to_resultsfile = args.resultsfile
mode = args.mode
lower_then = args.lower_then
greater_then = args.greater_then
path_output = args.output
path_stack = args.stack
field = args.field
field_index_error = 1
field_index_dr_ratio = 3
if field == "ERROR":
field = field_index_error
elif field == "DR_RATIO":
field = field_index_dr_ratio
# Read in error file
results = np.loadtxt(path_to_resultsfile, delimiter=",")
# Identify relevant particles
relevant_selection = [
a and b for a, b in zip(results[:, field] <= lower_then,
results[:, field] >= greater_then)
]
# Write stack
number_of_particles = EMAN2.EMUtil.get_image_count(path_stack)
local_bdb_stack = EMAN2db.db_open_dict(path_output)
sp_global_def.write_command(EMAN2db.db_parse_path(path_output)[0])
num_particles_relevant = 0
for particle_index in range(number_of_particles):
particle = sp_ctf_refine_io.read_particle(path_stack,
particle_index,
header_only=True)
particle_header = particle.get_attr_dict()
if relevant_selection[particle_index]:
pctf = particle_header["ctf"]
pctf.defocus = results[particle_index, 2]
particle_header["ctf"] = pctf
num_particles_relevant = num_particles_relevant + 1
if mode == "UPDATE":
local_bdb_stack[particle_index] = particle_header
elif mode == "EXTRACT" and relevant_selection[particle_index]:
local_bdb_stack[num_particles_relevant - 1] = particle_header
EMAN2db.db_close_dict(local_bdb_stack)
sp_global_def.sxprint("Particles updated/extracted",
num_particles_relevant)
def main():
arglist = []
for arg in sys.argv:
arglist.append( arg )
progname = os.path.basename( arglist[0] )
usage = progname + " stack --params='parm1 parm2 parm3 ...' --zero --one --set=number --randomize --rand_alpha --import=file --export=file --print --backup --suffix --restore --delete"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--params", type="string", default=None, help="parameter list")
parser.add_option("--zero", action="store_true", default=False, help="set parameter to zero")
parser.add_option("--one", action="store_true", default=False, help="set parameter to one")
parser.add_option("--set", type="float", default=0.0, help="set parameter to a value (different from 0.0)")
parser.add_option("--randomize", action="store_true", default=False, help="set parameter to randomized value")
parser.add_option("--rand_alpha", action="store_true", default=False, help="set all angles to randomized value")
parser.add_option("--import", type="string", dest="fimport", default=None, help="import parameters from file")
parser.add_option("--export", type="string", dest="fexport", default=None, help="export parameters to file")
parser.add_option("--print", action="store_true", dest="fprint", default=False, help="print parameters")
parser.add_option("--backup", action="store_true", default=False, help="backup parameters")
parser.add_option("--suffix", type="string", default="_backup", help="suffix for xform name in backup")
parser.add_option("--restore", action="store_true", default=False, help="restore parameters")
parser.add_option("--delete", action="store_true", default=False, help="delete parameters")
parser.add_option("--consecutive", action="store_true", default=False, help="set selected parameter to consecutive integers starting from 0")
parser.add_option("--list", type="string", default=None, help="Indices list containing the same amount of rows as the import file")
(options,args) = parser.parse_args( arglist[1:] )
if not options.fprint:
sp_global_def.print_timestamp( "Start" )
sp_global_def.write_command()
if len(args) != 1 :
sxprint( "Usage: " + usage )
ERROR( "Invalid number of parameters provided. Please see usage information above." )
return
if options.params == None:
sxprint( "Usage: " + usage )
ERROR( "No parameters provided. Please see usage information above." )
return
if sp_global_def.CACHE_DISABLE:
from sp_utilities import disable_bdb_cache
disable_bdb_cache()
from sp_applications import header
header(args[0], options.params, options.zero, options.one, options.set, options.randomize, options.rand_alpha, options.fimport, options.fexport, \
options.fprint, options.backup, options.suffix, options.restore, options.delete, options.consecutive, options.list)
if not options.fprint:
sp_global_def.print_timestamp( "Finish" )
def prepare_outdir(outdir='.', verbose=False, main=True):
"""
Create directory if it doesn't exist.
Arguments:
outdir : Output directory
verbose : (boolean) Whether to write additional information to screen
main : (boolean) Whether main MPI process
"""
# If using MPI, only need to check once
if main:
if os.path.isdir(outdir):
if verbose: sxprint("Writing to output directory: %s" % outdir)
else:
if verbose: sxprint("Created output directory: %s" % outdir)
os.makedirs(outdir) # os.mkdir() can only operate one directory deep
sp_global_def.write_command(outdir)
def main():
# Read arguments
args = argparser.parse_args()
config_path = args.config_path
target_dir = args.target_dir
model_path = args.model_path
output_dir = args.output_dir
confidence_threshold = args.confidence_threshold
do_filament_mode = args.filament_mode
filament_width = args.filament_width
min_box_per_filament = args.min_box_per_filament
box_distance = args.box_distance
gpu_fraction = 1.0
if args.gpu_fraction < 1.0 and args.gpu_fraction > 0.0:
gpu_fraction = args.gpu_fraction
num_cpu = args.num_cpu
str_gpus = [str(entry).strip() for entry in args.gpu.split(",")]
arg_gpu = ' '.join(str_gpus)
no_merging = args.nomerging
no_split = args.nosplit
cryolo_predict_path = args.cryolo_predict_path
# Run the training
complete_command = ['cryolo_predict.py']
if cryolo_predict_path is not None:
complete_command = [cryolo_predict_path]
config_argument = "-c=" + str(config_path)
complete_command.append(config_argument)
weights_argument = "-w=" + str(model_path)
complete_command.append(weights_argument)
input_argument = "-i=" + str(target_dir)
complete_command.append(input_argument)
output_argument = "-o=" + str(output_dir)
complete_command.append(output_argument)
thresh_argument = "-t=" + str(confidence_threshold)
complete_command.append(thresh_argument)
if args.min_distance > 0:
min_dist_arg = "-d=" + str(int(args.min_distance))
complete_command.append(min_dist_arg)
gpu_argument = "-g " + arg_gpu
complete_command.append(gpu_argument)
gpu_fraction_arg = "--gpu_fraction=" + str(gpu_fraction)
complete_command.append(gpu_fraction_arg)
if num_cpu != -1:
num_cpu_arg = "--num_cpu=" + str(num_cpu)
complete_command.append(num_cpu_arg)
if args.otf:
complete_command.append("--otf")
if do_filament_mode:
complete_command.append("--filament")
complete_command.append("-fw=" + str(filament_width))
complete_command.append("-mn=" + str(min_box_per_filament))
if box_distance > 0:
complete_command.append("-bd=" + str(box_distance))
if no_merging:
complete_command.append("--nomerging")
if no_split:
complete_command.append("--nosplit")
sp_global_def.write_command()
subprocess.check_call(complete_command)
sp_global_def.write_command(output_dir)
def sanity_checks(args):
"""
Check if the settings are valid.
Arguments:
args - Command line arguments parsed via argparse
Returns:
Output file name
"""
file_exists_check = [
args.particle_stack,
args.partres_file,
args.params_2d_file,
args.params_3d_file,
args.params_3d_index_file,
args.list,
args.exlist,
]
stack_dependency_check = [
args.params_2d_file,
args.params_3d_file,
args.params_3d_index_file,
args.list,
args.exlist,
]
if not args.particle_stack and not args.partres_file:
sp_global_def.ERROR(
"Particle_stack or partres_file option needs to be present!",
"sp_sphire2relion",
1,
)
for option in stack_dependency_check:
if option and not args.particle_stack:
sp_global_def.ERROR(
"{0} requires particle stack option!".format(option),
"sp_sphire2relion",
1,
)
for option in file_exists_check:
if option:
if option.startswith("bdb:"):
if "#" in option:
raw_dirnames, basename = option.split("#")
dirnames = raw_dirnames[4:]
else:
dirnames = os.path.dirname(option[4:])
if not dirnames:
dirnames = "."
basename = os.path.basename(option[4:])
option = "{0}/EMAN2DB/{1}.bdb".format(dirnames, basename)
if not os.path.isfile(option):
sp_global_def.ERROR("{0} stack must exist!".format(option),
"sp_sphire2relion", 1)
if args.list and args.exlist:
sp_global_def.ERROR(
"Arguments list and exlist cannot be used at the same time.",
"sp_sphire2relion",
1,
)
if args.params_2d_file and args.params_3d_file:
sp_global_def.ERROR(
"Arguments params_2d_file and params_3d_file cannot be used at the same time.",
"sp_sphire2relion",
1,
)
if args.params_3d_index_file and not args.params_3d_file:
sp_global_def.ERROR(
"Arguments params_3d_index_file requires params_3d_file to be set.",
"sp_sphire2relion",
1,
)
if args.params_3d_chunk_file_0 and not args.params_3d_file:
sp_global_def.ERROR(
"Arguments params_3d_chunk_files requires params_3d_file to be set.",
"sp_sphire2relion",
1,
)
if args.params_3d_chunk_file_1 and not args.params_3d_file:
sp_global_def.ERROR(
"Arguments params_3d_chunk_files requires params_3d_file to be set.",
"sp_sphire2relion",
1,
)
try:
os.makedirs(args.output_directory)
except OSError:
pass
sp_global_def.write_command(args.output_directory)
output_path = os.path.join(args.output_directory, args.output_name)
if os.path.exists(output_path) and args.force:
pass
elif os.path.exists(output_path) and not args.force:
sp_global_def.ERROR(
"Output file {0} must not exist! Use the --force flag to overwrite existing files"
.format(output_path),
"sp_sphire2relion",
1,
)
else:
pass
return output_path
def resample( prjfile, outdir, bufprefix, nbufvol, nvol, seedbase,\
delta, d, snr, CTF, npad,\
MPI, myid, ncpu, verbose = 0 ):
from sp_utilities import even_angles
from random import seed, jumpahead, shuffle
import os
from sys import exit
nprj = EMUtil.get_image_count( prjfile )
if MPI:
if myid == 0:
if os.path.exists(outdir): nx = 1
else: nx = 0
else: nx = 0
ny = bcast_number_to_all(nx, source_node = 0)
if ny == 1: ERROR('Output directory exists, please change the name and restart the program', "resample", 1,myid)
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
if myid == 0:
os.makedirs(outdir)
sp_global_def.write_command(outdir)
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
else:
if os.path.exists(outdir):
ERROR('Output directory exists, please change the name and restart the program', "resample", 1,0)
os.makedirs(outdir)
sp_global_def.write_command(outdir)
if(verbose == 1): finfo=open( os.path.join(outdir, "progress%04d.txt" % myid), "w" )
else: finfo = None
#print " before evenangles",myid
from sp_utilities import getvec
from numpy import array, reshape
refa = even_angles(delta)
nrefa = len(refa)
refnormal = zeros((nrefa,3),'float32')
tetref = [0.0]*nrefa
for i in range(nrefa):
tr = getvec( refa[i][0], refa[i][1] )
for j in range(3): refnormal[i][j] = tr[j]
tetref[i] = refa[i][1]
del refa
vct = array([0.0]*(3*nprj),'float32')
if myid == 0:
sxprint(" will read ",myid)
tr = EMUtil.get_all_attributes(prjfile,'xform.projection')
tetprj = [0.0]*nprj
for i in range(nprj):
temp = tr[i].get_params("spider")
tetprj[i] = temp["theta"]
if(tetprj[i] > 90.0): tetprj[i] = 180.0 - tetprj[i]
vct[3*i+0] = tr[i].at(2,0)
vct[3*i+1] = tr[i].at(2,1)
vct[3*i+2] = tr[i].at(2,2)
del tr
else:
tetprj = [0.0]*nprj
#print " READ ",myid
if MPI:
#print " will bcast",myid
vct = mpi.mpi_bcast( vct, len(vct), mpi.MPI_FLOAT, 0, mpi.MPI_COMM_WORLD )
from sp_utilities import bcast_list_to_all
tetprj = bcast_list_to_all(tetprj, myid, 0)
#print " reshape ",myid
vct = reshape(vct,(nprj,3))
assignments = [[] for i in range(nrefa)]
dspn = 1.25*delta
for k in range(nprj):
best_s = -1.0
best_i = -1
for i in range( nrefa ):
if(abs(tetprj[k] - tetref[i]) <= dspn):
s = abs(refnormal[i][0]*vct[k][0] + refnormal[i][1]*vct[k][1] + refnormal[i][2]*vct[k][2])
if s > best_s:
best_s = s
best_i = i
assignments[best_i].append(k)
am = len(assignments[0])
mufur = 1.0/am
for i in range(1,len(assignments)):
ti = len(assignments[i])
am = min(am, ti)
if(ti>0): mufur += 1.0/ti
del tetprj,tetref
dp = 1.0 - d # keep that many in each direction
keep = int(am*dp +0.5)
mufur = keep*nrefa/(1.0 - mufur*keep/float(nrefa))
if myid == 0:
sxprint(" Number of projections ",nprj,". Number of reference directions ",nrefa,", multiplicative factor for the variance ",mufur)
sxprint(" Minimum number of assignments ",am," Number of projections used per stratum ", keep," Number of projections in resampled structure ",int(am*dp +0.5)*nrefa)
if am <2 or am == keep:
sxprint("incorrect settings")
exit() # FIX
if(seedbase < 1):
seed()
jumpahead(17*myid+123)
else:
seed(seedbase)
jumpahead(17*myid+123)
volfile = os.path.join(outdir, "bsvol%04d.hdf" % myid)
from random import randint
niter = nvol/ncpu/nbufvol
for kiter in range(niter):
if(verbose == 1):
finfo.write( "Iteration %d: \n" % kiter )
finfo.flush()
iter_start = time()
# the following has to be converted to resample mults=1 means take given projection., mults=0 means omit
mults = [ [0]*nprj for i in range(nbufvol) ]
for i in range(nbufvol):
for l in range(nrefa):
mass = assignments[l][:]
shuffle(mass)
mass = mass[:keep]
mass.sort()
#print l, " * ",mass
for k in range(keep):
mults[i][mass[k]] = 1
'''
lout = []
for l in xrange(len(mults[i])):
if mults[i][l] == 1: lout.append(l)
write_text_file(lout, os.path.join(outdir, "list%04d_%03d.txt" %(i, myid)))
del lout
'''
del mass
rectors, fftvols, wgtvols = resample_prepare( prjfile, nbufvol, snr, CTF, npad )
resample_insert( bufprefix, fftvols, wgtvols, mults, CTF, npad, finfo )
del mults
resample_finish( rectors, fftvols, wgtvols, volfile, kiter, nprj, finfo )
rectors = None
fftvols = None
wgtvols = None
if(verbose == 1):
finfo.write( "time for iteration: %10.3f\n" % (time() - iter_start) )
finfo.flush()
def main(args):
progname = optparse.os.path.basename(sys.argv[0])
usage = (
progname +
" stack [output_directory] --ir=inner_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translational_search_step --delta=angular_step --center=center_type --maxit1=max_iter1 --maxit2=max_iter2 --L2threshold=0.1 --ref_a=S --sym=c1"
)
usage += """
stack 2D images in a stack file: (default required string)
directory output directory name: into which the results will be written (if it does not exist, it will be created, if it does exist, the results will be written possibly overwriting previous results) (default required string)
"""
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option(
"--radius",
type="int",
default=29,
help=
"radius of the particle: has to be less than < int(nx/2)-1 (default 29)",
)
parser.add_option(
"--xr",
type="string",
default="0",
help=
"range for translation search in x direction: search is +/xr in pixels (default '0')",
)
parser.add_option(
"--yr",
type="string",
default="0",
help=
"range for translation search in y direction: if omitted will be set to xr, search is +/yr in pixels (default '0')",
)
parser.add_option("--mask3D",
type="string",
default=None,
help="3D mask file: (default sphere)")
parser.add_option(
"--moon_elimination",
type="string",
default="",
help=
"elimination of disconnected pieces: two arguments: mass in KDa and pixel size in px/A separated by comma, no space (default none)",
)
parser.add_option(
"--ir",
type="int",
default=1,
help="inner radius for rotational search: > 0 (default 1)",
)
# 'radius' and 'ou' are the same as per Pawel's request; 'ou' is hidden from the user
# the 'ou' variable is not changed to 'radius' in the 'sparx' program. This change is at interface level only for sxviper.
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--ou",
type="int",
default=-1,
help=optparse.SUPPRESS_HELP)
parser.add_option(
"--rs",
type="int",
default=1,
help="step between rings in rotational search: >0 (default 1)",
)
parser.add_option(
"--ts",
type="string",
default="1.0",
help=
"step size of the translation search in x-y directions: search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional (default '1.0')",
)
parser.add_option(
"--delta",
type="string",
default="2.0",
help="angular step of reference projections: (default '2.0')",
)
parser.add_option(
"--center",
type="float",
default=-1.0,
help=
"centering of 3D template: average shift method; 0: no centering; 1: center of gravity (default -1.0)",
)
parser.add_option(
"--maxit1",
type="int",
default=400,
help=
"maximum number of iterations performed for the GA part: (default 400)",
)
parser.add_option(
"--maxit2",
type="int",
default=50,
help=
"maximum number of iterations performed for the finishing up part: (default 50)",
)
parser.add_option(
"--L2threshold",
type="float",
default=0.03,
help=
"stopping criterion of GA: given as a maximum relative dispersion of volumes' L2 norms: (default 0.03)",
)
parser.add_option(
"--ref_a",
type="string",
default="S",
help=
"method for generating the quasi-uniformly distributed projection directions: (default S)",
)
parser.add_option(
"--sym",
type="string",
default="c1",
help="point-group symmetry of the structure: (default c1)",
)
# parser.add_option("--function", type="string", default="ref_ali3d", help="name of the reference preparation function (ref_ali3d by default)")
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--function",
type="string",
default="ref_ali3d",
help=optparse.SUPPRESS_HELP)
parser.add_option(
"--nruns",
type="int",
default=6,
help=
"GA population: aka number of quasi-independent volumes (default 6)",
)
parser.add_option(
"--doga",
type="float",
default=0.1,
help=
"do GA when fraction of orientation changes less than 1.0 degrees is at least doga: (default 0.1)",
)
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option(
"--npad",
type="int",
default=2,
help="padding size for 3D reconstruction (default=2)",
)
parser.add_option(
"--fl",
type="float",
default=0.25,
help=
"cut-off frequency applied to the template volume: using a hyperbolic tangent low-pass filter (default 0.25)",
)
parser.add_option(
"--aa",
type="float",
default=0.1,
help="fall-off of hyperbolic tangent low-pass filter: (default 0.1)",
)
parser.add_option(
"--pwreference",
type="string",
default="",
help="text file with a reference power spectrum: (default none)",
)
parser.add_option(
"--debug",
action="store_true",
default=False,
help="debug info printout: (default False)",
)
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option(
"--return_options",
action="store_true",
dest="return_options",
default=False,
help=optparse.SUPPRESS_HELP,
)
# parser.add_option("--an", type="string", default= "-1", help="NOT USED angular neighborhood for local searches (phi and theta)")
# parser.add_option("--CTF", action="store_true", default=False, help="NOT USED Consider CTF correction during the alignment ")
# parser.add_option("--snr", type="float", default= 1.0, help="NOT USED Signal-to-Noise Ratio of the data (default 1.0)")
# (options, args) = parser.parse_args(sys.argv[1:])
required_option_list = ["radius"]
(options, args) = parser.parse_args(args)
# option_dict = vars(options)
# print parser
if options.return_options:
return parser
if options.moon_elimination == "":
options.moon_elimination = []
else:
options.moon_elimination = list(
map(float, options.moon_elimination.split(",")))
# Making sure all required options appeared.
for required_option in required_option_list:
if not options.__dict__[required_option]:
sp_global_def.sxprint("\n ==%s== mandatory option is missing.\n" %
required_option)
sp_global_def.sxprint("Please run '" + progname +
" -h' for detailed options")
sp_global_def.ERROR("Missing parameter. Please see above")
return
if len(args) < 2 or len(args) > 3:
sp_global_def.sxprint("Usage: " + usage)
sp_global_def.sxprint("Please run '" + progname +
" -h' for detailed options")
sp_global_def.ERROR(
"Invalid number of parameters used. Please see usage information above."
)
return
log = sp_logger.Logger(sp_logger.BaseLogger_Files())
# 'radius' and 'ou' are the same as per Pawel's request; 'ou' is hidden from the user
# the 'ou' variable is not changed to 'radius' in the 'sparx' program. This change is at interface level only for sxviper.
options.ou = options.radius
runs_count = options.nruns
mpi_rank = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
mpi_size = mpi.mpi_comm_size(
mpi.MPI_COMM_WORLD
) # Total number of processes, passed by --np option.
if mpi_rank == 0:
all_projs = EMAN2_cppwrap.EMData.read_images(args[0])
subset = list(range(len(all_projs)))
# if mpi_size > len(all_projs):
# ERROR('Number of processes supplied by --np needs to be less than or equal to %d (total number of images) ' % len(all_projs), 'sxviper', 1)
# mpi.mpi_finalize()
# return
else:
all_projs = None
subset = None
outdir = args[1]
error = 0
if mpi_rank == 0:
if mpi_size % options.nruns != 0:
sp_global_def.ERROR(
"Number of processes needs to be a multiple of total number of runs. Total runs by default are 3, you can change it by specifying --nruns option.",
action=0,
)
error = 1
if optparse.os.path.exists(outdir):
sp_global_def.ERROR(
"Output directory '%s' exists, please change the name and restart the program"
% outdir,
action=0,
)
error = 1
sp_global_def.LOGFILE = optparse.os.path.join(outdir,
sp_global_def.LOGFILE)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
error = sp_utilities.bcast_number_to_all(error,
source_node=0,
mpi_comm=mpi.MPI_COMM_WORLD)
if error == 1:
return
if mpi_rank == 0:
optparse.os.makedirs(outdir)
sp_global_def.write_command(outdir)
if outdir[-1] != "/":
outdir += "/"
log.prefix = outdir
# if len(args) > 2:
# ref_vol = get_im(args[2])
# else:
# ref_vol = None
options.user_func = sp_user_functions.factory[options.function]
options.CTF = False
options.snr = 1.0
options.an = -1.0
out_params, out_vol, out_peaks = sp_multi_shc.multi_shc(
all_projs,
subset,
runs_count,
options,
mpi_comm=mpi.MPI_COMM_WORLD,
log=log)
def main():
arglist = []
for arg in sys.argv:
arglist.append(arg)
progname = os.path.basename(arglist[0])
usage = progname + """ firstvolume secondvolume maskfile directory --prefix --wn --step --cutoff --radius --fsc --res_overall --out_ang_res --apix --MPI
Compute local resolution in real space within area outlined by the maskfile and within regions wn x wn x wn
"""
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option("--prefix",
type="str",
default='localres',
help="Prefix for the output files. (default localres)")
parser.add_option(
"--wn",
type="int",
default=7,
help=
"Size of window within which local real-space FSC is computed. (default 7)"
)
parser.add_option(
"--step",
type="float",
default=1.0,
help="Shell step in Fourier size in pixels. (default 1.0)")
parser.add_option("--cutoff",
type="float",
default=0.143,
help="Resolution cut-off for FSC. (default 0.143)")
parser.add_option(
"--radius",
type="int",
default=-1,
help=
"If there is no maskfile, sphere with r=radius will be used. By default, the radius is nx/2-wn (default -1)"
)
parser.add_option(
"--fsc",
type="string",
default=None,
help=
"Save overall FSC curve (might be truncated). By default, the program does not save the FSC curve. (default none)"
)
parser.add_option(
"--res_overall",
type="float",
default=-1.0,
help=
"Overall resolution at the cutoff level estimated by the user [abs units]. (default None)"
)
parser.add_option(
"--out_ang_res",
action="store_true",
default=False,
help=
"Additionally creates a local resolution file in Angstroms. (default False)"
)
parser.add_option(
"--apix",
type="float",
default=1.0,
help=
"Pixel size in Angstrom. Effective only with --out_ang_res options. (default 1.0)"
)
parser.add_option("--MPI",
action="store_true",
default=False,
help="Use MPI version.")
(options, args) = parser.parse_args(arglist[1:])
if len(args) < 3 or len(args) > 4:
sxprint("Usage: " + usage)
ERROR(
"Invalid number of parameters used. Please see usage information above."
)
return
if sp_global_def.CACHE_DISABLE:
sp_utilities.disable_bdb_cache()
res_overall = options.res_overall
if options.MPI:
number_of_proc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
main_node = 0
sp_global_def.MPI = True
cutoff = options.cutoff
nk = int(options.wn)
if (myid == main_node):
#print sys.argv
vi = sp_utilities.get_im(sys.argv[1])
ui = sp_utilities.get_im(sys.argv[2])
nx = vi.get_xsize()
ny = vi.get_ysize()
nz = vi.get_zsize()
dis = [nx, ny, nz]
else:
dis = [0, 0, 0, 0]
sp_global_def.BATCH = True
dis = sp_utilities.bcast_list_to_all(dis, myid, source_node=main_node)
if (myid != main_node):
nx = int(dis[0])
ny = int(dis[1])
nz = int(dis[2])
vi = sp_utilities.model_blank(nx, ny, nz)
ui = sp_utilities.model_blank(nx, ny, nz)
if len(args) == 3:
m = sp_utilities.model_circle((min(nx, ny, nz) - nk) // 2, nx, ny,
nz)
outdir = args[2]
elif len(args) == 4:
if (myid == main_node):
m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5)
else:
m = sp_utilities.model_blank(nx, ny, nz)
outdir = args[3]
if os.path.exists(outdir) and myid == 0:
sp_global_def.ERROR('Output directory already exists!')
elif myid == 0:
os.makedirs(outdir)
sp_global_def.write_command(outdir)
sp_utilities.bcast_EMData_to_all(m, myid, main_node)
"""Multiline Comment0"""
freqvol, resolut = sp_statistics.locres(vi, ui, m, nk, cutoff,
options.step, myid, main_node,
number_of_proc)
if (myid == 0):
# Remove outliers based on the Interquartile range
output_volume(freqvol, resolut, options.apix, outdir,
options.prefix, options.fsc, options.out_ang_res, nx,
ny, nz, res_overall)
else:
cutoff = options.cutoff
vi = sp_utilities.get_im(args[0])
ui = sp_utilities.get_im(args[1])
nn = vi.get_xsize()
nx = nn
ny = nn
nz = nn
nk = int(options.wn)
if len(args) == 3:
m = sp_utilities.model_circle((nn - nk) // 2, nn, nn, nn)
outdir = args[2]
elif len(args) == 4:
m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5)
outdir = args[3]
if os.path.exists(outdir):
sp_global_def.ERROR('Output directory already exists!')
else:
os.makedirs(outdir)
sp_global_def.write_command(outdir)
mc = sp_utilities.model_blank(nn, nn, nn, 1.0) - m
vf = sp_fundamentals.fft(vi)
uf = sp_fundamentals.fft(ui)
"""Multiline Comment1"""
lp = int(nn / 2 / options.step + 0.5)
step = 0.5 / lp
freqvol = sp_utilities.model_blank(nn, nn, nn)
resolut = []
for i in range(1, lp):
fl = step * i
fh = fl + step
#print(lp,i,step,fl,fh)
v = sp_fundamentals.fft(sp_filter.filt_tophatb(vf, fl, fh))
u = sp_fundamentals.fft(sp_filter.filt_tophatb(uf, fl, fh))
tmp1 = EMAN2_cppwrap.Util.muln_img(v, v)
tmp2 = EMAN2_cppwrap.Util.muln_img(u, u)
do = EMAN2_cppwrap.Util.infomask(
sp_morphology.square_root(
sp_morphology.threshold(
EMAN2_cppwrap.Util.muln_img(tmp1, tmp2))), m, True)[0]
tmp3 = EMAN2_cppwrap.Util.muln_img(u, v)
dp = EMAN2_cppwrap.Util.infomask(tmp3, m, True)[0]
resolut.append([i, (fl + fh) / 2.0, dp / do])
tmp1 = EMAN2_cppwrap.Util.box_convolution(tmp1, nk)
tmp2 = EMAN2_cppwrap.Util.box_convolution(tmp2, nk)
tmp3 = EMAN2_cppwrap.Util.box_convolution(tmp3, nk)
EMAN2_cppwrap.Util.mul_img(tmp1, tmp2)
tmp1 = sp_morphology.square_root(sp_morphology.threshold(tmp1))
EMAN2_cppwrap.Util.mul_img(tmp1, m)
EMAN2_cppwrap.Util.add_img(tmp1, mc)
EMAN2_cppwrap.Util.mul_img(tmp3, m)
EMAN2_cppwrap.Util.add_img(tmp3, mc)
EMAN2_cppwrap.Util.div_img(tmp3, tmp1)
EMAN2_cppwrap.Util.mul_img(tmp3, m)
freq = (fl + fh) / 2.0
bailout = True
for x in range(nn):
for y in range(nn):
for z in range(nn):
if (m.get_value_at(x, y, z) > 0.5):
if (freqvol.get_value_at(x, y, z) == 0.0):
if (tmp3.get_value_at(x, y, z) < cutoff):
freqvol.set_value_at(x, y, z, freq)
bailout = False
else:
bailout = False
if (bailout): break
#print(len(resolut))
# remove outliers
output_volume(freqvol, resolut, options.apix, outdir, options.prefix,
options.fsc, options.out_ang_res, nx, ny, nz,
res_overall)
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + " stack [output_directory] initial_volume --ir=inner_radius --ou=outer_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translational_search_step --delta=angular_step --an=angular_neighborhood --CTF --fl --aa --ref_a=S --sym=c1"
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--ir", type= "int", default= 1, help="inner radius for rotational correlation > 0 (set to 1)")
parser.add_option("--ou", type= "int", default= -1, help="outer radius for rotational correlation < int(nx/2)-1 (set to the radius of the particle)")
parser.add_option("--rs", type= "int", default= 1, help="step between rings in rotational correlation >0 (set to 1)" )
parser.add_option("--xr", type="string", default= "-1", help="range for translation search in x direction, search is +/xr (default 0)")
parser.add_option("--yr", type="string", default= "-1", help="range for translation search in y direction, search is +/yr (default = same as xr)")
parser.add_option("--ts", type="string", default= "1", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional")
parser.add_option("--delta", type="string", default= "-1", help="angular step of reference projections during initialization step (default automatically selected based on radius of the structure.)")
parser.add_option("--an", type="string", default= "-1", help="angular neighborhood for local searches (phi and theta) (Default exhaustive searches)")
parser.add_option("--CTF", action="store_true", default=False, help="Use CTF (Default no CTF correction)")
parser.add_option("--shrink", type="float", default= 1.0, help="Reduce data size by shrink factor (default 1.0)")
parser.add_option("--snr", type="float", default= 1.0, help="Signal-to-Noise Ratio of the data (default 1.0)")
parser.add_option("--ref_a", type="string", default= "S", help="method for generating the quasi-uniformly distributed projection directions (default S)")
parser.add_option("--sym", type="string", default= "c1", help="symmetry of the refined structure")
parser.add_option("--npad", type="int", default= 2, help="padding size for 3D reconstruction (default=2)")
#options introduced for the do_volume function
parser.add_option("--fl", type="float", default=0.12, help="cut-off frequency of hyperbolic tangent low-pass Fourier filte (default 0.12)")
parser.add_option("--aa", type="float", default=0.1, help="fall-off of hyperbolic tangent low-pass Fourier filter (default 0.1)")
parser.add_option("--pwreference", type="string", default="", help="text file with a reference power spectrum (default no power spectrum adjustment)")
parser.add_option("--mask3D", type="string", default=None, help="3D mask file (default a sphere WHAT RADIUS??)")
(options, args) = parser.parse_args(sys.argv[1:])
#print( " args ",args)
if( len(args) == 3):
volinit = args[2]
masterdir = args[1]
elif(len(args) == 2):
volinit = args[1]
masterdir = ""
else:
sxprint( "Usage: " + usage )
sxprint( "Please run \'" + progname + " -h\' for detailed options" )
ERROR( "Invalid number of parameters used. Please see usage information above." )
return
stack = args[0]
# INPUT PARAMETERS
radi = options.ou
sp_global_def.BATCH = True
ali3d_options.ir = options.ir
ali3d_options.rs = options.rs
ali3d_options.ou = options.ou
ali3d_options.xr = options.xr
ali3d_options.yr = options.yr
ali3d_options.ts = options.ts
ali3d_options.an = "-1"
ali3d_options.sym = options.sym
ali3d_options.delta = options.delta
ali3d_options.npad = options.npad
ali3d_options.CTF = options.CTF
ali3d_options.ref_a = options.ref_a
ali3d_options.snr = options.snr
ali3d_options.mask3D = options.mask3D
ali3d_options.pwreference = "" # It will have to be turned on after exhaustive done by setting to options.pwreference
ali3d_options.fl = 0.4
ali3d_options.initfl = 0.4
ali3d_options.aa = 0.1
nproc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
# Get the pixel size, if none set to 1.0, and the original image size
if(myid == main_node):
total_stack = EMUtil.get_image_count(stack)
a = get_im(stack)
nxinit = a.get_xsize()
if ali3d_options.CTF:
i = a.get_attr('ctf')
pixel_size = i.apix
fq = pixel_size/fq
else:
pixel_size = 1.0
# No pixel size, fusing computed as 5 Fourier pixels
fq = 5.0/nxinit
del a
else:
total_stack = 0
nxinit = 0
pixel_size = 1.0
total_stack = bcast_number_to_all(total_stack, source_node = main_node)
pixel_size = bcast_number_to_all(pixel_size, source_node = main_node)
nxinit = bcast_number_to_all(nxinit, source_node = main_node)
if(radi < 1): radi = nxinit//2-2
elif((2*radi+2)>nxinit):
ERROR("Particle radius set too large!", myid=myid )
ali3d_options.ou = radi
shrink = options.shrink
nxshrink = int(nxinit*shrink+0.5)
angular_neighborhood = "-1"
# MASTER DIRECTORY
if(myid == main_node):
sxprint( " masterdir ",masterdir)
if( masterdir == ""):
timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime())
masterdir = "master"+timestring
li = len(masterdir)
cmd = "{} {}".format("mkdir -p", masterdir)
junk = cmdexecute(cmd)
sp_global_def.write_command(masterdir)
else:
li = 0
li = mpi_bcast(li,1,MPI_INT,main_node,MPI_COMM_WORLD)[0]
if( li > 0 ):
masterdir = mpi_bcast(masterdir,li,MPI_CHAR,main_node,MPI_COMM_WORLD)
masterdir = string.join(masterdir,"")
nnxo = nxinit
# INITIALIZATION
initdir = masterdir
# This is initial setting, has to be initialized here, we do not want it to run too long.
# INITIALIZATION THAT FOLLOWS WILL HAVE TO BE CHANGED SO THE USER CAN PROVIDE INITIAL GUESS OF RESOLUTION
# If we new the initial resolution, it could be done more densely
if(options.xr == "-1"): xr = "%d"%((nnxo - (2*radi-1))//2)
else: xr = options.xr
if(options.yr == "-1"): yr = xr
else: yr = options.yr
delta = float(options.delta)
if(delta <= 0.0): delta = "%f"%round(degrees(atan(1.0/float(radi))), 2)
else: delta = "%f"%delta
paramsdict = { "stack":stack,"delta":delta, "ts":"1.0", "xr":xr, "an":angular_neighborhood, \
"center":"0", "maxit":1, "local":False,\
"lowpass":options.fl, "initialfl":0.4, "falloff":options.aa, "radius":radi, \
"nsoft":0, "delpreviousmax":True, "shrink":options.shrink, "saturatecrit":1.0, "pixercutoff":2.0,\
"refvol":volinit, "mask3D":options.mask3D}
partids = os.path.join(masterdir, "ids.txt")
partstack = os.path.join(masterdir, "paramszero.txt")
if( myid == main_node ):
write_text_file(list(range(total_stack)), partids)
write_text_row([[0.0,0.0,0.0,0.0,0.0] for i in range(total_stack) ], partstack)
run3Dalignment(paramsdict, partids, partstack, initdir, 0, myid, main_node, nproc)
mpi_barrier(MPI_COMM_WORLD)
sp_global_def.BATCH = True
if not os.path.exists(args.input_run_dir):
sp_global_def.ERROR("Input directory does not exist!", "sxbatch.py", 1)
qsub_dict = {
"qsub": glob.re.compile("Your job (\w+)"),
"sbatch": glob.re.compile("Submitted batch job (\w+)"),
}
if args.submission_command.split()[0] not in qsub_dict and args.hold_flag:
sp_global_def.ERROR(
"Qsub return output not known! Please contact the SPHIRE authors!",
"sxbatch.py",
1,
)
sp_global_def.write_command(".")
if args.first_hold_number:
prev_hold = args.first_hold_number
else:
prev_hold = "aaa"
for idx, file_name in enumerate(
sorted(glob.glob("{0}/*".format(args.input_run_dir)))):
command = args.submission_command.split()
if args.hold_flag and (idx != 0 or args.first_hold_number):
command.append("{0}{1}".format(args.hold_flag, prev_hold))
else:
pass
command.append(file_name)
if args.hold_flag:
def _main_():
_, child_argparser = setup_argparser()
# These global variables are kind of ugly, but are necessary in python2. Will be removed in
# python 3.
global STACK_FILE_PATH
global PROJECTION_PARAMETERS
global VOLUME1
global VOLUME2
global DEFOCUS_SEARCH_RANGE
global DEFOCUS_STEP_SIZE
global MASK_VOLUME
global RESOLUTION
global PIXEL_SIZE
global HALF_MAP_ASSIGNEMENTS
args = child_argparser.parse_args()
if "meridien" in sys.argv[1]:
meridien_path = args.meridien_path
files = sp_ctf_refine_io.read_meridien_data(meridien_path)
volume1_file_path = files["first_halfmap"]
volume2_file_path = files["second_halfmap"]
chunk_file_path = files["chunk1"]
params_file_path = files["final_params"]
else:
volume1_file_path = args.volume
volume2_file_path = args.volume2
params_file_path = args.params_path
chunk_file_path = args.chunk
if volume2_file_path is None and chunk_file_path is not None:
sp_global_def.ERROR(
"If chunk file is specified, you need to specify a second volume (-v2)"
)
STACK_FILE_PATH = args.inputstack # "bdb:name"
mask_file_path = args.mask
DEFOCUS_SEARCH_RANGE = args.range
DEFOCUS_STEP_SIZE = args.delta
output_folder = args.outputdir
if os.path.exists(output_folder):
sp_global_def.ERROR("Output folder already exists. Stop execution.")
else:
os.makedirs(output_folder)
sp_global_def.write_command(output_folder)
output_virtual_stack_path = "bdb:" + os.path.join(output_folder, "ctf_refined")
output_stats_path = os.path.join(output_folder, "statistics")
number_of_particles_to_read = args.number_particles
volume_nominal_resolution = args.resolution
RESOLUTION = volume_nominal_resolution
PIXEL_SIZE = args.apix
PROJECTION_PARAMETERS = sp_ctf_refine_io.read_meridien_params(params_file_path)
num_cpu = multiprocessing.cpu_count() - 1
volume1, volume2, MASK_VOLUME = sp_ctf_refine_io.read_volume(
path_vol_1=volume1_file_path,
path_vol_2=volume2_file_path,
path_mask=mask_file_path,
resolution=volume_nominal_resolution,
pixel_size=PIXEL_SIZE,
)
VOLUME1 = volume1
VOLUME2 = volume2
particle_chunks, number_of_particles = indices_to_chunks(
STACK_FILE_PATH,
chunk_size=100,
number_of_particles_to_read=number_of_particles_to_read,
)
if chunk_file_path:
HALF_MAP_ASSIGNEMENTS = get_half_map_assigments_per_particle(
stack_path=STACK_FILE_PATH,
chunk_path=chunk_file_path,
number_of_particles_to_read=number_of_particles_to_read,
)
sp_global_def.sxprint("####Start refinement####")
start = time.time()
# for chunk in particle_chunks:
# refine_set(chunk)
#####################################################################
# Python 2 workaround to get rid of a memory leak.
# Pool eats up memory and only give it back after it returns.
# So let it return more often by deviding chunks into chunks :-)
#####################################################################
num_chunks = len(particle_chunks)
refinement_results = []
with tqdm(total=num_chunks, file=sys.stdout) as pbar:
for i in range(0, num_chunks, num_cpu):
subset_chunk = particle_chunks[i: (i + num_cpu)]
pool = multiprocessing.Pool(num_cpu)
refinement_result = pool.map_async(refine_set, subset_chunk, chunksize=1)
pool.close()
# print_progress(refinement_result)
pool.join()
for res in refinement_result.get():
refinement_results.append(res)
pbar.update(len(subset_chunk))
#####################################################################
end = time.time()
sp_global_def.sxprint("Time for ", number_of_particles, " Particles:", end - start)
# Ouput results
refined_ctfs_as_list, refinement_results_per_micrograph = merge_ctf_refinement_results(
refinement_results=refinement_results
)
sp_ctf_refine_io.write_virtual_bdb_stack(
output_stack_path=output_virtual_stack_path,
origin_stack_path=STACK_FILE_PATH,
refined_ctfs=refined_ctfs_as_list,
number_of_particles=number_of_particles,
)
sp_global_def.sxprint("Write statistics...")
# WRITE STATISTICS
if not os.path.exists(output_stats_path):
os.makedirs(output_stats_path)
refinement_stats_per_micrograh = calc_statistics(refinement_results_per_micrograph)
sp_ctf_refine_io.write_statistics(output_stats_path, refinement_stats_per_micrograh)
# Estimate error Range
min_max_error, min_max_ratio = calculate_result_ranges(
refinement_results_per_micrograph
)
# Save particle plots
sp_global_def.sxprint("Write images...")
path_output_img = os.path.join(output_stats_path, "img/")
if not os.path.exists(path_output_img):
os.makedirs(path_output_img)
sp_ctf_refine_plotting.create_and_save_particle_plots(
path_output_img=path_output_img,
stack_file_path=STACK_FILE_PATH,
refinement_results_per_micrograph=refinement_results_per_micrograph,
min_max_error=min_max_error,
min_max_ratio=min_max_ratio,
)
sp_global_def.sxprint("Write other...")
refinement_results_matrix = get_refinement_results_matrix(
refinement_results_per_micrograph
)
particle_error_path = os.path.join(output_stats_path, "particle_results.txt")
refinement_results_matrix = refinement_results_matrix[
refinement_results_matrix[:, 0].argsort()
]
np.savetxt(
particle_error_path,
refinement_results_matrix,
delimiter=",",
fmt=["%d", "%f", "%f", "%f"],
)
sp_global_def.sxprint("Done")
def main():
# Read arguments
args = argparser.parse_args()
architecture = args.architecture
particle_diameter = args.particle_diameter
trainging_dir = args.training_dir
annot_dir = args.annot_dir
input_size = args.input_size
output_dir = args.output_directory
if os.path.exists(output_dir):
sp_global_def.ERROR("Output folder already exists. Stop execution.")
else:
os.makedirs(output_dir)
sp_global_def.write_command(output_dir)
num_patches = args.num_patches
overlap_patches = args.overlap_patches
train_times = args.train_times
saved_weights_name = args.saved_weights_name
saved_weights_name = os.path.join(output_dir, saved_weights_name)
pretrained_weights_name = args.pretrained_weights_name
batch_size = args.batch_size
learning_rate = args.learning_rate
np_epoch = args.np_epoch
object_scale = args.object_scale
no_object_scale = args.no_object_scale
coord_scale = args.coord_scale
valid_image_dir = args.valid_image_dir
valid_annot_dir = args.valid_annot_dir
fine_tune = args.fine_tune
gpu_fraction = args.gpu_fraction
num_cpu = args.num_cpu
cryolo_train_path = args.cryolo_train_path
skiplowpass = args.skiplowpass
cutoff = args.cutoff
filtered_dir = args.filtered_dir
warmup = args.warmup
early_stop = int(args.early)
str_gpus = [str(entry).strip() for entry in args.gpu.split(",")]
arg_gpu = ' '.join(str_gpus)
# Create config file
model_dict = {'architecture': architecture,
'input_size': input_size,
'anchors': [particle_diameter, particle_diameter],
'overlap_patches': overlap_patches,
'max_box_per_image': 1000,
'num_patches': num_patches}
if not skiplowpass:
model_dict['filter'] = [cutoff,filtered_dir]
else:
if args.usejanni:
model_dict['filter'] = [args.janni_model, args.janni_overlap, args.janni_batches, filtered_dir]
train_dict = {'train_image_folder': trainging_dir,
'train_annot_folder': annot_dir,
'train_times': train_times,
'pretrained_weights': pretrained_weights_name,
'batch_size': batch_size,
'learning_rate': learning_rate,
'nb_epoch': np_epoch,
'warmup_epochs': 0,
'object_scale': object_scale,
'no_object_scale': no_object_scale,
'coord_scale': coord_scale,
'class_scale': 1.0,
"saved_weights_name": saved_weights_name,
"debug": True,
"log_path": "cryolo_logs/"
}
valid_dict = {'valid_image_folder': valid_image_dir,
'valid_annot_folder': valid_annot_dir,
'valid_times': 1
}
dict = {"model": model_dict, "train": train_dict, "valid": valid_dict}
path = os.path.join(output_dir,"config_yolo.json")
with open(path, 'w') as f:
dump(dict, f, ensure_ascii=False, indent=4)
# Run the training
config_argument = "-c=" + path
warmup_argument = "-w=" + str(warmup)
gpu_argument = "-g " + arg_gpu
early_stop = "-e=" + str(early_stop)
fine_tune_argument = ""
if fine_tune:
fine_tune_argument = "--fine_tune"
gpu_fraction_arg = "--gpu_fraction=1.0"
if gpu_fraction < 1.0 and gpu_fraction > 0.0:
gpu_fraction_arg = "--gpu_fraction=" + str(gpu_fraction)
num_cpu_arg = ""
if num_cpu != -1:
num_cpu_arg = "--num_cpu="+str(num_cpu)
cryolo_ex_pth = "cryolo_train.py"
if cryolo_train_path:
cryolo_ex_pth = cryolo_train_path
'''
if not fine_tune:
command = [cryolo_ex_pth, "-c=config_yolo.json", warmup_argument, gpu_argument, early_stop, gpu_fraction_arg]
if num_cpu != -1:
command.append(num_cpu_arg)
subprocess.check_call(
command)
'''
if fine_tune:
warmup_argument = "-w=0"
command = [cryolo_ex_pth, config_argument, warmup_argument, gpu_argument, early_stop, gpu_fraction_arg]
if fine_tune:
command.append(fine_tune_argument)
if num_cpu != -1:
command.append(num_cpu_arg)
subprocess.check_call(
command)
def main():
# Parse the Options
progname = os.path.basename(sys.argv[0])
usage = (
progname
+ """ summovie_path input_micrograph_pattern input_shift_pattern output_directory
--selection_list
--nr_frames=nr_frames
--first
--last
--pixel_size=pixel_size
--nr_threads
--apply_dose_filter
--voltage=voltage
--exposure_per_frame=exposure_per_frame
--pre_exposure=pre_exposure
--dont_restore_noise
sxsummovie exists only in non-MPI version.
Perform summovie without dose filtering.
sxsummovie.py ~/my_app/summovie 'outdir_unblur/corrsum/micrograph_*_frames_sum.mrc' 'outdir_unblur/shift/micrograph_*_frames_shift.txt'
outdir_summovie --nr_frames=24 --pixel_size=1.19 --nr_threads=1
Perform summovie without dose filtering and with less frames.
sxsummovie.py ~/my_app/summovie 'outdir_unblur/corrsum/micrograph_*_frames_sum.mrc' 'outdir_unblur/shift/micrograph_*_frames_shift.txt'
outdir_summovie --nr_frames=24 --first=3 --last=15 --pixel_size=1.19 --nr_threads=1
Perform summovie with dose filtering and with less frames.
sxsummovie.py ~/my_app/summovie 'outdir_unblur/corrsum/micrograph_*_frames_sum.mrc' 'outdir_unblur/shift/micrograph_*_frames_shift.txt'
outdir_summovie --nr_frames=24 --first=3 --last=15 --pixel_size=1.19 --nr_threads=1 --apply_dose_filter --voltage=300 --exposure_per_frame=2 --pre_exposure=0
"""
)
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option(
"--selection_list",
type="str",
default="",
help="Micrograph selecting list (SPHIRE specific): Specify a name of micrograph selection list text file. The file extension must be '.txt'. If this is not provided, all files matched with the micrograph name pattern will be processed. (default none)",
)
parser.add_option(
"--nr_frames",
type="int",
default=3,
help="Number of movie frames: The number of movie frames in each input micrograph. (default 3)",
)
parser.add_option(
"--first",
type="int",
default=1,
help="First movie frame: First movie frame for summing. (default 1)",
)
parser.add_option(
"--last",
type="int",
default=-1,
help="Last movie frame: Last movie frame for summing. (default -1)",
)
parser.add_option(
"--sum_suffix", type="str", default="_sum", help=optparse.SUPPRESS_HELP
)
parser.add_option(
"--pixel_size",
type="float",
default=-1.0,
help="Pixel size [A]: The pixel size of input micrographs. (default required float)",
)
parser.add_option(
"--nr_threads",
type="int",
default=1,
help="Number of threads: The number of threads summovie can use. The higher the faster, but it requires larger memory. (default 1)",
)
parser.add_option(
"--apply_dose_filter",
action="store_true",
default=False,
help="Apply dose filter step: Requires voltage, exposure per frame, and pre exposure options. (default False)",
)
parser.add_option(
"--voltage",
type="float",
default=300.0,
help="Microscope voltage (dose filter) [kV]: The acceleration voltage of microscope used for imaging. (default 300.0)",
)
parser.add_option(
"--exposure_per_frame",
type="float",
default=2.0,
help="Per frame exposure (dose filter) [e/A^2]: The electron dose per frame in e/A^2. (default 2.0)",
)
parser.add_option(
"--pre_exposure",
type="float",
default=0.0,
help="Pre-exposure (dose filter) [e/A^2]: The electron does in e/A^2 used for exposure prior to imaging .(default 0.0)",
)
parser.add_option(
"--frc_suffix", type="string", default="_frc", help=optparse.SUPPRESS_HELP
)
parser.add_option(
"--dont_restore_noise",
action="store_true",
default=False,
help="Do not restore noise power: Do not restore noise power. (default False)",
)
parser.add_option(
"--summovie_ready",
action="store_true",
default=False,
help=optparse.SUPPRESS_HELP,
)
# list of the options and the arguments
(options, args) = parser.parse_args(sys.argv[1:])
sp_global_def.BATCH = True
# If there arent enough arguments, stop the script
if len(args) != 4:
sp_global_def.sxprint("Usage: " + usage)
sp_global_def.ERROR(
"Invalid number of parameters used. Please see usage information above."
)
return
# Convert the realtive parts to absolute ones
summovie_path = os.path.realpath(args[0]) # summovie_path
input_image = os.path.realpath(args[1]) # input_micrograph_pattern
input_shift = os.path.realpath(args[2]) # input_shift_pattern
output_dir = os.path.realpath(args[3]) # output_directory
# If the summovie executable file does not exists, stop the script
if not os.path.exists(summovie_path):
sp_global_def.ERROR(
"Summovie directory does not exist, please change the name and restart the program."
)
return
# If the output directory exists, stop the script
if os.path.exists(output_dir):
sp_global_def.ERROR(
"Output directory exists, please change the name and restart the program."
)
return
# If the input file does not exists, stop the script
file_list = glob.glob(input_image)
shift_list = glob.glob(input_shift)
if not file_list:
sp_global_def.ERROR(
"Input micrograph file(s) does not exist, please change the name and restart the program."
)
return
if not shift_list:
sp_global_def.ERROR(
"Input shift file(s) does not exist, please change the name and restart the program."
)
return
# Output paths
if options.apply_dose_filter:
output_path = "{:s}/corrsum_dose_filtered".format(output_dir)
else:
output_path = "{:s}/corrsum".format(output_dir)
frc_path = "{:s}/frc".format(output_dir)
log_path = "{:s}/logfiles".format(output_dir)
temp_path = "{0}/temp".format(output_dir)
# Split the path of the image name at the "/" Characters.
# The last entry contains the micrograph name.
# Split the micrograph name at the wildcard character for the
# prefix and suffix.
input_mic_split = input_image.split("/")
input_mic_name = input_mic_split[-1].split("*")
input_shift_name = input_shift.split("*")
if len(input_mic_name) != 2 or len(input_shift_name) != 2:
sp_global_def.ERROR(
"Too many wildcard arguments. Please use exactly one '*' in the pattern."
)
return
# Get the input directory
if len(input_mic_split) != 1:
input_dir = input_image[: -len(input_mic_split[-1])]
else:
input_dir = ""
# Create output directorys
if not os.path.exists(output_dir):
os.makedirs(output_dir)
sp_global_def.write_command(output_dir)
if not os.path.exists(output_path):
os.makedirs(output_path)
sp_global_def.write_command(output_path)
if not os.path.exists(frc_path):
os.mkdir(frc_path)
if not os.path.exists(temp_path) and not options.summovie_ready:
os.mkdir(temp_path)
if not os.path.exists(log_path):
os.mkdir(log_path)
# shift wildcard list
shift_wildcard = [
entry[len(input_shift_name[0]) : -len(input_shift_name[-1])]
for entry in shift_list
]
# Just use shifts that have a micrograph and vise versa
mic_list = [
entry
for entry in file_list
if entry[len(input_dir) + len(input_mic_name[0]) : -len(input_mic_name[-1])]
in shift_wildcard
]
# If micrograph list is provided just process the images in the list
selection_file = options.selection_list
if selection_file:
# Import list file
try:
selection = numpy.genfromtxt(selection_file, dtype=None)
except TypeError:
sp_global_def.ERROR(
"No entrys in micrograph list file '" + selection_file + "'"
)
return
# List of files which are in pattern and list
mic_list = [
entry
for entry in mic_list
if entry[len(input_dir) :] in selection and os.path.exists(entry)
]
# If no match is there abort
if len(mic_list) == 0:
sp_global_def.ERROR(
"No files in '"
+ selection_file
+ "' matched the micrograph file pattern"
)
return
option_dict = {
"summovie_path": summovie_path,
"mic_list": mic_list,
"mic_prefix": input_mic_name[0],
"mic_suffix": input_mic_name[1],
"shift_prefix": input_shift_name[0],
"shift_suffix": input_shift_name[1],
"input_dir": input_dir,
"output_dir": output_dir,
"output_path": output_path,
"frc_path": frc_path,
"log_path": log_path,
"temp_path": temp_path,
"nr_frames": options.nr_frames,
"sum_suffix": options.sum_suffix,
"first": options.first,
"last": options.last,
"pixel_size": options.pixel_size,
"apply_dose_filter": options.apply_dose_filter,
"exposure_per_frame": options.exposure_per_frame,
"voltage": options.voltage,
"pre_exposure": options.pre_exposure,
"frc_suffix": options.frc_suffix,
"dont_restore_noise": options.dont_restore_noise,
"nr_threads": options.nr_threads,
"summovie_ready": options.summovie_ready,
"verbose": True,
}
# Run summovie
run_summovie(opt=option_dict)
if not options.summovie_ready:
# Remove temp folder
for entry in glob.glob("{0}/*".format(temp_path)):
os.remove(entry)
os.rmdir(temp_path)
sp_global_def.sxprint("All Done!")
sp_global_def.BATCH = False
def main():
"""
Main function.
Arguments:
None
Returns:
None
"""
command_args = parse_command_line()
# Import volume
sp_global_def.sxprint("Import volume.")
input_vol = sp_utilities.get_im(command_args.input_volume)
# Sanity checks
sanity_checks(command_args, input_vol)
try:
os.makedirs(command_args.output_dir)
except OSError:
sp_global_def.sxprint(
"Output directory already exists. No need to create it.")
else:
sp_global_def.sxprint("Created output directory.")
sp_global_def.write_command(command_args.output_dir)
output_prefix = os.path.join(command_args.output_dir, command_args.prefix)
# Filter volume if specified
if command_args.low_pass_filter_resolution is not None:
sp_global_def.sxprint("Filter volume to {0}A.".format(
command_args.low_pass_filter_resolution))
input_vol = sp_filter.filt_tanl(
input_vol,
old_div(command_args.pixel_size,
command_args.low_pass_filter_resolution),
command_args.low_pass_filter_falloff,
)
input_vol.write_image(output_prefix + "_filtered_volume.hdf")
else:
sp_global_def.sxprint("Skip filter volume.")
# Create a mask based on the filtered volume
sp_global_def.sxprint("Create mask")
density_threshold = -9999.0
nsigma = 1.0
if command_args.mol_mass:
density_threshold = input_vol.find_3d_threshold(
command_args.mol_mass, command_args.pixel_size)
sp_global_def.sxprint(
"Mask molecular mass translated into binary threshold: ",
density_threshold)
elif command_args.threshold:
density_threshold = command_args.threshold
elif command_args.nsigma:
nsigma = command_args.nsigma
else:
assert False
if command_args.edge_type == "cosine":
mode = "C"
elif command_args.edge_type == "gaussian":
mode = "G"
else:
assert False
mask_first = sp_morphology.adaptive_mask_scipy(
input_vol,
nsigma=nsigma,
threshold=density_threshold,
ndilation=command_args.ndilation,
nerosion=command_args.nerosion,
edge_width=command_args.edge_width,
allow_disconnected=command_args.allow_disconnected,
mode=mode,
do_approx=command_args.do_old,
do_fill=command_args.fill_mask,
do_print=True,
)
# Create a second mask based on the filtered volume
s_mask = None
s_density_threshold = 1
s_nsigma = 1.0
if command_args.second_mask is not None:
sp_global_def.sxprint("Prepare second mask")
s_mask = sp_utilities.get_im(command_args.second_mask)
s_density_threshold = -9999.0
s_nsigma = 1.0
if command_args.s_mol_mass:
s_density_threshold = input_vol.find_3d_threshold(
command_args.s_mol_mass, command_args.s_pixel_size)
sp_global_def.sxprint(
"Second mask molecular mass translated into binary threshold: ",
s_density_threshold,
)
elif command_args.s_threshold:
s_density_threshold = command_args.s_threshold
elif command_args.s_nsigma:
s_nsigma = command_args.s_nsigma
else:
assert False
elif command_args.second_mask_shape is not None:
sp_global_def.sxprint("Prepare second mask")
nx = mask_first.get_xsize()
ny = mask_first.get_ysize()
nz = mask_first.get_zsize()
if command_args.second_mask_shape == "cube":
s_nx = command_args.s_nx
s_ny = command_args.s_ny
s_nz = command_args.s_nz
s_mask = sp_utilities.model_blank(s_nx, s_ny, s_nz, 1)
elif command_args.second_mask_shape == "cylinder":
s_radius = command_args.s_radius
s_nx = command_args.s_nx
s_ny = command_args.s_ny
s_nz = command_args.s_nz
try:
s_mask = sp_utilities.model_cylinder(s_radius, s_nx, s_ny,
s_nz)
except RuntimeError as e:
sp_global_def.sxprint(
"An error occured! Please check the error log")
raise
elif command_args.second_mask_shape == "sphere":
s_radius = command_args.s_radius
s_nx = command_args.s_nx
s_ny = command_args.s_ny
s_nz = command_args.s_nz
try:
s_mask = sp_utilities.model_circle(s_radius, s_nx, s_ny, s_nz)
except RuntimeError as e:
sp_global_def.sxprint(
"An error occured! Please check the error log")
raise
else:
assert False
s_mask = sp_utilities.pad(s_mask, nx, ny, nz, 0)
if s_mask is not None:
sp_global_def.sxprint("Create second mask")
if command_args.s_edge_type == "cosine":
mode = "C"
elif command_args.s_edge_type == "gaussian":
mode = "G"
else:
assert False
s_mask = sp_morphology.adaptive_mask_scipy(
s_mask,
nsigma=s_nsigma,
threshold=s_density_threshold,
ndilation=command_args.s_ndilation,
nerosion=command_args.s_nerosion,
edge_width=command_args.s_edge_width,
allow_disconnected=command_args.s_allow_disconnected,
mode=mode,
do_approx=command_args.s_do_old,
do_fill=command_args.s_fill_mask,
do_print=True,
)
if command_args.s_invert:
s_mask = 1 - s_mask
sp_global_def.sxprint("Write outputs.")
mask_first.write_image(output_prefix + "_mask_first.hdf")
s_mask.write_image(output_prefix + "_mask_second.hdf")
masked_combined = mask_first * s_mask
masked_combined.write_image(output_prefix + "_mask.hdf")
else:
sp_global_def.sxprint("Write outputs.")
mask_first.write_image(output_prefix + "_mask.hdf")
def main():
from time import sleep
from sp_logger import Logger, BaseLogger_Files
arglist = []
i = 0
while( i < len(sys.argv) ):
if sys.argv[i]=='-p4pg':
i = i+2
elif sys.argv[i]=='-p4wd':
i = i+2
else:
arglist.append( sys.argv[i] )
i = i+1
progname = os.path.basename(arglist[0])
usage = progname + " stack outdir <mask> --focus=3Dmask --radius=outer_radius --delta=angular_step" +\
"--an=angular_neighborhood --maxit=max_iter --CTF --sym=c1 --function=user_function --independent=indenpendent_runs --number_of_images_per_group=number_of_images_per_group --low_pass_filter=.25 --seed=random_seed"
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--focus", type="string", default=None, help="3D mask for focused clustering ")
parser.add_option("--ir", type= "int", default= 1, help="inner radius for rotational correlation > 0 (set to 1)")
parser.add_option("--radius", type= "int", default=-1, help="outer radius for rotational correlation <nx-1 (set to the radius of the particle)")
parser.add_option("--maxit", type= "int", default=50, help="maximum number of iteration")
parser.add_option("--rs", type= "int", default=1, help="step between rings in rotational correlation >0 (set to 1)" )
parser.add_option("--xr", type="string", default='1', help="range for translation search in x direction, search is +/-xr ")
parser.add_option("--yr", type="string", default='-1', help="range for translation search in y direction, search is +/-yr (default = same as xr)")
parser.add_option("--ts", type="string", default='0.25', help="step size of the translation search in both directions direction, search is -xr, -xr+ts, 0, xr-ts, xr ")
parser.add_option("--delta", type="string", default='2', help="angular step of reference projections")
parser.add_option("--an", type="string", default='-1', help="angular neighborhood for local searches")
parser.add_option("--center", type="int", default=0, help="0 - if you do not want the volume to be centered, 1 - center the volume using cog (default=0)")
parser.add_option("--nassign", type="int", default=1, help="number of reassignment iterations performed for each angular step (set to 3) ")
parser.add_option("--nrefine", type="int", default=0, help="number of alignment iterations performed for each angular step (set to 1) ")
parser.add_option("--CTF", action="store_true", default=False, help="Consider CTF correction during the alignment ")
parser.add_option("--stoprnct", type="float", default=3.0, help="Minimum percentage of assignment change to stop the program")
parser.add_option("--sym", type="string", default='c1', help="symmetry of the structure ")
parser.add_option("--function", type="string", default='do_volume_mrk05', help="name of the reference preparation function")
parser.add_option("--independent", type="int", default= 3, help="number of independent run")
parser.add_option("--number_of_images_per_group", type='int', default=1000, help="number of images per groups")
parser.add_option("--low_pass_filter", type="float", default=-1.0, help="absolute frequency of low-pass filter for 3d sorting on the original image size" )
parser.add_option("--nxinit", type="int", default=64, help="initial image size for sorting" )
parser.add_option("--unaccounted", action="store_true", default=False, help="reconstruct the unaccounted images")
parser.add_option("--seed", type="int", default=-1, help="random seed for create initial random assignment for EQ Kmeans")
parser.add_option("--smallest_group", type="int", default=500, help="minimum members for identified group" )
parser.add_option("--previous_run1", type="string", default='', help="two previous runs" )
parser.add_option("--previous_run2", type="string", default='', help="two previous runs" )
parser.add_option("--group_size_for_unaccounted", type="int", default=500, help="size for unaccounted particles" )
parser.add_option("--chunkdir", type="string", default='', help="chunkdir for computing margin of error")
parser.add_option("--sausage", action="store_true", default=False, help="way of filter volume")
parser.add_option("--PWadjustment", type="string", default='', help="1-D power spectrum of PDB file used for EM volume power spectrum correction")
parser.add_option("--protein_shape", type="string", default='g', help="protein shape. It defines protein preferred orientation angles. Currently it has g and f two types ")
parser.add_option("--upscale", type="float", default=0.5, help=" scaling parameter to adjust the power spectrum of EM volumes")
parser.add_option("--wn", type="int", default=0, help="optimal window size for data processing")
parser.add_option("--interpolation", type="string", default="4nn", help="3-d reconstruction interpolation method, two options, trl and 4nn")
(options, args) = parser.parse_args(arglist[1:])
if len(args) < 1 or len(args) > 4:
sxprint("Usage: " + usage)
sxprint("Please run \'" + progname + " -h\' for detailed options")
ERROR( "Invalid number of parameters used. Please see usage information above." )
return
else:
if len(args)>2:
mask_file = args[2]
else:
mask_file = None
orgstack =args[0]
masterdir =args[1]
sp_global_def.BATCH = True
#---initialize MPI related variables
nproc = mpi.mpi_comm_size( mpi.MPI_COMM_WORLD )
myid = mpi.mpi_comm_rank( mpi.MPI_COMM_WORLD )
mpi_comm = mpi.MPI_COMM_WORLD
main_node = 0
# Create the main log file
from sp_logger import Logger,BaseLogger_Files
if myid ==main_node:
log_main=Logger(BaseLogger_Files())
log_main.prefix=masterdir+"/"
else:
log_main = None
#--- fill input parameters into dictionary named after Constants
Constants ={}
Constants["stack"] =args[0]
Constants["masterdir"] =masterdir
Constants["mask3D"] =mask_file
Constants["focus3Dmask"] =options.focus
Constants["indep_runs"] =options.independent
Constants["stoprnct"] =options.stoprnct
Constants["number_of_images_per_group"] =options.number_of_images_per_group
Constants["CTF"] =options.CTF
Constants["maxit"] =options.maxit
Constants["ir"] =options.ir
Constants["radius"] =options.radius
Constants["nassign"] =options.nassign
Constants["rs"] =options.rs
Constants["xr"] =options.xr
Constants["yr"] =options.yr
Constants["ts"] =options.ts
Constants["delta"] =options.delta
Constants["an"] =options.an
Constants["sym"] =options.sym
Constants["center"] =options.center
Constants["nrefine"] =options.nrefine
Constants["user_func"] =options.function
Constants["low_pass_filter"] =options.low_pass_filter # enforced low_pass_filter
Constants["main_log_prefix"] =args[1]
#Constants["importali3d"] =options.importali3d
Constants["myid"] =myid
Constants["main_node"] =main_node
Constants["nproc"] =nproc
Constants["log_main"] =log_main
Constants["nxinit"] =options.nxinit
Constants["unaccounted"] =options.unaccounted
Constants["seed"] =options.seed
Constants["smallest_group"] =options.smallest_group
Constants["previous_runs"] =options.previous_run1+" "+options.previous_run2
Constants["sausage"] =options.sausage
Constants["chunkdir"] =options.chunkdir
Constants["PWadjustment"] =options.PWadjustment
Constants["upscale"] =options.upscale
Constants["wn"] =options.wn
Constants["3d-interpolation"] =options.interpolation
Constants["protein_shape"] =options.protein_shape
#Constants["frequency_stop_search"] =options.frequency_stop_search
#Constants["scale_of_number"] =options.scale_of_number
# -------------------------------------------------------------
#
# Create and initialize Tracker dictionary with input options
Tracker = {}
Tracker["constants"] = Constants
Tracker["maxit"] = Tracker["constants"]["maxit"]
Tracker["radius"] = Tracker["constants"]["radius"]
#Tracker["xr"] = ""
#Tracker["yr"] = "-1" # Do not change!
#Tracker["ts"] = 1
#Tracker["an"] = "-1"
#Tracker["delta"] = "2.0"
#Tracker["zoom"] = True
#Tracker["nsoft"] = 0
#Tracker["local"] = False
Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"]
Tracker["upscale"] = Tracker["constants"]["upscale"]
Tracker["applyctf"] = False # Should the data be premultiplied by the CTF. Set to False for local continuous.
#Tracker["refvol"] = None
Tracker["nxinit"] = Tracker["constants"]["nxinit"]
#Tracker["nxstep"] = 32
Tracker["icurrentres"] = -1
#Tracker["ireachedres"] = -1
Tracker["lowpass"] = Tracker["constants"]["low_pass_filter"]
Tracker["falloff"] = 0.1
#Tracker["inires"] = options.inires # Now in A, convert to absolute before using
Tracker["fuse_freq"] = 50 # Now in A, convert to absolute before using
#Tracker["delpreviousmax"]= False
#Tracker["anger"] = -1.0
#Tracker["shifter"] = -1.0
#Tracker["saturatecrit"] = 0.95
#Tracker["pixercutoff"] = 2.0
#Tracker["directory"] = ""
#Tracker["previousoutputdir"] = ""
#Tracker["eliminated-outliers"] = False
#Tracker["mainiteration"] = 0
#Tracker["movedback"] = False
#Tracker["state"] = Tracker["constants"]["states"][0]
#Tracker["global_resolution"] = 0.0
Tracker["orgstack"] = orgstack
#--------------------------------------------------------------------
# import from utilities
from sp_utilities import sample_down_1D_curve,get_initial_ID,remove_small_groups,print_upper_triangular_matrix,print_a_line_with_timestamp
from sp_utilities import convertasi,prepare_ptp,print_dict,get_resolution_mrk01,partition_to_groups,partition_independent_runs,get_outliers
from sp_utilities import merge_groups, save_alist, margin_of_error, get_margin_of_error, do_two_way_comparison, select_two_runs, get_ali3d_params
from sp_utilities import counting_projections, unload_dict, load_dict, get_stat_proj, create_random_list, get_number_of_groups, recons_mref
from sp_utilities import apply_low_pass_filter, get_groups_from_partition, get_number_of_groups, get_complementary_elements_total, update_full_dict
from sp_utilities import count_chunk_members, set_filter_parameters_from_adjusted_fsc, get_two_chunks_from_stack
####------------------------------------------------------------------
# another part
from sp_utilities import get_class_members, remove_small_groups, get_number_of_groups, get_stable_members_from_two_runs
from sp_utilities import two_way_comparison_single, get_leftover_from_stable, get_initial_ID, Kmeans_exhaustive_run
from sp_utilities import print_a_line_with_timestamp, split_a_group
#
# Get the pixel size; if none, set to 1.0, and the original image size
from sp_utilities import get_shrink_data_huang
from time import sleep
import sp_user_functions
user_func = sp_user_functions.factory[Tracker["constants"]["user_func"]]
if(myid == main_node):
line = ''
sxprint((line+"Initialization of 3-D sorting"))
a = get_im(Tracker["orgstack"])
nnxo = a.get_xsize()
if( Tracker["nxinit"] > nnxo ):
ERROR( "Image size less than minimum permitted $d"%Tracker["nxinit"] )
nnxo = -1 # we break here, so not sure what this is supposed to accomplish
return
else:
if Tracker["constants"]["CTF"]:
i = a.get_attr('ctf')
pixel_size = i.apix
fq = pixel_size/Tracker["fuse_freq"]
else:
pixel_size = 1.0
# No pixel size, fusing computed as 5 Fourier pixels
fq = 5.0/nnxo
del a
else:
nnxo = 0
fq = 0.0
pixel_size = 1.0
nnxo = bcast_number_to_all(nnxo, source_node = main_node)
if( nnxo < 0 ):
return
pixel_size = bcast_number_to_all(pixel_size, source_node = main_node)
fq = bcast_number_to_all(fq, source_node = main_node)
if Tracker["constants"]["wn"]==0:
Tracker["constants"]["nnxo"] = nnxo
else:
Tracker["constants"]["nnxo"] = Tracker["constants"]["wn"]
nnxo= Tracker["constants"]["wn"]
Tracker["constants"]["pixel_size"] = pixel_size
Tracker["fuse_freq"] = fq
del fq, nnxo, pixel_size
if(Tracker["constants"]["radius"] < 1):
Tracker["constants"]["radius"] = Tracker["constants"]["nnxo"]//2-2
elif((2*Tracker["constants"]["radius"] +2) > Tracker["constants"]["nnxo"]):
ERROR( "Particle radius set too large!", myid=myid )
return
####-----------------------------------------------------------------------------------------
# create the master directory
if myid == main_node:
if masterdir =="":
timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime())
masterdir ="master_sort3d"+timestring
li =len(masterdir)
else:
li = 0
cmd="{} {}".format("mkdir -p", masterdir)
os.system(cmd)
sp_global_def.write_command(masterdir)
else:
li=0
li = mpi.mpi_bcast( li, 1, mpi.MPI_INT, main_node, mpi.MPI_COMM_WORLD )[0]
if li>0:
masterdir = mpi.mpi_bcast( masterdir, li,MPI_CHAR, main_node, mpi.MPI_COMM_WORLD )
masterdir = string.join(masterdir,"")
####--- masterdir done!
if myid == main_node:
print_dict(Tracker["constants"],"Permanent settings of 3-D sorting program")
from time import sleep
while not os.path.exists(masterdir): # Be sure each proc is able to access the created dir
sxprint("Node ",myid," waiting...")
sleep(5)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
######### create a vstack from input stack to the local stack in masterdir
# stack name set to default
Tracker["constants"]["stack"] = "bdb:"+masterdir+"/rdata"
Tracker["constants"]["ali3d"] = os.path.join(masterdir, "ali3d_init.txt")
Tracker["constants"]["partstack"] = Tracker["constants"]["ali3d"]
Tracker["constants"]["ctf_params"] = os.path.join(masterdir, "ctf_params.txt")
######
if myid == main_node:
if(Tracker["orgstack"][:4] == "bdb:"): cmd = "{} {} {}".format("e2bdb.py", Tracker["orgstack"],"--makevstack="+Tracker["constants"]["stack"])
else: cmd = "{} {} {}".format("sp_cpy.py", orgstack, Tracker["constants"]["stack"])
cmdexecute(cmd)
cmd = "{} {} {} {} ".format("sp_header.py", Tracker["constants"]["stack"],"--params=xform.projection","--export="+Tracker["constants"]["ali3d"])
cmdexecute(cmd)
cmd = "{} {} {} {} ".format("sp_header.py", Tracker["constants"]["stack"],"--params=ctf","--export="+Tracker["constants"]["ctf_params"])
cmdexecute(cmd)
#keepchecking = False
total_stack = EMUtil.get_image_count(Tracker["orgstack"])
else:
total_stack =0
total_stack = bcast_number_to_all(total_stack, source_node = main_node)
"""
if myid==main_node:
from EMAN2db import db_open_dict
OB = db_open_dict(orgstack)
DB = db_open_dict(Tracker["constants"]["stack"])
for i in xrange(total_stack):
DB[i] = OB[i]
OB.close()
DB.close()
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
if myid==main_node:
params= []
for i in xrange(total_stack):
e=get_im(orgstack,i)
phi,theta,psi,s2x,s2y = get_params_proj(e)
params.append([phi,theta,psi,s2x,s2y])
write_text_row(params,Tracker["constants"]["ali3d"])
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
"""
#Tracker["total_stack"] = total_stack
Tracker["constants"]["total_stack"] = total_stack
Tracker["shrinkage"] = float(Tracker["nxinit"])/Tracker["constants"]["nnxo"]
#####------------------------------------------------------------------------------
if Tracker["constants"]["mask3D"]:
Tracker["mask3D"] = os.path.join(masterdir,"smask.hdf")
else:Tracker["mask3D"] = None
if Tracker["constants"]["focus3Dmask"]: Tracker["focus3D"]=os.path.join(masterdir,"sfocus.hdf")
else: Tracker["focus3D"] = None
if myid ==main_node:
if Tracker["constants"]["mask3D"]:
get_shrink_3dmask(Tracker["nxinit"],Tracker["constants"]["mask3D"]).write_image(Tracker["mask3D"])
if Tracker["constants"]["focus3Dmask"]:
mask_3D = get_shrink_3dmask(Tracker["nxinit"],Tracker["constants"]["focus3Dmask"])
st = Util.infomask(mask_3D, None, True)
if( st[0] == 0.0 ):
ERROR( "sxrsort3d","incorrect focused mask, after binarize all values zero" )
mask_3D.write_image(Tracker["focus3D"])
del mask_3D
if Tracker["constants"]["PWadjustment"]:
PW_dict={}
nxinit_pwsp=sample_down_1D_curve(Tracker["constants"]["nxinit"],Tracker["constants"]["nnxo"],Tracker["constants"]["PWadjustment"])
Tracker["nxinit_PW"] = os.path.join(masterdir,"spwp.txt")
if myid ==main_node:
write_text_file(nxinit_pwsp,Tracker["nxinit_PW"])
PW_dict[Tracker["constants"]["nnxo"]] =Tracker["constants"]["PWadjustment"]
PW_dict[Tracker["constants"]["nxinit"]] =Tracker["nxinit_PW"]
Tracker["PW_dict"] = PW_dict
###----------------------------------------------------------------------------------
####--------------------------- Extract the previous results #####################################################
from random import shuffle
if myid ==main_node:
log_main.add(" Sphire rsort3d ")
log_main.add("extract stable groups from two previous runs")
stable_member_list = get_stable_members_from_two_runs(Tracker["constants"]["previous_runs"], Tracker["constants"]["total_stack"], log_main)
Tracker["this_unaccounted_list"], new_stable_P1 = get_leftover_from_stable(stable_member_list, Tracker["constants"]["total_stack"], Tracker["constants"]["smallest_group"])
Tracker["this_unaccounted_list"].sort()
Tracker["total_stack"] = len(Tracker["this_unaccounted_list"])
log_main.add("new stable is %d"%len(new_stable_P1))
else:
Tracker["total_stack"] = 0
Tracker["this_unaccounted_list"] = 0
stable_member_list =0
stable_member_list = wrap_mpi_bcast(stable_member_list, main_node)
Tracker["total_stack"] = bcast_number_to_all(Tracker["total_stack"], source_node = main_node)
left_one_from_old_two_runs = wrap_mpi_bcast(Tracker["this_unaccounted_list"], main_node)
if myid ==main_node:
write_text_file(left_one_from_old_two_runs, os.path.join(masterdir,"unaccounted_from_two_previous_runs.txt"))
sxprint(" Extracting results of two previous runs is done!")
#################################### Estimate resolution----------------------#############
#### make chunkdir dictionary for computing margin of error
chunk_list = []
if Tracker["constants"]["chunkdir"] !="": ##inhere previous random assignment of odd and even
if myid == main_node:
chunk_one = read_text_file(os.path.join(Tracker["constants"]["chunkdir"],"chunk0.txt"))
chunk_two = read_text_file(os.path.join(Tracker["constants"]["chunkdir"],"chunk1.txt"))
else:
chunk_one = 0
chunk_two = 0
chunk_one = wrap_mpi_bcast(chunk_one, main_node)
chunk_two = wrap_mpi_bcast(chunk_two, main_node)
else: ## if traces are lost, then creating new random assignment of odd, even particles
chunks = list(range(Tracker["constants"]["total_stack"]))
shuffle(chunks)
chunk_one =chunks[0:Tracker["constants"]["total_stack"]//2]
chunk_two =chunks[Tracker["constants"]["total_stack"]//2:Tracker["constants"]["total_stack"]]
chunk_one = wrap_mpi_bcast(chunk_one, main_node)
chunk_two = wrap_mpi_bcast(chunk_two, main_node)
###### Fill chunk ID into headers when calling get_shrink_data_huang
if myid ==main_node:
sxprint(" random odd and even assignment done !")
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
#------------------------------------------------------------------------------
Tracker["chunk_dict"] = {}
for element in chunk_one: Tracker["chunk_dict"][element] = 0
for element in chunk_two: Tracker["chunk_dict"][element] = 1
Tracker["P_chunk0"] = len(chunk_one)/float(Tracker["constants"]["total_stack"])
Tracker["P_chunk1"] = len(chunk_two)/float(Tracker["constants"]["total_stack"])
### create two volumes to estimate resolution
if myid == main_node:
write_text_file(chunk_one, os.path.join(masterdir,"chunk0.txt"))
write_text_file(chunk_two, os.path.join(masterdir,"chunk1.txt"))
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
vols = []
for index in range(2):
data1,old_shifts1 = get_shrink_data_huang(Tracker,Tracker["constants"]["nxinit"], os.path.join(masterdir,"chunk%d.txt"%index), Tracker["constants"]["partstack"], myid, main_node, nproc, preshift = True)
vol1 = recons3d_4nn_ctf_MPI(myid=myid, prjlist=data1, symmetry=Tracker["constants"]["sym"], finfo=None)
if myid ==main_node:
vol1_file_name = os.path.join(masterdir, "vol%d.hdf"%index)
vol1.write_image(vol1_file_name)
vols.append(vol1)
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
if myid ==main_node:
low_pass, falloff, currentres = get_resolution_mrk01(vols, Tracker["constants"]["radius"]*Tracker["shrinkage"], Tracker["constants"]["nxinit"], masterdir,Tracker["mask3D"])
if low_pass > Tracker["constants"]["low_pass_filter"]: low_pass = Tracker["constants"]["low_pass_filter"]
else:
low_pass = 0.0
falloff = 0.0
currentres = 0.0
currentres = bcast_number_to_all(currentres,source_node = main_node)
low_pass = bcast_number_to_all(low_pass,source_node = main_node)
falloff = bcast_number_to_all(falloff,source_node = main_node)
Tracker["currentres"] = currentres
####################################################################
Tracker["falloff"] = falloff
if Tracker["constants"]["low_pass_filter"] == -1.0:
Tracker["low_pass_filter"] = low_pass*Tracker["shrinkage"]
else:
Tracker["low_pass_filter"] = Tracker["constants"]["low_pass_filter"]/Tracker["shrinkage"]
Tracker["lowpass"] = Tracker["low_pass_filter"]
Tracker["falloff"] = 0.1
Tracker["global_fsc"] = os.path.join(masterdir,"fsc.txt")
##################################################################
if myid ==main_node:
log_main.add("The command-line inputs are :")
log_main.add("**********************************************************")
for a in sys.argv:
log_main.add(a)
log_main.add("**********************************************************")
from sp_filter import filt_tanl
##################### START 3-D sorting ##########################
if myid ==main_node:
log_main.add("----------3-D sorting program------- ")
log_main.add("current resolution %6.3f for images of original size in terms of absolute frequency"%Tracker["currentres"])
log_main.add("equivalent to %f Angstrom resolution"%(round((Tracker["constants"]["pixel_size"]/Tracker["currentres"]/Tracker["shrinkage"]),4)))
filt_tanl(get_im(os.path.join(masterdir, "vol0.hdf")), Tracker["low_pass_filter"], 0.1).write_image(os.path.join(masterdir, "volf0.hdf"))
filt_tanl(get_im(os.path.join(masterdir, "vol1.hdf")), Tracker["low_pass_filter"], 0.1).write_image(os.path.join(masterdir, "volf1.hdf"))
sxprint(" random odd and even assignment done !")
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
## ---------------------------------------------------------------------------------------------########
## Stop program and output results when the leftover from two sort3d runs is not sufficient for a new run ########
## --------------------------------------------------- --------------------------------------- ######
Tracker["number_of_groups"] = get_number_of_groups(len(left_one_from_old_two_runs), Tracker["constants"]["number_of_images_per_group"])
if Tracker["number_of_groups"] <=1 : # programs finishes
if myid == main_node:
log_main.add("the unaccounted ones are no sufficient for a simple two-group run, output results!")
log_main.add("this implies your two sort3d runs already achieved high reproducibale ratio. ")
log_main.add("Or your number_of_images_per_group is too large ")
log_main.add("the final reproducibility is %f"%((Tracker["constants"]["total_stack"]-len(Tracker["this_unaccounted_list"]))/float(Tracker["constants"]["total_stack"])))
for i in range(len(stable_member_list)): write_text_file(stable_member_list[i], os.path.join(masterdir,"P2_final_class%d.txt"%i))
mask3d = get_im(Tracker["constants"]["mask3D"])
else:
mask3d = model_blank(Tracker["constants"]["nnxo"],Tracker["constants"]["nnxo"],Tracker["constants"]["nnxo"])
bcast_EMData_to_all(mask3d, myid, main_node)
for igrp in range(len(stable_member_list)):
#name_of_class_file = os.path.join(masterdir, "P2_final_class%d.txt"%igrp)
data, old_shifts = get_shrink_data_huang(Tracker,Tracker["constants"]["nnxo"], os.path.join(masterdir, "P2_final_class%d.txt"%igrp), Tracker["constants"]["partstack"], myid, main_node, nproc,preshift = True)
if Tracker["constants"]["CTF"]:
volref, fscc = rec3D_two_chunks_MPI(data, 1.0, Tracker["constants"]["sym"], mask3d,os.path.join(masterdir,"resolution_%02d.txt"%igrp), myid, main_node, index =-1, npad=2)
else:
sxprint("Missing CTF flag!")
return
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
#nx_of_image=volref.get_xsize()
if Tracker["constants"]["PWadjustment"] : Tracker["PWadjustment"] = Tracker["PW_dict"][Tracker["constants"]["nnxo"]]
else: Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"]
if myid ==main_node:
try:
lowpass = search_lowpass(fscc)
falloff = 0.1
except:
lowpass = 0.4
falloff = 0.1
log_main.add(" lowpass and falloff from fsc are %f %f"%(lowpass, falloff))
lowpass = round(lowpass,4)
falloff = round(min(0.1,falloff),4)
Tracker["lowpass"] = lowpass
Tracker["falloff"] = falloff
refdata = [None]*4
refdata[0] = volref
refdata[1] = Tracker
refdata[2] = Tracker["constants"]["myid"]
refdata[3] = Tracker["constants"]["nproc"]
volref = user_func(refdata)
cutoff = Tracker["constants"]["pixel_size"]/lowpass
log_main.add("%d vol low pass filer %f %f cut to %f Angstrom"%(igrp,Tracker["lowpass"],Tracker["falloff"],cutoff))
volref.write_image(os.path.join(masterdir,"volf_final%d.hdf"%igrp))
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
return
else: # Continue clustering on unaccounted ones that produced by two_way comparison of two previous runs
#########################################################################################################################
#if Tracker["constants"]["number_of_images_per_group"] ==-1: # Estimate number of images per group from delta, and scale up
# or down by scale_of_number
# number_of_images_per_group = int(Tracker["constants"]["scale_of_number"]*len(n_angles))
#
#########################################################################################################################P2
if myid ==main_node:
sxprint(" Now continue clustering on accounted ones because they can make at least two groups!")
P2_partitions = []
number_of_P2_runs = 2 # Notice P2 start from two P1 runs
### input list_to_be_processed
import copy
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
for iter_P2_run in range(number_of_P2_runs): # two runs such that one can obtain reproducibility
list_to_be_processed = left_one_from_old_two_runs[:]#Tracker["this_unaccounted_list"][:]
Tracker["this_unaccounted_list"] = left_one_from_old_two_runs[:]
if myid == main_node : new_stable1 = new_stable_P1[:]
total_stack = len(list_to_be_processed) # This is the input from two P1 runs
#number_of_images_per_group = Tracker["constants"]["number_of_images_per_group"]
P2_run_dir = os.path.join(masterdir, "P2_run%d"%iter_P2_run)
Tracker["number_of_groups"] = get_number_of_groups(total_stack, Tracker["constants"]["number_of_images_per_group"])
if myid == main_node:
cmd="{} {}".format("mkdir", P2_run_dir)
os.system(cmd)
log_main.add("----------------P2 independent run %d--------------"%iter_P2_run)
log_main.add("user provided number_of_images_per_group %d"%Tracker["constants"]["number_of_images_per_group"])
sxprint("----------------P2 independent run %d--------------"%iter_P2_run)
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
#
#Tracker["number_of_groups"] = get_number_of_groups(total_stack,Tracker["constants"]["number_of_images_per_group"])
generation = 0
if myid == main_node:
log_main.add("number of groups is %d"%Tracker["number_of_groups"])
log_main.add("total stack %d"%total_stack)
while( Tracker["number_of_groups"]>=2 ):
partition_dict = {}
full_dict = {}
workdir = os.path.join(P2_run_dir,"generation%03d"%generation)
Tracker["this_dir"] = workdir
if myid ==main_node:
cmd="{} {}".format("mkdir", workdir)
os.system(cmd)
log_main.add("---- generation %5d"%generation)
log_main.add("number of images per group is set as %d"%Tracker["constants"]["number_of_images_per_group"])
log_main.add("the initial number of groups is %d "%Tracker["number_of_groups"])
log_main.add(" the number to be processed in this generation is %d"%len(list_to_be_processed))
sxprint("---- generation %5d"%generation)
#core=read_text_row(Tracker["constants"]["ali3d"],-1)
#write_text_row(core, os.path.join(workdir,"node%d.txt"%myid))
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
Tracker["this_data_list"] = list_to_be_processed # leftover of P1 runs
Tracker["total_stack"] = len(list_to_be_processed)
create_random_list(Tracker)
###------ For super computer ##############
update_full_dict(list_to_be_processed, Tracker)
###----
##### ----------------Independent runs for EQ-Kmeans ------------------------------------
for indep_run in range(Tracker["constants"]["indep_runs"]):
Tracker["this_particle_list"] = Tracker["this_indep_list"][indep_run]
ref_vol = recons_mref(Tracker)
if myid ==main_node: log_main.add("independent run %10d"%indep_run)
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
#this_particle_text_file = # for get_shrink_data
if myid ==main_node:
write_text_file(list_to_be_processed, os.path.join(workdir, "independent_list_%03d.txt"%indep_run))
mref_ali3d_EQ_Kmeans(ref_vol, os.path.join(workdir, "EQ_Kmeans%03d"%indep_run), os.path.join(workdir, "independent_list_%03d.txt"%indep_run), Tracker)
partition_dict[indep_run] = Tracker["this_partition"]
del ref_vol
Tracker["partition_dict"] = partition_dict
Tracker["this_total_stack"] = Tracker["total_stack"]
do_two_way_comparison(Tracker)
##############################
if myid ==main_node: log_main.add("Now calculate stable volumes")
if myid ==main_node:
for igrp in range(len(Tracker["two_way_stable_member"])):
Tracker["this_data_list"] = Tracker["two_way_stable_member"][igrp]
write_text_file(Tracker["this_data_list"], os.path.join(workdir,"stable_class%d.txt"%igrp))
Tracker["this_data_list_file"] = -1
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
###
number_of_ref_class = []
ref_vol_list = []
for igrp in range(len(Tracker["two_way_stable_member"])):
data, old_shifts = get_shrink_data_huang(Tracker,Tracker["nxinit"], os.path.join(workdir, "stable_class%d.txt"%igrp), Tracker["constants"]["partstack"], myid, main_node, nproc, preshift = True)
volref = recons3d_4nn_ctf_MPI(myid=myid,prjlist=data,symmetry=Tracker["constants"]["sym"],finfo = None)
ref_vol_list.append(volref)
number_of_ref_class.append(len(Tracker["this_data_list"]))
if myid ==main_node:
log_main.add("group %d members %d "%(igrp,len(Tracker["this_data_list"])))
#ref_vol_list=apply_low_pass_filter(ref_vol_list,Tracker)
for iref in range(len(ref_vol_list)): ref_vol_list[iref].write_image(os.path.join(workdir,"vol_stable.hdf"),iref)
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
################################
Tracker["number_of_ref_class"] = number_of_ref_class
Tracker["this_data_list"] = Tracker["this_accounted_list"]
outdir = os.path.join(workdir, "Kmref")
empty_groups,res_classes,final_list = ali3d_mref_Kmeans_MPI(ref_vol_list, outdir, os.path.join(workdir,"Accounted.txt"), Tracker)
Tracker["this_unaccounted_list"] = get_complementary_elements(list_to_be_processed,final_list)
if myid == main_node: log_main.add("the number of particles not processed is %d"%len(Tracker["this_unaccounted_list"]))
update_full_dict(Tracker["this_unaccounted_list"], Tracker)
if myid == main_node: write_text_file(Tracker["this_unaccounted_list"], Tracker["this_unaccounted_text"])
Tracker["number_of_groups"] = len(res_classes)
### Update data
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
if myid == main_node:
number_of_ref_class=[]
log_main.add(" Compute volumes of original size")
for igrp in range(Tracker["number_of_groups"]):
if os.path.exists( os.path.join( outdir,"Class%d.txt"%igrp ) ):
new_stable1.append( read_text_file( os.path.join( outdir, "Class%d.txt"%igrp ) ) )
log_main.add(" read Class file %d"%igrp)
number_of_ref_class.append(len(new_stable1))
else: number_of_ref_class = 0
number_of_ref_class = wrap_mpi_bcast(number_of_ref_class,main_node)
################################
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
if myid ==main_node:
vol_list = []
for igrp in range(Tracker["number_of_groups"]):
if myid ==main_node: log_main.add("start vol %d"%igrp)
data,old_shifts = get_shrink_data_huang(Tracker,Tracker["constants"]["nnxo"], os.path.join(outdir,"Class%d.txt"%igrp), Tracker["constants"]["partstack"],myid, main_node, nproc, preshift = True)
volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist = data, symmetry = Tracker["constants"]["sym"],finfo= None)
if myid == main_node:
vol_list.append(volref)
log_main.add(" vol %d is done"%igrp)
Tracker["number_of_ref_class"] = number_of_ref_class
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
generation +=1
#################################
if myid ==main_node:
for ivol in range(len(vol_list)): vol_list[ivol].write_image(os.path.join(workdir, "vol_of_Classes.hdf"),ivol)
filt_tanl(vol_list[ivol],Tracker["constants"]["low_pass_filter"],.1).write_image(os.path.join(workdir, "volf_of_Classes.hdf"),ivol)
log_main.add("number of unaccounted particles %10d"%len(Tracker["this_unaccounted_list"]))
log_main.add("number of accounted particles %10d"%len(Tracker["this_accounted_list"]))
del vol_list
Tracker["this_data_list"] = Tracker["this_unaccounted_list"]
Tracker["total_stack"] = len(Tracker["this_unaccounted_list"])
Tracker["this_total_stack"] = Tracker["total_stack"]
#update_full_dict(complementary)
#number_of_groups = int(float(len(Tracker["this_unaccounted_list"]))/number_of_images_per_group)
del list_to_be_processed
list_to_be_processed = copy.deepcopy(Tracker["this_unaccounted_list"])
Tracker["number_of_groups"] = get_number_of_groups(len(list_to_be_processed),Tracker["constants"]["number_of_images_per_group"])
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
#############################################################################################################################
### Reconstruct the unaccounted is only done once
if (Tracker["constants"]["unaccounted"] and (len(Tracker["this_unaccounted_list"]) != 0)):
data,old_shifts = get_shrink_data_huang(Tracker,Tracker["constants"]["nnxo"],Tracker["this_unaccounted_text"],Tracker["constants"]["partstack"],myid,main_node,nproc,preshift = True)
volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist = data, symmetry=Tracker["constants"]["sym"],finfo=None)
volref = filt_tanl(volref, Tracker["constants"]["low_pass_filter"],.1)
if myid ==main_node: volref.write_image(os.path.join(workdir, "volf_unaccounted.hdf"))
######## Exhaustive Kmeans #############################################
if myid ==main_node:
if len(Tracker["this_unaccounted_list"])>=Tracker["constants"]["smallest_group"]:
new_stable1.append(Tracker["this_unaccounted_list"])
unaccounted = get_complementary_elements_total(Tracker["constants"]["total_stack"], final_list)
Tracker["number_of_groups"] = len(new_stable1)
log_main.add("----------------Exhaustive Kmeans------------------")
log_main.add("number_of_groups is %d"%Tracker["number_of_groups"])
else: Tracker["number_of_groups"] = 0
### prepare references for final K-means
if myid == main_node:
final_list =[]
for alist in new_stable1:
for element in alist:final_list.append(int(element))
unaccounted = get_complementary_elements_total(Tracker["constants"]["total_stack"],final_list)
if len(unaccounted) > Tracker["constants"]["smallest_group"]: # treat unaccounted ones also as a group if it is not too small.
new_stable1.append(unaccounted)
Tracker["number_of_groups"] = len(new_stable1)
for any in unaccounted:final_list.append(any)
log_main.add("total number %d"%len(final_list))
else: final_list = 0
Tracker["number_of_groups"] = bcast_number_to_all(Tracker["number_of_groups"],source_node = main_node)
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
final_list = wrap_mpi_bcast(final_list, main_node)
workdir = os.path.join(P2_run_dir,"Exhaustive_Kmeans") # new workdir
if myid==main_node:
os.mkdir(workdir)
write_text_file(final_list, os.path.join(workdir,"final_list.txt"))
else: new_stable1 = 0
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
## Create reference volumes
if myid == main_node:
number_of_ref_class = []
for igrp in range(Tracker["number_of_groups"]):
class_file = os.path.join(workdir,"final_class%d.txt"%igrp)
write_text_file(new_stable1[igrp],class_file)
log_main.add(" group %d number of particles %d"%(igrp,len(new_stable1[igrp])))
number_of_ref_class.append(len(new_stable1[igrp]))
else: number_of_ref_class= 0
number_of_ref_class = wrap_mpi_bcast(number_of_ref_class,main_node)
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
ref_vol_list = []
for igrp in range(Tracker["number_of_groups"]):
if myid ==main_node : sxprint(" prepare reference %d"%igrp)
#Tracker["this_data_list_file"] = os.path.join(workdir,"final_class%d.txt"%igrp)
data,old_shifts = get_shrink_data_huang(Tracker, Tracker["nxinit"],os.path.join(workdir,"final_class%d.txt"%igrp), Tracker["constants"]["partstack"], myid,main_node,nproc,preshift = True)
volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist = data, symmetry=Tracker["constants"]["sym"], finfo = None)
#volref = filt_tanl(volref, Tracker["low_pass_filter"],.1)
#if myid == main_node:
# volref.write_image(os.path.join(masterdir,"volf_stable.hdf"),iref)
#volref = resample(volref,Tracker["shrinkage"])
bcast_EMData_to_all(volref, myid, main_node)
ref_vol_list.append(volref)
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
### -------variables used in Kmeans_exhaustive_run-----
Tracker["number_of_ref_class"] = number_of_ref_class
Tracker["this_data_list"] = final_list
Tracker["total_stack"] = len(final_list)
Tracker["this_dir"] = workdir
Tracker["this_data_list_file"] = os.path.join(workdir,"final_list.txt")
KE_group = Kmeans_exhaustive_run(ref_vol_list,Tracker) #
P2_partitions.append(KE_group[:][:])
if myid ==main_node:
log_main.add(" the number of groups after exhaustive Kmeans is %d"%len(KE_group))
for ike in range(len(KE_group)):log_main.add(" group %d number of objects %d"%(ike,len(KE_group[ike])))
del new_stable1
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
if myid == main_node: log_main.add("P2 runs are done, now start two-way comparision to exclude those that are not reproduced ")
reproduced_groups = two_way_comparison_single(P2_partitions[0],P2_partitions[1],Tracker)# Here partition IDs are original indexes.
###### ----------------Reconstruct reproduced groups------------------------#######
######
if myid == main_node:
for index_of_reproduced_groups in range(len(reproduced_groups)):
name_of_class_file = os.path.join(masterdir, "P2_final_class%d.txt"%index_of_reproduced_groups)
write_text_file(reproduced_groups[index_of_reproduced_groups],name_of_class_file)
log_main.add("-------start to reconstruct reproduced volumes individully to orignal size-----------")
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
if Tracker["constants"]["mask3D"]: mask_3d = get_shrink_3dmask(Tracker["constants"]["nnxo"],Tracker["constants"]["mask3D"])
else: mask_3d = None
for igrp in range(len(reproduced_groups)):
data,old_shifts = get_shrink_data_huang(Tracker,Tracker["constants"]["nnxo"],os.path.join(masterdir, "P2_final_class%d.txt"%igrp),Tracker["constants"]["partstack"],myid,main_node,nproc,preshift = True)
#volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist = data, symmetry=Tracker["constants"]["sym"], finfo=None)
if Tracker["constants"]["CTF"]:
volref, fscc = rec3D_two_chunks_MPI(data,1.0,Tracker["constants"]["sym"],mask_3d, \
os.path.join(masterdir,"resolution_%02d.txt"%igrp),myid,main_node,index =-1,npad =2,finfo=None)
else:
sxprint("Missing CTF flag!")
return
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
fscc = read_text_file(os.path.join(masterdir, "resolution_%02d.txt"%igrp),-1)
nx_of_image = volref.get_xsize()
if Tracker["constants"]["PWadjustment"]: Tracker["PWadjustment"] = Tracker["PW_dict"][nx_of_image]
else: Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"]
try:
lowpass = search_lowpass(fscc)
falloff = 0.1
except:
lowpass= 0.4
falloff= 0.1
sxprint(lowpass)
lowpass=round(lowpass,4)
falloff=round(min(.1,falloff),4)
Tracker["lowpass"]= lowpass
Tracker["falloff"]= falloff
if myid == main_node:
refdata =[None]*4
refdata[0] = volref
refdata[1] = Tracker
refdata[2] = Tracker["constants"]["myid"]
refdata[3] = Tracker["constants"]["nproc"]
volref = user_func(refdata)
cutoff = Tracker["constants"]["pixel_size"]/lowpass
log_main.add("%d vol low pass filer %f %f cut to %f Angstrom"%(igrp,Tracker["lowpass"],Tracker["falloff"],cutoff))
volref.write_image(os.path.join(masterdir,"volf_final%d.hdf"%igrp))
if myid==main_node: log_main.add(" sxsort3d_P2 finishes. ")
# Finish program
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
return
args = parser.parse_args()
sp_global_def.BATCH = True
if not os.path.exists(args.input_run_dir):
sp_global_def.ERROR('Input directory does not exist!', 'sxbatch.py', 1)
qsub_dict = {
'qsub': re.compile('Your job (\w+)'),
'sbatch': re.compile('Submitted batch job (\w+)'),
}
if args.submission_command.split()[0] not in qsub_dict and args.hold_flag:
sp_global_def.ERROR(
'Qsub return output not known! Please contact the SPHIRE authors!',
'sxbatch.py', 1)
sp_global_def.write_command('.')
if args.first_hold_number:
prev_hold = args.first_hold_number
else:
prev_hold = 'aaa'
for idx, file_name in enumerate(
sorted(glob.glob('{0}/*'.format(args.input_run_dir)))):
command = args.submission_command.split()
if args.hold_flag and (idx != 0 or args.first_hold_number):
command.append('{0}{1}'.format(args.hold_flag, prev_hold))
else:
pass
command.append(file_name)
if args.hold_flag:
def main(args):
"""
Main function
Arguments:
args - Arguments as dictionary
Returns:
None
"""
main_mpi_proc = 0
my_mpi_proc_id = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
n_mpi_procs = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
# Import the file names
sanity_checks(args, my_mpi_proc_id)
if my_mpi_proc_id == main_mpi_proc:
if args['Expert options']:
sp_global_def.sxprint(
'Expert option detected! The program will enable expert mode!')
if args['Magnification correction']:
sp_global_def.sxprint(
'Magnification correction option detected! The program will enable magnification correction mode!'
)
file_names = load_file_names_by_pattern(
args['input_micrograph_pattern'], args['selection_file'])
else:
file_names = []
file_names = sp_utilities.wrap_mpi_bcast(file_names, main_mpi_proc)
# Split the list indices by node
max_proc = min(n_mpi_procs, len(file_names))
if my_mpi_proc_id in list(range(max_proc)):
idx_start, idx_end = sp_applications.MPI_start_end(
len(file_names), max_proc, my_mpi_proc_id)
else:
idx_start = 0
idx_end = 0
nima = idx_end - idx_start
max_nima_list = sp_utilities.wrap_mpi_gatherv([nima], main_mpi_proc,
mpi.MPI_COMM_WORLD)
max_nima_list = sp_utilities.wrap_mpi_bcast(max_nima_list, main_mpi_proc,
mpi.MPI_COMM_WORLD)
max_nima = max(max_nima_list)
mpi_print_id = max_nima_list.index(max_nima)
try:
os.makedirs(args['output_directory'])
except OSError:
pass
sp_global_def.write_command(args['output_directory'])
start_unblur = time.time.time()
for idx, file_path in enumerate(file_names[idx_start:idx_end]):
if my_mpi_proc_id == mpi_print_id:
total_time = time.time.time() - start_unblur
if idx == 0:
average_time = 0
else:
average_time = total_time / float(idx)
sp_global_def.sxprint(
'{0: 6.2f}% => Elapsed time: {1: 6.2f}min | Estimated total time: {2: 6.2f}min | Time per micrograph: {3: 5.2f}min/mic'
.format(
100 * idx / float(max_nima),
total_time / float(60),
(max_nima) * average_time / float(60),
average_time / float(60),
))
file_name = os.path.basename(os.path.splitext(file_path)[0])
file_name_out = '{0}.mrc'.format(file_name)
file_name_log = '{0}.log'.format(file_name)
file_name_err = '{0}.err'.format(file_name)
output_dir_name = os.path.join(args['output_directory'], 'corrsum')
output_dir_name_log = os.path.join(args['output_directory'],
'corrsum_log')
output_dir_name_dw = os.path.join(args['output_directory'],
'corrsum_dw')
output_dir_name_dw_log = os.path.join(args['output_directory'],
'corrsum_dw_log')
if args['additional_dose_unadjusted']:
unblur_list = (
(True, output_dir_name_dw, output_dir_name_dw_log),
(False, output_dir_name, output_dir_name_log),
)
elif args['skip_dose_adjustment']:
unblur_list = ((False, output_dir_name, output_dir_name_log), )
else:
unblur_list = ((True, output_dir_name_dw,
output_dir_name_dw_log), )
for dose_adjustment, dir_name, log_dir_name in unblur_list:
try:
os.makedirs(dir_name)
except OSError:
pass
try:
os.makedirs(log_dir_name)
except OSError:
pass
output_name = os.path.join(dir_name, file_name_out)
output_name_log = os.path.join(log_dir_name, file_name_log)
output_name_err = os.path.join(log_dir_name, file_name_err)
unblur_command = create_unblur_command(
file_path,
output_name,
args['pixel_size'],
args['bin_factor'],
dose_adjustment,
args['voltage'],
args['exposure_per_frame'],
args['pre_exposure'],
args['Expert options'],
args['min_shift_initial'],
args['outer_radius'],
args['b_factor'],
args['half_width_vert'],
args['half_width_hor'],
args['termination'],
args['max_iterations'],
bool(not args['dont_restore_noise_power']),
args['gain_file'],
args['first_frame'],
args['last_frame'],
args['Magnification correction'],
args['distortion_angle'],
args['major_scale'],
args['minor_scale'],
)
execute_command = r'echo "{0}" | {1}'.format(
unblur_command, args['unblur_path'])
with open(output_name_log, 'w') as log, open(output_name_err,
'w') as err:
start = time.time.time()
child = subprocess.Popen(execute_command,
shell=True,
stdout=log,
stderr=err)
child.wait()
if child.returncode != 0:
sp_global_def.sxprint(
'Process failed for image {0}.\nPlease make sure that the unblur path is correct\nand check the respective logfile.'
.format(file_path))
log.write('Time => {0:.2f} for command: {1}'.format(
time.time.time() - start, execute_command))
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if my_mpi_proc_id == mpi_print_id:
idx = idx + 1
total_time = time.time.time() - start_unblur
average_time = total_time / float(idx)
sp_global_def.sxprint(
'{0: 6.2f}% => Elapsed time: {1: 6.2f}min | Estimated total time: {2: 6.2f}min | Time per micrograph: {3: 5.2f}min/mic'
.format(
100 * idx / float(max_nima),
total_time / float(60),
(max_nima) * average_time / float(60),
average_time / float(60),
))
def main(args_as_dict):
if subprocess.os.path.exists(args_as_dict["output_directory"]):
sp_global_def.sxprint(
"Output directory already exists! Please choose another one.")
sys.exit(1)
# show_variables()
function_dict = collections.OrderedDict()
# [Function name, MPI support]
function_dict["do_unblur"] = [get_unblur_cmd, True]
function_dict["do_cter"] = [get_cter_cmd, True]
function_dict["do_cryolo"] = [get_cryolo_predict, False]
function_dict["do_window"] = [get_window, True]
function_dict["do_window_stack"] = [get_window_stack, False]
function_dict["do_isac2"] = [get_isac2, True]
function_dict["do_cinderella"] = [get_cinderella_predict, False]
function_dict["do_isac2_substack"] = [get_isac2_substack, False]
function_dict["do_rviper"] = [get_rviper, True]
function_dict["do_adjust_rviper"] = [get_adjustment, False]
function_dict["do_mask_rviper"] = [get_mask_rviper, False]
function_dict["do_meridien"] = [get_meridien, True]
function_dict["do_sharpening_meridien"] = [get_sharpening_meridien, False]
function_dict["do_restack_import_params"] = [
get_restack_import_params, False
]
function_dict["do_restack_box_files"] = [get_restack_box_files, False]
function_dict["do_restack_window"] = [get_restack_window, True]
function_dict["do_restack_stack"] = [get_restack_stack, False]
function_dict["do_restack_meridien"] = [get_restack_meridien, True]
function_dict["do_restack_sharpening"] = [get_restack_sharpening, False]
function_dict["do_ctf_refine_import_params"] = [
get_ctf_import_params, False
]
function_dict["do_ctf_refine"] = [get_ctf_refine, False]
function_dict["do_ctf_refine_meridien"] = [get_ctf_meridien, True]
function_dict["do_ctf_refine_sharpening"] = [get_ctf_sharpening, False]
do_dict = {}
for key, value in args_as_dict.items():
if key.startswith("skip") and isinstance(value, bool):
do_dict[key.replace("skip", "do")] = bool(not value)
if key == "skip_window":
do_dict["{0}_stack".format(key.replace(
"skip", "do"))] = bool(not value)
elif key == "skip_isac2":
do_dict["{0}_substack".format(key.replace(
"skip", "do"))] = bool(not value)
elif key == "skip_restack":
do_dict["{0}_import_params".format(key.replace(
"skip", "do"))] = bool(not value)
do_dict["{0}_box_files".format(key.replace(
"skip", "do"))] = bool(not value)
do_dict["{0}_window".format(key.replace(
"skip", "do"))] = bool(not value)
do_dict["{0}_stack".format(key.replace(
"skip", "do"))] = bool(not value)
do_dict["{0}_meridien".format(key.replace(
"skip", "do"))] = bool(not value)
do_dict["{0}_sharpening".format(key.replace(
"skip", "do"))] = bool(not value)
elif key == "skip_ctf_refine":
do_dict["{0}_import_params".format(key.replace(
"skip", "do"))] = bool(not value)
do_dict["{0}_meridien".format(key.replace(
"skip", "do"))] = bool(not value)
do_dict["{0}_sharpening".format(key.replace(
"skip", "do"))] = bool(not value)
phase_plate = args_as_dict["phase_plate"]
negative_stain = args_as_dict["negative_stain"]
fill_rviper_mask = args_as_dict["fill_rviper_mask"]
do_gain = bool(args_as_dict["XXX_SP_UNBLUR_GAIN_FILE_XXX"] is not None)
mpi_procs = args_as_dict["mpi_procs"]
mpi_submission = args_as_dict["mpi_submission_template"]
out_submission = "{0}/submission_script.sh".format(
args_as_dict["output_directory"])
prev_line = "echo $(date): {0}\n"
check_line = "if [[ ${{?}} != 0 ]]; then echo $(date): '{0}: Failure!'; exit 1; else echo $(date): '{0}: Success!'; fi\n"
cmds = []
dict_idx_dict = {}
running_idx = 0
current_idx = 0
for key in function_dict:
if do_dict[key]:
cmds.append(prev_line.format(key))
dict_idx_dict[running_idx] = current_idx
running_idx += 1
return_value = [
entry for entry in function_dict[key][0](
phase_plate=phase_plate,
negative_stain=negative_stain,
fill_rviper_mask=fill_rviper_mask,
status_dict=do_dict,
do_gain=do_gain,
) if entry.strip()
]
return_value.insert(0, [function_dict[key][1], key])
cmds.append(return_value)
dict_idx_dict[running_idx] = current_idx
running_idx += 1
cmds.append(check_line.format(key))
dict_idx_dict[running_idx] = current_idx
running_idx += 1
current_idx += 1
with open(mpi_submission) as read:
lines = read.readlines()
found_lines = []
for idx, entry in enumerate(lines):
if "XXX_SXCMD_LINE_XXX" in entry and "mpirun" in entry:
found_lines.append(idx)
if not found_lines:
sp_global_def.sxprint(
"Could not find a suitable command line for exchange.")
sp_global_def.sxprint(
"The line should contain XXX_SXMPI_NPROC_XXX and XXX_SXCMD_LINE_XXX."
)
sys.exit(1)
line = (lines[found_lines[-1]].replace("{", "{{").replace(
"}", "}}").replace("XXX_SXMPI_NPROC_XXX", str(mpi_procs)).replace(
"XXX_SXCMD_LINE_XXX", "'{0}' >> {1}_out.txt 2>>{1}_err.txt"))
line_no_mpi = "'{0}' >>{1}_out.txt 2>>{1}_err.txt\n"
remove_quote_list = (
"XXX_SP_WINDOW_OUTPUT_DIR_XXX/mpi_proc_*",
"XXX_SP_RESTACK_WINDOW_OUTPUT_DIR_XXX/mpi_proc_*",
"XXX_SP_MERIDIEN_OUTPUT_DIR_XXX/vol_*_unfil_*.hdf",
"--import=$(ls XXX_SP_MERIDIEN_OUTPUT_DIR_XXX/final_params_*.txt)",
"--import=$(ls XXX_SP_RESTACK_MERIDIEN_OUTPUT_DIR_XXX/final_params_*.txt)",
"XXX_SP_RESTACK_MERIDIEN_OUTPUT_DIR_XXX/vol_*_unfil_*.hdf",
"XXX_SP_CTF_MERIDIEN_OUTPUT_DIR_XXX/vol_*_unfil_*.hdf",
)
final_lines = []
output_folder_dict = {}
for idx, entry in enumerate(cmds):
if isinstance(entry, list) and entry[0][0]:
tmp = line.format(
"' '".join(entry[1:]),
subprocess.os.path.join(
args_as_dict["output_directory"],
"{0:04d}_{1}".format(dict_idx_dict[idx], entry[0][1]),
),
)
elif isinstance(entry, list):
tmp = line_no_mpi.format(
"' '".join(entry[1:]),
subprocess.os.path.join(
args_as_dict["output_directory"],
"{0:04d}_{1}".format(dict_idx_dict[idx], entry[0][1]),
),
)
else:
tmp = entry
for entry in remove_quote_list:
tmp = tmp.replace("'{0}'".format(entry), entry)
for key, value in args_as_dict.items():
if "_OUTPUT_DIR_" in key and key in tmp:
if key in output_folder_dict:
value = output_folder_dict[key]
else:
value = subprocess.os.path.join(
args_as_dict["output_directory"],
"{0:04d}_{1}".format(dict_idx_dict[idx], value),
)
output_folder_dict[key] = value
tmp = tmp.replace(key, value)
final_lines.append(tmp)
final_line = "\n".join(final_lines)
for key, value in args_as_dict.items():
if key.startswith("XXX"):
if "_ADDITION_XXX" in key:
value = "' '".join(value.split(" "))
final_line = final_line.replace(key, str(value))
lines[found_lines[-1]] = final_line.replace(" ''", "").replace("';'", ";")
subprocess.os.mkdir(args_as_dict["output_directory"])
sp_global_def.write_command(args_as_dict["output_directory"])
with open(out_submission, "w") as w:
w.write("".join(lines).replace("XXX_SXMPI_NPROC_XXX",
str(mpi_procs)).replace(
"XXX_SXMPI_JOB_NAME_XXX",
args_as_dict["mpi_job_name"]))
if not args_as_dict["dry_run"]:
sp_global_def.sxprint(
subprocess.check_output(
[args_as_dict["mpi_submission_command"], out_submission]))
def separate_class(classavgstack, instack, options, outdir='.', verbose=False):
"""
Main function overseeing various projection-comparison modes.
Arguments:
classavgstack : Input image stack
classmap : Class-to-particle lookup table. Each (long) line contains particles assigned to a class, one file for all classes
options : (list) Command-line options, run 'sxproj_compare.py -h' for an exhaustive list
outdir : Output directory
verbose : (boolean) Whether to write additional information to screen
"""
# Set output directory and log file name
prepare_outdir(outdir, verbose)
write_command(outdir)
log, verbose = prepare_log(outdir, verbose)
prepare_outdir(os.path.join(outdir, DOCFILEDIR))
# Set filename for copy of input BDB
bdb_path = os.path.join(outdir, 'EMAN2DB', BIGSTACKCOPY + '.bdb')
if os.path.exists(bdb_path):
os.remove(bdb_path) # will otherwise merge with pre-existing file
stackcp = 'bdb:' + outdir + '#' + BIGSTACKCOPY
# Expand paths for outputs
classmap = os.path.join(outdir, DOCFILEDIR, CLASSMAPFILE)
classdoc_template = os.path.join(outdir, DOCFILEDIR,
CLASSDOCPREFIX + '{0:03d}.txt')
class_init_bdb_template = os.path.join(outdir, CLASSORIGPREFIX + '{0:03d}')
class_filt_bdb_template = os.path.join(outdir,
CLASSFINALPREFIX + '{0:03d}')
if options.align_isac_dir or options.filtrad or options.shrink:
prepare_outdir(os.path.join(outdir, STACKFILEDIR))
outali = os.path.join(outdir, STACKFILEDIR,
ALIGNSTACKPREFIX + '{0:03d}' + options.format)
outflt = os.path.join(outdir, STACKFILEDIR,
FILTSTACKPREFIX + '{0:03d}' + options.format)
if options.align_isac_dir:
chains_params = os.path.join(options.align_isac_dir,
'chains_params.txt')
classed_imgs = os.path.join(options.align_isac_dir,
'processed_images.txt')
num_classes = EMUtil.get_image_count(classavgstack)
# Generate class-to-particle lookup table and class-selection lists
vomq(classavgstack, classmap, classdoc_template, log=log, verbose=verbose)
if options.filtrad:
if options.apix:
filtrad = options.apix / options.filtrad
else:
filtrad = options.filtrad
print_log_msg("Will low-pass filter to %s px^-1" % options.filtrad,
log, verbose)
if options.shrink:
print_log_msg(
"Will downsample stacks by a factor of %s" % options.shrink, log,
verbose)
if options.nvec != None and options.align_isac_dir == None:
sp_global_def.ERROR(
"\nERROR!! To compute eigenimages, need to specify --align_isac_dir",
__file__, 1)
exit()
if options.nvec:
print_log_msg('Writing %s eigenimages per class' % options.nvec, log,
verbose)
tot_parts = 0
if options.align_isac_dir:
if options.debug:
num_tot_images = EMUtil.get_image_count(instack)
print('num_tot_images', num_tot_images)
if os.path.basename(
classavgstack
) == 'ordered_class_averages.hdf' and os.path.exists(chains_params):
# Make substack with processed images
print_log_msg(
"Making substack %s from original stack %s using subset in %s"
% (stackcp, instack, classed_imgs), log, verbose)
cmd = "e2bdb.py %s --makevstack %s --list %s" % (instack, stackcp,
classed_imgs)
else:
# Simply copy image stack
print_log_msg(
"Copying %s to virtual stack %s" % (instack, stackcp), log,
verbose)
cmd = "e2bdb.py %s --makevstack %s" % (instack, stackcp)
print_log_msg(cmd, log, verbose)
os.system(cmd)
# Combine alignment parameters
combined_params_file = os.path.join(outdir, COMBINEDPARAMS)
combine_isac_params(options.align_isac_dir, classavgstack,
chains_params, classed_imgs, classdoc_template,
combined_params_file, log, verbose)
# Import alignment parameters
cmd = "sp_header.py %s --params=xform.align2d --import=%s\n" % (
stackcp, combined_params_file)
print_log_msg(cmd, log, verbose)
header(stackcp, 'xform.align2d', fimport=combined_params_file)
if options.debug:
test_params_file = os.path.join(outdir, TESTPARAMSOUT)
print_log_msg("Writing imported parameters to %s" %
test_params_file)
header(stackcp, 'xform.align2d', fexport=test_params_file)
stack2split = stackcp
# If not aligning images
else:
stack2split = instack
print_log_msg("Writing %s class stacks" % num_classes, log, verbose)
if options.align_isac_dir:
print_log_msg('Writing aligned images', log, verbose)
# Loop through classes
for class_num in xrange(num_classes):
# No aligned images
if not options.align_isac_dir:
# Write class stack
cmd = "e2bdb.py %s --makevstack bdb:%s --list %s" % (
stack2split, class_init_bdb_template.format(class_num),
classdoc_template.format(class_num))
print_log_msg(cmd, log, verbose)
os.system(cmd)
num_class_imgs = EMUtil.get_image_count(
'bdb:' + class_init_bdb_template.format(class_num))
tot_parts += num_class_imgs
# Optional filtered stack
if options.filtrad or options.shrink:
cmd = "e2proc2d.py bdb:%s %s --inplace" % (
class_init_bdb_template.format(class_num),
outflt.format(class_num))
# --inplace overwrites existing images
if options.filtrad:
cmd = cmd + " --process=filter.lowpass.gauss:cutoff_freq=%s" % filtrad
if options.shrink:
cmd = cmd + " --meanshrink=%s" % options.shrink
print_log_msg(cmd, log, verbose)
os.system(cmd)
# Optionally apply alignment
else:
# Write class stack
class_init_bdb_name = 'bdb:' + class_init_bdb_template.format(
class_num)
cmd = "e2bdb.py %s --makevstack %s --list %s" % (
stack2split, class_init_bdb_name,
classdoc_template.format(class_num))
print_log_msg(cmd, log, verbose)
os.system(cmd)
# Set filenames
aligned_stack_path = outali.format(class_num)
class_filt_bdb_name = 'bdb:' + class_filt_bdb_template.format(
class_num)
# PCA works better if application of alignment parameters is done internally.
# So, alignment will be applied afterward.
if options.debug and class_num == 0:
verbosity = True
else:
verbosity = False
if options.nvec != None:
num_class_imgs = filter_shrink(class_init_bdb_name,
aligned_stack_path,
class_filt_bdb_name,
alignYN=False,
filtrad=filtrad,
shrink=options.shrink,
verbose=verbosity)
tmp_classavg, class_stack_list = prepare_2d_forPCA(
class_filt_bdb_name, mode='a', CTF=False)
eig_list = pca(class_stack_list, nvec=options.nvec)
montage_file = os.path.join(outdir, 'stkeigen.hdf')
avg_img, var_img = ave_var(
class_filt_bdb_name
) # needs to be a BDB, aligned_stack_obj didn't work
# Not computing eigenimages
else:
eig_list = []
montage_file = os.path.join(outdir, 'stkavgvar.hdf')
avg_img, var_img = ave_var(
class_init_bdb_name
) # needs to be a BDB, aligned_stack_obj didn't work
montage_list = [avg_img] + [var_img] + eig_list
montage_row = montage_scale(montage_list, scale=True)
montage_row.write_image(montage_file, class_num)
# Apply alignments
num_class_imgs = filter_shrink(class_init_bdb_name,
aligned_stack_path,
class_filt_bdb_name,
alignYN=True,
filtrad=filtrad,
shrink=options.shrink,
verbose=verbosity)
tot_parts += num_class_imgs
print_log_msg("Done! Separated %s particles from %s classes\n" %
(tot_parts, num_classes), log, verbose) #=True)
def main():
# Parse the Options
progname = os.path.basename(sys.argv[0])
usage = (progname +
""" unblur_path input_micrograph_pattern output_directory
--summovie_path
--selection_list
--nr_frames=nr_frames
--pixel_size=pixel_size
--voltage=voltage
--exposure_per_frame=exposure_per_frame
--pre_exposure=pre_exposure
--nr_threads
--save_frames
--skip_dose_filter
--expert_mode
--shift_initial=shift_initial
--shift_radius=shift_radius
--b_factor=b_factor
--fourier_vertical=fourier_vertical
--fourier_horizontal=fourier_horizontal
--shift_threshold=shift_threshold
--iterations=iterations
--dont_restore_noise
--verbose
sxunblur exists only in non-MPI version.
Perform unblur and with dose filtering and summovie without dose filtering.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--summovie_path=~/my_app/summovie
--nr_frames=25 --pixel_size=1.19 --exposure_per_frame=1.0
--voltage=300.0 --pre_exposure=0.0 --nr_threads=1
Perform unblur with dose filtering and summovie without dose filtering with selection list.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--summovie_path=~/my_app/summovie
--selection_list=selected_micrograph_file
--nr_frames=25 --pixel_size=1.19 --exposure_per_frame=1.0
--voltage=300.0 --pre_exposure=0.0 --nr_threads=1
Perform unblur without dose filtering.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--nr_frames=25 --pixel_size=1.19 --skip_dose_filter --nr_threads=1
Perform unblur without dose filtering and save the frames.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--nr_frames=25 --pixel_size=1.19 --skip_dose_filter --save_frames --nr_threads=1
Perform unblur with dose filtering and summovie without dose filtering with all options.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--summovie_path=~/my_app/summovie
--nr_frames=25 --pixel_size=1.19 --exposure_per_frame=1.0
--voltage=300.0 --pre_exposure=0.0 --save_frames --expert_mode
--shift_initial=2.0 --shift_radius=200.0 --b_factor=1500.0
--fourier_vertical=1 --fourier_horizontal=1 --shift_threshold=0.1
--iterations=10 --verbose --nr_threads=1
""")
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option(
"--summovie_path",
type="str",
default="",
help=
"summovie executable path (SPHIRE specific): Specify the file path of summovie executable. (default none)",
)
parser.add_option(
"--selection_list",
type="str",
default="",
help=
"Micrograph selecting list (SPHIRE specific): Specify a name of micrograph selection list text file. The file extension must be '.txt'. If this is not provided, all files matched with the micrograph name pattern will be processed. (default none)",
)
parser.add_option(
"--nr_frames",
type="int",
default=3,
help=
"Number of movie frames: The number of movie frames in each input micrograph. (default 3)",
)
parser.add_option("--sum_suffix",
type="str",
default="_sum",
help=optparse.SUPPRESS_HELP)
parser.add_option("--shift_suffix",
type="str",
default="_shift",
help=optparse.SUPPRESS_HELP)
parser.add_option(
"--pixel_size",
type="float",
default=-1.0,
help=
"Pixel size [A]: The pixel size of input micrographs. (default required float)",
)
parser.add_option(
"--voltage",
type="float",
default=300.0,
help=
"Microscope voltage [kV]: The acceleration voltage of microscope used for imaging. (default 300.0)",
)
parser.add_option(
"--exposure_per_frame",
type="float",
default=2.0,
help=
"Per frame exposure [e/A^2]: The electron dose per frame in e/A^2. (default 2.0)",
)
parser.add_option(
"--pre_exposure",
type="float",
default=0.0,
help=
"Pre-exposure [e/A^2]: The electron does in e/A^2 used for exposure prior to imaging .(default 0.0)",
)
parser.add_option(
"--nr_threads",
type="int",
default=1,
help=
"Number of threads: The number of threads unblur can use. The higher the faster, but it requires larger memory. (default 1)",
)
parser.add_option(
"--save_frames",
action="store_true",
default=False,
help=
"Save aligned movie frames: Save aligned movie frames. This option slows down the process. (default False)",
)
parser.add_option("--frames_suffix",
type="string",
default="_frames",
help=optparse.SUPPRESS_HELP)
parser.add_option(
"--skip_dose_filter",
action="store_true",
default=False,
help=
"Skip dose filter step: With this option, voltage, exposure per frame, and pre exposure will be ignored. (default False)",
)
parser.add_option(
"--expert_mode",
action="store_true",
default=False,
help=
"Use expert mode: Requires initial shift, shift radius, b-factor, fourier_vertical, fourier_horizontal, shift threshold, iterations, restore noise, and verbosity options. (default False)",
)
parser.add_option("--frc_suffix",
type="string",
default="_frc",
help=optparse.SUPPRESS_HELP)
parser.add_option(
"--shift_initial",
type="float",
default=2.0,
help=
"Minimum shift for initial search [A] (expert mode): Effective with unblur expert mode. (default 2.0)",
)
parser.add_option(
"--shift_radius",
type="float",
default=200.0,
help=
"Outer radius shift limit [A] (expert mode): Effective with unblur expert mode. (default 200.0)",
)
parser.add_option(
"--b_factor",
type="float",
default=1500.0,
help=
"Apply B-factor to images [A^2] (expert mode): Effective with unblur expert mode. (default 1500.0)",
)
parser.add_option(
"--fourier_vertical",
type="int",
default=1,
help=
"Vertical Fourier central mask size (expert mode): The half-width of central vertical line of Fourier mask. Effective with unblur expert mode. (default 1)",
)
parser.add_option(
"--fourier_horizontal",
type="int",
default=1,
help=
"Horizontal Fourier central mask size (expert mode): The half-width of central horizontal line of Fourier mask. Effective with unblur expert mode. (default 1)",
)
parser.add_option(
"--shift_threshold",
type="float",
default=0.1,
help=
"Termination shift threshold (expert mode): Effective with unblur expert mode. (default 0.1)",
)
parser.add_option(
"--iterations",
type="int",
default=10,
help=
"Maximum iterations (expert mode): Effective with unblur expert mode. (default 10)",
)
parser.add_option(
"--dont_restore_noise",
action="store_true",
default=False,
help=
"Do not restore noise power (expert mode): Effective with unblur expert mode. (default False)",
)
parser.add_option(
"--verbose",
action="store_true",
default=False,
help=
"Verbose (expert mode): Effective with unblur expert mode. (default False)",
)
parser.add_option(
"--unblur_ready",
action="store_true",
default=False,
help=optparse.SUPPRESS_HELP,
)
# list of the options and the arguments
(options, args) = parser.parse_args(sys.argv[1:])
sp_global_def.BATCH = True
# If there arent enough arguments, stop the script
if len(args) != 3:
sp_global_def.sxprint("Usage: " + usage)
sp_global_def.ERROR(
"Invalid number of parameters used. Please see usage information above."
)
return
# Convert the realtive parts to absolute ones
unblur_path = os.path.realpath(args[0]) # unblur_path
input_image = os.path.realpath(args[1]) # input_micrograph_pattern
output_dir = os.path.realpath(args[2]) # output_directory
# If the unblur executable file does not exists, stop the script
if not os.path.exists(unblur_path):
sp_global_def.ERROR(
"Unblur directory does not exist, please change the name and restart the program"
)
return
# If the output directory exists, stop the script
if os.path.exists(output_dir):
sp_global_def.ERROR(
"Output directory exists, please change the name and restart the program"
)
return
# If the input file does not exists, stop the script
file_list = glob.glob(input_image)
if not file_list:
sp_global_def.ERROR(
"Input file does not exist, please change the name and restart the program"
)
return
# If the skip_dose_filter option is false, the summovie path is necessary
if not options.skip_dose_filter and not os.path.exists(
options.summovie_path):
sp_global_def.ERROR(
"Path to the SumMovie executable is necessary when dose weighting is performed"
)
return
# Output paths
corrected_path = "{:s}/corrsum_dose_filtered".format(output_dir)
uncorrected_path = "{:s}/corrsum".format(output_dir)
shift_path = "{:s}/shift".format(output_dir)
frc_path = "{:s}/frc".format(output_dir)
log_path = "{:s}/logfiles".format(output_dir)
temp_path = "{0}/temp".format(output_dir)
# Split the path of the image name at the "/" Characters.
# The last entry contains the micrograph name.
# Split the micrograph name at the wildcard character for the
# prefix and suffix.
input_split = input_image.split("/")
input_name = input_split[-1].split("*")
# Get the input directory
if len(input_split) != 1:
input_dir = input_image[:-len(input_split[-1])]
else:
input_dir = ""
# Create output directorys
if not os.path.exists(output_dir):
os.makedirs(output_dir)
sp_global_def.write_command(output_dir)
if not os.path.exists(uncorrected_path):
os.mkdir(uncorrected_path)
if not os.path.exists(shift_path):
os.mkdir(shift_path)
if not os.path.exists(corrected_path):
if not options.skip_dose_filter:
os.mkdir(corrected_path)
if not os.path.exists(frc_path):
if options.expert_mode or not options.skip_dose_filter:
os.mkdir(frc_path)
if not os.path.exists(temp_path) and not options.unblur_ready:
os.mkdir(temp_path)
if not os.path.exists(log_path):
os.mkdir(log_path)
# Run unblur
run_unblur(
unblur_path=unblur_path,
input_image=input_image,
input_dir=input_dir,
output_dir=output_dir,
corrected_path=corrected_path,
uncorrected_path=uncorrected_path,
shift_path=shift_path,
frc_path=frc_path,
temp_path=temp_path,
log_path=log_path,
file_list=file_list,
options=options,
)
if not options.unblur_ready:
# Remove temp folder
for entry in glob.glob("{0}/*".format(temp_path)):
os.remove(entry)
os.rmdir(temp_path)
sp_global_def.sxprint("All Done!")
sp_global_def.BATCH = False
def main():
global Tracker, Blockdata
progname = os.path.basename(sys.argv[0])
usage = progname + " --output_dir=output_dir --isac_dir=output_dir_of_isac "
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option(
"--pw_adjustment",
type="string",
default="analytical_model",
help=
"adjust power spectrum of 2-D averages to an analytic model. Other opions: no_adjustment; bfactor; a text file of 1D rotationally averaged PW",
)
#### Four options for --pw_adjustment:
# 1> analytical_model(default);
# 2> no_adjustment;
# 3> bfactor;
# 4> adjust_to_given_pw2(user has to provide a text file that contains 1D rotationally averaged PW)
# options in common
parser.add_option(
"--isac_dir",
type="string",
default="",
help="ISAC run output directory, input directory for this command",
)
parser.add_option(
"--output_dir",
type="string",
default="",
help="output directory where computed averages are saved",
)
parser.add_option(
"--pixel_size",
type="float",
default=-1.0,
help=
"pixel_size of raw images. one can put 1.0 in case of negative stain data",
)
parser.add_option(
"--fl",
type="float",
default=-1.0,
help=
"low pass filter, = -1.0, not applied; =0.0, using FH1 (initial resolution), = 1.0 using FH2 (resolution after local alignment), or user provided value in absolute freqency [0.0:0.5]",
)
parser.add_option("--stack",
type="string",
default="",
help="data stack used in ISAC")
parser.add_option("--radius", type="int", default=-1, help="radius")
parser.add_option("--xr",
type="float",
default=-1.0,
help="local alignment search range")
# parser.add_option("--ts", type ="float", default =1.0, help= "local alignment search step")
parser.add_option(
"--fh",
type="float",
default=-1.0,
help="local alignment high frequencies limit",
)
# parser.add_option("--maxit", type ="int", default =5, help= "local alignment iterations")
parser.add_option("--navg",
type="int",
default=1000000,
help="number of aveages")
parser.add_option(
"--local_alignment",
action="store_true",
default=False,
help="do local alignment",
)
parser.add_option(
"--noctf",
action="store_true",
default=False,
help=
"no ctf correction, useful for negative stained data. always ctf for cryo data",
)
parser.add_option(
"--B_start",
type="float",
default=45.0,
help=
"start frequency (Angstrom) of power spectrum for B_factor estimation",
)
parser.add_option(
"--Bfactor",
type="float",
default=-1.0,
help=
"User defined bactors (e.g. 25.0[A^2]). By default, the program automatically estimates B-factor. ",
)
(options, args) = parser.parse_args(sys.argv[1:])
adjust_to_analytic_model = (True if options.pw_adjustment
== "analytical_model" else False)
no_adjustment = True if options.pw_adjustment == "no_adjustment" else False
B_enhance = True if options.pw_adjustment == "bfactor" else False
adjust_to_given_pw2 = (
True if not (adjust_to_analytic_model or no_adjustment or B_enhance)
else False)
# mpi
nproc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
Blockdata = {}
Blockdata["nproc"] = nproc
Blockdata["myid"] = myid
Blockdata["main_node"] = 0
Blockdata["shared_comm"] = mpi.mpi_comm_split_type(
mpi.MPI_COMM_WORLD, mpi.MPI_COMM_TYPE_SHARED, 0, mpi.MPI_INFO_NULL)
Blockdata["myid_on_node"] = mpi.mpi_comm_rank(Blockdata["shared_comm"])
Blockdata["no_of_processes_per_group"] = mpi.mpi_comm_size(
Blockdata["shared_comm"])
masters_from_groups_vs_everything_else_comm = mpi.mpi_comm_split(
mpi.MPI_COMM_WORLD,
Blockdata["main_node"] == Blockdata["myid_on_node"],
Blockdata["myid_on_node"],
)
Blockdata["color"], Blockdata[
"no_of_groups"], balanced_processor_load_on_nodes = sp_utilities.get_colors_and_subsets(
Blockdata["main_node"],
mpi.MPI_COMM_WORLD,
Blockdata["myid"],
Blockdata["shared_comm"],
Blockdata["myid_on_node"],
masters_from_groups_vs_everything_else_comm,
)
# We need two nodes for processing of volumes
Blockdata["node_volume"] = [
Blockdata["no_of_groups"] - 3,
Blockdata["no_of_groups"] - 2,
Blockdata["no_of_groups"] - 1,
] # For 3D stuff take three last nodes
# We need two CPUs for processing of volumes, they are taken to be main CPUs on each volume
# We have to send the two myids to all nodes so we can identify main nodes on two selected groups.
Blockdata["nodes"] = [
Blockdata["node_volume"][0] * Blockdata["no_of_processes_per_group"],
Blockdata["node_volume"][1] * Blockdata["no_of_processes_per_group"],
Blockdata["node_volume"][2] * Blockdata["no_of_processes_per_group"],
]
# End of Blockdata: sorting requires at least three nodes, and the used number of nodes be integer times of three
sp_global_def.BATCH = True
sp_global_def.MPI = True
if adjust_to_given_pw2:
checking_flag = 0
if Blockdata["myid"] == Blockdata["main_node"]:
if not os.path.exists(options.pw_adjustment):
checking_flag = 1
checking_flag = sp_utilities.bcast_number_to_all(
checking_flag, Blockdata["main_node"], mpi.MPI_COMM_WORLD)
if checking_flag == 1:
sp_global_def.ERROR("User provided power spectrum does not exist",
myid=Blockdata["myid"])
Tracker = {}
Constants = {}
Constants["isac_dir"] = options.isac_dir
Constants["masterdir"] = options.output_dir
Constants["pixel_size"] = options.pixel_size
Constants["orgstack"] = options.stack
Constants["radius"] = options.radius
Constants["xrange"] = options.xr
Constants["FH"] = options.fh
Constants["low_pass_filter"] = options.fl
# Constants["maxit"] = options.maxit
Constants["navg"] = options.navg
Constants["B_start"] = options.B_start
Constants["Bfactor"] = options.Bfactor
if adjust_to_given_pw2:
Constants["modelpw"] = options.pw_adjustment
Tracker["constants"] = Constants
# -------------------------------------------------------------
#
# Create and initialize Tracker dictionary with input options # State Variables
# <<<---------------------->>>imported functions<<<---------------------------------------------
# x_range = max(Tracker["constants"]["xrange"], int(1./Tracker["ini_shrink"])+1)
# y_range = x_range
####-----------------------------------------------------------
# Create Master directory and associated subdirectories
line = time.strftime("%Y-%m-%d_%H:%M:%S", time.localtime()) + " =>"
if Tracker["constants"]["masterdir"] == Tracker["constants"]["isac_dir"]:
masterdir = os.path.join(Tracker["constants"]["isac_dir"], "sharpen")
else:
masterdir = Tracker["constants"]["masterdir"]
if Blockdata["myid"] == Blockdata["main_node"]:
msg = "Postprocessing ISAC 2D averages starts"
sp_global_def.sxprint(line, "Postprocessing ISAC 2D averages starts")
if not masterdir:
timestring = time.strftime("_%d_%b_%Y_%H_%M_%S", time.localtime())
masterdir = "sharpen_" + Tracker["constants"]["isac_dir"]
os.makedirs(masterdir)
else:
if os.path.exists(masterdir):
sp_global_def.sxprint("%s already exists" % masterdir)
else:
os.makedirs(masterdir)
sp_global_def.write_command(masterdir)
subdir_path = os.path.join(masterdir, "ali2d_local_params_avg")
if not os.path.exists(subdir_path):
os.mkdir(subdir_path)
subdir_path = os.path.join(masterdir, "params_avg")
if not os.path.exists(subdir_path):
os.mkdir(subdir_path)
li = len(masterdir)
else:
li = 0
li = mpi.mpi_bcast(li, 1, mpi.MPI_INT, Blockdata["main_node"],
mpi.MPI_COMM_WORLD)[0]
masterdir = mpi.mpi_bcast(masterdir, li, mpi.MPI_CHAR,
Blockdata["main_node"], mpi.MPI_COMM_WORLD)
masterdir = b"".join(masterdir).decode('latin1')
Tracker["constants"]["masterdir"] = masterdir
log_main = sp_logger.Logger(sp_logger.BaseLogger_Files())
log_main.prefix = Tracker["constants"]["masterdir"] + "/"
while not os.path.exists(Tracker["constants"]["masterdir"]):
sp_global_def.sxprint(
"Node ",
Blockdata["myid"],
" waiting...",
Tracker["constants"]["masterdir"],
)
time.sleep(1)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if Blockdata["myid"] == Blockdata["main_node"]:
init_dict = {}
sp_global_def.sxprint(Tracker["constants"]["isac_dir"])
Tracker["directory"] = os.path.join(Tracker["constants"]["isac_dir"],
"2dalignment")
core = sp_utilities.read_text_row(
os.path.join(Tracker["directory"], "initial2Dparams.txt"))
for im in range(len(core)):
init_dict[im] = core[im]
del core
else:
init_dict = 0
init_dict = sp_utilities.wrap_mpi_bcast(init_dict,
Blockdata["main_node"],
communicator=mpi.MPI_COMM_WORLD)
###
do_ctf = True
if options.noctf:
do_ctf = False
if Blockdata["myid"] == Blockdata["main_node"]:
if do_ctf:
sp_global_def.sxprint("CTF correction is on")
else:
sp_global_def.sxprint("CTF correction is off")
if options.local_alignment:
sp_global_def.sxprint("local refinement is on")
else:
sp_global_def.sxprint("local refinement is off")
if B_enhance:
sp_global_def.sxprint("Bfactor is to be applied on averages")
elif adjust_to_given_pw2:
sp_global_def.sxprint(
"PW of averages is adjusted to a given 1D PW curve")
elif adjust_to_analytic_model:
sp_global_def.sxprint(
"PW of averages is adjusted to analytical model")
else:
sp_global_def.sxprint("PW of averages is not adjusted")
# Tracker["constants"]["orgstack"] = "bdb:"+ os.path.join(Tracker["constants"]["isac_dir"],"../","sparx_stack")
image = sp_utilities.get_im(Tracker["constants"]["orgstack"], 0)
Tracker["constants"]["nnxo"] = image.get_xsize()
if Tracker["constants"]["pixel_size"] == -1.0:
sp_global_def.sxprint(
"Pixel size value is not provided by user. extracting it from ctf header entry of the original stack."
)
try:
ctf_params = image.get_attr("ctf")
Tracker["constants"]["pixel_size"] = ctf_params.apix
except:
sp_global_def.ERROR(
"Pixel size could not be extracted from the original stack.",
myid=Blockdata["myid"],
)
## Now fill in low-pass filter
isac_shrink_path = os.path.join(Tracker["constants"]["isac_dir"],
"README_shrink_ratio.txt")
if not os.path.exists(isac_shrink_path):
sp_global_def.ERROR(
"%s does not exist in the specified ISAC run output directory"
% (isac_shrink_path),
myid=Blockdata["myid"],
)
isac_shrink_file = open(isac_shrink_path, "r")
isac_shrink_lines = isac_shrink_file.readlines()
isac_shrink_ratio = float(
isac_shrink_lines[5]
) # 6th line: shrink ratio (= [target particle radius]/[particle radius]) used in the ISAC run
isac_radius = float(
isac_shrink_lines[6]
) # 7th line: particle radius at original pixel size used in the ISAC run
isac_shrink_file.close()
print("Extracted parameter values")
print("ISAC shrink ratio : {0}".format(isac_shrink_ratio))
print("ISAC particle radius : {0}".format(isac_radius))
Tracker["ini_shrink"] = isac_shrink_ratio
else:
Tracker["ini_shrink"] = 0.0
Tracker = sp_utilities.wrap_mpi_bcast(Tracker,
Blockdata["main_node"],
communicator=mpi.MPI_COMM_WORLD)
# print(Tracker["constants"]["pixel_size"], "pixel_size")
x_range = max(
Tracker["constants"]["xrange"],
int(old_div(1.0, Tracker["ini_shrink"]) + 0.99999),
)
a_range = y_range = x_range
if Blockdata["myid"] == Blockdata["main_node"]:
parameters = sp_utilities.read_text_row(
os.path.join(Tracker["constants"]["isac_dir"],
"all_parameters.txt"))
else:
parameters = 0
parameters = sp_utilities.wrap_mpi_bcast(parameters,
Blockdata["main_node"],
communicator=mpi.MPI_COMM_WORLD)
params_dict = {}
list_dict = {}
# parepare params_dict
# navg = min(Tracker["constants"]["navg"]*Blockdata["nproc"], EMUtil.get_image_count(os.path.join(Tracker["constants"]["isac_dir"], "class_averages.hdf")))
navg = min(
Tracker["constants"]["navg"],
EMAN2_cppwrap.EMUtil.get_image_count(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf")),
)
global_dict = {}
ptl_list = []
memlist = []
if Blockdata["myid"] == Blockdata["main_node"]:
sp_global_def.sxprint("Number of averages computed in this run is %d" %
navg)
for iavg in range(navg):
params_of_this_average = []
image = sp_utilities.get_im(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf"),
iavg,
)
members = sorted(image.get_attr("members"))
memlist.append(members)
for im in range(len(members)):
abs_id = members[im]
global_dict[abs_id] = [iavg, im]
P = sp_utilities.combine_params2(
init_dict[abs_id][0],
init_dict[abs_id][1],
init_dict[abs_id][2],
init_dict[abs_id][3],
parameters[abs_id][0],
old_div(parameters[abs_id][1], Tracker["ini_shrink"]),
old_div(parameters[abs_id][2], Tracker["ini_shrink"]),
parameters[abs_id][3],
)
if parameters[abs_id][3] == -1:
sp_global_def.sxprint(
"WARNING: Image #{0} is an unaccounted particle with invalid 2D alignment parameters and should not be the member of any classes. Please check the consitency of input dataset."
.format(abs_id)
) # How to check what is wrong about mirror = -1 (Toshio 2018/01/11)
params_of_this_average.append([P[0], P[1], P[2], P[3], 1.0])
ptl_list.append(abs_id)
params_dict[iavg] = params_of_this_average
list_dict[iavg] = members
sp_utilities.write_text_row(
params_of_this_average,
os.path.join(
Tracker["constants"]["masterdir"],
"params_avg",
"params_avg_%03d.txt" % iavg,
),
)
ptl_list.sort()
init_params = [None for im in range(len(ptl_list))]
for im in range(len(ptl_list)):
init_params[im] = [ptl_list[im]] + params_dict[global_dict[
ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
sp_utilities.write_text_row(
init_params,
os.path.join(Tracker["constants"]["masterdir"],
"init_isac_params.txt"),
)
else:
params_dict = 0
list_dict = 0
memlist = 0
params_dict = sp_utilities.wrap_mpi_bcast(params_dict,
Blockdata["main_node"],
communicator=mpi.MPI_COMM_WORLD)
list_dict = sp_utilities.wrap_mpi_bcast(list_dict,
Blockdata["main_node"],
communicator=mpi.MPI_COMM_WORLD)
memlist = sp_utilities.wrap_mpi_bcast(memlist,
Blockdata["main_node"],
communicator=mpi.MPI_COMM_WORLD)
# Now computing!
del init_dict
tag_sharpen_avg = 1000
## always apply low pass filter to B_enhanced images to suppress noise in high frequencies
enforced_to_H1 = False
if B_enhance:
if Tracker["constants"]["low_pass_filter"] == -1.0:
enforced_to_H1 = True
# distribute workload among mpi processes
image_start, image_end = sp_applications.MPI_start_end(
navg, Blockdata["nproc"], Blockdata["myid"])
if Blockdata["myid"] == Blockdata["main_node"]:
cpu_dict = {}
for iproc in range(Blockdata["nproc"]):
local_image_start, local_image_end = sp_applications.MPI_start_end(
navg, Blockdata["nproc"], iproc)
for im in range(local_image_start, local_image_end):
cpu_dict[im] = iproc
else:
cpu_dict = 0
cpu_dict = sp_utilities.wrap_mpi_bcast(cpu_dict,
Blockdata["main_node"],
communicator=mpi.MPI_COMM_WORLD)
slist = [None for im in range(navg)]
ini_list = [None for im in range(navg)]
avg1_list = [None for im in range(navg)]
avg2_list = [None for im in range(navg)]
data_list = [None for im in range(navg)]
plist_dict = {}
if Blockdata["myid"] == Blockdata["main_node"]:
if B_enhance:
sp_global_def.sxprint(
"Avg ID B-factor FH1(Res before ali) FH2(Res after ali)")
else:
sp_global_def.sxprint(
"Avg ID FH1(Res before ali) FH2(Res after ali)")
FH_list = [[0, 0.0, 0.0] for im in range(navg)]
for iavg in range(image_start, image_end):
mlist = EMAN2_cppwrap.EMData.read_images(
Tracker["constants"]["orgstack"], list_dict[iavg])
for im in range(len(mlist)):
sp_utilities.set_params2D(mlist[im],
params_dict[iavg][im],
xform="xform.align2d")
if options.local_alignment:
new_avg, plist, FH2 = sp_applications.refinement_2d_local(
mlist,
Tracker["constants"]["radius"],
a_range,
x_range,
y_range,
CTF=do_ctf,
SNR=1.0e10,
)
plist_dict[iavg] = plist
FH1 = -1.0
else:
new_avg, frc, plist = compute_average(
mlist, Tracker["constants"]["radius"], do_ctf)
FH1 = get_optimistic_res(frc)
FH2 = -1.0
FH_list[iavg] = [iavg, FH1, FH2]
if B_enhance:
new_avg, gb = apply_enhancement(
new_avg,
Tracker["constants"]["B_start"],
Tracker["constants"]["pixel_size"],
Tracker["constants"]["Bfactor"],
)
sp_global_def.sxprint(" %6d %6.3f %4.3f %4.3f" %
(iavg, gb, FH1, FH2))
elif adjust_to_given_pw2:
roo = sp_utilities.read_text_file(Tracker["constants"]["modelpw"],
-1)
roo = roo[0] # always on the first column
new_avg = adjust_pw_to_model(new_avg,
Tracker["constants"]["pixel_size"],
roo)
sp_global_def.sxprint(" %6d %4.3f %4.3f " %
(iavg, FH1, FH2))
elif adjust_to_analytic_model:
new_avg = adjust_pw_to_model(new_avg,
Tracker["constants"]["pixel_size"],
None)
sp_global_def.sxprint(" %6d %4.3f %4.3f " %
(iavg, FH1, FH2))
elif no_adjustment:
pass
if Tracker["constants"]["low_pass_filter"] != -1.0:
if Tracker["constants"]["low_pass_filter"] == 0.0:
low_pass_filter = FH1
elif Tracker["constants"]["low_pass_filter"] == 1.0:
low_pass_filter = FH2
if not options.local_alignment:
low_pass_filter = FH1
else:
low_pass_filter = Tracker["constants"]["low_pass_filter"]
if low_pass_filter >= 0.45:
low_pass_filter = 0.45
new_avg = sp_filter.filt_tanl(new_avg, low_pass_filter, 0.02)
else: # No low pass filter but if enforced
if enforced_to_H1:
new_avg = sp_filter.filt_tanl(new_avg, FH1, 0.02)
if B_enhance:
new_avg = sp_fundamentals.fft(new_avg)
new_avg.set_attr("members", list_dict[iavg])
new_avg.set_attr("n_objects", len(list_dict[iavg]))
slist[iavg] = new_avg
sp_global_def.sxprint(
time.strftime("%Y-%m-%d_%H:%M:%S", time.localtime()) + " =>",
"Refined average %7d" % iavg,
)
## send to main node to write
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
for im in range(navg):
# avg
if (cpu_dict[im] == Blockdata["myid"]
and Blockdata["myid"] != Blockdata["main_node"]):
sp_utilities.send_EMData(slist[im], Blockdata["main_node"],
tag_sharpen_avg)
elif (cpu_dict[im] == Blockdata["myid"]
and Blockdata["myid"] == Blockdata["main_node"]):
slist[im].set_attr("members", memlist[im])
slist[im].set_attr("n_objects", len(memlist[im]))
slist[im].write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"),
im,
)
elif (cpu_dict[im] != Blockdata["myid"]
and Blockdata["myid"] == Blockdata["main_node"]):
new_avg_other_cpu = sp_utilities.recv_EMData(
cpu_dict[im], tag_sharpen_avg)
new_avg_other_cpu.set_attr("members", memlist[im])
new_avg_other_cpu.set_attr("n_objects", len(memlist[im]))
new_avg_other_cpu.write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"),
im,
)
if options.local_alignment:
if cpu_dict[im] == Blockdata["myid"]:
sp_utilities.write_text_row(
plist_dict[im],
os.path.join(
Tracker["constants"]["masterdir"],
"ali2d_local_params_avg",
"ali2d_local_params_avg_%03d.txt" % im,
),
)
if (cpu_dict[im] == Blockdata["myid"]
and cpu_dict[im] != Blockdata["main_node"]):
sp_utilities.wrap_mpi_send(plist_dict[im],
Blockdata["main_node"],
mpi.MPI_COMM_WORLD)
sp_utilities.wrap_mpi_send(FH_list, Blockdata["main_node"],
mpi.MPI_COMM_WORLD)
elif (cpu_dict[im] != Blockdata["main_node"]
and Blockdata["myid"] == Blockdata["main_node"]):
dummy = sp_utilities.wrap_mpi_recv(cpu_dict[im],
mpi.MPI_COMM_WORLD)
plist_dict[im] = dummy
dummy = sp_utilities.wrap_mpi_recv(cpu_dict[im],
mpi.MPI_COMM_WORLD)
FH_list[im] = dummy[im]
else:
if (cpu_dict[im] == Blockdata["myid"]
and cpu_dict[im] != Blockdata["main_node"]):
sp_utilities.wrap_mpi_send(FH_list, Blockdata["main_node"],
mpi.MPI_COMM_WORLD)
elif (cpu_dict[im] != Blockdata["main_node"]
and Blockdata["myid"] == Blockdata["main_node"]):
dummy = sp_utilities.wrap_mpi_recv(cpu_dict[im],
mpi.MPI_COMM_WORLD)
FH_list[im] = dummy[im]
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if options.local_alignment:
if Blockdata["myid"] == Blockdata["main_node"]:
ali3d_local_params = [None for im in range(len(ptl_list))]
for im in range(len(ptl_list)):
ali3d_local_params[im] = [ptl_list[im]] + plist_dict[
global_dict[ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
sp_utilities.write_text_row(
ali3d_local_params,
os.path.join(Tracker["constants"]["masterdir"],
"ali2d_local_params.txt"),
)
sp_utilities.write_text_row(
FH_list,
os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))
else:
if Blockdata["myid"] == Blockdata["main_node"]:
sp_utilities.write_text_row(
FH_list,
os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
target_xr = 3
target_yr = 3
if Blockdata["myid"] == 0:
cmd = "{} {} {} {} {} {} {} {} {} {}".format(
"sp_chains.py",
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"),
os.path.join(Tracker["constants"]["masterdir"], "junk.hdf"),
os.path.join(Tracker["constants"]["masterdir"],
"ordered_class_averages.hdf"),
"--circular",
"--radius=%d" % Tracker["constants"]["radius"],
"--xr=%d" % (target_xr + 1),
"--yr=%d" % (target_yr + 1),
"--align",
">/dev/null",
)
junk = sp_utilities.cmdexecute(cmd)
cmd = "{} {}".format(
"rm -rf",
os.path.join(Tracker["constants"]["masterdir"], "junk.hdf"))
junk = sp_utilities.cmdexecute(cmd)
return
def main():
arglist = []
for arg in sys.argv:
arglist.append(arg)
progname = optparse.os.path.basename(arglist[0])
usage = (
progname
+ " prj_stack volume [begin end step] --CTF --npad=ntimes_padding --list=file --group=ID --snr=SNR --sym=symmetry --verbose=(0|1) --xysize --MPI"
)
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option(
"--CTF", action="store_true", default=False, help="apply CTF correction"
)
parser.add_option("--snr", type="float", default=1.0, help="Signal-to-Noise Ratio")
parser.add_option("--sym", type="string", default="c1", help="symmetry")
parser.add_option(
"--list",
type="string",
help="file with list of images to be used in the first column",
)
parser.add_option(
"--group",
type="int",
default=-1,
help="perform reconstruction using images for a given group number (group is attribute in the header)",
)
parser.add_option(
"--MPI", action="store_true", default=False, help="use MPI version "
)
parser.add_option(
"--npad", type="int", default=2, help="number of times padding (default 2)"
)
parser.add_option(
"--verbose",
type="int",
default=0,
help="verbose level: 0 no verbose, 1 verbose",
)
parser.add_option(
"--xysize", type="int", default=-1, help="user expected size at xy direction"
)
parser.add_option(
"--zsize", type="int", default=-1, help="user expected size at z direction"
)
parser.add_option(
"--smearstep",
type="float",
default=0.0,
help="Rotational smear step (default 0.0, no smear)",
)
parser.add_option(
"--interpolation_method",
type="string",
default="4nn",
help="4nn, or tril: nearest neighbor, or trilinear interpolation",
)
parser.add_option(
"--niter",
type="int",
default=10,
help="number of iterations for iterative reconstruction",
)
parser.add_option(
"--upweighted",
action="store_true",
default=False,
help="apply background noise",
)
parser.add_option(
"--compensate",
action="store_true",
default=False,
help="compensate in reconstruction",
)
parser.add_option(
"--chunk_id",
type="int",
default=-1,
help="reconstruct both odd and even groups of particles",
)
parser.add_option(
"--target_window_size",
type="int",
default=-1,
help=" size of the targeted reconstruction ",
)
(options, args) = parser.parse_args(arglist[1:])
if sp_global_def.CACHE_DISABLE:
sp_utilities.disable_bdb_cache()
if len(args) == 2:
prj_stack = args[0]
vol_stack = args[1]
nimage = EMAN2_cppwrap.EMUtil.get_image_count(prj_stack)
pid_list = list(range(0, nimage))
elif len(args) == 5:
prj_stack = args[0]
vol_stack = args[1]
begin = string.atoi(args[2])
end = string.atoi(args[3])
step = string.atoi(args[4])
pid_list = list(range(begin, end, step))
else:
sp_global_def.ERROR("Incomplete list of arguments")
return
if options.list and options.group > -1:
sp_global_def.ERROR("options group and list cannot be used together")
return
sp_global_def.BATCH = True
if options.interpolation_method == "4nn":
sp_applications.recons3d_n(
prj_stack,
pid_list,
vol_stack,
options.CTF,
options.snr,
1,
options.npad,
options.sym,
options.list,
options.group,
options.verbose,
options.MPI,
options.xysize,
options.zsize,
options.smearstep,
options.upweighted,
options.compensate,
options.chunk_id,
)
elif options.interpolation_method == "tril":
if options.MPI is False:
sp_global_def.ERROR(
"Trilinear interpolation reconstruction has MPI version only!"
)
return
sp_applications.recons3d_trl_MPI(
prj_stack,
pid_list,
vol_stack,
options.CTF,
options.snr,
1,
options.npad,
options.sym,
options.verbose,
options.niter,
options.compensate,
options.target_window_size,
)
else:
sp_global_def.ERROR(
"Wrong interpolation method. The current options are 4nn, and tril. 4nn is the default one."
)
return
sp_global_def.write_command(optparse.os.path.dirname(vol_stack))
sp_global_def.BATCH = False
def main():
def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
# the final ali2d parameters already combine shifts operation first and rotation operation second for parameters converted from 3D
if mirror:
m = 1
alpha, sx, sy, scalen = sp_utilities.compose_transform2(
0, s2x, s2y, 1.0, 540.0 - psi, 0, 0, 1.0)
else:
m = 0
alpha, sx, sy, scalen = sp_utilities.compose_transform2(
0, s2x, s2y, 1.0, 360.0 - psi, 0, 0, 1.0)
return alpha, sx, sy, m
progname = optparse.os.path.basename(sys.argv[0])
usage = (
progname +
" prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl= --aa= --sym=symmetry --CTF"
)
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option("--output_dir",
type="string",
default="./",
help="Output directory")
parser.add_option(
"--ave2D",
type="string",
default=False,
help="Write to the disk a stack of 2D averages",
)
parser.add_option(
"--var2D",
type="string",
default=False,
help="Write to the disk a stack of 2D variances",
)
parser.add_option(
"--ave3D",
type="string",
default=False,
help="Write to the disk reconstructed 3D average",
)
parser.add_option(
"--var3D",
type="string",
default=False,
help="Compute 3D variability (time consuming!)",
)
parser.add_option(
"--img_per_grp",
type="int",
default=100,
help="Number of neighbouring projections.(Default is 100)",
)
parser.add_option(
"--no_norm",
action="store_true",
default=False,
help="Do not use normalization.(Default is to apply normalization)",
)
# parser.add_option("--radius", type="int" , default=-1 , help="radius for 3D variability" )
parser.add_option(
"--npad",
type="int",
default=2,
help=
"Number of time to pad the original images.(Default is 2 times padding)",
)
parser.add_option("--sym",
type="string",
default="c1",
help="Symmetry. (Default is no symmetry)")
parser.add_option(
"--fl",
type="float",
default=0.0,
help=
"Low pass filter cutoff in absolute frequency (0.0 - 0.5) and is applied to decimated images. (Default - no filtration)",
)
parser.add_option(
"--aa",
type="float",
default=0.02,
help=
"Fall off of the filter. Use default value if user has no clue about falloff (Default value is 0.02)",
)
parser.add_option(
"--CTF",
action="store_true",
default=False,
help="Use CFT correction.(Default is no CTF correction)",
)
# parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
# parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" )
# parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" )
# parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
# parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
parser.add_option(
"--VAR",
action="store_true",
default=False,
help="Stack of input consists of 2D variances (Default False)",
)
parser.add_option(
"--decimate",
type="float",
default=0.25,
help="Image decimate rate, a number less than 1. (Default is 0.25)",
)
parser.add_option(
"--window",
type="int",
default=0,
help=
"Target image size relative to original image size. (Default value is zero.)",
)
# parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)")
# parser.add_option("--nvec", type="int" , default=0 , help="Number of eigenvectors, (Default = 0 meaning no PCA calculated)")
parser.add_option(
"--symmetrize",
action="store_true",
default=False,
help="Prepare input stack for handling symmetry (Default False)",
)
parser.add_option("--overhead",
type="float",
default=0.5,
help="python overhead per CPU.")
(options, args) = parser.parse_args()
#####
# from mpi import *
# This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015
# Set up global variables related to bdb cache
if sp_global_def.CACHE_DISABLE:
sp_utilities.disable_bdb_cache()
# Set up global variables related to ERROR function
sp_global_def.BATCH = True
# detect if program is running under MPI
RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in optparse.os.environ
if RUNNING_UNDER_MPI:
sp_global_def.MPI = True
if options.output_dir == "./":
current_output_dir = optparse.os.path.abspath(options.output_dir)
else:
current_output_dir = options.output_dir
if options.symmetrize:
if mpi.mpi_comm_size(mpi.MPI_COMM_WORLD) > 1:
sp_global_def.ERROR(
"Cannot use more than one CPU for symmetry preparation")
if not optparse.os.path.exists(current_output_dir):
optparse.os.makedirs(current_output_dir)
sp_global_def.write_command(current_output_dir)
if optparse.os.path.exists(
optparse.os.path.join(current_output_dir, "log.txt")):
optparse.os.remove(
optparse.os.path.join(current_output_dir, "log.txt"))
log_main = sp_logger.Logger(sp_logger.BaseLogger_Files())
log_main.prefix = optparse.os.path.join(current_output_dir, "./")
instack = args[0]
sym = options.sym.lower()
if sym == "c1":
sp_global_def.ERROR(
"There is no need to symmetrize stack for C1 symmetry")
line = ""
for a in sys.argv:
line += " " + a
log_main.add(line)
if instack[:4] != "bdb:":
# if output_dir =="./": stack = "bdb:data"
stack = "bdb:" + current_output_dir + "/data"
sp_utilities.delete_bdb(stack)
junk = sp_utilities.cmdexecute("sp_cpy.py " + instack + " " +
stack)
else:
stack = instack
qt = EMAN2_cppwrap.EMUtil.get_all_attributes(stack, "xform.projection")
na = len(qt)
ts = sp_utilities.get_symt(sym)
ks = len(ts)
angsa = [None] * na
for k in range(ks):
# Qfile = "Q%1d"%k
# if options.output_dir!="./": Qfile = os.path.join(options.output_dir,"Q%1d"%k)
Qfile = optparse.os.path.join(current_output_dir, "Q%1d" % k)
# delete_bdb("bdb:Q%1d"%k)
sp_utilities.delete_bdb("bdb:" + Qfile)
# junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
junk = sp_utilities.cmdexecute("e2bdb.py " + stack +
" --makevstack=bdb:" + Qfile)
# DB = db_open_dict("bdb:Q%1d"%k)
DB = EMAN2db.db_open_dict("bdb:" + Qfile)
for i in range(na):
ut = qt[i] * ts[k]
DB.set_attr(i, "xform.projection", ut)
# bt = ut.get_params("spider")
# angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
# write_text_row(angsa, 'ptsma%1d.txt'%k)
# junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
# junk = cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k))
DB.close()
# if options.output_dir =="./": delete_bdb("bdb:sdata")
sp_utilities.delete_bdb("bdb:" + current_output_dir + "/" + "sdata")
# junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
sdata = "bdb:" + current_output_dir + "/" + "sdata"
sp_global_def.sxprint(sdata)
junk = sp_utilities.cmdexecute("e2bdb.py " + current_output_dir +
" --makevstack=" + sdata + " --filt=Q")
# junk = cmdexecute("ls EMAN2DB/sdata*")
# a = get_im("bdb:sdata")
a = sp_utilities.get_im(sdata)
a.set_attr("variabilitysymmetry", sym)
# a.write_image("bdb:sdata")
a.write_image(sdata)
else:
myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
number_of_proc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
main_node = 0
shared_comm = mpi.mpi_comm_split_type(mpi.MPI_COMM_WORLD,
mpi.MPI_COMM_TYPE_SHARED, 0,
mpi.MPI_INFO_NULL)
myid_on_node = mpi.mpi_comm_rank(shared_comm)
no_of_processes_per_group = mpi.mpi_comm_size(shared_comm)
masters_from_groups_vs_everything_else_comm = mpi.mpi_comm_split(
mpi.MPI_COMM_WORLD, main_node == myid_on_node, myid_on_node)
color, no_of_groups, balanced_processor_load_on_nodes = sp_utilities.get_colors_and_subsets(
main_node,
mpi.MPI_COMM_WORLD,
myid,
shared_comm,
myid_on_node,
masters_from_groups_vs_everything_else_comm,
)
overhead_loading = options.overhead * number_of_proc
# memory_per_node = options.memory_per_node
# if memory_per_node == -1.: memory_per_node = 2.*no_of_processes_per_group
keepgoing = 1
current_window = options.window
current_decimate = options.decimate
if len(args) == 1:
stack = args[0]
else:
sp_global_def.sxprint("Usage: " + usage)
sp_global_def.sxprint("Please run '" + progname +
" -h' for detailed options")
sp_global_def.ERROR(
"Invalid number of parameters used. Please see usage information above."
)
return
t0 = time.time()
# obsolete flags
options.MPI = True
# options.nvec = 0
options.radiuspca = -1
options.iter = 40
options.abs = 0.0
options.squ = 0.0
if options.fl > 0.0 and options.aa == 0.0:
sp_global_def.ERROR(
"Fall off has to be given for the low-pass filter", myid=myid)
# if options.VAR and options.SND:
# ERROR( "Only one of var and SND can be set!",myid=myid )
if options.VAR and (options.ave2D or options.ave3D or options.var2D):
sp_global_def.ERROR(
"When VAR is set, the program cannot output ave2D, ave3D or var2D",
myid=myid,
)
# if options.SND and (options.ave2D or options.ave3D):
# ERROR( "When SND is set, the program cannot output ave2D or ave3D", myid=myid )
# if options.nvec > 0 :
# ERROR( "PCA option not implemented", myid=myid )
# if options.nvec > 0 and options.ave3D == None:
# ERROR( "When doing PCA analysis, one must set ave3D", myid=myid )
if current_decimate > 1.0 or current_decimate < 0.0:
sp_global_def.ERROR(
"Decimate rate should be a value between 0.0 and 1.0",
myid=myid)
if current_window < 0.0:
sp_global_def.ERROR(
"Target window size should be always larger than zero",
myid=myid)
if myid == main_node:
img = sp_utilities.get_image(stack, 0)
nx = img.get_xsize()
ny = img.get_ysize()
if min(nx, ny) < current_window:
keepgoing = 0
keepgoing = sp_utilities.bcast_number_to_all(keepgoing, main_node,
mpi.MPI_COMM_WORLD)
if keepgoing == 0:
sp_global_def.ERROR(
"The target window size cannot be larger than the size of decimated image",
myid=myid,
)
options.sym = options.sym.lower()
# if global_def.CACHE_DISABLE:
# from utilities import disable_bdb_cache
# disable_bdb_cache()
# global_def.BATCH = True
if myid == main_node:
if not optparse.os.path.exists(current_output_dir):
optparse.os.makedirs(
current_output_dir
) # Never delete output_dir in the program!
img_per_grp = options.img_per_grp
# nvec = options.nvec
radiuspca = options.radiuspca
# if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt"))
log_main = sp_logger.Logger(sp_logger.BaseLogger_Files())
log_main.prefix = optparse.os.path.join(current_output_dir, "./")
if myid == main_node:
line = ""
for a in sys.argv:
line += " " + a
log_main.add(line)
log_main.add("-------->>>Settings given by all options<<<-------")
log_main.add("Symmetry : %s" % options.sym)
log_main.add("Input stack : %s" % stack)
log_main.add("Output_dir : %s" % current_output_dir)
if options.ave3D:
log_main.add("Ave3d : %s" % options.ave3D)
if options.var3D:
log_main.add("Var3d : %s" % options.var3D)
if options.ave2D:
log_main.add("Ave2D : %s" % options.ave2D)
if options.var2D:
log_main.add("Var2D : %s" % options.var2D)
if options.VAR:
log_main.add("VAR : True")
else:
log_main.add("VAR : False")
if options.CTF:
log_main.add("CTF correction : True ")
else:
log_main.add("CTF correction : False ")
log_main.add("Image per group : %5d" % options.img_per_grp)
log_main.add("Image decimate rate : %4.3f" % current_decimate)
log_main.add("Low pass filter : %4.3f" % options.fl)
current_fl = options.fl
if current_fl == 0.0:
current_fl = 0.5
log_main.add(
"Current low pass filter is equivalent to cutoff frequency %4.3f for original image size"
% round((current_fl * current_decimate), 3))
log_main.add("Window size : %5d " % current_window)
log_main.add("sx3dvariability begins")
symbaselen = 0
if myid == main_node:
nima = EMAN2_cppwrap.EMUtil.get_image_count(stack)
img = sp_utilities.get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
nnxo = nx
nnyo = ny
if options.sym != "c1":
imgdata = sp_utilities.get_im(stack)
try:
i = imgdata.get_attr("variabilitysymmetry").lower()
if i != options.sym:
sp_global_def.ERROR(
"The symmetry provided does not agree with the symmetry of the input stack",
myid=myid,
)
except:
sp_global_def.ERROR(
"Input stack is not prepared for symmetry, please follow instructions",
myid=myid,
)
i = len(sp_utilities.get_symt(options.sym))
if (old_div(nima, i)) * i != nima:
sp_global_def.ERROR(
"The length of the input stack is incorrect for symmetry processing",
myid=myid,
)
symbaselen = old_div(nima, i)
else:
symbaselen = nima
else:
nima = 0
nx = 0
ny = 0
nnxo = 0
nnyo = 0
nima = sp_utilities.bcast_number_to_all(nima)
nx = sp_utilities.bcast_number_to_all(nx)
ny = sp_utilities.bcast_number_to_all(ny)
nnxo = sp_utilities.bcast_number_to_all(nnxo)
nnyo = sp_utilities.bcast_number_to_all(nnyo)
if current_window > max(nx, ny):
sp_global_def.ERROR(
"Window size is larger than the original image size")
if current_decimate == 1.0:
if current_window != 0:
nx = current_window
ny = current_window
else:
if current_window == 0:
nx = int(nx * current_decimate + 0.5)
ny = int(ny * current_decimate + 0.5)
else:
nx = int(current_window * current_decimate + 0.5)
ny = nx
symbaselen = sp_utilities.bcast_number_to_all(symbaselen)
# check FFT prime number
is_fft_friendly = nx == sp_fundamentals.smallprime(nx)
if not is_fft_friendly:
if myid == main_node:
log_main.add(
"The target image size is not a product of small prime numbers"
)
log_main.add("Program adjusts the input settings!")
### two cases
if current_decimate == 1.0:
nx = sp_fundamentals.smallprime(nx)
ny = nx
current_window = nx # update
if myid == main_node:
log_main.add("The window size is updated to %d." %
current_window)
else:
if current_window == 0:
nx = sp_fundamentals.smallprime(
int(nx * current_decimate + 0.5))
current_decimate = old_div(float(nx), nnxo)
ny = nx
if myid == main_node:
log_main.add("The decimate rate is updated to %f." %
current_decimate)
else:
nx = sp_fundamentals.smallprime(
int(current_window * current_decimate + 0.5))
ny = nx
current_window = int(old_div(nx, current_decimate) + 0.5)
if myid == main_node:
log_main.add("The window size is updated to %d." %
current_window)
if myid == main_node:
log_main.add("The target image size is %d" % nx)
if radiuspca == -1:
radiuspca = old_div(nx, 2) - 2
if myid == main_node:
log_main.add("%-70s: %d\n" % ("Number of projection", nima))
img_begin, img_end = sp_applications.MPI_start_end(
nima, number_of_proc, myid)
"""Multiline Comment0"""
"""
Comments from adnan, replace index_of_proj to index_of_particle, index_of_proj was not defined
also varList is not defined not made an empty list there
"""
if options.VAR: # 2D variance images have no shifts
varList = []
# varList = EMData.read_images(stack, range(img_begin, img_end))
for index_of_particle in range(img_begin, img_end):
image = sp_utilities.get_im(stack, index_of_particle)
if current_window > 0:
varList.append(
sp_fundamentals.fdecimate(
sp_fundamentals.window2d(image, current_window,
current_window),
nx,
ny,
))
else:
varList.append(sp_fundamentals.fdecimate(image, nx, ny))
else:
if myid == main_node:
t1 = time.time()
proj_angles = []
aveList = []
tab = EMAN2_cppwrap.EMUtil.get_all_attributes(
stack, "xform.projection")
for i in range(nima):
t = tab[i].get_params("spider")
phi = t["phi"]
theta = t["theta"]
psi = t["psi"]
x = theta
if x > 90.0:
x = 180.0 - x
x = x * 10000 + psi
proj_angles.append([x, t["phi"], t["theta"], t["psi"], i])
t2 = time.time()
log_main.add(
"%-70s: %d\n" %
("Number of neighboring projections", img_per_grp))
log_main.add("...... Finding neighboring projections\n")
log_main.add("Number of images per group: %d" % img_per_grp)
log_main.add("Now grouping projections")
proj_angles.sort()
proj_angles_list = numpy.full((nima, 4),
0.0,
dtype=numpy.float32)
for i in range(nima):
proj_angles_list[i][0] = proj_angles[i][1]
proj_angles_list[i][1] = proj_angles[i][2]
proj_angles_list[i][2] = proj_angles[i][3]
proj_angles_list[i][3] = proj_angles[i][4]
else:
proj_angles_list = 0
proj_angles_list = sp_utilities.wrap_mpi_bcast(
proj_angles_list, main_node, mpi.MPI_COMM_WORLD)
proj_angles = []
for i in range(nima):
proj_angles.append([
proj_angles_list[i][0],
proj_angles_list[i][1],
proj_angles_list[i][2],
int(proj_angles_list[i][3]),
])
del proj_angles_list
proj_list, mirror_list = sp_utilities.nearest_proj(
proj_angles, img_per_grp, range(img_begin, img_end))
all_proj = []
for im in proj_list:
for jm in im:
all_proj.append(proj_angles[jm][3])
all_proj = list(set(all_proj))
index = {}
for i in range(len(all_proj)):
index[all_proj[i]] = i
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if myid == main_node:
log_main.add("%-70s: %.2f\n" %
("Finding neighboring projections lasted [s]",
time.time() - t2))
log_main.add("%-70s: %d\n" %
("Number of groups processed on the main node",
len(proj_list)))
log_main.add("Grouping projections took: %12.1f [m]" %
(old_div((time.time() - t2), 60.0)))
log_main.add("Number of groups on main node: ", len(proj_list))
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if myid == main_node:
log_main.add("...... Calculating the stack of 2D variances \n")
# Memory estimation. There are two memory consumption peaks
# peak 1. Compute ave, var;
# peak 2. Var volume reconstruction;
# proj_params = [0.0]*(nima*5)
aveList = []
varList = []
# if nvec > 0: eigList = [[] for i in range(nvec)]
dnumber = len(
all_proj) # all neighborhood set for assigned to myid
pnumber = len(proj_list) * 2.0 + img_per_grp # aveList and varList
tnumber = dnumber + pnumber
vol_size2 = old_div(nx**3 * 4.0 * 8, 1.0e9)
vol_size1 = old_div(2.0 * nnxo**3 * 4.0 * 8, 1.0e9)
proj_size = old_div(nnxo * nnyo * len(proj_list) * 4.0 * 2.0,
1.0e9) # both aveList and varList
orig_data_size = old_div(nnxo * nnyo * 4.0 * tnumber, 1.0e9)
reduced_data_size = old_div(nx * nx * 4.0 * tnumber, 1.0e9)
full_data = numpy.full((number_of_proc, 2),
-1.0,
dtype=numpy.float16)
full_data[myid] = orig_data_size, reduced_data_size
if myid != main_node:
sp_utilities.wrap_mpi_send(full_data, main_node,
mpi.MPI_COMM_WORLD)
if myid == main_node:
for iproc in range(number_of_proc):
if iproc != main_node:
dummy = sp_utilities.wrap_mpi_recv(
iproc, mpi.MPI_COMM_WORLD)
full_data[numpy.where(dummy > -1)] = dummy[numpy.where(
dummy > -1)]
del dummy
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
full_data = sp_utilities.wrap_mpi_bcast(full_data, main_node,
mpi.MPI_COMM_WORLD)
# find the CPU with heaviest load
minindx = numpy.argsort(full_data, 0)
heavy_load_myid = minindx[-1][1]
total_mem = sum(full_data)
if myid == main_node:
if current_window == 0:
log_main.add(
"Nx: current image size = %d. Decimated by %f from %d"
% (nx, current_decimate, nnxo))
else:
log_main.add(
"Nx: current image size = %d. Windowed to %d, and decimated by %f from %d"
% (nx, current_window, current_decimate, nnxo))
log_main.add("Nproj: number of particle images.")
log_main.add("Navg: number of 2D average images.")
log_main.add("Nvar: number of 2D variance images.")
log_main.add(
"Img_per_grp: user defined image per group for averaging = %d"
% img_per_grp)
log_main.add(
"Overhead: total python overhead memory consumption = %f"
% overhead_loading)
log_main.add(
"Total memory) = 4.0*nx^2*(nproj + navg +nvar+ img_per_grp)/1.0e9 + overhead: %12.3f [GB]"
% (total_mem[1] + overhead_loading))
del full_data
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if myid == heavy_load_myid:
log_main.add(
"Begin reading and preprocessing images on processor. Wait... "
)
ttt = time.time()
# imgdata = EMData.read_images(stack, all_proj)
imgdata = [None for im in range(len(all_proj))]
for index_of_proj in range(len(all_proj)):
# image = get_im(stack, all_proj[index_of_proj])
if current_window > 0:
imgdata[index_of_proj] = sp_fundamentals.fdecimate(
sp_fundamentals.window2d(
sp_utilities.get_im(stack,
all_proj[index_of_proj]),
current_window,
current_window,
),
nx,
ny,
)
else:
imgdata[index_of_proj] = sp_fundamentals.fdecimate(
sp_utilities.get_im(stack, all_proj[index_of_proj]),
nx, ny)
if current_decimate > 0.0 and options.CTF:
ctf = imgdata[index_of_proj].get_attr("ctf")
ctf.apix = old_div(ctf.apix, current_decimate)
imgdata[index_of_proj].set_attr("ctf", ctf)
if myid == heavy_load_myid and index_of_proj % 100 == 0:
log_main.add(
" ...... %6.2f%% " %
(old_div(index_of_proj, float(len(all_proj))) * 100.0))
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if myid == heavy_load_myid:
log_main.add("All_proj preprocessing cost %7.2f m" % (old_div(
(time.time() - ttt), 60.0)))
log_main.add("Wait untill reading on all CPUs done...")
"""Multiline Comment1"""
if not options.no_norm:
mask = sp_utilities.model_circle(old_div(nx, 2) - 2, nx, nx)
if myid == heavy_load_myid:
log_main.add("Start computing 2D aveList and varList. Wait...")
ttt = time.time()
inner = old_div(nx, 2) - 4
outer = inner + 2
xform_proj_for_2D = [None for i in range(len(proj_list))]
for i in range(len(proj_list)):
ki = proj_angles[proj_list[i][0]][3]
if ki >= symbaselen:
continue
mi = index[ki]
dpar = EMAN2_cppwrap.Util.get_transform_params(
imgdata[mi], "xform.projection", "spider")
phiM, thetaM, psiM, s2xM, s2yM = (
dpar["phi"],
dpar["theta"],
dpar["psi"],
-dpar["tx"] * current_decimate,
-dpar["ty"] * current_decimate,
)
grp_imgdata = []
for j in range(img_per_grp):
mj = index[proj_angles[proj_list[i][j]][3]]
cpar = EMAN2_cppwrap.Util.get_transform_params(
imgdata[mj], "xform.projection", "spider")
alpha, sx, sy, mirror = params_3D_2D_NEW(
cpar["phi"],
cpar["theta"],
cpar["psi"],
-cpar["tx"] * current_decimate,
-cpar["ty"] * current_decimate,
mirror_list[i][j],
)
if thetaM <= 90:
if mirror == 0:
alpha, sx, sy, scale = sp_utilities.compose_transform2(
alpha, sx, sy, 1.0, phiM - cpar["phi"], 0.0,
0.0, 1.0)
else:
alpha, sx, sy, scale = sp_utilities.compose_transform2(
alpha,
sx,
sy,
1.0,
180 - (phiM - cpar["phi"]),
0.0,
0.0,
1.0,
)
else:
if mirror == 0:
alpha, sx, sy, scale = sp_utilities.compose_transform2(
alpha, sx, sy, 1.0, -(phiM - cpar["phi"]), 0.0,
0.0, 1.0)
else:
alpha, sx, sy, scale = sp_utilities.compose_transform2(
alpha,
sx,
sy,
1.0,
-(180 - (phiM - cpar["phi"])),
0.0,
0.0,
1.0,
)
imgdata[mj].set_attr(
"xform.align2d",
EMAN2_cppwrap.Transform({
"type": "2D",
"alpha": alpha,
"tx": sx,
"ty": sy,
"mirror": mirror,
"scale": 1.0,
}),
)
grp_imgdata.append(imgdata[mj])
if not options.no_norm:
for k in range(img_per_grp):
ave, std, minn, maxx = EMAN2_cppwrap.Util.infomask(
grp_imgdata[k], mask, False)
grp_imgdata[k] -= ave
grp_imgdata[k] = old_div(grp_imgdata[k], std)
if options.fl > 0.0:
for k in range(img_per_grp):
grp_imgdata[k] = sp_filter.filt_tanl(
grp_imgdata[k], options.fl, options.aa)
# Because of background issues, only linear option works.
if options.CTF:
ave, var = sp_statistics.aves_wiener(
grp_imgdata, SNR=1.0e5, interpolation_method="linear")
else:
ave, var = sp_statistics.ave_var(grp_imgdata)
# Switch to std dev
# threshold is not really needed,it is just in case due to numerical accuracy something turns out negative.
var = sp_morphology.square_root(sp_morphology.threshold(var))
sp_utilities.set_params_proj(ave,
[phiM, thetaM, 0.0, 0.0, 0.0])
sp_utilities.set_params_proj(var,
[phiM, thetaM, 0.0, 0.0, 0.0])
aveList.append(ave)
varList.append(var)
xform_proj_for_2D[i] = [phiM, thetaM, 0.0, 0.0, 0.0]
"""Multiline Comment2"""
if (myid == heavy_load_myid) and (i % 100 == 0):
log_main.add(" ......%6.2f%% " %
(old_div(i, float(len(proj_list))) * 100.0))
del imgdata, grp_imgdata, cpar, dpar, all_proj, proj_angles, index
if not options.no_norm:
del mask
if myid == main_node:
del tab
# At this point, all averages and variances are computed
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if myid == heavy_load_myid:
log_main.add("Computing aveList and varList took %12.1f [m]" %
(old_div((time.time() - ttt), 60.0)))
xform_proj_for_2D = sp_utilities.wrap_mpi_gatherv(
xform_proj_for_2D, main_node, mpi.MPI_COMM_WORLD)
if myid == main_node:
sp_utilities.write_text_row(
[str(entry) for entry in xform_proj_for_2D],
optparse.os.path.join(current_output_dir, "params.txt"),
)
del xform_proj_for_2D
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if options.ave2D:
if myid == main_node:
log_main.add("Compute ave2D ... ")
km = 0
for i in range(number_of_proc):
if i == main_node:
for im in range(len(aveList)):
aveList[im].write_image(
optparse.os.path.join(
current_output_dir, options.ave2D),
km,
)
km += 1
else:
nl = mpi.mpi_recv(
1,
mpi.MPI_INT,
i,
sp_global_def.SPARX_MPI_TAG_UNIVERSAL,
mpi.MPI_COMM_WORLD,
)
nl = int(nl[0])
for im in range(nl):
ave = sp_utilities.recv_EMData(
i, im + i + 70000)
"""Multiline Comment3"""
tmpvol = sp_fundamentals.fpol(ave, nx, nx, 1)
tmpvol.write_image(
optparse.os.path.join(
current_output_dir, options.ave2D),
km,
)
km += 1
else:
mpi.mpi_send(
len(aveList),
1,
mpi.MPI_INT,
main_node,
sp_global_def.SPARX_MPI_TAG_UNIVERSAL,
mpi.MPI_COMM_WORLD,
)
for im in range(len(aveList)):
sp_utilities.send_EMData(aveList[im], main_node,
im + myid + 70000)
"""Multiline Comment4"""
if myid == main_node:
sp_applications.header(
optparse.os.path.join(current_output_dir,
options.ave2D),
params="xform.projection",
fimport=optparse.os.path.join(current_output_dir,
"params.txt"),
)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if options.ave3D:
t5 = time.time()
if myid == main_node:
log_main.add("Reconstruct ave3D ... ")
ave3D = sp_reconstruction.recons3d_4nn_MPI(
myid, aveList, symmetry=options.sym, npad=options.npad)
sp_utilities.bcast_EMData_to_all(ave3D, myid)
if myid == main_node:
if current_decimate != 1.0:
ave3D = sp_fundamentals.resample(
ave3D, old_div(1.0, current_decimate))
ave3D = sp_fundamentals.fpol(
ave3D, nnxo, nnxo,
nnxo) # always to the orignal image size
sp_utilities.set_pixel_size(ave3D, 1.0)
ave3D.write_image(
optparse.os.path.join(current_output_dir,
options.ave3D))
log_main.add("Ave3D reconstruction took %12.1f [m]" %
(old_div((time.time() - t5), 60.0)))
log_main.add("%-70s: %s\n" %
("The reconstructed ave3D is saved as ",
options.ave3D))
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
del ave, var, proj_list, stack, alpha, sx, sy, mirror, aveList
"""Multiline Comment5"""
if options.ave3D:
del ave3D
if options.var2D:
if myid == main_node:
log_main.add("Compute var2D...")
km = 0
for i in range(number_of_proc):
if i == main_node:
for im in range(len(varList)):
tmpvol = sp_fundamentals.fpol(
varList[im], nx, nx, 1)
tmpvol.write_image(
optparse.os.path.join(
current_output_dir, options.var2D),
km,
)
km += 1
else:
nl = mpi.mpi_recv(
1,
mpi.MPI_INT,
i,
sp_global_def.SPARX_MPI_TAG_UNIVERSAL,
mpi.MPI_COMM_WORLD,
)
nl = int(nl[0])
for im in range(nl):
ave = sp_utilities.recv_EMData(
i, im + i + 70000)
tmpvol = sp_fundamentals.fpol(ave, nx, nx, 1)
tmpvol.write_image(
optparse.os.path.join(
current_output_dir, options.var2D),
km,
)
km += 1
else:
mpi.mpi_send(
len(varList),
1,
mpi.MPI_INT,
main_node,
sp_global_def.SPARX_MPI_TAG_UNIVERSAL,
mpi.MPI_COMM_WORLD,
)
for im in range(len(varList)):
sp_utilities.send_EMData(
varList[im], main_node,
im + myid + 70000) # What with the attributes??
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if myid == main_node:
sp_applications.header(
optparse.os.path.join(current_output_dir,
options.var2D),
params="xform.projection",
fimport=optparse.os.path.join(current_output_dir,
"params.txt"),
)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if options.var3D:
if myid == main_node:
log_main.add("Reconstruct var3D ...")
t6 = time.time()
# radiusvar = options.radius
# if( radiusvar < 0 ): radiusvar = nx//2 -3
res = sp_reconstruction.recons3d_4nn_MPI(myid,
varList,
symmetry=options.sym,
npad=options.npad)
# res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
if myid == main_node:
if current_decimate != 1.0:
res = sp_fundamentals.resample(
res, old_div(1.0, current_decimate))
res = sp_fundamentals.fpol(res, nnxo, nnxo, nnxo)
sp_utilities.set_pixel_size(res, 1.0)
res.write_image(os.path.join(current_output_dir,
options.var3D))
log_main.add(
"%-70s: %s\n" %
("The reconstructed var3D is saved as ", options.var3D))
log_main.add("Var3D reconstruction took %f12.1 [m]" % (old_div(
(time.time() - t6), 60.0)))
log_main.add("Total computation time %f12.1 [m]" % (old_div(
(time.time() - t0), 60.0)))
log_main.add("sx3dvariability finishes")
if RUNNING_UNDER_MPI:
sp_global_def.MPI = False
sp_global_def.BATCH = False
def main():
arglist = []
for arg in sys.argv:
arglist.append(arg)
progname = optparse.os.path.basename(arglist[0])
usage = progname + """ inputvolume locresvolume maskfile outputfile --radius --falloff --MPI
Locally filer a volume based on local resolution volume (sxlocres.py) within area outlined by the maskfile
"""
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option(
"--radius",
type="int",
default=-1,
help=
"if there is no maskfile, sphere with r=radius will be used, by default the radius is nx/2-1"
)
parser.add_option("--falloff",
type="float",
default=0.1,
help="falloff of tanl filter (default 0.1)")
parser.add_option("--MPI",
action="store_true",
default=False,
help="use MPI version")
(options, args) = parser.parse_args(arglist[1:])
if len(args) < 3 or len(args) > 4:
sp_global_def.sxprint("See usage " + usage)
sp_global_def.ERROR(
"Wrong number of parameters. Please see usage information above.")
return
if sp_global_def.CACHE_DISABLE:
pass #IMPORTIMPORTIMPORT from sp_utilities import disable_bdb_cache
sp_utilities.disable_bdb_cache()
if options.MPI:
number_of_proc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
main_node = 0
if (myid == main_node):
#print sys.argv
vi = sp_utilities.get_im(sys.argv[1])
ui = sp_utilities.get_im(sys.argv[2])
#print Util.infomask(ui, None, True)
radius = options.radius
nx = vi.get_xsize()
ny = vi.get_ysize()
nz = vi.get_zsize()
dis = [nx, ny, nz]
else:
falloff = 0.0
radius = 0
dis = [0, 0, 0]
vi = None
ui = None
dis = sp_utilities.bcast_list_to_all(dis, myid, source_node=main_node)
if (myid != main_node):
nx = int(dis[0])
ny = int(dis[1])
nz = int(dis[2])
radius = sp_utilities.bcast_number_to_all(radius, main_node)
if len(args) == 3:
if (radius == -1):
radius = min(nx, ny, nz) // 2 - 1
m = sp_utilities.model_circle(radius, nx, ny, nz)
outvol = args[2]
elif len(args) == 4:
if (myid == main_node):
m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5)
else:
m = sp_utilities.model_blank(nx, ny, nz)
outvol = args[3]
sp_utilities.bcast_EMData_to_all(m, myid, main_node)
pass #IMPORTIMPORTIMPORT from sp_filter import filterlocal
filteredvol = sp_filter.filterlocal(ui, vi, m, options.falloff, myid,
main_node, number_of_proc)
if (myid == 0):
filteredvol.write_image(outvol)
else:
vi = sp_utilities.get_im(args[0])
ui = sp_utilities.get_im(
args[1]
) # resolution volume, values are assumed to be from 0 to 0.5
nn = vi.get_xsize()
falloff = options.falloff
if len(args) == 3:
radius = options.radius
if (radius == -1):
radius = nn // 2 - 1
m = sp_utilities.model_circle(radius, nn, nn, nn)
outvol = args[2]
elif len(args) == 4:
m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5)
outvol = args[3]
sp_fundamentals.fftip(vi) # this is the volume to be filtered
# Round all resolution numbers to two digits
for x in range(nn):
for y in range(nn):
for z in range(nn):
ui.set_value_at_fast(x, y, z,
round(ui.get_value_at(x, y, z), 2))
st = EMAN2_cppwrap.Util.infomask(ui, m, True)
filteredvol = sp_utilities.model_blank(nn, nn, nn)
cutoff = max(st[2] - 0.01, 0.0)
while (cutoff < st[3]):
cutoff = round(cutoff + 0.01, 2)
pt = EMAN2_cppwrap.Util.infomask(
sp_morphology.threshold_outside(ui, cutoff - 0.00501,
cutoff + 0.005), m, True)
if (pt[0] != 0.0):
vovo = sp_fundamentals.fft(
sp_filter.filt_tanl(vi, cutoff, falloff))
for x in range(nn):
for y in range(nn):
for z in range(nn):
if (m.get_value_at(x, y, z) > 0.5):
if (round(ui.get_value_at(x, y, z),
2) == cutoff):
filteredvol.set_value_at_fast(
x, y, z, vovo.get_value_at(x, y, z))
sp_global_def.write_command(optparse.os.path.dirname(outvol))
filteredvol.write_image(outvol)
def main():
# Parse the Options
progname = path.basename(argv[0])
usage = progname + """ summovie_path input_micrograph_pattern input_shift_pattern output_directory
--selection_list
--nr_frames=nr_frames
--first
--last
--pixel_size=pixel_size
--nr_threads
--apply_dose_filter
--voltage=voltage
--exposure_per_frame=exposure_per_frame
--pre_exposure=pre_exposure
--dont_restore_noise
sxsummovie exists only in non-MPI version.
Perform summovie without dose filtering.
sxsummovie.py ~/my_app/summovie 'outdir_unblur/corrsum/micrograph_*_frames_sum.mrc' 'outdir_unblur/shift/micrograph_*_frames_shift.txt'
outdir_summovie --nr_frames=24 --pixel_size=1.19 --nr_threads=1
Perform summovie without dose filtering and with less frames.
sxsummovie.py ~/my_app/summovie 'outdir_unblur/corrsum/micrograph_*_frames_sum.mrc' 'outdir_unblur/shift/micrograph_*_frames_shift.txt'
outdir_summovie --nr_frames=24 --first=3 --last=15 --pixel_size=1.19 --nr_threads=1
Perform summovie with dose filtering and with less frames.
sxsummovie.py ~/my_app/summovie 'outdir_unblur/corrsum/micrograph_*_frames_sum.mrc' 'outdir_unblur/shift/micrograph_*_frames_shift.txt'
outdir_summovie --nr_frames=24 --first=3 --last=15 --pixel_size=1.19 --nr_threads=1 --apply_dose_filter --voltage=300 --exposure_per_frame=2 --pre_exposure=0
"""
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option('--selection_list', type='str', default='', help='Micrograph selecting list (SPHIRE specific): Specify a name of micrograph selection list text file. The file extension must be \'.txt\'. If this is not provided, all files matched with the micrograph name pattern will be processed. (default none)')
parser.add_option('--nr_frames', type='int', default=3, help='Number of movie frames: The number of movie frames in each input micrograph. (default 3)')
parser.add_option('--first', type='int', default=1, help='First movie frame: First movie frame for summing. (default 1)')
parser.add_option('--last', type='int', default=-1, help='Last movie frame: Last movie frame for summing. (default -1)')
parser.add_option('--sum_suffix', type='str', default='_sum', help=SUPPRESS_HELP)
parser.add_option('--pixel_size', type='float', default=-1.0, help='Pixel size [A]: The pixel size of input micrographs. (default required float)')
parser.add_option('--nr_threads', type='int', default=1, help='Number of threads: The number of threads summovie can use. The higher the faster, but it requires larger memory. (default 1)')
parser.add_option('--apply_dose_filter', action='store_true', default=False, help='Apply dose filter step: Requires voltage, exposure per frame, and pre exposure options. (default False)')
parser.add_option('--voltage', type='float', default=300.0, help='Microscope voltage (dose filter) [kV]: The acceleration voltage of microscope used for imaging. (default 300.0)')
parser.add_option('--exposure_per_frame', type='float', default=2.0, help='Per frame exposure (dose filter) [e/A^2]: The electron dose per frame in e/A^2. (default 2.0)')
parser.add_option('--pre_exposure', type='float', default=0.0, help='Pre-exposure (dose filter) [e/A^2]: The electron does in e/A^2 used for exposure prior to imaging .(default 0.0)')
parser.add_option('--frc_suffix', type='string', default='_frc', help=SUPPRESS_HELP)
parser.add_option('--dont_restore_noise', action='store_true', default=False, help='Do not restore noise power: Do not restore noise power. (default False)')
parser.add_option('--summovie_ready', action='store_true', default=False, help=SUPPRESS_HELP)
# list of the options and the arguments
(options, args) = parser.parse_args(argv[1:])
sp_global_def.BATCH = True
# If there arent enough arguments, stop the script
if len(args) != 4:
sxprint( "Usage: " + usage )
ERROR( "Invalid number of parameters used. Please see usage information above." )
return
# Convert the realtive parts to absolute ones
summovie_path = path.realpath(args[0]) # summovie_path
input_image = path.realpath(args[1]) # input_micrograph_pattern
input_shift = path.realpath(args[2]) # input_shift_pattern
output_dir = path.realpath(args[3]) # output_directory
# If the summovie executable file does not exists, stop the script
if not path.exists(summovie_path):
ERROR( "Summovie directory does not exist, please change the name and restart the program." )
return
# If the output directory exists, stop the script
if path.exists(output_dir):
ERROR( "Output directory exists, please change the name and restart the program." )
return
# If the input file does not exists, stop the script
file_list = glob(input_image)
shift_list = glob(input_shift)
if not file_list:
ERROR( "Input micrograph file(s) does not exist, please change the name and restart the program." )
return
if not shift_list:
ERROR( "Input shift file(s) does not exist, please change the name and restart the program." )
return
# Output paths
if options.apply_dose_filter:
output_path = '{:s}/corrsum_dose_filtered'.format(output_dir)
else:
output_path = '{:s}/corrsum'.format(output_dir)
frc_path = '{:s}/frc'.format(output_dir)
log_path = '{:s}/logfiles'.format(output_dir)
temp_path = '{0}/temp'.format(output_dir)
# Split the path of the image name at the "/" Characters.
# The last entry contains the micrograph name.
# Split the micrograph name at the wildcard character for the
# prefix and suffix.
input_mic_split = input_image.split('/')
input_mic_name = input_mic_split[-1].split('*')
input_shift_name = input_shift.split('*')
if len(input_mic_name) != 2 or len(input_shift_name) != 2:
ERROR( "Too many wildcard arguments. Please use exactly one \'*\' in the pattern." )
return
# Get the input directory
if len(input_mic_split) != 1:
input_dir = input_image[:-len(input_mic_split[-1])]
else:
input_dir = ''
# Create output directorys
if not path.exists(output_dir):
os.makedirs(output_dir)
sp_global_def.write_command(output_dir)
if not path.exists(output_path):
os.makedirs(output_path)
sp_global_def.write_command(output_path)
if not path.exists(frc_path):
mkdir(frc_path)
if not path.exists(temp_path) and not options.summovie_ready:
mkdir(temp_path)
if not path.exists(log_path):
mkdir(log_path)
# shift wildcard list
shift_wildcard = [entry[len(input_shift_name[0]):-len(input_shift_name[-1])] \
for entry in shift_list]
# Just use shifts that have a micrograph and vise versa
mic_list = [
entry for entry in file_list \
if entry[len(input_dir) + len(input_mic_name[0]):-len(input_mic_name[-1])]\
in shift_wildcard]
# If micrograph list is provided just process the images in the list
selection_file = options.selection_list
if selection_file:
# Import list file
try:
selection = genfromtxt(selection_file, dtype=None)
except TypeError:
ERROR( "No entrys in micrograph list file \'"+selection_file+"\'" )
return
# List of files which are in pattern and list
mic_list = [
entry for entry in mic_list \
if entry[len(input_dir):] in selection and \
path.exists(entry)
]
# If no match is there abort
if len(mic_list) == 0:
ERROR( "No files in \'"+selection_file+"\' matched the micrograph file pattern" )
return
option_dict = {
'summovie_path': summovie_path,
'mic_list': mic_list,
'mic_prefix': input_mic_name[0],
'mic_suffix': input_mic_name[1],
'shift_prefix': input_shift_name[0],
'shift_suffix': input_shift_name[1],
'input_dir': input_dir,
'output_dir': output_dir,
'output_path': output_path,
'frc_path': frc_path,
'log_path': log_path,
'temp_path': temp_path,
'nr_frames': options.nr_frames,
'sum_suffix': options.sum_suffix,
'first': options.first,
'last':options.last,
'pixel_size': options.pixel_size,
'apply_dose_filter': options.apply_dose_filter,
'exposure_per_frame': options.exposure_per_frame,
'voltage': options.voltage,
'pre_exposure': options.pre_exposure,
'frc_suffix': options.frc_suffix,
'dont_restore_noise': options.dont_restore_noise,
'nr_threads': options.nr_threads,
'summovie_ready': options.summovie_ready,
'verbose': True
}
# Run summovie
run_summovie( opt=option_dict )
if not options.summovie_ready:
# Remove temp folder
for entry in glob('{0}/*'.format(temp_path)):
remove(entry)
rmdir(temp_path)
sxprint( "All Done!" )
sp_global_def.BATCH = False
multiplicity = 0
for i in range(i + 1, len(U)):
if U[i] == u:
multiplicity += 1
else:
break
for i in range(i - 1, 0, -1):
if U[i] == u:
multiplicity += 1
else:
break
return multiplicity
def gaussian(sigma, a, mu, x):
from math import sqrt, pi
return a * exp(-(x - mu)**2 /
(2.0 * sigma**2)) * 1.0 / (sigma * sqrt(2 * pi))
if __name__ == "__main__":
sp_global_def.print_timestamp("Start")
sp_global_def.write_command()
main()
sp_global_def.print_timestamp("Finish")
mpi.mpi_finalize()
def main():
def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
# the final ali2d parameters already combine shifts operation first and rotation operation second for parameters converted from 3D
if mirror:
m = 1
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
540.0 - psi, 0, 0, 1.0)
else:
m = 0
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
360.0 - psi, 0, 0, 1.0)
return alpha, sx, sy, m
progname = os.path.basename(sys.argv[0])
usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl= --aa= --sym=symmetry --CTF"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--output_dir",
type="string",
default="./",
help="Output directory")
parser.add_option("--ave2D",
type="string",
default=False,
help="Write to the disk a stack of 2D averages")
parser.add_option("--var2D",
type="string",
default=False,
help="Write to the disk a stack of 2D variances")
parser.add_option("--ave3D",
type="string",
default=False,
help="Write to the disk reconstructed 3D average")
parser.add_option("--var3D",
type="string",
default=False,
help="Compute 3D variability (time consuming!)")
parser.add_option(
"--img_per_grp",
type="int",
default=100,
help="Number of neighbouring projections.(Default is 100)")
parser.add_option(
"--no_norm",
action="store_true",
default=False,
help="Do not use normalization.(Default is to apply normalization)")
#parser.add_option("--radius", type="int" , default=-1 , help="radius for 3D variability" )
parser.add_option(
"--npad",
type="int",
default=2,
help=
"Number of time to pad the original images.(Default is 2 times padding)"
)
parser.add_option("--sym",
type="string",
default="c1",
help="Symmetry. (Default is no symmetry)")
parser.add_option(
"--fl",
type="float",
default=0.0,
help=
"Low pass filter cutoff in absolute frequency (0.0 - 0.5) and is applied to decimated images. (Default - no filtration)"
)
parser.add_option(
"--aa",
type="float",
default=0.02,
help=
"Fall off of the filter. Use default value if user has no clue about falloff (Default value is 0.02)"
)
parser.add_option("--CTF",
action="store_true",
default=False,
help="Use CFT correction.(Default is no CTF correction)")
#parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
#parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" )
#parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" )
#parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
#parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
parser.add_option(
"--VAR",
action="store_true",
default=False,
help="Stack of input consists of 2D variances (Default False)")
parser.add_option(
"--decimate",
type="float",
default=0.25,
help="Image decimate rate, a number less than 1. (Default is 0.25)")
parser.add_option(
"--window",
type="int",
default=0,
help=
"Target image size relative to original image size. (Default value is zero.)"
)
#parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)")
#parser.add_option("--nvec", type="int" , default=0 , help="Number of eigenvectors, (Default = 0 meaning no PCA calculated)")
parser.add_option(
"--symmetrize",
action="store_true",
default=False,
help="Prepare input stack for handling symmetry (Default False)")
parser.add_option("--overhead",
type="float",
default=0.5,
help="python overhead per CPU.")
(options, args) = parser.parse_args()
#####
from mpi import mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD
from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
#from mpi import *
from sp_applications import MPI_start_end
from sp_reconstruction import recons3d_em, recons3d_em_MPI
from sp_reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
from sp_utilities import print_begin_msg, print_end_msg, print_msg
from sp_utilities import read_text_row, get_image, get_im, wrap_mpi_send, wrap_mpi_recv
from sp_utilities import bcast_EMData_to_all, bcast_number_to_all
from sp_utilities import get_symt
# This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015
from EMAN2db import db_open_dict
# Set up global variables related to bdb cache
if sp_global_def.CACHE_DISABLE:
from sp_utilities import disable_bdb_cache
disable_bdb_cache()
# Set up global variables related to ERROR function
sp_global_def.BATCH = True
# detect if program is running under MPI
RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ
if RUNNING_UNDER_MPI: sp_global_def.MPI = True
if options.output_dir == "./":
current_output_dir = os.path.abspath(options.output_dir)
else:
current_output_dir = options.output_dir
if options.symmetrize:
if mpi.mpi_comm_size(MPI_COMM_WORLD) > 1:
ERROR("Cannot use more than one CPU for symmetry preparation")
if not os.path.exists(current_output_dir):
os.makedirs(current_output_dir)
sp_global_def.write_command(current_output_dir)
from sp_logger import Logger, BaseLogger_Files
if os.path.exists(os.path.join(current_output_dir, "log.txt")):
os.remove(os.path.join(current_output_dir, "log.txt"))
log_main = Logger(BaseLogger_Files())
log_main.prefix = os.path.join(current_output_dir, "./")
instack = args[0]
sym = options.sym.lower()
if (sym == "c1"):
ERROR("There is no need to symmetrize stack for C1 symmetry")
line = ""
for a in sys.argv:
line += " " + a
log_main.add(line)
if (instack[:4] != "bdb:"):
#if output_dir =="./": stack = "bdb:data"
stack = "bdb:" + current_output_dir + "/data"
delete_bdb(stack)
junk = cmdexecute("sp_cpy.py " + instack + " " + stack)
else:
stack = instack
qt = EMUtil.get_all_attributes(stack, 'xform.projection')
na = len(qt)
ts = get_symt(sym)
ks = len(ts)
angsa = [None] * na
for k in range(ks):
#Qfile = "Q%1d"%k
#if options.output_dir!="./": Qfile = os.path.join(options.output_dir,"Q%1d"%k)
Qfile = os.path.join(current_output_dir, "Q%1d" % k)
#delete_bdb("bdb:Q%1d"%k)
delete_bdb("bdb:" + Qfile)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
junk = cmdexecute("e2bdb.py " + stack + " --makevstack=bdb:" +
Qfile)
#DB = db_open_dict("bdb:Q%1d"%k)
DB = db_open_dict("bdb:" + Qfile)
for i in range(na):
ut = qt[i] * ts[k]
DB.set_attr(i, "xform.projection", ut)
#bt = ut.get_params("spider")
#angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
#write_text_row(angsa, 'ptsma%1d.txt'%k)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
#junk = cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k))
DB.close()
#if options.output_dir =="./": delete_bdb("bdb:sdata")
delete_bdb("bdb:" + current_output_dir + "/" + "sdata")
#junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
sdata = "bdb:" + current_output_dir + "/" + "sdata"
sxprint(sdata)
junk = cmdexecute("e2bdb.py " + current_output_dir +
" --makevstack=" + sdata + " --filt=Q")
#junk = cmdexecute("ls EMAN2DB/sdata*")
#a = get_im("bdb:sdata")
a = get_im(sdata)
a.set_attr("variabilitysymmetry", sym)
#a.write_image("bdb:sdata")
a.write_image(sdata)
else:
from sp_fundamentals import window2d
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
shared_comm = mpi_comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED,
0, MPI_INFO_NULL)
myid_on_node = mpi_comm_rank(shared_comm)
no_of_processes_per_group = mpi_comm_size(shared_comm)
masters_from_groups_vs_everything_else_comm = mpi_comm_split(
MPI_COMM_WORLD, main_node == myid_on_node, myid_on_node)
color, no_of_groups, balanced_processor_load_on_nodes = get_colors_and_subsets(main_node, MPI_COMM_WORLD, myid, \
shared_comm, myid_on_node, masters_from_groups_vs_everything_else_comm)
overhead_loading = options.overhead * number_of_proc
#memory_per_node = options.memory_per_node
#if memory_per_node == -1.: memory_per_node = 2.*no_of_processes_per_group
keepgoing = 1
current_window = options.window
current_decimate = options.decimate
if len(args) == 1: stack = args[0]
else:
sxprint("Usage: " + usage)
sxprint("Please run \'" + progname + " -h\' for detailed options")
ERROR(
"Invalid number of parameters used. Please see usage information above."
)
return
t0 = time()
# obsolete flags
options.MPI = True
#options.nvec = 0
options.radiuspca = -1
options.iter = 40
options.abs = 0.0
options.squ = 0.0
if options.fl > 0.0 and options.aa == 0.0:
ERROR("Fall off has to be given for the low-pass filter",
myid=myid)
#if options.VAR and options.SND:
# ERROR( "Only one of var and SND can be set!",myid=myid )
if options.VAR and (options.ave2D or options.ave3D or options.var2D):
ERROR(
"When VAR is set, the program cannot output ave2D, ave3D or var2D",
myid=myid)
#if options.SND and (options.ave2D or options.ave3D):
# ERROR( "When SND is set, the program cannot output ave2D or ave3D", myid=myid )
#if options.nvec > 0 :
# ERROR( "PCA option not implemented", myid=myid )
#if options.nvec > 0 and options.ave3D == None:
# ERROR( "When doing PCA analysis, one must set ave3D", myid=myid )
if current_decimate > 1.0 or current_decimate < 0.0:
ERROR("Decimate rate should be a value between 0.0 and 1.0",
myid=myid)
if current_window < 0.0:
ERROR("Target window size should be always larger than zero",
myid=myid)
if myid == main_node:
img = get_image(stack, 0)
nx = img.get_xsize()
ny = img.get_ysize()
if (min(nx, ny) < current_window): keepgoing = 0
keepgoing = bcast_number_to_all(keepgoing, main_node, MPI_COMM_WORLD)
if keepgoing == 0:
ERROR(
"The target window size cannot be larger than the size of decimated image",
myid=myid)
import string
options.sym = options.sym.lower()
# if global_def.CACHE_DISABLE:
# from utilities import disable_bdb_cache
# disable_bdb_cache()
# global_def.BATCH = True
if myid == main_node:
if not os.path.exists(current_output_dir):
os.makedirs(current_output_dir
) # Never delete output_dir in the program!
img_per_grp = options.img_per_grp
#nvec = options.nvec
radiuspca = options.radiuspca
from sp_logger import Logger, BaseLogger_Files
#if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt"))
log_main = Logger(BaseLogger_Files())
log_main.prefix = os.path.join(current_output_dir, "./")
if myid == main_node:
line = ""
for a in sys.argv:
line += " " + a
log_main.add(line)
log_main.add("-------->>>Settings given by all options<<<-------")
log_main.add("Symmetry : %s" % options.sym)
log_main.add("Input stack : %s" % stack)
log_main.add("Output_dir : %s" % current_output_dir)
if options.ave3D:
log_main.add("Ave3d : %s" % options.ave3D)
if options.var3D:
log_main.add("Var3d : %s" % options.var3D)
if options.ave2D:
log_main.add("Ave2D : %s" % options.ave2D)
if options.var2D:
log_main.add("Var2D : %s" % options.var2D)
if options.VAR: log_main.add("VAR : True")
else: log_main.add("VAR : False")
if options.CTF: log_main.add("CTF correction : True ")
else: log_main.add("CTF correction : False ")
log_main.add("Image per group : %5d" % options.img_per_grp)
log_main.add("Image decimate rate : %4.3f" % current_decimate)
log_main.add("Low pass filter : %4.3f" % options.fl)
current_fl = options.fl
if current_fl == 0.0: current_fl = 0.5
log_main.add(
"Current low pass filter is equivalent to cutoff frequency %4.3f for original image size"
% round((current_fl * current_decimate), 3))
log_main.add("Window size : %5d " % current_window)
log_main.add("sx3dvariability begins")
symbaselen = 0
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
nnxo = nx
nnyo = ny
if options.sym != "c1":
imgdata = get_im(stack)
try:
i = imgdata.get_attr("variabilitysymmetry").lower()
if (i != options.sym):
ERROR(
"The symmetry provided does not agree with the symmetry of the input stack",
myid=myid)
except:
ERROR(
"Input stack is not prepared for symmetry, please follow instructions",
myid=myid)
from sp_utilities import get_symt
i = len(get_symt(options.sym))
if ((nima / i) * i != nima):
ERROR(
"The length of the input stack is incorrect for symmetry processing",
myid=myid)
symbaselen = nima / i
else:
symbaselen = nima
else:
nima = 0
nx = 0
ny = 0
nnxo = 0
nnyo = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
nnxo = bcast_number_to_all(nnxo)
nnyo = bcast_number_to_all(nnyo)
if current_window > max(nx, ny):
ERROR("Window size is larger than the original image size")
if current_decimate == 1.:
if current_window != 0:
nx = current_window
ny = current_window
else:
if current_window == 0:
nx = int(nx * current_decimate + 0.5)
ny = int(ny * current_decimate + 0.5)
else:
nx = int(current_window * current_decimate + 0.5)
ny = nx
symbaselen = bcast_number_to_all(symbaselen)
# check FFT prime number
from sp_fundamentals import smallprime
is_fft_friendly = (nx == smallprime(nx))
if not is_fft_friendly:
if myid == main_node:
log_main.add(
"The target image size is not a product of small prime numbers"
)
log_main.add("Program adjusts the input settings!")
### two cases
if current_decimate == 1.:
nx = smallprime(nx)
ny = nx
current_window = nx # update
if myid == main_node:
log_main.add("The window size is updated to %d." %
current_window)
else:
if current_window == 0:
nx = smallprime(int(nx * current_decimate + 0.5))
current_decimate = float(nx) / nnxo
ny = nx
if (myid == main_node):
log_main.add("The decimate rate is updated to %f." %
current_decimate)
else:
nx = smallprime(
int(current_window * current_decimate + 0.5))
ny = nx
current_window = int(nx / current_decimate + 0.5)
if (myid == main_node):
log_main.add("The window size is updated to %d." %
current_window)
if myid == main_node:
log_main.add("The target image size is %d" % nx)
if radiuspca == -1: radiuspca = nx / 2 - 2
if myid == main_node:
log_main.add("%-70s: %d\n" % ("Number of projection", nima))
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
"""
if options.SND:
from sp_projection import prep_vol, prgs
from sp_statistics import im_diff
from sp_utilities import get_im, model_circle, get_params_proj, set_params_proj
from sp_utilities import get_ctf, generate_ctf
from sp_filter import filt_ctf
imgdata = EMData.read_images(stack, range(img_begin, img_end))
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
bcast_EMData_to_all(vol, myid)
volft, kb = prep_vol(vol)
mask = model_circle(nx/2-2, nx, ny)
varList = []
for i in xrange(img_begin, img_end):
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
if options.CTF:
ctf_params = get_ctf(imgdata[i-img_begin])
ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
diff2 = diff*diff
set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
varList.append(diff2)
mpi_barrier(MPI_COMM_WORLD)
"""
if options.VAR: # 2D variance images have no shifts
#varList = EMData.read_images(stack, range(img_begin, img_end))
from EMAN2 import Region
for index_of_particle in range(img_begin, img_end):
image = get_im(stack, index_of_proj)
if current_window > 0:
varList.append(
fdecimate(
window2d(image, current_window, current_window),
nx, ny))
else:
varList.append(fdecimate(image, nx, ny))
else:
from sp_utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
from sp_utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
from sp_utilities import model_blank, nearest_proj, model_circle, write_text_row, wrap_mpi_gatherv
from sp_applications import pca
from sp_statistics import avgvar, avgvar_ctf, ccc
from sp_filter import filt_tanl
from sp_morphology import threshold, square_root
from sp_projection import project, prep_vol, prgs
from sets import Set
from sp_utilities import wrap_mpi_recv, wrap_mpi_bcast, wrap_mpi_send
import numpy as np
if myid == main_node:
t1 = time()
proj_angles = []
aveList = []
tab = EMUtil.get_all_attributes(stack, 'xform.projection')
for i in range(nima):
t = tab[i].get_params('spider')
phi = t['phi']
theta = t['theta']
psi = t['psi']
x = theta
if x > 90.0: x = 180.0 - x
x = x * 10000 + psi
proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
t2 = time()
log_main.add(
"%-70s: %d\n" %
("Number of neighboring projections", img_per_grp))
log_main.add("...... Finding neighboring projections\n")
log_main.add("Number of images per group: %d" % img_per_grp)
log_main.add("Now grouping projections")
proj_angles.sort()
proj_angles_list = np.full((nima, 4), 0.0, dtype=np.float32)
for i in range(nima):
proj_angles_list[i][0] = proj_angles[i][1]
proj_angles_list[i][1] = proj_angles[i][2]
proj_angles_list[i][2] = proj_angles[i][3]
proj_angles_list[i][3] = proj_angles[i][4]
else:
proj_angles_list = 0
proj_angles_list = wrap_mpi_bcast(proj_angles_list, main_node,
MPI_COMM_WORLD)
proj_angles = []
for i in range(nima):
proj_angles.append([
proj_angles_list[i][0], proj_angles_list[i][1],
proj_angles_list[i][2],
int(proj_angles_list[i][3])
])
del proj_angles_list
proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp,
range(img_begin, img_end))
all_proj = Set()
for im in proj_list:
for jm in im:
all_proj.add(proj_angles[jm][3])
all_proj = list(all_proj)
index = {}
for i in range(len(all_proj)):
index[all_proj[i]] = i
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
log_main.add("%-70s: %.2f\n" %
("Finding neighboring projections lasted [s]",
time() - t2))
log_main.add("%-70s: %d\n" %
("Number of groups processed on the main node",
len(proj_list)))
log_main.add("Grouping projections took: %12.1f [m]" %
((time() - t2) / 60.))
log_main.add("Number of groups on main node: ", len(proj_list))
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
log_main.add("...... Calculating the stack of 2D variances \n")
# Memory estimation. There are two memory consumption peaks
# peak 1. Compute ave, var;
# peak 2. Var volume reconstruction;
# proj_params = [0.0]*(nima*5)
aveList = []
varList = []
#if nvec > 0: eigList = [[] for i in range(nvec)]
dnumber = len(
all_proj) # all neighborhood set for assigned to myid
pnumber = len(proj_list) * 2. + img_per_grp # aveList and varList
tnumber = dnumber + pnumber
vol_size2 = nx**3 * 4. * 8 / 1.e9
vol_size1 = 2. * nnxo**3 * 4. * 8 / 1.e9
proj_size = nnxo * nnyo * len(
proj_list) * 4. * 2. / 1.e9 # both aveList and varList
orig_data_size = nnxo * nnyo * 4. * tnumber / 1.e9
reduced_data_size = nx * nx * 4. * tnumber / 1.e9
full_data = np.full((number_of_proc, 2), -1., dtype=np.float16)
full_data[myid] = orig_data_size, reduced_data_size
if myid != main_node:
wrap_mpi_send(full_data, main_node, MPI_COMM_WORLD)
if myid == main_node:
for iproc in range(number_of_proc):
if iproc != main_node:
dummy = wrap_mpi_recv(iproc, MPI_COMM_WORLD)
full_data[np.where(dummy > -1)] = dummy[np.where(
dummy > -1)]
del dummy
mpi_barrier(MPI_COMM_WORLD)
full_data = wrap_mpi_bcast(full_data, main_node, MPI_COMM_WORLD)
# find the CPU with heaviest load
minindx = np.argsort(full_data, 0)
heavy_load_myid = minindx[-1][1]
total_mem = sum(full_data)
if myid == main_node:
if current_window == 0:
log_main.add(
"Nx: current image size = %d. Decimated by %f from %d"
% (nx, current_decimate, nnxo))
else:
log_main.add(
"Nx: current image size = %d. Windowed to %d, and decimated by %f from %d"
% (nx, current_window, current_decimate, nnxo))
log_main.add("Nproj: number of particle images.")
log_main.add("Navg: number of 2D average images.")
log_main.add("Nvar: number of 2D variance images.")
log_main.add(
"Img_per_grp: user defined image per group for averaging = %d"
% img_per_grp)
log_main.add(
"Overhead: total python overhead memory consumption = %f"
% overhead_loading)
log_main.add("Total memory) = 4.0*nx^2*(nproj + navg +nvar+ img_per_grp)/1.0e9 + overhead: %12.3f [GB]"%\
(total_mem[1] + overhead_loading))
del full_data
mpi_barrier(MPI_COMM_WORLD)
if myid == heavy_load_myid:
log_main.add(
"Begin reading and preprocessing images on processor. Wait... "
)
ttt = time()
#imgdata = EMData.read_images(stack, all_proj)
imgdata = [None for im in range(len(all_proj))]
for index_of_proj in range(len(all_proj)):
#image = get_im(stack, all_proj[index_of_proj])
if (current_window > 0):
imgdata[index_of_proj] = fdecimate(
window2d(get_im(stack, all_proj[index_of_proj]),
current_window, current_window), nx, ny)
else:
imgdata[index_of_proj] = fdecimate(
get_im(stack, all_proj[index_of_proj]), nx, ny)
if (current_decimate > 0.0 and options.CTF):
ctf = imgdata[index_of_proj].get_attr("ctf")
ctf.apix = ctf.apix / current_decimate
imgdata[index_of_proj].set_attr("ctf", ctf)
if myid == heavy_load_myid and index_of_proj % 100 == 0:
log_main.add(" ...... %6.2f%% " %
(index_of_proj / float(len(all_proj)) * 100.))
mpi_barrier(MPI_COMM_WORLD)
if myid == heavy_load_myid:
log_main.add("All_proj preprocessing cost %7.2f m" %
((time() - ttt) / 60.))
log_main.add("Wait untill reading on all CPUs done...")
'''
imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
if options.fl > 0.0:
for k in xrange(len(imgdata2)):
imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
if myid == main_node:
vol.write_image("vol_ctf.hdf")
print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf"))
del vol, imgdata2
mpi_barrier(MPI_COMM_WORLD)
'''
from sp_applications import prepare_2d_forPCA
from sp_utilities import model_blank
from EMAN2 import Transform
if not options.no_norm:
mask = model_circle(nx / 2 - 2, nx, nx)
if options.CTF:
from sp_utilities import pad
from sp_filter import filt_ctf
from sp_filter import filt_tanl
if myid == heavy_load_myid:
log_main.add("Start computing 2D aveList and varList. Wait...")
ttt = time()
inner = nx // 2 - 4
outer = inner + 2
xform_proj_for_2D = [None for i in range(len(proj_list))]
for i in range(len(proj_list)):
ki = proj_angles[proj_list[i][0]][3]
if ki >= symbaselen: continue
mi = index[ki]
dpar = Util.get_transform_params(imgdata[mi],
"xform.projection", "spider")
phiM, thetaM, psiM, s2xM, s2yM = dpar["phi"], dpar[
"theta"], dpar[
"psi"], -dpar["tx"] * current_decimate, -dpar[
"ty"] * current_decimate
grp_imgdata = []
for j in range(img_per_grp):
mj = index[proj_angles[proj_list[i][j]][3]]
cpar = Util.get_transform_params(imgdata[mj],
"xform.projection",
"spider")
alpha, sx, sy, mirror = params_3D_2D_NEW(
cpar["phi"], cpar["theta"], cpar["psi"],
-cpar["tx"] * current_decimate,
-cpar["ty"] * current_decimate, mirror_list[i][j])
if thetaM <= 90:
if mirror == 0:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, phiM - cpar["phi"], 0.0,
0.0, 1.0)
else:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, 180 - (phiM - cpar["phi"]),
0.0, 0.0, 1.0)
else:
if mirror == 0:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, -(phiM - cpar["phi"]), 0.0,
0.0, 1.0)
else:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0,
-(180 - (phiM - cpar["phi"])), 0.0, 0.0, 1.0)
imgdata[mj].set_attr(
"xform.align2d",
Transform({
"type": "2D",
"alpha": alpha,
"tx": sx,
"ty": sy,
"mirror": mirror,
"scale": 1.0
}))
grp_imgdata.append(imgdata[mj])
if not options.no_norm:
for k in range(img_per_grp):
ave, std, minn, maxx = Util.infomask(
grp_imgdata[k], mask, False)
grp_imgdata[k] -= ave
grp_imgdata[k] /= std
if options.fl > 0.0:
for k in range(img_per_grp):
grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl,
options.aa)
# Because of background issues, only linear option works.
if options.CTF:
ave, var = aves_wiener(grp_imgdata,
SNR=1.0e5,
interpolation_method="linear")
else:
ave, var = ave_var(grp_imgdata)
# Switch to std dev
# threshold is not really needed,it is just in case due to numerical accuracy something turns out negative.
var = square_root(threshold(var))
set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])
aveList.append(ave)
varList.append(var)
xform_proj_for_2D[i] = [phiM, thetaM, 0.0, 0.0, 0.0]
'''
if nvec > 0:
eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True)
for k in range(nvec):
set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
eigList[k].append(eig[k])
"""
if myid == 0 and i == 0:
for k in xrange(nvec):
eig[k].write_image("eig.hdf", k)
"""
'''
if (myid == heavy_load_myid) and (i % 100 == 0):
log_main.add(" ......%6.2f%% " %
(i / float(len(proj_list)) * 100.))
del imgdata, grp_imgdata, cpar, dpar, all_proj, proj_angles, index
if not options.no_norm: del mask
if myid == main_node: del tab
# At this point, all averages and variances are computed
mpi_barrier(MPI_COMM_WORLD)
if (myid == heavy_load_myid):
log_main.add("Computing aveList and varList took %12.1f [m]" %
((time() - ttt) / 60.))
xform_proj_for_2D = wrap_mpi_gatherv(xform_proj_for_2D, main_node,
MPI_COMM_WORLD)
if (myid == main_node):
write_text_row([str(entry) for entry in xform_proj_for_2D],
os.path.join(current_output_dir, "params.txt"))
del xform_proj_for_2D
mpi_barrier(MPI_COMM_WORLD)
if options.ave2D:
from sp_fundamentals import fpol
from sp_applications import header
if myid == main_node:
log_main.add("Compute ave2D ... ")
km = 0
for i in range(number_of_proc):
if i == main_node:
for im in range(len(aveList)):
aveList[im].write_image(
os.path.join(current_output_dir,
options.ave2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i,
SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in range(nl):
ave = recv_EMData(i, im + i + 70000)
"""
nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('pix_err', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
"""
tmpvol = fpol(ave, nx, nx, 1)
tmpvol.write_image(
os.path.join(current_output_dir,
options.ave2D), km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in range(len(aveList)):
send_EMData(aveList[im], main_node, im + myid + 70000)
"""
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
members = aveList[im].get_attr('pix_err')
mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
except:
mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
"""
if myid == main_node:
header(os.path.join(current_output_dir, options.ave2D),
params='xform.projection',
fimport=os.path.join(current_output_dir,
"params.txt"))
mpi_barrier(MPI_COMM_WORLD)
if options.ave3D:
from sp_fundamentals import fpol
t5 = time()
if myid == main_node: log_main.add("Reconstruct ave3D ... ")
ave3D = recons3d_4nn_MPI(myid,
aveList,
symmetry=options.sym,
npad=options.npad)
bcast_EMData_to_all(ave3D, myid)
if myid == main_node:
if current_decimate != 1.0:
ave3D = resample(ave3D, 1. / current_decimate)
ave3D = fpol(ave3D, nnxo, nnxo,
nnxo) # always to the orignal image size
set_pixel_size(ave3D, 1.0)
ave3D.write_image(
os.path.join(current_output_dir, options.ave3D))
log_main.add("Ave3D reconstruction took %12.1f [m]" %
((time() - t5) / 60.0))
log_main.add("%-70s: %s\n" %
("The reconstructed ave3D is saved as ",
options.ave3D))
mpi_barrier(MPI_COMM_WORLD)
del ave, var, proj_list, stack, alpha, sx, sy, mirror, aveList
'''
if nvec > 0:
for k in range(nvec):
if myid == main_node:log_main.add("Reconstruction eigenvolumes", k)
cont = True
ITER = 0
mask2d = model_circle(radiuspca, nx, nx)
while cont:
#print "On node %d, iteration %d"%(myid, ITER)
eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad)
bcast_EMData_to_all(eig3D, myid, main_node)
if options.fl > 0.0:
eig3D = filt_tanl(eig3D, options.fl, options.aa)
if myid == main_node:
eig3D.write_image(os.path.join(options.outpout_dir, "eig3d_%03d.hdf"%(k, ITER)))
Util.mul_img( eig3D, model_circle(radiuspca, nx, nx, nx) )
eig3Df, kb = prep_vol(eig3D)
del eig3D
cont = False
icont = 0
for l in range(len(eigList[k])):
phi, theta, psi, s2x, s2y = get_params_proj(eigList[k][l])
proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y])
cl = ccc(proj, eigList[k][l], mask2d)
if cl < 0.0:
icont += 1
cont = True
eigList[k][l] *= -1.0
u = int(cont)
u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD)
icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
if myid == main_node:
u = int(u[0])
log_main.add(" Eigenvector: ",k," number changed ",int(icont[0]))
else: u = 0
u = bcast_number_to_all(u, main_node)
cont = bool(u)
ITER += 1
del eig3Df, kb
mpi_barrier(MPI_COMM_WORLD)
del eigList, mask2d
'''
if options.ave3D: del ave3D
if options.var2D:
from sp_fundamentals import fpol
from sp_applications import header
if myid == main_node:
log_main.add("Compute var2D...")
km = 0
for i in range(number_of_proc):
if i == main_node:
for im in range(len(varList)):
tmpvol = fpol(varList[im], nx, nx, 1)
tmpvol.write_image(
os.path.join(current_output_dir,
options.var2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i,
SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in range(nl):
ave = recv_EMData(i, im + i + 70000)
tmpvol = fpol(ave, nx, nx, 1)
tmpvol.write_image(
os.path.join(current_output_dir,
options.var2D), km)
km += 1
else:
mpi_send(len(varList), 1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in range(len(varList)):
send_EMData(varList[im], main_node, im + myid +
70000) # What with the attributes??
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
from sp_applications import header
header(os.path.join(current_output_dir, options.var2D),
params='xform.projection',
fimport=os.path.join(current_output_dir,
"params.txt"))
mpi_barrier(MPI_COMM_WORLD)
if options.var3D:
if myid == main_node: log_main.add("Reconstruct var3D ...")
t6 = time()
# radiusvar = options.radius
# if( radiusvar < 0 ): radiusvar = nx//2 -3
res = recons3d_4nn_MPI(myid,
varList,
symmetry=options.sym,
npad=options.npad)
#res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
if myid == main_node:
from sp_fundamentals import fpol
if current_decimate != 1.0:
res = resample(res, 1. / current_decimate)
res = fpol(res, nnxo, nnxo, nnxo)
set_pixel_size(res, 1.0)
res.write_image(os.path.join(current_output_dir,
options.var3D))
log_main.add(
"%-70s: %s\n" %
("The reconstructed var3D is saved as ", options.var3D))
log_main.add("Var3D reconstruction took %f12.1 [m]" %
((time() - t6) / 60.0))
log_main.add("Total computation time %f12.1 [m]" %
((time() - t0) / 60.0))
log_main.add("sx3dvariability finishes")
if RUNNING_UNDER_MPI:
sp_global_def.MPI = False
sp_global_def.BATCH = False
