def shrink_or_enlarge_or_stay_the_same(images, shrink_ratio, target_nx,
target_radius, newx, nima):
from fundamentals import resample
from utilities import pad
if newx >= target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
else:
msk = model_circle(newx // 2 - 2, newx, newx)
do_not_stay_the_same_size = shrink_ratio != 1.0
cut_to_window = newx > target_nx
need_padding = newx < target_nx
for im in xrange(nima):
if do_not_stay_the_same_size:
images[im] = resample(images[im], shrink_ratio)
if cut_to_window:
images[im] = Util.window(images[im], target_nx, target_nx, 1)
p = Util.infomask(images[im], msk, False)
images[im] -= p[0]
p = Util.infomask(images[im], msk, True)
images[im] /= p[1]
if need_padding:
images[im] = pad(images[im], target_nx, target_nx, 1, 0.0)
def generate_helimic(refvol, outdir, pixel, CTF=False, Cs=2.0,voltage = 200.0, ampcont = 10.0, nonoise = False, rand_seed=14567):
from utilities import model_blank, model_gauss, model_gauss_noise, pad, get_im
from random import random
from projection import prgs, prep_vol
from filter import filt_gaussl, filt_ctf
from EMAN2 import EMAN2Ctf
if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "sxhelical_demo", 1)
os.mkdir(outdir)
seed(rand_seed)
Util.set_randnum_seed(rand_seed)
angles =[]
for i in xrange(3):
angles.append( [0.0+60.0*i, 90.0-i*5, 0.0, 0.0, 0.0] )
nangle = len(angles)
volfts = get_im(refvol)
nx = volfts.get_xsize()
ny = volfts.get_ysize()
nz = volfts.get_zsize()
volfts, kbx, kby, kbz = prep_vol( volfts )
iprj = 0
width = 500
xstart = 0
ystart = 0
for idef in xrange(3,6):
mic = model_blank(2048, 2048)
#defocus = idef*0.2
defocus = idef*0.6 ##@ming
if CTF :
#ctf = EMAN2Ctf()
#ctf.from_dict( {"defocus":defocus, "cs":Cs, "voltage":voltage, "apix":pixel, "ampcont":ampcont, "bfactor":0.0} )
from utilities import generate_ctf
ctf = generate_ctf([defocus,2,200,1.84,0.0,ampcont,defocus*0.2,80]) ##@ming the range of astigmatism amplitude is between 10 percent and 22 percent. 20 percent is a good choice.
i = idef - 4
for k in xrange(1):
psi = 90 + 10*i
proj = prgs(volfts, kbz, [angles[idef-3][0], angles[idef-3][1], psi, 0.0, 0.0], kbx, kby)
proj = Util.window(proj, 320, nz)
mic += pad(proj, 2048, 2048, 1, 0.0, 750*i, 20*i, 0)
if not nonoise: mic += model_gauss_noise(30.0,2048,2048)
if CTF :
#apply CTF
mic = filt_ctf(mic, ctf)
if not nonoise: mic += filt_gaussl(model_gauss_noise(17.5,2048,2048), 0.3)
mic.write_image("%s/mic%1d.hdf"%(outdir, idef-3),0)
def test_one_wave(self):
test_data = \
"""
oo
FirmID VendorName ACSI_14 ACSI_15 ACSI_16 ACSI_14_BENCH ACSI_15_BENCH ACSI_16_BENCH
0 1 Apple [Cellular Telephones] 79 80.0 81.0 78 78.0 79.0
1 1 Apple [Personal Computers] 84 84.0 NaN 78 77.0 NaN
2 2 Google [Internet Search Engines and Information] 83 78.0 NaN 80 76.0 NaN
3 2 Google+ [Internet Social Media] 84 NaN NaN 71 NaN NaN
4 2 YouTube (Google) [Internet Social Media] 78 76.0 77.0 71 74.0 73.0
5 3 Bing (Microsoft) [Internet Search Engines and ... 73 72.0 75.0 80 76.0 77.0
6 3 Microsoft [Computer Software] 75 75.0 80.0 76 74.0 81.0
t
FirmID VendorName ACSI_14 ACSI_15 ACSI_16 ACSI_16_PADDED ACSI_14_BENCH ACSI_15_BENCH ACSI_16_BENCH ACSI_16_BENCH_PADDED
0 1 Apple [Cellular Telephones] 79 80.0 81.0 81.0 78 78.0 79.0 79.0
1 1 Apple [Personal Computers] 84 84.0 NaN 84.0 78 77.0 NaN 77.0
2 2 Google [Internet Search Engines and Information] 83 78.0 NaN 78.0 80 76.0 NaN 76.0
3 2 Google+ [Internet Social Media] 84 NaN NaN NaN 71 NaN NaN NaN
4 2 YouTube (Google) [Internet Social Media] 78 76.0 77.0 77.0 71 74.0 73.0 73.0
5 3 Bing (Microsoft) [Internet Search Engines and ... 73 72.0 75.0 75.0 80 76.0 77.0 77.0
6 3 Microsoft [Computer Software] 75 75.0 80.0 80.0 76 74.0 81.0 81.0
"""
ofn = 'test_pad_original_from_ACSI_clean_and_augmented.csv'
vendor = 'test_vendor'
self.orig = pd.read_csv(
os.path.join(os.path.abspath(path), vendor, ofn))
tfn = 'test_pad_target_ACSI_clean_and_augmented_1_wave.csv'
self.target = pd.read_csv(
os.path.join(os.path.abspath(path), vendor, tfn))
t = self.target
o = self.orig
result = pad(self.orig, waves_to_update=1, vendor='ACSI', wave=2016)
r = result
for col, dtype in zip(result.columns, result.dtypes):
if not dtype == object:
diff = result[col].round(5) - self.target[col].round(5)
self.assertFalse(diff.any())
a = 1
def shrink_or_enlarge_or_stay_the_same(images, shrink_ratio, target_nx, target_radius, newx, nima): from fundamentals import resample from utilities import pad if newx >= target_nx : msk = model_circle(target_radius, target_nx, target_nx) else: msk = model_circle(newx//2-2, newx, newx) do_not_stay_the_same_size = shrink_ratio != 1.0 cut_to_window = newx > target_nx need_padding = newx < target_nx for im in xrange(nima): if do_not_stay_the_same_size : images[im] = resample(images[im], shrink_ratio) if cut_to_window : images[im] = Util.window(images[im], target_nx, target_nx, 1) p = Util.infomask(images[im], msk, False) images[im] -= p[0] p = Util.infomask(images[im], msk, True) images[im] /= p[1] if need_padding : images[im] = pad(images[im], target_nx, target_nx, 1, 0.0)
def main(args):
progname = os.path.basename(sys.argv[0])
usage = ( progname + " stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter" +
" --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp" +
" --generation=generation --rand_seed=rand_seed>" )
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--radius", type="int", help="particle radius: there is no default, a sensible number has to be provided, units - pixels (default required int)")
parser.add_option("--target_radius", type="int", default=29, help="target particle radius: actual particle radius on which isac will process data. Images will be shrinked/enlarged to achieve this radius (default 29)")
parser.add_option("--target_nx", type="int", default=76, help="target particle image size: actual image size on which isac will process data. Images will be shrinked/enlarged according to target particle radius and then cut/padded to achieve target_nx size. When xr > 0, the final image size for isac processing is 'target_nx + xr - 1' (default 76)")
parser.add_option("--img_per_grp", type="int", default=100, help="number of images per class: in the ideal case (essentially maximum size of class) (default 100)")
parser.add_option("--CTF", action="store_true", default=False, help="apply phase-flip for CTF correction: if set the data will be phase-flipped using CTF information included in image headers (default False)")
parser.add_option("--ir", type="int", default=1, help="inner ring: of the resampling to polar coordinates. units - pixels (default 1)")
parser.add_option("--rs", type="int", default=1, help="ring step: of the resampling to polar coordinates. units - pixels (default 1)")
parser.add_option("--xr", type="int", default=1, help="x range: of translational search. By default, set by the program. (default 1)")
parser.add_option("--yr", type="int", default=-1, help="y range: of translational search. By default, same as xr. (default -1)")
parser.add_option("--ts", type="float", default=1.0, help="search step: of translational search: units - pixels (default 1.0)")
parser.add_option("--maxit", type="int", default=30, help="number of iterations for reference-free alignment: (default 30)")
#parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)")
parser.add_option("--center_method", type="int", default=-1, help="method for centering: of global 2D average during initial prealignment of data (0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7) (default -1)")
parser.add_option("--dst", type="float", default=90.0, help="discrete angle used in within group alignment: (default 90.0)")
parser.add_option("--FL", type="float", default=0.2, help="lowest stopband: frequency used in the tangent filter (default 0.2)")
parser.add_option("--FH", type="float", default=0.3, help="highest stopband: frequency used in the tangent filter (default 0.3)")
parser.add_option("--FF", type="float", default=0.2, help="fall-off of the tangent filter: (default 0.2)")
parser.add_option("--init_iter", type="int", default=3, help="SAC initialization iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC initialization (default 3)")
parser.add_option("--main_iter", type="int", default=3, help="SAC main iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC (default 3)")
parser.add_option("--iter_reali", type="int", default=1, help="SAC stability check interval: every iter_reali iterations of SAC stability checking is performed (default 1)")
parser.add_option("--match_first", type="int", default=1, help="number of iterations to run 2-way matching in the first phase: (default 1)")
parser.add_option("--max_round", type="int", default=20, help="maximum rounds: of generating candidate class averages in the first phase (default 20)")
parser.add_option("--match_second", type="int", default=5, help="number of iterations to run 2-way (or 3-way) matching in the second phase: (default 5)")
parser.add_option("--stab_ali", type="int", default=5, help="number of alignments when checking stability: (default 5)")
parser.add_option("--thld_err", type="float", default=0.7, help="threshold of pixel error when checking stability: equals root mean square of distances between corresponding pixels from set of found transformations and theirs average transformation, depends linearly on square of radius (parameter ou). units - pixels. (default 0.7)")
parser.add_option("--indep_run", type="int", default=4, help="level of m-way matching for reproducibility tests: By default, perform full ISAC to 4-way matching. Value indep_run=2 will restrict ISAC to 2-way matching and 3 to 3-way matching. Note the number of used MPI processes requested in mpirun must be a multiplicity of indep_run. (default 4)")
parser.add_option("--thld_grp", type="int", default=10, help="minimum size of reproducible class (default 10)")
parser.add_option("--n_generations", type="int", default=100, help="maximum number of generations: program stops when reaching this total number of generations: (default 100)")
#parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)")
parser.add_option("--rand_seed", type="int", help="random seed set before calculations: useful for testing purposes. By default, total randomness (type int)")
parser.add_option("--new", action="store_true", default=False, help="use new code: (default False)")
parser.add_option("--debug", action="store_true", default=False, help="debug info printout: (default False)")
# must be switched off in production
parser.add_option("--use_latest_master_directory",action="store_true", default=False, help="use latest master directory: when active, the program looks for the latest directory that starts with the word 'master', so the user does not need to provide a directory name. (default False)")
parser.add_option("--restart_section", type="string", default=' ', help="restart section: each generation (iteration) contains three sections: 'restart', 'candidate_class_averages', and 'reproducible_class_averages'. To restart from a particular step, for example, generation 4 and section 'candidate_class_averages' the following option is needed: '--restart_section=candidate_class_averages,4'. The option requires no white space before or after the comma. The default behavior is to restart execution from where it stopped intentionally or unintentionally. For default restart, it is assumed that the name of the directory is provided as argument. Alternatively, the '--use_latest_master_directory' option can be used. (default ' ')")
parser.add_option("--stop_after_candidates", action="store_true", default=False, help="stop after candidates: stops after the 'candidate_class_averages' section. (default False)")
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help = SUPPRESS_HELP)
parser.add_option("--skip_prealignment", action="store_true", default=False, help="skip pre-alignment step: to be used if images are already centered. 2dalignment directory will still be generated but the parameters will be zero. (default False)")
required_option_list = ['radius']
(options, args) = parser.parse_args(args)
if options.return_options:
return parser
if len(args) > 2:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
sys.exit()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
global_def.BATCH = True
from isac import iter_isac
from fundamentals import rot_shift2D, resample
from utilities import pad, combine_params2
command_line_provided_stack_filename = args[0]
main_node = 0
mpi_init(0, [])
myid = mpi_comm_rank(MPI_COMM_WORLD)
nproc = mpi_comm_size(MPI_COMM_WORLD)
mpi_barrier(MPI_COMM_WORLD)
if(myid == main_node):
print "****************************************************************"
Util.version()
print "****************************************************************"
sys.stdout.flush()
mpi_barrier(MPI_COMM_WORLD)
# Making sure all required options appeared.
for required_option in required_option_list:
if not options.__dict__[required_option]:
print "\n ==%s== mandatory option is missing.\n"%required_option
print "Please run '" + progname + " -h' for detailed options"
return 1
radi = options.radius
target_radius = options.target_radius
target_nx = options.target_nx
center_method = options.center_method
if(radi < 1): ERROR("Particle radius has to be provided!","sxisac",1,myid)
use_latest_master_directory = options.use_latest_master_directory
stop_after_candidates = options.stop_after_candidates
# program_state_stack.restart_location_title_from_command_line = options.restart_section
from utilities import qw
program_state_stack.PROGRAM_STATE_VARIABLES = set(qw("""
isac_generation
"""))
# create or reuse master directory
masterdir = ""
stack_processed_by_ali2d_base__filename = ""
stack_processed_by_ali2d_base__filename__without_master_dir = ""
error_status = 0
if len(args) == 2:
masterdir = args[1]
elif len(args) == 1:
if use_latest_master_directory:
all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)]
import re; r = re.compile("^master.*$")
all_dirs = filter(r.match, all_dirs)
if len(all_dirs)>0:
# all_dirs = max(all_dirs, key=os.path.getctime)
masterdir = max(all_dirs, key=os.path.getmtime)
#Create folder for all results or check if there is one created already
if(myid == main_node):
if( masterdir == ""):
timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime())
masterdir = "master"+timestring
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
elif not os.path.exists(masterdir):
# os.path.exists(masterdir) does not exist
masterdir = args[1]
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
if(args[0][:4] == "bdb:"): filename = args[0][4:]
else: filename = args[0][:-4]
filename = os.path.basename(filename)
stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join(masterdir, filename )
stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename
if_error_then_all_processes_exit_program(error_status)
# send masterdir to all processes
masterdir = send_string_to_all(masterdir)
if myid == 0:
if options.restart_section != " ":
if os.path.exists(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)):
stored_stack, stored_state = restore_program_stack_and_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE))
import re
if "," in options.restart_section:
parsed_restart_section_option = options.restart_section.split(",")
stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%parsed_restart_section_option[0], stored_state[-1]["location_in_program"])
generation_str_format = parsed_restart_section_option[1]
if generation_str_format != "":
isac_generation_from_command_line = int(generation_str_format)
stored_state[-1]["isac_generation"] = isac_generation_from_command_line
else:
isac_generation_from_command_line = 1
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = -1
stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%options.restart_section, stored_state[-1]["location_in_program"])
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
store_program_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE), stored_state, stored_stack)
else:
print "Please remove the restart_section option from the command line. The program must be started from the beginning."
mpi_finalize()
sys.exit()
else:
isac_generation_from_command_line = -1
program_state_stack(locals(), getframeinfo(currentframe()), os.path.join(masterdir,NAME_OF_JSON_STATE_FILE))
stack_processed_by_ali2d_base__filename = send_string_to_all(stack_processed_by_ali2d_base__filename)
stack_processed_by_ali2d_base__filename__without_master_dir = \
send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir)
# previous code 2016-05-05--20-14-12-153
# # PARAMETERS OF THE PROCEDURE
# if( options.xr == -1 ):
# # Default values
# # target_nx = 76
# # target_radius = 29
# target_xr = 1
# else: # nx//2
# # Check below!
# target_xr = options.xr
# # target_nx = 76 + target_xr - 1 # subtract one, which is default
# target_nx += target_xr - 1 # subtract one, which is default
# # target_radius = 29
target_xr = options.xr
target_nx += target_xr - 1 # subtract one, which is default
if (options.yr == -1):
yr = options.xr
else:
yr = options.yr
mpi_barrier(MPI_COMM_WORLD)
# Initialization of stacks
if(myid == main_node):
print "command_line_provided_stack_filename", command_line_provided_stack_filename
number_of_images_in_stack = EMUtil.get_image_count(command_line_provided_stack_filename)
else:
number_of_images_in_stack = 0
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node)
nxrsteps = 4
init2dir = os.path.join(masterdir,"2dalignment")
# from mpi import mpi_finalize
# mpi_finalize()
# sys.stdout.flush()
# sys.exit()
if not os.path.exists(os.path.join(init2dir, "Finished_initial_2d_alignment.txt")):
if(myid == 0):
import subprocess
from logger import Logger, BaseLogger_Files
# Create output directory
log2d = Logger(BaseLogger_Files())
log2d.prefix = os.path.join(init2dir)
cmd = "mkdir -p "+log2d.prefix
outcome = subprocess.call(cmd, shell=True)
log2d.prefix += "/"
# outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True)
else:
outcome = 0
log2d = None
if(myid == main_node):
a = get_im(command_line_provided_stack_filename)
nnxo = a.get_xsize()
else:
nnxo = 0
nnxo = bcast_number_to_all(nnxo, source_node = main_node)
image_start, image_end = MPI_start_end(number_of_images_in_stack, nproc, myid)
if options.skip_prealignment:
params2d = [[0.0,0.0,0.0,0] for i in xrange(image_start, image_end)]
else:
original_images = EMData.read_images(command_line_provided_stack_filename, range(image_start,image_end))
# We assume the target radius will be 29, and xr = 1.
shrink_ratio = float(target_radius)/float(radi)
for im in xrange(len(original_images)):
if(shrink_ratio != 1.0):
original_images[im] = resample(original_images[im], shrink_ratio)
nx = original_images[0].get_xsize()
# nx = int(nx*shrink_ratio + 0.5)
txrm = (nx - 2*(target_radius+1))//2
if(txrm < 0): ERROR( "ERROR!! Radius of the structure larger than the window data size permits %d"%(radi), "sxisac",1, myid)
if(txrm/nxrsteps>0):
tss = ""
txr = ""
while(txrm/nxrsteps>0):
tts=txrm/nxrsteps
tss += " %d"%tts
txr += " %d"%(tts*nxrsteps)
txrm =txrm//2
else:
tss = "1"
txr = "%d"%txrm
# print "nx, txr, txrm, tss", nx, txr, txrm, tss
# from mpi import mpi_finalize
# mpi_finalize()
# sys.stdout.flush()
# sys.exit()
# section ali2d_base
params2d = ali2d_base(original_images, init2dir, None, 1, target_radius, 1, txr, txr, tss, \
False, 90.0, center_method, 14, options.CTF, 1.0, False, \
"ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False)
del original_images
for i in xrange(len(params2d)):
alpha, sx, sy, mirror = combine_params2(0, params2d[i][1],params2d[i][2], 0, -params2d[i][0], 0, 0, 0)
sx /= shrink_ratio
sy /= shrink_ratio
params2d[i][0] = 0.0
params2d[i][1] = sx
params2d[i][2] = sy
params2d[i][3] = 0
#set_params2D(aligned_images[i],[0.0, sx,sy,0.,1.0])
mpi_barrier(MPI_COMM_WORLD)
tmp = params2d[:]
tmp = wrap_mpi_gatherv(tmp, main_node, MPI_COMM_WORLD)
if( myid == main_node ):
if options.skip_prealignment:
print "========================================="
print "Even though there is no alignment step, '%s' params are set to zero for later use."%os.path.join(init2dir, "initial2Dparams.txt")
print "========================================="
write_text_row(tmp,os.path.join(init2dir, "initial2Dparams.txt"))
del tmp
mpi_barrier(MPI_COMM_WORLD)
# We assume the target image size will be target_nx, radius will be 29, and xr = 1.
# Note images can be also padded, in which case shrink_ratio > 1.
shrink_ratio = float(target_radius)/float(radi)
aligned_images = EMData.read_images(command_line_provided_stack_filename, range(image_start,image_end))
nx = aligned_images[0].get_xsize()
nima = len(aligned_images)
newx = int(nx*shrink_ratio + 0.5)
while not os.path.exists(os.path.join(init2dir, "initial2Dparams.txt")):
import time
time.sleep(1)
mpi_barrier(MPI_COMM_WORLD)
params = read_text_row(os.path.join(init2dir, "initial2Dparams.txt"))
params = params[image_start:image_end]
msk = model_circle(radi, nx, nx)
for im in xrange(nima):
st = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= st[0]
if options.CTF:
aligned_images[im] = filt_ctf(aligned_images[im], aligned_images[im].get_attr("ctf"), binary = True)
if(shrink_ratio < 1.0):
if newx > target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
#aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx :
msk = model_circle(newx//2-2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
elif(shrink_ratio == 1.0):
if newx > target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx :
msk = model_circle(newx//2-2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
#aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
elif(shrink_ratio > 1.0):
if newx > target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx :
msk = model_circle(newx//2-2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
del msk
gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images, myid)
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node)
if( myid == main_node ):
for i in range(number_of_images_in_stack): aligned_images[i].write_image(stack_processed_by_ali2d_base__filename,i)
# It has to be explicitly closed
from EMAN2db import db_open_dict
DB = db_open_dict(stack_processed_by_ali2d_base__filename)
DB.close()
fp = open(os.path.join(masterdir,"README_shrink_ratio.txt"), "w")
output_text = """
Since, for processing purposes, isac changes the image dimensions,
adjustment of pixel size needs to be made in subsequent steps, (e.g.
running sxviper.py). The shrink ratio for this particular isac run is
--------
%.5f
%.5f
--------
To get the pixel size for the isac output the user needs to divide
the original pixel size by the above value. This info is saved in
the following file: README_shrink_ratio.txt
"""%(shrink_ratio, radi)
fp.write(output_text); fp.flush() ;fp.close()
print output_text
fp = open(os.path.join(init2dir, "Finished_initial_2d_alignment.txt"), "w"); fp.flush() ;fp.close()
else:
if( myid == main_node ):
print "Skipping 2d alignment since it was already done!"
mpi_barrier(MPI_COMM_WORLD)
# from mpi import mpi_finalize
# mpi_finalize()
# sys.stdout.flush()
# sys.exit()
os.chdir(masterdir)
if program_state_stack(locals(), getframeinfo(currentframe())):
# if 1:
pass
if (myid == main_node):
cmdexecute("sxheader.py --consecutive --params=originalid %s"%stack_processed_by_ali2d_base__filename__without_master_dir)
cmdexecute("e2bdb.py %s --makevstack=%s_000"%(stack_processed_by_ali2d_base__filename__without_master_dir, stack_processed_by_ali2d_base__filename__without_master_dir))
if (myid == main_node):
main_dir_no = get_latest_directory_increment_value("./", NAME_OF_MAIN_DIR, myformat="%04d")
print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no
if isac_generation_from_command_line < 0:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = -1
if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no:
for i in xrange(isac_generation_from_command_line+1, main_dir_no + 1):
if i == isac_generation_from_command_line+1:
backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%i + " 000_backup" + "%05d"%backup_dir_no)
cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%i)
# it includes both command line and json file
my_restart_section = stored_state[-1]["location_in_program"].split("___")[-1]
if "restart" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no)
cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%isac_generation_from_command_line )
elif "candidate_class_averages" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no)
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line)
# cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line)
elif "reproducible_class_averages" in my_restart_section:
cmdexecute("rm -rf " + NAME_OF_MAIN_DIR + "%04d/ali_params_generation_*"%isac_generation_from_command_line)
cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_generation*"%isac_generation_from_command_line)
else:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 0
isac_generation_from_command_line = mpi_bcast(isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]
isac_generation = isac_generation_from_command_line - 1
if (myid == main_node):
if isac_generation == 0:
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation)
write_text_file([1], os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_accounted.txt"%isac_generation))
write_text_file(range(number_of_images_in_stack), os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_unaccounted.txt"%isac_generation))
# Stopping criterion should be inside the program.
while True:
isac_generation += 1
if isac_generation > options.n_generations:
break
data64_stack_current = "bdb:../"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d"%isac_generation
program_state_stack.restart_location_title = "restart"
if program_state_stack(locals(), getframeinfo(currentframe())):
if (myid == main_node):
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation)
# reference the original stack
list_file = os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1), "generation_%d_unaccounted.txt"%(isac_generation - 1))
cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\
stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file))
mpi_barrier(MPI_COMM_WORLD)
os.chdir(NAME_OF_MAIN_DIR + "%04d"%isac_generation)
program_state_stack.restart_location_title = "candidate_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new)
# program_state_stack.restart_location_title = "stopped_program1"
# program_state_stack(locals(), getframeinfo(currentframe()))
program_state_stack.restart_location_title = "stop_after_candidates"
program_state_stack(locals(), getframeinfo(currentframe()))
if stop_after_candidates:
mpi_finalize()
sys.exit()
exit_program = 0
if(myid == main_node):
if not os.path.exists("class_averages_candidate_generation_%d.hdf"%isac_generation):
print "This generation (%d) no class average candidates were generated! Finishing."%isac_generation
exit_program = 1
exit_program = int(mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
if exit_program:
os.chdir("..")
break
program_state_stack.restart_location_title = "reproducible_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new)
pass
os.chdir("..")
if(myid == main_node):
accounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation),"generation_%d_accounted.txt"%(isac_generation)))
number_of_accounted_images = len(accounted_images)
un_accounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation),"generation_%d_unaccounted.txt"%(isac_generation)))
number_of_un_accounted_images = len(un_accounted_images)
else:
number_of_accounted_images = 0
number_of_un_accounted_images = 0
number_of_accounted_images = int(mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
number_of_un_accounted_images = int(mpi_bcast(number_of_un_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
if number_of_accounted_images == 0:
if(myid == main_node):
print "This generation (%d) there are no accounted images! Finishing."%isac_generation
break
while (myid == main_node):
def files_are_missing(isac_generation):
for i in xrange(1, isac_generation + 1):
if not os.path.exists("generation_%04d/class_averages_generation_%d.hdf"%(i,i)):
print "Error: generation_%04d/class_averages_generation_%d.hdf is missing! Exiting."%(i,i)
return 1
return 0
if files_are_missing(isac_generation):
break
cmdexecute("rm -f class_averages.hdf")
cpy(["generation_%04d/class_averages_generation_%d.hdf"%(i,i) for i in xrange(1, isac_generation + 1)], "class_averages.hdf")
break
if number_of_un_accounted_images == 0:
if(myid == main_node):
print "This generation (%d) there are no un accounted images! Finishing."%isac_generation
break
program_state_stack(locals(), getframeinfo(currentframe()), last_call="__LastCall")
mpi_barrier(MPI_COMM_WORLD)
mpi_finalize()
def helicalshiftali_MPI(stack,
maskfile=None,
maxit=100,
CTF=False,
snr=1.0,
Fourvar=False,
search_rng=-1):
from applications import MPI_start_end
from utilities import model_circle, model_blank, get_image, peak_search, get_im, pad
from utilities import reduce_EMData_to_root, bcast_EMData_to_all, send_attr_dict, file_type, bcast_number_to_all, bcast_list_to_all
from pap_statistics import varf2d_MPI
from fundamentals import fft, ccf, rot_shift3D, rot_shift2D, fshift
from utilities import get_params2D, set_params2D, chunks_distribution
from utilities import print_msg, print_begin_msg, print_end_msg
import os
import sys
from mpi import mpi_init, mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD
from mpi import mpi_reduce, mpi_bcast, mpi_barrier, mpi_gatherv
from mpi import MPI_SUM, MPI_FLOAT, MPI_INT
from time import time
from pixel_error import ordersegments
from math import sqrt, atan2, tan, pi
nproc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
ftp = file_type(stack)
if myid == main_node:
print_begin_msg("helical-shiftali_MPI")
max_iter = int(maxit)
if (myid == main_node):
infils = EMUtil.get_all_attributes(stack, "filament")
ptlcoords = EMUtil.get_all_attributes(stack, 'ptcl_source_coord')
filaments = ordersegments(infils, ptlcoords)
total_nfils = len(filaments)
inidl = [0] * total_nfils
for i in range(total_nfils):
inidl[i] = len(filaments[i])
linidl = sum(inidl)
nima = linidl
tfilaments = []
for i in range(total_nfils):
tfilaments += filaments[i]
del filaments
else:
total_nfils = 0
linidl = 0
total_nfils = bcast_number_to_all(total_nfils, source_node=main_node)
if myid != main_node:
inidl = [-1] * total_nfils
inidl = bcast_list_to_all(inidl, myid, source_node=main_node)
linidl = bcast_number_to_all(linidl, source_node=main_node)
if myid != main_node:
tfilaments = [-1] * linidl
tfilaments = bcast_list_to_all(tfilaments, myid, source_node=main_node)
filaments = []
iendi = 0
for i in range(total_nfils):
isti = iendi
iendi = isti + inidl[i]
filaments.append(tfilaments[isti:iendi])
del tfilaments, inidl
if myid == main_node:
print_msg("total number of filaments: %d" % total_nfils)
if total_nfils < nproc:
ERROR(
'number of CPUs (%i) is larger than the number of filaments (%i), please reduce the number of CPUs used'
% (nproc, total_nfils), "ehelix_MPI", 1, myid)
# balanced load
temp = chunks_distribution([[len(filaments[i]), i]
for i in range(len(filaments))],
nproc)[myid:myid + 1][0]
filaments = [filaments[temp[i][1]] for i in range(len(temp))]
nfils = len(filaments)
#filaments = [[0,1]]
#print "filaments",filaments
list_of_particles = []
indcs = []
k = 0
for i in range(nfils):
list_of_particles += filaments[i]
k1 = k + len(filaments[i])
indcs.append([k, k1])
k = k1
data = EMData.read_images(stack, list_of_particles)
ldata = len(data)
print("ldata=", ldata)
nx = data[0].get_xsize()
ny = data[0].get_ysize()
if maskfile == None:
mrad = min(nx, ny) // 2 - 2
mask = pad(model_blank(2 * mrad + 1, ny, 1, 1.0), nx, ny, 1, 0.0)
else:
mask = get_im(maskfile)
# apply initial xform.align2d parameters stored in header
init_params = []
for im in range(ldata):
t = data[im].get_attr('xform.align2d')
init_params.append(t)
p = t.get_params("2d")
data[im] = rot_shift2D(data[im], p['alpha'], p['tx'], p['ty'],
p['mirror'], p['scale'])
if CTF:
from filter import filt_ctf
from morphology import ctf_img
ctf_abs_sum = EMData(nx, ny, 1, False)
ctf_2_sum = EMData(nx, ny, 1, False)
else:
ctf_2_sum = None
ctf_abs_sum = None
from utilities import info
for im in range(ldata):
data[im].set_attr('ID', list_of_particles[im])
st = Util.infomask(data[im], mask, False)
data[im] -= st[0]
if CTF:
ctf_params = data[im].get_attr("ctf")
qctf = data[im].get_attr("ctf_applied")
if qctf == 0:
data[im] = filt_ctf(fft(data[im]), ctf_params)
data[im].set_attr('ctf_applied', 1)
elif qctf != 1:
ERROR('Incorrectly set qctf flag', "helicalshiftali_MPI", 1,
myid)
ctfimg = ctf_img(nx, ctf_params, ny=ny)
Util.add_img2(ctf_2_sum, ctfimg)
Util.add_img_abs(ctf_abs_sum, ctfimg)
else:
data[im] = fft(data[im])
del list_of_particles
if CTF:
reduce_EMData_to_root(ctf_2_sum, myid, main_node)
reduce_EMData_to_root(ctf_abs_sum, myid, main_node)
if CTF:
if myid != main_node:
del ctf_2_sum
del ctf_abs_sum
else:
temp = EMData(nx, ny, 1, False)
tsnr = 1. / snr
for i in range(0, nx + 2, 2):
for j in range(ny):
temp.set_value_at(i, j, tsnr)
temp.set_value_at(i + 1, j, 0.0)
#info(ctf_2_sum)
Util.add_img(ctf_2_sum, temp)
#info(ctf_2_sum)
del temp
total_iter = 0
shift_x = [0.0] * ldata
for Iter in range(max_iter):
if myid == main_node:
start_time = time()
print_msg("Iteration #%4d\n" % (total_iter))
total_iter += 1
avg = EMData(nx, ny, 1, False)
for im in range(ldata):
Util.add_img(avg, fshift(data[im], shift_x[im]))
reduce_EMData_to_root(avg, myid, main_node)
if myid == main_node:
if CTF: tavg = Util.divn_filter(avg, ctf_2_sum)
else: tavg = Util.mult_scalar(avg, 1.0 / float(nima))
else:
tavg = model_blank(nx, ny)
if Fourvar:
bcast_EMData_to_all(tavg, myid, main_node)
vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF)
if myid == main_node:
if Fourvar:
tavg = fft(Util.divn_img(fft(tavg), vav))
vav_r = Util.pack_complex_to_real(vav)
# normalize and mask tavg in real space
tavg = fft(tavg)
stat = Util.infomask(tavg, mask, False)
tavg -= stat[0]
Util.mul_img(tavg, mask)
tavg.write_image("tavg.hdf", Iter)
# For testing purposes: shift tavg to some random place and see if the centering is still correct
#tavg = rot_shift3D(tavg,sx=3,sy=-4)
if Fourvar: del vav
bcast_EMData_to_all(tavg, myid, main_node)
tavg = fft(tavg)
sx_sum = 0.0
nxc = nx // 2
for ifil in range(nfils):
"""
# Calculate filament average
avg = EMData(nx, ny, 1, False)
filnima = 0
for im in xrange(indcs[ifil][0], indcs[ifil][1]):
Util.add_img(avg, data[im])
filnima += 1
tavg = Util.mult_scalar(avg, 1.0/float(filnima))
"""
# Calculate 1D ccf between each segment and filament average
nsegms = indcs[ifil][1] - indcs[ifil][0]
ctx = [None] * nsegms
pcoords = [None] * nsegms
for im in range(indcs[ifil][0], indcs[ifil][1]):
ctx[im - indcs[ifil][0]] = Util.window(ccf(tavg, data[im]), nx,
1)
pcoords[im - indcs[ifil][0]] = data[im].get_attr(
'ptcl_source_coord')
#ctx[im-indcs[ifil][0]].write_image("ctx.hdf",im-indcs[ifil][0])
#print " CTX ",myid,im,Util.infomask(ctx[im-indcs[ifil][0]], None, True)
# search for best x-shift
cents = nsegms // 2
dst = sqrt(
max((pcoords[cents][0] - pcoords[0][0])**2 +
(pcoords[cents][1] - pcoords[0][1])**2,
(pcoords[cents][0] - pcoords[-1][0])**2 +
(pcoords[cents][1] - pcoords[-1][1])**2))
maxincline = atan2(ny // 2 - 2 - float(search_rng), dst)
kang = int(dst * tan(maxincline) + 0.5)
#print " settings ",nsegms,cents,dst,search_rng,maxincline,kang
# ## C code for alignment. @ming
results = [0.0] * 3
results = Util.helixshiftali(ctx, pcoords, nsegms, maxincline,
kang, search_rng, nxc)
sib = int(results[0])
bang = results[1]
qm = results[2]
#print qm, sib, bang
# qm = -1.e23
#
# for six in xrange(-search_rng, search_rng+1,1):
# q0 = ctx[cents].get_value_at(six+nxc)
# for incline in xrange(kang+1):
# qt = q0
# qu = q0
# if(kang>0): tang = tan(maxincline/kang*incline)
# else: tang = 0.0
# for kim in xrange(cents+1,nsegms):
# dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
# xl = dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# #print " A ", ifil,six,incline,kim,xl,ixl,dxl
# qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# xl = -dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# for kim in xrange(cents):
# dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
# xl = -dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# xl = dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# if( qt > qm ):
# qm = qt
# sib = six
# bang = tang
# if( qu > qm ):
# qm = qu
# sib = six
# bang = -tang
#if incline == 0: print "incline = 0 ",six,tang,qt,qu
#print qm,six,sib,bang
#print " got results ",indcs[ifil][0], indcs[ifil][1], ifil,myid,qm,sib,tang,bang,len(ctx),Util.infomask(ctx[0], None, True)
for im in range(indcs[ifil][0], indcs[ifil][1]):
kim = im - indcs[ifil][0]
dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 +
(pcoords[cents][1] - pcoords[kim][1])**2)
if (kim < cents): xl = -dst * bang + sib
else: xl = dst * bang + sib
shift_x[im] = xl
# Average shift
sx_sum += shift_x[indcs[ifil][0] + cents]
# #print myid,sx_sum,total_nfils
sx_sum = mpi_reduce(sx_sum, 1, MPI_FLOAT, MPI_SUM, main_node,
MPI_COMM_WORLD)
if myid == main_node:
sx_sum = float(sx_sum[0]) / total_nfils
print_msg("Average shift %6.2f\n" % (sx_sum))
else:
sx_sum = 0.0
sx_sum = 0.0
sx_sum = bcast_number_to_all(sx_sum, source_node=main_node)
for im in range(ldata):
shift_x[im] -= sx_sum
#print " %3d %6.3f"%(im,shift_x[im])
#exit()
# combine shifts found with the original parameters
for im in range(ldata):
t1 = Transform()
##import random
##shix=random.randint(-10, 10)
##t1.set_params({"type":"2D","tx":shix})
t1.set_params({"type": "2D", "tx": shift_x[im]})
# combine t0 and t1
tt = t1 * init_params[im]
data[im].set_attr("xform.align2d", tt)
# write out headers and STOP, under MPI writing has to be done sequentially
mpi_barrier(MPI_COMM_WORLD)
par_str = ["xform.align2d", "ID"]
if myid == main_node:
from utilities import file_type
if (file_type(stack) == "bdb"):
from utilities import recv_attr_dict_bdb
recv_attr_dict_bdb(main_node, stack, data, par_str, 0, ldata,
nproc)
else:
from utilities import recv_attr_dict
recv_attr_dict(main_node, stack, data, par_str, 0, ldata, nproc)
else:
send_attr_dict(main_node, data, par_str, 0, ldata)
if myid == main_node: print_end_msg("helical-shiftali_MPI")
def main():
def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
if mirror:
m = 1
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
540.0 - psi, 0, 0, 1.0)
else:
m = 0
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
360.0 - psi, 0, 0, 1.0)
return alpha, sx, sy, m
progname = os.path.basename(sys.argv[0])
usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl=15. --aa=0.01 --sym=symmetry --CTF"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--output_dir",
type="string",
default="./",
help="output directory")
parser.add_option("--ave2D",
type="string",
default=False,
help="write to the disk a stack of 2D averages")
parser.add_option("--var2D",
type="string",
default=False,
help="write to the disk a stack of 2D variances")
parser.add_option("--ave3D",
type="string",
default=False,
help="write to the disk reconstructed 3D average")
parser.add_option("--var3D",
type="string",
default=False,
help="compute 3D variability (time consuming!)")
parser.add_option("--img_per_grp",
type="int",
default=10,
help="number of neighbouring projections")
parser.add_option("--no_norm",
action="store_true",
default=False,
help="do not use normalization")
#parser.add_option("--radius", type="int" , default=-1 , help="radius for 3D variability" )
parser.add_option("--npad",
type="int",
default=2,
help="number of time to pad the original images")
parser.add_option("--sym", type="string", default="c1", help="symmetry")
parser.add_option(
"--fl",
type="float",
default=0.0,
help=
"cutoff freqency in absolute frequency (0.0-0.5). (Default - no filtration)"
)
parser.add_option(
"--aa",
type="float",
default=0.0,
help=
"fall off of the filter. Put 0.01 if user has no clue about falloff (Default - no filtration)"
)
parser.add_option("--CTF",
action="store_true",
default=False,
help="use CFT correction")
parser.add_option("--VERBOSE",
action="store_true",
default=False,
help="Long output for debugging")
#parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
#parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" )
#parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" )
#parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
#parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
parser.add_option(
"--VAR",
action="store_true",
default=False,
help="stack on input consists of 2D variances (Default False)")
parser.add_option(
"--decimate",
type="float",
default=1.0,
help=
"image decimate rate, a number larger (expand image) or less (shrink image) than 1. default is 1"
)
parser.add_option(
"--window",
type="int",
default=0,
help=
"reduce images to a small image size without changing pixel_size. Default value is zero."
)
#parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)")
parser.add_option(
"--nvec",
type="int",
default=0,
help="number of eigenvectors, default = 0 meaning no PCA calculated")
parser.add_option(
"--symmetrize",
action="store_true",
default=False,
help="Prepare input stack for handling symmetry (Default False)")
(options, args) = parser.parse_args()
#####
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD
from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
from applications import MPI_start_end
from reconstruction import recons3d_em, recons3d_em_MPI
from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
from utilities import print_begin_msg, print_end_msg, print_msg
from utilities import read_text_row, get_image, get_im
from utilities import bcast_EMData_to_all, bcast_number_to_all
from utilities import get_symt
# This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015
from EMAN2db import db_open_dict
# Set up global variables related to bdb cache
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
# Set up global variables related to ERROR function
global_def.BATCH = True
# detect if program is running under MPI
RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ
if RUNNING_UNDER_MPI:
global_def.MPI = True
if options.symmetrize:
if RUNNING_UNDER_MPI:
try:
sys.argv = mpi_init(len(sys.argv), sys.argv)
try:
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
if (number_of_proc > 1):
ERROR(
"Cannot use more than one CPU for symmetry prepration",
"sx3dvariability", 1)
except:
pass
except:
pass
if options.output_dir != "./" and not os.path.exists(
options.output_dir):
os.mkdir(options.output_dir)
# Input
#instack = "Clean_NORM_CTF_start_wparams.hdf"
#instack = "bdb:data"
from logger import Logger, BaseLogger_Files
if os.path.exists(os.path.join(options.output_dir, "log.txt")):
os.remove(os.path.join(options.output_dir, "log.txt"))
log_main = Logger(BaseLogger_Files())
log_main.prefix = os.path.join(options.output_dir, "./")
instack = args[0]
sym = options.sym.lower()
if (sym == "c1"):
ERROR("There is no need to symmetrize stack for C1 symmetry",
"sx3dvariability", 1)
line = ""
for a in sys.argv:
line += " " + a
log_main.add(line)
if (instack[:4] != "bdb:"):
if output_dir == "./": stack = "bdb:data"
else: stack = "bdb:" + options.output_dir + "/data"
delete_bdb(stack)
junk = cmdexecute("sxcpy.py " + instack + " " + stack)
else:
stack = instack
qt = EMUtil.get_all_attributes(stack, 'xform.projection')
na = len(qt)
ts = get_symt(sym)
ks = len(ts)
angsa = [None] * na
for k in xrange(ks):
#Qfile = "Q%1d"%k
if options.output_dir != "./":
Qfile = os.path.join(options.output_dir, "Q%1d" % k)
else:
Qfile = os.path.join(options.output_dir, "Q%1d" % k)
#delete_bdb("bdb:Q%1d"%k)
delete_bdb("bdb:" + Qfile)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
junk = cmdexecute("e2bdb.py " + stack + " --makevstack=bdb:" +
Qfile)
#DB = db_open_dict("bdb:Q%1d"%k)
DB = db_open_dict("bdb:" + Qfile)
for i in xrange(na):
ut = qt[i] * ts[k]
DB.set_attr(i, "xform.projection", ut)
#bt = ut.get_params("spider")
#angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
#write_text_row(angsa, 'ptsma%1d.txt'%k)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
#junk = cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k))
DB.close()
if options.output_dir == "./": delete_bdb("bdb:sdata")
else: delete_bdb("bdb:" + options.output_dir + "/" + "sdata")
#junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
sdata = "bdb:" + options.output_dir + "/" + "sdata"
print(sdata)
junk = cmdexecute("e2bdb.py " + options.output_dir +
" --makevstack=" + sdata + " --filt=Q")
#junk = cmdexecute("ls EMAN2DB/sdata*")
#a = get_im("bdb:sdata")
a = get_im(sdata)
a.set_attr("variabilitysymmetry", sym)
#a.write_image("bdb:sdata")
a.write_image(sdata)
else:
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
if len(args) == 1:
stack = args[0]
else:
print(("usage: " + usage))
print(("Please run '" + progname + " -h' for detailed options"))
return 1
t0 = time()
# obsolete flags
options.MPI = True
options.nvec = 0
options.radiuspca = -1
options.iter = 40
options.abs = 0.0
options.squ = 0.0
if options.fl > 0.0 and options.aa == 0.0:
ERROR("Fall off has to be given for the low-pass filter",
"sx3dvariability", 1, myid)
if options.VAR and options.SND:
ERROR("Only one of var and SND can be set!", "sx3dvariability",
myid)
exit()
if options.VAR and (options.ave2D or options.ave3D or options.var2D):
ERROR(
"When VAR is set, the program cannot output ave2D, ave3D or var2D",
"sx3dvariability", 1, myid)
exit()
#if options.SND and (options.ave2D or options.ave3D):
# ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid)
# exit()
if options.nvec > 0:
ERROR("PCA option not implemented", "sx3dvariability", 1, myid)
exit()
if options.nvec > 0 and options.ave3D == None:
ERROR("When doing PCA analysis, one must set ave3D",
"sx3dvariability",
myid=myid)
exit()
import string
options.sym = options.sym.lower()
# if global_def.CACHE_DISABLE:
# from utilities import disable_bdb_cache
# disable_bdb_cache()
# global_def.BATCH = True
if myid == main_node:
if options.output_dir != "./" and not os.path.exists(
options.output_dir):
os.mkdir(options.output_dir)
img_per_grp = options.img_per_grp
nvec = options.nvec
radiuspca = options.radiuspca
from logger import Logger, BaseLogger_Files
#if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt"))
log_main = Logger(BaseLogger_Files())
log_main.prefix = os.path.join(options.output_dir, "./")
if myid == main_node:
line = ""
for a in sys.argv:
line += " " + a
log_main.add(line)
log_main.add("-------->>>Settings given by all options<<<-------")
log_main.add("instack :" + stack)
log_main.add("output_dir :" + options.output_dir)
log_main.add("var3d :" + options.var3D)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
#print_begin_msg("sx3dvariability")
msg = "sx3dvariability"
log_main.add(msg)
print(line, msg)
msg = ("%-70s: %s\n" % ("Input stack", stack))
log_main.add(msg)
print(line, msg)
symbaselen = 0
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
if options.sym != "c1":
imgdata = get_im(stack)
try:
i = imgdata.get_attr("variabilitysymmetry").lower()
if (i != options.sym):
ERROR(
"The symmetry provided does not agree with the symmetry of the input stack",
"sx3dvariability",
myid=myid)
except:
ERROR(
"Input stack is not prepared for symmetry, please follow instructions",
"sx3dvariability",
myid=myid)
from utilities import get_symt
i = len(get_symt(options.sym))
if ((nima / i) * i != nima):
ERROR(
"The length of the input stack is incorrect for symmetry processing",
"sx3dvariability",
myid=myid)
symbaselen = nima / i
else:
symbaselen = nima
else:
nima = 0
nx = 0
ny = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
Tracker = {}
Tracker["total_stack"] = nima
if options.decimate == 1.:
if options.window != 0:
nx = options.window
ny = options.window
else:
if options.window == 0:
nx = int(nx * options.decimate)
ny = int(ny * options.decimate)
else:
nx = int(options.window * options.decimate)
ny = nx
Tracker["nx"] = nx
Tracker["ny"] = ny
Tracker["nz"] = nx
symbaselen = bcast_number_to_all(symbaselen)
if radiuspca == -1: radiuspca = nx / 2 - 2
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = "%-70s: %d\n" % ("Number of projection", nima)
log_main.add(msg)
print(line, msg)
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
"""
if options.SND:
from projection import prep_vol, prgs
from statistics import im_diff
from utilities import get_im, model_circle, get_params_proj, set_params_proj
from utilities import get_ctf, generate_ctf
from filter import filt_ctf
imgdata = EMData.read_images(stack, range(img_begin, img_end))
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
bcast_EMData_to_all(vol, myid)
volft, kb = prep_vol(vol)
mask = model_circle(nx/2-2, nx, ny)
varList = []
for i in xrange(img_begin, img_end):
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
if options.CTF:
ctf_params = get_ctf(imgdata[i-img_begin])
ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
diff2 = diff*diff
set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
varList.append(diff2)
mpi_barrier(MPI_COMM_WORLD)
"""
if options.VAR:
#varList = EMData.read_images(stack, range(img_begin, img_end))
varList = []
this_image = EMData()
for index_of_particle in xrange(img_begin, img_end):
this_image.read_image(stack, index_of_particle)
varList.append(
image_decimate_window_xform_ctf(this_image,
options.decimate,
options.window,
options.CTF))
else:
from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
from utilities import model_blank, nearest_proj, model_circle
from applications import pca
from statistics import avgvar, avgvar_ctf, ccc
from filter import filt_tanl
from morphology import threshold, square_root
from projection import project, prep_vol, prgs
from sets import Set
if myid == main_node:
t1 = time()
proj_angles = []
aveList = []
tab = EMUtil.get_all_attributes(stack, 'xform.projection')
for i in xrange(nima):
t = tab[i].get_params('spider')
phi = t['phi']
theta = t['theta']
psi = t['psi']
x = theta
if x > 90.0: x = 180.0 - x
x = x * 10000 + psi
proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
t2 = time()
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = "%-70s: %d\n" % ("Number of neighboring projections",
img_per_grp)
log_main.add(msg)
print(line, msg)
msg = "...... Finding neighboring projections\n"
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
msg = "Number of images per group: %d" % img_per_grp
log_main.add(msg)
print(line, msg)
msg = "Now grouping projections"
log_main.add(msg)
print(line, msg)
proj_angles.sort()
proj_angles_list = [0.0] * (nima * 4)
if myid == main_node:
for i in xrange(nima):
proj_angles_list[i * 4] = proj_angles[i][1]
proj_angles_list[i * 4 + 1] = proj_angles[i][2]
proj_angles_list[i * 4 + 2] = proj_angles[i][3]
proj_angles_list[i * 4 + 3] = proj_angles[i][4]
proj_angles_list = bcast_list_to_all(proj_angles_list, myid,
main_node)
proj_angles = []
for i in xrange(nima):
proj_angles.append([
proj_angles_list[i * 4], proj_angles_list[i * 4 + 1],
proj_angles_list[i * 4 + 2],
int(proj_angles_list[i * 4 + 3])
])
del proj_angles_list
proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp,
range(img_begin, img_end))
all_proj = Set()
for im in proj_list:
for jm in im:
all_proj.add(proj_angles[jm][3])
all_proj = list(all_proj)
if options.VERBOSE:
print("On node %2d, number of images needed to be read = %5d" %
(myid, len(all_proj)))
index = {}
for i in xrange(len(all_proj)):
index[all_proj[i]] = i
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("%-70s: %.2f\n" %
("Finding neighboring projections lasted [s]",
time() - t2))
log_main.add(msg)
print(msg)
msg = ("%-70s: %d\n" %
("Number of groups processed on the main node",
len(proj_list)))
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
print("Grouping projections took: ", (time() - t2) / 60,
"[min]")
print("Number of groups on main node: ", len(proj_list))
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("...... calculating the stack of 2D variances \n")
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
print("Now calculating the stack of 2D variances")
proj_params = [0.0] * (nima * 5)
aveList = []
varList = []
if nvec > 0:
eigList = [[] for i in xrange(nvec)]
if options.VERBOSE:
print("Begin to read images on processor %d" % (myid))
ttt = time()
#imgdata = EMData.read_images(stack, all_proj)
imgdata = []
for index_of_proj in xrange(len(all_proj)):
#img = EMData()
#img.read_image(stack, all_proj[index_of_proj])
dmg = image_decimate_window_xform_ctf(
get_im(stack, all_proj[index_of_proj]), options.decimate,
options.window, options.CTF)
#print dmg.get_xsize(), "init"
imgdata.append(dmg)
if options.VERBOSE:
print("Reading images on processor %d done, time = %.2f" %
(myid, time() - ttt))
print("On processor %d, we got %d images" %
(myid, len(imgdata)))
mpi_barrier(MPI_COMM_WORLD)
'''
imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
if options.fl > 0.0:
for k in xrange(len(imgdata2)):
imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
if myid == main_node:
vol.write_image("vol_ctf.hdf")
print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf"))
del vol, imgdata2
mpi_barrier(MPI_COMM_WORLD)
'''
from applications import prepare_2d_forPCA
from utilities import model_blank
for i in xrange(len(proj_list)):
ki = proj_angles[proj_list[i][0]][3]
if ki >= symbaselen: continue
mi = index[ki]
phiM, thetaM, psiM, s2xM, s2yM = get_params_proj(imgdata[mi])
grp_imgdata = []
for j in xrange(img_per_grp):
mj = index[proj_angles[proj_list[i][j]][3]]
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[mj])
alpha, sx, sy, mirror = params_3D_2D_NEW(
phi, theta, psi, s2x, s2y, mirror_list[i][j])
if thetaM <= 90:
if mirror == 0:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, phiM - phi, 0.0, 0.0, 1.0)
else:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, 180 - (phiM - phi), 0.0,
0.0, 1.0)
else:
if mirror == 0:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, -(phiM - phi), 0.0, 0.0,
1.0)
else:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, -(180 - (phiM - phi)), 0.0,
0.0, 1.0)
set_params2D(imgdata[mj], [alpha, sx, sy, mirror, 1.0])
grp_imgdata.append(imgdata[mj])
#print grp_imgdata[j].get_xsize(), imgdata[mj].get_xsize()
if not options.no_norm:
#print grp_imgdata[j].get_xsize()
mask = model_circle(nx / 2 - 2, nx, nx)
for k in xrange(img_per_grp):
ave, std, minn, maxx = Util.infomask(
grp_imgdata[k], mask, False)
grp_imgdata[k] -= ave
grp_imgdata[k] /= std
del mask
if options.fl > 0.0:
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2 * nx
ny2 = 2 * ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d(
fft(
filt_tanl(
filt_ctf(
fft(
pad(grp_imgdata[k], nx2, ny2,
1, 0.0)),
grp_imgdata[k].get_attr("ctf"),
binary=1), options.fl,
options.aa)), nx, ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
for k in xrange(img_per_grp):
grp_imgdata[k] = filt_tanl(grp_imgdata[k],
options.fl, options.aa)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
from utilities import pad, read_text_file
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2 * nx
ny2 = 2 * ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d(
fft(
filt_ctf(fft(
pad(grp_imgdata[k], nx2, ny2, 1, 0.0)),
grp_imgdata[k].get_attr("ctf"),
binary=1)), nx, ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
'''
if i < 10 and myid == main_node:
for k in xrange(10):
grp_imgdata[k].write_image("grp%03d.hdf"%i, k)
'''
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("pp.hdf", pp)
"""
ave, grp_imgdata = prepare_2d_forPCA(grp_imgdata)
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("qq.hdf", pp)
"""
var = model_blank(nx, ny)
for q in grp_imgdata:
Util.add_img2(var, q)
Util.mul_scalar(var, 1.0 / (len(grp_imgdata) - 1))
# Switch to std dev
var = square_root(threshold(var))
#if options.CTF: ave, var = avgvar_ctf(grp_imgdata, mode="a")
#else: ave, var = avgvar(grp_imgdata, mode="a")
"""
if myid == main_node:
ave.write_image("avgv.hdf",i)
var.write_image("varv.hdf",i)
"""
set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])
aveList.append(ave)
varList.append(var)
if options.VERBOSE:
print("%5.2f%% done on processor %d" %
(i * 100.0 / len(proj_list), myid))
if nvec > 0:
eig = pca(input_stacks=grp_imgdata,
subavg="",
mask_radius=radiuspca,
nvec=nvec,
incore=True,
shuffle=False,
genbuf=True)
for k in xrange(nvec):
set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
eigList[k].append(eig[k])
"""
if myid == 0 and i == 0:
for k in xrange(nvec):
eig[k].write_image("eig.hdf", k)
"""
del imgdata
# To this point, all averages, variances, and eigenvectors are computed
if options.ave2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node:
for im in xrange(len(aveList)):
aveList[im].write_image(
os.path.join(options.output_dir,
options.ave2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i,
SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im + i + 70000)
"""
nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('pix_err', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
"""
tmpvol = fpol(ave, Tracker["nx"],
Tracker["nx"], 1)
tmpvol.write_image(
os.path.join(options.output_dir,
options.ave2D), km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in xrange(len(aveList)):
send_EMData(aveList[im], main_node, im + myid + 70000)
"""
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
members = aveList[im].get_attr('pix_err')
mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
except:
mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
"""
if options.ave3D:
from fundamentals import fpol
if options.VERBOSE:
print("Reconstructing 3D average volume")
ave3D = recons3d_4nn_MPI(myid,
aveList,
symmetry=options.sym,
npad=options.npad)
bcast_EMData_to_all(ave3D, myid)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
ave3D = fpol(ave3D, Tracker["nx"], Tracker["nx"],
Tracker["nx"])
ave3D.write_image(
os.path.join(options.output_dir, options.ave3D))
msg = ("%-70s: %s\n" % (
"Writing to the disk volume reconstructed from averages as",
options.ave3D))
log_main.add(msg)
print(line, msg)
del ave, var, proj_list, stack, phi, theta, psi, s2x, s2y, alpha, sx, sy, mirror, aveList
if nvec > 0:
for k in xrange(nvec):
if options.VERBOSE:
print("Reconstruction eigenvolumes", k)
cont = True
ITER = 0
mask2d = model_circle(radiuspca, nx, nx)
while cont:
#print "On node %d, iteration %d"%(myid, ITER)
eig3D = recons3d_4nn_MPI(myid,
eigList[k],
symmetry=options.sym,
npad=options.npad)
bcast_EMData_to_all(eig3D, myid, main_node)
if options.fl > 0.0:
eig3D = filt_tanl(eig3D, options.fl, options.aa)
if myid == main_node:
eig3D.write_image(
os.path.join(options.outpout_dir,
"eig3d_%03d.hdf" % (k, ITER)))
Util.mul_img(eig3D,
model_circle(radiuspca, nx, nx, nx))
eig3Df, kb = prep_vol(eig3D)
del eig3D
cont = False
icont = 0
for l in xrange(len(eigList[k])):
phi, theta, psi, s2x, s2y = get_params_proj(
eigList[k][l])
proj = prgs(eig3Df, kb,
[phi, theta, psi, s2x, s2y])
cl = ccc(proj, eigList[k][l], mask2d)
if cl < 0.0:
icont += 1
cont = True
eigList[k][l] *= -1.0
u = int(cont)
u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node,
MPI_COMM_WORLD)
icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM,
main_node, MPI_COMM_WORLD)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S",
localtime()) + " =>"
u = int(u[0])
msg = (" Eigenvector: ", k, " number changed ",
int(icont[0]))
log_main.add(msg)
print(line, msg)
else:
u = 0
u = bcast_number_to_all(u, main_node)
cont = bool(u)
ITER += 1
del eig3Df, kb
mpi_barrier(MPI_COMM_WORLD)
del eigList, mask2d
if options.ave3D: del ave3D
if options.var2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node:
for im in xrange(len(varList)):
tmpvol = fpol(varList[im], Tracker["nx"],
Tracker["nx"], 1)
tmpvol.write_image(
os.path.join(options.output_dir,
options.var2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i,
SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im + i + 70000)
tmpvol = fpol(ave, Tracker["nx"],
Tracker["nx"], 1)
tmpvol.write_image(
os.path.join(options.output_dir,
options.var2D, km))
km += 1
else:
mpi_send(len(varList), 1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in xrange(len(varList)):
send_EMData(varList[im], main_node, im + myid +
70000) # What with the attributes??
mpi_barrier(MPI_COMM_WORLD)
if options.var3D:
if myid == main_node and options.VERBOSE:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("Reconstructing 3D variability volume")
log_main.add(msg)
print(line, msg)
t6 = time()
# radiusvar = options.radius
# if( radiusvar < 0 ): radiusvar = nx//2 -3
res = recons3d_4nn_MPI(myid,
varList,
symmetry=options.sym,
npad=options.npad)
#res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
if myid == main_node:
from fundamentals import fpol
res = fpol(res, Tracker["nx"], Tracker["nx"], Tracker["nx"])
res.write_image(os.path.join(options.output_dir,
options.var3D))
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("%-70s: %.2f\n" %
("Reconstructing 3D variability took [s]", time() - t6))
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
print("Reconstruction took: %.2f [min]" %
((time() - t6) / 60))
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("%-70s: %.2f\n" %
("Total time for these computations [s]", time() - t0))
print(line, msg)
log_main.add(msg)
if options.VERBOSE:
print("Total time for these computations: %.2f [min]" %
((time() - t0) / 60))
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("sx3dvariability")
print(line, msg)
log_main.add(msg)
from mpi import mpi_finalize
mpi_finalize()
if RUNNING_UNDER_MPI:
global_def.MPI = False
global_def.BATCH = False
def prep_vol(vol, npad=2, interpolation_method=-1):
"""
Name
prep_vol - prepare the volume for calculation of gridding projections and generate the interpolants.
Input
vol: input volume for which projections will be calculated using prgs (interpolation_method=-1) or prgl (interpolation_method>0)
interpolation_method = -1 gridding
interpolation_method = 0 NN
interpolation_method = 1 trilinear
Output
volft: volume prepared for gridding projections using prgs
kb: interpolants (tabulated Kaiser-Bessel function) when the volume is cubic.
kbx,kby: interpolants along x, y and z direction (tabulated Kaiser-Bessel function) when the volume is rectangular
"""
# prepare the volume
Mx = vol.get_xsize()
My = vol.get_ysize()
Mz = vol.get_zsize()
# gridding
if interpolation_method == -1:
K = 6
alpha = 1.75
assert npad == 2
if (Mx == Mz & My == Mz):
M = vol.get_xsize()
# padd two times
N = M * npad
# support of the window
kb = Util.KaiserBessel(alpha, K, M / 2, K / (2. * N), N)
volft = vol.copy()
volft.divkbsinh(kb)
volft = volft.norm_pad(False, npad)
volft.do_fft_inplace()
volft.center_origin_fft()
volft.fft_shuffle()
return volft, kb
else:
Nx = Mx * npad
Ny = My * npad
Nz = Mz * npad
# support of the window
kbx = Util.KaiserBessel(alpha, K, Mx / 2, K / (2. * Nx), Nx)
kby = Util.KaiserBessel(alpha, K, My / 2, K / (2. * Ny), Ny)
kbz = Util.KaiserBessel(alpha, K, Mz / 2, K / (2. * Nz), Nz)
volft = vol.copy()
volft.divkbsinh_rect(kbx, kby, kbz)
volft = volft.norm_pad(False, npad)
volft.do_fft_inplace()
volft.center_origin_fft()
volft.fft_shuffle()
return volft, kbx, kby, kbz
else:
# NN and trilinear
assert interpolation_method >= 0
from utilities import pad
volft = pad(vol, Mx * npad, My * npad, My * npad, 0.0)
volft.set_attr("npad", npad)
volft.div_sinc(interpolation_method)
volft = volft.norm_pad(False, 1)
volft.do_fft_inplace()
volft.center_origin_fft()
volft.fft_shuffle()
volft.set_attr("npad", npad)
return volft
def prep_vol(vol, npad = 2, interpolation_method = -1):
"""
Name
prep_vol - prepare the volume for calculation of gridding projections and generate the interpolants.
Input
vol: input volume for which projections will be calculated using prgs (interpolation_method=-1) or prgl (interpolation_method>0)
interpolation_method = -1 gridding
interpolation_method = 0 NN
interpolation_method = 1 trilinear
Output
volft: volume prepared for gridding projections using prgs
kb: interpolants (tabulated Kaiser-Bessel function) when the volume is cubic.
kbx,kby: interpolants along x, y and z direction (tabulated Kaiser-Bessel function) when the volume is rectangular
"""
# prepare the volume
Mx = vol.get_xsize()
My = vol.get_ysize()
Mz = vol.get_zsize()
# gridding
if interpolation_method == -1:
K = 6
alpha = 1.75
assert npad == 2
if(Mx==Mz&My==Mz):
M = vol.get_xsize()
# padd two times
N = M*npad
# support of the window
kb = Util.KaiserBessel(alpha, K, M/2, K/(2.*N), N)
volft = vol.copy()
volft.divkbsinh(kb)
volft = volft.norm_pad(False, npad)
volft.do_fft_inplace()
volft.center_origin_fft()
volft.fft_shuffle()
return volft,kb
else:
Nx = Mx*npad
Ny = My*npad
Nz = Mz*npad
# support of the window
kbx = Util.KaiserBessel(alpha, K, Mx/2, K/(2.*Nx), Nx)
kby = Util.KaiserBessel(alpha, K, My/2, K/(2.*Ny), Ny)
kbz = Util.KaiserBessel(alpha, K, Mz/2, K/(2.*Nz), Nz)
volft = vol.copy()
volft.divkbsinh_rect(kbx,kby,kbz)
volft = volft.norm_pad(False, npad)
volft.do_fft_inplace()
volft.center_origin_fft()
volft.fft_shuffle()
return volft,kbx,kby,kbz
else:
# NN and trilinear
assert interpolation_method >= 0
from utilities import pad
volft = pad(vol, Mx*npad, My*npad, My*npad, 0.0)
volft.div_sinc(interpolation_method)
volft = volft.norm_pad(False, 1)
volft.set_attr("npad", npad)
volft.do_fft_inplace()
volft.center_origin_fft()
volft.fft_shuffle()
return volft
def main(args):
progname = os.path.basename(sys.argv[0])
usage = (
progname +
" stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter"
+
" --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp"
+ " --generation=generation --rand_seed=rand_seed>")
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--radius",
type="int",
default=-1,
help="<Particle radius>, it has to be provided.")
parser.add_option(
"--img_per_grp",
type="int",
default=100,
help=
"<number of images per group> in the ideal case (essentially maximum size of class) (100)"
)
parser.add_option("--CTF",
action="store_true",
default=False,
help="<CTF flag>, if set the data will be phase-flipped")
parser.add_option(
"--ir",
type="int",
default=1,
help="<inner ring> of the resampling to polar coordinates (1)")
parser.add_option(
"--rs",
type="int",
default=1,
help="<ring step> of the resampling to polar coordinates (1)")
parser.add_option(
"--xr",
type="int",
default=-1,
help=
"<x range> of translational search (By default set by the program) (advanced)"
)
parser.add_option(
"--yr",
type="int",
default=-1,
help="<y range> of translational search (same as xr) (advanced)")
parser.add_option("--ts",
type="float",
default=1.0,
help="<search step> of translational search (1.0)")
parser.add_option(
"--maxit",
type="int",
default=30,
help="number of iterations for reference-free alignment (30)")
#parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)")
parser.add_option(
"--center_method",
type="int",
default=7,
help=
"<Method for centering> of global 2D average during initial prealignment of data (default : 7; 0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7)"
)
parser.add_option("--dst",
type="float",
default=90.0,
help="discrete angle used in within group alignment ")
parser.add_option(
"--FL",
type="float",
default=0.2,
help="<lowest stopband> frequency used in the tangent filter (0.2)")
parser.add_option(
"--FH",
type="float",
default=0.3,
help="<highest stopband> frequency used in the tangent filter (0.3)")
parser.add_option("--FF",
type="float",
default=0.2,
help="<fall-off of the tangent> filter (0.2)")
parser.add_option(
"--init_iter",
type="int",
default=3,
help=
"<init_iter> number of iterations of ISAC program in initialization (3)"
)
parser.add_option(
"--main_iter",
type="int",
default=3,
help="<main_iter> number of iterations of ISAC program in main part (3)"
)
parser.add_option(
"--iter_reali",
type="int",
default=1,
help=
"<iter_reali> number of iterations in ISAC before checking stability (1)"
)
parser.add_option(
"--match_first",
type="int",
default=1,
help="number of iterations to run 2-way matching in the first phase (1)"
)
parser.add_option(
"--max_round",
type="int",
default=20,
help=
"maximum rounds of generating candidate averages in the first phase (20)"
)
parser.add_option(
"--match_second",
type="int",
default=5,
help=
"number of iterations to run 2-way (or 3-way) matching in the second phase (5)"
)
parser.add_option("--stab_ali",
type="int",
default=5,
help="number of alignments when checking stability (5)")
parser.add_option(
"--thld_err",
type="float",
default=0.7,
help="the threshold of pixel error when checking stability (0.7)")
parser.add_option(
"--indep_run",
type="int",
default=4,
help=
"number of independent runs for reproducibility (default=4, only values 2, 3 and 4 are supported (4)"
)
parser.add_option("--thld_grp",
type="int",
default=10,
help="minimum size of class (10)")
parser.add_option(
"--n_generations",
type="int",
default=100,
help=
"<n_generations> program stops when reaching this total number of generations (advanced)"
)
#parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)")
parser.add_option(
"--rand_seed",
type="int",
default=None,
help=
"random seed set before calculations, useful for testing purposes (default None - total randomness)"
)
parser.add_option("--new",
action="store_true",
default=False,
help="use new code (default = False)")
parser.add_option("--debug",
action="store_true",
default=False,
help="debug info printout (default = False)")
# must be switched off in production
parser.add_option("--use_latest_master_directory",
action="store_true",
dest="use_latest_master_directory",
default=False)
parser.add_option(
"--restart_section",
type="string",
default="",
help=
"<restart section name> (no spaces) followed immediately by comma, followed immediately by generation to restart, example: \n--restart_section=candidate_class_averages,1 (Sections: restart, candidate_class_averages, reproducible_class_averages)"
)
parser.add_option(
"--stop_after_candidates",
action="store_true",
default=False,
help=
"<stop_after_candidates> stops after the 'candidate_class_averages' section"
)
parser.add_option("--return_options",
action="store_true",
dest="return_options",
default=False,
help=SUPPRESS_HELP)
(options, args) = parser.parse_args(args)
if options.return_options:
return parser
if len(args) > 2:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
sys.exit()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
from isac import iter_isac
global_def.BATCH = True
global_def.BATCH = True
command_line_provided_stack_filename = args[0]
global_def.BATCH = True
main_node = 0
mpi_init(0, [])
myid = mpi_comm_rank(MPI_COMM_WORLD)
nproc = mpi_comm_size(MPI_COMM_WORLD)
radi = options.radius
center_method = options.center_method
if (radi < 1):
ERROR("Particle radius has to be provided!", "sxisac", 1, myid)
use_latest_master_directory = options.use_latest_master_directory
stop_after_candidates = options.stop_after_candidates
program_state_stack.restart_location_title_from_command_line = options.restart_section
from utilities import qw
program_state_stack.PROGRAM_STATE_VARIABLES = set(
qw("""
isac_generation
"""))
# create or reuse master directory
masterdir = ""
stack_processed_by_ali2d_base__filename = ""
stack_processed_by_ali2d_base__filename__without_master_dir = ""
error_status = 0
if len(args) == 2:
masterdir = args[1]
elif len(args) == 1:
if use_latest_master_directory:
all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)]
import re
r = re.compile("^master.*$")
all_dirs = filter(r.match, all_dirs)
if len(all_dirs) > 0:
# all_dirs = max(all_dirs, key=os.path.getctime)
masterdir = max(all_dirs, key=os.path.getmtime)
#Create folder for all results or check if there is one created already
if (myid == main_node):
if (masterdir == ""):
timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM,
localtime())
masterdir = "master" + timestring
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
elif not os.path.exists(masterdir):
# os.path.exists(masterdir) does not exist
masterdir = args[1]
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
if (args[0][:4] == "bdb:"): filename = args[0][4:]
else: filename = args[0][:-4]
filename = os.path.basename(filename)
stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join(
masterdir, filename)
stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename
if_error_all_processes_quit_program(error_status)
# send masterdir to all processes
masterdir = send_string_to_all(masterdir)
if myid == 0:
if options.restart_section != "":
if os.path.exists(os.path.join(masterdir,
NAME_OF_JSON_STATE_FILE)):
stored_stack, stored_state = restore_program_stack_and_state(
os.path.join(masterdir, NAME_OF_JSON_STATE_FILE))
import re
if "," in options.restart_section:
parsed_restart_section_option = options.restart_section.split(
",")
stored_state[-1]["location_in_program"] = re.sub(
r"___.*$", "___%s" % parsed_restart_section_option[0],
stored_state[-1]["location_in_program"])
generation_str_format = parsed_restart_section_option[1]
if generation_str_format != "":
isac_generation_from_command_line = int(
generation_str_format)
stored_state[-1][
"isac_generation"] = isac_generation_from_command_line
else:
isac_generation_from_command_line = 1
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = -1
stored_state[-1]["location_in_program"] = re.sub(
r"___.*$", "___%s" % options.restart_section,
stored_state[-1]["location_in_program"])
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
store_program_state(
os.path.join(masterdir, NAME_OF_JSON_STATE_FILE),
stored_state, stored_stack)
else:
print "Please remove the restart_section option from the command line. The program must be started from the beginning."
mpi_finalize()
sys.exit()
else:
isac_generation_from_command_line = -1
program_state_stack(locals(), getframeinfo(currentframe()),
os.path.join(masterdir, NAME_OF_JSON_STATE_FILE))
stack_processed_by_ali2d_base__filename = send_string_to_all(
stack_processed_by_ali2d_base__filename)
stack_processed_by_ali2d_base__filename__without_master_dir = \
send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir)
# PARAMETERS OF THE PROCEDURE
if (options.xr == -1):
# Default values
target_nx = 76
target_radius = 29
target_xr = 1
else: # nx//2
# Check below!
target_xr = options.xr
target_nx = 76 + target_xr - 1 # subtract one, which is default
target_radius = 29
mpi_barrier(MPI_COMM_WORLD)
# Initialization of stacks
if (myid == main_node):
number_of_images_in_stack = EMUtil.get_image_count(
command_line_provided_stack_filename)
else:
number_of_images_in_stack = 0
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack,
source_node=main_node)
nxrsteps = 4
init2dir = os.path.join(masterdir, "2dalignment")
if (myid == 0):
import subprocess
from logger import Logger, BaseLogger_Files
# Create output directory
log2d = Logger(BaseLogger_Files())
log2d.prefix = os.path.join(init2dir)
cmd = "mkdir -p " + log2d.prefix
outcome = subprocess.call(cmd, shell=True)
log2d.prefix += "/"
# outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True)
else:
outcome = 0
log2d = None
if (myid == main_node):
a = get_im(command_line_provided_stack_filename)
nnxo = a.get_xsize()
else:
nnxo = 0
nnxo = bcast_number_to_all(nnxo, source_node=main_node)
txrm = (nnxo - 2 * (radi + 1)) // 2
if (txrm < 0):
ERROR(
"ERROR!! Radius of the structure larger than the window data size permits %d"
% (radi), "sxisac", 1, myid)
if (txrm / nxrsteps > 0):
tss = ""
txr = ""
while (txrm / nxrsteps > 0):
tts = txrm / nxrsteps
tss += " %d" % tts
txr += " %d" % (tts * nxrsteps)
txrm = txrm // 2
else:
tss = "1"
txr = "%d" % txrm
# section ali2d_base
# centering method is set to #7
params2d, aligned_images = ali2d_base(command_line_provided_stack_filename, init2dir, None, 1, radi, 1, txr, txr, tss, \
False, 90.0, center_method, 14, options.CTF, 1.0, False, \
"ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False)
if (myid == main_node):
write_text_row(params2d, os.path.join(init2dir, "initial2Dparams.txt"))
del params2d
mpi_barrier(MPI_COMM_WORLD)
# We assume the target image size will be target_nx, radius will be 29, and xr = 1.
# Note images can be also padded, in which case shrink_ratio > 1.
shrink_ratio = float(target_radius) / float(radi)
nx = aligned_images[0].get_xsize()
nima = len(aligned_images)
newx = int(nx * shrink_ratio + 0.5)
from fundamentals import rot_shift2D, resample
from utilities import pad, combine_params2
if (shrink_ratio < 1.0):
if newx > target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
aligned_images[im] = Util.window(aligned_images[im], target_nx,
target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx:
msk = model_circle(nx // 2 - 2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx,
target_nx, 1, 0.0)
elif (shrink_ratio == 1.0):
if newx > target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
aligned_images[im] = Util.window(aligned_images[im], target_nx,
target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx:
msk = model_circle(nx // 2 - 2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx,
target_nx, 1, 0.0)
elif (shrink_ratio > 1.0):
if newx > target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
aligned_images[im] = Util.window(aligned_images[im], target_nx,
target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx:
msk = model_circle(target_radius, nx, nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx,
target_nx, 1, 0.0)
del msk
gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images,
myid)
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack,
source_node=main_node)
if (myid == main_node):
for i in range(number_of_images_in_stack):
aligned_images[i].write_image(
stack_processed_by_ali2d_base__filename, i)
# It has to be explicitly closed
from EMAN2db import db_open_dict
DB = db_open_dict(stack_processed_by_ali2d_base__filename)
DB.close()
mpi_barrier(MPI_COMM_WORLD)
global_def.BATCH = True
os.chdir(masterdir)
if program_state_stack(locals(), getframeinfo(currentframe())):
# if 1:
pass
if (myid == main_node):
cmdexecute(
"sxheader.py --consecutive --params=originalid %s" %
stack_processed_by_ali2d_base__filename__without_master_dir)
cmdexecute(
"e2bdb.py %s --makevstack=%s_000" %
(stack_processed_by_ali2d_base__filename__without_master_dir,
stack_processed_by_ali2d_base__filename__without_master_dir))
if (myid == main_node):
main_dir_no = get_latest_directory_increment_value("./",
NAME_OF_MAIN_DIR,
myformat="%04d")
print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no
if isac_generation_from_command_line < 0:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(
NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1][
"isac_generation"]
else:
isac_generation_from_command_line = -1
if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no:
for i in xrange(isac_generation_from_command_line + 1,
main_dir_no + 1):
if i == isac_generation_from_command_line + 1:
backup_dir_no = get_nonexistent_directory_increment_value(
"./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" +
"%05d" % backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % i +
" 000_backup" + "%05d" % backup_dir_no)
cmdexecute(
"rm " + "EMAN2DB/" +
stack_processed_by_ali2d_base__filename__without_master_dir[
4:] + "_%03d.bdb" % i)
# it includes both command line and json file
my_restart_section = stored_state[-1]["location_in_program"].split(
"___")[-1]
if "restart" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value(
"./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" +
"%05d" % backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation_from_command_line +
" 000_backup" + "%05d" % backup_dir_no)
cmdexecute(
"rm " + "EMAN2DB/" +
stack_processed_by_ali2d_base__filename__without_master_dir[
4:] + "_%03d.bdb" % isac_generation_from_command_line)
elif "candidate_class_averages" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value(
"./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" +
"%05d" % backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation_from_command_line +
" 000_backup" + "%05d" % backup_dir_no)
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation_from_command_line)
# cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line)
elif "reproducible_class_averages" in my_restart_section:
cmdexecute("rm -rf " + NAME_OF_MAIN_DIR +
"%04d/ali_params_generation_*" %
isac_generation_from_command_line)
cmdexecute("rm -f " + NAME_OF_MAIN_DIR +
"%04d/class_averages_generation*" %
isac_generation_from_command_line)
else:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(
NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1][
"isac_generation"]
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 0
isac_generation_from_command_line = mpi_bcast(
isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]
isac_generation = isac_generation_from_command_line - 1
if (myid == main_node):
if isac_generation == 0:
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation)
write_text_file(
[1],
os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation,
"generation_%d_accounted.txt" % isac_generation))
write_text_file(
range(number_of_images_in_stack),
os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation,
"generation_%d_unaccounted.txt" %
isac_generation))
# Stopping criterion should be inside the program.
while True:
isac_generation += 1
if isac_generation > options.n_generations:
break
data64_stack_current = "bdb:../" + stack_processed_by_ali2d_base__filename__without_master_dir[
4:] + "_%03d" % isac_generation
if (myid == main_node):
accounted_images = read_text_file(
os.path.join(
NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1),
"generation_%d_accounted.txt" % (isac_generation - 1)))
number_of_accounted_images = len(accounted_images)
# unaccounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1),"generation_%d_unaccounted.txt"%(isac_generation - 1)))
# number_of_unaccounted_images = len(unaccounted_images)
else:
number_of_accounted_images = 0
number_of_accounted_images = int(
mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0,
MPI_COMM_WORLD)[0])
if number_of_accounted_images == 0:
os.chdir("..")
break
program_state_stack.restart_location_title = "restart"
if program_state_stack(locals(), getframeinfo(currentframe())):
if (myid == main_node):
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation)
# reference the original stack
list_file = os.path.join(
NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1),
"generation_%d_unaccounted.txt" % (isac_generation - 1))
cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\
stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file))
mpi_barrier(MPI_COMM_WORLD)
os.chdir(NAME_OF_MAIN_DIR + "%04d" % isac_generation)
program_state_stack.restart_location_title = "candidate_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new)
# program_state_stack.restart_location_title = "stopped_program1"
# program_state_stack(locals(), getframeinfo(currentframe()))
program_state_stack.restart_location_title = "stop_after_candidates"
program_state_stack(locals(), getframeinfo(currentframe()))
if stop_after_candidates:
mpi_finalize()
sys.exit()
exit_program = 0
if (myid == main_node):
if not os.path.exists(
"class_averages_candidate_generation_%d.hdf" %
isac_generation):
print "This generation (%d) no class averages were generated!" % isac_generation
exit_program = 1
exit_program = int(
mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
if exit_program:
os.chdir("..")
break
program_state_stack.restart_location_title = "reproducible_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new)
pass
os.chdir("..")
if (myid == main_node):
cmdexecute("rm -f class_averages.hdf")
cpy([
"generation_%04d/class_averages_generation_%d.hdf" % (i, i)
for i in xrange(1, isac_generation)
], "class_averages.hdf")
# program_state_stack.restart_location_title = "stopped_program2"
# program_state_stack(locals(), getframeinfo(currentframe()))
program_state_stack(locals(),
getframeinfo(currentframe()),
last_call="__LastCall")
mpi_finalize()
def generate_helimic(refvol,
outdir,
pixel,
CTF=False,
Cs=2.0,
voltage=200.0,
ampcont=10.0,
nonoise=False,
rand_seed=14567):
from utilities import model_blank, model_gauss, model_gauss_noise, pad, get_im
from random import random
from projection import prgs, prep_vol
from filter import filt_gaussl, filt_ctf
from EMAN2 import EMAN2Ctf
if os.path.exists(outdir):
ERROR(
'Output directory exists, please change the name and restart the program',
"sxhelical_demo", 1)
os.mkdir(outdir)
seed(rand_seed)
Util.set_randnum_seed(rand_seed)
angles = []
for i in range(3):
angles.append([0.0 + 60.0 * i, 90.0 - i * 5, 0.0, 0.0, 0.0])
nangle = len(angles)
volfts = get_im(refvol)
nx = volfts.get_xsize()
ny = volfts.get_ysize()
nz = volfts.get_zsize()
volfts, kbx, kby, kbz = prep_vol(volfts)
iprj = 0
width = 500
xstart = 0
ystart = 0
for idef in range(3, 6):
mic = model_blank(2048, 2048)
#defocus = idef*0.2
defocus = idef * 0.6 ##@ming
if CTF:
#ctf = EMAN2Ctf()
#ctf.from_dict( {"defocus":defocus, "cs":Cs, "voltage":voltage, "apix":pixel, "ampcont":ampcont, "bfactor":0.0} )
from utilities import generate_ctf
ctf = generate_ctf(
[defocus, 2, 200, 1.84, 0.0, ampcont, defocus * 0.2, 80]
) ##@ming the range of astigmatism amplitude is between 10 percent and 22 percent. 20 percent is a good choice.
i = idef - 4
for k in range(1):
psi = 90 + 10 * i
proj = prgs(
volfts, kbz,
[angles[idef - 3][0], angles[idef - 3][1], psi, 0.0, 0.0], kbx,
kby)
proj = Util.window(proj, 320, nz)
mic += pad(proj, 2048, 2048, 1, 0.0, 750 * i, 20 * i, 0)
if not nonoise: mic += model_gauss_noise(30.0, 2048, 2048)
if CTF:
#apply CTF
mic = filt_ctf(mic, ctf)
if not nonoise:
mic += filt_gaussl(model_gauss_noise(17.5, 2048, 2048), 0.3)
mic.write_image("%s/mic%1d.hdf" % (outdir, idef - 3), 0)
def main():
import sys
import os
import math
import random
import pyemtbx.options
import time
from random import random, seed, randint
from optparse import OptionParser
progname = os.path.basename(sys.argv[0])
usage = progname + """ [options] <inputfile> <outputfile>
Generic 2-D image processing programs.
Functionality:
1. Phase flip a stack of images and write output to new file:
sxprocess.py input_stack.hdf output_stack.hdf --phase_flip
2. Resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.
The window size will change accordingly.
sxprocess input.hdf output.hdf --changesize --ratio=0.5
3. Compute average power spectrum of a stack of 2D images with optional padding (option wn) with zeroes or a 3-D volume.
sxprocess.py input_stack.hdf powerspectrum.hdf --pw [--wn=1024]
4. Generate a stack of projections bdb:data and micrographs with prefix mic (i.e., mic0.hdf, mic1.hdf etc) from structure input_structure.hdf, with CTF applied to both projections and micrographs:
sxprocess.py input_structure.hdf data mic --generate_projections format="bdb":apix=5.2:CTF=True:boxsize=64
5. Retrieve original image numbers in the selected ISAC group (here group 12 from generation 3):
sxprocess.py bdb:test3 class_averages_generation_3.hdf list3_12.txt --isacgroup=12 --params=originalid
6. Retrieve original image numbers of images listed in ISAC output stack of averages:
sxprocess.py select1.hdf ohk.txt
7. Adjust rotationally averaged power spectrum of an image to that of a reference image or a reference 1D power spectrum stored in an ASCII file.
Optionally use a tangent low-pass filter. Also works for a stack of images, in which case the output is also a stack.
sxprocess.py vol.hdf ref.hdf avol.hdf < 0.25 0.2> --adjpw
sxprocess.py vol.hdf pw.txt avol.hdf < 0.25 0.2> --adjpw
8. Generate a 1D rotationally averaged power spectrum of an image.
sxprocess.py vol.hdf --rotwp=rotpw.txt
# Output will contain three columns:
(1) rotationally averaged power spectrum
(2) logarithm of the rotationally averaged power spectrum
(3) integer line number (from zero to approximately to half the image size)
9. Apply 3D transformation (rotation and/or shift) to a set of orientation parameters associated with projection data.
sxprocess.py --transfromparams=phi,theta,psi,tx,ty,tz input.txt output.txt
The output file is then imported and 3D transformed volume computed:
sxheader.py bdb:p --params=xform.projection --import=output.txt
mpirun -np 2 sxrecons3d_n.py bdb:p tvol.hdf --MPI
The reconstructed volume is in the position of the volume computed using the input.txt parameters and then
transformed with rot_shift3D(vol, phi,theta,psi,tx,ty,tz)
10. Import ctf parameters from the output of sxcter into windowed particle headers.
There are three possible input files formats: (1) all particles are in one stack, (2 aor 3) particles are in stacks, each stack corresponds to a single micrograph.
In each case the particles should contain a name of the micrograph of origin stores using attribute name 'ptcl_source_image'.
Normally this is done by e2boxer.py during windowing.
Particles whose defocus or astigmatism error exceed set thresholds will be skipped, otherwise, virtual stacks with the original way preceded by G will be created.
sxprocess.py --input=bdb:data --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0
# Output will be a vritual stack bdb:Gdata
sxprocess.py --input="bdb:directory/stacks*" --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0
To concatenate output files:
cd directory
e2bdb.py . --makevstack=bdb:allparticles --filt=G
IMPORTANT: Please do not move (or remove!) any input/intermediate EMAN2DB files as the information is linked between them.
11. Scale 3D shifts. The shifts in the input five columns text file with 3D orientation parameters will be DIVIDED by the scale factor
sxprocess.py orientationparams.txt scaledparams.txt scale=0.5
12. Generate 3D mask from a given 3-D volume automatically or using threshold provided by user.
13. Postprocess 3-D or 2-D images:
for 3-D volumes: calculate FSC with provided mask; weight summed volume with FSC; estimate B-factor from FSC weighted summed two volumes; apply negative B-factor to the weighted volume.
for 2-D images: calculate B-factor and apply negative B-factor to 2-D images.
14. Winow stack file -reduce size of images without changing the pixel size.
"""
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--order", action="store_true", help="Two arguments are required: name of input stack and desired name of output stack. The output stack is the input stack sorted by similarity in terms of cross-correlation coefficent.", default=False)
parser.add_option("--order_lookup", action="store_true", help="Test/Debug.", default=False)
parser.add_option("--order_metropolis", action="store_true", help="Test/Debug.", default=False)
parser.add_option("--order_pca", action="store_true", help="Test/Debug.", default=False)
parser.add_option("--initial", type="int", default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)")
parser.add_option("--circular", action="store_true", help="Select circular ordering (fisr image has to be similar to the last", default=False)
parser.add_option("--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering")
parser.add_option("--changesize", action="store_true", help="resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.", default=False)
parser.add_option("--ratio", type="float", default=1.0, help="The ratio of new to old image size (if <1 the pixel size will increase and image size decrease, if>1, the other way round")
parser.add_option("--pw", action="store_true", help="compute average power spectrum of a stack of 2-D images with optional padding (option wn) with zeroes", default=False)
parser.add_option("--wn", type="int", default=-1, help="Size of window to use (should be larger/equal than particle box size, default padding to max(nx,ny))")
parser.add_option("--phase_flip", action="store_true", help="Phase flip the input stack", default=False)
parser.add_option("--makedb", metavar="param1=value1:param2=value2", type="string",
action="append", help="One argument is required: name of key with which the database will be created. Fill in database with parameters specified as follows: --makedb param1=value1:param2=value2, e.g. 'gauss_width'=1.0:'pixel_input'=5.2:'pixel_output'=5.2:'thr_low'=1.0")
parser.add_option("--generate_projections", metavar="param1=value1:param2=value2", type="string",
action="append", help="Three arguments are required: name of input structure from which to generate projections, desired name of output projection stack, and desired prefix for micrographs (e.g. if prefix is 'mic', then micrographs mic0.hdf, mic1.hdf etc will be generated). Optional arguments specifying format, apix, box size and whether to add CTF effects can be entered as follows after --generate_projections: format='bdb':apix=5.2:CTF=True:boxsize=100, or format='hdf', etc., where format is bdb or hdf, apix (pixel size) is a float, CTF is True or False, and boxsize denotes the dimension of the box (assumed to be a square). If an optional parameter is not specified, it will default as follows: format='bdb', apix=2.5, CTF=False, boxsize=64.")
parser.add_option("--isacgroup", type="int", help="Retrieve original image numbers in the selected ISAC group. See ISAC documentation for details.", default=-1)
parser.add_option("--isacselect", action="store_true", help="Retrieve original image numbers of images listed in ISAC output stack of averages. See ISAC documentation for details.", default=False)
parser.add_option("--params", type="string", default=None, help="Name of header of parameter, which one depends on specific option")
parser.add_option("--adjpw", action="store_true", help="Adjust rotationally averaged power spectrum of an image", default=False)
parser.add_option("--rotpw", type="string", default=None, help="Name of the text file to contain rotationally averaged power spectrum of the input image.")
parser.add_option("--transformparams", type="string", default=None, help="Transform 3D projection orientation parameters using six 3D parameters (phi, theta,psi,sx,sy,sz). Input: --transformparams=45.,66.,12.,-2,3,-5.5 desired six transformation of the reconstructed structure. Output: file with modified orientation parameters.")
# import ctf estimates done using cter
parser.add_option("--input", type="string", default= None, help="Input particles.")
parser.add_option("--importctf", type="string", default= None, help="Name of the file containing CTF parameters produced by sxcter.")
parser.add_option("--defocuserror", type="float", default=1000000.0, help="Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent. The error is computed as (std dev defocus)/defocus*100%")
parser.add_option("--astigmatismerror", type="float", default=360.0, help="Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees.")
# import ctf estimates done using cter
parser.add_option("--scale", type="float", default=-1.0, help="Divide shifts in the input 3D orientation parameters text file by the scale factor.")
# generate adaptive mask from an given 3-D volume
parser.add_option("--adaptive_mask", action="store_true", help="create adavptive 3-D mask from a given volume", default=False)
parser.add_option("--nsigma", type="float", default= 1., help="number of times of sigma of the input volume to obtain the the large density cluster")
parser.add_option("--ndilation", type="int", default= 3, help="number of times of dilation applied to the largest cluster of density")
parser.add_option("--kernel_size", type="int", default= 11, help="convolution kernel for smoothing the edge of the mask")
parser.add_option("--gauss_standard_dev", type="int", default= 9, help="stanadard deviation value to generate Gaussian edge")
parser.add_option("--threshold", type="float", default= 9999., help="threshold provided by user to binarize input volume")
parser.add_option("--ne", type="int", default= 0, help="number of times to erode the binarized input image")
parser.add_option("--nd", type="int", default= 0, help="number of times to dilate the binarized input image")
parser.add_option("--postprocess", action="store_true", help="postprocess unfiltered odd, even 3-D volumes",default=False)
parser.add_option("--fsc_weighted", action="store_true", help="postprocess unfiltered odd, even 3-D volumes")
parser.add_option("--low_pass_filter", action="store_true", default=False, help="postprocess unfiltered odd, even 3-D volumes")
parser.add_option("--ff", type="float", default=.25, help="low pass filter stop band frequency in absolute unit")
parser.add_option("--aa", type="float", default=.1, help="low pass filter falloff" )
parser.add_option("--mask", type="string", help="input mask file", default=None)
parser.add_option("--output", type="string", help="output file name", default="postprocessed.hdf")
parser.add_option("--pixel_size", type="float", help="pixel size of the data", default=1.0)
parser.add_option("--B_start", type="float", help="starting frequency in Angstrom for B-factor estimation", default=10.)
parser.add_option("--FSC_cutoff", type="float", help="stop frequency in Angstrom for B-factor estimation", default=0.143)
parser.add_option("--2d", action="store_true", help="postprocess isac 2-D averaged images",default=False)
parser.add_option("--window_stack", action="store_true", help="window stack images using a smaller window size", default=False)
parser.add_option("--box", type="int", default= 0, help="the new window size ")
(options, args) = parser.parse_args()
global_def.BATCH = True
if options.phase_flip:
nargs = len(args)
if nargs != 2:
print "must provide name of input and output file!"
return
from EMAN2 import Processor
instack = args[0]
outstack = args[1]
nima = EMUtil.get_image_count(instack)
from filter import filt_ctf
for i in xrange(nima):
img = EMData()
img.read_image(instack, i)
try:
ctf = img.get_attr('ctf')
except:
print "no ctf information in input stack! Exiting..."
return
dopad = True
sign = 1
binary = 1 # phase flip
assert img.get_ysize() > 1
dict = ctf.to_dict()
dz = dict["defocus"]
cs = dict["cs"]
voltage = dict["voltage"]
pixel_size = dict["apix"]
b_factor = dict["bfactor"]
ampcont = dict["ampcont"]
dza = dict["dfdiff"]
azz = dict["dfang"]
if dopad and not img.is_complex(): ip = 1
else: ip = 0
params = {"filter_type": Processor.fourier_filter_types.CTF_,
"defocus" : dz,
"Cs": cs,
"voltage": voltage,
"Pixel_size": pixel_size,
"B_factor": b_factor,
"amp_contrast": ampcont,
"dopad": ip,
"binary": binary,
"sign": sign,
"dza": dza,
"azz":azz}
tmp = Processor.EMFourierFilter(img, params)
tmp.set_attr_dict({"ctf": ctf})
tmp.write_image(outstack, i)
elif options.changesize:
nargs = len(args)
if nargs != 2:
ERROR("must provide name of input and output file!", "change size", 1)
return
from utilities import get_im
instack = args[0]
outstack = args[1]
sub_rate = float(options.ratio)
nima = EMUtil.get_image_count(instack)
from fundamentals import resample
for i in xrange(nima):
resample(get_im(instack, i), sub_rate).write_image(outstack, i)
elif options.isacgroup>-1:
nargs = len(args)
if nargs != 3:
ERROR("Three files needed on input!", "isacgroup", 1)
return
from utilities import get_im
instack = args[0]
m=get_im(args[1],int(options.isacgroup)).get_attr("members")
l = []
for k in m:
l.append(int(get_im(args[0],k).get_attr(options.params)))
from utilities import write_text_file
write_text_file(l, args[2])
elif options.isacselect:
nargs = len(args)
if nargs != 2:
ERROR("Two files needed on input!", "isacgroup", 1)
return
from utilities import get_im
nima = EMUtil.get_image_count(args[0])
m = []
for k in xrange(nima):
m += get_im(args[0],k).get_attr("members")
m.sort()
from utilities import write_text_file
write_text_file(m, args[1])
elif options.pw:
nargs = len(args)
if nargs < 2:
ERROR("must provide name of input and output file!", "pw", 1)
return
from utilities import get_im, write_text_file
from fundamentals import rops_table
d = get_im(args[0])
ndim = d.get_ndim()
if ndim ==3:
pw = rops_table(d)
write_text_file(pw, args[1])
else:
nx = d.get_xsize()
ny = d.get_ysize()
if nargs ==3: mask = get_im(args[2])
wn = int(options.wn)
if wn == -1:
wn = max(nx, ny)
else:
if( (wn<nx) or (wn<ny) ): ERROR("window size cannot be smaller than the image size","pw",1)
n = EMUtil.get_image_count(args[0])
from utilities import model_blank, model_circle, pad
from EMAN2 import periodogram
p = model_blank(wn,wn)
for i in xrange(n):
d = get_im(args[0], i)
if nargs==3:
d *=mask
st = Util.infomask(d, None, True)
d -= st[0]
p += periodogram(pad(d, wn, wn, 1, 0.))
p /= n
p.write_image(args[1])
elif options.adjpw:
if len(args) < 3:
ERROR("filt_by_rops input target output fl aa (the last two are optional parameters of a low-pass filter)","adjpw",1)
return
img_stack = args[0]
from math import sqrt
from fundamentals import rops_table, fft
from utilities import read_text_file, get_im
from filter import filt_tanl, filt_table
if( args[1][-3:] == 'txt'):
rops_dst = read_text_file( args[1] )
else:
rops_dst = rops_table(get_im( args[1] ))
out_stack = args[2]
if(len(args) >4):
fl = float(args[3])
aa = float(args[4])
else:
fl = -1.0
aa = 0.0
nimage = EMUtil.get_image_count( img_stack )
for i in xrange(nimage):
img = fft(get_im(img_stack, i) )
rops_src = rops_table(img)
assert len(rops_dst) == len(rops_src)
table = [0.0]*len(rops_dst)
for j in xrange( len(rops_dst) ):
table[j] = sqrt( rops_dst[j]/rops_src[j] )
if( fl > 0.0):
img = filt_tanl(img, fl, aa)
img = fft(filt_table(img, table))
img.write_image(out_stack, i)
elif options.rotpw != None:
if len(args) != 1:
ERROR("Only one input permitted","rotpw",1)
return
from utilities import write_text_file, get_im
from fundamentals import rops_table
from math import log10
t = rops_table(get_im(args[0]))
x = range(len(t))
r = [0.0]*len(x)
for i in x: r[i] = log10(t[i])
write_text_file([t,r,x],options.rotpw)
elif options.transformparams != None:
if len(args) != 2:
ERROR("Please provide names of input and output files with orientation parameters","transformparams",1)
return
from utilities import read_text_row, write_text_row
transf = [0.0]*6
spl=options.transformparams.split(',')
for i in xrange(len(spl)): transf[i] = float(spl[i])
write_text_row( rotate_shift_params(read_text_row(args[0]), transf) , args[1])
elif options.makedb != None:
nargs = len(args)
if nargs != 1:
print "must provide exactly one argument denoting database key under which the input params will be stored"
return
dbkey = args[0]
print "database key under which params will be stored: ", dbkey
gbdb = js_open_dict("e2boxercache/gauss_box_DB.json")
parmstr = 'dummy:'+options.makedb[0]
(processorname, param_dict) = parsemodopt(parmstr)
dbdict = {}
for pkey in param_dict:
if (pkey == 'invert_contrast') or (pkey == 'use_variance'):
if param_dict[pkey] == 'True':
dbdict[pkey] = True
else:
dbdict[pkey] = False
else:
dbdict[pkey] = param_dict[pkey]
gbdb[dbkey] = dbdict
elif options.generate_projections:
nargs = len(args)
if nargs != 3:
ERROR("Must provide name of input structure(s) from which to generate projections, name of output projection stack, and prefix for output micrographs."\
"sxprocess - generate projections",1)
return
inpstr = args[0]
outstk = args[1]
micpref = args[2]
parmstr = 'dummy:'+options.generate_projections[0]
(processorname, param_dict) = parsemodopt(parmstr)
parm_CTF = False
parm_format = 'bdb'
parm_apix = 2.5
if 'CTF' in param_dict:
if param_dict['CTF'] == 'True':
parm_CTF = True
if 'format' in param_dict:
parm_format = param_dict['format']
if 'apix' in param_dict:
parm_apix = float(param_dict['apix'])
boxsize = 64
if 'boxsize' in param_dict:
boxsize = int(param_dict['boxsize'])
print "pixel size: ", parm_apix, " format: ", parm_format, " add CTF: ", parm_CTF, " box size: ", boxsize
scale_mult = 2500
sigma_add = 1.5
sigma_proj = 30.0
sigma2_proj = 17.5
sigma_gauss = 0.3
sigma_mic = 30.0
sigma2_mic = 17.5
sigma_gauss_mic = 0.3
if 'scale_mult' in param_dict:
scale_mult = float(param_dict['scale_mult'])
if 'sigma_add' in param_dict:
sigma_add = float(param_dict['sigma_add'])
if 'sigma_proj' in param_dict:
sigma_proj = float(param_dict['sigma_proj'])
if 'sigma2_proj' in param_dict:
sigma2_proj = float(param_dict['sigma2_proj'])
if 'sigma_gauss' in param_dict:
sigma_gauss = float(param_dict['sigma_gauss'])
if 'sigma_mic' in param_dict:
sigma_mic = float(param_dict['sigma_mic'])
if 'sigma2_mic' in param_dict:
sigma2_mic = float(param_dict['sigma2_mic'])
if 'sigma_gauss_mic' in param_dict:
sigma_gauss_mic = float(param_dict['sigma_gauss_mic'])
from filter import filt_gaussl, filt_ctf
from utilities import drop_spider_doc, even_angles, model_gauss, delete_bdb, model_blank,pad,model_gauss_noise,set_params2D, set_params_proj
from projection import prep_vol,prgs
seed(14567)
delta = 29
angles = even_angles(delta, 0.0, 89.9, 0.0, 359.9, "S")
nangle = len(angles)
modelvol = []
nvlms = EMUtil.get_image_count(inpstr)
from utilities import get_im
for k in xrange(nvlms): modelvol.append(get_im(inpstr,k))
nx = modelvol[0].get_xsize()
if nx != boxsize:
ERROR("Requested box dimension does not match dimension of the input model.", \
"sxprocess - generate projections",1)
nvol = 10
volfts = [[] for k in xrange(nvlms)]
for k in xrange(nvlms):
for i in xrange(nvol):
sigma = sigma_add + random() # 1.5-2.5
addon = model_gauss(sigma, boxsize, boxsize, boxsize, sigma, sigma, 38, 38, 40 )
scale = scale_mult * (0.5+random())
vf, kb = prep_vol(modelvol[k] + scale*addon)
volfts[k].append(vf)
del vf, modelvol
if parm_format == "bdb":
stack_data = "bdb:"+outstk
delete_bdb(stack_data)
else:
stack_data = outstk + ".hdf"
Cs = 2.0
pixel = parm_apix
voltage = 120.0
ampcont = 10.0
ibd = 4096/2-boxsize
iprj = 0
width = 240
xstart = 8 + boxsize/2
ystart = 8 + boxsize/2
rowlen = 17
from random import randint
params = []
for idef in xrange(3, 8):
irow = 0
icol = 0
mic = model_blank(4096, 4096)
defocus = idef * 0.5#0.2
if parm_CTF:
astampl=defocus*0.15
astangl=50.0
ctf = generate_ctf([defocus, Cs, voltage, pixel, ampcont, 0.0, astampl, astangl])
for i in xrange(nangle):
for k in xrange(12):
dphi = 8.0*(random()-0.5)
dtht = 8.0*(random()-0.5)
psi = 360.0*random()
phi = angles[i][0]+dphi
tht = angles[i][1]+dtht
s2x = 4.0*(random()-0.5)
s2y = 4.0*(random()-0.5)
params.append([phi, tht, psi, s2x, s2y])
ivol = iprj % nvol
#imgsrc = randint(0,nvlms-1)
imgsrc = iprj % nvlms
proj = prgs(volfts[imgsrc][ivol], kb, [phi, tht, psi, -s2x, -s2y])
x = xstart + irow * width
y = ystart + icol * width
mic += pad(proj, 4096, 4096, 1, 0.0, x-2048, y-2048, 0)
proj = proj + model_gauss_noise( sigma_proj, nx, nx )
if parm_CTF:
proj = filt_ctf(proj, ctf)
proj.set_attr_dict({"ctf":ctf, "ctf_applied":0})
proj = proj + filt_gaussl(model_gauss_noise(sigma2_proj, nx, nx), sigma_gauss)
proj.set_attr("origimgsrc",imgsrc)
proj.set_attr("test_id", iprj)
# flags describing the status of the image (1 = true, 0 = false)
set_params2D(proj, [0.0, 0.0, 0.0, 0, 1.0])
set_params_proj(proj, [phi, tht, psi, s2x, s2y])
proj.write_image(stack_data, iprj)
icol += 1
if icol == rowlen:
icol = 0
irow += 1
iprj += 1
mic += model_gauss_noise(sigma_mic,4096,4096)
if parm_CTF:
#apply CTF
mic = filt_ctf(mic, ctf)
mic += filt_gaussl(model_gauss_noise(sigma2_mic, 4096, 4096), sigma_gauss_mic)
mic.write_image(micpref + "%1d.hdf" % (idef-3), 0)
drop_spider_doc("params.txt", params)
elif options.importctf != None:
print ' IMPORTCTF '
from utilities import read_text_row,write_text_row
from random import randint
import subprocess
grpfile = 'groupid%04d'%randint(1000,9999)
ctfpfile = 'ctfpfile%04d'%randint(1000,9999)
cterr = [options.defocuserror/100.0, options.astigmatismerror]
ctfs = read_text_row(options.importctf)
for kk in xrange(len(ctfs)):
root,name = os.path.split(ctfs[kk][-1])
ctfs[kk][-1] = name[:-4]
if(options.input[:4] != 'bdb:'):
ERROR('Sorry, only bdb files implemented','importctf',1)
d = options.input[4:]
#try: str = d.index('*')
#except: str = -1
from string import split
import glob
uu = os.path.split(d)
uu = os.path.join(uu[0],'EMAN2DB',uu[1]+'.bdb')
flist = glob.glob(uu)
for i in xrange(len(flist)):
root,name = os.path.split(flist[i])
root = root[:-7]
name = name[:-4]
fil = 'bdb:'+os.path.join(root,name)
sourcemic = EMUtil.get_all_attributes(fil,'ptcl_source_image')
nn = len(sourcemic)
gctfp = []
groupid = []
for kk in xrange(nn):
junk,name2 = os.path.split(sourcemic[kk])
name2 = name2[:-4]
ctfp = [-1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]
for ll in xrange(len(ctfs)):
if(name2 == ctfs[ll][-1]):
# found correct
if(ctfs[ll][8]/ctfs[ll][0] <= cterr[0]):
# acceptable defocus error
ctfp = ctfs[ll][:8]
if(ctfs[ll][10] > cterr[1] ):
# error of astigmatism exceed the threshold, set astigmatism to zero.
ctfp[6] = 0.0
ctfp[7] = 0.0
gctfp.append(ctfp)
groupid.append(kk)
break
if(len(groupid) > 0):
write_text_row(groupid, grpfile)
write_text_row(gctfp, ctfpfile)
cmd = "{} {} {} {}".format('e2bdb.py',fil,'--makevstack=bdb:'+root+'G'+name,'--list='+grpfile)
#print cmd
subprocess.call(cmd, shell=True)
cmd = "{} {} {} {}".format('sxheader.py','bdb:'+root+'G'+name,'--params=ctf','--import='+ctfpfile)
#print cmd
subprocess.call(cmd, shell=True)
else:
print ' >>> Group ',name,' skipped.'
cmd = "{} {} {}".format("rm -f",grpfile,ctfpfile)
subprocess.call(cmd, shell=True)
elif options.scale > 0.0:
from utilities import read_text_row,write_text_row
scale = options.scale
nargs = len(args)
if nargs != 2:
print "Please provide names of input and output file!"
return
p = read_text_row(args[0])
for i in xrange(len(p)):
p[i][3] /= scale
p[i][4] /= scale
write_text_row(p, args[1])
elif options.adaptive_mask:
from utilities import get_im
from morphology import adaptive_mask, binarize, erosion, dilation
nsigma = options.nsigma
ndilation = options.ndilation
kernel_size = options.kernel_size
gauss_standard_dev = options.gauss_standard_dev
nargs = len(args)
if nargs ==0:
print " Create 3D mask from a given volume, either automatically or from the user provided threshold."
elif nargs > 2:
print "Too many inputs are given, try again!"
return
else:
inputvol = get_im(args[0])
input_path, input_file_name = os.path.split(args[0])
input_file_name_root,ext=os.path.splitext(input_file_name)
if nargs == 2: mask_file_name = args[1]
else: mask_file_name = "adaptive_mask_for_"+input_file_name_root+".hdf" # Only hdf file is output.
if options.threshold !=9999.:
mask3d = binarize(inputvol, options.threshold)
for i in xrange(options.ne): mask3d = erosion(mask3d)
for i in xrange(options.nd): mask3d = dilation(mask3d)
else:
mask3d = adaptive_mask(inputvol, nsigma, ndilation, kernel_size, gauss_standard_dev)
mask3d.write_image(mask_file_name)
elif options.postprocess:
from utilities import get_im
from fundamentals import rot_avg_table
from morphology import compute_bfactor,power
from statistics import fsc
from filter import filt_table, filt_gaussinv
from EMAN2 import periodogram
e1 = get_im(args[0],0)
if e1.get_zsize()==1:
nimage = EMUtil.get_image_count(args[0])
if options.mask !=None: m = get_im(options.mask)
else: m = None
for i in xrange(nimage):
e1 = get_im(args[0],i)
if m: e1 *=m
guinerline = rot_avg_table(power(periodogram(e1),.5))
freq_max = 1/(2.*pixel_size)
freq_min = 1./options.B_start
b,junk=compute_bfactor(guinerline, freq_min, freq_max, pixel_size)
tmp = b/pixel_size**2
sigma_of_inverse=sqrt(2./tmp)
e1 = filt_gaussinv(e1,sigma_of_inverse)
if options.low_pass_filter:
from filter import filt_tanl
e1 =filt_tanl(e1,options.ff, options.aa)
e1.write_image(options.output)
else:
nargs = len(args)
e1 = get_im(args[0])
if nargs >1: e2 = get_im(args[1])
if options.mask !=None: m = get_im(options.mask)
else: m =None
pixel_size = options.pixel_size
from math import sqrt
if m !=None:
e1 *=m
if nargs >1 :e2 *=m
if options.fsc_weighted:
frc = fsc(e1,e2,1)
## FSC is done on masked two images
#### FSC weighting sqrt((2.*fsc)/(1+fsc));
fil = len(frc[1])*[None]
for i in xrange(len(fil)):
if frc[1][i]>=options.FSC_cutoff: tmp = frc[1][i]
else: tmp = 0.0
fil[i] = sqrt(2.*tmp/(1.+tmp))
if nargs>1: e1 +=e2
if options.fsc_weighted: e1=filt_table(e1,fil)
guinerline = rot_avg_table(power(periodogram(e1),.5))
freq_max = 1/(2.*pixel_size)
freq_min = 1./options.B_start
b,junk = compute_bfactor(guinerline, freq_min, freq_max, pixel_size)
tmp = b/pixel_size**2
sigma_of_inverse=sqrt(2./tmp)
e1 = filt_gaussinv(e1,sigma_of_inverse)
if options.low_pass_filter:
from filter import filt_tanl
e1 =filt_tanl(e1,options.ff, options.aa)
e1.write_image(options.output)
elif options.window_stack:
nargs = len(args)
if nargs ==0:
print " Reduce image size of a stack"
return
else:
output_stack_name = None
inputstack = args[0]
if nargs ==2:output_stack_name = args[1]
input_path,input_file_name=os.path.split(inputstack)
input_file_name_root,ext=os.path.splitext(input_file_name)
if input_file_name_root[0:3]=="bdb":stack_is_bdb= True
else: stack_is_bdb= False
if output_stack_name is None:
if stack_is_bdb: output_stack_name ="bdb:reduced_"+input_file_name_root[4:]
else:output_stack_name = "reduced_"+input_file_name_root+".hdf" # Only hdf file is output.
nimage = EMUtil.get_image_count(inputstack)
from fundamentals import window2d
for i in xrange(nimage):
image = EMData()
image.read_image(inputstack,i)
w = window2d(image,options.box,options.box)
w.write_image(output_stack_name,i)
else: ERROR("Please provide option name","sxprocess.py",1)
def helicalshiftali_MPI(stack, maskfile=None, maxit=100, CTF=False, snr=1.0, Fourvar=False, search_rng=-1):
from applications import MPI_start_end
from utilities import model_circle, model_blank, get_image, peak_search, get_im, pad
from utilities import reduce_EMData_to_root, bcast_EMData_to_all, send_attr_dict, file_type, bcast_number_to_all, bcast_list_to_all
from statistics import varf2d_MPI
from fundamentals import fft, ccf, rot_shift3D, rot_shift2D, fshift
from utilities import get_params2D, set_params2D, chunks_distribution
from utilities import print_msg, print_begin_msg, print_end_msg
import os
import sys
from mpi import mpi_init, mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD
from mpi import mpi_reduce, mpi_bcast, mpi_barrier, mpi_gatherv
from mpi import MPI_SUM, MPI_FLOAT, MPI_INT
from time import time
from pixel_error import ordersegments
from math import sqrt, atan2, tan, pi
nproc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
ftp = file_type(stack)
if myid == main_node:
print_begin_msg("helical-shiftali_MPI")
max_iter=int(maxit)
if( myid == main_node):
infils = EMUtil.get_all_attributes(stack, "filament")
ptlcoords = EMUtil.get_all_attributes(stack, 'ptcl_source_coord')
filaments = ordersegments(infils, ptlcoords)
total_nfils = len(filaments)
inidl = [0]*total_nfils
for i in xrange(total_nfils): inidl[i] = len(filaments[i])
linidl = sum(inidl)
nima = linidl
tfilaments = []
for i in xrange(total_nfils): tfilaments += filaments[i]
del filaments
else:
total_nfils = 0
linidl = 0
total_nfils = bcast_number_to_all(total_nfils, source_node = main_node)
if myid != main_node:
inidl = [-1]*total_nfils
inidl = bcast_list_to_all(inidl, myid, source_node = main_node)
linidl = bcast_number_to_all(linidl, source_node = main_node)
if myid != main_node:
tfilaments = [-1]*linidl
tfilaments = bcast_list_to_all(tfilaments, myid, source_node = main_node)
filaments = []
iendi = 0
for i in xrange(total_nfils):
isti = iendi
iendi = isti+inidl[i]
filaments.append(tfilaments[isti:iendi])
del tfilaments,inidl
if myid == main_node:
print_msg( "total number of filaments: %d"%total_nfils)
if total_nfils< nproc:
ERROR('number of CPUs (%i) is larger than the number of filaments (%i), please reduce the number of CPUs used'%(nproc, total_nfils), "ehelix_MPI", 1,myid)
# balanced load
temp = chunks_distribution([[len(filaments[i]), i] for i in xrange(len(filaments))], nproc)[myid:myid+1][0]
filaments = [filaments[temp[i][1]] for i in xrange(len(temp))]
nfils = len(filaments)
#filaments = [[0,1]]
#print "filaments",filaments
list_of_particles = []
indcs = []
k = 0
for i in xrange(nfils):
list_of_particles += filaments[i]
k1 = k+len(filaments[i])
indcs.append([k,k1])
k = k1
data = EMData.read_images(stack, list_of_particles)
ldata = len(data)
print "ldata=", ldata
nx = data[0].get_xsize()
ny = data[0].get_ysize()
if maskfile == None:
mrad = min(nx, ny)//2-2
mask = pad( model_blank(2*mrad+1, ny, 1, 1.0), nx, ny, 1, 0.0)
else:
mask = get_im(maskfile)
# apply initial xform.align2d parameters stored in header
init_params = []
for im in xrange(ldata):
t = data[im].get_attr('xform.align2d')
init_params.append(t)
p = t.get_params("2d")
data[im] = rot_shift2D(data[im], p['alpha'], p['tx'], p['ty'], p['mirror'], p['scale'])
if CTF:
from filter import filt_ctf
from morphology import ctf_img
ctf_abs_sum = EMData(nx, ny, 1, False)
ctf_2_sum = EMData(nx, ny, 1, False)
else:
ctf_2_sum = None
ctf_abs_sum = None
from utilities import info
for im in xrange(ldata):
data[im].set_attr('ID', list_of_particles[im])
st = Util.infomask(data[im], mask, False)
data[im] -= st[0]
if CTF:
ctf_params = data[im].get_attr("ctf")
qctf = data[im].get_attr("ctf_applied")
if qctf == 0:
data[im] = filt_ctf(fft(data[im]), ctf_params)
data[im].set_attr('ctf_applied', 1)
elif qctf != 1:
ERROR('Incorrectly set qctf flag', "helicalshiftali_MPI", 1,myid)
ctfimg = ctf_img(nx, ctf_params, ny=ny)
Util.add_img2(ctf_2_sum, ctfimg)
Util.add_img_abs(ctf_abs_sum, ctfimg)
else: data[im] = fft(data[im])
del list_of_particles
if CTF:
reduce_EMData_to_root(ctf_2_sum, myid, main_node)
reduce_EMData_to_root(ctf_abs_sum, myid, main_node)
if CTF:
if myid != main_node:
del ctf_2_sum
del ctf_abs_sum
else:
temp = EMData(nx, ny, 1, False)
tsnr = 1./snr
for i in xrange(0,nx+2,2):
for j in xrange(ny):
temp.set_value_at(i,j,tsnr)
temp.set_value_at(i+1,j,0.0)
#info(ctf_2_sum)
Util.add_img(ctf_2_sum, temp)
#info(ctf_2_sum)
del temp
total_iter = 0
shift_x = [0.0]*ldata
for Iter in xrange(max_iter):
if myid == main_node:
start_time = time()
print_msg("Iteration #%4d\n"%(total_iter))
total_iter += 1
avg = EMData(nx, ny, 1, False)
for im in xrange(ldata):
Util.add_img(avg, fshift(data[im], shift_x[im]))
reduce_EMData_to_root(avg, myid, main_node)
if myid == main_node:
if CTF: tavg = Util.divn_filter(avg, ctf_2_sum)
else: tavg = Util.mult_scalar(avg, 1.0/float(nima))
else:
tavg = model_blank(nx,ny)
if Fourvar:
bcast_EMData_to_all(tavg, myid, main_node)
vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF)
if myid == main_node:
if Fourvar:
tavg = fft(Util.divn_img(fft(tavg), vav))
vav_r = Util.pack_complex_to_real(vav)
# normalize and mask tavg in real space
tavg = fft(tavg)
stat = Util.infomask( tavg, mask, False )
tavg -= stat[0]
Util.mul_img(tavg, mask)
tavg.write_image("tavg.hdf",Iter)
# For testing purposes: shift tavg to some random place and see if the centering is still correct
#tavg = rot_shift3D(tavg,sx=3,sy=-4)
if Fourvar: del vav
bcast_EMData_to_all(tavg, myid, main_node)
tavg = fft(tavg)
sx_sum = 0.0
nxc = nx//2
for ifil in xrange(nfils):
"""
# Calculate filament average
avg = EMData(nx, ny, 1, False)
filnima = 0
for im in xrange(indcs[ifil][0], indcs[ifil][1]):
Util.add_img(avg, data[im])
filnima += 1
tavg = Util.mult_scalar(avg, 1.0/float(filnima))
"""
# Calculate 1D ccf between each segment and filament average
nsegms = indcs[ifil][1]-indcs[ifil][0]
ctx = [None]*nsegms
pcoords = [None]*nsegms
for im in xrange(indcs[ifil][0], indcs[ifil][1]):
ctx[im-indcs[ifil][0]] = Util.window(ccf(tavg, data[im]), nx, 1)
pcoords[im-indcs[ifil][0]] = data[im].get_attr('ptcl_source_coord')
#ctx[im-indcs[ifil][0]].write_image("ctx.hdf",im-indcs[ifil][0])
#print " CTX ",myid,im,Util.infomask(ctx[im-indcs[ifil][0]], None, True)
# search for best x-shift
cents = nsegms//2
dst = sqrt(max((pcoords[cents][0] - pcoords[0][0])**2 + (pcoords[cents][1] - pcoords[0][1])**2, (pcoords[cents][0] - pcoords[-1][0])**2 + (pcoords[cents][1] - pcoords[-1][1])**2))
maxincline = atan2(ny//2-2-float(search_rng),dst)
kang = int(dst*tan(maxincline)+0.5)
#print " settings ",nsegms,cents,dst,search_rng,maxincline,kang
# ## C code for alignment. @ming
results = [0.0]*3;
results = Util.helixshiftali(ctx, pcoords, nsegms, maxincline, kang, search_rng,nxc)
sib = int(results[0])
bang = results[1]
qm = results[2]
#print qm, sib, bang
# qm = -1.e23
#
# for six in xrange(-search_rng, search_rng+1,1):
# q0 = ctx[cents].get_value_at(six+nxc)
# for incline in xrange(kang+1):
# qt = q0
# qu = q0
# if(kang>0): tang = tan(maxincline/kang*incline)
# else: tang = 0.0
# for kim in xrange(cents+1,nsegms):
# dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
# xl = dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# #print " A ", ifil,six,incline,kim,xl,ixl,dxl
# qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# xl = -dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# for kim in xrange(cents):
# dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
# xl = -dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# xl = dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# if( qt > qm ):
# qm = qt
# sib = six
# bang = tang
# if( qu > qm ):
# qm = qu
# sib = six
# bang = -tang
#if incline == 0: print "incline = 0 ",six,tang,qt,qu
#print qm,six,sib,bang
#print " got results ",indcs[ifil][0], indcs[ifil][1], ifil,myid,qm,sib,tang,bang,len(ctx),Util.infomask(ctx[0], None, True)
for im in xrange(indcs[ifil][0], indcs[ifil][1]):
kim = im-indcs[ifil][0]
dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
if(kim < cents): xl = -dst*bang+sib
else: xl = dst*bang+sib
shift_x[im] = xl
# Average shift
sx_sum += shift_x[indcs[ifil][0]+cents]
# #print myid,sx_sum,total_nfils
sx_sum = mpi_reduce(sx_sum, 1, MPI_FLOAT, MPI_SUM, main_node, MPI_COMM_WORLD)
if myid == main_node:
sx_sum = float(sx_sum[0])/total_nfils
print_msg("Average shift %6.2f\n"%(sx_sum))
else:
sx_sum = 0.0
sx_sum = 0.0
sx_sum = bcast_number_to_all(sx_sum, source_node = main_node)
for im in xrange(ldata):
shift_x[im] -= sx_sum
#print " %3d %6.3f"%(im,shift_x[im])
#exit()
# combine shifts found with the original parameters
for im in xrange(ldata):
t1 = Transform()
##import random
##shix=random.randint(-10, 10)
##t1.set_params({"type":"2D","tx":shix})
t1.set_params({"type":"2D","tx":shift_x[im]})
# combine t0 and t1
tt = t1*init_params[im]
data[im].set_attr("xform.align2d", tt)
# write out headers and STOP, under MPI writing has to be done sequentially
mpi_barrier(MPI_COMM_WORLD)
par_str = ["xform.align2d", "ID"]
if myid == main_node:
from utilities import file_type
if(file_type(stack) == "bdb"):
from utilities import recv_attr_dict_bdb
recv_attr_dict_bdb(main_node, stack, data, par_str, 0, ldata, nproc)
else:
from utilities import recv_attr_dict
recv_attr_dict(main_node, stack, data, par_str, 0, ldata, nproc)
else: send_attr_dict(main_node, data, par_str, 0, ldata)
if myid == main_node: print_end_msg("helical-shiftali_MPI")
def main(args):
progname = os.path.basename(sys.argv[0])
usage = ( progname + " stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter" +
" --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp" +
" --generation=generation --rand_seed=rand_seed>" )
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--radius", type="int", default=-1, help="<Particle radius>, it has to be provided.")
parser.add_option("--img_per_grp", type="int", default=100, help="<number of images per group> in the ideal case (essentially maximum size of class) (100)")
parser.add_option("--CTF", action="store_true", default=False, help="<CTF flag>, if set the data will be phase-flipped")
parser.add_option("--ir", type="int", default=1, help="<inner ring> of the resampling to polar coordinates (1)")
parser.add_option("--rs", type="int", default=1, help="<ring step> of the resampling to polar coordinates (1)")
parser.add_option("--xr", type="int", default=-1, help="<x range> of translational search (By default set by the program) (advanced)")
parser.add_option("--yr", type="int", default=-1, help="<y range> of translational search (same as xr) (advanced)")
parser.add_option("--ts", type="float", default=1.0, help="<search step> of translational search (1.0)")
parser.add_option("--maxit", type="int", default=30, help="number of iterations for reference-free alignment (30)")
#parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)")
parser.add_option("--center_method", type="int", default=7, help="<Method for centering> of global 2D average during initial prealignment of data (default : 7; 0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7)")
parser.add_option("--dst", type="float", default=90.0, help="discrete angle used in within group alignment ")
parser.add_option("--FL", type="float", default=0.2, help="<lowest stopband> frequency used in the tangent filter (0.2)")
parser.add_option("--FH", type="float", default=0.3, help="<highest stopband> frequency used in the tangent filter (0.3)")
parser.add_option("--FF", type="float", default=0.2, help="<fall-off of the tangent> filter (0.2)")
parser.add_option("--init_iter", type="int", default=3, help="<init_iter> number of iterations of ISAC program in initialization (3)")
parser.add_option("--main_iter", type="int", default=3, help="<main_iter> number of iterations of ISAC program in main part (3)")
parser.add_option("--iter_reali", type="int", default=1, help="<iter_reali> number of iterations in ISAC before checking stability (1)")
parser.add_option("--match_first", type="int", default=1, help="number of iterations to run 2-way matching in the first phase (1)")
parser.add_option("--max_round", type="int", default=20, help="maximum rounds of generating candidate averages in the first phase (20)")
parser.add_option("--match_second", type="int", default=5, help="number of iterations to run 2-way (or 3-way) matching in the second phase (5)")
parser.add_option("--stab_ali", type="int", default=5, help="number of alignments when checking stability (5)")
parser.add_option("--thld_err", type="float", default=0.7, help="the threshold of pixel error when checking stability (0.7)")
parser.add_option("--indep_run", type="int", default=4, help="number of independent runs for reproducibility (default=4, only values 2, 3 and 4 are supported (4)")
parser.add_option("--thld_grp", type="int", default=10, help="minimum size of class (10)")
parser.add_option("--n_generations", type="int", default=100, help="<n_generations> program stops when reaching this total number of generations (advanced)")
#parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)")
parser.add_option("--rand_seed", type="int", default=None, help="random seed set before calculations, useful for testing purposes (default None - total randomness)")
parser.add_option("--new", action="store_true", default=False, help="use new code (default = False)")
parser.add_option("--debug", action="store_true", default=False, help="debug info printout (default = False)")
# must be switched off in production
parser.add_option("--use_latest_master_directory", action="store_true", dest="use_latest_master_directory", default=False)
parser.add_option("--restart_section", type="string", default="", help="<restart section name> (no spaces) followed immediately by comma, followed immediately by generation to restart, example: \n--restart_section=candidate_class_averages,1 (Sections: restart, candidate_class_averages, reproducible_class_averages)")
parser.add_option("--stop_after_candidates", action="store_true", default=False, help="<stop_after_candidates> stops after the 'candidate_class_averages' section")
parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help = SUPPRESS_HELP)
(options, args) = parser.parse_args(args)
if options.return_options:
return parser
if len(args) > 2:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
sys.exit()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
from isac import iter_isac
global_def.BATCH = True
global_def.BATCH = True
command_line_provided_stack_filename = args[0]
global_def.BATCH = True
main_node = 0
mpi_init(0, [])
myid = mpi_comm_rank(MPI_COMM_WORLD)
nproc = mpi_comm_size(MPI_COMM_WORLD)
radi = options.radius
center_method = options.center_method
if(radi < 1): ERROR("Particle radius has to be provided!","sxisac",1,myid)
use_latest_master_directory = options.use_latest_master_directory
stop_after_candidates = options.stop_after_candidates
program_state_stack.restart_location_title_from_command_line = options.restart_section
from utilities import qw
program_state_stack.PROGRAM_STATE_VARIABLES = set(qw("""
isac_generation
"""))
# create or reuse master directory
masterdir = ""
stack_processed_by_ali2d_base__filename = ""
stack_processed_by_ali2d_base__filename__without_master_dir = ""
error_status = 0
if len(args) == 2:
masterdir = args[1]
elif len(args) == 1:
if use_latest_master_directory:
all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)]
import re; r = re.compile("^master.*$")
all_dirs = filter(r.match, all_dirs)
if len(all_dirs)>0:
# all_dirs = max(all_dirs, key=os.path.getctime)
masterdir = max(all_dirs, key=os.path.getmtime)
#Create folder for all results or check if there is one created already
if(myid == main_node):
if( masterdir == ""):
timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime())
masterdir = "master"+timestring
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
elif not os.path.exists(masterdir):
# os.path.exists(masterdir) does not exist
masterdir = args[1]
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
if(args[0][:4] == "bdb:"): filename = args[0][4:]
else: filename = args[0][:-4]
filename = os.path.basename(filename)
stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join(masterdir, filename )
stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename
if_error_all_processes_quit_program(error_status)
# send masterdir to all processes
masterdir = send_string_to_all(masterdir)
if myid == 0:
if options.restart_section != "":
if os.path.exists(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)):
stored_stack, stored_state = restore_program_stack_and_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE))
import re
if "," in options.restart_section:
parsed_restart_section_option = options.restart_section.split(",")
stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%parsed_restart_section_option[0], stored_state[-1]["location_in_program"])
generation_str_format = parsed_restart_section_option[1]
if generation_str_format != "":
isac_generation_from_command_line = int(generation_str_format)
stored_state[-1]["isac_generation"] = isac_generation_from_command_line
else:
isac_generation_from_command_line = 1
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = -1
stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%options.restart_section, stored_state[-1]["location_in_program"])
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
store_program_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE), stored_state, stored_stack)
else:
print "Please remove the restart_section option from the command line. The program must be started from the beginning."
mpi_finalize()
sys.exit()
else:
isac_generation_from_command_line = -1
program_state_stack(locals(), getframeinfo(currentframe()), os.path.join(masterdir,NAME_OF_JSON_STATE_FILE))
stack_processed_by_ali2d_base__filename = send_string_to_all(stack_processed_by_ali2d_base__filename)
stack_processed_by_ali2d_base__filename__without_master_dir = \
send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir)
# PARAMETERS OF THE PROCEDURE
if( options.xr == -1 ):
# Default values
target_nx = 76
target_radius = 29
target_xr = 1
else: # nx//2
# Check below!
target_xr = options.xr
target_nx = 76 + target_xr - 1 # subtract one, which is default
target_radius = 29
mpi_barrier(MPI_COMM_WORLD)
# Initialization of stacks
if(myid == main_node):
number_of_images_in_stack = EMUtil.get_image_count(command_line_provided_stack_filename)
else:
number_of_images_in_stack = 0
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node)
nxrsteps = 4
init2dir = os.path.join(masterdir,"2dalignment")
if(myid == 0):
import subprocess
from logger import Logger, BaseLogger_Files
# Create output directory
log2d = Logger(BaseLogger_Files())
log2d.prefix = os.path.join(init2dir)
cmd = "mkdir -p "+log2d.prefix
outcome = subprocess.call(cmd, shell=True)
log2d.prefix += "/"
# outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True)
else:
outcome = 0
log2d = None
if(myid == main_node):
a = get_im(command_line_provided_stack_filename)
nnxo = a.get_xsize()
else:
nnxo = 0
nnxo = bcast_number_to_all(nnxo, source_node = main_node)
txrm = (nnxo - 2*(radi+1))//2
if(txrm < 0): ERROR( "ERROR!! Radius of the structure larger than the window data size permits %d"%(radi), "sxisac",1, myid)
if(txrm/nxrsteps>0):
tss = ""
txr = ""
while(txrm/nxrsteps>0):
tts=txrm/nxrsteps
tss += " %d"%tts
txr += " %d"%(tts*nxrsteps)
txrm =txrm//2
else:
tss = "1"
txr = "%d"%txrm
# section ali2d_base
# centering method is set to #7
params2d, aligned_images = ali2d_base(command_line_provided_stack_filename, init2dir, None, 1, radi, 1, txr, txr, tss, \
False, 90.0, center_method, 14, options.CTF, 1.0, False, \
"ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False)
if( myid == main_node ):
write_text_row(params2d,os.path.join(init2dir, "initial2Dparams.txt"))
del params2d
mpi_barrier(MPI_COMM_WORLD)
# We assume the target image size will be target_nx, radius will be 29, and xr = 1.
# Note images can be also padded, in which case shrink_ratio > 1.
shrink_ratio = float(target_radius)/float(radi)
nx = aligned_images[0].get_xsize()
nima = len(aligned_images)
newx = int(nx*shrink_ratio + 0.5)
from fundamentals import rot_shift2D, resample
from utilities import pad, combine_params2
if(shrink_ratio < 1.0):
if newx > target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx :
msk = model_circle(newx//2-2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
elif(shrink_ratio == 1.0):
if newx > target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx :
msk = model_circle(nx//2-2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
#aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
elif(shrink_ratio > 1.0):
target_radius = radi
msk = model_circle(target_radius, nx, nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
del msk
gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images, myid)
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node)
if( myid == main_node ):
for i in range(number_of_images_in_stack): aligned_images[i].write_image(stack_processed_by_ali2d_base__filename,i)
# It has to be explicitly closed
from EMAN2db import db_open_dict
DB = db_open_dict(stack_processed_by_ali2d_base__filename)
DB.close()
fp = open("README_shrink_ratio.txt", "w")
output_text = """
Since, for processing purposes, isac changes the image dimensions,
adjustment of pixel size needs to be made in subsequent steps, (e.g.
running sxviper.py). The shrink ratio for this particular isac run is
--------
%.5f
--------
To get the pixel size for the isac output the user needs to divide
the original pixel size by the above value. This info is saved in
the following file: README_shrink_ratio.txt
"""%shrink_ratio
fp.write(output_text); fp.flush() ;fp.close()
print output_text
mpi_barrier(MPI_COMM_WORLD)
global_def.BATCH = True
os.chdir(masterdir)
if program_state_stack(locals(), getframeinfo(currentframe())):
# if 1:
pass
if (myid == main_node):
cmdexecute("sxheader.py --consecutive --params=originalid %s"%stack_processed_by_ali2d_base__filename__without_master_dir)
cmdexecute("e2bdb.py %s --makevstack=%s_000"%(stack_processed_by_ali2d_base__filename__without_master_dir, stack_processed_by_ali2d_base__filename__without_master_dir))
if (myid == main_node):
main_dir_no = get_latest_directory_increment_value("./", NAME_OF_MAIN_DIR, myformat="%04d")
print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no
if isac_generation_from_command_line < 0:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = -1
if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no:
for i in xrange(isac_generation_from_command_line+1, main_dir_no + 1):
if i == isac_generation_from_command_line+1:
backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%i + " 000_backup" + "%05d"%backup_dir_no)
cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%i)
# it includes both command line and json file
my_restart_section = stored_state[-1]["location_in_program"].split("___")[-1]
if "restart" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no)
cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%isac_generation_from_command_line )
elif "candidate_class_averages" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no)
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line)
# cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line)
elif "reproducible_class_averages" in my_restart_section:
cmdexecute("rm -rf " + NAME_OF_MAIN_DIR + "%04d/ali_params_generation_*"%isac_generation_from_command_line)
cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_generation*"%isac_generation_from_command_line)
else:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 0
isac_generation_from_command_line = mpi_bcast(isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]
isac_generation = isac_generation_from_command_line - 1
if (myid == main_node):
if isac_generation == 0:
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation)
write_text_file([1], os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_accounted.txt"%isac_generation))
write_text_file(range(number_of_images_in_stack), os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_unaccounted.txt"%isac_generation))
# Stopping criterion should be inside the program.
while True:
isac_generation += 1
if isac_generation > options.n_generations:
break
data64_stack_current = "bdb:../"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d"%isac_generation
if(myid == main_node):
accounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1),"generation_%d_accounted.txt"%(isac_generation - 1)))
number_of_accounted_images = len(accounted_images)
# unaccounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1),"generation_%d_unaccounted.txt"%(isac_generation - 1)))
# number_of_unaccounted_images = len(unaccounted_images)
else:
number_of_accounted_images = 0
number_of_accounted_images = int(mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
if number_of_accounted_images == 0:
os.chdir("..")
break
program_state_stack.restart_location_title = "restart"
if program_state_stack(locals(), getframeinfo(currentframe())):
if (myid == main_node):
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation)
# reference the original stack
list_file = os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1), "generation_%d_unaccounted.txt"%(isac_generation - 1))
cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\
stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file))
mpi_barrier(MPI_COMM_WORLD)
os.chdir(NAME_OF_MAIN_DIR + "%04d"%isac_generation)
program_state_stack.restart_location_title = "candidate_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new)
# program_state_stack.restart_location_title = "stopped_program1"
# program_state_stack(locals(), getframeinfo(currentframe()))
program_state_stack.restart_location_title = "stop_after_candidates"
program_state_stack(locals(), getframeinfo(currentframe()))
if stop_after_candidates:
mpi_finalize()
sys.exit()
exit_program = 0
if(myid == main_node):
if not os.path.exists("class_averages_candidate_generation_%d.hdf"%isac_generation):
print "This generation (%d) no class averages were generated!"%isac_generation
exit_program = 1
exit_program = int(mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
if exit_program:
os.chdir("..")
break
program_state_stack.restart_location_title = "reproducible_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new)
pass
os.chdir("..")
if (myid == main_node):
cmdexecute("rm -f class_averages.hdf")
cpy(["generation_%04d/class_averages_generation_%d.hdf"%(i,i) for i in xrange(1, isac_generation)], "class_averages.hdf")
# program_state_stack.restart_location_title = "stopped_program2"
# program_state_stack(locals(), getframeinfo(currentframe()))
program_state_stack(locals(), getframeinfo(currentframe()), last_call="__LastCall")
mpi_finalize()
def main(args):
progname = os.path.basename(sys.argv[0])
usage = (
progname +
" stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter"
+
" --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp"
+ " --generation=generation --rand_seed=rand_seed>")
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option(
"--radius",
type="int",
help=
"particle radius: there is no default, a sensible number has to be provided, units - pixels (default required int)"
)
parser.add_option(
"--target_radius",
type="int",
default=29,
help=
"target particle radius: actual particle radius on which isac will process data. Images will be shrinked/enlarged to achieve this radius (default 29)"
)
parser.add_option(
"--target_nx",
type="int",
default=76,
help=
"target particle image size: actual image size on which isac will process data. Images will be shrinked/enlarged according to target particle radius and then cut/padded to achieve target_nx size. When xr > 0, the final image size for isac processing is 'target_nx + xr - 1' (default 76)"
)
parser.add_option(
"--img_per_grp",
type="int",
default=100,
help=
"number of images per class: in the ideal case (essentially maximum size of class) (default 100)"
)
parser.add_option(
"--CTF",
action="store_true",
default=False,
help=
"apply phase-flip for CTF correction: if set the data will be phase-flipped using CTF information included in image headers (default False)"
)
parser.add_option(
"--ir",
type="int",
default=1,
help=
"inner ring: of the resampling to polar coordinates. units - pixels (default 1)"
)
parser.add_option(
"--rs",
type="int",
default=1,
help=
"ring step: of the resampling to polar coordinates. units - pixels (default 1)"
)
parser.add_option(
"--xr",
type="int",
default=1,
help=
"x range: of translational search. By default, set by the program. (default 1)"
)
parser.add_option(
"--yr",
type="int",
default=-1,
help=
"y range: of translational search. By default, same as xr. (default -1)"
)
parser.add_option(
"--ts",
type="float",
default=1.0,
help=
"search step: of translational search: units - pixels (default 1.0)")
parser.add_option(
"--maxit",
type="int",
default=30,
help="number of iterations for reference-free alignment: (default 30)")
#parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)")
parser.add_option(
"--center_method",
type="int",
default=-1,
help=
"method for centering: of global 2D average during initial prealignment of data (0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7) (default -1)"
)
parser.add_option(
"--dst",
type="float",
default=90.0,
help="discrete angle used in within group alignment: (default 90.0)")
parser.add_option(
"--FL",
type="float",
default=0.2,
help=
"lowest stopband: frequency used in the tangent filter (default 0.2)")
parser.add_option(
"--FH",
type="float",
default=0.3,
help=
"highest stopband: frequency used in the tangent filter (default 0.3)")
parser.add_option("--FF",
type="float",
default=0.2,
help="fall-off of the tangent filter: (default 0.2)")
parser.add_option(
"--init_iter",
type="int",
default=3,
help=
"SAC initialization iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC initialization (default 3)"
)
parser.add_option(
"--main_iter",
type="int",
default=3,
help=
"SAC main iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC (default 3)"
)
parser.add_option(
"--iter_reali",
type="int",
default=1,
help=
"SAC stability check interval: every iter_reali iterations of SAC stability checking is performed (default 1)"
)
parser.add_option(
"--match_first",
type="int",
default=1,
help=
"number of iterations to run 2-way matching in the first phase: (default 1)"
)
parser.add_option(
"--max_round",
type="int",
default=20,
help=
"maximum rounds: of generating candidate class averages in the first phase (default 20)"
)
parser.add_option(
"--match_second",
type="int",
default=5,
help=
"number of iterations to run 2-way (or 3-way) matching in the second phase: (default 5)"
)
parser.add_option(
"--stab_ali",
type="int",
default=5,
help="number of alignments when checking stability: (default 5)")
parser.add_option(
"--thld_err",
type="float",
default=0.7,
help=
"threshold of pixel error when checking stability: equals root mean square of distances between corresponding pixels from set of found transformations and theirs average transformation, depends linearly on square of radius (parameter ou). units - pixels. (default 0.7)"
)
parser.add_option(
"--indep_run",
type="int",
default=4,
help=
"level of m-way matching for reproducibility tests: By default, perform full ISAC to 4-way matching. Value indep_run=2 will restrict ISAC to 2-way matching and 3 to 3-way matching. Note the number of used MPI processes requested in mpirun must be a multiplicity of indep_run. (default 4)"
)
parser.add_option("--thld_grp",
type="int",
default=10,
help="minimum size of reproducible class (default 10)")
parser.add_option(
"--n_generations",
type="int",
default=10,
help=
"maximum number of generations: program stops when reaching this total number of generations: (default 10)"
)
#parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)")
parser.add_option(
"--rand_seed",
type="int",
help=
"random seed set before calculations: useful for testing purposes. By default, total randomness (type int)"
)
parser.add_option("--new",
action="store_true",
default=False,
help="use new code: (default False)")
parser.add_option("--debug",
action="store_true",
default=False,
help="debug info printout: (default False)")
# must be switched off in production
parser.add_option(
"--use_latest_master_directory",
action="store_true",
default=False,
help=
"use latest master directory: when active, the program looks for the latest directory that starts with the word 'master', so the user does not need to provide a directory name. (default False)"
)
parser.add_option(
"--restart_section",
type="string",
default=' ',
help=
"restart section: each generation (iteration) contains three sections: 'restart', 'candidate_class_averages', and 'reproducible_class_averages'. To restart from a particular step, for example, generation 4 and section 'candidate_class_averages' the following option is needed: '--restart_section=candidate_class_averages,4'. The option requires no white space before or after the comma. The default behavior is to restart execution from where it stopped intentionally or unintentionally. For default restart, it is assumed that the name of the directory is provided as argument. Alternatively, the '--use_latest_master_directory' option can be used. (default ' ')"
)
parser.add_option(
"--stop_after_candidates",
action="store_true",
default=False,
help=
"stop after candidates: stops after the 'candidate_class_averages' section. (default False)"
)
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--return_options",
action="store_true",
dest="return_options",
default=False,
help=SUPPRESS_HELP)
parser.add_option(
"--skip_prealignment",
action="store_true",
default=False,
help=
"skip pre-alignment step: to be used if images are already centered. 2dalignment directory will still be generated but the parameters will be zero. (default False)"
)
required_option_list = ['radius']
(options, args) = parser.parse_args(args)
if options.return_options:
return parser
if len(args) > 2:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
sys.exit()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
global_def.BATCH = True
from isac import iter_isac
from fundamentals import rot_shift2D, resample
from utilities import pad, combine_params2
command_line_provided_stack_filename = args[0]
main_node = 0
mpi_init(0, [])
myid = mpi_comm_rank(MPI_COMM_WORLD)
nproc = mpi_comm_size(MPI_COMM_WORLD)
mpi_barrier(MPI_COMM_WORLD)
if (myid == main_node):
print "****************************************************************"
Util.version()
print "****************************************************************"
sys.stdout.flush()
mpi_barrier(MPI_COMM_WORLD)
# Making sure all required options appeared.
for required_option in required_option_list:
if not options.__dict__[required_option]:
print "\n ==%s== mandatory option is missing.\n" % required_option
print "Please run '" + progname + " -h' for detailed options"
return 1
radi = options.radius
target_radius = options.target_radius
target_nx = options.target_nx
center_method = options.center_method
if (radi < 1):
ERROR("Particle radius has to be provided!", "sxisac", 1, myid)
use_latest_master_directory = options.use_latest_master_directory
stop_after_candidates = options.stop_after_candidates
# program_state_stack.restart_location_title_from_command_line = options.restart_section
from utilities import qw
program_state_stack.PROGRAM_STATE_VARIABLES = set(
qw("""
isac_generation
"""))
# create or reuse master directory
masterdir = ""
stack_processed_by_ali2d_base__filename = ""
stack_processed_by_ali2d_base__filename__without_master_dir = ""
error_status = 0
if len(args) == 2:
masterdir = args[1]
elif len(args) == 1:
if use_latest_master_directory:
all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)]
import re
r = re.compile("^master.*$")
all_dirs = filter(r.match, all_dirs)
if len(all_dirs) > 0:
# all_dirs = max(all_dirs, key=os.path.getctime)
masterdir = max(all_dirs, key=os.path.getmtime)
#Create folder for all results or check if there is one created already
if (myid == main_node):
if (masterdir == ""):
timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM,
localtime())
masterdir = "master" + timestring
cmd = "{} {}".format("mkdir", masterdir)
junk = cmdexecute(cmd)
elif not os.path.exists(masterdir):
# os.path.exists(masterdir) does not exist
masterdir = args[1]
cmd = "{} {}".format("mkdir", masterdir)
junk = cmdexecute(cmd)
if (args[0][:4] == "bdb:"): filename = args[0][4:]
else: filename = args[0][:-4]
filename = os.path.basename(filename)
stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join(
masterdir, filename)
stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename
if_error_then_all_processes_exit_program(error_status)
# send masterdir to all processes
masterdir = send_string_to_all(masterdir)
if myid == 0:
if options.restart_section != " ":
if os.path.exists(os.path.join(masterdir,
NAME_OF_JSON_STATE_FILE)):
stored_stack, stored_state = restore_program_stack_and_state(
os.path.join(masterdir, NAME_OF_JSON_STATE_FILE))
import re
if "," in options.restart_section:
parsed_restart_section_option = options.restart_section.split(
",")
stored_state[-1]["location_in_program"] = re.sub(
r"___.*$", "___%s" % parsed_restart_section_option[0],
stored_state[-1]["location_in_program"])
generation_str_format = parsed_restart_section_option[1]
if generation_str_format != "":
isac_generation_from_command_line = int(
generation_str_format)
stored_state[-1][
"isac_generation"] = isac_generation_from_command_line
else:
isac_generation_from_command_line = 1
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = -1
stored_state[-1]["location_in_program"] = re.sub(
r"___.*$", "___%s" % options.restart_section,
stored_state[-1]["location_in_program"])
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
store_program_state(
os.path.join(masterdir, NAME_OF_JSON_STATE_FILE),
stored_state, stored_stack)
else:
print "Please remove the restart_section option from the command line. The program must be started from the beginning."
mpi_finalize()
sys.exit()
else:
isac_generation_from_command_line = -1
program_state_stack(locals(), getframeinfo(currentframe()),
os.path.join(masterdir, NAME_OF_JSON_STATE_FILE))
stack_processed_by_ali2d_base__filename = send_string_to_all(
stack_processed_by_ali2d_base__filename)
stack_processed_by_ali2d_base__filename__without_master_dir = \
send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir)
# previous code 2016-05-05--20-14-12-153
# # PARAMETERS OF THE PROCEDURE
# if( options.xr == -1 ):
# # Default values
# # target_nx = 76
# # target_radius = 29
# target_xr = 1
# else: # nx//2
# # Check below!
# target_xr = options.xr
# # target_nx = 76 + target_xr - 1 # subtract one, which is default
# target_nx += target_xr - 1 # subtract one, which is default
# # target_radius = 29
target_xr = options.xr
target_nx += target_xr - 1 # subtract one, which is default
if (options.yr == -1):
yr = options.xr
else:
yr = options.yr
mpi_barrier(MPI_COMM_WORLD)
# Initialization of stacks
if (myid == main_node):
print "command_line_provided_stack_filename", command_line_provided_stack_filename
number_of_images_in_stack = EMUtil.get_image_count(
command_line_provided_stack_filename)
else:
number_of_images_in_stack = 0
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack,
source_node=main_node)
nxrsteps = 4
init2dir = os.path.join(masterdir, "2dalignment")
# from mpi import mpi_finalize
# mpi_finalize()
# sys.stdout.flush()
# sys.exit()
if not os.path.exists(
os.path.join(init2dir, "Finished_initial_2d_alignment.txt")):
if (myid == 0):
import subprocess
from logger import Logger, BaseLogger_Files
# Create output directory
log2d = Logger(BaseLogger_Files())
log2d.prefix = os.path.join(init2dir)
cmd = "mkdir -p " + log2d.prefix
outcome = subprocess.call(cmd, shell=True)
log2d.prefix += "/"
# outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True)
else:
outcome = 0
log2d = None
if (myid == main_node):
a = get_im(command_line_provided_stack_filename)
nnxo = a.get_xsize()
else:
nnxo = 0
nnxo = bcast_number_to_all(nnxo, source_node=main_node)
image_start, image_end = MPI_start_end(number_of_images_in_stack,
nproc, myid)
if options.skip_prealignment:
params2d = [[0.0, 0.0, 0.0, 0]
for i in xrange(image_start, image_end)]
else:
original_images = EMData.read_images(
command_line_provided_stack_filename,
range(image_start, image_end))
# We assume the target radius will be 29, and xr = 1.
shrink_ratio = float(target_radius) / float(radi)
for im in xrange(len(original_images)):
if (shrink_ratio != 1.0):
original_images[im] = resample(original_images[im],
shrink_ratio)
nx = original_images[0].get_xsize()
# nx = int(nx*shrink_ratio + 0.5)
txrm = (nx - 2 * (target_radius + 1)) // 2
if (txrm < 0):
ERROR(
"ERROR!! Radius of the structure larger than the window data size permits %d"
% (radi), "sxisac", 1, myid)
if (txrm / nxrsteps > 0):
tss = ""
txr = ""
while (txrm / nxrsteps > 0):
tts = txrm / nxrsteps
tss += " %d" % tts
txr += " %d" % (tts * nxrsteps)
txrm = txrm // 2
else:
tss = "1"
txr = "%d" % txrm
# print "nx, txr, txrm, tss", nx, txr, txrm, tss
# from mpi import mpi_finalize
# mpi_finalize()
# sys.stdout.flush()
# sys.exit()
# section ali2d_base
params2d = ali2d_base(original_images, init2dir, None, 1, target_radius, 1, txr, txr, tss, \
False, 90.0, center_method, 14, options.CTF, 1.0, False, \
"ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False)
del original_images
for i in xrange(len(params2d)):
alpha, sx, sy, mirror = combine_params2(
0, params2d[i][1], params2d[i][2], 0, -params2d[i][0], 0,
0, 0)
sx /= shrink_ratio
sy /= shrink_ratio
params2d[i][0] = 0.0
params2d[i][1] = sx
params2d[i][2] = sy
params2d[i][3] = 0
#set_params2D(aligned_images[i],[0.0, sx,sy,0.,1.0])
mpi_barrier(MPI_COMM_WORLD)
tmp = params2d[:]
tmp = wrap_mpi_gatherv(tmp, main_node, MPI_COMM_WORLD)
if (myid == main_node):
if options.skip_prealignment:
print "========================================="
print "Even though there is no alignment step, '%s' params are set to zero for later use." % os.path.join(
init2dir, "initial2Dparams.txt")
print "========================================="
write_text_row(tmp, os.path.join(init2dir, "initial2Dparams.txt"))
del tmp
mpi_barrier(MPI_COMM_WORLD)
# We assume the target image size will be target_nx, radius will be 29, and xr = 1.
# Note images can be also padded, in which case shrink_ratio > 1.
shrink_ratio = float(target_radius) / float(radi)
aligned_images = EMData.read_images(
command_line_provided_stack_filename,
range(image_start, image_end))
nx = aligned_images[0].get_xsize()
nima = len(aligned_images)
newx = int(nx * shrink_ratio + 0.5)
while not os.path.exists(os.path.join(init2dir,
"initial2Dparams.txt")):
import time
time.sleep(1)
mpi_barrier(MPI_COMM_WORLD)
params = read_text_row(os.path.join(init2dir, "initial2Dparams.txt"))
params = params[image_start:image_end]
msk = model_circle(radi, nx, nx)
for im in xrange(nima):
st = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= st[0]
if options.CTF:
aligned_images[im] = filt_ctf(
aligned_images[im],
aligned_images[im].get_attr("ctf"),
binary=True)
if (shrink_ratio < 1.0):
if newx > target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
#aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = resample(aligned_images[im],
shrink_ratio)
aligned_images[im] = Util.window(aligned_images[im],
target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = resample(aligned_images[im],
shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx:
msk = model_circle(newx // 2 - 2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = resample(aligned_images[im],
shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx,
target_nx, 1, 0.0)
elif (shrink_ratio == 1.0):
if newx > target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = Util.window(aligned_images[im],
target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx:
msk = model_circle(newx // 2 - 2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
#aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx,
target_nx, 1, 0.0)
elif (shrink_ratio > 1.0):
if newx > target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = resample(aligned_images[im],
shrink_ratio)
aligned_images[im] = Util.window(aligned_images[im],
target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = resample(aligned_images[im],
shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx:
msk = model_circle(newx // 2 - 2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = resample(aligned_images[im],
shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx,
target_nx, 1, 0.0)
del msk
gather_compacted_EMData_to_root(number_of_images_in_stack,
aligned_images, myid)
number_of_images_in_stack = bcast_number_to_all(
number_of_images_in_stack, source_node=main_node)
if (myid == main_node):
for i in range(number_of_images_in_stack):
aligned_images[i].write_image(
stack_processed_by_ali2d_base__filename, i)
# It has to be explicitly closed
from EMAN2db import db_open_dict
DB = db_open_dict(stack_processed_by_ali2d_base__filename)
DB.close()
fp = open(os.path.join(masterdir, "README_shrink_ratio.txt"), "w")
output_text = """
Since, for processing purposes, isac changes the image dimensions,
adjustment of pixel size needs to be made in subsequent steps, (e.g.
running sxviper.py). The shrink ratio for this particular isac run is
--------
%.5f
%.5f
--------
To get the pixel size for the isac output the user needs to divide
the original pixel size by the above value. This info is saved in
the following file: README_shrink_ratio.txt
""" % (shrink_ratio, radi)
fp.write(output_text)
fp.flush()
fp.close()
print output_text
fp = open(
os.path.join(init2dir, "Finished_initial_2d_alignment.txt"),
"w")
fp.flush()
fp.close()
else:
if (myid == main_node):
print "Skipping 2d alignment since it was already done!"
mpi_barrier(MPI_COMM_WORLD)
# from mpi import mpi_finalize
# mpi_finalize()
# sys.stdout.flush()
# sys.exit()
os.chdir(masterdir)
if program_state_stack(locals(), getframeinfo(currentframe())):
# if 1:
pass
if (myid == main_node):
junk = cmdexecute(
"sxheader.py --consecutive --params=originalid %s" %
stack_processed_by_ali2d_base__filename__without_master_dir)
junk = cmdexecute(
"e2bdb.py %s --makevstack=%s_000" %
(stack_processed_by_ali2d_base__filename__without_master_dir,
stack_processed_by_ali2d_base__filename__without_master_dir))
if (myid == main_node):
main_dir_no = get_latest_directory_increment_value("./",
NAME_OF_MAIN_DIR,
myformat="%04d")
print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no
if isac_generation_from_command_line < 0:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(
NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1][
"isac_generation"]
else:
isac_generation_from_command_line = -1
if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no:
for i in xrange(isac_generation_from_command_line + 1,
main_dir_no + 1):
if i == isac_generation_from_command_line + 1:
backup_dir_no = get_nonexistent_directory_increment_value(
"./", "000_backup", myformat="%05d", start_value=1)
junk = cmdexecute("mkdir -p " + "000_backup" +
"%05d" % backup_dir_no)
junk = cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % i +
" 000_backup" + "%05d" % backup_dir_no)
junk = cmdexecute(
"rm " + "EMAN2DB/" +
stack_processed_by_ali2d_base__filename__without_master_dir[
4:] + "_%03d.bdb" % i)
# it includes both command line and json file
my_restart_section = stored_state[-1]["location_in_program"].split(
"___")[-1]
if "restart" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value(
"./", "000_backup", myformat="%05d", start_value=1)
junk = cmdexecute("mkdir -p " + "000_backup" +
"%05d" % backup_dir_no)
junk = cmdexecute("mv " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation_from_command_line +
" 000_backup" + "%05d" % backup_dir_no)
junk = cmdexecute(
"rm " + "EMAN2DB/" +
stack_processed_by_ali2d_base__filename__without_master_dir[
4:] + "_%03d.bdb" % isac_generation_from_command_line)
elif "candidate_class_averages" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value(
"./", "000_backup", myformat="%05d", start_value=1)
junk = cmdexecute("mkdir -p " + "000_backup" +
"%05d" % backup_dir_no)
junk = cmdexecute("mv " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation_from_command_line +
" 000_backup" + "%05d" % backup_dir_no)
junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation_from_command_line)
# junk = cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line)
elif "reproducible_class_averages" in my_restart_section:
junk = cmdexecute("rm -rf " + NAME_OF_MAIN_DIR +
"%04d/ali_params_generation_*" %
isac_generation_from_command_line)
junk = cmdexecute("rm -f " + NAME_OF_MAIN_DIR +
"%04d/class_averages_generation*" %
isac_generation_from_command_line)
else:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(
NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1][
"isac_generation"]
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 0
isac_generation_from_command_line = mpi_bcast(
isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]
isac_generation = isac_generation_from_command_line - 1
if (myid == main_node):
if isac_generation == 0:
junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation)
write_text_file(
[1],
os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation,
"generation_%d_accounted.txt" % isac_generation))
write_text_file(
range(number_of_images_in_stack),
os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation,
"generation_%d_unaccounted.txt" %
isac_generation))
# Stopping criterion should be inside the program.
while True:
isac_generation += 1
if isac_generation > options.n_generations:
break
data64_stack_current = "bdb:../" + stack_processed_by_ali2d_base__filename__without_master_dir[
4:] + "_%03d" % isac_generation
program_state_stack.restart_location_title = "restart"
if program_state_stack(locals(), getframeinfo(currentframe())):
if (myid == main_node):
junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation)
# reference the original stack
list_file = os.path.join(
NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1),
"generation_%d_unaccounted.txt" % (isac_generation - 1))
junk = cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\
stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file))
mpi_barrier(MPI_COMM_WORLD)
os.chdir(NAME_OF_MAIN_DIR + "%04d" % isac_generation)
program_state_stack.restart_location_title = "candidate_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new)
# program_state_stack.restart_location_title = "stopped_program1"
# program_state_stack(locals(), getframeinfo(currentframe()))
program_state_stack.restart_location_title = "stop_after_candidates"
program_state_stack(locals(), getframeinfo(currentframe()))
if stop_after_candidates:
mpi_finalize()
sys.exit()
exit_program = 0
if (myid == main_node):
if not os.path.exists(
"class_averages_candidate_generation_%d.hdf" %
isac_generation):
print "This generation (%d) no class average candidates were generated! Finishing." % isac_generation
exit_program = 1
exit_program = int(
mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
if exit_program:
os.chdir("..")
break
program_state_stack.restart_location_title = "reproducible_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new)
pass
os.chdir("..")
if (myid == main_node):
accounted_images = read_text_file(
os.path.join(NAME_OF_MAIN_DIR + "%04d" % (isac_generation),
"generation_%d_accounted.txt" %
(isac_generation)))
number_of_accounted_images = len(accounted_images)
un_accounted_images = read_text_file(
os.path.join(
NAME_OF_MAIN_DIR + "%04d" % (isac_generation),
"generation_%d_unaccounted.txt" % (isac_generation)))
number_of_un_accounted_images = len(un_accounted_images)
else:
number_of_accounted_images = 0
number_of_un_accounted_images = 0
number_of_accounted_images = int(
mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0,
MPI_COMM_WORLD)[0])
number_of_un_accounted_images = int(
mpi_bcast(number_of_un_accounted_images, 1, MPI_INT, 0,
MPI_COMM_WORLD)[0])
if number_of_accounted_images == 0:
if (myid == main_node):
print "This generation (%d) there are no accounted images! Finishing." % isac_generation
break
while (myid == main_node):
def files_are_missing(isac_generation):
for i in xrange(1, isac_generation + 1):
if not os.path.exists(
"generation_%04d/class_averages_generation_%d.hdf"
% (i, i)):
print "Error: generation_%04d/class_averages_generation_%d.hdf is missing! Exiting." % (
i, i)
return 1
return 0
if files_are_missing(isac_generation):
break
junk = cmdexecute("rm -f class_averages.hdf")
cpy([
"generation_%04d/class_averages_generation_%d.hdf" % (i, i)
for i in xrange(1, isac_generation + 1)
], "class_averages.hdf")
break
if number_of_un_accounted_images == 0:
if (myid == main_node):
print "This generation (%d) there are no un accounted images! Finishing." % isac_generation
break
program_state_stack(locals(),
getframeinfo(currentframe()),
last_call="__LastCall")
mpi_barrier(MPI_COMM_WORLD)
mpi_finalize()
def main():
def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
if mirror:
m = 1
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 540.0-psi, 0, 0, 1.0)
else:
m = 0
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 360.0-psi, 0, 0, 1.0)
return alpha, sx, sy, m
progname = os.path.basename(sys.argv[0])
usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl=0.2 --aa=0.1 --sym=symmetry --CTF"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--ave2D", type="string" , default=False, help="write to the disk a stack of 2D averages")
parser.add_option("--var2D", type="string" , default=False, help="write to the disk a stack of 2D variances")
parser.add_option("--ave3D", type="string" , default=False, help="write to the disk reconstructed 3D average")
parser.add_option("--var3D", type="string" , default=False, help="compute 3D variability (time consuming!)")
parser.add_option("--img_per_grp", type="int" , default=10 , help="number of neighbouring projections")
parser.add_option("--no_norm", action="store_true", default=False, help="do not use normalization")
parser.add_option("--radiusvar", type="int" , default=-1 , help="radius for 3D var" )
parser.add_option("--npad", type="int" , default=2 , help="number of time to pad the original images")
parser.add_option("--sym" , type="string" , default="c1" , help="symmetry")
parser.add_option("--fl", type="float" , default=0.0 , help="stop-band frequency (Default - no filtration)")
parser.add_option("--aa", type="float" , default=0.0 , help="fall off of the filter (Default - no filtration)")
parser.add_option("--CTF", action="store_true", default=False, help="use CFT correction")
parser.add_option("--VERBOSE", action="store_true", default=False, help="Long output for debugging")
#parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
#parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" )
#parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" )
#parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
#parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
parser.add_option("--VAR" , action="store_true", default=False, help="stack on input consists of 2D variances (Default False)")
parser.add_option("--decimate", type="float", default=1.0, help="image decimate rate, a number large than 1. default is 1")
parser.add_option("--window", type="int", default=0, help="reduce images to a small image size without changing pixel_size. Default value is zero.")
#parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)")
parser.add_option("--nvec", type="int" , default=0 , help="number of eigenvectors, default = 0 meaning no PCA calculated")
parser.add_option("--symmetrize", action="store_true", default=False, help="Prepare input stack for handling symmetry (Default False)")
(options,args) = parser.parse_args()
#####
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD, MPI_TAG_UB
from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
from applications import MPI_start_end
from reconstruction import recons3d_em, recons3d_em_MPI
from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
from utilities import print_begin_msg, print_end_msg, print_msg
from utilities import read_text_row, get_image, get_im
from utilities import bcast_EMData_to_all, bcast_number_to_all
from utilities import get_symt
# This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015
from EMAN2db import db_open_dict
if options.symmetrize :
try:
sys.argv = mpi_init(len(sys.argv), sys.argv)
try:
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
if( number_of_proc > 1 ):
ERROR("Cannot use more than one CPU for symmetry prepration","sx3dvariability",1)
except:
pass
except:
pass
# Input
#instack = "Clean_NORM_CTF_start_wparams.hdf"
#instack = "bdb:data"
instack = args[0]
sym = options.sym
if( sym == "c1" ):
ERROR("Thre is no need to symmetrize stack for C1 symmetry","sx3dvariability",1)
if(instack[:4] !="bdb:"):
stack = "bdb:data"
delete_bdb(stack)
cmdexecute("sxcpy.py "+instack+" "+stack)
else:
stack = instack
qt = EMUtil.get_all_attributes(stack,'xform.projection')
na = len(qt)
ts = get_symt(sym)
ks = len(ts)
angsa = [None]*na
for k in xrange(ks):
delete_bdb("bdb:Q%1d"%k)
cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
DB = db_open_dict("bdb:Q%1d"%k)
for i in xrange(na):
ut = qt[i]*ts[k]
DB.set_attr(i, "xform.projection", ut)
#bt = ut.get_params("spider")
#angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
#write_text_row(angsa, 'ptsma%1d.txt'%k)
#cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
#cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k))
DB.close()
delete_bdb("bdb:sdata")
cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
#cmdexecute("ls EMAN2DB/sdata*")
a = get_im("bdb:sdata")
a.set_attr("variabilitysymmetry",sym)
a.write_image("bdb:sdata")
else:
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
if len(args) == 1:
stack = args[0]
else:
print( "usage: " + usage)
print( "Please run '" + progname + " -h' for detailed options")
return 1
t0 = time()
# obsolete flags
options.MPI = True
options.nvec = 0
options.radiuspca = -1
options.iter = 40
options.abs = 0.0
options.squ = 0.0
if options.fl > 0.0 and options.aa == 0.0:
ERROR("Fall off has to be given for the low-pass filter", "sx3dvariability", 1, myid)
if options.VAR and options.SND:
ERROR("Only one of var and SND can be set!", "sx3dvariability", myid)
exit()
if options.VAR and (options.ave2D or options.ave3D or options.var2D):
ERROR("When VAR is set, the program cannot output ave2D, ave3D or var2D", "sx3dvariability", 1, myid)
exit()
#if options.SND and (options.ave2D or options.ave3D):
# ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid)
# exit()
if options.nvec > 0 :
ERROR("PCA option not implemented", "sx3dvariability", 1, myid)
exit()
if options.nvec > 0 and options.ave3D == None:
ERROR("When doing PCA analysis, one must set ave3D", "sx3dvariability", myid=myid)
exit()
import string
options.sym = options.sym.lower()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
global_def.BATCH = True
if myid == main_node:
print_begin_msg("sx3dvariability")
print_msg("%-70s: %s\n"%("Input stack", stack))
img_per_grp = options.img_per_grp
nvec = options.nvec
radiuspca = options.radiuspca
symbaselen = 0
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
if options.sym != "c1" :
imgdata = get_im(stack)
try:
i = imgdata.get_attr("variabilitysymmetry")
if(i != options.sym):
ERROR("The symmetry provided does not agree with the symmetry of the input stack", "sx3dvariability", myid=myid)
except:
ERROR("Input stack is not prepared for symmetry, please follow instructions", "sx3dvariability", myid=myid)
from utilities import get_symt
i = len(get_symt(options.sym))
if((nima/i)*i != nima):
ERROR("The length of the input stack is incorrect for symmetry processing", "sx3dvariability", myid=myid)
symbaselen = nima/i
else: symbaselen = nima
else:
nima = 0
nx = 0
ny = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
Tracker ={}
Tracker["nx"] =nx
Tracker["ny"] =ny
Tracker["total_stack"]=nima
if options.decimate==1.:
if options.window !=0:
nx = options.window
ny = options.window
else:
if options.window ==0:
nx = int(nx/options.decimate)
ny = int(ny/options.decimate)
else:
nx = int(options.window/options.decimate)
ny = nx
symbaselen = bcast_number_to_all(symbaselen)
if radiuspca == -1: radiuspca = nx/2-2
if myid == main_node:
print_msg("%-70s: %d\n"%("Number of projection", nima))
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
"""
if options.SND:
from projection import prep_vol, prgs
from statistics import im_diff
from utilities import get_im, model_circle, get_params_proj, set_params_proj
from utilities import get_ctf, generate_ctf
from filter import filt_ctf
imgdata = EMData.read_images(stack, range(img_begin, img_end))
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
bcast_EMData_to_all(vol, myid)
volft, kb = prep_vol(vol)
mask = model_circle(nx/2-2, nx, ny)
varList = []
for i in xrange(img_begin, img_end):
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
if options.CTF:
ctf_params = get_ctf(imgdata[i-img_begin])
ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
diff2 = diff*diff
set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
varList.append(diff2)
mpi_barrier(MPI_COMM_WORLD)
"""
if options.VAR:
#varList = EMData.read_images(stack, range(img_begin, img_end))
varList = []
this_image = EMData()
for index_of_particle in xrange(img_begin,img_end):
this_image.read_image(stack,index_of_particle)
varList.append(image_decimate_window_xform_ctf(img,options.decimate,options.window,options.CTF))
else:
from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
from utilities import model_blank, nearest_proj, model_circle
from applications import pca
from statistics import avgvar, avgvar_ctf, ccc
from filter import filt_tanl
from morphology import threshold, square_root
from projection import project, prep_vol, prgs
from sets import Set
if myid == main_node:
t1 = time()
proj_angles = []
aveList = []
tab = EMUtil.get_all_attributes(stack, 'xform.projection')
for i in xrange(nima):
t = tab[i].get_params('spider')
phi = t['phi']
theta = t['theta']
psi = t['psi']
x = theta
if x > 90.0: x = 180.0 - x
x = x*10000+psi
proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
t2 = time()
print_msg("%-70s: %d\n"%("Number of neighboring projections", img_per_grp))
print_msg("...... Finding neighboring projections\n")
if options.VERBOSE:
print "Number of images per group: ", img_per_grp
print "Now grouping projections"
proj_angles.sort()
proj_angles_list = [0.0]*(nima*4)
if myid == main_node:
for i in xrange(nima):
proj_angles_list[i*4] = proj_angles[i][1]
proj_angles_list[i*4+1] = proj_angles[i][2]
proj_angles_list[i*4+2] = proj_angles[i][3]
proj_angles_list[i*4+3] = proj_angles[i][4]
proj_angles_list = bcast_list_to_all(proj_angles_list, myid, main_node)
proj_angles = []
for i in xrange(nima):
proj_angles.append([proj_angles_list[i*4], proj_angles_list[i*4+1], proj_angles_list[i*4+2], int(proj_angles_list[i*4+3])])
del proj_angles_list
proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp, range(img_begin, img_end))
all_proj = Set()
for im in proj_list:
for jm in im:
all_proj.add(proj_angles[jm][3])
all_proj = list(all_proj)
if options.VERBOSE:
print "On node %2d, number of images needed to be read = %5d"%(myid, len(all_proj))
index = {}
for i in xrange(len(all_proj)): index[all_proj[i]] = i
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
print_msg("%-70s: %.2f\n"%("Finding neighboring projections lasted [s]", time()-t2))
print_msg("%-70s: %d\n"%("Number of groups processed on the main node", len(proj_list)))
if options.VERBOSE:
print "Grouping projections took: ", (time()-t2)/60 , "[min]"
print "Number of groups on main node: ", len(proj_list)
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
print_msg("...... calculating the stack of 2D variances \n")
if options.VERBOSE:
print "Now calculating the stack of 2D variances"
proj_params = [0.0]*(nima*5)
aveList = []
varList = []
if nvec > 0:
eigList = [[] for i in xrange(nvec)]
if options.VERBOSE: print "Begin to read images on processor %d"%(myid)
ttt = time()
#imgdata = EMData.read_images(stack, all_proj)
img = EMData()
imgdata = []
for index_of_proj in xrange(len(all_proj)):
img.read_image(stack, all_proj[index_of_proj])
dmg = image_decimate_window_xform_ctf(img,options.decimate,options.window,options.CTF)
#print dmg.get_xsize(), "init"
imgdata.append(dmg)
if options.VERBOSE:
print "Reading images on processor %d done, time = %.2f"%(myid, time()-ttt)
print "On processor %d, we got %d images"%(myid, len(imgdata))
mpi_barrier(MPI_COMM_WORLD)
'''
imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
if options.fl > 0.0:
for k in xrange(len(imgdata2)):
imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
if myid == main_node:
vol.write_image("vol_ctf.hdf")
print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf"))
del vol, imgdata2
mpi_barrier(MPI_COMM_WORLD)
'''
from applications import prepare_2d_forPCA
from utilities import model_blank
for i in xrange(len(proj_list)):
ki = proj_angles[proj_list[i][0]][3]
if ki >= symbaselen: continue
mi = index[ki]
phiM, thetaM, psiM, s2xM, s2yM = get_params_proj(imgdata[mi])
grp_imgdata = []
for j in xrange(img_per_grp):
mj = index[proj_angles[proj_list[i][j]][3]]
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[mj])
alpha, sx, sy, mirror = params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror_list[i][j])
if thetaM <= 90:
if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, phiM-phi, 0.0, 0.0, 1.0)
else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, 180-(phiM-phi), 0.0, 0.0, 1.0)
else:
if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(phiM-phi), 0.0, 0.0, 1.0)
else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(180-(phiM-phi)), 0.0, 0.0, 1.0)
set_params2D(imgdata[mj], [alpha, sx, sy, mirror, 1.0])
grp_imgdata.append(imgdata[mj])
#print grp_imgdata[j].get_xsize(), imgdata[mj].get_xsize()
if not options.no_norm:
#print grp_imgdata[j].get_xsize()
mask = model_circle(nx/2-2, nx, nx)
for k in xrange(img_per_grp):
ave, std, minn, maxx = Util.infomask(grp_imgdata[k], mask, False)
grp_imgdata[k] -= ave
grp_imgdata[k] /= std
del mask
if options.fl > 0.0:
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2*nx
ny2 = 2*ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d(fft( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa) ),nx,ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
for k in xrange(img_per_grp):
grp_imgdata[k] = filt_tanl( grp_imgdata[k], options.fl, options.aa)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
from utilities import pad, read_text_file
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2*nx
ny2 = 2*ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d( fft( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1) ) , nx,ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
'''
if i < 10 and myid == main_node:
for k in xrange(10):
grp_imgdata[k].write_image("grp%03d.hdf"%i, k)
'''
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("pp.hdf", pp)
"""
ave, grp_imgdata = prepare_2d_forPCA(grp_imgdata)
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("qq.hdf", pp)
"""
var = model_blank(nx,ny)
for q in grp_imgdata: Util.add_img2( var, q )
Util.mul_scalar( var, 1.0/(len(grp_imgdata)-1))
# Switch to std dev
var = square_root(threshold(var))
#if options.CTF: ave, var = avgvar_ctf(grp_imgdata, mode="a")
#else: ave, var = avgvar(grp_imgdata, mode="a")
"""
if myid == main_node:
ave.write_image("avgv.hdf",i)
var.write_image("varv.hdf",i)
"""
set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])
aveList.append(ave)
varList.append(var)
if options.VERBOSE:
print "%5.2f%% done on processor %d"%(i*100.0/len(proj_list), myid)
if nvec > 0:
eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True)
for k in xrange(nvec):
set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
eigList[k].append(eig[k])
"""
if myid == 0 and i == 0:
for k in xrange(nvec):
eig[k].write_image("eig.hdf", k)
"""
del imgdata
# To this point, all averages, variances, and eigenvectors are computed
if options.ave2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node :
for im in xrange(len(aveList)):
aveList[im].write_image(options.ave2D, km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im+i+70000)
"""
nm = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('pix_err', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
"""
tmpvol=fpol(ave, Tracker["nx"],Tracker["nx"],Tracker["nx"])
tmpvol.write_image(options.ave2D, km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
for im in xrange(len(aveList)):
send_EMData(aveList[im], main_node,im+myid+70000)
"""
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
members = aveList[im].get_attr('pix_err')
mpi_send(members, len(members), MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
except:
mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
"""
if options.ave3D:
from fundamentals import fpol
if options.VERBOSE:
print "Reconstructing 3D average volume"
ave3D = recons3d_4nn_MPI(myid, aveList, symmetry=options.sym, npad=options.npad)
bcast_EMData_to_all(ave3D, myid)
if myid == main_node:
ave3D=fpol(ave3D,Tracker["nx"],Tracker["nx"],Tracker["nx"])
ave3D.write_image(options.ave3D)
print_msg("%-70s: %s\n"%("Writing to the disk volume reconstructed from averages as", options.ave3D))
del ave, var, proj_list, stack, phi, theta, psi, s2x, s2y, alpha, sx, sy, mirror, aveList
if nvec > 0:
for k in xrange(nvec):
if options.VERBOSE:
print "Reconstruction eigenvolumes", k
cont = True
ITER = 0
mask2d = model_circle(radiuspca, nx, nx)
while cont:
#print "On node %d, iteration %d"%(myid, ITER)
eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad)
bcast_EMData_to_all(eig3D, myid, main_node)
if options.fl > 0.0:
eig3D = filt_tanl(eig3D, options.fl, options.aa)
if myid == main_node:
eig3D.write_image("eig3d_%03d.hdf"%k, ITER)
Util.mul_img( eig3D, model_circle(radiuspca, nx, nx, nx) )
eig3Df, kb = prep_vol(eig3D)
del eig3D
cont = False
icont = 0
for l in xrange(len(eigList[k])):
phi, theta, psi, s2x, s2y = get_params_proj(eigList[k][l])
proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y])
cl = ccc(proj, eigList[k][l], mask2d)
if cl < 0.0:
icont += 1
cont = True
eigList[k][l] *= -1.0
u = int(cont)
u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD)
icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
if myid == main_node:
u = int(u[0])
print " Eigenvector: ",k," number changed ",int(icont[0])
else: u = 0
u = bcast_number_to_all(u, main_node)
cont = bool(u)
ITER += 1
del eig3Df, kb
mpi_barrier(MPI_COMM_WORLD)
del eigList, mask2d
if options.ave3D: del ave3D
if options.var2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node :
for im in xrange(len(varList)):
tmpvol=fpol(varList[im], Tracker["nx"], Tracker["nx"],1)
tmpvol.write_image(options.var2D, km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im+i+70000)
tmpvol=fpol(ave, Tracker["nx"], Tracker["nx"],1)
tmpvol.write_image(options.var2D, km)
km += 1
else:
mpi_send(len(varList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
for im in xrange(len(varList)):
send_EMData(varList[im], main_node, im+myid+70000)# What with the attributes??
mpi_barrier(MPI_COMM_WORLD)
if options.var3D:
if myid == main_node and options.VERBOSE:
print "Reconstructing 3D variability volume"
t6 = time()
radiusvar = options.radiusvar
if( radiusvar < 0 ): radiusvar = nx//2 -3
res = recons3d_4nn_MPI(myid, varList, symmetry=options.sym, npad=options.npad)
#res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
if myid == main_node:
from fundamentals import fpol
res =fpol(res, Tracker["nx"], Tracker["nx"], Tracker["nx"])
res.write_image(options.var3D)
if myid == main_node:
print_msg("%-70s: %.2f\n"%("Reconstructing 3D variability took [s]", time()-t6))
if options.VERBOSE:
print "Reconstruction took: %.2f [min]"%((time()-t6)/60)
if myid == main_node:
print_msg("%-70s: %.2f\n"%("Total time for these computations [s]", time()-t0))
if options.VERBOSE:
print "Total time for these computations: %.2f [min]"%((time()-t0)/60)
print_end_msg("sx3dvariability")
global_def.BATCH = False
from mpi import mpi_finalize
mpi_finalize()
def main():
"""
Main function.
Arguments:
None
Returns:
None
"""
command_args = parse_command_line()
# Import volume
print('Import volume.')
input_vol = utilities.get_im(command_args.input_volume)
# Sanity checks
sanity_checks(command_args, input_vol)
try:
os.makedirs(command_args.output_dir)
except OSError:
print('Output directory already exists. No need to create it.')
else:
print('Created output directory.')
output_prefix = os.path.join(command_args.output_dir, command_args.prefix)
# Filter volume if specified
if command_args.low_pass_filter_resolution is not None:
print('Filter volume to {0}A.'.format(
command_args.low_pass_filter_resolution))
input_vol = sparx_filter.filt_tanl(
input_vol,
command_args.pixel_size / command_args.low_pass_filter_resolution,
command_args.low_pass_filter_falloff)
input_vol.write_image(output_prefix + '_filtered_volume.hdf')
else:
print('Skip filter volume.')
# Create a mask based on the filtered volume
print('Create mask')
density_threshold = -9999.0
nsigma = 1.0
if command_args.mol_mass:
density_threshold = input_vol.find_3d_threshold(
command_args.mol_mass, command_args.pixel_size)
elif command_args.threshold:
density_threshold = command_args.threshold
elif command_args.nsigma:
nsigma = command_args.nsigma
else:
assert False
if command_args.edge_type == 'cosine':
mode = 'C'
elif command_args.edge_type == 'gaussian':
mode = 'G'
else:
assert False
mask_first = morphology.adaptive_mask_scipy(
input_vol,
nsigma=nsigma,
threshold=density_threshold,
ndilation=command_args.ndilation,
nerosion=command_args.nerosion,
edge_width=command_args.edge_width,
allow_disconnected=command_args.allow_disconnected,
mode=mode,
do_approx=command_args.do_old,
)
# Create a second mask based on the filtered volume
s_mask = None
s_density_threshold = 1
s_nsigma = 1.0
if command_args.second_mask is not None:
s_mask = utilities.get_im(command_args.second_mask)
density_threshold = -9999.0
nsigma = 1.0
if command_args.s_mol_mass:
s_density_threshold = input_vol.find_3d_threshold(
command_args.s_mol_mass, command_args.s_pixel_size)
elif command_args.s_threshold:
s_density_threshold = command_args.s_threshold
elif command_args.s_nsigma:
s_nsigma = command_args.s_nsigma
else:
assert False
elif command_args.second_mask_shape is not None:
nx = mask_first.get_xsize()
ny = mask_first.get_ysize()
nz = mask_first.get_zsize()
if command_args.second_mask_shape == 'cube':
s_nx = command_args.s_nx
s_ny = command_args.s_ny
s_nz = command_args.s_nz
s_mask = utilities.model_blank(s_nx, s_ny, s_nz, 1)
elif command_args.second_mask_shape == 'cylinder':
s_radius = command_args.s_radius
s_nx = command_args.s_nx
s_ny = command_args.s_ny
s_nz = command_args.s_nz
s_mask = utilities.model_cylinder(s_radius, s_nx, s_ny, s_nz)
elif command_args.second_mask_shape == 'sphere':
s_radius = command_args.s_radius
s_nx = command_args.s_nx
s_ny = command_args.s_ny
s_nz = command_args.s_nz
s_mask = utilities.model_circle(s_radius, s_nx, s_ny, s_nz)
else:
assert False
s_mask = utilities.pad(s_mask, nx, ny, nz, 0)
if s_mask is not None:
print('Create second mask')
if command_args.s_edge_type == 'cosine':
mode = 'C'
elif command_args.s_edge_type == 'gaussian':
mode = 'G'
else:
assert False
s_mask = morphology.adaptive_mask_scipy(
s_mask,
nsigma=s_nsigma,
threshold=s_density_threshold,
ndilation=command_args.s_ndilation,
nerosion=command_args.s_nerosion,
edge_width=command_args.s_edge_width,
allow_disconnected=command_args.s_allow_disconnected,
mode=mode,
do_approx=command_args.s_do_old)
if command_args.s_invert:
s_mask = 1 - s_mask
mask_first.write_image(output_prefix + '_mask_first.hdf')
s_mask.write_image(output_prefix + '_mask_second.hdf')
masked_combined = mask_first * s_mask
masked_combined.write_image(output_prefix + '_mask.hdf')
else:
mask_first.write_image(output_prefix + '_mask.hdf')
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + """ Input Output [options]
Generate three micrographs, each micrograph contains one projection of a long filament.
Input: Reference Volume, output directory
Output: Three micrographs stored in output directory
sxhelical_demo.py tmp.hdf mic --generate_micrograph --CTF --apix=1.84
Generate noisy cylinder ini.hdf with radius 35 pixels and box size 100 by 100 by 200
sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35
Generate rectangular 2D mask mask2d.hdf with width 60 pixels and image size 200 by 200 pixels
sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=60
Apply the centering parameters to bdb:adata, normalize using average and standard deviation outside the mask, and output the new images to bdb:data
sxhelical_demo.py bdb:adata bdb:data mask2d.hdf --applyparams
Generate run through example script for helicon
sxhelical_demo.py --generate_script --filename=run --seg_ny=180 --ptcl_dist=15 --fract=0.35
"""
parser = OptionParser(usage,version=SPARXVERSION)
# helicise the Atom coordinates
# generate micrographs of helical filament
parser.add_option("--generate_micrograph", action="store_true", default=False, help="Generate three micrographs where each micrograph contains one projection of a long filament. \n Input: Reference Volume, output directory \n Output: Three micrographs containing helical filament projections stored in output directory")
parser.add_option("--CTF", action="store_true", default=False, help="Use CTF correction")
parser.add_option("--apix", type="float", default= -1, help="pixel size in Angstroms")
parser.add_option("--rand_seed", type="int", default=14567, help="the seed used for generating random numbers (default 14567) for adding noise to the generated micrographs.")
parser.add_option("--Cs", type="float", default= 2.0, help="Microscope Cs (spherical aberation)")
parser.add_option("--voltage", type="float", default=200.0, help="Microscope voltage in KV")
parser.add_option("--ac", type="float", default=10.0, help="Amplitude contrast (percentage, default=10)")
parser.add_option("--nonoise", action="store_true", default=False, help="Do not add noise to the micrograph.")
# generate initial volume
parser.add_option("--generate_noisycyl", action="store_true", default=False, help="Generate initial volume of noisy cylinder.")
parser.add_option("--boxsize", type="string", default="100,100,200", help="String containing x , y, z dimensions (separated by comma) in pixels")
parser.add_option("--rad", type="int", default=35, help="Radius of initial volume in pixels")
# generate 2D mask
parser.add_option("--generate_mask", action="store_true", default=False, help="Generate 2D rectangular mask.")
parser.add_option("--masksize", type="string", default="200,200", help="String containing x and y dimensions (separated by comma) in pixels")
parser.add_option("--maskwidth", type="int", default=60, help="Width of rectangular mask")
# Apply 2D alignment parameters to input stack and output new images to output stack
parser.add_option("--applyparams", action="store_true", default=False, help="Apply the centering parameters to input stack, normalize using average and standard deviation outside the mask, and output the new images to output stack")
# Generate run script
parser.add_option("--generate_script", action="store_true", default=False, help="Generate script for helicon run through example")
parser.add_option("--filename", type="string", default="runhelicon", help="Name of run script to generate")
parser.add_option("--seg_ny", type="int", default=180, help="y-dimension of segment used for refinement")
parser.add_option("--ptcl_dist", type="int", default=15, help="Distance in pixels between adjacent segments windowed from same filament")
parser.add_option("--fract", type="float", default=0.35, help="Fraction of the volume used for applying helical symmetry.")
(options, args) = parser.parse_args()
if len(args) > 3:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
else:
if options.generate_script:
generate_runscript(options.filename, options.seg_ny, options.ptcl_dist, options.fract)
if options.generate_micrograph:
if options.apix <= 0:
print "Please enter pixel size."
sys.exit()
generate_helimic(args[0], args[1], options.apix, options.CTF, options.Cs, options.voltage, options.ac, options.nonoise, options.rand_seed)
if options.generate_noisycyl:
from utilities import model_cylinder, model_gauss_noise
outvol = args[0]
boxdims = options.boxsize.split(',')
if len(boxdims) < 1 or len(boxdims) > 3:
print "Enter box size as string containing x , y, z dimensions (separated by comma) in pixels. E.g.: --boxsize='100,100,200'"
sys.exit()
nx= int(boxdims[0])
if len(boxdims) == 1:
ny = nx
nz = nx
else:
ny = int(boxdims[1])
if len(boxdims) == 3:
nz = int(boxdims[2])
(model_cylinder(options.rad,nx, ny, nz)*model_gauss_noise(1.0, nx, ny, nz) ).write_image(outvol)
if options.generate_mask:
from utilities import model_blank, pad
outvol = args[0]
maskdims = options.masksize.split(',')
if len(maskdims) < 1 or len(maskdims) > 2:
print "Enter box size as string containing x , y dimensions (separated by comma) in pixels. E.g.: --boxsize='200,200'"
sys.exit()
nx= int(maskdims[0])
if len(maskdims) == 1:
ny = nx
else:
ny = int(maskdims[1])
mask = pad(model_blank(options.maskwidth, ny, 1, 1.0), nx, ny, 1, 0.0)
mask.write_image(outvol)
if options.applyparams:
from utilities import get_im, get_params2D, set_params2D
from fundamentals import cyclic_shift
stack = args[0]
newstack = args[1]
mask = get_im(args[2])
nima = EMUtil.get_image_count(stack)
for im in xrange(nima):
prj = get_im(stack,im)
alpha, sx, sy, mirror, scale = get_params2D(prj)
prj = cyclic_shift(prj, int(sx))
set_params2D(prj, [0.0,0.,0.0,0,1])
stat = Util.infomask(prj , mask, False )
prj= (prj-stat[0])/stat[1]
ctf_params = prj.get_attr("ctf")
prj.set_attr('ctf_applied', 0)
prj.write_image(newstack, im)
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + """ Input Output [options]
Generate three micrographs, each micrograph contains one projection of a long filament.
Input: Reference Volume, output directory
Output: Three micrographs stored in output directory
sxhelical_demo.py tmp.hdf mic --generate_micrograph --CTF --apix=1.84
Generate noisy cylinder ini.hdf with radius 35 pixels and box size 100 by 100 by 200
sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35
Generate rectangular 2D mask mask2d.hdf with width 60 pixels and image size 200 by 200 pixels
sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=60
Apply the centering parameters to bdb:adata, normalize using average and standard deviation outside the mask, and output the new images to bdb:data
sxhelical_demo.py bdb:adata bdb:data mask2d.hdf --applyparams
Generate run through example script for helicon
sxhelical_demo.py --generate_script --filename=run --seg_ny=180 --ptcl_dist=15 --fract=0.35
"""
parser = OptionParser(usage, version=SPARXVERSION)
# helicise the Atom coordinates
# generate micrographs of helical filament
parser.add_option(
"--generate_micrograph",
action="store_true",
default=False,
help=
"Generate three micrographs where each micrograph contains one projection of a long filament. \n Input: Reference Volume, output directory \n Output: Three micrographs containing helical filament projections stored in output directory"
)
parser.add_option("--CTF",
action="store_true",
default=False,
help="Use CTF correction")
parser.add_option("--apix",
type="float",
default=-1,
help="pixel size in Angstroms")
parser.add_option(
"--rand_seed",
type="int",
default=14567,
help=
"the seed used for generating random numbers (default 14567) for adding noise to the generated micrographs."
)
parser.add_option("--Cs",
type="float",
default=2.0,
help="Microscope Cs (spherical aberation)")
parser.add_option("--voltage",
type="float",
default=200.0,
help="Microscope voltage in KV")
parser.add_option("--ac",
type="float",
default=10.0,
help="Amplitude contrast (percentage, default=10)")
parser.add_option("--nonoise",
action="store_true",
default=False,
help="Do not add noise to the micrograph.")
# generate initial volume
parser.add_option("--generate_noisycyl",
action="store_true",
default=False,
help="Generate initial volume of noisy cylinder.")
parser.add_option(
"--boxsize",
type="string",
default="100,100,200",
help=
"String containing x , y, z dimensions (separated by comma) in pixels")
parser.add_option("--rad",
type="int",
default=35,
help="Radius of initial volume in pixels")
# generate 2D mask
parser.add_option("--generate_mask",
action="store_true",
default=False,
help="Generate 2D rectangular mask.")
parser.add_option(
"--masksize",
type="string",
default="200,200",
help=
"String containing x and y dimensions (separated by comma) in pixels")
parser.add_option("--maskwidth",
type="int",
default=60,
help="Width of rectangular mask")
# Apply 2D alignment parameters to input stack and output new images to output stack
parser.add_option(
"--applyparams",
action="store_true",
default=False,
help=
"Apply the centering parameters to input stack, normalize using average and standard deviation outside the mask, and output the new images to output stack"
)
# Generate run script
parser.add_option("--generate_script",
action="store_true",
default=False,
help="Generate script for helicon run through example")
parser.add_option("--filename",
type="string",
default="runhelicon",
help="Name of run script to generate")
parser.add_option("--seg_ny",
type="int",
default=180,
help="y-dimension of segment used for refinement")
parser.add_option(
"--ptcl_dist",
type="int",
default=15,
help=
"Distance in pixels between adjacent segments windowed from same filament"
)
parser.add_option(
"--fract",
type="float",
default=0.35,
help="Fraction of the volume used for applying helical symmetry.")
(options, args) = parser.parse_args()
if len(args) > 3:
print("usage: " + usage)
print("Please run '" + progname + " -h' for detailed options")
else:
if options.generate_script:
generate_runscript(options.filename, options.seg_ny,
options.ptcl_dist, options.fract)
if options.generate_micrograph:
if options.apix <= 0:
print("Please enter pixel size.")
sys.exit()
generate_helimic(args[0], args[1], options.apix, options.CTF,
options.Cs, options.voltage, options.ac,
options.nonoise, options.rand_seed)
if options.generate_noisycyl:
from utilities import model_cylinder, model_gauss_noise
outvol = args[0]
boxdims = options.boxsize.split(',')
if len(boxdims) < 1 or len(boxdims) > 3:
print(
"Enter box size as string containing x , y, z dimensions (separated by comma) in pixels. E.g.: --boxsize='100,100,200'"
)
sys.exit()
nx = int(boxdims[0])
if len(boxdims) == 1:
ny = nx
nz = nx
else:
ny = int(boxdims[1])
if len(boxdims) == 3:
nz = int(boxdims[2])
(model_cylinder(options.rad, nx, ny, nz) *
model_gauss_noise(1.0, nx, ny, nz)).write_image(outvol)
if options.generate_mask:
from utilities import model_blank, pad
outvol = args[0]
maskdims = options.masksize.split(',')
if len(maskdims) < 1 or len(maskdims) > 2:
print(
"Enter box size as string containing x , y dimensions (separated by comma) in pixels. E.g.: --boxsize='200,200'"
)
sys.exit()
nx = int(maskdims[0])
if len(maskdims) == 1:
ny = nx
else:
ny = int(maskdims[1])
mask = pad(model_blank(options.maskwidth, ny, 1, 1.0), nx, ny, 1,
0.0)
mask.write_image(outvol)
if options.applyparams:
from utilities import get_im, get_params2D, set_params2D
from fundamentals import cyclic_shift
stack = args[0]
newstack = args[1]
mask = get_im(args[2])
nima = EMUtil.get_image_count(stack)
for im in range(nima):
prj = get_im(stack, im)
alpha, sx, sy, mirror, scale = get_params2D(prj)
prj = cyclic_shift(prj, int(sx))
set_params2D(prj, [0.0, 0., 0.0, 0, 1])
stat = Util.infomask(prj, mask, False)
prj = old_div((prj - stat[0]), stat[1])
ctf_params = prj.get_attr("ctf")
prj.set_attr('ctf_applied', 0)
prj.write_image(newstack, im)
def main():
import sys
import os
import math
import random
import pyemtbx.options
import time
from random import random, seed, randint
from optparse import OptionParser
progname = os.path.basename(sys.argv[0])
usage = progname + """ [options] <inputfile> <outputfile>
Generic 2-D image processing programs.
Functionality:
1. Phase flip a stack of images and write output to new file:
sxprocess.py input_stack.hdf output_stack.hdf --phase_flip
2. Resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.
The window size will change accordingly.
sxprocess input.hdf output.hdf --changesize --ratio=0.5
3. Compute average power spectrum of a stack of 2D images with optional padding (option wn) with zeroes.
sxprocess.py input_stack.hdf powerspectrum.hdf --pw [--wn=1024]
4. Generate a stack of projections bdb:data and micrographs with prefix mic (i.e., mic0.hdf, mic1.hdf etc) from structure input_structure.hdf, with CTF applied to both projections and micrographs:
sxprocess.py input_structure.hdf data mic --generate_projections format="bdb":apix=5.2:CTF=True:boxsize=64
5. Retrieve original image numbers in the selected ISAC group (here group 12 from generation 3):
sxprocess.py bdb:test3 class_averages_generation_3.hdf list3_12.txt --isacgroup=12 --params=originalid
6. Retrieve original image numbers of images listed in ISAC output stack of averages:
sxprocess.py select1.hdf ohk.txt
7. Adjust rotationally averaged power spectrum of an image to that of a reference image or a reference 1D power spectrum stored in an ASCII file.
Optionally use a tangent low-pass filter. Also works for a stack of images, in which case the output is also a stack.
sxprocess.py vol.hdf ref.hdf avol.hdf < 0.25 0.2> --adjpw
sxprocess.py vol.hdf pw.txt avol.hdf < 0.25 0.2> --adjpw
8. Generate a 1D rotationally averaged power spectrum of an image.
sxprocess.py vol.hdf --rotwp=rotpw.txt
# Output will contain three columns:
(1) rotationally averaged power spectrum
(2) logarithm of the rotationally averaged power spectrum
(3) integer line number (from zero to approximately to half the image size)
9. Apply 3D transformation (rotation and/or shift) to a set of orientation parameters associated with projection data.
sxprocess.py --transfromparams=phi,theta,psi,tx,ty,tz input.txt output.txt
The output file is then imported and 3D transformed volume computed:
sxheader.py bdb:p --params=xform.projection --import=output.txt
mpirun -np 2 sxrecons3d_n.py bdb:p tvol.hdf --MPI
The reconstructed volume is in the position of the volume computed using the input.txt parameters and then
transformed with rot_shift3D(vol, phi,theta,psi,tx,ty,tz)
10. Import ctf parameters from the output of sxcter into windowed particle headers.
There are three possible input files formats: (1) all particles are in one stack, (2 aor 3) particles are in stacks, each stack corresponds to a single micrograph.
In each case the particles should contain a name of the micrograph of origin stores using attribute name 'ptcl_source_image'.
Normally this is done by e2boxer.py during windowing.
Particles whose defocus or astigmatism error exceed set thresholds will be skipped, otherwise, virtual stacks with the original way preceded by G will be created.
sxprocess.py --input=bdb:data --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0
# Output will be a vritual stack bdb:Gdata
sxprocess.py --input="bdb:directory/stacks*" --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0
To concatenate output files:
cd directory
e2bdb.py . --makevstack=bdb:allparticles --filt=G
IMPORTANT: Please do not move (or remove!) any input/intermediate EMAN2DB files as the information is linked between them.
11. Scale 3D shifts. The shifts in the input five columns text file with 3D orientation parameters will be DIVIDED by the scale factor
sxprocess.py orientationparams.txt scaledparams.txt scale=0.5
12. Generate adaptive mask from a given 3-D volume.
"""
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option(
"--order",
action="store_true",
help=
"Two arguments are required: name of input stack and desired name of output stack. The output stack is the input stack sorted by similarity in terms of cross-correlation coefficent.",
default=False)
parser.add_option("--order_lookup",
action="store_true",
help="Test/Debug.",
default=False)
parser.add_option("--order_metropolis",
action="store_true",
help="Test/Debug.",
default=False)
parser.add_option("--order_pca",
action="store_true",
help="Test/Debug.",
default=False)
parser.add_option(
"--initial",
type="int",
default=-1,
help=
"Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)"
)
parser.add_option(
"--circular",
action="store_true",
help=
"Select circular ordering (fisr image has to be similar to the last",
default=False)
parser.add_option(
"--radius",
type="int",
default=-1,
help="Radius of a circular mask for similarity based ordering")
parser.add_option(
"--changesize",
action="store_true",
help=
"resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.",
default=False)
parser.add_option(
"--ratio",
type="float",
default=1.0,
help=
"The ratio of new to old image size (if <1 the pixel size will increase and image size decrease, if>1, the other way round"
)
parser.add_option(
"--pw",
action="store_true",
help=
"compute average power spectrum of a stack of 2-D images with optional padding (option wn) with zeroes",
default=False)
parser.add_option(
"--wn",
type="int",
default=-1,
help=
"Size of window to use (should be larger/equal than particle box size, default padding to max(nx,ny))"
)
parser.add_option("--phase_flip",
action="store_true",
help="Phase flip the input stack",
default=False)
parser.add_option(
"--makedb",
metavar="param1=value1:param2=value2",
type="string",
action="append",
help=
"One argument is required: name of key with which the database will be created. Fill in database with parameters specified as follows: --makedb param1=value1:param2=value2, e.g. 'gauss_width'=1.0:'pixel_input'=5.2:'pixel_output'=5.2:'thr_low'=1.0"
)
parser.add_option(
"--generate_projections",
metavar="param1=value1:param2=value2",
type="string",
action="append",
help=
"Three arguments are required: name of input structure from which to generate projections, desired name of output projection stack, and desired prefix for micrographs (e.g. if prefix is 'mic', then micrographs mic0.hdf, mic1.hdf etc will be generated). Optional arguments specifying format, apix, box size and whether to add CTF effects can be entered as follows after --generate_projections: format='bdb':apix=5.2:CTF=True:boxsize=100, or format='hdf', etc., where format is bdb or hdf, apix (pixel size) is a float, CTF is True or False, and boxsize denotes the dimension of the box (assumed to be a square). If an optional parameter is not specified, it will default as follows: format='bdb', apix=2.5, CTF=False, boxsize=64."
)
parser.add_option(
"--isacgroup",
type="int",
help=
"Retrieve original image numbers in the selected ISAC group. See ISAC documentation for details.",
default=-1)
parser.add_option(
"--isacselect",
action="store_true",
help=
"Retrieve original image numbers of images listed in ISAC output stack of averages. See ISAC documentation for details.",
default=False)
parser.add_option(
"--params",
type="string",
default=None,
help="Name of header of parameter, which one depends on specific option"
)
parser.add_option(
"--adjpw",
action="store_true",
help="Adjust rotationally averaged power spectrum of an image",
default=False)
parser.add_option(
"--rotpw",
type="string",
default=None,
help=
"Name of the text file to contain rotationally averaged power spectrum of the input image."
)
parser.add_option(
"--transformparams",
type="string",
default=None,
help=
"Transform 3D projection orientation parameters using six 3D parameters (phi, theta,psi,sx,sy,sz). Input: --transformparams=45.,66.,12.,-2,3,-5.5 desired six transformation of the reconstructed structure. Output: file with modified orientation parameters."
)
# import ctf estimates done using cter
parser.add_option("--input",
type="string",
default=None,
help="Input particles.")
parser.add_option(
"--importctf",
type="string",
default=None,
help="Name of the file containing CTF parameters produced by sxcter.")
parser.add_option(
"--defocuserror",
type="float",
default=1000000.0,
help=
"Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent. The error is computed as (std dev defocus)/defocus*100%"
)
parser.add_option(
"--astigmatismerror",
type="float",
default=360.0,
help=
"Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees."
)
# import ctf estimates done using cter
parser.add_option(
"--scale",
type="float",
default=-1.0,
help=
"Divide shifts in the input 3D orientation parameters text file by the scale factor."
)
# generate adaptive mask from an given 3-Db volue
parser.add_option("--adaptive_mask",
action="store_true",
help="create adavptive 3-D mask from a given volume",
default=False)
parser.add_option(
"--nsigma",
type="float",
default=1.,
help=
"number of times of sigma of the input volume to obtain the the large density cluster"
)
parser.add_option(
"--ndilation",
type="int",
default=3,
help=
"number of times of dilation applied to the largest cluster of density"
)
parser.add_option(
"--kernel_size",
type="int",
default=11,
help="convolution kernel for smoothing the edge of the mask")
parser.add_option(
"--gauss_standard_dev",
type="int",
default=9,
help="stanadard deviation value to generate Gaussian edge")
(options, args) = parser.parse_args()
global_def.BATCH = True
if options.phase_flip:
nargs = len(args)
if nargs != 2:
print "must provide name of input and output file!"
return
from EMAN2 import Processor
instack = args[0]
outstack = args[1]
nima = EMUtil.get_image_count(instack)
from filter import filt_ctf
for i in xrange(nima):
img = EMData()
img.read_image(instack, i)
try:
ctf = img.get_attr('ctf')
except:
print "no ctf information in input stack! Exiting..."
return
dopad = True
sign = 1
binary = 1 # phase flip
assert img.get_ysize() > 1
dict = ctf.to_dict()
dz = dict["defocus"]
cs = dict["cs"]
voltage = dict["voltage"]
pixel_size = dict["apix"]
b_factor = dict["bfactor"]
ampcont = dict["ampcont"]
dza = dict["dfdiff"]
azz = dict["dfang"]
if dopad and not img.is_complex(): ip = 1
else: ip = 0
params = {
"filter_type": Processor.fourier_filter_types.CTF_,
"defocus": dz,
"Cs": cs,
"voltage": voltage,
"Pixel_size": pixel_size,
"B_factor": b_factor,
"amp_contrast": ampcont,
"dopad": ip,
"binary": binary,
"sign": sign,
"dza": dza,
"azz": azz
}
tmp = Processor.EMFourierFilter(img, params)
tmp.set_attr_dict({"ctf": ctf})
tmp.write_image(outstack, i)
elif options.changesize:
nargs = len(args)
if nargs != 2:
ERROR("must provide name of input and output file!", "change size",
1)
return
from utilities import get_im
instack = args[0]
outstack = args[1]
sub_rate = float(options.ratio)
nima = EMUtil.get_image_count(instack)
from fundamentals import resample
for i in xrange(nima):
resample(get_im(instack, i), sub_rate).write_image(outstack, i)
elif options.isacgroup > -1:
nargs = len(args)
if nargs != 3:
ERROR("Three files needed on input!", "isacgroup", 1)
return
from utilities import get_im
instack = args[0]
m = get_im(args[1], int(options.isacgroup)).get_attr("members")
l = []
for k in m:
l.append(int(get_im(args[0], k).get_attr(options.params)))
from utilities import write_text_file
write_text_file(l, args[2])
elif options.isacselect:
nargs = len(args)
if nargs != 2:
ERROR("Two files needed on input!", "isacgroup", 1)
return
from utilities import get_im
nima = EMUtil.get_image_count(args[0])
m = []
for k in xrange(nima):
m += get_im(args[0], k).get_attr("members")
m.sort()
from utilities import write_text_file
write_text_file(m, args[1])
elif options.pw:
nargs = len(args)
if nargs < 2:
ERROR("must provide name of input and output file!", "pw", 1)
return
from utilities import get_im
d = get_im(args[0])
nx = d.get_xsize()
ny = d.get_ysize()
if nargs == 3: mask = get_im(args[2])
wn = int(options.wn)
if wn == -1:
wn = max(nx, ny)
else:
if ((wn < nx) or (wn < ny)):
ERROR("window size cannot be smaller than the image size",
"pw", 1)
n = EMUtil.get_image_count(args[0])
from utilities import model_blank, model_circle, pad
from EMAN2 import periodogram
p = model_blank(wn, wn)
for i in xrange(n):
d = get_im(args[0], i)
if nargs == 3:
d *= mask
st = Util.infomask(d, None, True)
d -= st[0]
p += periodogram(pad(d, wn, wn, 1, 0.))
p /= n
p.write_image(args[1])
elif options.adjpw:
if len(args) < 3:
ERROR(
"filt_by_rops input target output fl aa (the last two are optional parameters of a low-pass filter)",
"adjpw", 1)
return
img_stack = args[0]
from math import sqrt
from fundamentals import rops_table, fft
from utilities import read_text_file, get_im
from filter import filt_tanl, filt_table
if (args[1][-3:] == 'txt'):
rops_dst = read_text_file(args[1])
else:
rops_dst = rops_table(get_im(args[1]))
out_stack = args[2]
if (len(args) > 4):
fl = float(args[3])
aa = float(args[4])
else:
fl = -1.0
aa = 0.0
nimage = EMUtil.get_image_count(img_stack)
for i in xrange(nimage):
img = fft(get_im(img_stack, i))
rops_src = rops_table(img)
assert len(rops_dst) == len(rops_src)
table = [0.0] * len(rops_dst)
for j in xrange(len(rops_dst)):
table[j] = sqrt(rops_dst[j] / rops_src[j])
if (fl > 0.0):
img = filt_tanl(img, fl, aa)
img = fft(filt_table(img, table))
img.write_image(out_stack, i)
elif options.rotpw != None:
if len(args) != 1:
ERROR("Only one input permitted", "rotpw", 1)
return
from utilities import write_text_file, get_im
from fundamentals import rops_table
from math import log10
t = rops_table(get_im(args[0]))
x = range(len(t))
r = [0.0] * len(x)
for i in x:
r[i] = log10(t[i])
write_text_file([t, r, x], options.rotpw)
elif options.transformparams != None:
if len(args) != 2:
ERROR(
"Please provide names of input and output files with orientation parameters",
"transformparams", 1)
return
from utilities import read_text_row, write_text_row
transf = [0.0] * 6
spl = options.transformparams.split(',')
for i in xrange(len(spl)):
transf[i] = float(spl[i])
write_text_row(rotate_shift_params(read_text_row(args[0]), transf),
args[1])
elif options.makedb != None:
nargs = len(args)
if nargs != 1:
print "must provide exactly one argument denoting database key under which the input params will be stored"
return
dbkey = args[0]
print "database key under which params will be stored: ", dbkey
gbdb = js_open_dict("e2boxercache/gauss_box_DB.json")
parmstr = 'dummy:' + options.makedb[0]
(processorname, param_dict) = parsemodopt(parmstr)
dbdict = {}
for pkey in param_dict:
if (pkey == 'invert_contrast') or (pkey == 'use_variance'):
if param_dict[pkey] == 'True':
dbdict[pkey] = True
else:
dbdict[pkey] = False
else:
dbdict[pkey] = param_dict[pkey]
gbdb[dbkey] = dbdict
elif options.generate_projections:
nargs = len(args)
if nargs != 3:
ERROR("Must provide name of input structure(s) from which to generate projections, name of output projection stack, and prefix for output micrographs."\
"sxprocess - generate projections",1)
return
inpstr = args[0]
outstk = args[1]
micpref = args[2]
parmstr = 'dummy:' + options.generate_projections[0]
(processorname, param_dict) = parsemodopt(parmstr)
parm_CTF = False
parm_format = 'bdb'
parm_apix = 2.5
if 'CTF' in param_dict:
if param_dict['CTF'] == 'True':
parm_CTF = True
if 'format' in param_dict:
parm_format = param_dict['format']
if 'apix' in param_dict:
parm_apix = float(param_dict['apix'])
boxsize = 64
if 'boxsize' in param_dict:
boxsize = int(param_dict['boxsize'])
print "pixel size: ", parm_apix, " format: ", parm_format, " add CTF: ", parm_CTF, " box size: ", boxsize
scale_mult = 2500
sigma_add = 1.5
sigma_proj = 30.0
sigma2_proj = 17.5
sigma_gauss = 0.3
sigma_mic = 30.0
sigma2_mic = 17.5
sigma_gauss_mic = 0.3
if 'scale_mult' in param_dict:
scale_mult = float(param_dict['scale_mult'])
if 'sigma_add' in param_dict:
sigma_add = float(param_dict['sigma_add'])
if 'sigma_proj' in param_dict:
sigma_proj = float(param_dict['sigma_proj'])
if 'sigma2_proj' in param_dict:
sigma2_proj = float(param_dict['sigma2_proj'])
if 'sigma_gauss' in param_dict:
sigma_gauss = float(param_dict['sigma_gauss'])
if 'sigma_mic' in param_dict:
sigma_mic = float(param_dict['sigma_mic'])
if 'sigma2_mic' in param_dict:
sigma2_mic = float(param_dict['sigma2_mic'])
if 'sigma_gauss_mic' in param_dict:
sigma_gauss_mic = float(param_dict['sigma_gauss_mic'])
from filter import filt_gaussl, filt_ctf
from utilities import drop_spider_doc, even_angles, model_gauss, delete_bdb, model_blank, pad, model_gauss_noise, set_params2D, set_params_proj
from projection import prep_vol, prgs
seed(14567)
delta = 29
angles = even_angles(delta, 0.0, 89.9, 0.0, 359.9, "S")
nangle = len(angles)
modelvol = []
nvlms = EMUtil.get_image_count(inpstr)
from utilities import get_im
for k in xrange(nvlms):
modelvol.append(get_im(inpstr, k))
nx = modelvol[0].get_xsize()
if nx != boxsize:
ERROR("Requested box dimension does not match dimension of the input model.", \
"sxprocess - generate projections",1)
nvol = 10
volfts = [[] for k in xrange(nvlms)]
for k in xrange(nvlms):
for i in xrange(nvol):
sigma = sigma_add + random() # 1.5-2.5
addon = model_gauss(sigma, boxsize, boxsize, boxsize, sigma,
sigma, 38, 38, 40)
scale = scale_mult * (0.5 + random())
vf, kb = prep_vol(modelvol[k] + scale * addon)
volfts[k].append(vf)
del vf, modelvol
if parm_format == "bdb":
stack_data = "bdb:" + outstk
delete_bdb(stack_data)
else:
stack_data = outstk + ".hdf"
Cs = 2.0
pixel = parm_apix
voltage = 120.0
ampcont = 10.0
ibd = 4096 / 2 - boxsize
iprj = 0
width = 240
xstart = 8 + boxsize / 2
ystart = 8 + boxsize / 2
rowlen = 17
from random import randint
params = []
for idef in xrange(3, 8):
irow = 0
icol = 0
mic = model_blank(4096, 4096)
defocus = idef * 0.5 #0.2
if parm_CTF:
astampl = defocus * 0.15
astangl = 50.0
ctf = generate_ctf([
defocus, Cs, voltage, pixel, ampcont, 0.0, astampl, astangl
])
for i in xrange(nangle):
for k in xrange(12):
dphi = 8.0 * (random() - 0.5)
dtht = 8.0 * (random() - 0.5)
psi = 360.0 * random()
phi = angles[i][0] + dphi
tht = angles[i][1] + dtht
s2x = 4.0 * (random() - 0.5)
s2y = 4.0 * (random() - 0.5)
params.append([phi, tht, psi, s2x, s2y])
ivol = iprj % nvol
#imgsrc = randint(0,nvlms-1)
imgsrc = iprj % nvlms
proj = prgs(volfts[imgsrc][ivol], kb,
[phi, tht, psi, -s2x, -s2y])
x = xstart + irow * width
y = ystart + icol * width
mic += pad(proj, 4096, 4096, 1, 0.0, x - 2048, y - 2048, 0)
proj = proj + model_gauss_noise(sigma_proj, nx, nx)
if parm_CTF:
proj = filt_ctf(proj, ctf)
proj.set_attr_dict({"ctf": ctf, "ctf_applied": 0})
proj = proj + filt_gaussl(
model_gauss_noise(sigma2_proj, nx, nx), sigma_gauss)
proj.set_attr("origimgsrc", imgsrc)
proj.set_attr("test_id", iprj)
# flags describing the status of the image (1 = true, 0 = false)
set_params2D(proj, [0.0, 0.0, 0.0, 0, 1.0])
set_params_proj(proj, [phi, tht, psi, s2x, s2y])
proj.write_image(stack_data, iprj)
icol += 1
if icol == rowlen:
icol = 0
irow += 1
iprj += 1
mic += model_gauss_noise(sigma_mic, 4096, 4096)
if parm_CTF:
#apply CTF
mic = filt_ctf(mic, ctf)
mic += filt_gaussl(model_gauss_noise(sigma2_mic, 4096, 4096),
sigma_gauss_mic)
mic.write_image(micpref + "%1d.hdf" % (idef - 3), 0)
drop_spider_doc("params.txt", params)
elif options.importctf != None:
print ' IMPORTCTF '
from utilities import read_text_row, write_text_row
from random import randint
import subprocess
grpfile = 'groupid%04d' % randint(1000, 9999)
ctfpfile = 'ctfpfile%04d' % randint(1000, 9999)
cterr = [options.defocuserror / 100.0, options.astigmatismerror]
ctfs = read_text_row(options.importctf)
for kk in xrange(len(ctfs)):
root, name = os.path.split(ctfs[kk][-1])
ctfs[kk][-1] = name[:-4]
if (options.input[:4] != 'bdb:'):
ERROR('Sorry, only bdb files implemented', 'importctf', 1)
d = options.input[4:]
#try: str = d.index('*')
#except: str = -1
from string import split
import glob
uu = os.path.split(d)
uu = os.path.join(uu[0], 'EMAN2DB', uu[1] + '.bdb')
flist = glob.glob(uu)
for i in xrange(len(flist)):
root, name = os.path.split(flist[i])
root = root[:-7]
name = name[:-4]
fil = 'bdb:' + os.path.join(root, name)
sourcemic = EMUtil.get_all_attributes(fil, 'ptcl_source_image')
nn = len(sourcemic)
gctfp = []
groupid = []
for kk in xrange(nn):
junk, name2 = os.path.split(sourcemic[kk])
name2 = name2[:-4]
ctfp = [-1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
for ll in xrange(len(ctfs)):
if (name2 == ctfs[ll][-1]):
# found correct
if (ctfs[ll][8] / ctfs[ll][0] <= cterr[0]):
# acceptable defocus error
ctfp = ctfs[ll][:8]
if (ctfs[ll][10] > cterr[1]):
# error of astigmatism exceed the threshold, set astigmatism to zero.
ctfp[6] = 0.0
ctfp[7] = 0.0
gctfp.append(ctfp)
groupid.append(kk)
break
if (len(groupid) > 0):
write_text_row(groupid, grpfile)
write_text_row(gctfp, ctfpfile)
cmd = "{} {} {} {}".format(
'e2bdb.py', fil, '--makevstack=bdb:' + root + 'G' + name,
'--list=' + grpfile)
#print cmd
subprocess.call(cmd, shell=True)
cmd = "{} {} {} {}".format('sxheader.py',
'bdb:' + root + 'G' + name,
'--params=ctf',
'--import=' + ctfpfile)
#print cmd
subprocess.call(cmd, shell=True)
else:
print ' >>> Group ', name, ' skipped.'
cmd = "{} {} {}".format("rm -f", grpfile, ctfpfile)
subprocess.call(cmd, shell=True)
elif options.scale > 0.0:
from utilities import read_text_row, write_text_row
scale = options.scale
nargs = len(args)
if nargs != 2:
print "Please provide names of input and output file!"
return
p = read_text_row(args[0])
for i in xrange(len(p)):
p[i][3] /= scale
p[i][4] /= scale
write_text_row(p, args[1])
elif options.adaptive_mask:
from utilities import get_im
from morphology import adaptive_mask
nsigma = options.nsigma
ndilation = options.ndilation
kernel_size = options.kernel_size
gauss_standard_dev = options.gauss_standard_dev
nargs = len(args)
if nargs > 2:
print "Too many inputs are given, try again!"
return
else:
inputvol = get_im(args[0])
input_path, input_file_name = os.path.split(args[0])
input_file_name_root, ext = os.path.splitext(input_file_name)
if nargs == 2: mask_file_name = args[1]
else:
mask_file_name = "adaptive_mask_for" + input_file_name_root + ".hdf" # Only hdf file is output.
mask3d = adaptive_mask(inputvol, nsigma, ndilation, kernel_size,
gauss_standard_dev)
mask3d.write_image(mask_file_name)
else:
ERROR("Please provide option name", "sxprocess.py", 1)
