Esempio n. 1
0
def shrink_or_enlarge_or_stay_the_same(images, shrink_ratio, target_nx,
                                       target_radius, newx, nima):
    from fundamentals import resample
    from utilities import pad

    if newx >= target_nx:
        msk = model_circle(target_radius, target_nx, target_nx)
    else:
        msk = model_circle(newx // 2 - 2, newx, newx)

    do_not_stay_the_same_size = shrink_ratio != 1.0
    cut_to_window = newx > target_nx
    need_padding = newx < target_nx

    for im in xrange(nima):
        if do_not_stay_the_same_size:
            images[im] = resample(images[im], shrink_ratio)
        if cut_to_window:
            images[im] = Util.window(images[im], target_nx, target_nx, 1)

        p = Util.infomask(images[im], msk, False)
        images[im] -= p[0]
        p = Util.infomask(images[im], msk, True)
        images[im] /= p[1]

        if need_padding:
            images[im] = pad(images[im], target_nx, target_nx, 1, 0.0)
Esempio n. 2
0
def generate_helimic(refvol, outdir, pixel, CTF=False, Cs=2.0,voltage = 200.0, ampcont = 10.0, nonoise = False, rand_seed=14567):
	
	from utilities	 import model_blank, model_gauss, model_gauss_noise, pad, get_im
	from random 	 import random
	from projection  import prgs, prep_vol
	from filter	     import filt_gaussl, filt_ctf
	from EMAN2 	     import EMAN2Ctf
	
	if os.path.exists(outdir):   ERROR('Output directory exists, please change the name and restart the program', "sxhelical_demo", 1)
	os.mkdir(outdir)
	seed(rand_seed)
	Util.set_randnum_seed(rand_seed)
	angles =[]
	for i in xrange(3):
		angles.append( [0.0+60.0*i, 90.0-i*5, 0.0, 0.0, 0.0] )

	nangle   = len(angles)

	volfts = get_im(refvol)
	nx = volfts.get_xsize()
	ny = volfts.get_ysize()
	nz = volfts.get_zsize()
	volfts, kbx, kby, kbz = prep_vol( volfts )
	iprj   = 0
	width  = 500
	xstart = 0
	ystart = 0

	for idef in xrange(3,6):
		mic = model_blank(2048, 2048)
		#defocus = idef*0.2
		defocus = idef*0.6     ##@ming
		if CTF :
			#ctf = EMAN2Ctf()
			#ctf.from_dict( {"defocus":defocus, "cs":Cs, "voltage":voltage, "apix":pixel, "ampcont":ampcont, "bfactor":0.0} )
			from utilities import generate_ctf
			ctf = generate_ctf([defocus,2,200,1.84,0.0,ampcont,defocus*0.2,80])   ##@ming   the range of astigmatism amplitude is between 10 percent and 22 percent. 20 percent is a good choice.
		i = idef - 4
		for k in xrange(1):
			psi  = 90 + 10*i			
 			proj = prgs(volfts, kbz, [angles[idef-3][0], angles[idef-3][1], psi, 0.0, 0.0], kbx, kby)
			proj = Util.window(proj, 320, nz)		
			mic += pad(proj, 2048, 2048, 1, 0.0, 750*i, 20*i, 0)

		if not nonoise:  mic += model_gauss_noise(30.0,2048,2048)
		if CTF :
			#apply CTF
			mic = filt_ctf(mic, ctf)

		if not nonoise:  mic += filt_gaussl(model_gauss_noise(17.5,2048,2048), 0.3)

		mic.write_image("%s/mic%1d.hdf"%(outdir, idef-3),0)
Esempio n. 3
0
    def test_one_wave(self):
        test_data = \
            """
		    oo
		    FirmID                                         VendorName  ACSI_14  ACSI_15  ACSI_16  ACSI_14_BENCH  ACSI_15_BENCH  ACSI_16_BENCH
		 0       1                        Apple [Cellular Telephones]       79     80.0     81.0             78           78.0           79.0
		 1       1                         Apple [Personal Computers]       84     84.0      NaN             78           77.0            NaN
		 2       2   Google [Internet Search Engines and Information]       83     78.0      NaN             80           76.0            NaN
		 3       2                    Google+ [Internet Social Media]       84      NaN      NaN             71            NaN            NaN
		 4       2           YouTube (Google) [Internet Social Media]       78     76.0     77.0             71           74.0           73.0
		 5       3  Bing (Microsoft) [Internet Search Engines and ...       73     72.0     75.0             80           76.0           77.0
		 6       3                      Microsoft [Computer Software]       75     75.0     80.0             76           74.0           81.0

		 t
		    FirmID                                         VendorName  ACSI_14  ACSI_15  ACSI_16  ACSI_16_PADDED  ACSI_14_BENCH  ACSI_15_BENCH  ACSI_16_BENCH  ACSI_16_BENCH_PADDED
		 0       1                        Apple [Cellular Telephones]       79     80.0     81.0            81.0             78           78.0           79.0                  79.0
		 1       1                         Apple [Personal Computers]       84     84.0      NaN            84.0             78           77.0            NaN                  77.0
		 2       2   Google [Internet Search Engines and Information]       83     78.0      NaN            78.0             80           76.0            NaN                  76.0
		 3       2                    Google+ [Internet Social Media]       84      NaN      NaN             NaN             71            NaN            NaN                   NaN
		 4       2           YouTube (Google) [Internet Social Media]       78     76.0     77.0            77.0             71           74.0           73.0                  73.0
		 5       3  Bing (Microsoft) [Internet Search Engines and ...       73     72.0     75.0            75.0             80           76.0           77.0                  77.0
		 6       3                      Microsoft [Computer Software]       75     75.0     80.0            80.0             76           74.0           81.0                  81.0


		    """
        ofn = 'test_pad_original_from_ACSI_clean_and_augmented.csv'
        vendor = 'test_vendor'
        self.orig = pd.read_csv(
            os.path.join(os.path.abspath(path), vendor, ofn))

        tfn = 'test_pad_target_ACSI_clean_and_augmented_1_wave.csv'
        self.target = pd.read_csv(
            os.path.join(os.path.abspath(path), vendor, tfn))

        t = self.target
        o = self.orig

        result = pad(self.orig, waves_to_update=1, vendor='ACSI', wave=2016)

        r = result

        for col, dtype in zip(result.columns, result.dtypes):
            if not dtype == object:
                diff = result[col].round(5) - self.target[col].round(5)
                self.assertFalse(diff.any())
        a = 1
Esempio n. 4
0
def shrink_or_enlarge_or_stay_the_same(images, shrink_ratio, target_nx, target_radius, newx, nima):
	from fundamentals import resample
	from utilities import pad
	
	if newx >= target_nx  :
		msk = model_circle(target_radius, target_nx, target_nx)
	else:
		msk = model_circle(newx//2-2, newx,  newx)
		
	do_not_stay_the_same_size = shrink_ratio != 1.0 
	cut_to_window = newx > target_nx 
	need_padding = newx < target_nx 

	for im in xrange(nima):
		if do_not_stay_the_same_size : images[im]  = resample(images[im], shrink_ratio)
		if cut_to_window : images[im] = Util.window(images[im], target_nx, target_nx, 1)
		
		p = Util.infomask(images[im], msk, False)
		images[im] -= p[0]
		p = Util.infomask(images[im], msk, True)
		images[im] /= p[1]
		
		if need_padding : images[im] = pad(images[im], target_nx, target_nx, 1, 0.0)
Esempio n. 5
0
def main(args):
	progname = os.path.basename(sys.argv[0])
	usage = ( progname + " stack_file  output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter" +
			" --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp" +
			"  --generation=generation  --rand_seed=rand_seed>" )
	
	parser = OptionParser(usage,version=SPARXVERSION)
	parser.add_option("--radius",                type="int",           help="particle radius: there is no default, a sensible number has to be provided, units - pixels (default required int)")
	parser.add_option("--target_radius",         type="int",           default=29,         help="target particle radius: actual particle radius on which isac will process data. Images will be shrinked/enlarged to achieve this radius (default 29)")
	parser.add_option("--target_nx",             type="int",           default=76,         help="target particle image size: actual image size on which isac will process data. Images will be shrinked/enlarged according to target particle radius and then cut/padded to achieve target_nx size. When xr > 0, the final image size for isac processing is 'target_nx + xr - 1'  (default 76)")
	parser.add_option("--img_per_grp",           type="int",           default=100,        help="number of images per class: in the ideal case (essentially maximum size of class) (default 100)")
	parser.add_option("--CTF",                   action="store_true",  default=False,      help="apply phase-flip for CTF correction: if set the data will be phase-flipped using CTF information included in image headers (default False)")
	parser.add_option("--ir",                    type="int",           default=1,          help="inner ring: of the resampling to polar coordinates. units - pixels (default 1)")
	parser.add_option("--rs",                    type="int",           default=1,          help="ring step: of the resampling to polar coordinates. units - pixels (default 1)")
	parser.add_option("--xr",                    type="int",           default=1,         help="x range: of translational search. By default, set by the program. (default 1)")
	parser.add_option("--yr",                    type="int",           default=-1,         help="y range: of translational search. By default, same as xr. (default -1)")
	parser.add_option("--ts",                    type="float",         default=1.0,        help="search step: of translational search: units - pixels (default 1.0)")
	parser.add_option("--maxit",                 type="int",           default=30,         help="number of iterations for reference-free alignment: (default 30)")
	#parser.add_option("--snr",            type="float",        default=1.0,     help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)")
	parser.add_option("--center_method",         type="int",           default=-1,         help="method for centering: of global 2D average during initial prealignment of data (0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7) (default -1)")
	parser.add_option("--dst",                   type="float",         default=90.0,       help="discrete angle used in within group alignment: (default 90.0)")
	parser.add_option("--FL",                    type="float",         default=0.2,        help="lowest stopband: frequency used in the tangent filter (default 0.2)")
	parser.add_option("--FH",                    type="float",         default=0.3,        help="highest stopband: frequency used in the tangent filter (default 0.3)")
	parser.add_option("--FF",                    type="float",         default=0.2,        help="fall-off of the tangent filter: (default 0.2)")
	parser.add_option("--init_iter",             type="int",           default=3,          help="SAC initialization iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC initialization (default 3)")
	parser.add_option("--main_iter",             type="int",           default=3,          help="SAC main iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC (default 3)")
	parser.add_option("--iter_reali",            type="int",           default=1,          help="SAC stability check interval: every iter_reali iterations of SAC stability checking is performed (default 1)")
	parser.add_option("--match_first",           type="int",           default=1,          help="number of iterations to run 2-way matching in the first phase: (default 1)")
	parser.add_option("--max_round",             type="int",           default=20,         help="maximum rounds: of generating candidate class averages in the first phase (default 20)")
	parser.add_option("--match_second",          type="int",           default=5,          help="number of iterations to run 2-way (or 3-way) matching in the second phase: (default 5)")
	parser.add_option("--stab_ali",              type="int",           default=5,          help="number of alignments when checking stability: (default 5)")
	parser.add_option("--thld_err",              type="float",         default=0.7,        help="threshold of pixel error when checking stability: equals root mean square of distances between corresponding pixels from set of found transformations and theirs average transformation, depends linearly on square of radius (parameter ou). units - pixels. (default 0.7)")
	parser.add_option("--indep_run",             type="int",           default=4,          help="level of m-way matching for reproducibility tests: By default, perform full ISAC to 4-way matching. Value indep_run=2 will restrict ISAC to 2-way matching and 3 to 3-way matching.  Note the number of used MPI processes requested in mpirun must be a multiplicity of indep_run. (default 4)")
	parser.add_option("--thld_grp",              type="int",           default=10,         help="minimum size of reproducible class (default 10)")
	parser.add_option("--n_generations",         type="int",           default=100,        help="maximum number of generations: program stops when reaching this total number of generations: (default 100)")
	#parser.add_option("--candidatesexist",action="store_true", default=False,   help="Candidate class averages exist use them (default False)")
	parser.add_option("--rand_seed",             type="int",           help="random seed set before calculations: useful for testing purposes. By default, total randomness (type int)")
	parser.add_option("--new",                   action="store_true",  default=False,      help="use new code: (default False)")
	parser.add_option("--debug",                 action="store_true",  default=False,      help="debug info printout: (default False)")

	# must be switched off in production
	parser.add_option("--use_latest_master_directory",action="store_true",  default=False,      help="use latest master directory: when active, the program looks for the latest directory that starts with the word 'master', so the user does not need to provide a directory name. (default False)")
	
	parser.add_option("--restart_section",       type="string",        default=' ',        help="restart section: each generation (iteration) contains three sections: 'restart', 'candidate_class_averages', and 'reproducible_class_averages'. To restart from a particular step, for example, generation 4 and section 'candidate_class_averages' the following option is needed: '--restart_section=candidate_class_averages,4'. The option requires no white space before or after the comma. The default behavior is to restart execution from where it stopped intentionally or unintentionally. For default restart, it is assumed that the name of the directory is provided as argument. Alternatively, the '--use_latest_master_directory' option can be used. (default ' ')")
	parser.add_option("--stop_after_candidates", action="store_true",  default=False,      help="stop after candidates: stops after the 'candidate_class_averages' section. (default False)")

	##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
	parser.add_option("--return_options",        action="store_true", dest="return_options", default=False, help = SUPPRESS_HELP)
	parser.add_option("--skip_prealignment",     action="store_true",  default=False,      help="skip pre-alignment step: to be used if images are already centered. 2dalignment directory will still be generated but the parameters will be zero. (default False)")

	required_option_list = ['radius']
	(options, args) = parser.parse_args(args)

	if options.return_options:
		return parser
	
	if len(args) > 2:
		print "usage: " + usage
		print "Please run '" + progname + " -h' for detailed options"
		sys.exit()
	
	if global_def.CACHE_DISABLE:
		from utilities import disable_bdb_cache
		disable_bdb_cache()
	global_def.BATCH = True
	
	from isac import iter_isac
	from fundamentals import rot_shift2D, resample
	from utilities import pad, combine_params2

	command_line_provided_stack_filename = args[0]
	
	main_node = 0
	mpi_init(0, [])
	myid = mpi_comm_rank(MPI_COMM_WORLD)
	nproc = mpi_comm_size(MPI_COMM_WORLD)
	
	mpi_barrier(MPI_COMM_WORLD)
	if(myid == main_node):
		print "****************************************************************"
		Util.version()
		print "****************************************************************"
		sys.stdout.flush()
	mpi_barrier(MPI_COMM_WORLD)
	

	# Making sure all required options appeared.
	for required_option in required_option_list:
		if not options.__dict__[required_option]:
			print "\n ==%s== mandatory option is missing.\n"%required_option
			print "Please run '" + progname + " -h' for detailed options"
			return 1

	radi  = options.radius
	target_radius  = options.target_radius
	target_nx  = options.target_nx
	center_method  = options.center_method
	if(radi < 1):  ERROR("Particle radius has to be provided!","sxisac",1,myid)

	
	use_latest_master_directory = options.use_latest_master_directory
	stop_after_candidates = options.stop_after_candidates
	# program_state_stack.restart_location_title_from_command_line = options.restart_section
	
	from utilities import qw
	program_state_stack.PROGRAM_STATE_VARIABLES = set(qw("""
		isac_generation
	"""))

	# create or reuse master directory
	masterdir = ""
	stack_processed_by_ali2d_base__filename = ""
	stack_processed_by_ali2d_base__filename__without_master_dir = ""
	error_status = 0
	if len(args) == 2:
		masterdir = args[1]
	elif len(args) == 1:
		if use_latest_master_directory:
			all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)]
			import re; r = re.compile("^master.*$")
			all_dirs = filter(r.match, all_dirs)
			if len(all_dirs)>0:
				# all_dirs = max(all_dirs, key=os.path.getctime)
				masterdir = max(all_dirs, key=os.path.getmtime)
				
	#Create folder for all results or check if there is one created already
	if(myid == main_node):
		if( masterdir == ""):
			timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime())
			masterdir = "master"+timestring
			cmd = "{} {}".format("mkdir", masterdir)
			cmdexecute(cmd)
		elif not os.path.exists(masterdir):
			# os.path.exists(masterdir) does not exist
			masterdir = args[1]
			cmd = "{} {}".format("mkdir", masterdir)
			cmdexecute(cmd)

		if(args[0][:4] == "bdb:"): filename = args[0][4:]
		else:                      filename = args[0][:-4]
		filename = os.path.basename(filename)
		stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join(masterdir, filename )
		stack_processed_by_ali2d_base__filename__without_master_dir  = "bdb:" + filename
	if_error_then_all_processes_exit_program(error_status)

	# send masterdir to all processes
	masterdir = send_string_to_all(masterdir)

	if myid == 0:
		if options.restart_section != " ":
			if os.path.exists(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)):
				stored_stack, stored_state = restore_program_stack_and_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE))
				import re
				if "," in options.restart_section:
					parsed_restart_section_option = options.restart_section.split(",")
					stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%parsed_restart_section_option[0], stored_state[-1]["location_in_program"])
					generation_str_format = parsed_restart_section_option[1]
					if generation_str_format != "":
						isac_generation_from_command_line = int(generation_str_format)
						stored_state[-1]["isac_generation"] = isac_generation_from_command_line 
					else:
						isac_generation_from_command_line = 1
						if "isac_generation" in stored_state[-1]:
							del stored_state[-1]["isac_generation"]
				else:
					isac_generation_from_command_line = -1
					stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%options.restart_section, stored_state[-1]["location_in_program"])
					if "isac_generation" in stored_state[-1]:
						del stored_state[-1]["isac_generation"]
				store_program_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE), stored_state, stored_stack)
			else:
				print "Please remove the restart_section option from the command line. The program must be started from the beginning."			
				mpi_finalize()
				sys.exit()
		else:
			isac_generation_from_command_line = -1
	
	program_state_stack(locals(), getframeinfo(currentframe()), os.path.join(masterdir,NAME_OF_JSON_STATE_FILE))
	

	stack_processed_by_ali2d_base__filename = send_string_to_all(stack_processed_by_ali2d_base__filename)
	stack_processed_by_ali2d_base__filename__without_master_dir = \
		send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir)

	# previous code 2016-05-05--20-14-12-153
	# #  PARAMETERS OF THE PROCEDURE
	# if( options.xr == -1 ):
	# 	#  Default values
	# 	# target_nx = 76
	# 	# target_radius = 29
	# 	target_xr = 1
	# else:  #  nx//2
	# 	#  Check below!
	# 	target_xr = options.xr
	# 	# target_nx = 76 + target_xr - 1 # subtract one, which is default
	# 	target_nx += target_xr - 1 # subtract one, which is default
	# 	# target_radius = 29

	target_xr = options.xr
	target_nx += target_xr - 1 # subtract one, which is default
	
	if (options.yr == -1):
		yr = options.xr
	else:
		yr = options.yr


	mpi_barrier(MPI_COMM_WORLD)

	# Initialization of stacks
	if(myid == main_node):
		print "command_line_provided_stack_filename", command_line_provided_stack_filename
		number_of_images_in_stack = EMUtil.get_image_count(command_line_provided_stack_filename)
	else:
		number_of_images_in_stack = 0

	number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node)
	
	nxrsteps = 4
	
	init2dir = os.path.join(masterdir,"2dalignment")
	
	# from mpi import mpi_finalize
	# mpi_finalize()
	# sys.stdout.flush()
	# sys.exit()
	
	
	if not os.path.exists(os.path.join(init2dir, "Finished_initial_2d_alignment.txt")):
	
		if(myid == 0):
			import subprocess
			from logger import Logger, BaseLogger_Files
			#  Create output directory
			log2d = Logger(BaseLogger_Files())
			log2d.prefix = os.path.join(init2dir)
			cmd = "mkdir -p "+log2d.prefix
			outcome = subprocess.call(cmd, shell=True)
			log2d.prefix += "/"
			# outcome = subprocess.call("sxheader.py  "+command_line_provided_stack_filename+"   --params=xform.align2d  --zero", shell=True)
		else:
			outcome = 0
			log2d = None

		if(myid == main_node):
			a = get_im(command_line_provided_stack_filename)
			nnxo = a.get_xsize()
		else:
			nnxo = 0
		nnxo = bcast_number_to_all(nnxo, source_node = main_node)

		image_start, image_end = MPI_start_end(number_of_images_in_stack, nproc, myid)

		if options.skip_prealignment:
			params2d = [[0.0,0.0,0.0,0] for i in xrange(image_start, image_end)]
		else:

			original_images = EMData.read_images(command_line_provided_stack_filename, range(image_start,image_end))
			#  We assume the target radius will be 29, and xr = 1.  
			shrink_ratio = float(target_radius)/float(radi)

			for im in xrange(len(original_images)):
				if(shrink_ratio != 1.0):
					original_images[im]  = resample(original_images[im], shrink_ratio)

			nx = original_images[0].get_xsize()
			# nx = int(nx*shrink_ratio + 0.5)

			txrm = (nx - 2*(target_radius+1))//2
			if(txrm < 0):  			ERROR( "ERROR!!   Radius of the structure larger than the window data size permits   %d"%(radi), "sxisac",1, myid)
			if(txrm/nxrsteps>0):
				tss = ""
				txr = ""
				while(txrm/nxrsteps>0):
					tts=txrm/nxrsteps
					tss += "  %d"%tts
					txr += "  %d"%(tts*nxrsteps)
					txrm =txrm//2
			else:
				tss = "1"
				txr = "%d"%txrm
			
			# print "nx, txr, txrm, tss", nx, txr, txrm, tss
		# from mpi import mpi_finalize
		# mpi_finalize()
		# sys.stdout.flush()
		# sys.exit()



			# section ali2d_base

			params2d = ali2d_base(original_images, init2dir, None, 1, target_radius, 1, txr, txr, tss, \
				False, 90.0, center_method, 14, options.CTF, 1.0, False, \
				"ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False)
			
			del original_images
			
			for i in xrange(len(params2d)):
				alpha, sx, sy, mirror = combine_params2(0, params2d[i][1],params2d[i][2], 0, -params2d[i][0], 0, 0, 0)
				sx /= shrink_ratio
				sy /= shrink_ratio
				params2d[i][0] = 0.0
				params2d[i][1] = sx
				params2d[i][2] = sy
				params2d[i][3] = 0
				#set_params2D(aligned_images[i],[0.0, sx,sy,0.,1.0])

		mpi_barrier(MPI_COMM_WORLD)
		tmp = params2d[:]
		tmp = wrap_mpi_gatherv(tmp, main_node, MPI_COMM_WORLD)
		if( myid == main_node ):		
			if options.skip_prealignment:
				print "========================================="
				print "Even though there is no alignment step, '%s' params are set to zero for later use."%os.path.join(init2dir, "initial2Dparams.txt")
				print "========================================="
			write_text_row(tmp,os.path.join(init2dir, "initial2Dparams.txt"))
		del tmp
		mpi_barrier(MPI_COMM_WORLD)
	
		#  We assume the target image size will be target_nx, radius will be 29, and xr = 1.  
		#  Note images can be also padded, in which case shrink_ratio > 1.
		shrink_ratio = float(target_radius)/float(radi)
		
		aligned_images = EMData.read_images(command_line_provided_stack_filename, range(image_start,image_end))
		nx = aligned_images[0].get_xsize()
		nima = len(aligned_images)
		newx = int(nx*shrink_ratio + 0.5)


		
		while not os.path.exists(os.path.join(init2dir, "initial2Dparams.txt")):
			import time
			time.sleep(1)
		mpi_barrier(MPI_COMM_WORLD)
		
		params = read_text_row(os.path.join(init2dir, "initial2Dparams.txt"))
		params = params[image_start:image_end]


		msk = model_circle(radi, nx, nx)
		for im in xrange(nima):
			st = Util.infomask(aligned_images[im], msk, False)
			aligned_images[im] -= st[0]
			if options.CTF:
				aligned_images[im] = filt_ctf(aligned_images[im], aligned_images[im].get_attr("ctf"), binary = True)
	
		if(shrink_ratio < 1.0):
			if    newx > target_nx  :
				msk = model_circle(target_radius, target_nx, target_nx)
				for im in xrange(nima):
					#  Here we should use only shifts
					#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
					#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
					#aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
					aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
					aligned_images[im]  = resample(aligned_images[im], shrink_ratio)
					aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
					p = Util.infomask(aligned_images[im], msk, False)
					aligned_images[im] -= p[0]
					p = Util.infomask(aligned_images[im], msk, True)
					aligned_images[im] /= p[1]
			elif  newx == target_nx :
				msk = model_circle(target_radius, target_nx, target_nx)
				for im in xrange(nima):
					#  Here we should use only shifts
					#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
					#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
					aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
					aligned_images[im]  = resample(aligned_images[im], shrink_ratio)
					p = Util.infomask(aligned_images[im], msk, False)
					aligned_images[im] -= p[0]
					p = Util.infomask(aligned_images[im], msk, True)
					aligned_images[im] /= p[1]
			elif  newx < target_nx  :	
				msk = model_circle(newx//2-2, newx,  newx)
				for im in xrange(nima):
					#  Here we should use only shifts
					#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
					#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
					aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
					aligned_images[im]  = resample(aligned_images[im], shrink_ratio)
					p = Util.infomask(aligned_images[im], msk, False)
					aligned_images[im] -= p[0]
					p = Util.infomask(aligned_images[im], msk, True)
					aligned_images[im] /= p[1]
					aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
		elif(shrink_ratio == 1.0):
			if    newx > target_nx  :
				msk = model_circle(target_radius, target_nx, target_nx)
				for im in xrange(nima):
					#  Here we should use only shifts
					#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
					#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
					aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
					aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
					p = Util.infomask(aligned_images[im], msk, False)
					aligned_images[im] -= p[0]
					p = Util.infomask(aligned_images[im], msk, True)
					aligned_images[im] /= p[1]
			elif  newx == target_nx :
				msk = model_circle(target_radius, target_nx, target_nx)
				for im in xrange(nima):
					#  Here we should use only shifts
					#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
					#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
					aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
					p = Util.infomask(aligned_images[im], msk, False)
					aligned_images[im] -= p[0]
					p = Util.infomask(aligned_images[im], msk, True)
					aligned_images[im] /= p[1]
			elif  newx < target_nx  :			
				msk = model_circle(newx//2-2, newx,  newx)
				for im in xrange(nima):
					#  Here we should use only shifts
					#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
					#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
					aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
					#aligned_images[im]  = resample(aligned_images[im], shrink_ratio)
					p = Util.infomask(aligned_images[im], msk, False)
					aligned_images[im] -= p[0]
					p = Util.infomask(aligned_images[im], msk, True)
					aligned_images[im] /= p[1]
					aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
		elif(shrink_ratio > 1.0):
			if    newx > target_nx  :
				msk = model_circle(target_radius, target_nx, target_nx)
				for im in xrange(nima):
					#  Here we should use only shifts
					#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
					#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
					aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
					aligned_images[im]  = resample(aligned_images[im], shrink_ratio)
					aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
					p = Util.infomask(aligned_images[im], msk, False)
					aligned_images[im] -= p[0]
					p = Util.infomask(aligned_images[im], msk, True)
					aligned_images[im] /= p[1]
			elif  newx == target_nx :
				msk = model_circle(target_radius, target_nx, target_nx)
				for im in xrange(nima):
					#  Here we should use only shifts
					#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
					#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
					aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
					aligned_images[im]  = resample(aligned_images[im], shrink_ratio)
					p = Util.infomask(aligned_images[im], msk, False)
					aligned_images[im] -= p[0]
					p = Util.infomask(aligned_images[im], msk, True)
					aligned_images[im] /= p[1]
			elif  newx < target_nx  :
				msk = model_circle(newx//2-2, newx,  newx)
				for im in xrange(nima):
					#  Here we should use only shifts
					#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
					#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
					aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
					aligned_images[im]  = resample(aligned_images[im], shrink_ratio)
					p = Util.infomask(aligned_images[im], msk, False)
					aligned_images[im] -= p[0]
					p = Util.infomask(aligned_images[im], msk, True)
					aligned_images[im] /= p[1]
					aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
		del msk
	
		gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images, myid)
		number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node)
	
		if( myid == main_node ):
			for i in range(number_of_images_in_stack):  aligned_images[i].write_image(stack_processed_by_ali2d_base__filename,i)
			#  It has to be explicitly closed
			from EMAN2db import db_open_dict
			DB = db_open_dict(stack_processed_by_ali2d_base__filename)
			DB.close()
	
			fp = open(os.path.join(masterdir,"README_shrink_ratio.txt"), "w")
			output_text = """
			Since, for processing purposes, isac changes the image dimensions,
			adjustment of pixel size needs to be made in subsequent steps, (e.g.
			running sxviper.py). The shrink ratio for this particular isac run is
			--------
			%.5f
			%.5f
			--------
			To get the pixel size for the isac output the user needs to divide
			the original pixel size by the above value. This info is saved in
			the following file: README_shrink_ratio.txt
			"""%(shrink_ratio, radi)
			fp.write(output_text); fp.flush() ;fp.close()
			print output_text
			fp = open(os.path.join(init2dir, "Finished_initial_2d_alignment.txt"), "w"); fp.flush() ;fp.close()
	else:
		if( myid == main_node ):
			print "Skipping 2d alignment since it was already done!"

	mpi_barrier(MPI_COMM_WORLD)
	
	
	# from mpi import mpi_finalize
	# mpi_finalize()
	# sys.stdout.flush()
	# sys.exit()
	

	os.chdir(masterdir)
	
	if program_state_stack(locals(), getframeinfo(currentframe())):
	# if 1:
		pass
		if (myid == main_node):
			cmdexecute("sxheader.py  --consecutive  --params=originalid   %s"%stack_processed_by_ali2d_base__filename__without_master_dir)
			cmdexecute("e2bdb.py %s --makevstack=%s_000"%(stack_processed_by_ali2d_base__filename__without_master_dir, stack_processed_by_ali2d_base__filename__without_master_dir))

	if (myid == main_node):
		main_dir_no = get_latest_directory_increment_value("./", NAME_OF_MAIN_DIR, myformat="%04d")
		print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no
		if isac_generation_from_command_line < 0:
			if os.path.exists(NAME_OF_JSON_STATE_FILE):
				stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE)
				if "isac_generation" in stored_state[-1]:
					isac_generation_from_command_line = stored_state[-1]["isac_generation"]
				else:
					isac_generation_from_command_line = -1
		if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no: 
			for i in xrange(isac_generation_from_command_line+1, main_dir_no + 1):
				if i == isac_generation_from_command_line+1:
					backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
					cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
				cmdexecute("mv  " + NAME_OF_MAIN_DIR + "%04d"%i +  " 000_backup" + "%05d"%backup_dir_no)
				cmdexecute("rm  " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%i)
				
			# it includes both command line and json file
			my_restart_section = stored_state[-1]["location_in_program"].split("___")[-1]
			if "restart" in my_restart_section:
				if "backup_dir_no" not in locals():
					backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
					cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
				cmdexecute("mv  " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line +  " 000_backup" + "%05d"%backup_dir_no)
				cmdexecute("rm  " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%isac_generation_from_command_line )
			elif "candidate_class_averages" in my_restart_section:
				if "backup_dir_no" not in locals():
					backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
					cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
				cmdexecute("mv  " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line +  " 000_backup" + "%05d"%backup_dir_no)
				cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line)
				# cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line)
			elif "reproducible_class_averages" in my_restart_section:
				cmdexecute("rm -rf " + NAME_OF_MAIN_DIR + "%04d/ali_params_generation_*"%isac_generation_from_command_line)
				cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_generation*"%isac_generation_from_command_line)
		else:
			if os.path.exists(NAME_OF_JSON_STATE_FILE):
				stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE)
				if "isac_generation" in stored_state[-1]:
					isac_generation_from_command_line = stored_state[-1]["isac_generation"]
				else:
					isac_generation_from_command_line = 1
			else:
				isac_generation_from_command_line = 1
	else:
		isac_generation_from_command_line = 0
		
		
		
	isac_generation_from_command_line = mpi_bcast(isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]
	isac_generation = isac_generation_from_command_line - 1
	
	if (myid == main_node):
		if isac_generation == 0:
			cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation)
			write_text_file([1], os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_accounted.txt"%isac_generation))
			write_text_file(range(number_of_images_in_stack), os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_unaccounted.txt"%isac_generation))

	#  Stopping criterion should be inside the program.
	while True:
		isac_generation += 1
		if isac_generation > options.n_generations:
			break

		data64_stack_current = "bdb:../"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d"%isac_generation

		program_state_stack.restart_location_title = "restart"
		if program_state_stack(locals(), getframeinfo(currentframe())):
			if (myid == main_node):
				cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation)
				# reference the original stack
				list_file = os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1), "generation_%d_unaccounted.txt"%(isac_generation - 1))
				cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\
						stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file))
			mpi_barrier(MPI_COMM_WORLD)

		os.chdir(NAME_OF_MAIN_DIR + "%04d"%isac_generation)

		program_state_stack.restart_location_title = "candidate_class_averages"
		if program_state_stack(locals(), getframeinfo(currentframe())):

			iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\
				options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
				options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
				options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new)

		# program_state_stack.restart_location_title = "stopped_program1"
		# program_state_stack(locals(), getframeinfo(currentframe()))
		
		program_state_stack.restart_location_title = "stop_after_candidates"
		program_state_stack(locals(), getframeinfo(currentframe()))
		if stop_after_candidates:
			mpi_finalize()
			sys.exit()

		exit_program = 0
		if(myid == main_node):
			if not os.path.exists("class_averages_candidate_generation_%d.hdf"%isac_generation):
				print "This generation (%d) no class average candidates were generated! Finishing."%isac_generation
				exit_program = 1
		exit_program = int(mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
		if exit_program:
			os.chdir("..")
			break

		program_state_stack.restart_location_title = "reproducible_class_averages"
		if program_state_stack(locals(), getframeinfo(currentframe())):


			iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\
				options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
				options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
				options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new)
			pass

		os.chdir("..")

		if(myid == main_node):
			accounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation),"generation_%d_accounted.txt"%(isac_generation)))
			number_of_accounted_images = len(accounted_images)
			un_accounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation),"generation_%d_unaccounted.txt"%(isac_generation)))
			number_of_un_accounted_images = len(un_accounted_images)
		else:
			number_of_accounted_images = 0
			number_of_un_accounted_images = 0

		number_of_accounted_images = int(mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
		number_of_un_accounted_images = int(mpi_bcast(number_of_un_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
		
		if number_of_accounted_images == 0:
			if(myid == main_node):
				print "This generation (%d) there are no accounted images! Finishing."%isac_generation
			break

		while (myid == main_node):
			def files_are_missing(isac_generation):
				for i in xrange(1, isac_generation + 1):
					if not os.path.exists("generation_%04d/class_averages_generation_%d.hdf"%(i,i)):
						print "Error: generation_%04d/class_averages_generation_%d.hdf is missing! Exiting."%(i,i)
						return 1
				return 0
			
			if files_are_missing(isac_generation):
				break
				
			cmdexecute("rm -f class_averages.hdf")
			cpy(["generation_%04d/class_averages_generation_%d.hdf"%(i,i) for i in xrange(1, isac_generation + 1)], "class_averages.hdf")
			
			break

		if number_of_un_accounted_images == 0:
			if(myid == main_node):
				print "This generation (%d) there are no un accounted images! Finishing."%isac_generation
			break


	program_state_stack(locals(), getframeinfo(currentframe()), last_call="__LastCall")

	mpi_barrier(MPI_COMM_WORLD)
	mpi_finalize()
Esempio n. 6
0
def helicalshiftali_MPI(stack,
                        maskfile=None,
                        maxit=100,
                        CTF=False,
                        snr=1.0,
                        Fourvar=False,
                        search_rng=-1):
    from applications import MPI_start_end
    from utilities import model_circle, model_blank, get_image, peak_search, get_im, pad
    from utilities import reduce_EMData_to_root, bcast_EMData_to_all, send_attr_dict, file_type, bcast_number_to_all, bcast_list_to_all
    from pap_statistics import varf2d_MPI
    from fundamentals import fft, ccf, rot_shift3D, rot_shift2D, fshift
    from utilities import get_params2D, set_params2D, chunks_distribution
    from utilities import print_msg, print_begin_msg, print_end_msg
    import os
    import sys
    from mpi import mpi_init, mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD
    from mpi import mpi_reduce, mpi_bcast, mpi_barrier, mpi_gatherv
    from mpi import MPI_SUM, MPI_FLOAT, MPI_INT
    from time import time
    from pixel_error import ordersegments
    from math import sqrt, atan2, tan, pi

    nproc = mpi_comm_size(MPI_COMM_WORLD)
    myid = mpi_comm_rank(MPI_COMM_WORLD)
    main_node = 0

    ftp = file_type(stack)

    if myid == main_node:
        print_begin_msg("helical-shiftali_MPI")

    max_iter = int(maxit)
    if (myid == main_node):
        infils = EMUtil.get_all_attributes(stack, "filament")
        ptlcoords = EMUtil.get_all_attributes(stack, 'ptcl_source_coord')
        filaments = ordersegments(infils, ptlcoords)
        total_nfils = len(filaments)
        inidl = [0] * total_nfils
        for i in range(total_nfils):
            inidl[i] = len(filaments[i])
        linidl = sum(inidl)
        nima = linidl
        tfilaments = []
        for i in range(total_nfils):
            tfilaments += filaments[i]
        del filaments
    else:
        total_nfils = 0
        linidl = 0
    total_nfils = bcast_number_to_all(total_nfils, source_node=main_node)
    if myid != main_node:
        inidl = [-1] * total_nfils
    inidl = bcast_list_to_all(inidl, myid, source_node=main_node)
    linidl = bcast_number_to_all(linidl, source_node=main_node)
    if myid != main_node:
        tfilaments = [-1] * linidl
    tfilaments = bcast_list_to_all(tfilaments, myid, source_node=main_node)
    filaments = []
    iendi = 0
    for i in range(total_nfils):
        isti = iendi
        iendi = isti + inidl[i]
        filaments.append(tfilaments[isti:iendi])
    del tfilaments, inidl

    if myid == main_node:
        print_msg("total number of filaments: %d" % total_nfils)
    if total_nfils < nproc:
        ERROR(
            'number of CPUs (%i) is larger than the number of filaments (%i), please reduce the number of CPUs used'
            % (nproc, total_nfils), "ehelix_MPI", 1, myid)

    #  balanced load
    temp = chunks_distribution([[len(filaments[i]), i]
                                for i in range(len(filaments))],
                               nproc)[myid:myid + 1][0]
    filaments = [filaments[temp[i][1]] for i in range(len(temp))]
    nfils = len(filaments)

    #filaments = [[0,1]]
    #print "filaments",filaments
    list_of_particles = []
    indcs = []
    k = 0
    for i in range(nfils):
        list_of_particles += filaments[i]
        k1 = k + len(filaments[i])
        indcs.append([k, k1])
        k = k1
    data = EMData.read_images(stack, list_of_particles)
    ldata = len(data)
    print("ldata=", ldata)
    nx = data[0].get_xsize()
    ny = data[0].get_ysize()
    if maskfile == None:
        mrad = min(nx, ny) // 2 - 2
        mask = pad(model_blank(2 * mrad + 1, ny, 1, 1.0), nx, ny, 1, 0.0)
    else:
        mask = get_im(maskfile)

    # apply initial xform.align2d parameters stored in header
    init_params = []
    for im in range(ldata):
        t = data[im].get_attr('xform.align2d')
        init_params.append(t)
        p = t.get_params("2d")
        data[im] = rot_shift2D(data[im], p['alpha'], p['tx'], p['ty'],
                               p['mirror'], p['scale'])

    if CTF:
        from filter import filt_ctf
        from morphology import ctf_img
        ctf_abs_sum = EMData(nx, ny, 1, False)
        ctf_2_sum = EMData(nx, ny, 1, False)
    else:
        ctf_2_sum = None
        ctf_abs_sum = None

    from utilities import info

    for im in range(ldata):
        data[im].set_attr('ID', list_of_particles[im])
        st = Util.infomask(data[im], mask, False)
        data[im] -= st[0]
        if CTF:
            ctf_params = data[im].get_attr("ctf")
            qctf = data[im].get_attr("ctf_applied")
            if qctf == 0:
                data[im] = filt_ctf(fft(data[im]), ctf_params)
                data[im].set_attr('ctf_applied', 1)
            elif qctf != 1:
                ERROR('Incorrectly set qctf flag', "helicalshiftali_MPI", 1,
                      myid)
            ctfimg = ctf_img(nx, ctf_params, ny=ny)
            Util.add_img2(ctf_2_sum, ctfimg)
            Util.add_img_abs(ctf_abs_sum, ctfimg)
        else:
            data[im] = fft(data[im])

    del list_of_particles

    if CTF:
        reduce_EMData_to_root(ctf_2_sum, myid, main_node)
        reduce_EMData_to_root(ctf_abs_sum, myid, main_node)
    if CTF:
        if myid != main_node:
            del ctf_2_sum
            del ctf_abs_sum
        else:
            temp = EMData(nx, ny, 1, False)
            tsnr = 1. / snr
            for i in range(0, nx + 2, 2):
                for j in range(ny):
                    temp.set_value_at(i, j, tsnr)
                    temp.set_value_at(i + 1, j, 0.0)
            #info(ctf_2_sum)
            Util.add_img(ctf_2_sum, temp)
            #info(ctf_2_sum)
            del temp

    total_iter = 0
    shift_x = [0.0] * ldata

    for Iter in range(max_iter):
        if myid == main_node:
            start_time = time()
            print_msg("Iteration #%4d\n" % (total_iter))
        total_iter += 1
        avg = EMData(nx, ny, 1, False)
        for im in range(ldata):
            Util.add_img(avg, fshift(data[im], shift_x[im]))

        reduce_EMData_to_root(avg, myid, main_node)

        if myid == main_node:
            if CTF: tavg = Util.divn_filter(avg, ctf_2_sum)
            else: tavg = Util.mult_scalar(avg, 1.0 / float(nima))
        else:
            tavg = model_blank(nx, ny)

        if Fourvar:
            bcast_EMData_to_all(tavg, myid, main_node)
            vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF)

        if myid == main_node:
            if Fourvar:
                tavg = fft(Util.divn_img(fft(tavg), vav))
                vav_r = Util.pack_complex_to_real(vav)
            # normalize and mask tavg in real space
            tavg = fft(tavg)
            stat = Util.infomask(tavg, mask, False)
            tavg -= stat[0]
            Util.mul_img(tavg, mask)
            tavg.write_image("tavg.hdf", Iter)
            # For testing purposes: shift tavg to some random place and see if the centering is still correct
            #tavg = rot_shift3D(tavg,sx=3,sy=-4)

        if Fourvar: del vav
        bcast_EMData_to_all(tavg, myid, main_node)
        tavg = fft(tavg)

        sx_sum = 0.0
        nxc = nx // 2

        for ifil in range(nfils):
            """
			# Calculate filament average
			avg = EMData(nx, ny, 1, False)
			filnima = 0
			for im in xrange(indcs[ifil][0], indcs[ifil][1]):
				Util.add_img(avg, data[im])
				filnima += 1
			tavg = Util.mult_scalar(avg, 1.0/float(filnima))
			"""
            # Calculate 1D ccf between each segment and filament average
            nsegms = indcs[ifil][1] - indcs[ifil][0]
            ctx = [None] * nsegms
            pcoords = [None] * nsegms
            for im in range(indcs[ifil][0], indcs[ifil][1]):
                ctx[im - indcs[ifil][0]] = Util.window(ccf(tavg, data[im]), nx,
                                                       1)
                pcoords[im - indcs[ifil][0]] = data[im].get_attr(
                    'ptcl_source_coord')
                #ctx[im-indcs[ifil][0]].write_image("ctx.hdf",im-indcs[ifil][0])
                #print "  CTX  ",myid,im,Util.infomask(ctx[im-indcs[ifil][0]], None, True)
            # search for best x-shift
            cents = nsegms // 2

            dst = sqrt(
                max((pcoords[cents][0] - pcoords[0][0])**2 +
                    (pcoords[cents][1] - pcoords[0][1])**2,
                    (pcoords[cents][0] - pcoords[-1][0])**2 +
                    (pcoords[cents][1] - pcoords[-1][1])**2))
            maxincline = atan2(ny // 2 - 2 - float(search_rng), dst)
            kang = int(dst * tan(maxincline) + 0.5)
            #print  "  settings ",nsegms,cents,dst,search_rng,maxincline,kang

            # ## C code for alignment. @ming
            results = [0.0] * 3
            results = Util.helixshiftali(ctx, pcoords, nsegms, maxincline,
                                         kang, search_rng, nxc)
            sib = int(results[0])
            bang = results[1]
            qm = results[2]
            #print qm, sib, bang

            # qm = -1.e23
            #
            # 			for six in xrange(-search_rng, search_rng+1,1):
            # 				q0 = ctx[cents].get_value_at(six+nxc)
            # 				for incline in xrange(kang+1):
            # 					qt = q0
            # 					qu = q0
            # 					if(kang>0):  tang = tan(maxincline/kang*incline)
            # 					else:        tang = 0.0
            # 					for kim in xrange(cents+1,nsegms):
            # 						dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
            # 						xl = dst*tang+six+nxc
            # 						ixl = int(xl)
            # 						dxl = xl - ixl
            # 						#print "  A  ", ifil,six,incline,kim,xl,ixl,dxl
            # 						qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
            # 						xl = -dst*tang+six+nxc
            # 						ixl = int(xl)
            # 						dxl = xl - ixl
            # 						qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
            # 					for kim in xrange(cents):
            # 						dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
            # 						xl = -dst*tang+six+nxc
            # 						ixl = int(xl)
            # 						dxl = xl - ixl
            # 						qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
            # 						xl =  dst*tang+six+nxc
            # 						ixl = int(xl)
            # 						dxl = xl - ixl
            # 						qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
            # 					if( qt > qm ):
            # 						qm = qt
            # 						sib = six
            # 						bang = tang
            # 					if( qu > qm ):
            # 						qm = qu
            # 						sib = six
            # 						bang = -tang
            #if incline == 0:  print  "incline = 0  ",six,tang,qt,qu
            #print qm,six,sib,bang
            #print " got results   ",indcs[ifil][0], indcs[ifil][1], ifil,myid,qm,sib,tang,bang,len(ctx),Util.infomask(ctx[0], None, True)
            for im in range(indcs[ifil][0], indcs[ifil][1]):
                kim = im - indcs[ifil][0]
                dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 +
                           (pcoords[cents][1] - pcoords[kim][1])**2)
                if (kim < cents): xl = -dst * bang + sib
                else: xl = dst * bang + sib
                shift_x[im] = xl

            # Average shift
            sx_sum += shift_x[indcs[ifil][0] + cents]

        # #print myid,sx_sum,total_nfils
        sx_sum = mpi_reduce(sx_sum, 1, MPI_FLOAT, MPI_SUM, main_node,
                            MPI_COMM_WORLD)
        if myid == main_node:
            sx_sum = float(sx_sum[0]) / total_nfils
            print_msg("Average shift  %6.2f\n" % (sx_sum))
        else:
            sx_sum = 0.0
        sx_sum = 0.0
        sx_sum = bcast_number_to_all(sx_sum, source_node=main_node)
        for im in range(ldata):
            shift_x[im] -= sx_sum
            #print  "   %3d  %6.3f"%(im,shift_x[im])
        #exit()

    # combine shifts found with the original parameters
    for im in range(ldata):
        t1 = Transform()
        ##import random
        ##shix=random.randint(-10, 10)
        ##t1.set_params({"type":"2D","tx":shix})
        t1.set_params({"type": "2D", "tx": shift_x[im]})
        # combine t0 and t1
        tt = t1 * init_params[im]
        data[im].set_attr("xform.align2d", tt)
    # write out headers and STOP, under MPI writing has to be done sequentially
    mpi_barrier(MPI_COMM_WORLD)
    par_str = ["xform.align2d", "ID"]
    if myid == main_node:
        from utilities import file_type
        if (file_type(stack) == "bdb"):
            from utilities import recv_attr_dict_bdb
            recv_attr_dict_bdb(main_node, stack, data, par_str, 0, ldata,
                               nproc)
        else:
            from utilities import recv_attr_dict
            recv_attr_dict(main_node, stack, data, par_str, 0, ldata, nproc)
    else:
        send_attr_dict(main_node, data, par_str, 0, ldata)
    if myid == main_node: print_end_msg("helical-shiftali_MPI")
Esempio n. 7
0
def main():
    def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
        if mirror:
            m = 1
            alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
                                                       540.0 - psi, 0, 0, 1.0)
        else:
            m = 0
            alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
                                                       360.0 - psi, 0, 0, 1.0)
        return alpha, sx, sy, m

    progname = os.path.basename(sys.argv[0])
    usage = progname + " prj_stack  --ave2D= --var2D=  --ave3D= --var3D= --img_per_grp= --fl=15. --aa=0.01  --sym=symmetry --CTF"
    parser = OptionParser(usage, version=SPARXVERSION)

    parser.add_option("--output_dir",
                      type="string",
                      default="./",
                      help="output directory")
    parser.add_option("--ave2D",
                      type="string",
                      default=False,
                      help="write to the disk a stack of 2D averages")
    parser.add_option("--var2D",
                      type="string",
                      default=False,
                      help="write to the disk a stack of 2D variances")
    parser.add_option("--ave3D",
                      type="string",
                      default=False,
                      help="write to the disk reconstructed 3D average")
    parser.add_option("--var3D",
                      type="string",
                      default=False,
                      help="compute 3D variability (time consuming!)")
    parser.add_option("--img_per_grp",
                      type="int",
                      default=10,
                      help="number of neighbouring projections")
    parser.add_option("--no_norm",
                      action="store_true",
                      default=False,
                      help="do not use normalization")
    #parser.add_option("--radius", 	    type="int"         ,	default=-1   ,				help="radius for 3D variability" )
    parser.add_option("--npad",
                      type="int",
                      default=2,
                      help="number of time to pad the original images")
    parser.add_option("--sym", type="string", default="c1", help="symmetry")
    parser.add_option(
        "--fl",
        type="float",
        default=0.0,
        help=
        "cutoff freqency in absolute frequency (0.0-0.5). (Default - no filtration)"
    )
    parser.add_option(
        "--aa",
        type="float",
        default=0.0,
        help=
        "fall off of the filter. Put 0.01 if user has no clue about falloff (Default - no filtration)"
    )
    parser.add_option("--CTF",
                      action="store_true",
                      default=False,
                      help="use CFT correction")
    parser.add_option("--VERBOSE",
                      action="store_true",
                      default=False,
                      help="Long output for debugging")
    #parser.add_option("--MPI" , 		action="store_true",	default=False,				help="use MPI version")
    #parser.add_option("--radiuspca", 	type="int"         ,	default=-1   ,				help="radius for PCA" )
    #parser.add_option("--iter", 		type="int"         ,	default=40   ,				help="maximum number of iterations (stop criterion of reconstruction process)" )
    #parser.add_option("--abs", 		type="float"   ,        default=0.0  ,				help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
    #parser.add_option("--squ", 		type="float"   ,	    default=0.0  ,				help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
    parser.add_option(
        "--VAR",
        action="store_true",
        default=False,
        help="stack on input consists of 2D variances (Default False)")
    parser.add_option(
        "--decimate",
        type="float",
        default=1.0,
        help=
        "image decimate rate, a number larger (expand image) or less (shrink image) than 1. default is 1"
    )
    parser.add_option(
        "--window",
        type="int",
        default=0,
        help=
        "reduce images to a small image size without changing pixel_size. Default value is zero."
    )
    #parser.add_option("--SND",			action="store_true",	default=False,				help="compute squared normalized differences (Default False)")
    parser.add_option(
        "--nvec",
        type="int",
        default=0,
        help="number of eigenvectors, default = 0 meaning no PCA calculated")
    parser.add_option(
        "--symmetrize",
        action="store_true",
        default=False,
        help="Prepare input stack for handling symmetry (Default False)")

    (options, args) = parser.parse_args()
    #####
    from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD
    from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
    from applications import MPI_start_end
    from reconstruction import recons3d_em, recons3d_em_MPI
    from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
    from utilities import print_begin_msg, print_end_msg, print_msg
    from utilities import read_text_row, get_image, get_im
    from utilities import bcast_EMData_to_all, bcast_number_to_all
    from utilities import get_symt

    #  This is code for handling symmetries by the above program.  To be incorporated. PAP 01/27/2015

    from EMAN2db import db_open_dict

    # Set up global variables related to bdb cache
    if global_def.CACHE_DISABLE:
        from utilities import disable_bdb_cache
        disable_bdb_cache()

    # Set up global variables related to ERROR function
    global_def.BATCH = True

    # detect if program is running under MPI
    RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ
    if RUNNING_UNDER_MPI:
        global_def.MPI = True

    if options.symmetrize:
        if RUNNING_UNDER_MPI:
            try:
                sys.argv = mpi_init(len(sys.argv), sys.argv)
                try:
                    number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
                    if (number_of_proc > 1):
                        ERROR(
                            "Cannot use more than one CPU for symmetry prepration",
                            "sx3dvariability", 1)
                except:
                    pass
            except:
                pass
        if options.output_dir != "./" and not os.path.exists(
                options.output_dir):
            os.mkdir(options.output_dir)
        #  Input
        #instack = "Clean_NORM_CTF_start_wparams.hdf"
        #instack = "bdb:data"

        from logger import Logger, BaseLogger_Files
        if os.path.exists(os.path.join(options.output_dir, "log.txt")):
            os.remove(os.path.join(options.output_dir, "log.txt"))
        log_main = Logger(BaseLogger_Files())
        log_main.prefix = os.path.join(options.output_dir, "./")

        instack = args[0]
        sym = options.sym.lower()
        if (sym == "c1"):
            ERROR("There is no need to symmetrize stack for C1 symmetry",
                  "sx3dvariability", 1)

        line = ""
        for a in sys.argv:
            line += " " + a
        log_main.add(line)

        if (instack[:4] != "bdb:"):
            if output_dir == "./": stack = "bdb:data"
            else: stack = "bdb:" + options.output_dir + "/data"
            delete_bdb(stack)
            junk = cmdexecute("sxcpy.py  " + instack + "  " + stack)
        else:
            stack = instack

        qt = EMUtil.get_all_attributes(stack, 'xform.projection')

        na = len(qt)
        ts = get_symt(sym)
        ks = len(ts)
        angsa = [None] * na

        for k in xrange(ks):
            #Qfile = "Q%1d"%k
            if options.output_dir != "./":
                Qfile = os.path.join(options.output_dir, "Q%1d" % k)
            else:
                Qfile = os.path.join(options.output_dir, "Q%1d" % k)
            #delete_bdb("bdb:Q%1d"%k)
            delete_bdb("bdb:" + Qfile)
            #junk = cmdexecute("e2bdb.py  "+stack+"  --makevstack=bdb:Q%1d"%k)
            junk = cmdexecute("e2bdb.py  " + stack + "  --makevstack=bdb:" +
                              Qfile)
            #DB = db_open_dict("bdb:Q%1d"%k)
            DB = db_open_dict("bdb:" + Qfile)
            for i in xrange(na):
                ut = qt[i] * ts[k]
                DB.set_attr(i, "xform.projection", ut)
                #bt = ut.get_params("spider")
                #angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
            #write_text_row(angsa, 'ptsma%1d.txt'%k)
            #junk = cmdexecute("e2bdb.py  "+stack+"  --makevstack=bdb:Q%1d"%k)
            #junk = cmdexecute("sxheader.py  bdb:Q%1d  --params=xform.projection  --import=ptsma%1d.txt"%(k,k))
            DB.close()
        if options.output_dir == "./": delete_bdb("bdb:sdata")
        else: delete_bdb("bdb:" + options.output_dir + "/" + "sdata")
        #junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
        sdata = "bdb:" + options.output_dir + "/" + "sdata"
        print(sdata)
        junk = cmdexecute("e2bdb.py   " + options.output_dir +
                          "  --makevstack=" + sdata + " --filt=Q")
        #junk = cmdexecute("ls  EMAN2DB/sdata*")
        #a = get_im("bdb:sdata")
        a = get_im(sdata)
        a.set_attr("variabilitysymmetry", sym)
        #a.write_image("bdb:sdata")
        a.write_image(sdata)

    else:

        sys.argv = mpi_init(len(sys.argv), sys.argv)
        myid = mpi_comm_rank(MPI_COMM_WORLD)
        number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
        main_node = 0

        if len(args) == 1:
            stack = args[0]
        else:
            print(("usage: " + usage))
            print(("Please run '" + progname + " -h' for detailed options"))
            return 1

        t0 = time()
        # obsolete flags
        options.MPI = True
        options.nvec = 0
        options.radiuspca = -1
        options.iter = 40
        options.abs = 0.0
        options.squ = 0.0

        if options.fl > 0.0 and options.aa == 0.0:
            ERROR("Fall off has to be given for the low-pass filter",
                  "sx3dvariability", 1, myid)
        if options.VAR and options.SND:
            ERROR("Only one of var and SND can be set!", "sx3dvariability",
                  myid)
            exit()
        if options.VAR and (options.ave2D or options.ave3D or options.var2D):
            ERROR(
                "When VAR is set, the program cannot output ave2D, ave3D or var2D",
                "sx3dvariability", 1, myid)
            exit()
        #if options.SND and (options.ave2D or options.ave3D):
        #	ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid)
        #	exit()
        if options.nvec > 0:
            ERROR("PCA option not implemented", "sx3dvariability", 1, myid)
            exit()
        if options.nvec > 0 and options.ave3D == None:
            ERROR("When doing PCA analysis, one must set ave3D",
                  "sx3dvariability",
                  myid=myid)
            exit()
        import string
        options.sym = options.sym.lower()

        # if global_def.CACHE_DISABLE:
        # 	from utilities import disable_bdb_cache
        # 	disable_bdb_cache()
        # global_def.BATCH = True

        if myid == main_node:
            if options.output_dir != "./" and not os.path.exists(
                    options.output_dir):
                os.mkdir(options.output_dir)

        img_per_grp = options.img_per_grp
        nvec = options.nvec
        radiuspca = options.radiuspca

        from logger import Logger, BaseLogger_Files
        #if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt"))
        log_main = Logger(BaseLogger_Files())
        log_main.prefix = os.path.join(options.output_dir, "./")

        if myid == main_node:
            line = ""
            for a in sys.argv:
                line += " " + a
            log_main.add(line)
            log_main.add("-------->>>Settings given by all options<<<-------")
            log_main.add("instack  		    :" + stack)
            log_main.add("output_dir        :" + options.output_dir)
            log_main.add("var3d   		    :" + options.var3D)

        if myid == main_node:
            line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
            #print_begin_msg("sx3dvariability")
            msg = "sx3dvariability"
            log_main.add(msg)
            print(line, msg)
            msg = ("%-70s:  %s\n" % ("Input stack", stack))
            log_main.add(msg)
            print(line, msg)

        symbaselen = 0
        if myid == main_node:
            nima = EMUtil.get_image_count(stack)
            img = get_image(stack)
            nx = img.get_xsize()
            ny = img.get_ysize()
            if options.sym != "c1":
                imgdata = get_im(stack)
                try:
                    i = imgdata.get_attr("variabilitysymmetry").lower()
                    if (i != options.sym):
                        ERROR(
                            "The symmetry provided does not agree with the symmetry of the input stack",
                            "sx3dvariability",
                            myid=myid)
                except:
                    ERROR(
                        "Input stack is not prepared for symmetry, please follow instructions",
                        "sx3dvariability",
                        myid=myid)
                from utilities import get_symt
                i = len(get_symt(options.sym))
                if ((nima / i) * i != nima):
                    ERROR(
                        "The length of the input stack is incorrect for symmetry processing",
                        "sx3dvariability",
                        myid=myid)
                symbaselen = nima / i
            else:
                symbaselen = nima
        else:
            nima = 0
            nx = 0
            ny = 0
        nima = bcast_number_to_all(nima)
        nx = bcast_number_to_all(nx)
        ny = bcast_number_to_all(ny)
        Tracker = {}
        Tracker["total_stack"] = nima
        if options.decimate == 1.:
            if options.window != 0:
                nx = options.window
                ny = options.window
        else:
            if options.window == 0:
                nx = int(nx * options.decimate)
                ny = int(ny * options.decimate)
            else:
                nx = int(options.window * options.decimate)
                ny = nx
        Tracker["nx"] = nx
        Tracker["ny"] = ny
        Tracker["nz"] = nx
        symbaselen = bcast_number_to_all(symbaselen)
        if radiuspca == -1: radiuspca = nx / 2 - 2

        if myid == main_node:
            line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
            msg = "%-70s:  %d\n" % ("Number of projection", nima)
            log_main.add(msg)
            print(line, msg)
        img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
        """
		if options.SND:
			from projection		import prep_vol, prgs
			from statistics		import im_diff
			from utilities		import get_im, model_circle, get_params_proj, set_params_proj
			from utilities		import get_ctf, generate_ctf
			from filter			import filt_ctf
		
			imgdata = EMData.read_images(stack, range(img_begin, img_end))

			if options.CTF:
				vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
			else:
				vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)

			bcast_EMData_to_all(vol, myid)
			volft, kb = prep_vol(vol)

			mask = model_circle(nx/2-2, nx, ny)
			varList = []
			for i in xrange(img_begin, img_end):
				phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
				ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
				if options.CTF:
					ctf_params = get_ctf(imgdata[i-img_begin])
					ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
				diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
				diff2 = diff*diff
				set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
				varList.append(diff2)
			mpi_barrier(MPI_COMM_WORLD)
		"""
        if options.VAR:
            #varList   = EMData.read_images(stack, range(img_begin, img_end))
            varList = []
            this_image = EMData()
            for index_of_particle in xrange(img_begin, img_end):
                this_image.read_image(stack, index_of_particle)
                varList.append(
                    image_decimate_window_xform_ctf(this_image,
                                                    options.decimate,
                                                    options.window,
                                                    options.CTF))
        else:
            from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
            from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
            from utilities import model_blank, nearest_proj, model_circle
            from applications import pca
            from statistics import avgvar, avgvar_ctf, ccc
            from filter import filt_tanl
            from morphology import threshold, square_root
            from projection import project, prep_vol, prgs
            from sets import Set

            if myid == main_node:
                t1 = time()
                proj_angles = []
                aveList = []
                tab = EMUtil.get_all_attributes(stack, 'xform.projection')
                for i in xrange(nima):
                    t = tab[i].get_params('spider')
                    phi = t['phi']
                    theta = t['theta']
                    psi = t['psi']
                    x = theta
                    if x > 90.0: x = 180.0 - x
                    x = x * 10000 + psi
                    proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
                t2 = time()
                line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
                msg = "%-70s:  %d\n" % ("Number of neighboring projections",
                                        img_per_grp)
                log_main.add(msg)
                print(line, msg)
                msg = "...... Finding neighboring projections\n"
                log_main.add(msg)
                print(line, msg)
                if options.VERBOSE:
                    msg = "Number of images per group: %d" % img_per_grp
                    log_main.add(msg)
                    print(line, msg)
                    msg = "Now grouping projections"
                    log_main.add(msg)
                    print(line, msg)
                proj_angles.sort()
            proj_angles_list = [0.0] * (nima * 4)
            if myid == main_node:
                for i in xrange(nima):
                    proj_angles_list[i * 4] = proj_angles[i][1]
                    proj_angles_list[i * 4 + 1] = proj_angles[i][2]
                    proj_angles_list[i * 4 + 2] = proj_angles[i][3]
                    proj_angles_list[i * 4 + 3] = proj_angles[i][4]
            proj_angles_list = bcast_list_to_all(proj_angles_list, myid,
                                                 main_node)
            proj_angles = []
            for i in xrange(nima):
                proj_angles.append([
                    proj_angles_list[i * 4], proj_angles_list[i * 4 + 1],
                    proj_angles_list[i * 4 + 2],
                    int(proj_angles_list[i * 4 + 3])
                ])
            del proj_angles_list
            proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp,
                                                  range(img_begin, img_end))

            all_proj = Set()
            for im in proj_list:
                for jm in im:
                    all_proj.add(proj_angles[jm][3])

            all_proj = list(all_proj)
            if options.VERBOSE:
                print("On node %2d, number of images needed to be read = %5d" %
                      (myid, len(all_proj)))

            index = {}
            for i in xrange(len(all_proj)):
                index[all_proj[i]] = i
            mpi_barrier(MPI_COMM_WORLD)

            if myid == main_node:
                line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
                msg = ("%-70s:  %.2f\n" %
                       ("Finding neighboring projections lasted [s]",
                        time() - t2))
                log_main.add(msg)
                print(msg)
                msg = ("%-70s:  %d\n" %
                       ("Number of groups processed on the main node",
                        len(proj_list)))
                log_main.add(msg)
                print(line, msg)
                if options.VERBOSE:
                    print("Grouping projections took: ", (time() - t2) / 60,
                          "[min]")
                    print("Number of groups on main node: ", len(proj_list))
            mpi_barrier(MPI_COMM_WORLD)

            if myid == main_node:
                line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
                msg = ("...... calculating the stack of 2D variances \n")
                log_main.add(msg)
                print(line, msg)
                if options.VERBOSE:
                    print("Now calculating the stack of 2D variances")

            proj_params = [0.0] * (nima * 5)
            aveList = []
            varList = []
            if nvec > 0:
                eigList = [[] for i in xrange(nvec)]

            if options.VERBOSE:
                print("Begin to read images on processor %d" % (myid))
            ttt = time()
            #imgdata = EMData.read_images(stack, all_proj)
            imgdata = []
            for index_of_proj in xrange(len(all_proj)):
                #img     = EMData()
                #img.read_image(stack, all_proj[index_of_proj])
                dmg = image_decimate_window_xform_ctf(
                    get_im(stack, all_proj[index_of_proj]), options.decimate,
                    options.window, options.CTF)
                #print dmg.get_xsize(), "init"
                imgdata.append(dmg)
            if options.VERBOSE:
                print("Reading images on processor %d done, time = %.2f" %
                      (myid, time() - ttt))
                print("On processor %d, we got %d images" %
                      (myid, len(imgdata)))
            mpi_barrier(MPI_COMM_WORLD)
            '''	
			imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
			if options.fl > 0.0:
				for k in xrange(len(imgdata2)):
					imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
			if options.CTF:
				vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
			else:
				vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
			if myid == main_node:
				vol.write_image("vol_ctf.hdf")
				print_msg("Writing to the disk volume reconstructed from averages as		:  %s\n"%("vol_ctf.hdf"))
			del vol, imgdata2
			mpi_barrier(MPI_COMM_WORLD)
			'''
            from applications import prepare_2d_forPCA
            from utilities import model_blank
            for i in xrange(len(proj_list)):
                ki = proj_angles[proj_list[i][0]][3]
                if ki >= symbaselen: continue
                mi = index[ki]
                phiM, thetaM, psiM, s2xM, s2yM = get_params_proj(imgdata[mi])

                grp_imgdata = []
                for j in xrange(img_per_grp):
                    mj = index[proj_angles[proj_list[i][j]][3]]
                    phi, theta, psi, s2x, s2y = get_params_proj(imgdata[mj])
                    alpha, sx, sy, mirror = params_3D_2D_NEW(
                        phi, theta, psi, s2x, s2y, mirror_list[i][j])
                    if thetaM <= 90:
                        if mirror == 0:
                            alpha, sx, sy, scale = compose_transform2(
                                alpha, sx, sy, 1.0, phiM - phi, 0.0, 0.0, 1.0)
                        else:
                            alpha, sx, sy, scale = compose_transform2(
                                alpha, sx, sy, 1.0, 180 - (phiM - phi), 0.0,
                                0.0, 1.0)
                    else:
                        if mirror == 0:
                            alpha, sx, sy, scale = compose_transform2(
                                alpha, sx, sy, 1.0, -(phiM - phi), 0.0, 0.0,
                                1.0)
                        else:
                            alpha, sx, sy, scale = compose_transform2(
                                alpha, sx, sy, 1.0, -(180 - (phiM - phi)), 0.0,
                                0.0, 1.0)
                    set_params2D(imgdata[mj], [alpha, sx, sy, mirror, 1.0])
                    grp_imgdata.append(imgdata[mj])
                    #print grp_imgdata[j].get_xsize(), imgdata[mj].get_xsize()

                if not options.no_norm:
                    #print grp_imgdata[j].get_xsize()
                    mask = model_circle(nx / 2 - 2, nx, nx)
                    for k in xrange(img_per_grp):
                        ave, std, minn, maxx = Util.infomask(
                            grp_imgdata[k], mask, False)
                        grp_imgdata[k] -= ave
                        grp_imgdata[k] /= std
                    del mask

                if options.fl > 0.0:
                    from filter import filt_ctf, filt_table
                    from fundamentals import fft, window2d
                    nx2 = 2 * nx
                    ny2 = 2 * ny
                    if options.CTF:
                        from utilities import pad
                        for k in xrange(img_per_grp):
                            grp_imgdata[k] = window2d(
                                fft(
                                    filt_tanl(
                                        filt_ctf(
                                            fft(
                                                pad(grp_imgdata[k], nx2, ny2,
                                                    1, 0.0)),
                                            grp_imgdata[k].get_attr("ctf"),
                                            binary=1), options.fl,
                                        options.aa)), nx, ny)
                            #grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
                            #grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
                    else:
                        for k in xrange(img_per_grp):
                            grp_imgdata[k] = filt_tanl(grp_imgdata[k],
                                                       options.fl, options.aa)
                            #grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
                            #grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
                else:
                    from utilities import pad, read_text_file
                    from filter import filt_ctf, filt_table
                    from fundamentals import fft, window2d
                    nx2 = 2 * nx
                    ny2 = 2 * ny
                    if options.CTF:
                        from utilities import pad
                        for k in xrange(img_per_grp):
                            grp_imgdata[k] = window2d(
                                fft(
                                    filt_ctf(fft(
                                        pad(grp_imgdata[k], nx2, ny2, 1, 0.0)),
                                             grp_imgdata[k].get_attr("ctf"),
                                             binary=1)), nx, ny)
                            #grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
                            #grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
                '''
				if i < 10 and myid == main_node:
					for k in xrange(10):
						grp_imgdata[k].write_image("grp%03d.hdf"%i, k)
				'''
                """
				if myid == main_node and i==0:
					for pp in xrange(len(grp_imgdata)):
						grp_imgdata[pp].write_image("pp.hdf", pp)
				"""
                ave, grp_imgdata = prepare_2d_forPCA(grp_imgdata)
                """
				if myid == main_node and i==0:
					for pp in xrange(len(grp_imgdata)):
						grp_imgdata[pp].write_image("qq.hdf", pp)
				"""

                var = model_blank(nx, ny)
                for q in grp_imgdata:
                    Util.add_img2(var, q)
                Util.mul_scalar(var, 1.0 / (len(grp_imgdata) - 1))
                # Switch to std dev
                var = square_root(threshold(var))
                #if options.CTF:	ave, var = avgvar_ctf(grp_imgdata, mode="a")
                #else:	            ave, var = avgvar(grp_imgdata, mode="a")
                """
				if myid == main_node:
					ave.write_image("avgv.hdf",i)
					var.write_image("varv.hdf",i)
				"""

                set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
                set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])

                aveList.append(ave)
                varList.append(var)

                if options.VERBOSE:
                    print("%5.2f%% done on processor %d" %
                          (i * 100.0 / len(proj_list), myid))
                if nvec > 0:
                    eig = pca(input_stacks=grp_imgdata,
                              subavg="",
                              mask_radius=radiuspca,
                              nvec=nvec,
                              incore=True,
                              shuffle=False,
                              genbuf=True)
                    for k in xrange(nvec):
                        set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
                        eigList[k].append(eig[k])
                    """
					if myid == 0 and i == 0:
						for k in xrange(nvec):
							eig[k].write_image("eig.hdf", k)
					"""

            del imgdata
            #  To this point, all averages, variances, and eigenvectors are computed

            if options.ave2D:
                from fundamentals import fpol
                if myid == main_node:
                    km = 0
                    for i in xrange(number_of_proc):
                        if i == main_node:
                            for im in xrange(len(aveList)):
                                aveList[im].write_image(
                                    os.path.join(options.output_dir,
                                                 options.ave2D), km)
                                km += 1
                        else:
                            nl = mpi_recv(1, MPI_INT, i,
                                          SPARX_MPI_TAG_UNIVERSAL,
                                          MPI_COMM_WORLD)
                            nl = int(nl[0])
                            for im in xrange(nl):
                                ave = recv_EMData(i, im + i + 70000)
                                """
								nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
								nm = int(nm[0])
								members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
								ave.set_attr('members', map(int, members))
								members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
								ave.set_attr('pix_err', map(float, members))
								members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
								ave.set_attr('refprojdir', map(float, members))
								"""
                                tmpvol = fpol(ave, Tracker["nx"],
                                              Tracker["nx"], 1)
                                tmpvol.write_image(
                                    os.path.join(options.output_dir,
                                                 options.ave2D), km)
                                km += 1
                else:
                    mpi_send(len(aveList), 1, MPI_INT, main_node,
                             SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
                    for im in xrange(len(aveList)):
                        send_EMData(aveList[im], main_node, im + myid + 70000)
                        """
						members = aveList[im].get_attr('members')
						mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						members = aveList[im].get_attr('pix_err')
						mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						try:
							members = aveList[im].get_attr('refprojdir')
							mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						except:
							mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						"""

            if options.ave3D:
                from fundamentals import fpol
                if options.VERBOSE:
                    print("Reconstructing 3D average volume")
                ave3D = recons3d_4nn_MPI(myid,
                                         aveList,
                                         symmetry=options.sym,
                                         npad=options.npad)
                bcast_EMData_to_all(ave3D, myid)
                if myid == main_node:
                    line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
                    ave3D = fpol(ave3D, Tracker["nx"], Tracker["nx"],
                                 Tracker["nx"])
                    ave3D.write_image(
                        os.path.join(options.output_dir, options.ave3D))
                    msg = ("%-70s:  %s\n" % (
                        "Writing to the disk volume reconstructed from averages as",
                        options.ave3D))
                    log_main.add(msg)
                    print(line, msg)
            del ave, var, proj_list, stack, phi, theta, psi, s2x, s2y, alpha, sx, sy, mirror, aveList

            if nvec > 0:
                for k in xrange(nvec):
                    if options.VERBOSE:
                        print("Reconstruction eigenvolumes", k)
                    cont = True
                    ITER = 0
                    mask2d = model_circle(radiuspca, nx, nx)
                    while cont:
                        #print "On node %d, iteration %d"%(myid, ITER)
                        eig3D = recons3d_4nn_MPI(myid,
                                                 eigList[k],
                                                 symmetry=options.sym,
                                                 npad=options.npad)
                        bcast_EMData_to_all(eig3D, myid, main_node)
                        if options.fl > 0.0:
                            eig3D = filt_tanl(eig3D, options.fl, options.aa)
                        if myid == main_node:
                            eig3D.write_image(
                                os.path.join(options.outpout_dir,
                                             "eig3d_%03d.hdf" % (k, ITER)))
                        Util.mul_img(eig3D,
                                     model_circle(radiuspca, nx, nx, nx))
                        eig3Df, kb = prep_vol(eig3D)
                        del eig3D
                        cont = False
                        icont = 0
                        for l in xrange(len(eigList[k])):
                            phi, theta, psi, s2x, s2y = get_params_proj(
                                eigList[k][l])
                            proj = prgs(eig3Df, kb,
                                        [phi, theta, psi, s2x, s2y])
                            cl = ccc(proj, eigList[k][l], mask2d)
                            if cl < 0.0:
                                icont += 1
                                cont = True
                                eigList[k][l] *= -1.0
                        u = int(cont)
                        u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node,
                                       MPI_COMM_WORLD)
                        icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM,
                                           main_node, MPI_COMM_WORLD)

                        if myid == main_node:
                            line = strftime("%Y-%m-%d_%H:%M:%S",
                                            localtime()) + " =>"
                            u = int(u[0])
                            msg = (" Eigenvector: ", k, " number changed ",
                                   int(icont[0]))
                            log_main.add(msg)
                            print(line, msg)
                        else:
                            u = 0
                        u = bcast_number_to_all(u, main_node)
                        cont = bool(u)
                        ITER += 1

                    del eig3Df, kb
                    mpi_barrier(MPI_COMM_WORLD)
                del eigList, mask2d

            if options.ave3D: del ave3D
            if options.var2D:
                from fundamentals import fpol
                if myid == main_node:
                    km = 0
                    for i in xrange(number_of_proc):
                        if i == main_node:
                            for im in xrange(len(varList)):
                                tmpvol = fpol(varList[im], Tracker["nx"],
                                              Tracker["nx"], 1)
                                tmpvol.write_image(
                                    os.path.join(options.output_dir,
                                                 options.var2D), km)
                                km += 1
                        else:
                            nl = mpi_recv(1, MPI_INT, i,
                                          SPARX_MPI_TAG_UNIVERSAL,
                                          MPI_COMM_WORLD)
                            nl = int(nl[0])
                            for im in xrange(nl):
                                ave = recv_EMData(i, im + i + 70000)
                                tmpvol = fpol(ave, Tracker["nx"],
                                              Tracker["nx"], 1)
                                tmpvol.write_image(
                                    os.path.join(options.output_dir,
                                                 options.var2D, km))
                                km += 1
                else:
                    mpi_send(len(varList), 1, MPI_INT, main_node,
                             SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
                    for im in xrange(len(varList)):
                        send_EMData(varList[im], main_node, im + myid +
                                    70000)  #  What with the attributes??

            mpi_barrier(MPI_COMM_WORLD)

        if options.var3D:
            if myid == main_node and options.VERBOSE:
                line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
                msg = ("Reconstructing 3D variability volume")
                log_main.add(msg)
                print(line, msg)
            t6 = time()
            # radiusvar = options.radius
            # if( radiusvar < 0 ):  radiusvar = nx//2 -3
            res = recons3d_4nn_MPI(myid,
                                   varList,
                                   symmetry=options.sym,
                                   npad=options.npad)
            #res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
            if myid == main_node:
                from fundamentals import fpol
                res = fpol(res, Tracker["nx"], Tracker["nx"], Tracker["nx"])
                res.write_image(os.path.join(options.output_dir,
                                             options.var3D))

            if myid == main_node:
                line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
                msg = ("%-70s:  %.2f\n" %
                       ("Reconstructing 3D variability took [s]", time() - t6))
                log_main.add(msg)
                print(line, msg)
                if options.VERBOSE:
                    print("Reconstruction took: %.2f [min]" %
                          ((time() - t6) / 60))

            if myid == main_node:
                line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
                msg = ("%-70s:  %.2f\n" %
                       ("Total time for these computations [s]", time() - t0))
                print(line, msg)
                log_main.add(msg)
                if options.VERBOSE:
                    print("Total time for these computations: %.2f [min]" %
                          ((time() - t0) / 60))
                line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
                msg = ("sx3dvariability")
                print(line, msg)
                log_main.add(msg)

        from mpi import mpi_finalize
        mpi_finalize()

        if RUNNING_UNDER_MPI:
            global_def.MPI = False

        global_def.BATCH = False
Esempio n. 8
0
def prep_vol(vol, npad=2, interpolation_method=-1):
    """
		Name
			prep_vol - prepare the volume for calculation of gridding projections and generate the interpolants.
		Input
			vol: input volume for which projections will be calculated using prgs (interpolation_method=-1) or prgl (interpolation_method>0)
			interpolation_method = -1  gridding
			interpolation_method =  0  NN
			interpolation_method =  1  trilinear
		Output
			volft: volume prepared for gridding projections using prgs
			kb: interpolants (tabulated Kaiser-Bessel function) when the volume is cubic.
			kbx,kby: interpolants along x, y and z direction (tabulated Kaiser-Bessel function) when the volume is rectangular 
	"""
    # prepare the volume
    Mx = vol.get_xsize()
    My = vol.get_ysize()
    Mz = vol.get_zsize()
    #  gridding
    if interpolation_method == -1:
        K = 6
        alpha = 1.75
        assert npad == 2
        if (Mx == Mz & My == Mz):
            M = vol.get_xsize()
            # padd two times
            N = M * npad
            # support of the window
            kb = Util.KaiserBessel(alpha, K, M / 2, K / (2. * N), N)
            volft = vol.copy()
            volft.divkbsinh(kb)
            volft = volft.norm_pad(False, npad)
            volft.do_fft_inplace()
            volft.center_origin_fft()
            volft.fft_shuffle()
            return volft, kb
        else:
            Nx = Mx * npad
            Ny = My * npad
            Nz = Mz * npad
            # support of the window
            kbx = Util.KaiserBessel(alpha, K, Mx / 2, K / (2. * Nx), Nx)
            kby = Util.KaiserBessel(alpha, K, My / 2, K / (2. * Ny), Ny)
            kbz = Util.KaiserBessel(alpha, K, Mz / 2, K / (2. * Nz), Nz)
            volft = vol.copy()
            volft.divkbsinh_rect(kbx, kby, kbz)
            volft = volft.norm_pad(False, npad)
            volft.do_fft_inplace()
            volft.center_origin_fft()
            volft.fft_shuffle()
            return volft, kbx, kby, kbz
    else:
        # NN and trilinear
        assert interpolation_method >= 0
        from utilities import pad
        volft = pad(vol, Mx * npad, My * npad, My * npad, 0.0)
        volft.set_attr("npad", npad)
        volft.div_sinc(interpolation_method)
        volft = volft.norm_pad(False, 1)
        volft.do_fft_inplace()
        volft.center_origin_fft()
        volft.fft_shuffle()
        volft.set_attr("npad", npad)
        return volft
Esempio n. 9
0
def prep_vol(vol, npad = 2, interpolation_method = -1):
	"""
		Name
			prep_vol - prepare the volume for calculation of gridding projections and generate the interpolants.
		Input
			vol: input volume for which projections will be calculated using prgs (interpolation_method=-1) or prgl (interpolation_method>0)
			interpolation_method = -1  gridding
			interpolation_method =  0  NN
			interpolation_method =  1  trilinear
		Output
			volft: volume prepared for gridding projections using prgs
			kb: interpolants (tabulated Kaiser-Bessel function) when the volume is cubic.
			kbx,kby: interpolants along x, y and z direction (tabulated Kaiser-Bessel function) when the volume is rectangular 
	"""
	# prepare the volume
	Mx = vol.get_xsize()
	My = vol.get_ysize()
	Mz = vol.get_zsize()
	#  gridding
	if interpolation_method == -1:
		K     = 6
		alpha = 1.75
		assert npad  == 2
		if(Mx==Mz&My==Mz):
			M     = vol.get_xsize()
			# padd two times
			N     = M*npad
			# support of the window
			kb    = Util.KaiserBessel(alpha, K, M/2, K/(2.*N), N)
			volft = vol.copy()
			volft.divkbsinh(kb)
			volft = volft.norm_pad(False, npad)
			volft.do_fft_inplace()
			volft.center_origin_fft()
			volft.fft_shuffle()
			return  volft,kb
		else:
			Nx     = Mx*npad
			Ny     = My*npad
			Nz     = Mz*npad
			# support of the window
			kbx    = Util.KaiserBessel(alpha, K, Mx/2, K/(2.*Nx), Nx)
			kby    = Util.KaiserBessel(alpha, K, My/2, K/(2.*Ny), Ny)
			kbz    = Util.KaiserBessel(alpha, K, Mz/2, K/(2.*Nz), Nz)
			volft = vol.copy()
			volft.divkbsinh_rect(kbx,kby,kbz)
			volft = volft.norm_pad(False, npad)
			volft.do_fft_inplace()
			volft.center_origin_fft()
			volft.fft_shuffle()
			return  volft,kbx,kby,kbz
	else:
		# NN and trilinear
		assert  interpolation_method >= 0
		from utilities import pad
		volft = pad(vol, Mx*npad, My*npad, My*npad, 0.0)
		volft.div_sinc(interpolation_method)
		volft = volft.norm_pad(False, 1)
		volft.set_attr("npad", npad)
		volft.do_fft_inplace()
		volft.center_origin_fft()
		volft.fft_shuffle()
		return  volft
Esempio n. 10
0
def main(args):
    progname = os.path.basename(sys.argv[0])
    usage = (
        progname +
        " stack_file  output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter"
        +
        " --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp"
        + "  --generation=generation  --rand_seed=rand_seed>")

    parser = OptionParser(usage, version=SPARXVERSION)
    parser.add_option("--radius",
                      type="int",
                      default=-1,
                      help="<Particle radius>, it has to be provided.")
    parser.add_option(
        "--img_per_grp",
        type="int",
        default=100,
        help=
        "<number of images per group> in the ideal case (essentially maximum size of class) (100)"
    )
    parser.add_option("--CTF",
                      action="store_true",
                      default=False,
                      help="<CTF flag>, if set the data will be phase-flipped")
    parser.add_option(
        "--ir",
        type="int",
        default=1,
        help="<inner ring> of the resampling to polar coordinates (1)")
    parser.add_option(
        "--rs",
        type="int",
        default=1,
        help="<ring step> of the resampling to polar coordinates (1)")
    parser.add_option(
        "--xr",
        type="int",
        default=-1,
        help=
        "<x range> of translational search (By default set by the program) (advanced)"
    )
    parser.add_option(
        "--yr",
        type="int",
        default=-1,
        help="<y range> of translational search (same as xr) (advanced)")
    parser.add_option("--ts",
                      type="float",
                      default=1.0,
                      help="<search step> of translational search (1.0)")
    parser.add_option(
        "--maxit",
        type="int",
        default=30,
        help="number of iterations for reference-free alignment (30)")
    #parser.add_option("--snr",            type="float",        default=1.0,     help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)")
    parser.add_option(
        "--center_method",
        type="int",
        default=7,
        help=
        "<Method for centering> of global 2D average during initial prealignment of data (default : 7; 0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7)"
    )
    parser.add_option("--dst",
                      type="float",
                      default=90.0,
                      help="discrete angle used in within group alignment ")
    parser.add_option(
        "--FL",
        type="float",
        default=0.2,
        help="<lowest stopband> frequency used in the tangent filter (0.2)")
    parser.add_option(
        "--FH",
        type="float",
        default=0.3,
        help="<highest stopband> frequency used in the tangent filter (0.3)")
    parser.add_option("--FF",
                      type="float",
                      default=0.2,
                      help="<fall-off of the tangent> filter (0.2)")
    parser.add_option(
        "--init_iter",
        type="int",
        default=3,
        help=
        "<init_iter> number of iterations of ISAC program in initialization (3)"
    )
    parser.add_option(
        "--main_iter",
        type="int",
        default=3,
        help="<main_iter> number of iterations of ISAC program in main part (3)"
    )
    parser.add_option(
        "--iter_reali",
        type="int",
        default=1,
        help=
        "<iter_reali> number of iterations in ISAC before checking stability (1)"
    )
    parser.add_option(
        "--match_first",
        type="int",
        default=1,
        help="number of iterations to run 2-way matching in the first phase (1)"
    )
    parser.add_option(
        "--max_round",
        type="int",
        default=20,
        help=
        "maximum rounds of generating candidate averages in the first phase (20)"
    )
    parser.add_option(
        "--match_second",
        type="int",
        default=5,
        help=
        "number of iterations to run 2-way (or 3-way) matching in the second phase (5)"
    )
    parser.add_option("--stab_ali",
                      type="int",
                      default=5,
                      help="number of alignments when checking stability (5)")
    parser.add_option(
        "--thld_err",
        type="float",
        default=0.7,
        help="the threshold of pixel error when checking stability (0.7)")
    parser.add_option(
        "--indep_run",
        type="int",
        default=4,
        help=
        "number of independent runs for reproducibility (default=4, only values 2, 3 and 4 are supported (4)"
    )
    parser.add_option("--thld_grp",
                      type="int",
                      default=10,
                      help="minimum size of class (10)")
    parser.add_option(
        "--n_generations",
        type="int",
        default=100,
        help=
        "<n_generations> program stops when reaching this total number of generations (advanced)"
    )
    #parser.add_option("--candidatesexist",action="store_true", default=False,   help="Candidate class averages exist use them (default False)")
    parser.add_option(
        "--rand_seed",
        type="int",
        default=None,
        help=
        "random seed set before calculations, useful for testing purposes (default None - total randomness)"
    )
    parser.add_option("--new",
                      action="store_true",
                      default=False,
                      help="use new code (default = False)")
    parser.add_option("--debug",
                      action="store_true",
                      default=False,
                      help="debug info printout (default = False)")

    # must be switched off in production
    parser.add_option("--use_latest_master_directory",
                      action="store_true",
                      dest="use_latest_master_directory",
                      default=False)

    parser.add_option(
        "--restart_section",
        type="string",
        default="",
        help=
        "<restart section name> (no spaces) followed immediately by comma, followed immediately by generation to restart, example: \n--restart_section=candidate_class_averages,1         (Sections: restart, candidate_class_averages, reproducible_class_averages)"
    )
    parser.add_option(
        "--stop_after_candidates",
        action="store_true",
        default=False,
        help=
        "<stop_after_candidates> stops after the 'candidate_class_averages' section"
    )

    parser.add_option("--return_options",
                      action="store_true",
                      dest="return_options",
                      default=False,
                      help=SUPPRESS_HELP)

    (options, args) = parser.parse_args(args)

    if options.return_options:
        return parser

    if len(args) > 2:
        print "usage: " + usage
        print "Please run '" + progname + " -h' for detailed options"
        sys.exit()

    if global_def.CACHE_DISABLE:
        from utilities import disable_bdb_cache
        disable_bdb_cache()

    from isac import iter_isac
    global_def.BATCH = True

    global_def.BATCH = True

    command_line_provided_stack_filename = args[0]
    global_def.BATCH = True

    main_node = 0
    mpi_init(0, [])
    myid = mpi_comm_rank(MPI_COMM_WORLD)
    nproc = mpi_comm_size(MPI_COMM_WORLD)

    radi = options.radius
    center_method = options.center_method
    if (radi < 1):
        ERROR("Particle radius has to be provided!", "sxisac", 1, myid)

    use_latest_master_directory = options.use_latest_master_directory
    stop_after_candidates = options.stop_after_candidates
    program_state_stack.restart_location_title_from_command_line = options.restart_section

    from utilities import qw
    program_state_stack.PROGRAM_STATE_VARIABLES = set(
        qw("""
		isac_generation
	"""))

    # create or reuse master directory
    masterdir = ""
    stack_processed_by_ali2d_base__filename = ""
    stack_processed_by_ali2d_base__filename__without_master_dir = ""
    error_status = 0
    if len(args) == 2:
        masterdir = args[1]
    elif len(args) == 1:
        if use_latest_master_directory:
            all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)]
            import re
            r = re.compile("^master.*$")
            all_dirs = filter(r.match, all_dirs)
            if len(all_dirs) > 0:
                # all_dirs = max(all_dirs, key=os.path.getctime)
                masterdir = max(all_dirs, key=os.path.getmtime)

    #Create folder for all results or check if there is one created already
    if (myid == main_node):
        if (masterdir == ""):
            timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM,
                                  localtime())
            masterdir = "master" + timestring
            cmd = "{} {}".format("mkdir", masterdir)
            cmdexecute(cmd)
        elif not os.path.exists(masterdir):
            # os.path.exists(masterdir) does not exist
            masterdir = args[1]
            cmd = "{} {}".format("mkdir", masterdir)
            cmdexecute(cmd)

        if (args[0][:4] == "bdb:"): filename = args[0][4:]
        else: filename = args[0][:-4]
        filename = os.path.basename(filename)
        stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join(
            masterdir, filename)
        stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename
    if_error_all_processes_quit_program(error_status)

    # send masterdir to all processes
    masterdir = send_string_to_all(masterdir)

    if myid == 0:
        if options.restart_section != "":
            if os.path.exists(os.path.join(masterdir,
                                           NAME_OF_JSON_STATE_FILE)):
                stored_stack, stored_state = restore_program_stack_and_state(
                    os.path.join(masterdir, NAME_OF_JSON_STATE_FILE))
                import re
                if "," in options.restart_section:
                    parsed_restart_section_option = options.restart_section.split(
                        ",")
                    stored_state[-1]["location_in_program"] = re.sub(
                        r"___.*$", "___%s" % parsed_restart_section_option[0],
                        stored_state[-1]["location_in_program"])
                    generation_str_format = parsed_restart_section_option[1]
                    if generation_str_format != "":
                        isac_generation_from_command_line = int(
                            generation_str_format)
                        stored_state[-1][
                            "isac_generation"] = isac_generation_from_command_line
                    else:
                        isac_generation_from_command_line = 1
                        if "isac_generation" in stored_state[-1]:
                            del stored_state[-1]["isac_generation"]
                else:
                    isac_generation_from_command_line = -1
                    stored_state[-1]["location_in_program"] = re.sub(
                        r"___.*$", "___%s" % options.restart_section,
                        stored_state[-1]["location_in_program"])
                    if "isac_generation" in stored_state[-1]:
                        del stored_state[-1]["isac_generation"]
                store_program_state(
                    os.path.join(masterdir, NAME_OF_JSON_STATE_FILE),
                    stored_state, stored_stack)
            else:
                print "Please remove the restart_section option from the command line. The program must be started from the beginning."
                mpi_finalize()
                sys.exit()
        else:
            isac_generation_from_command_line = -1

    program_state_stack(locals(), getframeinfo(currentframe()),
                        os.path.join(masterdir, NAME_OF_JSON_STATE_FILE))

    stack_processed_by_ali2d_base__filename = send_string_to_all(
        stack_processed_by_ali2d_base__filename)
    stack_processed_by_ali2d_base__filename__without_master_dir = \
     send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir)

    #  PARAMETERS OF THE PROCEDURE
    if (options.xr == -1):
        #  Default values
        target_nx = 76
        target_radius = 29
        target_xr = 1
    else:  #  nx//2
        #  Check below!
        target_xr = options.xr
        target_nx = 76 + target_xr - 1  # subtract one, which is default
        target_radius = 29

    mpi_barrier(MPI_COMM_WORLD)

    # Initialization of stacks
    if (myid == main_node):
        number_of_images_in_stack = EMUtil.get_image_count(
            command_line_provided_stack_filename)
    else:
        number_of_images_in_stack = 0

    number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack,
                                                    source_node=main_node)

    nxrsteps = 4

    init2dir = os.path.join(masterdir, "2dalignment")

    if (myid == 0):
        import subprocess
        from logger import Logger, BaseLogger_Files
        #  Create output directory
        log2d = Logger(BaseLogger_Files())
        log2d.prefix = os.path.join(init2dir)
        cmd = "mkdir -p " + log2d.prefix
        outcome = subprocess.call(cmd, shell=True)
        log2d.prefix += "/"
        # outcome = subprocess.call("sxheader.py  "+command_line_provided_stack_filename+"   --params=xform.align2d  --zero", shell=True)
    else:
        outcome = 0
        log2d = None

    if (myid == main_node):
        a = get_im(command_line_provided_stack_filename)
        nnxo = a.get_xsize()
    else:
        nnxo = 0
    nnxo = bcast_number_to_all(nnxo, source_node=main_node)

    txrm = (nnxo - 2 * (radi + 1)) // 2
    if (txrm < 0):
        ERROR(
            "ERROR!!   Radius of the structure larger than the window data size permits   %d"
            % (radi), "sxisac", 1, myid)
    if (txrm / nxrsteps > 0):
        tss = ""
        txr = ""
        while (txrm / nxrsteps > 0):
            tts = txrm / nxrsteps
            tss += "  %d" % tts
            txr += "  %d" % (tts * nxrsteps)
            txrm = txrm // 2
    else:
        tss = "1"
        txr = "%d" % txrm

    # section ali2d_base

    #  centering method is set to #7
    params2d, aligned_images = ali2d_base(command_line_provided_stack_filename, init2dir, None, 1, radi, 1, txr, txr, tss, \
       False, 90.0, center_method, 14, options.CTF, 1.0, False, \
       "ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False)

    if (myid == main_node):
        write_text_row(params2d, os.path.join(init2dir, "initial2Dparams.txt"))
    del params2d
    mpi_barrier(MPI_COMM_WORLD)

    #  We assume the target image size will be target_nx, radius will be 29, and xr = 1.
    #  Note images can be also padded, in which case shrink_ratio > 1.
    shrink_ratio = float(target_radius) / float(radi)
    nx = aligned_images[0].get_xsize()
    nima = len(aligned_images)
    newx = int(nx * shrink_ratio + 0.5)

    from fundamentals import rot_shift2D, resample
    from utilities import pad, combine_params2
    if (shrink_ratio < 1.0):
        if newx > target_nx:
            msk = model_circle(target_radius, target_nx, target_nx)
            for im in xrange(nima):
                #  Here we should use only shifts
                alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
                alpha, sx, sy, mirror = combine_params2(
                    0, sx, sy, 0, -alpha, 0, 0, 0)
                aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
                                                 0)
                aligned_images[im] = resample(aligned_images[im], shrink_ratio)
                aligned_images[im] = Util.window(aligned_images[im], target_nx,
                                                 target_nx, 1)
                p = Util.infomask(aligned_images[im], msk, False)
                aligned_images[im] -= p[0]
                p = Util.infomask(aligned_images[im], msk, True)
                aligned_images[im] /= p[1]
        elif newx == target_nx:
            msk = model_circle(target_radius, target_nx, target_nx)
            for im in xrange(nima):
                #  Here we should use only shifts
                alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
                alpha, sx, sy, mirror = combine_params2(
                    0, sx, sy, 0, -alpha, 0, 0, 0)
                aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
                                                 0)
                aligned_images[im] = resample(aligned_images[im], shrink_ratio)
                p = Util.infomask(aligned_images[im], msk, False)
                aligned_images[im] -= p[0]
                p = Util.infomask(aligned_images[im], msk, True)
                aligned_images[im] /= p[1]
        elif newx < target_nx:
            msk = model_circle(nx // 2 - 2, newx, newx)
            for im in xrange(nima):
                #  Here we should use only shifts
                alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
                alpha, sx, sy, mirror = combine_params2(
                    0, sx, sy, 0, -alpha, 0, 0, 0)
                aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
                                                 0)
                aligned_images[im] = resample(aligned_images[im], shrink_ratio)
                p = Util.infomask(aligned_images[im], msk, False)
                aligned_images[im] -= p[0]
                p = Util.infomask(aligned_images[im], msk, True)
                aligned_images[im] /= p[1]
                aligned_images[im] = pad(aligned_images[im], target_nx,
                                         target_nx, 1, 0.0)
    elif (shrink_ratio == 1.0):
        if newx > target_nx:
            msk = model_circle(target_radius, target_nx, target_nx)
            for im in xrange(nima):
                #  Here we should use only shifts
                alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
                alpha, sx, sy, mirror = combine_params2(
                    0, sx, sy, 0, -alpha, 0, 0, 0)
                aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
                                                 0)
                aligned_images[im] = Util.window(aligned_images[im], target_nx,
                                                 target_nx, 1)
                p = Util.infomask(aligned_images[im], msk, False)
                aligned_images[im] -= p[0]
                p = Util.infomask(aligned_images[im], msk, True)
                aligned_images[im] /= p[1]
        elif newx == target_nx:
            msk = model_circle(target_radius, target_nx, target_nx)
            for im in xrange(nima):
                #  Here we should use only shifts
                alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
                alpha, sx, sy, mirror = combine_params2(
                    0, sx, sy, 0, -alpha, 0, 0, 0)
                aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
                                                 0)
                p = Util.infomask(aligned_images[im], msk, False)
                aligned_images[im] -= p[0]
                p = Util.infomask(aligned_images[im], msk, True)
                aligned_images[im] /= p[1]
        elif newx < target_nx:
            msk = model_circle(nx // 2 - 2, newx, newx)
            for im in xrange(nima):
                #  Here we should use only shifts
                alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
                alpha, sx, sy, mirror = combine_params2(
                    0, sx, sy, 0, -alpha, 0, 0, 0)
                aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
                                                 0)
                aligned_images[im] = resample(aligned_images[im], shrink_ratio)
                p = Util.infomask(aligned_images[im], msk, False)
                aligned_images[im] -= p[0]
                p = Util.infomask(aligned_images[im], msk, True)
                aligned_images[im] /= p[1]
                aligned_images[im] = pad(aligned_images[im], target_nx,
                                         target_nx, 1, 0.0)
    elif (shrink_ratio > 1.0):
        if newx > target_nx:
            msk = model_circle(target_radius, target_nx, target_nx)
            for im in xrange(nima):
                #  Here we should use only shifts
                alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
                alpha, sx, sy, mirror = combine_params2(
                    0, sx, sy, 0, -alpha, 0, 0, 0)
                aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
                                                 0)
                aligned_images[im] = Util.window(aligned_images[im], target_nx,
                                                 target_nx, 1)
                p = Util.infomask(aligned_images[im], msk, False)
                aligned_images[im] -= p[0]
                p = Util.infomask(aligned_images[im], msk, True)
                aligned_images[im] /= p[1]
        elif newx == target_nx:
            msk = model_circle(target_radius, target_nx, target_nx)
            for im in xrange(nima):
                #  Here we should use only shifts
                alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
                alpha, sx, sy, mirror = combine_params2(
                    0, sx, sy, 0, -alpha, 0, 0, 0)
                aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
                                                 0)
                p = Util.infomask(aligned_images[im], msk, False)
                aligned_images[im] -= p[0]
                p = Util.infomask(aligned_images[im], msk, True)
                aligned_images[im] /= p[1]
        elif newx < target_nx:
            msk = model_circle(target_radius, nx, nx)
            for im in xrange(nima):
                #  Here we should use only shifts
                alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
                alpha, sx, sy, mirror = combine_params2(
                    0, sx, sy, 0, -alpha, 0, 0, 0)
                aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
                                                 0)
                p = Util.infomask(aligned_images[im], msk, False)
                aligned_images[im] -= p[0]
                p = Util.infomask(aligned_images[im], msk, True)
                aligned_images[im] /= p[1]
                aligned_images[im] = pad(aligned_images[im], target_nx,
                                         target_nx, 1, 0.0)
    del msk

    gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images,
                                    myid)
    number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack,
                                                    source_node=main_node)

    if (myid == main_node):
        for i in range(number_of_images_in_stack):
            aligned_images[i].write_image(
                stack_processed_by_ali2d_base__filename, i)
        #  It has to be explicitly closed
        from EMAN2db import db_open_dict
        DB = db_open_dict(stack_processed_by_ali2d_base__filename)
        DB.close()

    mpi_barrier(MPI_COMM_WORLD)

    global_def.BATCH = True

    os.chdir(masterdir)

    if program_state_stack(locals(), getframeinfo(currentframe())):
        # if 1:
        pass
        if (myid == main_node):
            cmdexecute(
                "sxheader.py  --consecutive  --params=originalid   %s" %
                stack_processed_by_ali2d_base__filename__without_master_dir)
            cmdexecute(
                "e2bdb.py %s --makevstack=%s_000" %
                (stack_processed_by_ali2d_base__filename__without_master_dir,
                 stack_processed_by_ali2d_base__filename__without_master_dir))

    if (myid == main_node):
        main_dir_no = get_latest_directory_increment_value("./",
                                                           NAME_OF_MAIN_DIR,
                                                           myformat="%04d")
        print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no
        if isac_generation_from_command_line < 0:
            if os.path.exists(NAME_OF_JSON_STATE_FILE):
                stored_stack, stored_state = restore_program_stack_and_state(
                    NAME_OF_JSON_STATE_FILE)
                if "isac_generation" in stored_state[-1]:
                    isac_generation_from_command_line = stored_state[-1][
                        "isac_generation"]
                else:
                    isac_generation_from_command_line = -1
        if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no:
            for i in xrange(isac_generation_from_command_line + 1,
                            main_dir_no + 1):
                if i == isac_generation_from_command_line + 1:
                    backup_dir_no = get_nonexistent_directory_increment_value(
                        "./", "000_backup", myformat="%05d", start_value=1)
                    cmdexecute("mkdir -p " + "000_backup" +
                               "%05d" % backup_dir_no)
                cmdexecute("mv  " + NAME_OF_MAIN_DIR + "%04d" % i +
                           " 000_backup" + "%05d" % backup_dir_no)
                cmdexecute(
                    "rm  " + "EMAN2DB/" +
                    stack_processed_by_ali2d_base__filename__without_master_dir[
                        4:] + "_%03d.bdb" % i)

            # it includes both command line and json file
            my_restart_section = stored_state[-1]["location_in_program"].split(
                "___")[-1]
            if "restart" in my_restart_section:
                if "backup_dir_no" not in locals():
                    backup_dir_no = get_nonexistent_directory_increment_value(
                        "./", "000_backup", myformat="%05d", start_value=1)
                    cmdexecute("mkdir -p " + "000_backup" +
                               "%05d" % backup_dir_no)
                cmdexecute("mv  " + NAME_OF_MAIN_DIR +
                           "%04d" % isac_generation_from_command_line +
                           " 000_backup" + "%05d" % backup_dir_no)
                cmdexecute(
                    "rm  " + "EMAN2DB/" +
                    stack_processed_by_ali2d_base__filename__without_master_dir[
                        4:] + "_%03d.bdb" % isac_generation_from_command_line)
            elif "candidate_class_averages" in my_restart_section:
                if "backup_dir_no" not in locals():
                    backup_dir_no = get_nonexistent_directory_increment_value(
                        "./", "000_backup", myformat="%05d", start_value=1)
                    cmdexecute("mkdir -p " + "000_backup" +
                               "%05d" % backup_dir_no)
                cmdexecute("mv  " + NAME_OF_MAIN_DIR +
                           "%04d" % isac_generation_from_command_line +
                           " 000_backup" + "%05d" % backup_dir_no)
                cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
                           "%04d" % isac_generation_from_command_line)
                # cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line)
            elif "reproducible_class_averages" in my_restart_section:
                cmdexecute("rm -rf " + NAME_OF_MAIN_DIR +
                           "%04d/ali_params_generation_*" %
                           isac_generation_from_command_line)
                cmdexecute("rm -f " + NAME_OF_MAIN_DIR +
                           "%04d/class_averages_generation*" %
                           isac_generation_from_command_line)
        else:
            if os.path.exists(NAME_OF_JSON_STATE_FILE):
                stored_stack, stored_state = restore_program_stack_and_state(
                    NAME_OF_JSON_STATE_FILE)
                if "isac_generation" in stored_state[-1]:
                    isac_generation_from_command_line = stored_state[-1][
                        "isac_generation"]
                else:
                    isac_generation_from_command_line = 1
            else:
                isac_generation_from_command_line = 1
    else:
        isac_generation_from_command_line = 0

    isac_generation_from_command_line = mpi_bcast(
        isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]
    isac_generation = isac_generation_from_command_line - 1

    if (myid == main_node):
        if isac_generation == 0:
            cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
                       "%04d" % isac_generation)
            write_text_file(
                [1],
                os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation,
                             "generation_%d_accounted.txt" % isac_generation))
            write_text_file(
                range(number_of_images_in_stack),
                os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation,
                             "generation_%d_unaccounted.txt" %
                             isac_generation))

    #  Stopping criterion should be inside the program.
    while True:
        isac_generation += 1
        if isac_generation > options.n_generations:
            break

        data64_stack_current = "bdb:../" + stack_processed_by_ali2d_base__filename__without_master_dir[
            4:] + "_%03d" % isac_generation

        if (myid == main_node):
            accounted_images = read_text_file(
                os.path.join(
                    NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1),
                    "generation_%d_accounted.txt" % (isac_generation - 1)))
            number_of_accounted_images = len(accounted_images)
            # unaccounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1),"generation_%d_unaccounted.txt"%(isac_generation - 1)))
            # number_of_unaccounted_images = len(unaccounted_images)
        else:
            number_of_accounted_images = 0

        number_of_accounted_images = int(
            mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0,
                      MPI_COMM_WORLD)[0])
        if number_of_accounted_images == 0:
            os.chdir("..")
            break

        program_state_stack.restart_location_title = "restart"
        if program_state_stack(locals(), getframeinfo(currentframe())):
            if (myid == main_node):
                cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
                           "%04d" % isac_generation)
                # reference the original stack
                list_file = os.path.join(
                    NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1),
                    "generation_%d_unaccounted.txt" % (isac_generation - 1))
                cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\
                  stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file))
            mpi_barrier(MPI_COMM_WORLD)

        os.chdir(NAME_OF_MAIN_DIR + "%04d" % isac_generation)

        program_state_stack.restart_location_title = "candidate_class_averages"
        if program_state_stack(locals(), getframeinfo(currentframe())):

            iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\
             options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
             options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
             options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new)

        # program_state_stack.restart_location_title = "stopped_program1"
        # program_state_stack(locals(), getframeinfo(currentframe()))

        program_state_stack.restart_location_title = "stop_after_candidates"
        program_state_stack(locals(), getframeinfo(currentframe()))
        if stop_after_candidates:
            mpi_finalize()
            sys.exit()

        exit_program = 0
        if (myid == main_node):
            if not os.path.exists(
                    "class_averages_candidate_generation_%d.hdf" %
                    isac_generation):
                print "This generation (%d) no class averages were generated!" % isac_generation
                exit_program = 1
        exit_program = int(
            mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
        if exit_program:
            os.chdir("..")
            break

        program_state_stack.restart_location_title = "reproducible_class_averages"
        if program_state_stack(locals(), getframeinfo(currentframe())):


            iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\
             options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
             options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
             options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new)
            pass

        os.chdir("..")

        if (myid == main_node):
            cmdexecute("rm -f class_averages.hdf")
            cpy([
                "generation_%04d/class_averages_generation_%d.hdf" % (i, i)
                for i in xrange(1, isac_generation)
            ], "class_averages.hdf")

        # program_state_stack.restart_location_title = "stopped_program2"
        # program_state_stack(locals(), getframeinfo(currentframe()))

    program_state_stack(locals(),
                        getframeinfo(currentframe()),
                        last_call="__LastCall")

    mpi_finalize()
Esempio n. 11
0
def generate_helimic(refvol,
                     outdir,
                     pixel,
                     CTF=False,
                     Cs=2.0,
                     voltage=200.0,
                     ampcont=10.0,
                     nonoise=False,
                     rand_seed=14567):

    from utilities import model_blank, model_gauss, model_gauss_noise, pad, get_im
    from random import random
    from projection import prgs, prep_vol
    from filter import filt_gaussl, filt_ctf
    from EMAN2 import EMAN2Ctf

    if os.path.exists(outdir):
        ERROR(
            'Output directory exists, please change the name and restart the program',
            "sxhelical_demo", 1)
    os.mkdir(outdir)
    seed(rand_seed)
    Util.set_randnum_seed(rand_seed)
    angles = []
    for i in range(3):
        angles.append([0.0 + 60.0 * i, 90.0 - i * 5, 0.0, 0.0, 0.0])

    nangle = len(angles)

    volfts = get_im(refvol)
    nx = volfts.get_xsize()
    ny = volfts.get_ysize()
    nz = volfts.get_zsize()
    volfts, kbx, kby, kbz = prep_vol(volfts)
    iprj = 0
    width = 500
    xstart = 0
    ystart = 0

    for idef in range(3, 6):
        mic = model_blank(2048, 2048)
        #defocus = idef*0.2
        defocus = idef * 0.6  ##@ming
        if CTF:
            #ctf = EMAN2Ctf()
            #ctf.from_dict( {"defocus":defocus, "cs":Cs, "voltage":voltage, "apix":pixel, "ampcont":ampcont, "bfactor":0.0} )
            from utilities import generate_ctf
            ctf = generate_ctf(
                [defocus, 2, 200, 1.84, 0.0, ampcont, defocus * 0.2, 80]
            )  ##@ming   the range of astigmatism amplitude is between 10 percent and 22 percent. 20 percent is a good choice.
        i = idef - 4
        for k in range(1):
            psi = 90 + 10 * i
            proj = prgs(
                volfts, kbz,
                [angles[idef - 3][0], angles[idef - 3][1], psi, 0.0, 0.0], kbx,
                kby)
            proj = Util.window(proj, 320, nz)
            mic += pad(proj, 2048, 2048, 1, 0.0, 750 * i, 20 * i, 0)

        if not nonoise: mic += model_gauss_noise(30.0, 2048, 2048)
        if CTF:
            #apply CTF
            mic = filt_ctf(mic, ctf)

        if not nonoise:
            mic += filt_gaussl(model_gauss_noise(17.5, 2048, 2048), 0.3)

        mic.write_image("%s/mic%1d.hdf" % (outdir, idef - 3), 0)
Esempio n. 12
0
def main():
	import sys
	import os
	import math
	import random
	import pyemtbx.options
	import time
	from   random   import random, seed, randint
	from   optparse import OptionParser

	progname = os.path.basename(sys.argv[0])
	usage = progname + """ [options] <inputfile> <outputfile>

	Generic 2-D image processing programs.

	Functionality:

	1.  Phase flip a stack of images and write output to new file:
		sxprocess.py input_stack.hdf output_stack.hdf --phase_flip
	
	2.  Resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.
	    The window size will change accordingly.
		sxprocess input.hdf output.hdf  --changesize --ratio=0.5

	3.  Compute average power spectrum of a stack of 2D images with optional padding (option wn) with zeroes or a 3-D volume.
		sxprocess.py input_stack.hdf powerspectrum.hdf --pw [--wn=1024]

	4.  Generate a stack of projections bdb:data and micrographs with prefix mic (i.e., mic0.hdf, mic1.hdf etc) from structure input_structure.hdf, with CTF applied to both projections and micrographs:
		sxprocess.py input_structure.hdf data mic --generate_projections format="bdb":apix=5.2:CTF=True:boxsize=64

    5.  Retrieve original image numbers in the selected ISAC group (here group 12 from generation 3):
    	sxprocess.py  bdb:test3 class_averages_generation_3.hdf  list3_12.txt --isacgroup=12 --params=originalid

    6.  Retrieve original image numbers of images listed in ISAC output stack of averages:
    	sxprocess.py  select1.hdf  ohk.txt

    7.  Adjust rotationally averaged power spectrum of an image to that of a reference image or a reference 1D power spectrum stored in an ASCII file.
    	Optionally use a tangent low-pass filter.  Also works for a stack of images, in which case the output is also a stack.
    	sxprocess.py  vol.hdf ref.hdf  avol.hdf < 0.25 0.2> --adjpw
   	 	sxprocess.py  vol.hdf pw.txt   avol.hdf < 0.25 0.2> --adjpw

    8.  Generate a 1D rotationally averaged power spectrum of an image.
		sxprocess.py  vol.hdf --rotwp=rotpw.txt
    	# Output will contain three columns:
       (1) rotationally averaged power spectrum
       (2) logarithm of the rotationally averaged power spectrum
       (3) integer line number (from zero to approximately to half the image size)

    9.  Apply 3D transformation (rotation and/or shift) to a set of orientation parameters associated with projection data.
    	sxprocess.py  --transfromparams=phi,theta,psi,tx,ty,tz      input.txt  output.txt
    	The output file is then imported and 3D transformed volume computed:
    	sxheader.py  bdb:p  --params=xform.projection  --import=output.txt
    	mpirun -np 2 sxrecons3d_n.py  bdb:p tvol.hdf --MPI
    	The reconstructed volume is in the position of the volume computed using the input.txt parameters and then
    	transformed with rot_shift3D(vol, phi,theta,psi,tx,ty,tz)

   10.  Import ctf parameters from the output of sxcter into windowed particle headers.
	    There are three possible input files formats:  (1) all particles are in one stack, (2 aor 3) particles are in stacks, each stack corresponds to a single micrograph.
	    In each case the particles should contain a name of the micrograph of origin stores using attribute name 'ptcl_source_image'.
        Normally this is done by e2boxer.py during windowing.
	    Particles whose defocus or astigmatism error exceed set thresholds will be skipped, otherwise, virtual stacks with the original way preceded by G will be created.
		sxprocess.py  --input=bdb:data  --importctf=outdir/partres  --defocuserror=10.0  --astigmatismerror=5.0
		#  Output will be a vritual stack bdb:Gdata
		sxprocess.py  --input="bdb:directory/stacks*"  --importctf=outdir/partres  --defocuserror=10.0  --astigmatismerror=5.0
		To concatenate output files:
		cd directory
		e2bdb.py . --makevstack=bdb:allparticles  --filt=G
		IMPORTANT:  Please do not move (or remove!) any input/intermediate EMAN2DB files as the information is linked between them.

   11. Scale 3D shifts.  The shifts in the input five columns text file with 3D orientation parameters will be DIVIDED by the scale factor
		sxprocess.py  orientationparams.txt  scaledparams.txt  scale=0.5
   
   12. Generate 3D mask from a given 3-D volume automatically or using threshold provided by user.
   
   13. Postprocess 3-D or 2-D images: 
   			for 3-D volumes: calculate FSC with provided mask; weight summed volume with FSC; estimate B-factor from FSC weighted summed two volumes; apply negative B-factor to the weighted volume. 
   			for 2-D images:  calculate B-factor and apply negative B-factor to 2-D images.
   14. Winow stack file -reduce size of images without changing the pixel size. 


"""

	parser = OptionParser(usage,version=SPARXVERSION)
	parser.add_option("--order", 				action="store_true", help="Two arguments are required: name of input stack and desired name of output stack. The output stack is the input stack sorted by similarity in terms of cross-correlation coefficent.", default=False)
	parser.add_option("--order_lookup", 		action="store_true", help="Test/Debug.", default=False)
	parser.add_option("--order_metropolis", 	action="store_true", help="Test/Debug.", default=False)
	parser.add_option("--order_pca", 			action="store_true", help="Test/Debug.", default=False)
	parser.add_option("--initial",				type="int", 		default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)")
	parser.add_option("--circular", 			action="store_true", help="Select circular ordering (fisr image has to be similar to the last", default=False)
	parser.add_option("--radius", 				type="int", 		default=-1, help="Radius of a circular mask for similarity based ordering")
	parser.add_option("--changesize", 			action="store_true", help="resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.", default=False)
	parser.add_option("--ratio", 				type="float", 		default=1.0, help="The ratio of new to old image size (if <1 the pixel size will increase and image size decrease, if>1, the other way round")
	parser.add_option("--pw", 					action="store_true", help="compute average power spectrum of a stack of 2-D images with optional padding (option wn) with zeroes", default=False)
	parser.add_option("--wn", 					type="int", 		default=-1, help="Size of window to use (should be larger/equal than particle box size, default padding to max(nx,ny))")
	parser.add_option("--phase_flip", 			action="store_true", help="Phase flip the input stack", default=False)
	parser.add_option("--makedb", 				metavar="param1=value1:param2=value2", type="string",
					action="append",  help="One argument is required: name of key with which the database will be created. Fill in database with parameters specified as follows: --makedb param1=value1:param2=value2, e.g. 'gauss_width'=1.0:'pixel_input'=5.2:'pixel_output'=5.2:'thr_low'=1.0")
	parser.add_option("--generate_projections", metavar="param1=value1:param2=value2", type="string",
					action="append", help="Three arguments are required: name of input structure from which to generate projections, desired name of output projection stack, and desired prefix for micrographs (e.g. if prefix is 'mic', then micrographs mic0.hdf, mic1.hdf etc will be generated). Optional arguments specifying format, apix, box size and whether to add CTF effects can be entered as follows after --generate_projections: format='bdb':apix=5.2:CTF=True:boxsize=100, or format='hdf', etc., where format is bdb or hdf, apix (pixel size) is a float, CTF is True or False, and boxsize denotes the dimension of the box (assumed to be a square). If an optional parameter is not specified, it will default as follows: format='bdb', apix=2.5, CTF=False, boxsize=64.")
	parser.add_option("--isacgroup", 			type="int", 		help="Retrieve original image numbers in the selected ISAC group. See ISAC documentation for details.", default=-1)
	parser.add_option("--isacselect", 			action="store_true", 		help="Retrieve original image numbers of images listed in ISAC output stack of averages. See ISAC documentation for details.", default=False)
	parser.add_option("--params",	   			type="string",      default=None,    help="Name of header of parameter, which one depends on specific option")
	parser.add_option("--adjpw", 				action="store_true",	help="Adjust rotationally averaged power spectrum of an image", default=False)
	parser.add_option("--rotpw", 				type="string",   	default=None,    help="Name of the text file to contain rotationally averaged power spectrum of the input image.")
	parser.add_option("--transformparams",		type="string",   	default=None,    help="Transform 3D projection orientation parameters using six 3D parameters (phi, theta,psi,sx,sy,sz).  Input: --transformparams=45.,66.,12.,-2,3,-5.5 desired six transformation of the reconstructed structure. Output: file with modified orientation parameters.")

	
	# import ctf estimates done using cter
	parser.add_option("--input",              	type="string",		default= None,     		  help="Input particles.")
	parser.add_option("--importctf",          	type="string",		default= None,     		  help="Name of the file containing CTF parameters produced by sxcter.")
	parser.add_option("--defocuserror",       	type="float",  		default=1000000.0,        help="Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent.  The error is computed as (std dev defocus)/defocus*100%")
	parser.add_option("--astigmatismerror",   	type="float",  		default=360.0,            help="Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees.")

	# import ctf estimates done using cter
	parser.add_option("--scale",              	type="float", 		default=-1.0,      		  help="Divide shifts in the input 3D orientation parameters text file by the scale factor.")
	
	# generate adaptive mask from an given 3-D volume
	parser.add_option("--adaptive_mask",        action="store_true",                      help="create adavptive 3-D mask from a given volume", default=False)
	parser.add_option("--nsigma",              	type="float",	default= 1.,     	      help="number of times of sigma of the input volume to obtain the the large density cluster")
	parser.add_option("--ndilation",            type="int",		default= 3,     		  help="number of times of dilation applied to the largest cluster of density")
	parser.add_option("--kernel_size",          type="int",		default= 11,     		  help="convolution kernel for smoothing the edge of the mask")
	parser.add_option("--gauss_standard_dev",   type="int",		default= 9,     		  help="stanadard deviation value to generate Gaussian edge")
	parser.add_option("--threshold",            type="float",	default= 9999.,           help="threshold provided by user to binarize input volume")
	parser.add_option("--ne",                   type="int",		default= 0,     		  help="number of times to erode the binarized  input image")
	parser.add_option("--nd",                   type="int",		default= 0,     		  help="number of times to dilate the binarized input image")
	parser.add_option("--postprocess",          action="store_true",                      help="postprocess unfiltered odd, even 3-D volumes",default=False)
	parser.add_option("--fsc_weighted",         action="store_true",                      help="postprocess unfiltered odd, even 3-D volumes")
	parser.add_option("--low_pass_filter",      action="store_true",      default=False,  help="postprocess unfiltered odd, even 3-D volumes")
	parser.add_option("--ff",                   type="float", default=.25,                help="low pass filter stop band frequency in absolute unit")
	parser.add_option("--aa",                   type="float", default=.1,                 help="low pass filter falloff" )
	parser.add_option("--mask",           type="string",                                  help="input mask file",  default=None)
	parser.add_option("--output",         type="string",                                  help="output file name", default="postprocessed.hdf")
	parser.add_option("--pixel_size",     type="float",                                   help="pixel size of the data", default=1.0)
	parser.add_option("--B_start",     type="float",                                      help="starting frequency in Angstrom for B-factor estimation", default=10.)
	parser.add_option("--FSC_cutoff",     type="float",                                   help="stop frequency in Angstrom for B-factor estimation", default=0.143)
	parser.add_option("--2d",          action="store_true",                      help="postprocess isac 2-D averaged images",default=False)
	parser.add_option("--window_stack",                     action="store_true",          help="window stack images using a smaller window size", default=False)
	parser.add_option("--box",           type="int",		default= 0,                   help="the new window size ") 
 	(options, args) = parser.parse_args()

	global_def.BATCH = True
		
	if options.phase_flip:
		nargs = len(args)
		if nargs != 2:
			print "must provide name of input and output file!"
			return
		from EMAN2 import Processor
		instack = args[0]
		outstack = args[1]
		nima = EMUtil.get_image_count(instack)
		from filter import filt_ctf
		for i in xrange(nima):
			img = EMData()
			img.read_image(instack, i)
			try:
				ctf = img.get_attr('ctf')
			except:
				print "no ctf information in input stack! Exiting..."
				return
			
			dopad = True
			sign = 1
			binary = 1  # phase flip
				
			assert img.get_ysize() > 1	
			dict = ctf.to_dict()
			dz = dict["defocus"]
			cs = dict["cs"]
			voltage = dict["voltage"]
			pixel_size = dict["apix"]
			b_factor = dict["bfactor"]
			ampcont = dict["ampcont"]
			dza = dict["dfdiff"]
			azz = dict["dfang"]
			
			if dopad and not img.is_complex(): ip = 1
			else:                             ip = 0
	
	
			params = {"filter_type": Processor.fourier_filter_types.CTF_,
	 			"defocus" : dz,
				"Cs": cs,
				"voltage": voltage,
				"Pixel_size": pixel_size,
				"B_factor": b_factor,
				"amp_contrast": ampcont,
				"dopad": ip,
				"binary": binary,
				"sign": sign,
				"dza": dza,
				"azz":azz}
			
			tmp = Processor.EMFourierFilter(img, params)
			tmp.set_attr_dict({"ctf": ctf})
			
			tmp.write_image(outstack, i)

	elif options.changesize:
		nargs = len(args)
		if nargs != 2:
			ERROR("must provide name of input and output file!", "change size", 1)
			return
		from utilities import get_im
		instack = args[0]
		outstack = args[1]
		sub_rate = float(options.ratio)
			
		nima = EMUtil.get_image_count(instack)
		from fundamentals import resample
		for i in xrange(nima):
			resample(get_im(instack, i), sub_rate).write_image(outstack, i)

	elif options.isacgroup>-1:
		nargs = len(args)
		if nargs != 3:
			ERROR("Three files needed on input!", "isacgroup", 1)
			return
		from utilities import get_im
		instack = args[0]
		m=get_im(args[1],int(options.isacgroup)).get_attr("members")
		l = []
		for k in m:
			l.append(int(get_im(args[0],k).get_attr(options.params)))
		from utilities import write_text_file
		write_text_file(l, args[2])

	elif options.isacselect:
		nargs = len(args)
		if nargs != 2:
			ERROR("Two files needed on input!", "isacgroup", 1)
			return
		from utilities import get_im
		nima = EMUtil.get_image_count(args[0])
		m = []
		for k in xrange(nima):
			m += get_im(args[0],k).get_attr("members")
		m.sort()
		from utilities import write_text_file
		write_text_file(m, args[1])

	elif options.pw:
		nargs = len(args)
		if nargs < 2:
			ERROR("must provide name of input and output file!", "pw", 1)
			return
		from utilities import get_im, write_text_file
		from fundamentals import rops_table
		d = get_im(args[0])
		ndim = d.get_ndim()
		if ndim ==3:
			pw = rops_table(d)
			write_text_file(pw, args[1])			
		else:
			nx = d.get_xsize()
			ny = d.get_ysize()
			if nargs ==3: mask = get_im(args[2])
			wn = int(options.wn)
			if wn == -1:
				wn = max(nx, ny)
			else:
				if( (wn<nx) or (wn<ny) ):  ERROR("window size cannot be smaller than the image size","pw",1)
			n = EMUtil.get_image_count(args[0])
			from utilities import model_blank, model_circle, pad
			from EMAN2 import periodogram
			p = model_blank(wn,wn)
		
			for i in xrange(n):
				d = get_im(args[0], i)
				if nargs==3:
					d *=mask
				st = Util.infomask(d, None, True)
				d -= st[0]
				p += periodogram(pad(d, wn, wn, 1, 0.))
			p /= n
			p.write_image(args[1])

	elif options.adjpw:

		if len(args) < 3:
			ERROR("filt_by_rops input target output fl aa (the last two are optional parameters of a low-pass filter)","adjpw",1)
			return
		img_stack = args[0]
		from math         import sqrt
		from fundamentals import rops_table, fft
		from utilities    import read_text_file, get_im
		from filter       import  filt_tanl, filt_table
		if(  args[1][-3:] == 'txt'):
			rops_dst = read_text_file( args[1] )
		else:
			rops_dst = rops_table(get_im( args[1] ))

		out_stack = args[2]
		if(len(args) >4):
			fl = float(args[3])
			aa = float(args[4])
		else:
			fl = -1.0
			aa = 0.0

		nimage = EMUtil.get_image_count( img_stack )

		for i in xrange(nimage):
			img = fft(get_im(img_stack, i) )
			rops_src = rops_table(img)

			assert len(rops_dst) == len(rops_src)

			table = [0.0]*len(rops_dst)
			for j in xrange( len(rops_dst) ):
				table[j] = sqrt( rops_dst[j]/rops_src[j] )

			if( fl > 0.0):
				img = filt_tanl(img, fl, aa)
			img = fft(filt_table(img, table))
			img.write_image(out_stack, i)

	elif options.rotpw != None:

		if len(args) != 1:
			ERROR("Only one input permitted","rotpw",1)
			return
		from utilities import write_text_file, get_im
		from fundamentals import rops_table
		from math import log10
		t = rops_table(get_im(args[0]))
		x = range(len(t))
		r = [0.0]*len(x)
		for i in x:  r[i] = log10(t[i])
		write_text_file([t,r,x],options.rotpw)

	elif options.transformparams != None:
		if len(args) != 2:
			ERROR("Please provide names of input and output files with orientation parameters","transformparams",1)
			return
		from utilities import read_text_row, write_text_row
		transf = [0.0]*6
		spl=options.transformparams.split(',')
		for i in xrange(len(spl)):  transf[i] = float(spl[i])

		write_text_row( rotate_shift_params(read_text_row(args[0]), transf)	, args[1])

	elif options.makedb != None:
		nargs = len(args)
		if nargs != 1:
			print "must provide exactly one argument denoting database key under which the input params will be stored"
			return
		dbkey = args[0]
		print "database key under which params will be stored: ", dbkey
		gbdb = js_open_dict("e2boxercache/gauss_box_DB.json")
				
		parmstr = 'dummy:'+options.makedb[0]
		(processorname, param_dict) = parsemodopt(parmstr)
		dbdict = {}
		for pkey in param_dict:
			if (pkey == 'invert_contrast') or (pkey == 'use_variance'):
				if param_dict[pkey] == 'True':
					dbdict[pkey] = True
				else:
					dbdict[pkey] = False
			else:		
				dbdict[pkey] = param_dict[pkey]
		gbdb[dbkey] = dbdict

	elif options.generate_projections:
		nargs = len(args)
		if nargs != 3:
			ERROR("Must provide name of input structure(s) from which to generate projections, name of output projection stack, and prefix for output micrographs."\
			"sxprocess - generate projections",1)
			return
		inpstr  = args[0]
		outstk  = args[1]
		micpref = args[2]

		parmstr = 'dummy:'+options.generate_projections[0]
		(processorname, param_dict) = parsemodopt(parmstr)

		parm_CTF    = False
		parm_format = 'bdb'
		parm_apix   = 2.5

		if 'CTF' in param_dict:
			if param_dict['CTF'] == 'True':
				parm_CTF = True

		if 'format' in param_dict:
			parm_format = param_dict['format']

		if 'apix' in param_dict:
			parm_apix = float(param_dict['apix'])

		boxsize = 64
		if 'boxsize' in param_dict:
			boxsize = int(param_dict['boxsize'])

		print "pixel size: ", parm_apix, " format: ", parm_format, " add CTF: ", parm_CTF, " box size: ", boxsize

		scale_mult      = 2500
		sigma_add       = 1.5
		sigma_proj      = 30.0
		sigma2_proj     = 17.5
		sigma_gauss     = 0.3
		sigma_mic       = 30.0
		sigma2_mic      = 17.5
		sigma_gauss_mic = 0.3
		
		if 'scale_mult' in param_dict:
			scale_mult = float(param_dict['scale_mult'])
		if 'sigma_add' in param_dict:
			sigma_add = float(param_dict['sigma_add'])
		if 'sigma_proj' in param_dict:
			sigma_proj = float(param_dict['sigma_proj'])
		if 'sigma2_proj' in param_dict:
			sigma2_proj = float(param_dict['sigma2_proj'])
		if 'sigma_gauss' in param_dict:
			sigma_gauss = float(param_dict['sigma_gauss'])	
		if 'sigma_mic' in param_dict:
			sigma_mic = float(param_dict['sigma_mic'])
		if 'sigma2_mic' in param_dict:
			sigma2_mic = float(param_dict['sigma2_mic'])
		if 'sigma_gauss_mic' in param_dict:
			sigma_gauss_mic = float(param_dict['sigma_gauss_mic'])	
			
		from filter import filt_gaussl, filt_ctf
		from utilities import drop_spider_doc, even_angles, model_gauss, delete_bdb, model_blank,pad,model_gauss_noise,set_params2D, set_params_proj
		from projection import prep_vol,prgs
		seed(14567)
		delta = 29
		angles = even_angles(delta, 0.0, 89.9, 0.0, 359.9, "S")
		nangle = len(angles)
		
		modelvol = []
		nvlms = EMUtil.get_image_count(inpstr)
		from utilities import get_im
		for k in xrange(nvlms):  modelvol.append(get_im(inpstr,k))
		
		nx = modelvol[0].get_xsize()
		
		if nx != boxsize:
			ERROR("Requested box dimension does not match dimension of the input model.", \
			"sxprocess - generate projections",1)
		nvol = 10
		volfts = [[] for k in xrange(nvlms)]
		for k in xrange(nvlms):
			for i in xrange(nvol):
				sigma = sigma_add + random()  # 1.5-2.5
				addon = model_gauss(sigma, boxsize, boxsize, boxsize, sigma, sigma, 38, 38, 40 )
				scale = scale_mult * (0.5+random())
				vf, kb = prep_vol(modelvol[k] + scale*addon)
				volfts[k].append(vf)
		del vf, modelvol

		if parm_format == "bdb":
			stack_data = "bdb:"+outstk
			delete_bdb(stack_data)
		else:
			stack_data = outstk + ".hdf"
		Cs      = 2.0
		pixel   = parm_apix
		voltage = 120.0
		ampcont = 10.0
		ibd     = 4096/2-boxsize
		iprj    = 0

		width = 240
		xstart = 8 + boxsize/2
		ystart = 8 + boxsize/2
		rowlen = 17
		from random import randint
		params = []
		for idef in xrange(3, 8):

			irow = 0
			icol = 0

			mic = model_blank(4096, 4096)
			defocus = idef * 0.5#0.2
			if parm_CTF:
				astampl=defocus*0.15
				astangl=50.0
				ctf = generate_ctf([defocus, Cs, voltage,  pixel, ampcont, 0.0, astampl, astangl])

			for i in xrange(nangle):
				for k in xrange(12):
					dphi = 8.0*(random()-0.5)
					dtht = 8.0*(random()-0.5)
					psi  = 360.0*random()

					phi = angles[i][0]+dphi
					tht = angles[i][1]+dtht

					s2x = 4.0*(random()-0.5)
					s2y = 4.0*(random()-0.5)

					params.append([phi, tht, psi, s2x, s2y])

					ivol = iprj % nvol
					#imgsrc = randint(0,nvlms-1)
					imgsrc = iprj % nvlms
					proj = prgs(volfts[imgsrc][ivol], kb, [phi, tht, psi, -s2x, -s2y])

					x = xstart + irow * width
					y = ystart + icol * width

					mic += pad(proj, 4096, 4096, 1, 0.0, x-2048, y-2048, 0)

					proj = proj + model_gauss_noise( sigma_proj, nx, nx )
					if parm_CTF:
						proj = filt_ctf(proj, ctf)
						proj.set_attr_dict({"ctf":ctf, "ctf_applied":0})

					proj = proj + filt_gaussl(model_gauss_noise(sigma2_proj, nx, nx), sigma_gauss)
					proj.set_attr("origimgsrc",imgsrc)
					proj.set_attr("test_id", iprj)
					# flags describing the status of the image (1 = true, 0 = false)
					set_params2D(proj, [0.0, 0.0, 0.0, 0, 1.0])
					set_params_proj(proj, [phi, tht, psi, s2x, s2y])

					proj.write_image(stack_data, iprj)
			
					icol += 1
					if icol == rowlen:
						icol = 0
						irow += 1

					iprj += 1

			mic += model_gauss_noise(sigma_mic,4096,4096)
			if parm_CTF:
				#apply CTF
				mic = filt_ctf(mic, ctf)
			mic += filt_gaussl(model_gauss_noise(sigma2_mic, 4096, 4096), sigma_gauss_mic)
	
			mic.write_image(micpref + "%1d.hdf" % (idef-3), 0)
		
		drop_spider_doc("params.txt", params)

	elif options.importctf != None:
		print ' IMPORTCTF  '
		from utilities import read_text_row,write_text_row
		from random import randint
		import subprocess
		grpfile = 'groupid%04d'%randint(1000,9999)
		ctfpfile = 'ctfpfile%04d'%randint(1000,9999)
		cterr = [options.defocuserror/100.0, options.astigmatismerror]
		ctfs = read_text_row(options.importctf)
		for kk in xrange(len(ctfs)):
			root,name = os.path.split(ctfs[kk][-1])
			ctfs[kk][-1] = name[:-4]
		if(options.input[:4] != 'bdb:'):
			ERROR('Sorry, only bdb files implemented','importctf',1)
		d = options.input[4:]
		#try:     str = d.index('*')
		#except:  str = -1
		from string import split
		import glob
		uu = os.path.split(d)
		uu = os.path.join(uu[0],'EMAN2DB',uu[1]+'.bdb')
		flist = glob.glob(uu)
		for i in xrange(len(flist)):
			root,name = os.path.split(flist[i])
			root = root[:-7]
			name = name[:-4]
			fil = 'bdb:'+os.path.join(root,name)
			sourcemic = EMUtil.get_all_attributes(fil,'ptcl_source_image')
			nn = len(sourcemic)
			gctfp = []
			groupid = []
			for kk in xrange(nn):
				junk,name2 = os.path.split(sourcemic[kk])
				name2 = name2[:-4]
				ctfp = [-1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]
				for ll in xrange(len(ctfs)):
					if(name2 == ctfs[ll][-1]):
						#  found correct
						if(ctfs[ll][8]/ctfs[ll][0] <= cterr[0]):
							#  acceptable defocus error
							ctfp = ctfs[ll][:8]
							if(ctfs[ll][10] > cterr[1] ):
								# error of astigmatism exceed the threshold, set astigmatism to zero.
								ctfp[6] = 0.0
								ctfp[7] = 0.0
							gctfp.append(ctfp)
							groupid.append(kk)
						break
			if(len(groupid) > 0):
				write_text_row(groupid, grpfile)
				write_text_row(gctfp, ctfpfile)
				cmd = "{} {} {} {}".format('e2bdb.py',fil,'--makevstack=bdb:'+root+'G'+name,'--list='+grpfile)
				#print cmd
				subprocess.call(cmd, shell=True)
				cmd = "{} {} {} {}".format('sxheader.py','bdb:'+root+'G'+name,'--params=ctf','--import='+ctfpfile)
				#print cmd
				subprocess.call(cmd, shell=True)
			else:
				print  ' >>>  Group ',name,'  skipped.'
				
		cmd = "{} {} {}".format("rm -f",grpfile,ctfpfile)
		subprocess.call(cmd, shell=True)

	elif options.scale > 0.0:
		from utilities import read_text_row,write_text_row
		scale = options.scale
		nargs = len(args)
		if nargs != 2:
			print "Please provide names of input and output file!"
			return
		p = read_text_row(args[0])
		for i in xrange(len(p)):
			p[i][3] /= scale
			p[i][4] /= scale
		write_text_row(p, args[1])
		
	elif options.adaptive_mask:
		from utilities import get_im
		from morphology import adaptive_mask, binarize, erosion, dilation
		nsigma             = options.nsigma
		ndilation          = options.ndilation
		kernel_size        = options.kernel_size
		gauss_standard_dev = options.gauss_standard_dev
		nargs = len(args)
		if nargs ==0:
			print " Create 3D mask from a given volume, either automatically or from the user provided threshold."
		elif nargs > 2:
			print "Too many inputs are given, try again!"
			return
		else:
			inputvol = get_im(args[0])
			input_path, input_file_name = os.path.split(args[0])
			input_file_name_root,ext=os.path.splitext(input_file_name)
			if nargs == 2:  mask_file_name = args[1]
			else:           mask_file_name = "adaptive_mask_for_"+input_file_name_root+".hdf" # Only hdf file is output.
			if options.threshold !=9999.:
				mask3d = binarize(inputvol, options.threshold)
				for i in xrange(options.ne): mask3d = erosion(mask3d)
				for i in xrange(options.nd): mask3d = dilation(mask3d)
			else: 
				mask3d = adaptive_mask(inputvol, nsigma, ndilation, kernel_size, gauss_standard_dev)
			mask3d.write_image(mask_file_name)
			
	elif options.postprocess:
		from utilities    import get_im
		from fundamentals import rot_avg_table
		from morphology   import compute_bfactor,power
		from statistics   import fsc
		from filter       import filt_table, filt_gaussinv
		from EMAN2 import periodogram
		e1   = get_im(args[0],0)
		if e1.get_zsize()==1:
			nimage = EMUtil.get_image_count(args[0])
			if options.mask !=None: m = get_im(options.mask)
			else: m = None
			for i in xrange(nimage):
				e1 = get_im(args[0],i)
				if m: e1 *=m
				guinerline = rot_avg_table(power(periodogram(e1),.5))
				freq_max   =  1/(2.*pixel_size)
				freq_min   =  1./options.B_start
				b,junk=compute_bfactor(guinerline, freq_min, freq_max, pixel_size)
				tmp = b/pixel_size**2
				sigma_of_inverse=sqrt(2./tmp)
				e1 = filt_gaussinv(e1,sigma_of_inverse)
				if options.low_pass_filter:
					from filter import filt_tanl
					e1 =filt_tanl(e1,options.ff, options.aa)
				e1.write_image(options.output)							
		else:
			nargs = len(args)
			e1    = get_im(args[0])
			if nargs >1: e2 = get_im(args[1])
			if options.mask !=None: m = get_im(options.mask)
			else: m =None
			pixel_size = options.pixel_size
			from math import sqrt
			if m !=None:
				e1 *=m
				if nargs >1 :e2 *=m
			if options.fsc_weighted:
				frc = fsc(e1,e2,1)
				## FSC is done on masked two images
				#### FSC weighting sqrt((2.*fsc)/(1+fsc));
				fil = len(frc[1])*[None]
				for i in xrange(len(fil)):
					if frc[1][i]>=options.FSC_cutoff: tmp = frc[1][i]
					else: tmp = 0.0
					fil[i] = sqrt(2.*tmp/(1.+tmp))
			if nargs>1: e1 +=e2
			if options.fsc_weighted: e1=filt_table(e1,fil) 
			guinerline = rot_avg_table(power(periodogram(e1),.5))
			freq_max   = 1/(2.*pixel_size)
			freq_min   = 1./options.B_start
			b,junk     = compute_bfactor(guinerline, freq_min, freq_max, pixel_size)
			tmp        = b/pixel_size**2
			sigma_of_inverse=sqrt(2./tmp)
			e1  = filt_gaussinv(e1,sigma_of_inverse)
			if options.low_pass_filter:
				from filter       import filt_tanl
				e1 =filt_tanl(e1,options.ff, options.aa)
			e1.write_image(options.output)
		 
	elif options.window_stack:
		nargs = len(args)
		if nargs ==0:
			print "  Reduce image size of a stack"
			return
		else:
			output_stack_name = None
			inputstack = args[0]
			if nargs ==2:output_stack_name = args[1]
			input_path,input_file_name=os.path.split(inputstack)
			input_file_name_root,ext=os.path.splitext(input_file_name)
			if input_file_name_root[0:3]=="bdb":stack_is_bdb= True
			else: stack_is_bdb= False
			if output_stack_name is None:
				if stack_is_bdb: output_stack_name ="bdb:reduced_"+input_file_name_root[4:]
				else:output_stack_name = "reduced_"+input_file_name_root+".hdf" # Only hdf file is output.
			nimage = EMUtil.get_image_count(inputstack)
			from fundamentals import window2d
			for i in xrange(nimage):
				image = EMData()
				image.read_image(inputstack,i)
				w = window2d(image,options.box,options.box)
				w.write_image(output_stack_name,i)
	else:  ERROR("Please provide option name","sxprocess.py",1)	
Esempio n. 13
0
def helicalshiftali_MPI(stack, maskfile=None, maxit=100, CTF=False, snr=1.0, Fourvar=False, search_rng=-1):
	from applications import MPI_start_end
	from utilities    import model_circle, model_blank, get_image, peak_search, get_im, pad
	from utilities    import reduce_EMData_to_root, bcast_EMData_to_all, send_attr_dict, file_type, bcast_number_to_all, bcast_list_to_all
	from statistics   import varf2d_MPI
	from fundamentals import fft, ccf, rot_shift3D, rot_shift2D, fshift
	from utilities    import get_params2D, set_params2D, chunks_distribution
	from utilities    import print_msg, print_begin_msg, print_end_msg
	import os
	import sys
	from mpi 	  	  import mpi_init, mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD
	from mpi 	  	  import mpi_reduce, mpi_bcast, mpi_barrier, mpi_gatherv
	from mpi 	  	  import MPI_SUM, MPI_FLOAT, MPI_INT
	from time         import time	
	from pixel_error  import ordersegments
	from math         import sqrt, atan2, tan, pi
	
	nproc = mpi_comm_size(MPI_COMM_WORLD)
	myid = mpi_comm_rank(MPI_COMM_WORLD)
	main_node = 0
		
	ftp = file_type(stack)

	if myid == main_node:
		print_begin_msg("helical-shiftali_MPI")

	max_iter=int(maxit)
	if( myid == main_node):
		infils = EMUtil.get_all_attributes(stack, "filament")
		ptlcoords = EMUtil.get_all_attributes(stack, 'ptcl_source_coord')
		filaments = ordersegments(infils, ptlcoords)
		total_nfils = len(filaments)
		inidl = [0]*total_nfils
		for i in xrange(total_nfils):  inidl[i] = len(filaments[i])
		linidl = sum(inidl)
		nima = linidl
		tfilaments = []
		for i in xrange(total_nfils):  tfilaments += filaments[i]
		del filaments
	else:
		total_nfils = 0
		linidl = 0
	total_nfils = bcast_number_to_all(total_nfils, source_node = main_node)
	if myid != main_node:
		inidl = [-1]*total_nfils
	inidl = bcast_list_to_all(inidl, myid, source_node = main_node)
	linidl = bcast_number_to_all(linidl, source_node = main_node)
	if myid != main_node:
		tfilaments = [-1]*linidl
	tfilaments = bcast_list_to_all(tfilaments, myid, source_node = main_node)
	filaments = []
	iendi = 0
	for i in xrange(total_nfils):
		isti = iendi
		iendi = isti+inidl[i]
		filaments.append(tfilaments[isti:iendi])
	del tfilaments,inidl

	if myid == main_node:
		print_msg( "total number of filaments: %d"%total_nfils)
	if total_nfils< nproc:
		ERROR('number of CPUs (%i) is larger than the number of filaments (%i), please reduce the number of CPUs used'%(nproc, total_nfils), "ehelix_MPI", 1,myid)

	#  balanced load
	temp = chunks_distribution([[len(filaments[i]), i] for i in xrange(len(filaments))], nproc)[myid:myid+1][0]
	filaments = [filaments[temp[i][1]] for i in xrange(len(temp))]
	nfils     = len(filaments)

	#filaments = [[0,1]]
	#print "filaments",filaments
	list_of_particles = []
	indcs = []
	k = 0
	for i in xrange(nfils):
		list_of_particles += filaments[i]
		k1 = k+len(filaments[i])
		indcs.append([k,k1])
		k = k1
	data = EMData.read_images(stack, list_of_particles)
	ldata = len(data)
	print "ldata=", ldata
	nx = data[0].get_xsize()
	ny = data[0].get_ysize()
	if maskfile == None:
		mrad = min(nx, ny)//2-2
		mask = pad( model_blank(2*mrad+1, ny, 1, 1.0), nx, ny, 1, 0.0)
	else:
		mask = get_im(maskfile)

	# apply initial xform.align2d parameters stored in header
	init_params = []
	for im in xrange(ldata):
		t = data[im].get_attr('xform.align2d')
		init_params.append(t)
		p = t.get_params("2d")
		data[im] = rot_shift2D(data[im], p['alpha'], p['tx'], p['ty'], p['mirror'], p['scale'])

	if CTF:
		from filter import filt_ctf
		from morphology   import ctf_img
		ctf_abs_sum = EMData(nx, ny, 1, False)
		ctf_2_sum = EMData(nx, ny, 1, False)
	else:
		ctf_2_sum = None
		ctf_abs_sum = None



	from utilities import info

	for im in xrange(ldata):
		data[im].set_attr('ID', list_of_particles[im])
		st = Util.infomask(data[im], mask, False)
		data[im] -= st[0]
		if CTF:
			ctf_params = data[im].get_attr("ctf")
			qctf = data[im].get_attr("ctf_applied")
			if qctf == 0:
				data[im] = filt_ctf(fft(data[im]), ctf_params)
				data[im].set_attr('ctf_applied', 1)
			elif qctf != 1:
				ERROR('Incorrectly set qctf flag', "helicalshiftali_MPI", 1,myid)
			ctfimg = ctf_img(nx, ctf_params, ny=ny)
			Util.add_img2(ctf_2_sum, ctfimg)
			Util.add_img_abs(ctf_abs_sum, ctfimg)
		else:  data[im] = fft(data[im])

	del list_of_particles		

	if CTF:
		reduce_EMData_to_root(ctf_2_sum, myid, main_node)
		reduce_EMData_to_root(ctf_abs_sum, myid, main_node)
	if CTF:
		if myid != main_node:
			del ctf_2_sum
			del ctf_abs_sum
		else:
			temp = EMData(nx, ny, 1, False)
			tsnr = 1./snr
			for i in xrange(0,nx+2,2):
				for j in xrange(ny):
					temp.set_value_at(i,j,tsnr)
					temp.set_value_at(i+1,j,0.0)
			#info(ctf_2_sum)
			Util.add_img(ctf_2_sum, temp)
			#info(ctf_2_sum)
			del temp

	total_iter = 0
	shift_x = [0.0]*ldata

	for Iter in xrange(max_iter):
		if myid == main_node:
			start_time = time()
			print_msg("Iteration #%4d\n"%(total_iter))
		total_iter += 1
		avg = EMData(nx, ny, 1, False)
		for im in xrange(ldata):
			Util.add_img(avg, fshift(data[im], shift_x[im]))

		reduce_EMData_to_root(avg, myid, main_node)

		if myid == main_node:
			if CTF:  tavg = Util.divn_filter(avg, ctf_2_sum)
			else:    tavg = Util.mult_scalar(avg, 1.0/float(nima))
		else:
			tavg = model_blank(nx,ny)

		if Fourvar:
			bcast_EMData_to_all(tavg, myid, main_node)
			vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF)

		if myid == main_node:
			if Fourvar:
				tavg    = fft(Util.divn_img(fft(tavg), vav))
				vav_r	= Util.pack_complex_to_real(vav)
			# normalize and mask tavg in real space
			tavg = fft(tavg)
			stat = Util.infomask( tavg, mask, False )
			tavg -= stat[0]
			Util.mul_img(tavg, mask)
			tavg.write_image("tavg.hdf",Iter)
			# For testing purposes: shift tavg to some random place and see if the centering is still correct
			#tavg = rot_shift3D(tavg,sx=3,sy=-4)

		if Fourvar:  del vav
		bcast_EMData_to_all(tavg, myid, main_node)
		tavg = fft(tavg)

		sx_sum = 0.0
		nxc = nx//2
		
		for ifil in xrange(nfils):
			"""
			# Calculate filament average
			avg = EMData(nx, ny, 1, False)
			filnima = 0
			for im in xrange(indcs[ifil][0], indcs[ifil][1]):
				Util.add_img(avg, data[im])
				filnima += 1
			tavg = Util.mult_scalar(avg, 1.0/float(filnima))
			"""
			# Calculate 1D ccf between each segment and filament average
			nsegms = indcs[ifil][1]-indcs[ifil][0]
			ctx = [None]*nsegms
			pcoords = [None]*nsegms
			for im in xrange(indcs[ifil][0], indcs[ifil][1]):
				ctx[im-indcs[ifil][0]] = Util.window(ccf(tavg, data[im]), nx, 1)
				pcoords[im-indcs[ifil][0]] = data[im].get_attr('ptcl_source_coord')
				#ctx[im-indcs[ifil][0]].write_image("ctx.hdf",im-indcs[ifil][0])
				#print "  CTX  ",myid,im,Util.infomask(ctx[im-indcs[ifil][0]], None, True)
			# search for best x-shift
			cents = nsegms//2
			
			dst = sqrt(max((pcoords[cents][0] - pcoords[0][0])**2 + (pcoords[cents][1] - pcoords[0][1])**2, (pcoords[cents][0] - pcoords[-1][0])**2 + (pcoords[cents][1] - pcoords[-1][1])**2))
			maxincline = atan2(ny//2-2-float(search_rng),dst)
			kang = int(dst*tan(maxincline)+0.5)
			#print  "  settings ",nsegms,cents,dst,search_rng,maxincline,kang
			
			# ## C code for alignment. @ming
 			results = [0.0]*3;
 			results = Util.helixshiftali(ctx, pcoords, nsegms, maxincline, kang, search_rng,nxc)
			sib = int(results[0])
 			bang = results[1]
 			qm = results[2]
			#print qm, sib, bang
			
			# qm = -1.e23	
# 				
# 			for six in xrange(-search_rng, search_rng+1,1):
# 				q0 = ctx[cents].get_value_at(six+nxc)
# 				for incline in xrange(kang+1):
# 					qt = q0
# 					qu = q0
# 					if(kang>0):  tang = tan(maxincline/kang*incline)
# 					else:        tang = 0.0
# 					for kim in xrange(cents+1,nsegms):
# 						dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
# 						xl = dst*tang+six+nxc
# 						ixl = int(xl)
# 						dxl = xl - ixl
# 						#print "  A  ", ifil,six,incline,kim,xl,ixl,dxl
# 						qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# 						xl = -dst*tang+six+nxc
# 						ixl = int(xl)
# 						dxl = xl - ixl
# 						qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# 					for kim in xrange(cents):
# 						dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
# 						xl = -dst*tang+six+nxc
# 						ixl = int(xl)
# 						dxl = xl - ixl
# 						qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# 						xl =  dst*tang+six+nxc
# 						ixl = int(xl)
# 						dxl = xl - ixl
# 						qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# 					if( qt > qm ):
# 						qm = qt
# 						sib = six
# 						bang = tang
# 					if( qu > qm ):
# 						qm = qu
# 						sib = six
# 						bang = -tang
					#if incline == 0:  print  "incline = 0  ",six,tang,qt,qu
			#print qm,six,sib,bang
			#print " got results   ",indcs[ifil][0], indcs[ifil][1], ifil,myid,qm,sib,tang,bang,len(ctx),Util.infomask(ctx[0], None, True)
			for im in xrange(indcs[ifil][0], indcs[ifil][1]):
				kim = im-indcs[ifil][0]
				dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
				if(kim < cents):  xl = -dst*bang+sib
				else:             xl =  dst*bang+sib
				shift_x[im] = xl
							
			# Average shift
			sx_sum += shift_x[indcs[ifil][0]+cents]
			
			
		# #print myid,sx_sum,total_nfils
		sx_sum = mpi_reduce(sx_sum, 1, MPI_FLOAT, MPI_SUM, main_node, MPI_COMM_WORLD)
		if myid == main_node:
			sx_sum = float(sx_sum[0])/total_nfils
			print_msg("Average shift  %6.2f\n"%(sx_sum))
		else:
			sx_sum = 0.0
		sx_sum = 0.0
		sx_sum = bcast_number_to_all(sx_sum, source_node = main_node)
		for im in xrange(ldata):
			shift_x[im] -= sx_sum
			#print  "   %3d  %6.3f"%(im,shift_x[im])
		#exit()


			
	# combine shifts found with the original parameters
	for im in xrange(ldata):		
		t1 = Transform()
		##import random
		##shix=random.randint(-10, 10)
		##t1.set_params({"type":"2D","tx":shix})
		t1.set_params({"type":"2D","tx":shift_x[im]})
		# combine t0 and t1
		tt = t1*init_params[im]
		data[im].set_attr("xform.align2d", tt)
	# write out headers and STOP, under MPI writing has to be done sequentially
	mpi_barrier(MPI_COMM_WORLD)
	par_str = ["xform.align2d", "ID"]
	if myid == main_node:
		from utilities import file_type
		if(file_type(stack) == "bdb"):
			from utilities import recv_attr_dict_bdb
			recv_attr_dict_bdb(main_node, stack, data, par_str, 0, ldata, nproc)
		else:
			from utilities import recv_attr_dict
			recv_attr_dict(main_node, stack, data, par_str, 0, ldata, nproc)
	else:           send_attr_dict(main_node, data, par_str, 0, ldata)
	if myid == main_node: print_end_msg("helical-shiftali_MPI")				
Esempio n. 14
0
def main(args):
	progname = os.path.basename(sys.argv[0])
	usage = ( progname + " stack_file  output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter" +
			" --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp" +
			"  --generation=generation  --rand_seed=rand_seed>" )
	
	parser = OptionParser(usage,version=SPARXVERSION)
	parser.add_option("--radius",         type="int",          default=-1,      help="<Particle radius>, it has to be provided.")
	parser.add_option("--img_per_grp",    type="int",          default=100,     help="<number of images per group> in the ideal case (essentially maximum size of class) (100)")
	parser.add_option("--CTF",            action="store_true", default=False,   help="<CTF flag>, if set the data will be phase-flipped")
	parser.add_option("--ir",             type="int",          default=1,       help="<inner ring> of the resampling to polar coordinates (1)")
	parser.add_option("--rs",             type="int",          default=1,       help="<ring step> of the resampling to polar coordinates (1)")
	parser.add_option("--xr",             type="int",          default=-1,      help="<x range> of translational search (By default set by the program) (advanced)")
	parser.add_option("--yr",             type="int",          default=-1,      help="<y range> of translational search (same as xr) (advanced)")
	parser.add_option("--ts",             type="float",        default=1.0,     help="<search step> of translational search (1.0)")
	parser.add_option("--maxit",          type="int",          default=30,      help="number of iterations for reference-free alignment (30)")
	#parser.add_option("--snr",            type="float",        default=1.0,     help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)")
	parser.add_option("--center_method",  type="int",          default=7,       help="<Method for centering> of global 2D average during initial prealignment of data (default : 7; 0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7)")
	parser.add_option("--dst",            type="float",        default=90.0,    help="discrete angle used in within group alignment ")
	parser.add_option("--FL",             type="float",        default=0.2,     help="<lowest stopband> frequency used in the tangent filter (0.2)")
	parser.add_option("--FH",             type="float",        default=0.3,     help="<highest stopband> frequency used in the tangent filter (0.3)")
	parser.add_option("--FF",             type="float",        default=0.2,     help="<fall-off of the tangent> filter (0.2)")
	parser.add_option("--init_iter",      type="int",          default=3,       help="<init_iter> number of iterations of ISAC program in initialization (3)")
	parser.add_option("--main_iter",      type="int",          default=3,       help="<main_iter> number of iterations of ISAC program in main part (3)")
	parser.add_option("--iter_reali",     type="int",          default=1,       help="<iter_reali> number of iterations in ISAC before checking stability (1)")
	parser.add_option("--match_first",    type="int",          default=1,       help="number of iterations to run 2-way matching in the first phase (1)")
	parser.add_option("--max_round",      type="int",          default=20,      help="maximum rounds of generating candidate averages in the first phase (20)")
	parser.add_option("--match_second",   type="int",          default=5,       help="number of iterations to run 2-way (or 3-way) matching in the second phase (5)")
	parser.add_option("--stab_ali",       type="int",          default=5,       help="number of alignments when checking stability (5)")
	parser.add_option("--thld_err",       type="float",        default=0.7,     help="the threshold of pixel error when checking stability (0.7)")
	parser.add_option("--indep_run",      type="int",          default=4,       help="number of independent runs for reproducibility (default=4, only values 2, 3 and 4 are supported (4)")
	parser.add_option("--thld_grp",       type="int",          default=10,      help="minimum size of class (10)")
	parser.add_option("--n_generations",     type="int",          default=100,       help="<n_generations> program stops when reaching this total number of generations (advanced)")
	#parser.add_option("--candidatesexist",action="store_true", default=False,   help="Candidate class averages exist use them (default False)")
	parser.add_option("--rand_seed",      type="int",          default=None,    help="random seed set before calculations, useful for testing purposes (default None - total randomness)")
	parser.add_option("--new",            action="store_true", default=False,   help="use new code (default = False)")
	parser.add_option("--debug",          action="store_true", default=False,   help="debug info printout (default = False)")

	# must be switched off in production
	parser.add_option("--use_latest_master_directory", action="store_true", dest="use_latest_master_directory", default=False)
	
	parser.add_option("--restart_section", type="string", default="", help="<restart section name> (no spaces) followed immediately by comma, followed immediately by generation to restart, example: \n--restart_section=candidate_class_averages,1         (Sections: restart, candidate_class_averages, reproducible_class_averages)")
	parser.add_option("--stop_after_candidates",          action="store_true", default=False,   help="<stop_after_candidates> stops after the 'candidate_class_averages' section")

	parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help = SUPPRESS_HELP)

	(options, args) = parser.parse_args(args)

	if options.return_options:
		return parser
	
	if len(args) > 2:
		print "usage: " + usage
		print "Please run '" + progname + " -h' for detailed options"
		sys.exit()
	
	if global_def.CACHE_DISABLE:
		from utilities import disable_bdb_cache
		disable_bdb_cache()
	
	from isac import iter_isac
	global_def.BATCH = True

	global_def.BATCH = True
	
	command_line_provided_stack_filename = args[0]
	global_def.BATCH = True

	main_node = 0
	mpi_init(0, [])
	myid = mpi_comm_rank(MPI_COMM_WORLD)
	nproc = mpi_comm_size(MPI_COMM_WORLD)

	radi  = options.radius
	center_method  = options.center_method
	if(radi < 1):  ERROR("Particle radius has to be provided!","sxisac",1,myid)

	
	use_latest_master_directory = options.use_latest_master_directory
	stop_after_candidates = options.stop_after_candidates
	program_state_stack.restart_location_title_from_command_line = options.restart_section
	
	from utilities import qw
	program_state_stack.PROGRAM_STATE_VARIABLES = set(qw("""
		isac_generation
	"""))

	# create or reuse master directory
	masterdir = ""
	stack_processed_by_ali2d_base__filename = ""
	stack_processed_by_ali2d_base__filename__without_master_dir = ""
	error_status = 0
	if len(args) == 2:
		masterdir = args[1]
	elif len(args) == 1:
		if use_latest_master_directory:
			all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)]
			import re; r = re.compile("^master.*$")
			all_dirs = filter(r.match, all_dirs)
			if len(all_dirs)>0:
				# all_dirs = max(all_dirs, key=os.path.getctime)
				masterdir = max(all_dirs, key=os.path.getmtime)
				
	#Create folder for all results or check if there is one created already
	if(myid == main_node):
		if( masterdir == ""):
			timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime())
			masterdir = "master"+timestring
			cmd = "{} {}".format("mkdir", masterdir)
			cmdexecute(cmd)
		elif not os.path.exists(masterdir):
			# os.path.exists(masterdir) does not exist
			masterdir = args[1]
			cmd = "{} {}".format("mkdir", masterdir)
			cmdexecute(cmd)

		if(args[0][:4] == "bdb:"): filename = args[0][4:]
		else:                      filename = args[0][:-4]
		filename = os.path.basename(filename)
		stack_processed_by_ali2d_base__filename  = "bdb:" + os.path.join(masterdir, filename )
		stack_processed_by_ali2d_base__filename__without_master_dir  = "bdb:" + filename
	if_error_all_processes_quit_program(error_status)

	# send masterdir to all processes
	masterdir = send_string_to_all(masterdir)

	if myid == 0:
		if options.restart_section != "":
			if os.path.exists(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)):
				stored_stack, stored_state = restore_program_stack_and_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE))
				import re
				if "," in options.restart_section:
					parsed_restart_section_option = options.restart_section.split(",")
					stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%parsed_restart_section_option[0], stored_state[-1]["location_in_program"])
					generation_str_format = parsed_restart_section_option[1]
					if generation_str_format != "":
						isac_generation_from_command_line = int(generation_str_format)
						stored_state[-1]["isac_generation"] = isac_generation_from_command_line 
					else:
						isac_generation_from_command_line = 1
						if "isac_generation" in stored_state[-1]:
							del stored_state[-1]["isac_generation"]
				else:
					isac_generation_from_command_line = -1
					stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%options.restart_section, stored_state[-1]["location_in_program"])
					if "isac_generation" in stored_state[-1]:
						del stored_state[-1]["isac_generation"]
				store_program_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE), stored_state, stored_stack)
			else:
				print "Please remove the restart_section option from the command line. The program must be started from the beginning."			
				mpi_finalize()
				sys.exit()
		else:
			isac_generation_from_command_line = -1
	
	program_state_stack(locals(), getframeinfo(currentframe()), os.path.join(masterdir,NAME_OF_JSON_STATE_FILE))	

	stack_processed_by_ali2d_base__filename = send_string_to_all(stack_processed_by_ali2d_base__filename)
	stack_processed_by_ali2d_base__filename__without_master_dir = \
		send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir)

	#  PARAMETERS OF THE PROCEDURE
	if( options.xr == -1 ):
		#  Default values
		target_nx = 76
		target_radius = 29
		target_xr = 1
	else:  #  nx//2
		#  Check below!
		target_xr = options.xr
		target_nx = 76 + target_xr - 1 # subtract one, which is default
		target_radius = 29

	mpi_barrier(MPI_COMM_WORLD)

	# Initialization of stacks
	if(myid == main_node):
		number_of_images_in_stack = EMUtil.get_image_count(command_line_provided_stack_filename)
	else:
		number_of_images_in_stack = 0

	number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node)
	
	nxrsteps = 4
	
	init2dir = os.path.join(masterdir,"2dalignment")

	if(myid == 0):
		import subprocess
		from logger import Logger, BaseLogger_Files
		#  Create output directory
		log2d = Logger(BaseLogger_Files())
		log2d.prefix = os.path.join(init2dir)
		cmd = "mkdir -p "+log2d.prefix
		outcome = subprocess.call(cmd, shell=True)
		log2d.prefix += "/"
		# outcome = subprocess.call("sxheader.py  "+command_line_provided_stack_filename+"   --params=xform.align2d  --zero", shell=True)
	else:
		outcome = 0
		log2d = None

	if(myid == main_node):
		a = get_im(command_line_provided_stack_filename)
		nnxo = a.get_xsize()
	else:
		nnxo = 0
	nnxo = bcast_number_to_all(nnxo, source_node = main_node)

	txrm = (nnxo - 2*(radi+1))//2
	if(txrm < 0):  			ERROR( "ERROR!!   Radius of the structure larger than the window data size permits   %d"%(radi), "sxisac",1, myid)
	if(txrm/nxrsteps>0):
		tss = ""
		txr = ""
		while(txrm/nxrsteps>0):
			tts=txrm/nxrsteps
			tss += "  %d"%tts
			txr += "  %d"%(tts*nxrsteps)
			txrm =txrm//2
	else:
		tss = "1"
		txr = "%d"%txrm

	# section ali2d_base

	#  centering method is set to #7
	params2d, aligned_images = ali2d_base(command_line_provided_stack_filename, init2dir, None, 1, radi, 1, txr, txr, tss, \
				False, 90.0, center_method, 14, options.CTF, 1.0, False, \
				"ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False)

	if( myid == main_node ):
		write_text_row(params2d,os.path.join(init2dir, "initial2Dparams.txt"))
	del params2d
	mpi_barrier(MPI_COMM_WORLD)

	#  We assume the target image size will be target_nx, radius will be 29, and xr = 1.  
	#  Note images can be also padded, in which case shrink_ratio > 1.
	shrink_ratio = float(target_radius)/float(radi)
	nx = aligned_images[0].get_xsize()
	nima = len(aligned_images)
	newx = int(nx*shrink_ratio + 0.5)

	from fundamentals import rot_shift2D, resample
	from utilities import pad, combine_params2
	if(shrink_ratio < 1.0):
		if    newx > target_nx  :
			msk = model_circle(target_radius, target_nx, target_nx)
			for im in xrange(nima):
				#  Here we should use only shifts
				alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
				alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
				aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
				aligned_images[im]  = resample(aligned_images[im], shrink_ratio)
				aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
				p = Util.infomask(aligned_images[im], msk, False)
				aligned_images[im] -= p[0]
				p = Util.infomask(aligned_images[im], msk, True)
				aligned_images[im] /= p[1]
		elif  newx == target_nx :
			msk = model_circle(target_radius, target_nx, target_nx)
			for im in xrange(nima):
				#  Here we should use only shifts
				alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
				alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
				aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
				aligned_images[im]  = resample(aligned_images[im], shrink_ratio)
				p = Util.infomask(aligned_images[im], msk, False)
				aligned_images[im] -= p[0]
				p = Util.infomask(aligned_images[im], msk, True)
				aligned_images[im] /= p[1]
		elif  newx < target_nx  :	
			msk = model_circle(newx//2-2, newx,  newx)
			for im in xrange(nima):
				#  Here we should use only shifts
				alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
				alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
				aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
				aligned_images[im]  = resample(aligned_images[im], shrink_ratio)
				p = Util.infomask(aligned_images[im], msk, False)
				aligned_images[im] -= p[0]
				p = Util.infomask(aligned_images[im], msk, True)
				aligned_images[im] /= p[1]
				aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
	elif(shrink_ratio == 1.0):
		if    newx > target_nx  :
			msk = model_circle(target_radius, target_nx, target_nx)
			for im in xrange(nima):
				#  Here we should use only shifts
				alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
				alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
				aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
				aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
				p = Util.infomask(aligned_images[im], msk, False)
				aligned_images[im] -= p[0]
				p = Util.infomask(aligned_images[im], msk, True)
				aligned_images[im] /= p[1]
		elif  newx == target_nx :
			msk = model_circle(target_radius, target_nx, target_nx)
			for im in xrange(nima):
				#  Here we should use only shifts
				alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
				alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
				aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
				p = Util.infomask(aligned_images[im], msk, False)
				aligned_images[im] -= p[0]
				p = Util.infomask(aligned_images[im], msk, True)
				aligned_images[im] /= p[1]
		elif  newx < target_nx  :			
			msk = model_circle(nx//2-2, newx,  newx)
			for im in xrange(nima):
				#  Here we should use only shifts
				alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
				alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
				aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
				#aligned_images[im]  = resample(aligned_images[im], shrink_ratio)
				p = Util.infomask(aligned_images[im], msk, False)
				aligned_images[im] -= p[0]
				p = Util.infomask(aligned_images[im], msk, True)
				aligned_images[im] /= p[1]
				aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
	elif(shrink_ratio > 1.0):
		target_radius = radi
		msk = model_circle(target_radius, nx, nx)
		for im in xrange(nima):
			#  Here we should use only shifts
			alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
			alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
			aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
			p = Util.infomask(aligned_images[im], msk, False)
			aligned_images[im] -= p[0]
			p = Util.infomask(aligned_images[im], msk, True)
			aligned_images[im] /= p[1]
			aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
	del msk

	gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images, myid)
	number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node)

	if( myid == main_node ):
		for i in range(number_of_images_in_stack):  aligned_images[i].write_image(stack_processed_by_ali2d_base__filename,i)
		#  It has to be explicitly closed
		from EMAN2db import db_open_dict
		DB = db_open_dict(stack_processed_by_ali2d_base__filename)
		DB.close()
		
		fp = open("README_shrink_ratio.txt", "w")
		output_text = """
		Since, for processing purposes, isac changes the image dimensions,
		adjustment of pixel size needs to be made in subsequent steps, (e.g.
		running sxviper.py). The shrink ratio for this particular isac run is
		--------
		%.5f
		--------
		To get the pixel size for the isac output the user needs to divide
		the original pixel size by the above value. This info is saved in
		the following file: README_shrink_ratio.txt
		"""%shrink_ratio
		fp.write(output_text); fp.flush() ;fp.close()
		print output_text

	mpi_barrier(MPI_COMM_WORLD)

	global_def.BATCH = True

	os.chdir(masterdir)

	if program_state_stack(locals(), getframeinfo(currentframe())):
	# if 1:
		pass
		if (myid == main_node):
			cmdexecute("sxheader.py  --consecutive  --params=originalid   %s"%stack_processed_by_ali2d_base__filename__without_master_dir)
			cmdexecute("e2bdb.py %s --makevstack=%s_000"%(stack_processed_by_ali2d_base__filename__without_master_dir, stack_processed_by_ali2d_base__filename__without_master_dir))

	if (myid == main_node):
		main_dir_no = get_latest_directory_increment_value("./", NAME_OF_MAIN_DIR, myformat="%04d")
		print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no
		if isac_generation_from_command_line < 0:
			if os.path.exists(NAME_OF_JSON_STATE_FILE):
				stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE)
				if "isac_generation" in stored_state[-1]:
					isac_generation_from_command_line = stored_state[-1]["isac_generation"]
				else:
					isac_generation_from_command_line = -1
		if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no: 
			for i in xrange(isac_generation_from_command_line+1, main_dir_no + 1):
				if i == isac_generation_from_command_line+1:
					backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
					cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
				cmdexecute("mv  " + NAME_OF_MAIN_DIR + "%04d"%i +  " 000_backup" + "%05d"%backup_dir_no)
				cmdexecute("rm  " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%i)
				
			# it includes both command line and json file
			my_restart_section = stored_state[-1]["location_in_program"].split("___")[-1]
			if "restart" in my_restart_section:
				if "backup_dir_no" not in locals():
					backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
					cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
				cmdexecute("mv  " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line +  " 000_backup" + "%05d"%backup_dir_no)
				cmdexecute("rm  " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%isac_generation_from_command_line )
			elif "candidate_class_averages" in my_restart_section:
				if "backup_dir_no" not in locals():
					backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
					cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
				cmdexecute("mv  " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line +  " 000_backup" + "%05d"%backup_dir_no)
				cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line)
				# cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line)
			elif "reproducible_class_averages" in my_restart_section:
				cmdexecute("rm -rf " + NAME_OF_MAIN_DIR + "%04d/ali_params_generation_*"%isac_generation_from_command_line)
				cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_generation*"%isac_generation_from_command_line)
		else:
			if os.path.exists(NAME_OF_JSON_STATE_FILE):
				stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE)
				if "isac_generation" in stored_state[-1]:
					isac_generation_from_command_line = stored_state[-1]["isac_generation"]
				else:
					isac_generation_from_command_line = 1
			else:
				isac_generation_from_command_line = 1
	else:
		isac_generation_from_command_line = 0
		
		
		
	isac_generation_from_command_line = mpi_bcast(isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]
	isac_generation = isac_generation_from_command_line - 1
	
	if (myid == main_node):
		if isac_generation == 0:
			cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation)
			write_text_file([1], os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_accounted.txt"%isac_generation))
			write_text_file(range(number_of_images_in_stack), os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_unaccounted.txt"%isac_generation))

	#  Stopping criterion should be inside the program.
	while True:
		isac_generation += 1
		if isac_generation > options.n_generations:
			break

		data64_stack_current = "bdb:../"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d"%isac_generation

		if(myid == main_node):
			accounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1),"generation_%d_accounted.txt"%(isac_generation - 1)))
			number_of_accounted_images = len(accounted_images)
			# unaccounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1),"generation_%d_unaccounted.txt"%(isac_generation - 1)))
			# number_of_unaccounted_images = len(unaccounted_images)
		else:
			number_of_accounted_images = 0

		number_of_accounted_images = int(mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
		if number_of_accounted_images == 0:
			os.chdir("..")
			break

		program_state_stack.restart_location_title = "restart"
		if program_state_stack(locals(), getframeinfo(currentframe())):
			if (myid == main_node):
				cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation)
				# reference the original stack
				list_file = os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1), "generation_%d_unaccounted.txt"%(isac_generation - 1))
				cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\
						stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file))
			mpi_barrier(MPI_COMM_WORLD)

		os.chdir(NAME_OF_MAIN_DIR + "%04d"%isac_generation)

		program_state_stack.restart_location_title = "candidate_class_averages"
		if program_state_stack(locals(), getframeinfo(currentframe())):

			iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\
				options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
				options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
				options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new)

		# program_state_stack.restart_location_title = "stopped_program1"
		# program_state_stack(locals(), getframeinfo(currentframe()))
		
		program_state_stack.restart_location_title = "stop_after_candidates"
		program_state_stack(locals(), getframeinfo(currentframe()))
		if stop_after_candidates:
			mpi_finalize()
			sys.exit()

		exit_program = 0
		if(myid == main_node):
			if not os.path.exists("class_averages_candidate_generation_%d.hdf"%isac_generation):
				print "This generation (%d) no class averages were generated!"%isac_generation
				exit_program = 1
		exit_program = int(mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
		if exit_program:
			os.chdir("..")
			break

		program_state_stack.restart_location_title = "reproducible_class_averages"
		if program_state_stack(locals(), getframeinfo(currentframe())):


			iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\
				options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
				options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
				options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new)
			pass

		os.chdir("..")

		if (myid == main_node):
			cmdexecute("rm -f class_averages.hdf")
			cpy(["generation_%04d/class_averages_generation_%d.hdf"%(i,i) for i in xrange(1, isac_generation)], "class_averages.hdf")

		# program_state_stack.restart_location_title = "stopped_program2"
		# program_state_stack(locals(), getframeinfo(currentframe()))

	program_state_stack(locals(), getframeinfo(currentframe()), last_call="__LastCall")


	mpi_finalize()
Esempio n. 15
0
def main(args):
    progname = os.path.basename(sys.argv[0])
    usage = (
        progname +
        " stack_file  output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter"
        +
        " --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp"
        + "  --generation=generation  --rand_seed=rand_seed>")

    parser = OptionParser(usage, version=SPARXVERSION)
    parser.add_option(
        "--radius",
        type="int",
        help=
        "particle radius: there is no default, a sensible number has to be provided, units - pixels (default required int)"
    )
    parser.add_option(
        "--target_radius",
        type="int",
        default=29,
        help=
        "target particle radius: actual particle radius on which isac will process data. Images will be shrinked/enlarged to achieve this radius (default 29)"
    )
    parser.add_option(
        "--target_nx",
        type="int",
        default=76,
        help=
        "target particle image size: actual image size on which isac will process data. Images will be shrinked/enlarged according to target particle radius and then cut/padded to achieve target_nx size. When xr > 0, the final image size for isac processing is 'target_nx + xr - 1'  (default 76)"
    )
    parser.add_option(
        "--img_per_grp",
        type="int",
        default=100,
        help=
        "number of images per class: in the ideal case (essentially maximum size of class) (default 100)"
    )
    parser.add_option(
        "--CTF",
        action="store_true",
        default=False,
        help=
        "apply phase-flip for CTF correction: if set the data will be phase-flipped using CTF information included in image headers (default False)"
    )
    parser.add_option(
        "--ir",
        type="int",
        default=1,
        help=
        "inner ring: of the resampling to polar coordinates. units - pixels (default 1)"
    )
    parser.add_option(
        "--rs",
        type="int",
        default=1,
        help=
        "ring step: of the resampling to polar coordinates. units - pixels (default 1)"
    )
    parser.add_option(
        "--xr",
        type="int",
        default=1,
        help=
        "x range: of translational search. By default, set by the program. (default 1)"
    )
    parser.add_option(
        "--yr",
        type="int",
        default=-1,
        help=
        "y range: of translational search. By default, same as xr. (default -1)"
    )
    parser.add_option(
        "--ts",
        type="float",
        default=1.0,
        help=
        "search step: of translational search: units - pixels (default 1.0)")
    parser.add_option(
        "--maxit",
        type="int",
        default=30,
        help="number of iterations for reference-free alignment: (default 30)")
    #parser.add_option("--snr",            type="float",        default=1.0,     help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)")
    parser.add_option(
        "--center_method",
        type="int",
        default=-1,
        help=
        "method for centering: of global 2D average during initial prealignment of data (0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7) (default -1)"
    )
    parser.add_option(
        "--dst",
        type="float",
        default=90.0,
        help="discrete angle used in within group alignment: (default 90.0)")
    parser.add_option(
        "--FL",
        type="float",
        default=0.2,
        help=
        "lowest stopband: frequency used in the tangent filter (default 0.2)")
    parser.add_option(
        "--FH",
        type="float",
        default=0.3,
        help=
        "highest stopband: frequency used in the tangent filter (default 0.3)")
    parser.add_option("--FF",
                      type="float",
                      default=0.2,
                      help="fall-off of the tangent filter: (default 0.2)")
    parser.add_option(
        "--init_iter",
        type="int",
        default=3,
        help=
        "SAC initialization iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC initialization (default 3)"
    )
    parser.add_option(
        "--main_iter",
        type="int",
        default=3,
        help=
        "SAC main iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC (default 3)"
    )
    parser.add_option(
        "--iter_reali",
        type="int",
        default=1,
        help=
        "SAC stability check interval: every iter_reali iterations of SAC stability checking is performed (default 1)"
    )
    parser.add_option(
        "--match_first",
        type="int",
        default=1,
        help=
        "number of iterations to run 2-way matching in the first phase: (default 1)"
    )
    parser.add_option(
        "--max_round",
        type="int",
        default=20,
        help=
        "maximum rounds: of generating candidate class averages in the first phase (default 20)"
    )
    parser.add_option(
        "--match_second",
        type="int",
        default=5,
        help=
        "number of iterations to run 2-way (or 3-way) matching in the second phase: (default 5)"
    )
    parser.add_option(
        "--stab_ali",
        type="int",
        default=5,
        help="number of alignments when checking stability: (default 5)")
    parser.add_option(
        "--thld_err",
        type="float",
        default=0.7,
        help=
        "threshold of pixel error when checking stability: equals root mean square of distances between corresponding pixels from set of found transformations and theirs average transformation, depends linearly on square of radius (parameter ou). units - pixels. (default 0.7)"
    )
    parser.add_option(
        "--indep_run",
        type="int",
        default=4,
        help=
        "level of m-way matching for reproducibility tests: By default, perform full ISAC to 4-way matching. Value indep_run=2 will restrict ISAC to 2-way matching and 3 to 3-way matching.  Note the number of used MPI processes requested in mpirun must be a multiplicity of indep_run. (default 4)"
    )
    parser.add_option("--thld_grp",
                      type="int",
                      default=10,
                      help="minimum size of reproducible class (default 10)")
    parser.add_option(
        "--n_generations",
        type="int",
        default=10,
        help=
        "maximum number of generations: program stops when reaching this total number of generations: (default 10)"
    )
    #parser.add_option("--candidatesexist",action="store_true", default=False,   help="Candidate class averages exist use them (default False)")
    parser.add_option(
        "--rand_seed",
        type="int",
        help=
        "random seed set before calculations: useful for testing purposes. By default, total randomness (type int)"
    )
    parser.add_option("--new",
                      action="store_true",
                      default=False,
                      help="use new code: (default False)")
    parser.add_option("--debug",
                      action="store_true",
                      default=False,
                      help="debug info printout: (default False)")

    # must be switched off in production
    parser.add_option(
        "--use_latest_master_directory",
        action="store_true",
        default=False,
        help=
        "use latest master directory: when active, the program looks for the latest directory that starts with the word 'master', so the user does not need to provide a directory name. (default False)"
    )

    parser.add_option(
        "--restart_section",
        type="string",
        default=' ',
        help=
        "restart section: each generation (iteration) contains three sections: 'restart', 'candidate_class_averages', and 'reproducible_class_averages'. To restart from a particular step, for example, generation 4 and section 'candidate_class_averages' the following option is needed: '--restart_section=candidate_class_averages,4'. The option requires no white space before or after the comma. The default behavior is to restart execution from where it stopped intentionally or unintentionally. For default restart, it is assumed that the name of the directory is provided as argument. Alternatively, the '--use_latest_master_directory' option can be used. (default ' ')"
    )
    parser.add_option(
        "--stop_after_candidates",
        action="store_true",
        default=False,
        help=
        "stop after candidates: stops after the 'candidate_class_averages' section. (default False)"
    )

    ##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
    parser.add_option("--return_options",
                      action="store_true",
                      dest="return_options",
                      default=False,
                      help=SUPPRESS_HELP)
    parser.add_option(
        "--skip_prealignment",
        action="store_true",
        default=False,
        help=
        "skip pre-alignment step: to be used if images are already centered. 2dalignment directory will still be generated but the parameters will be zero. (default False)"
    )

    required_option_list = ['radius']
    (options, args) = parser.parse_args(args)

    if options.return_options:
        return parser

    if len(args) > 2:
        print "usage: " + usage
        print "Please run '" + progname + " -h' for detailed options"
        sys.exit()

    if global_def.CACHE_DISABLE:
        from utilities import disable_bdb_cache
        disable_bdb_cache()
    global_def.BATCH = True

    from isac import iter_isac
    from fundamentals import rot_shift2D, resample
    from utilities import pad, combine_params2

    command_line_provided_stack_filename = args[0]

    main_node = 0
    mpi_init(0, [])
    myid = mpi_comm_rank(MPI_COMM_WORLD)
    nproc = mpi_comm_size(MPI_COMM_WORLD)

    mpi_barrier(MPI_COMM_WORLD)
    if (myid == main_node):
        print "****************************************************************"
        Util.version()
        print "****************************************************************"
        sys.stdout.flush()
    mpi_barrier(MPI_COMM_WORLD)

    # Making sure all required options appeared.
    for required_option in required_option_list:
        if not options.__dict__[required_option]:
            print "\n ==%s== mandatory option is missing.\n" % required_option
            print "Please run '" + progname + " -h' for detailed options"
            return 1

    radi = options.radius
    target_radius = options.target_radius
    target_nx = options.target_nx
    center_method = options.center_method
    if (radi < 1):
        ERROR("Particle radius has to be provided!", "sxisac", 1, myid)

    use_latest_master_directory = options.use_latest_master_directory
    stop_after_candidates = options.stop_after_candidates
    # program_state_stack.restart_location_title_from_command_line = options.restart_section

    from utilities import qw
    program_state_stack.PROGRAM_STATE_VARIABLES = set(
        qw("""
		isac_generation
	"""))

    # create or reuse master directory
    masterdir = ""
    stack_processed_by_ali2d_base__filename = ""
    stack_processed_by_ali2d_base__filename__without_master_dir = ""
    error_status = 0
    if len(args) == 2:
        masterdir = args[1]
    elif len(args) == 1:
        if use_latest_master_directory:
            all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)]
            import re
            r = re.compile("^master.*$")
            all_dirs = filter(r.match, all_dirs)
            if len(all_dirs) > 0:
                # all_dirs = max(all_dirs, key=os.path.getctime)
                masterdir = max(all_dirs, key=os.path.getmtime)

    #Create folder for all results or check if there is one created already
    if (myid == main_node):
        if (masterdir == ""):
            timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM,
                                  localtime())
            masterdir = "master" + timestring
            cmd = "{} {}".format("mkdir", masterdir)
            junk = cmdexecute(cmd)
        elif not os.path.exists(masterdir):
            # os.path.exists(masterdir) does not exist
            masterdir = args[1]
            cmd = "{} {}".format("mkdir", masterdir)
            junk = cmdexecute(cmd)

        if (args[0][:4] == "bdb:"): filename = args[0][4:]
        else: filename = args[0][:-4]
        filename = os.path.basename(filename)
        stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join(
            masterdir, filename)
        stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename
    if_error_then_all_processes_exit_program(error_status)

    # send masterdir to all processes
    masterdir = send_string_to_all(masterdir)

    if myid == 0:
        if options.restart_section != " ":
            if os.path.exists(os.path.join(masterdir,
                                           NAME_OF_JSON_STATE_FILE)):
                stored_stack, stored_state = restore_program_stack_and_state(
                    os.path.join(masterdir, NAME_OF_JSON_STATE_FILE))
                import re
                if "," in options.restart_section:
                    parsed_restart_section_option = options.restart_section.split(
                        ",")
                    stored_state[-1]["location_in_program"] = re.sub(
                        r"___.*$", "___%s" % parsed_restart_section_option[0],
                        stored_state[-1]["location_in_program"])
                    generation_str_format = parsed_restart_section_option[1]
                    if generation_str_format != "":
                        isac_generation_from_command_line = int(
                            generation_str_format)
                        stored_state[-1][
                            "isac_generation"] = isac_generation_from_command_line
                    else:
                        isac_generation_from_command_line = 1
                        if "isac_generation" in stored_state[-1]:
                            del stored_state[-1]["isac_generation"]
                else:
                    isac_generation_from_command_line = -1
                    stored_state[-1]["location_in_program"] = re.sub(
                        r"___.*$", "___%s" % options.restart_section,
                        stored_state[-1]["location_in_program"])
                    if "isac_generation" in stored_state[-1]:
                        del stored_state[-1]["isac_generation"]
                store_program_state(
                    os.path.join(masterdir, NAME_OF_JSON_STATE_FILE),
                    stored_state, stored_stack)
            else:
                print "Please remove the restart_section option from the command line. The program must be started from the beginning."
                mpi_finalize()
                sys.exit()
        else:
            isac_generation_from_command_line = -1

    program_state_stack(locals(), getframeinfo(currentframe()),
                        os.path.join(masterdir, NAME_OF_JSON_STATE_FILE))

    stack_processed_by_ali2d_base__filename = send_string_to_all(
        stack_processed_by_ali2d_base__filename)
    stack_processed_by_ali2d_base__filename__without_master_dir = \
     send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir)

    # previous code 2016-05-05--20-14-12-153
    # #  PARAMETERS OF THE PROCEDURE
    # if( options.xr == -1 ):
    # 	#  Default values
    # 	# target_nx = 76
    # 	# target_radius = 29
    # 	target_xr = 1
    # else:  #  nx//2
    # 	#  Check below!
    # 	target_xr = options.xr
    # 	# target_nx = 76 + target_xr - 1 # subtract one, which is default
    # 	target_nx += target_xr - 1 # subtract one, which is default
    # 	# target_radius = 29

    target_xr = options.xr
    target_nx += target_xr - 1  # subtract one, which is default

    if (options.yr == -1):
        yr = options.xr
    else:
        yr = options.yr

    mpi_barrier(MPI_COMM_WORLD)

    # Initialization of stacks
    if (myid == main_node):
        print "command_line_provided_stack_filename", command_line_provided_stack_filename
        number_of_images_in_stack = EMUtil.get_image_count(
            command_line_provided_stack_filename)
    else:
        number_of_images_in_stack = 0

    number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack,
                                                    source_node=main_node)

    nxrsteps = 4

    init2dir = os.path.join(masterdir, "2dalignment")

    # from mpi import mpi_finalize
    # mpi_finalize()
    # sys.stdout.flush()
    # sys.exit()

    if not os.path.exists(
            os.path.join(init2dir, "Finished_initial_2d_alignment.txt")):

        if (myid == 0):
            import subprocess
            from logger import Logger, BaseLogger_Files
            #  Create output directory
            log2d = Logger(BaseLogger_Files())
            log2d.prefix = os.path.join(init2dir)
            cmd = "mkdir -p " + log2d.prefix
            outcome = subprocess.call(cmd, shell=True)
            log2d.prefix += "/"
            # outcome = subprocess.call("sxheader.py  "+command_line_provided_stack_filename+"   --params=xform.align2d  --zero", shell=True)
        else:
            outcome = 0
            log2d = None

        if (myid == main_node):
            a = get_im(command_line_provided_stack_filename)
            nnxo = a.get_xsize()
        else:
            nnxo = 0
        nnxo = bcast_number_to_all(nnxo, source_node=main_node)

        image_start, image_end = MPI_start_end(number_of_images_in_stack,
                                               nproc, myid)

        if options.skip_prealignment:
            params2d = [[0.0, 0.0, 0.0, 0]
                        for i in xrange(image_start, image_end)]
        else:

            original_images = EMData.read_images(
                command_line_provided_stack_filename,
                range(image_start, image_end))
            #  We assume the target radius will be 29, and xr = 1.
            shrink_ratio = float(target_radius) / float(radi)

            for im in xrange(len(original_images)):
                if (shrink_ratio != 1.0):
                    original_images[im] = resample(original_images[im],
                                                   shrink_ratio)

            nx = original_images[0].get_xsize()
            # nx = int(nx*shrink_ratio + 0.5)

            txrm = (nx - 2 * (target_radius + 1)) // 2
            if (txrm < 0):
                ERROR(
                    "ERROR!!   Radius of the structure larger than the window data size permits   %d"
                    % (radi), "sxisac", 1, myid)
            if (txrm / nxrsteps > 0):
                tss = ""
                txr = ""
                while (txrm / nxrsteps > 0):
                    tts = txrm / nxrsteps
                    tss += "  %d" % tts
                    txr += "  %d" % (tts * nxrsteps)
                    txrm = txrm // 2
            else:
                tss = "1"
                txr = "%d" % txrm

            # print "nx, txr, txrm, tss", nx, txr, txrm, tss
        # from mpi import mpi_finalize
        # mpi_finalize()
        # sys.stdout.flush()
        # sys.exit()

        # section ali2d_base

            params2d = ali2d_base(original_images, init2dir, None, 1, target_radius, 1, txr, txr, tss, \
             False, 90.0, center_method, 14, options.CTF, 1.0, False, \
             "ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False)

            del original_images

            for i in xrange(len(params2d)):
                alpha, sx, sy, mirror = combine_params2(
                    0, params2d[i][1], params2d[i][2], 0, -params2d[i][0], 0,
                    0, 0)
                sx /= shrink_ratio
                sy /= shrink_ratio
                params2d[i][0] = 0.0
                params2d[i][1] = sx
                params2d[i][2] = sy
                params2d[i][3] = 0
                #set_params2D(aligned_images[i],[0.0, sx,sy,0.,1.0])

        mpi_barrier(MPI_COMM_WORLD)
        tmp = params2d[:]
        tmp = wrap_mpi_gatherv(tmp, main_node, MPI_COMM_WORLD)
        if (myid == main_node):
            if options.skip_prealignment:
                print "========================================="
                print "Even though there is no alignment step, '%s' params are set to zero for later use." % os.path.join(
                    init2dir, "initial2Dparams.txt")
                print "========================================="
            write_text_row(tmp, os.path.join(init2dir, "initial2Dparams.txt"))
        del tmp
        mpi_barrier(MPI_COMM_WORLD)

        #  We assume the target image size will be target_nx, radius will be 29, and xr = 1.
        #  Note images can be also padded, in which case shrink_ratio > 1.
        shrink_ratio = float(target_radius) / float(radi)

        aligned_images = EMData.read_images(
            command_line_provided_stack_filename,
            range(image_start, image_end))
        nx = aligned_images[0].get_xsize()
        nima = len(aligned_images)
        newx = int(nx * shrink_ratio + 0.5)

        while not os.path.exists(os.path.join(init2dir,
                                              "initial2Dparams.txt")):
            import time
            time.sleep(1)
        mpi_barrier(MPI_COMM_WORLD)

        params = read_text_row(os.path.join(init2dir, "initial2Dparams.txt"))
        params = params[image_start:image_end]

        msk = model_circle(radi, nx, nx)
        for im in xrange(nima):
            st = Util.infomask(aligned_images[im], msk, False)
            aligned_images[im] -= st[0]
            if options.CTF:
                aligned_images[im] = filt_ctf(
                    aligned_images[im],
                    aligned_images[im].get_attr("ctf"),
                    binary=True)

        if (shrink_ratio < 1.0):
            if newx > target_nx:
                msk = model_circle(target_radius, target_nx, target_nx)
                for im in xrange(nima):
                    #  Here we should use only shifts
                    #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
                    #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
                    #aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
                    aligned_images[im] = rot_shift2D(aligned_images[im], 0,
                                                     params[im][1],
                                                     params[im][2], 0)
                    aligned_images[im] = resample(aligned_images[im],
                                                  shrink_ratio)
                    aligned_images[im] = Util.window(aligned_images[im],
                                                     target_nx, target_nx, 1)
                    p = Util.infomask(aligned_images[im], msk, False)
                    aligned_images[im] -= p[0]
                    p = Util.infomask(aligned_images[im], msk, True)
                    aligned_images[im] /= p[1]
            elif newx == target_nx:
                msk = model_circle(target_radius, target_nx, target_nx)
                for im in xrange(nima):
                    #  Here we should use only shifts
                    #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
                    #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
                    aligned_images[im] = rot_shift2D(aligned_images[im], 0,
                                                     params[im][1],
                                                     params[im][2], 0)
                    aligned_images[im] = resample(aligned_images[im],
                                                  shrink_ratio)
                    p = Util.infomask(aligned_images[im], msk, False)
                    aligned_images[im] -= p[0]
                    p = Util.infomask(aligned_images[im], msk, True)
                    aligned_images[im] /= p[1]
            elif newx < target_nx:
                msk = model_circle(newx // 2 - 2, newx, newx)
                for im in xrange(nima):
                    #  Here we should use only shifts
                    #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
                    #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
                    aligned_images[im] = rot_shift2D(aligned_images[im], 0,
                                                     params[im][1],
                                                     params[im][2], 0)
                    aligned_images[im] = resample(aligned_images[im],
                                                  shrink_ratio)
                    p = Util.infomask(aligned_images[im], msk, False)
                    aligned_images[im] -= p[0]
                    p = Util.infomask(aligned_images[im], msk, True)
                    aligned_images[im] /= p[1]
                    aligned_images[im] = pad(aligned_images[im], target_nx,
                                             target_nx, 1, 0.0)
        elif (shrink_ratio == 1.0):
            if newx > target_nx:
                msk = model_circle(target_radius, target_nx, target_nx)
                for im in xrange(nima):
                    #  Here we should use only shifts
                    #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
                    #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
                    aligned_images[im] = rot_shift2D(aligned_images[im], 0,
                                                     params[im][1],
                                                     params[im][2], 0)
                    aligned_images[im] = Util.window(aligned_images[im],
                                                     target_nx, target_nx, 1)
                    p = Util.infomask(aligned_images[im], msk, False)
                    aligned_images[im] -= p[0]
                    p = Util.infomask(aligned_images[im], msk, True)
                    aligned_images[im] /= p[1]
            elif newx == target_nx:
                msk = model_circle(target_radius, target_nx, target_nx)
                for im in xrange(nima):
                    #  Here we should use only shifts
                    #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
                    #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
                    aligned_images[im] = rot_shift2D(aligned_images[im], 0,
                                                     params[im][1],
                                                     params[im][2], 0)
                    p = Util.infomask(aligned_images[im], msk, False)
                    aligned_images[im] -= p[0]
                    p = Util.infomask(aligned_images[im], msk, True)
                    aligned_images[im] /= p[1]
            elif newx < target_nx:
                msk = model_circle(newx // 2 - 2, newx, newx)
                for im in xrange(nima):
                    #  Here we should use only shifts
                    #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
                    #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
                    aligned_images[im] = rot_shift2D(aligned_images[im], 0,
                                                     params[im][1],
                                                     params[im][2], 0)
                    #aligned_images[im]  = resample(aligned_images[im], shrink_ratio)
                    p = Util.infomask(aligned_images[im], msk, False)
                    aligned_images[im] -= p[0]
                    p = Util.infomask(aligned_images[im], msk, True)
                    aligned_images[im] /= p[1]
                    aligned_images[im] = pad(aligned_images[im], target_nx,
                                             target_nx, 1, 0.0)
        elif (shrink_ratio > 1.0):
            if newx > target_nx:
                msk = model_circle(target_radius, target_nx, target_nx)
                for im in xrange(nima):
                    #  Here we should use only shifts
                    #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
                    #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
                    aligned_images[im] = rot_shift2D(aligned_images[im], 0,
                                                     params[im][1],
                                                     params[im][2], 0)
                    aligned_images[im] = resample(aligned_images[im],
                                                  shrink_ratio)
                    aligned_images[im] = Util.window(aligned_images[im],
                                                     target_nx, target_nx, 1)
                    p = Util.infomask(aligned_images[im], msk, False)
                    aligned_images[im] -= p[0]
                    p = Util.infomask(aligned_images[im], msk, True)
                    aligned_images[im] /= p[1]
            elif newx == target_nx:
                msk = model_circle(target_radius, target_nx, target_nx)
                for im in xrange(nima):
                    #  Here we should use only shifts
                    #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
                    #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
                    aligned_images[im] = rot_shift2D(aligned_images[im], 0,
                                                     params[im][1],
                                                     params[im][2], 0)
                    aligned_images[im] = resample(aligned_images[im],
                                                  shrink_ratio)
                    p = Util.infomask(aligned_images[im], msk, False)
                    aligned_images[im] -= p[0]
                    p = Util.infomask(aligned_images[im], msk, True)
                    aligned_images[im] /= p[1]
            elif newx < target_nx:
                msk = model_circle(newx // 2 - 2, newx, newx)
                for im in xrange(nima):
                    #  Here we should use only shifts
                    #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
                    #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
                    aligned_images[im] = rot_shift2D(aligned_images[im], 0,
                                                     params[im][1],
                                                     params[im][2], 0)
                    aligned_images[im] = resample(aligned_images[im],
                                                  shrink_ratio)
                    p = Util.infomask(aligned_images[im], msk, False)
                    aligned_images[im] -= p[0]
                    p = Util.infomask(aligned_images[im], msk, True)
                    aligned_images[im] /= p[1]
                    aligned_images[im] = pad(aligned_images[im], target_nx,
                                             target_nx, 1, 0.0)
        del msk

        gather_compacted_EMData_to_root(number_of_images_in_stack,
                                        aligned_images, myid)
        number_of_images_in_stack = bcast_number_to_all(
            number_of_images_in_stack, source_node=main_node)

        if (myid == main_node):
            for i in range(number_of_images_in_stack):
                aligned_images[i].write_image(
                    stack_processed_by_ali2d_base__filename, i)
            #  It has to be explicitly closed
            from EMAN2db import db_open_dict
            DB = db_open_dict(stack_processed_by_ali2d_base__filename)
            DB.close()

            fp = open(os.path.join(masterdir, "README_shrink_ratio.txt"), "w")
            output_text = """
			Since, for processing purposes, isac changes the image dimensions,
			adjustment of pixel size needs to be made in subsequent steps, (e.g.
			running sxviper.py). The shrink ratio for this particular isac run is
			--------
			%.5f
			%.5f
			--------
			To get the pixel size for the isac output the user needs to divide
			the original pixel size by the above value. This info is saved in
			the following file: README_shrink_ratio.txt
			""" % (shrink_ratio, radi)
            fp.write(output_text)
            fp.flush()
            fp.close()
            print output_text
            fp = open(
                os.path.join(init2dir, "Finished_initial_2d_alignment.txt"),
                "w")
            fp.flush()
            fp.close()
    else:
        if (myid == main_node):
            print "Skipping 2d alignment since it was already done!"

    mpi_barrier(MPI_COMM_WORLD)

    # from mpi import mpi_finalize
    # mpi_finalize()
    # sys.stdout.flush()
    # sys.exit()

    os.chdir(masterdir)

    if program_state_stack(locals(), getframeinfo(currentframe())):
        # if 1:
        pass
        if (myid == main_node):
            junk = cmdexecute(
                "sxheader.py  --consecutive  --params=originalid   %s" %
                stack_processed_by_ali2d_base__filename__without_master_dir)
            junk = cmdexecute(
                "e2bdb.py %s --makevstack=%s_000" %
                (stack_processed_by_ali2d_base__filename__without_master_dir,
                 stack_processed_by_ali2d_base__filename__without_master_dir))

    if (myid == main_node):
        main_dir_no = get_latest_directory_increment_value("./",
                                                           NAME_OF_MAIN_DIR,
                                                           myformat="%04d")
        print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no
        if isac_generation_from_command_line < 0:
            if os.path.exists(NAME_OF_JSON_STATE_FILE):
                stored_stack, stored_state = restore_program_stack_and_state(
                    NAME_OF_JSON_STATE_FILE)
                if "isac_generation" in stored_state[-1]:
                    isac_generation_from_command_line = stored_state[-1][
                        "isac_generation"]
                else:
                    isac_generation_from_command_line = -1
        if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no:
            for i in xrange(isac_generation_from_command_line + 1,
                            main_dir_no + 1):
                if i == isac_generation_from_command_line + 1:
                    backup_dir_no = get_nonexistent_directory_increment_value(
                        "./", "000_backup", myformat="%05d", start_value=1)
                    junk = cmdexecute("mkdir -p " + "000_backup" +
                                      "%05d" % backup_dir_no)
                junk = cmdexecute("mv  " + NAME_OF_MAIN_DIR + "%04d" % i +
                                  " 000_backup" + "%05d" % backup_dir_no)
                junk = cmdexecute(
                    "rm  " + "EMAN2DB/" +
                    stack_processed_by_ali2d_base__filename__without_master_dir[
                        4:] + "_%03d.bdb" % i)

            # it includes both command line and json file
            my_restart_section = stored_state[-1]["location_in_program"].split(
                "___")[-1]
            if "restart" in my_restart_section:
                if "backup_dir_no" not in locals():
                    backup_dir_no = get_nonexistent_directory_increment_value(
                        "./", "000_backup", myformat="%05d", start_value=1)
                    junk = cmdexecute("mkdir -p " + "000_backup" +
                                      "%05d" % backup_dir_no)
                junk = cmdexecute("mv  " + NAME_OF_MAIN_DIR +
                                  "%04d" % isac_generation_from_command_line +
                                  " 000_backup" + "%05d" % backup_dir_no)
                junk = cmdexecute(
                    "rm  " + "EMAN2DB/" +
                    stack_processed_by_ali2d_base__filename__without_master_dir[
                        4:] + "_%03d.bdb" % isac_generation_from_command_line)
            elif "candidate_class_averages" in my_restart_section:
                if "backup_dir_no" not in locals():
                    backup_dir_no = get_nonexistent_directory_increment_value(
                        "./", "000_backup", myformat="%05d", start_value=1)
                    junk = cmdexecute("mkdir -p " + "000_backup" +
                                      "%05d" % backup_dir_no)
                junk = cmdexecute("mv  " + NAME_OF_MAIN_DIR +
                                  "%04d" % isac_generation_from_command_line +
                                  " 000_backup" + "%05d" % backup_dir_no)
                junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
                                  "%04d" % isac_generation_from_command_line)
                # junk = cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line)
            elif "reproducible_class_averages" in my_restart_section:
                junk = cmdexecute("rm -rf " + NAME_OF_MAIN_DIR +
                                  "%04d/ali_params_generation_*" %
                                  isac_generation_from_command_line)
                junk = cmdexecute("rm -f " + NAME_OF_MAIN_DIR +
                                  "%04d/class_averages_generation*" %
                                  isac_generation_from_command_line)
        else:
            if os.path.exists(NAME_OF_JSON_STATE_FILE):
                stored_stack, stored_state = restore_program_stack_and_state(
                    NAME_OF_JSON_STATE_FILE)
                if "isac_generation" in stored_state[-1]:
                    isac_generation_from_command_line = stored_state[-1][
                        "isac_generation"]
                else:
                    isac_generation_from_command_line = 1
            else:
                isac_generation_from_command_line = 1
    else:
        isac_generation_from_command_line = 0

    isac_generation_from_command_line = mpi_bcast(
        isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]
    isac_generation = isac_generation_from_command_line - 1

    if (myid == main_node):
        if isac_generation == 0:
            junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
                              "%04d" % isac_generation)
            write_text_file(
                [1],
                os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation,
                             "generation_%d_accounted.txt" % isac_generation))
            write_text_file(
                range(number_of_images_in_stack),
                os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation,
                             "generation_%d_unaccounted.txt" %
                             isac_generation))

    #  Stopping criterion should be inside the program.
    while True:
        isac_generation += 1
        if isac_generation > options.n_generations:
            break

        data64_stack_current = "bdb:../" + stack_processed_by_ali2d_base__filename__without_master_dir[
            4:] + "_%03d" % isac_generation

        program_state_stack.restart_location_title = "restart"
        if program_state_stack(locals(), getframeinfo(currentframe())):
            if (myid == main_node):
                junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
                                  "%04d" % isac_generation)
                # reference the original stack
                list_file = os.path.join(
                    NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1),
                    "generation_%d_unaccounted.txt" % (isac_generation - 1))
                junk = cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\
                  stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file))
            mpi_barrier(MPI_COMM_WORLD)

        os.chdir(NAME_OF_MAIN_DIR + "%04d" % isac_generation)

        program_state_stack.restart_location_title = "candidate_class_averages"
        if program_state_stack(locals(), getframeinfo(currentframe())):

            iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\
             options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
             options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
             options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new)

        # program_state_stack.restart_location_title = "stopped_program1"
        # program_state_stack(locals(), getframeinfo(currentframe()))

        program_state_stack.restart_location_title = "stop_after_candidates"
        program_state_stack(locals(), getframeinfo(currentframe()))
        if stop_after_candidates:
            mpi_finalize()
            sys.exit()

        exit_program = 0
        if (myid == main_node):
            if not os.path.exists(
                    "class_averages_candidate_generation_%d.hdf" %
                    isac_generation):
                print "This generation (%d) no class average candidates were generated! Finishing." % isac_generation
                exit_program = 1
        exit_program = int(
            mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
        if exit_program:
            os.chdir("..")
            break

        program_state_stack.restart_location_title = "reproducible_class_averages"
        if program_state_stack(locals(), getframeinfo(currentframe())):


            iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\
             options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
             options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
             options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new)
            pass

        os.chdir("..")

        if (myid == main_node):
            accounted_images = read_text_file(
                os.path.join(NAME_OF_MAIN_DIR + "%04d" % (isac_generation),
                             "generation_%d_accounted.txt" %
                             (isac_generation)))
            number_of_accounted_images = len(accounted_images)
            un_accounted_images = read_text_file(
                os.path.join(
                    NAME_OF_MAIN_DIR + "%04d" % (isac_generation),
                    "generation_%d_unaccounted.txt" % (isac_generation)))
            number_of_un_accounted_images = len(un_accounted_images)
        else:
            number_of_accounted_images = 0
            number_of_un_accounted_images = 0

        number_of_accounted_images = int(
            mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0,
                      MPI_COMM_WORLD)[0])
        number_of_un_accounted_images = int(
            mpi_bcast(number_of_un_accounted_images, 1, MPI_INT, 0,
                      MPI_COMM_WORLD)[0])

        if number_of_accounted_images == 0:
            if (myid == main_node):
                print "This generation (%d) there are no accounted images! Finishing." % isac_generation
            break

        while (myid == main_node):

            def files_are_missing(isac_generation):
                for i in xrange(1, isac_generation + 1):
                    if not os.path.exists(
                            "generation_%04d/class_averages_generation_%d.hdf"
                            % (i, i)):
                        print "Error: generation_%04d/class_averages_generation_%d.hdf is missing! Exiting." % (
                            i, i)
                        return 1
                return 0

            if files_are_missing(isac_generation):
                break

            junk = cmdexecute("rm -f class_averages.hdf")
            cpy([
                "generation_%04d/class_averages_generation_%d.hdf" % (i, i)
                for i in xrange(1, isac_generation + 1)
            ], "class_averages.hdf")

            break

        if number_of_un_accounted_images == 0:
            if (myid == main_node):
                print "This generation (%d) there are no un accounted images! Finishing." % isac_generation
            break

    program_state_stack(locals(),
                        getframeinfo(currentframe()),
                        last_call="__LastCall")

    mpi_barrier(MPI_COMM_WORLD)
    mpi_finalize()
Esempio n. 16
0
def main():

	def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
		if mirror:
			m = 1
			alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 540.0-psi, 0, 0, 1.0)
		else:
			m = 0
			alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 360.0-psi, 0, 0, 1.0)
		return  alpha, sx, sy, m
	
	progname = os.path.basename(sys.argv[0])
	usage = progname + " prj_stack  --ave2D= --var2D=  --ave3D= --var3D= --img_per_grp= --fl=0.2 --aa=0.1  --sym=symmetry --CTF"
	parser = OptionParser(usage, version=SPARXVERSION)

	parser.add_option("--ave2D",		type="string"	   ,	default=False,				help="write to the disk a stack of 2D averages")
	parser.add_option("--var2D",		type="string"	   ,	default=False,				help="write to the disk a stack of 2D variances")
	parser.add_option("--ave3D",		type="string"	   ,	default=False,				help="write to the disk reconstructed 3D average")
	parser.add_option("--var3D",		type="string"	   ,	default=False,				help="compute 3D variability (time consuming!)")
	parser.add_option("--img_per_grp",	type="int"         ,	default=10   ,				help="number of neighbouring projections")
	parser.add_option("--no_norm",		action="store_true",	default=False,				help="do not use normalization")
	parser.add_option("--radiusvar", 	type="int"         ,	default=-1   ,				help="radius for 3D var" )
	parser.add_option("--npad",			type="int"         ,	default=2    ,				help="number of time to pad the original images")
	parser.add_option("--sym" , 		type="string"      ,	default="c1" ,				help="symmetry")
	parser.add_option("--fl",			type="float"       ,	default=0.0  ,				help="stop-band frequency (Default - no filtration)")
	parser.add_option("--aa",			type="float"       ,	default=0.0  ,				help="fall off of the filter (Default - no filtration)")
	parser.add_option("--CTF",			action="store_true",	default=False,				help="use CFT correction")
	parser.add_option("--VERBOSE",		action="store_true",	default=False,				help="Long output for debugging")
	#parser.add_option("--MPI" , 		action="store_true",	default=False,				help="use MPI version")
	#parser.add_option("--radiuspca", 	type="int"         ,	default=-1   ,				help="radius for PCA" )
	#parser.add_option("--iter", 		type="int"         ,	default=40   ,				help="maximum number of iterations (stop criterion of reconstruction process)" )
	#parser.add_option("--abs", 			type="float"       ,	default=0.0  ,				help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
	#parser.add_option("--squ", 			type="float"       ,	default=0.0  ,				help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
	parser.add_option("--VAR" , 		action="store_true",	default=False,				help="stack on input consists of 2D variances (Default False)")
	parser.add_option("--decimate",     type="float",           default=1.0,                 help="image decimate rate, a number large than 1. default is 1")
	parser.add_option("--window",       type="int",             default=0,                   help="reduce images to a small image size without changing pixel_size. Default value is zero.")
	#parser.add_option("--SND",			action="store_true",	default=False,				help="compute squared normalized differences (Default False)")
	parser.add_option("--nvec",			type="int"         ,	default=0    ,				help="number of eigenvectors, default = 0 meaning no PCA calculated")
	parser.add_option("--symmetrize",	action="store_true",	default=False,				help="Prepare input stack for handling symmetry (Default False)")
	
	(options,args) = parser.parse_args()
	#####
	from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD, MPI_TAG_UB
	from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
	from applications import MPI_start_end
	from reconstruction import recons3d_em, recons3d_em_MPI
	from reconstruction	import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
	from utilities import print_begin_msg, print_end_msg, print_msg
	from utilities import read_text_row, get_image, get_im
	from utilities import bcast_EMData_to_all, bcast_number_to_all
	from utilities import get_symt

	#  This is code for handling symmetries by the above program.  To be incorporated. PAP 01/27/2015

	from EMAN2db import db_open_dict
	
	if options.symmetrize :
		try:
			sys.argv = mpi_init(len(sys.argv), sys.argv)
			try:	
				number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
				if( number_of_proc > 1 ):
					ERROR("Cannot use more than one CPU for symmetry prepration","sx3dvariability",1)
			except:
				pass
		except:
			pass

		#  Input
		#instack = "Clean_NORM_CTF_start_wparams.hdf"
		#instack = "bdb:data"
		instack = args[0]
		sym = options.sym
		if( sym == "c1" ):
			ERROR("Thre is no need to symmetrize stack for C1 symmetry","sx3dvariability",1)

		if(instack[:4] !="bdb:"):
			stack = "bdb:data"
			delete_bdb(stack)
			cmdexecute("sxcpy.py  "+instack+"  "+stack)
		else:
			stack = instack

		qt = EMUtil.get_all_attributes(stack,'xform.projection')

		na = len(qt)
		ts = get_symt(sym)
		ks = len(ts)
		angsa = [None]*na
		for k in xrange(ks):
			delete_bdb("bdb:Q%1d"%k)
			cmdexecute("e2bdb.py  "+stack+"  --makevstack=bdb:Q%1d"%k)
			DB = db_open_dict("bdb:Q%1d"%k)
			for i in xrange(na):
				ut = qt[i]*ts[k]
				DB.set_attr(i, "xform.projection", ut)
				#bt = ut.get_params("spider")
				#angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
			#write_text_row(angsa, 'ptsma%1d.txt'%k)
			#cmdexecute("e2bdb.py  "+stack+"  --makevstack=bdb:Q%1d"%k)
			#cmdexecute("sxheader.py  bdb:Q%1d  --params=xform.projection  --import=ptsma%1d.txt"%(k,k))
			DB.close()
		delete_bdb("bdb:sdata")
		cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
		#cmdexecute("ls  EMAN2DB/sdata*")
		a = get_im("bdb:sdata")
		a.set_attr("variabilitysymmetry",sym)
		a.write_image("bdb:sdata")


	else:

		sys.argv = mpi_init(len(sys.argv), sys.argv)
		myid     = mpi_comm_rank(MPI_COMM_WORLD)
		number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
		main_node = 0

		if len(args) == 1:
			stack = args[0]
		else:
			print( "usage: " + usage)
			print( "Please run '" + progname + " -h' for detailed options")
			return 1

		t0 = time()
	
		# obsolete flags
		options.MPI = True
		options.nvec = 0
		options.radiuspca = -1
		options.iter = 40
		options.abs = 0.0
		options.squ = 0.0

		if options.fl > 0.0 and options.aa == 0.0:
			ERROR("Fall off has to be given for the low-pass filter", "sx3dvariability", 1, myid)
		if options.VAR and options.SND:
			ERROR("Only one of var and SND can be set!", "sx3dvariability", myid)
			exit()
		if options.VAR and (options.ave2D or options.ave3D or options.var2D): 
			ERROR("When VAR is set, the program cannot output ave2D, ave3D or var2D", "sx3dvariability", 1, myid)
			exit()
		#if options.SND and (options.ave2D or options.ave3D):
		#	ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid)
		#	exit()
		if options.nvec > 0 :
			ERROR("PCA option not implemented", "sx3dvariability", 1, myid)
			exit()
		if options.nvec > 0 and options.ave3D == None:
			ERROR("When doing PCA analysis, one must set ave3D", "sx3dvariability", myid=myid)
			exit()
		import string
		options.sym = options.sym.lower()
		 
		if global_def.CACHE_DISABLE:
			from utilities import disable_bdb_cache
			disable_bdb_cache()
		global_def.BATCH = True

		if myid == main_node:
			print_begin_msg("sx3dvariability")
			print_msg("%-70s:  %s\n"%("Input stack", stack))
	
		img_per_grp = options.img_per_grp
		nvec = options.nvec
		radiuspca = options.radiuspca

		symbaselen = 0
		if myid == main_node:
			nima = EMUtil.get_image_count(stack)
			img  = get_image(stack)
			nx   = img.get_xsize()
			ny   = img.get_ysize()
			if options.sym != "c1" :
				imgdata = get_im(stack)
				try:
					i = imgdata.get_attr("variabilitysymmetry")
					if(i != options.sym):
						ERROR("The symmetry provided does not agree with the symmetry of the input stack", "sx3dvariability", myid=myid)
				except:
					ERROR("Input stack is not prepared for symmetry, please follow instructions", "sx3dvariability", myid=myid)
				from utilities import get_symt
				i = len(get_symt(options.sym))
				if((nima/i)*i != nima):
					ERROR("The length of the input stack is incorrect for symmetry processing", "sx3dvariability", myid=myid)
				symbaselen = nima/i
			else:  symbaselen = nima
		else:
			nima = 0
			nx = 0
			ny = 0
		nima = bcast_number_to_all(nima)
		nx   = bcast_number_to_all(nx)
		ny   = bcast_number_to_all(ny)
		Tracker ={}
		Tracker["nx"]  =nx
		Tracker["ny"]  =ny
		Tracker["total_stack"]=nima
		if options.decimate==1.:
			if options.window !=0:
				nx = options.window
				ny = options.window
		else:
			if options.window ==0:
				nx = int(nx/options.decimate)
				ny = int(ny/options.decimate)
			else:
				nx = int(options.window/options.decimate)
				ny = nx
		symbaselen = bcast_number_to_all(symbaselen)
		if radiuspca == -1: radiuspca = nx/2-2

		if myid == main_node:
			print_msg("%-70s:  %d\n"%("Number of projection", nima))
		
		img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
		"""
		if options.SND:
			from projection		import prep_vol, prgs
			from statistics		import im_diff
			from utilities		import get_im, model_circle, get_params_proj, set_params_proj
			from utilities		import get_ctf, generate_ctf
			from filter			import filt_ctf
		
			imgdata = EMData.read_images(stack, range(img_begin, img_end))

			if options.CTF:
				vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
			else:
				vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)

			bcast_EMData_to_all(vol, myid)
			volft, kb = prep_vol(vol)

			mask = model_circle(nx/2-2, nx, ny)
			varList = []
			for i in xrange(img_begin, img_end):
				phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
				ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
				if options.CTF:
					ctf_params = get_ctf(imgdata[i-img_begin])
					ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
				diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
				diff2 = diff*diff
				set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
				varList.append(diff2)
			mpi_barrier(MPI_COMM_WORLD)
		"""
		if options.VAR:
			#varList = EMData.read_images(stack, range(img_begin, img_end))
			varList = []
			this_image = EMData()
			for index_of_particle in xrange(img_begin,img_end):
				this_image.read_image(stack,index_of_particle)
				varList.append(image_decimate_window_xform_ctf(img,options.decimate,options.window,options.CTF))
		else:
			from utilities		import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
			from utilities		import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
			from utilities		import model_blank, nearest_proj, model_circle
			from applications	import pca
			from statistics		import avgvar, avgvar_ctf, ccc
			from filter		    import filt_tanl
			from morphology		import threshold, square_root
			from projection 	import project, prep_vol, prgs
			from sets		    import Set

			if myid == main_node:
				t1 = time()
				proj_angles = []
				aveList = []
				tab = EMUtil.get_all_attributes(stack, 'xform.projection')
				for i in xrange(nima):
					t     = tab[i].get_params('spider')
					phi   = t['phi']
					theta = t['theta']
					psi   = t['psi']
					x     = theta
					if x > 90.0: x = 180.0 - x
					x = x*10000+psi
					proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
				t2 = time()
				print_msg("%-70s:  %d\n"%("Number of neighboring projections", img_per_grp))
				print_msg("...... Finding neighboring projections\n")
				if options.VERBOSE:
					print "Number of images per group: ", img_per_grp
					print "Now grouping projections"
				proj_angles.sort()

			proj_angles_list = [0.0]*(nima*4)
			if myid == main_node:
				for i in xrange(nima):
					proj_angles_list[i*4]   = proj_angles[i][1]
					proj_angles_list[i*4+1] = proj_angles[i][2]
					proj_angles_list[i*4+2] = proj_angles[i][3]
					proj_angles_list[i*4+3] = proj_angles[i][4]
			proj_angles_list = bcast_list_to_all(proj_angles_list, myid, main_node)
			proj_angles = []
			for i in xrange(nima):
				proj_angles.append([proj_angles_list[i*4], proj_angles_list[i*4+1], proj_angles_list[i*4+2], int(proj_angles_list[i*4+3])])
			del proj_angles_list

			proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp, range(img_begin, img_end))

			all_proj = Set()
			for im in proj_list:
				for jm in im:
					all_proj.add(proj_angles[jm][3])

			all_proj = list(all_proj)
			if options.VERBOSE:
				print "On node %2d, number of images needed to be read = %5d"%(myid, len(all_proj))

			index = {}
			for i in xrange(len(all_proj)): index[all_proj[i]] = i
			mpi_barrier(MPI_COMM_WORLD)

			if myid == main_node:
				print_msg("%-70s:  %.2f\n"%("Finding neighboring projections lasted [s]", time()-t2))
				print_msg("%-70s:  %d\n"%("Number of groups processed on the main node", len(proj_list)))
				if options.VERBOSE:
					print "Grouping projections took: ", (time()-t2)/60	, "[min]"
					print "Number of groups on main node: ", len(proj_list)
			mpi_barrier(MPI_COMM_WORLD)

			if myid == main_node:
				print_msg("...... calculating the stack of 2D variances \n")
				if options.VERBOSE:
					print "Now calculating the stack of 2D variances"

			proj_params = [0.0]*(nima*5)
			aveList = []
			varList = []				
			if nvec > 0:
				eigList = [[] for i in xrange(nvec)]

			if options.VERBOSE: 	print "Begin to read images on processor %d"%(myid)
			ttt = time()
			#imgdata = EMData.read_images(stack, all_proj)
			img     = EMData()
			imgdata = []
			for index_of_proj in xrange(len(all_proj)):
				img.read_image(stack, all_proj[index_of_proj])
				dmg = image_decimate_window_xform_ctf(img,options.decimate,options.window,options.CTF)
				#print dmg.get_xsize(), "init"
				imgdata.append(dmg)
			if options.VERBOSE:
				print "Reading images on processor %d done, time = %.2f"%(myid, time()-ttt)
				print "On processor %d, we got %d images"%(myid, len(imgdata))
			mpi_barrier(MPI_COMM_WORLD)

			'''	
			imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
			if options.fl > 0.0:
				for k in xrange(len(imgdata2)):
					imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
			if options.CTF:
				vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
			else:
				vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
			if myid == main_node:
				vol.write_image("vol_ctf.hdf")
				print_msg("Writing to the disk volume reconstructed from averages as		:  %s\n"%("vol_ctf.hdf"))
			del vol, imgdata2
			mpi_barrier(MPI_COMM_WORLD)
			'''
			from applications import prepare_2d_forPCA
			from utilities import model_blank
			for i in xrange(len(proj_list)):
				ki = proj_angles[proj_list[i][0]][3]
				if ki >= symbaselen:  continue
				mi = index[ki]
				phiM, thetaM, psiM, s2xM, s2yM = get_params_proj(imgdata[mi])

				grp_imgdata = []
				for j in xrange(img_per_grp):
					mj = index[proj_angles[proj_list[i][j]][3]]
					phi, theta, psi, s2x, s2y = get_params_proj(imgdata[mj])
					alpha, sx, sy, mirror = params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror_list[i][j])
					if thetaM <= 90:
						if mirror == 0:  alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, phiM-phi, 0.0, 0.0, 1.0)
						else:            alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, 180-(phiM-phi), 0.0, 0.0, 1.0)
					else:
						if mirror == 0:  alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(phiM-phi), 0.0, 0.0, 1.0)
						else:            alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(180-(phiM-phi)), 0.0, 0.0, 1.0)
					set_params2D(imgdata[mj], [alpha, sx, sy, mirror, 1.0])
					grp_imgdata.append(imgdata[mj])
					#print grp_imgdata[j].get_xsize(), imgdata[mj].get_xsize()

				if not options.no_norm:
					#print grp_imgdata[j].get_xsize()
					mask = model_circle(nx/2-2, nx, nx)
					for k in xrange(img_per_grp):
						ave, std, minn, maxx = Util.infomask(grp_imgdata[k], mask, False)
						grp_imgdata[k] -= ave
						grp_imgdata[k] /= std
					del mask

				if options.fl > 0.0:
					from filter import filt_ctf, filt_table
					from fundamentals import fft, window2d
					nx2 = 2*nx
					ny2 = 2*ny
					if options.CTF:
						from utilities import pad
						for k in xrange(img_per_grp):
							grp_imgdata[k] = window2d(fft( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa) ),nx,ny)
							#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
							#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
					else:
						for k in xrange(img_per_grp):
							grp_imgdata[k] = filt_tanl( grp_imgdata[k], options.fl, options.aa)
							#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
							#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
				else:
					from utilities import pad, read_text_file
					from filter import filt_ctf, filt_table
					from fundamentals import fft, window2d
					nx2 = 2*nx
					ny2 = 2*ny
					if options.CTF:
						from utilities import pad
						for k in xrange(img_per_grp):
							grp_imgdata[k] = window2d( fft( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1) ) , nx,ny)
							#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
							#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)

				'''
				if i < 10 and myid == main_node:
					for k in xrange(10):
						grp_imgdata[k].write_image("grp%03d.hdf"%i, k)
				'''
				"""
				if myid == main_node and i==0:
					for pp in xrange(len(grp_imgdata)):
						grp_imgdata[pp].write_image("pp.hdf", pp)
				"""
				ave, grp_imgdata = prepare_2d_forPCA(grp_imgdata)
				"""
				if myid == main_node and i==0:
					for pp in xrange(len(grp_imgdata)):
						grp_imgdata[pp].write_image("qq.hdf", pp)
				"""

				var = model_blank(nx,ny)
				for q in grp_imgdata:  Util.add_img2( var, q )
				Util.mul_scalar( var, 1.0/(len(grp_imgdata)-1))
				# Switch to std dev
				var = square_root(threshold(var))
				#if options.CTF:	ave, var = avgvar_ctf(grp_imgdata, mode="a")
				#else:	            ave, var = avgvar(grp_imgdata, mode="a")
				"""
				if myid == main_node:
					ave.write_image("avgv.hdf",i)
					var.write_image("varv.hdf",i)
				"""
			
				set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
				set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])

				aveList.append(ave)
				varList.append(var)

				if options.VERBOSE:
					print "%5.2f%% done on processor %d"%(i*100.0/len(proj_list), myid)
				if nvec > 0:
					eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True)
					for k in xrange(nvec):
						set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
						eigList[k].append(eig[k])
					"""
					if myid == 0 and i == 0:
						for k in xrange(nvec):
							eig[k].write_image("eig.hdf", k)
					"""

			del imgdata
			#  To this point, all averages, variances, and eigenvectors are computed

			if options.ave2D:
				from fundamentals import fpol
				if myid == main_node:
					km = 0
					for i in xrange(number_of_proc):
						if i == main_node :
							for im in xrange(len(aveList)):
								aveList[im].write_image(options.ave2D, km)
								km += 1
						else:
							nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
							nl = int(nl[0])
							for im in xrange(nl):
								ave = recv_EMData(i, im+i+70000)
								"""
								nm = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
								nm = int(nm[0])
								members = mpi_recv(nm, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
								ave.set_attr('members', map(int, members))
								members = mpi_recv(nm, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
								ave.set_attr('pix_err', map(float, members))
								members = mpi_recv(3, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
								ave.set_attr('refprojdir', map(float, members))
								"""
								tmpvol=fpol(ave, Tracker["nx"],Tracker["nx"],Tracker["nx"])								
								tmpvol.write_image(options.ave2D, km)
								km += 1
				else:
					mpi_send(len(aveList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
					for im in xrange(len(aveList)):
						send_EMData(aveList[im], main_node,im+myid+70000)
						"""
						members = aveList[im].get_attr('members')
						mpi_send(len(members), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
						mpi_send(members, len(members), MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
						members = aveList[im].get_attr('pix_err')
						mpi_send(members, len(members), MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
						try:
							members = aveList[im].get_attr('refprojdir')
							mpi_send(members, 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
						except:
							mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
						"""

			if options.ave3D:
				from fundamentals import fpol
				if options.VERBOSE:
					print "Reconstructing 3D average volume"
				ave3D = recons3d_4nn_MPI(myid, aveList, symmetry=options.sym, npad=options.npad)
				bcast_EMData_to_all(ave3D, myid)
				if myid == main_node:
					ave3D=fpol(ave3D,Tracker["nx"],Tracker["nx"],Tracker["nx"])
					ave3D.write_image(options.ave3D)
					print_msg("%-70s:  %s\n"%("Writing to the disk volume reconstructed from averages as", options.ave3D))
			del ave, var, proj_list, stack, phi, theta, psi, s2x, s2y, alpha, sx, sy, mirror, aveList

			if nvec > 0:
				for k in xrange(nvec):
					if options.VERBOSE:
						print "Reconstruction eigenvolumes", k
					cont = True
					ITER = 0
					mask2d = model_circle(radiuspca, nx, nx)
					while cont:
						#print "On node %d, iteration %d"%(myid, ITER)
						eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad)
						bcast_EMData_to_all(eig3D, myid, main_node)
						if options.fl > 0.0:
							eig3D = filt_tanl(eig3D, options.fl, options.aa)
						if myid == main_node:
							eig3D.write_image("eig3d_%03d.hdf"%k, ITER)
						Util.mul_img( eig3D, model_circle(radiuspca, nx, nx, nx) )
						eig3Df, kb = prep_vol(eig3D)
						del eig3D
						cont = False
						icont = 0
						for l in xrange(len(eigList[k])):
							phi, theta, psi, s2x, s2y = get_params_proj(eigList[k][l])
							proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y])
							cl = ccc(proj, eigList[k][l], mask2d)
							if cl < 0.0:
								icont += 1
								cont = True
								eigList[k][l] *= -1.0
						u = int(cont)
						u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD)
						icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)

						if myid == main_node:
							u = int(u[0])
							print " Eigenvector: ",k," number changed ",int(icont[0])
						else: u = 0
						u = bcast_number_to_all(u, main_node)
						cont = bool(u)
						ITER += 1

					del eig3Df, kb
					mpi_barrier(MPI_COMM_WORLD)
				del eigList, mask2d

			if options.ave3D: del ave3D
			if options.var2D:
				from fundamentals import fpol 
				if myid == main_node:
					km = 0
					for i in xrange(number_of_proc):
						if i == main_node :
							for im in xrange(len(varList)):
								tmpvol=fpol(varList[im], Tracker["nx"], Tracker["nx"],1)
								tmpvol.write_image(options.var2D, km)
								km += 1
						else:
							nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
							nl = int(nl[0])
							for im in xrange(nl):
								ave = recv_EMData(i, im+i+70000)
								tmpvol=fpol(ave, Tracker["nx"], Tracker["nx"],1)
								tmpvol.write_image(options.var2D, km)
								km += 1
				else:
					mpi_send(len(varList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
					for im in xrange(len(varList)):
						send_EMData(varList[im], main_node, im+myid+70000)#  What with the attributes??

			mpi_barrier(MPI_COMM_WORLD)

		if  options.var3D:
			if myid == main_node and options.VERBOSE:
				print "Reconstructing 3D variability volume"

			t6 = time()
			radiusvar = options.radiusvar
			if( radiusvar < 0 ):  radiusvar = nx//2 -3
			res = recons3d_4nn_MPI(myid, varList, symmetry=options.sym, npad=options.npad)
			#res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
			if myid == main_node:
				from fundamentals import fpol
				res =fpol(res, Tracker["nx"], Tracker["nx"], Tracker["nx"])
				res.write_image(options.var3D)

			if myid == main_node:
				print_msg("%-70s:  %.2f\n"%("Reconstructing 3D variability took [s]", time()-t6))
				if options.VERBOSE:
					print "Reconstruction took: %.2f [min]"%((time()-t6)/60)

			if myid == main_node:
				print_msg("%-70s:  %.2f\n"%("Total time for these computations [s]", time()-t0))
				if options.VERBOSE:
					print "Total time for these computations: %.2f [min]"%((time()-t0)/60)
				print_end_msg("sx3dvariability")

		global_def.BATCH = False

		from mpi import mpi_finalize
		mpi_finalize()
Esempio n. 17
0
def main():
    """
	Main function.

	Arguments:
	None

	Returns:
	None
	"""

    command_args = parse_command_line()

    # Import volume
    print('Import volume.')
    input_vol = utilities.get_im(command_args.input_volume)

    # Sanity checks
    sanity_checks(command_args, input_vol)

    try:
        os.makedirs(command_args.output_dir)
    except OSError:
        print('Output directory already exists. No need to create it.')
    else:
        print('Created output directory.')
    output_prefix = os.path.join(command_args.output_dir, command_args.prefix)

    # Filter volume if specified
    if command_args.low_pass_filter_resolution is not None:
        print('Filter volume to {0}A.'.format(
            command_args.low_pass_filter_resolution))
        input_vol = sparx_filter.filt_tanl(
            input_vol,
            command_args.pixel_size / command_args.low_pass_filter_resolution,
            command_args.low_pass_filter_falloff)
        input_vol.write_image(output_prefix + '_filtered_volume.hdf')
    else:
        print('Skip filter volume.')

    # Create a mask based on the filtered volume
    print('Create mask')
    density_threshold = -9999.0
    nsigma = 1.0
    if command_args.mol_mass:
        density_threshold = input_vol.find_3d_threshold(
            command_args.mol_mass, command_args.pixel_size)
    elif command_args.threshold:
        density_threshold = command_args.threshold
    elif command_args.nsigma:
        nsigma = command_args.nsigma
    else:
        assert False

    if command_args.edge_type == 'cosine':
        mode = 'C'
    elif command_args.edge_type == 'gaussian':
        mode = 'G'
    else:
        assert False

    mask_first = morphology.adaptive_mask_scipy(
        input_vol,
        nsigma=nsigma,
        threshold=density_threshold,
        ndilation=command_args.ndilation,
        nerosion=command_args.nerosion,
        edge_width=command_args.edge_width,
        allow_disconnected=command_args.allow_disconnected,
        mode=mode,
        do_approx=command_args.do_old,
    )

    # Create a second mask based on the filtered volume
    s_mask = None
    s_density_threshold = 1
    s_nsigma = 1.0
    if command_args.second_mask is not None:
        s_mask = utilities.get_im(command_args.second_mask)
        density_threshold = -9999.0
        nsigma = 1.0
        if command_args.s_mol_mass:
            s_density_threshold = input_vol.find_3d_threshold(
                command_args.s_mol_mass, command_args.s_pixel_size)
        elif command_args.s_threshold:
            s_density_threshold = command_args.s_threshold
        elif command_args.s_nsigma:
            s_nsigma = command_args.s_nsigma
        else:
            assert False
    elif command_args.second_mask_shape is not None:
        nx = mask_first.get_xsize()
        ny = mask_first.get_ysize()
        nz = mask_first.get_zsize()
        if command_args.second_mask_shape == 'cube':
            s_nx = command_args.s_nx
            s_ny = command_args.s_ny
            s_nz = command_args.s_nz
            s_mask = utilities.model_blank(s_nx, s_ny, s_nz, 1)
        elif command_args.second_mask_shape == 'cylinder':
            s_radius = command_args.s_radius
            s_nx = command_args.s_nx
            s_ny = command_args.s_ny
            s_nz = command_args.s_nz
            s_mask = utilities.model_cylinder(s_radius, s_nx, s_ny, s_nz)
        elif command_args.second_mask_shape == 'sphere':
            s_radius = command_args.s_radius
            s_nx = command_args.s_nx
            s_ny = command_args.s_ny
            s_nz = command_args.s_nz
            s_mask = utilities.model_circle(s_radius, s_nx, s_ny, s_nz)
        else:
            assert False
        s_mask = utilities.pad(s_mask, nx, ny, nz, 0)

    if s_mask is not None:
        print('Create second mask')

        if command_args.s_edge_type == 'cosine':
            mode = 'C'
        elif command_args.s_edge_type == 'gaussian':
            mode = 'G'
        else:
            assert False

        s_mask = morphology.adaptive_mask_scipy(
            s_mask,
            nsigma=s_nsigma,
            threshold=s_density_threshold,
            ndilation=command_args.s_ndilation,
            nerosion=command_args.s_nerosion,
            edge_width=command_args.s_edge_width,
            allow_disconnected=command_args.s_allow_disconnected,
            mode=mode,
            do_approx=command_args.s_do_old)
        if command_args.s_invert:
            s_mask = 1 - s_mask
        mask_first.write_image(output_prefix + '_mask_first.hdf')
        s_mask.write_image(output_prefix + '_mask_second.hdf')
        masked_combined = mask_first * s_mask
        masked_combined.write_image(output_prefix + '_mask.hdf')
    else:
        mask_first.write_image(output_prefix + '_mask.hdf')
Esempio n. 18
0
def main():
	progname = os.path.basename(sys.argv[0])
	usage = progname + """ Input Output [options]
	
	Generate three micrographs, each micrograph contains one projection of a long filament.
	Input: Reference Volume, output directory 
	Output: Three micrographs stored in output directory		
				 
		sxhelical_demo.py tmp.hdf  mic --generate_micrograph --CTF --apix=1.84	
	
	Generate noisy cylinder ini.hdf with radius 35 pixels and box size 100 by 100 by 200
	
		sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35
	
	Generate rectangular 2D mask mask2d.hdf with width 60 pixels and image size 200 by 200 pixels
	
		sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=60
	
	Apply the centering parameters to bdb:adata, normalize using average and standard deviation outside the mask, and output the new images to bdb:data
		
		sxhelical_demo.py bdb:adata bdb:data mask2d.hdf --applyparams
	
	Generate run through example script for helicon
	
		sxhelical_demo.py --generate_script --filename=run --seg_ny=180 --ptcl_dist=15 --fract=0.35
	"""
	parser = OptionParser(usage,version=SPARXVERSION)
	
	# helicise the Atom coordinates
	
	# generate micrographs of helical filament
	parser.add_option("--generate_micrograph",    action="store_true",      default=False,      		  	 help="Generate three micrographs where each micrograph contains one projection of a long filament. \n Input: Reference Volume, output directory \n Output: Three micrographs containing helical filament projections stored in output directory")
	parser.add_option("--CTF",              	  action="store_true",  	default=False,   				 help="Use CTF correction")
	parser.add_option("--apix",               	  type="float",			 	default= -1,               	     help="pixel size in Angstroms")   
	parser.add_option("--rand_seed",              type="int",			    default=14567,              	 help="the seed used for generating random numbers (default 14567) for adding noise to the generated micrographs.")
	parser.add_option("--Cs",               	  type="float",			 	default= 2.0,               	 help="Microscope Cs (spherical aberation)")
	parser.add_option("--voltage",				  type="float",				default=200.0, 					 help="Microscope voltage in KV")
	parser.add_option("--ac",					  type="float",				default=10.0, 					 help="Amplitude contrast (percentage, default=10)")
	parser.add_option("--nonoise",                action="store_true",      default=False,      		  	 help="Do not add noise to the micrograph.")
	
	# generate initial volume
	parser.add_option("--generate_noisycyl",      action="store_true",      default=False,      		  	 help="Generate initial volume of noisy cylinder.")
	parser.add_option("--boxsize",                type="string",		    default="100,100,200",           help="String containing x , y, z dimensions (separated by comma) in pixels")
	parser.add_option("--rad",                    type="int",			    default=35,              	 	 help="Radius of initial volume in pixels")
	
	# generate 2D mask 
	parser.add_option("--generate_mask",          action="store_true",      default=False,      		  	 help="Generate 2D rectangular mask.")
	parser.add_option("--masksize",               type="string",		    default="200,200",               help="String containing x and y dimensions (separated by comma) in pixels")
	parser.add_option("--maskwidth",              type="int",			    default=60,              	 	 help="Width of rectangular mask")
	
	# Apply 2D alignment parameters to input stack and output new images to output stack
	parser.add_option("--applyparams",            action="store_true",      default=False,      		  	 help="Apply the centering parameters to input stack, normalize using average and standard deviation outside the mask, and output the new images to output stack")
	
	# Generate run script
	parser.add_option("--generate_script",        action="store_true",      default=False,      		  	 help="Generate script for helicon run through example")
	parser.add_option("--filename",               type="string",		    default="runhelicon",            help="Name of run script to generate")
	parser.add_option("--seg_ny",                 type="int",			    default=180,              	     help="y-dimension of segment used for refinement")
	parser.add_option("--ptcl_dist",              type="int",			    default=15,              	     help="Distance in pixels between adjacent segments windowed from same filament")
	parser.add_option("--fract",               	  type="float",			 	default=0.35,               	 help="Fraction of the volume used for applying helical symmetry.")
	
	(options, args) = parser.parse_args()
	if len(args) > 3:
		print "usage: " + usage
		print "Please run '" + progname + " -h' for detailed options"
	else:
		if options.generate_script:
			generate_runscript(options.filename, options.seg_ny, options.ptcl_dist, options.fract)

		if options.generate_micrograph:
			if options.apix <= 0:
				print "Please enter pixel size."
				sys.exit()
			generate_helimic(args[0], args[1], options.apix, options.CTF, options.Cs, options.voltage, options.ac, options.nonoise, options.rand_seed)

		if options.generate_noisycyl:
			from utilities import model_cylinder, model_gauss_noise
			outvol = args[0]
			boxdims = options.boxsize.split(',')
			if len(boxdims) < 1 or len(boxdims) > 3:
				print "Enter box size as string containing x , y, z dimensions (separated by comma) in pixels. E.g.: --boxsize='100,100,200'"
				sys.exit()
			nx= int(boxdims[0])
			if len(boxdims) == 1:
				ny = nx
				nz = nx
			else:
				ny = int(boxdims[1])
				if len(boxdims) == 3:
					nz = int(boxdims[2])
					
			(model_cylinder(options.rad,nx, ny, nz)*model_gauss_noise(1.0, nx, ny, nz) ).write_image(outvol)

		if options.generate_mask:
			from utilities import model_blank, pad
			outvol = args[0]
			maskdims = options.masksize.split(',')
			if len(maskdims) < 1 or len(maskdims) > 2:
				print "Enter box size as string containing x , y dimensions (separated by comma) in pixels. E.g.: --boxsize='200,200'"
				sys.exit()
			nx= int(maskdims[0])
			if len(maskdims) == 1:
				ny = nx
			else:
				ny = int(maskdims[1])
					
			mask = pad(model_blank(options.maskwidth, ny, 1, 1.0), nx, ny, 1, 0.0)
			mask.write_image(outvol)
		
		if options.applyparams:
			from utilities    import get_im, get_params2D, set_params2D
			from fundamentals import cyclic_shift
			stack = args[0]
			newstack = args[1]
			mask = get_im(args[2])
			nima = EMUtil.get_image_count(stack)
			for im in xrange(nima):
				prj = get_im(stack,im)
				alpha, sx, sy, mirror, scale = get_params2D(prj)
				prj = cyclic_shift(prj, int(sx))
				set_params2D(prj, [0.0,0.,0.0,0,1])
				stat = Util.infomask(prj , mask, False )
				prj= (prj-stat[0])/stat[1]
				ctf_params = prj.get_attr("ctf")
				prj.set_attr('ctf_applied', 0)
				prj.write_image(newstack, im)
Esempio n. 19
0
def main():
    progname = os.path.basename(sys.argv[0])
    usage = progname + """ Input Output [options]
	
	Generate three micrographs, each micrograph contains one projection of a long filament.
	Input: Reference Volume, output directory 
	Output: Three micrographs stored in output directory		
				 
		sxhelical_demo.py tmp.hdf  mic --generate_micrograph --CTF --apix=1.84	
	
	Generate noisy cylinder ini.hdf with radius 35 pixels and box size 100 by 100 by 200
	
		sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35
	
	Generate rectangular 2D mask mask2d.hdf with width 60 pixels and image size 200 by 200 pixels
	
		sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=60
	
	Apply the centering parameters to bdb:adata, normalize using average and standard deviation outside the mask, and output the new images to bdb:data
		
		sxhelical_demo.py bdb:adata bdb:data mask2d.hdf --applyparams
	
	Generate run through example script for helicon
	
		sxhelical_demo.py --generate_script --filename=run --seg_ny=180 --ptcl_dist=15 --fract=0.35
	"""
    parser = OptionParser(usage, version=SPARXVERSION)

    # helicise the Atom coordinates

    # generate micrographs of helical filament
    parser.add_option(
        "--generate_micrograph",
        action="store_true",
        default=False,
        help=
        "Generate three micrographs where each micrograph contains one projection of a long filament. \n Input: Reference Volume, output directory \n Output: Three micrographs containing helical filament projections stored in output directory"
    )
    parser.add_option("--CTF",
                      action="store_true",
                      default=False,
                      help="Use CTF correction")
    parser.add_option("--apix",
                      type="float",
                      default=-1,
                      help="pixel size in Angstroms")
    parser.add_option(
        "--rand_seed",
        type="int",
        default=14567,
        help=
        "the seed used for generating random numbers (default 14567) for adding noise to the generated micrographs."
    )
    parser.add_option("--Cs",
                      type="float",
                      default=2.0,
                      help="Microscope Cs (spherical aberation)")
    parser.add_option("--voltage",
                      type="float",
                      default=200.0,
                      help="Microscope voltage in KV")
    parser.add_option("--ac",
                      type="float",
                      default=10.0,
                      help="Amplitude contrast (percentage, default=10)")
    parser.add_option("--nonoise",
                      action="store_true",
                      default=False,
                      help="Do not add noise to the micrograph.")

    # generate initial volume
    parser.add_option("--generate_noisycyl",
                      action="store_true",
                      default=False,
                      help="Generate initial volume of noisy cylinder.")
    parser.add_option(
        "--boxsize",
        type="string",
        default="100,100,200",
        help=
        "String containing x , y, z dimensions (separated by comma) in pixels")
    parser.add_option("--rad",
                      type="int",
                      default=35,
                      help="Radius of initial volume in pixels")

    # generate 2D mask
    parser.add_option("--generate_mask",
                      action="store_true",
                      default=False,
                      help="Generate 2D rectangular mask.")
    parser.add_option(
        "--masksize",
        type="string",
        default="200,200",
        help=
        "String containing x and y dimensions (separated by comma) in pixels")
    parser.add_option("--maskwidth",
                      type="int",
                      default=60,
                      help="Width of rectangular mask")

    # Apply 2D alignment parameters to input stack and output new images to output stack
    parser.add_option(
        "--applyparams",
        action="store_true",
        default=False,
        help=
        "Apply the centering parameters to input stack, normalize using average and standard deviation outside the mask, and output the new images to output stack"
    )

    # Generate run script
    parser.add_option("--generate_script",
                      action="store_true",
                      default=False,
                      help="Generate script for helicon run through example")
    parser.add_option("--filename",
                      type="string",
                      default="runhelicon",
                      help="Name of run script to generate")
    parser.add_option("--seg_ny",
                      type="int",
                      default=180,
                      help="y-dimension of segment used for refinement")
    parser.add_option(
        "--ptcl_dist",
        type="int",
        default=15,
        help=
        "Distance in pixels between adjacent segments windowed from same filament"
    )
    parser.add_option(
        "--fract",
        type="float",
        default=0.35,
        help="Fraction of the volume used for applying helical symmetry.")

    (options, args) = parser.parse_args()
    if len(args) > 3:
        print("usage: " + usage)
        print("Please run '" + progname + " -h' for detailed options")
    else:
        if options.generate_script:
            generate_runscript(options.filename, options.seg_ny,
                               options.ptcl_dist, options.fract)

        if options.generate_micrograph:
            if options.apix <= 0:
                print("Please enter pixel size.")
                sys.exit()
            generate_helimic(args[0], args[1], options.apix, options.CTF,
                             options.Cs, options.voltage, options.ac,
                             options.nonoise, options.rand_seed)

        if options.generate_noisycyl:
            from utilities import model_cylinder, model_gauss_noise
            outvol = args[0]
            boxdims = options.boxsize.split(',')
            if len(boxdims) < 1 or len(boxdims) > 3:
                print(
                    "Enter box size as string containing x , y, z dimensions (separated by comma) in pixels. E.g.: --boxsize='100,100,200'"
                )
                sys.exit()
            nx = int(boxdims[0])
            if len(boxdims) == 1:
                ny = nx
                nz = nx
            else:
                ny = int(boxdims[1])
                if len(boxdims) == 3:
                    nz = int(boxdims[2])

            (model_cylinder(options.rad, nx, ny, nz) *
             model_gauss_noise(1.0, nx, ny, nz)).write_image(outvol)

        if options.generate_mask:
            from utilities import model_blank, pad
            outvol = args[0]
            maskdims = options.masksize.split(',')
            if len(maskdims) < 1 or len(maskdims) > 2:
                print(
                    "Enter box size as string containing x , y dimensions (separated by comma) in pixels. E.g.: --boxsize='200,200'"
                )
                sys.exit()
            nx = int(maskdims[0])
            if len(maskdims) == 1:
                ny = nx
            else:
                ny = int(maskdims[1])

            mask = pad(model_blank(options.maskwidth, ny, 1, 1.0), nx, ny, 1,
                       0.0)
            mask.write_image(outvol)

        if options.applyparams:
            from utilities import get_im, get_params2D, set_params2D
            from fundamentals import cyclic_shift
            stack = args[0]
            newstack = args[1]
            mask = get_im(args[2])
            nima = EMUtil.get_image_count(stack)
            for im in range(nima):
                prj = get_im(stack, im)
                alpha, sx, sy, mirror, scale = get_params2D(prj)
                prj = cyclic_shift(prj, int(sx))
                set_params2D(prj, [0.0, 0., 0.0, 0, 1])
                stat = Util.infomask(prj, mask, False)
                prj = old_div((prj - stat[0]), stat[1])
                ctf_params = prj.get_attr("ctf")
                prj.set_attr('ctf_applied', 0)
                prj.write_image(newstack, im)
Esempio n. 20
0
def main():
    import sys
    import os
    import math
    import random
    import pyemtbx.options
    import time
    from random import random, seed, randint
    from optparse import OptionParser

    progname = os.path.basename(sys.argv[0])
    usage = progname + """ [options] <inputfile> <outputfile>

	Generic 2-D image processing programs.

	Functionality:

	1.  Phase flip a stack of images and write output to new file:
		sxprocess.py input_stack.hdf output_stack.hdf --phase_flip
	
	2.  Resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.
	    The window size will change accordingly.
		sxprocess input.hdf output.hdf  --changesize --ratio=0.5

	3.  Compute average power spectrum of a stack of 2D images with optional padding (option wn) with zeroes.
		sxprocess.py input_stack.hdf powerspectrum.hdf --pw [--wn=1024]

	4.  Generate a stack of projections bdb:data and micrographs with prefix mic (i.e., mic0.hdf, mic1.hdf etc) from structure input_structure.hdf, with CTF applied to both projections and micrographs:
		sxprocess.py input_structure.hdf data mic --generate_projections format="bdb":apix=5.2:CTF=True:boxsize=64

    5.  Retrieve original image numbers in the selected ISAC group (here group 12 from generation 3):
    	sxprocess.py  bdb:test3 class_averages_generation_3.hdf  list3_12.txt --isacgroup=12 --params=originalid

    6.  Retrieve original image numbers of images listed in ISAC output stack of averages:
    	sxprocess.py  select1.hdf  ohk.txt

    7.  Adjust rotationally averaged power spectrum of an image to that of a reference image or a reference 1D power spectrum stored in an ASCII file.
    	Optionally use a tangent low-pass filter.  Also works for a stack of images, in which case the output is also a stack.
    	sxprocess.py  vol.hdf ref.hdf  avol.hdf < 0.25 0.2> --adjpw
   	 	sxprocess.py  vol.hdf pw.txt   avol.hdf < 0.25 0.2> --adjpw

    8.  Generate a 1D rotationally averaged power spectrum of an image.
		sxprocess.py  vol.hdf --rotwp=rotpw.txt
    	# Output will contain three columns:
       (1) rotationally averaged power spectrum
       (2) logarithm of the rotationally averaged power spectrum
       (3) integer line number (from zero to approximately to half the image size)

    9.  Apply 3D transformation (rotation and/or shift) to a set of orientation parameters associated with projection data.
    	sxprocess.py  --transfromparams=phi,theta,psi,tx,ty,tz      input.txt  output.txt
    	The output file is then imported and 3D transformed volume computed:
    	sxheader.py  bdb:p  --params=xform.projection  --import=output.txt
    	mpirun -np 2 sxrecons3d_n.py  bdb:p tvol.hdf --MPI
    	The reconstructed volume is in the position of the volume computed using the input.txt parameters and then
    	transformed with rot_shift3D(vol, phi,theta,psi,tx,ty,tz)

   10.  Import ctf parameters from the output of sxcter into windowed particle headers.
	    There are three possible input files formats:  (1) all particles are in one stack, (2 aor 3) particles are in stacks, each stack corresponds to a single micrograph.
	    In each case the particles should contain a name of the micrograph of origin stores using attribute name 'ptcl_source_image'.
        Normally this is done by e2boxer.py during windowing.
	    Particles whose defocus or astigmatism error exceed set thresholds will be skipped, otherwise, virtual stacks with the original way preceded by G will be created.
		sxprocess.py  --input=bdb:data  --importctf=outdir/partres  --defocuserror=10.0  --astigmatismerror=5.0
		#  Output will be a vritual stack bdb:Gdata
		sxprocess.py  --input="bdb:directory/stacks*"  --importctf=outdir/partres  --defocuserror=10.0  --astigmatismerror=5.0
		To concatenate output files:
		cd directory
		e2bdb.py . --makevstack=bdb:allparticles  --filt=G
		IMPORTANT:  Please do not move (or remove!) any input/intermediate EMAN2DB files as the information is linked between them.

   11. Scale 3D shifts.  The shifts in the input five columns text file with 3D orientation parameters will be DIVIDED by the scale factor
		sxprocess.py  orientationparams.txt  scaledparams.txt  scale=0.5
   
   12. Generate adaptive mask from a given 3-D volume. 


"""

    parser = OptionParser(usage, version=SPARXVERSION)
    parser.add_option(
        "--order",
        action="store_true",
        help=
        "Two arguments are required: name of input stack and desired name of output stack. The output stack is the input stack sorted by similarity in terms of cross-correlation coefficent.",
        default=False)
    parser.add_option("--order_lookup",
                      action="store_true",
                      help="Test/Debug.",
                      default=False)
    parser.add_option("--order_metropolis",
                      action="store_true",
                      help="Test/Debug.",
                      default=False)
    parser.add_option("--order_pca",
                      action="store_true",
                      help="Test/Debug.",
                      default=False)
    parser.add_option(
        "--initial",
        type="int",
        default=-1,
        help=
        "Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)"
    )
    parser.add_option(
        "--circular",
        action="store_true",
        help=
        "Select circular ordering (fisr image has to be similar to the last",
        default=False)
    parser.add_option(
        "--radius",
        type="int",
        default=-1,
        help="Radius of a circular mask for similarity based ordering")
    parser.add_option(
        "--changesize",
        action="store_true",
        help=
        "resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.",
        default=False)
    parser.add_option(
        "--ratio",
        type="float",
        default=1.0,
        help=
        "The ratio of new to old image size (if <1 the pixel size will increase and image size decrease, if>1, the other way round"
    )
    parser.add_option(
        "--pw",
        action="store_true",
        help=
        "compute average power spectrum of a stack of 2-D images with optional padding (option wn) with zeroes",
        default=False)
    parser.add_option(
        "--wn",
        type="int",
        default=-1,
        help=
        "Size of window to use (should be larger/equal than particle box size, default padding to max(nx,ny))"
    )
    parser.add_option("--phase_flip",
                      action="store_true",
                      help="Phase flip the input stack",
                      default=False)
    parser.add_option(
        "--makedb",
        metavar="param1=value1:param2=value2",
        type="string",
        action="append",
        help=
        "One argument is required: name of key with which the database will be created. Fill in database with parameters specified as follows: --makedb param1=value1:param2=value2, e.g. 'gauss_width'=1.0:'pixel_input'=5.2:'pixel_output'=5.2:'thr_low'=1.0"
    )
    parser.add_option(
        "--generate_projections",
        metavar="param1=value1:param2=value2",
        type="string",
        action="append",
        help=
        "Three arguments are required: name of input structure from which to generate projections, desired name of output projection stack, and desired prefix for micrographs (e.g. if prefix is 'mic', then micrographs mic0.hdf, mic1.hdf etc will be generated). Optional arguments specifying format, apix, box size and whether to add CTF effects can be entered as follows after --generate_projections: format='bdb':apix=5.2:CTF=True:boxsize=100, or format='hdf', etc., where format is bdb or hdf, apix (pixel size) is a float, CTF is True or False, and boxsize denotes the dimension of the box (assumed to be a square). If an optional parameter is not specified, it will default as follows: format='bdb', apix=2.5, CTF=False, boxsize=64."
    )
    parser.add_option(
        "--isacgroup",
        type="int",
        help=
        "Retrieve original image numbers in the selected ISAC group. See ISAC documentation for details.",
        default=-1)
    parser.add_option(
        "--isacselect",
        action="store_true",
        help=
        "Retrieve original image numbers of images listed in ISAC output stack of averages. See ISAC documentation for details.",
        default=False)
    parser.add_option(
        "--params",
        type="string",
        default=None,
        help="Name of header of parameter, which one depends on specific option"
    )
    parser.add_option(
        "--adjpw",
        action="store_true",
        help="Adjust rotationally averaged power spectrum of an image",
        default=False)
    parser.add_option(
        "--rotpw",
        type="string",
        default=None,
        help=
        "Name of the text file to contain rotationally averaged power spectrum of the input image."
    )
    parser.add_option(
        "--transformparams",
        type="string",
        default=None,
        help=
        "Transform 3D projection orientation parameters using six 3D parameters (phi, theta,psi,sx,sy,sz).  Input: --transformparams=45.,66.,12.,-2,3,-5.5 desired six transformation of the reconstructed structure. Output: file with modified orientation parameters."
    )

    # import ctf estimates done using cter
    parser.add_option("--input",
                      type="string",
                      default=None,
                      help="Input particles.")
    parser.add_option(
        "--importctf",
        type="string",
        default=None,
        help="Name of the file containing CTF parameters produced by sxcter.")
    parser.add_option(
        "--defocuserror",
        type="float",
        default=1000000.0,
        help=
        "Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent.  The error is computed as (std dev defocus)/defocus*100%"
    )
    parser.add_option(
        "--astigmatismerror",
        type="float",
        default=360.0,
        help=
        "Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees."
    )

    # import ctf estimates done using cter
    parser.add_option(
        "--scale",
        type="float",
        default=-1.0,
        help=
        "Divide shifts in the input 3D orientation parameters text file by the scale factor."
    )

    # generate adaptive mask from an given 3-Db volue
    parser.add_option("--adaptive_mask",
                      action="store_true",
                      help="create adavptive 3-D mask from a given volume",
                      default=False)
    parser.add_option(
        "--nsigma",
        type="float",
        default=1.,
        help=
        "number of times of sigma of the input volume to obtain the the large density cluster"
    )
    parser.add_option(
        "--ndilation",
        type="int",
        default=3,
        help=
        "number of times of dilation applied to the largest cluster of density"
    )
    parser.add_option(
        "--kernel_size",
        type="int",
        default=11,
        help="convolution kernel for smoothing the edge of the mask")
    parser.add_option(
        "--gauss_standard_dev",
        type="int",
        default=9,
        help="stanadard deviation value to generate Gaussian edge")

    (options, args) = parser.parse_args()

    global_def.BATCH = True

    if options.phase_flip:
        nargs = len(args)
        if nargs != 2:
            print "must provide name of input and output file!"
            return
        from EMAN2 import Processor
        instack = args[0]
        outstack = args[1]
        nima = EMUtil.get_image_count(instack)
        from filter import filt_ctf
        for i in xrange(nima):
            img = EMData()
            img.read_image(instack, i)
            try:
                ctf = img.get_attr('ctf')
            except:
                print "no ctf information in input stack! Exiting..."
                return

            dopad = True
            sign = 1
            binary = 1  # phase flip

            assert img.get_ysize() > 1
            dict = ctf.to_dict()
            dz = dict["defocus"]
            cs = dict["cs"]
            voltage = dict["voltage"]
            pixel_size = dict["apix"]
            b_factor = dict["bfactor"]
            ampcont = dict["ampcont"]
            dza = dict["dfdiff"]
            azz = dict["dfang"]

            if dopad and not img.is_complex(): ip = 1
            else: ip = 0

            params = {
                "filter_type": Processor.fourier_filter_types.CTF_,
                "defocus": dz,
                "Cs": cs,
                "voltage": voltage,
                "Pixel_size": pixel_size,
                "B_factor": b_factor,
                "amp_contrast": ampcont,
                "dopad": ip,
                "binary": binary,
                "sign": sign,
                "dza": dza,
                "azz": azz
            }

            tmp = Processor.EMFourierFilter(img, params)
            tmp.set_attr_dict({"ctf": ctf})

            tmp.write_image(outstack, i)

    elif options.changesize:
        nargs = len(args)
        if nargs != 2:
            ERROR("must provide name of input and output file!", "change size",
                  1)
            return
        from utilities import get_im
        instack = args[0]
        outstack = args[1]
        sub_rate = float(options.ratio)

        nima = EMUtil.get_image_count(instack)
        from fundamentals import resample
        for i in xrange(nima):
            resample(get_im(instack, i), sub_rate).write_image(outstack, i)

    elif options.isacgroup > -1:
        nargs = len(args)
        if nargs != 3:
            ERROR("Three files needed on input!", "isacgroup", 1)
            return
        from utilities import get_im
        instack = args[0]
        m = get_im(args[1], int(options.isacgroup)).get_attr("members")
        l = []
        for k in m:
            l.append(int(get_im(args[0], k).get_attr(options.params)))
        from utilities import write_text_file
        write_text_file(l, args[2])

    elif options.isacselect:
        nargs = len(args)
        if nargs != 2:
            ERROR("Two files needed on input!", "isacgroup", 1)
            return
        from utilities import get_im
        nima = EMUtil.get_image_count(args[0])
        m = []
        for k in xrange(nima):
            m += get_im(args[0], k).get_attr("members")
        m.sort()
        from utilities import write_text_file
        write_text_file(m, args[1])

    elif options.pw:
        nargs = len(args)
        if nargs < 2:
            ERROR("must provide name of input and output file!", "pw", 1)
            return
        from utilities import get_im
        d = get_im(args[0])
        nx = d.get_xsize()
        ny = d.get_ysize()
        if nargs == 3: mask = get_im(args[2])
        wn = int(options.wn)
        if wn == -1:
            wn = max(nx, ny)
        else:
            if ((wn < nx) or (wn < ny)):
                ERROR("window size cannot be smaller than the image size",
                      "pw", 1)
        n = EMUtil.get_image_count(args[0])
        from utilities import model_blank, model_circle, pad
        from EMAN2 import periodogram
        p = model_blank(wn, wn)

        for i in xrange(n):
            d = get_im(args[0], i)
            if nargs == 3:
                d *= mask
            st = Util.infomask(d, None, True)
            d -= st[0]
            p += periodogram(pad(d, wn, wn, 1, 0.))
        p /= n
        p.write_image(args[1])

    elif options.adjpw:

        if len(args) < 3:
            ERROR(
                "filt_by_rops input target output fl aa (the last two are optional parameters of a low-pass filter)",
                "adjpw", 1)
            return
        img_stack = args[0]
        from math import sqrt
        from fundamentals import rops_table, fft
        from utilities import read_text_file, get_im
        from filter import filt_tanl, filt_table
        if (args[1][-3:] == 'txt'):
            rops_dst = read_text_file(args[1])
        else:
            rops_dst = rops_table(get_im(args[1]))

        out_stack = args[2]
        if (len(args) > 4):
            fl = float(args[3])
            aa = float(args[4])
        else:
            fl = -1.0
            aa = 0.0

        nimage = EMUtil.get_image_count(img_stack)

        for i in xrange(nimage):
            img = fft(get_im(img_stack, i))
            rops_src = rops_table(img)

            assert len(rops_dst) == len(rops_src)

            table = [0.0] * len(rops_dst)
            for j in xrange(len(rops_dst)):
                table[j] = sqrt(rops_dst[j] / rops_src[j])

            if (fl > 0.0):
                img = filt_tanl(img, fl, aa)
            img = fft(filt_table(img, table))
            img.write_image(out_stack, i)

    elif options.rotpw != None:

        if len(args) != 1:
            ERROR("Only one input permitted", "rotpw", 1)
            return
        from utilities import write_text_file, get_im
        from fundamentals import rops_table
        from math import log10
        t = rops_table(get_im(args[0]))
        x = range(len(t))
        r = [0.0] * len(x)
        for i in x:
            r[i] = log10(t[i])
        write_text_file([t, r, x], options.rotpw)

    elif options.transformparams != None:
        if len(args) != 2:
            ERROR(
                "Please provide names of input and output files with orientation parameters",
                "transformparams", 1)
            return
        from utilities import read_text_row, write_text_row
        transf = [0.0] * 6
        spl = options.transformparams.split(',')
        for i in xrange(len(spl)):
            transf[i] = float(spl[i])

        write_text_row(rotate_shift_params(read_text_row(args[0]), transf),
                       args[1])

    elif options.makedb != None:
        nargs = len(args)
        if nargs != 1:
            print "must provide exactly one argument denoting database key under which the input params will be stored"
            return
        dbkey = args[0]
        print "database key under which params will be stored: ", dbkey
        gbdb = js_open_dict("e2boxercache/gauss_box_DB.json")

        parmstr = 'dummy:' + options.makedb[0]
        (processorname, param_dict) = parsemodopt(parmstr)
        dbdict = {}
        for pkey in param_dict:
            if (pkey == 'invert_contrast') or (pkey == 'use_variance'):
                if param_dict[pkey] == 'True':
                    dbdict[pkey] = True
                else:
                    dbdict[pkey] = False
            else:
                dbdict[pkey] = param_dict[pkey]
        gbdb[dbkey] = dbdict

    elif options.generate_projections:
        nargs = len(args)
        if nargs != 3:
            ERROR("Must provide name of input structure(s) from which to generate projections, name of output projection stack, and prefix for output micrographs."\
            "sxprocess - generate projections",1)
            return
        inpstr = args[0]
        outstk = args[1]
        micpref = args[2]

        parmstr = 'dummy:' + options.generate_projections[0]
        (processorname, param_dict) = parsemodopt(parmstr)

        parm_CTF = False
        parm_format = 'bdb'
        parm_apix = 2.5

        if 'CTF' in param_dict:
            if param_dict['CTF'] == 'True':
                parm_CTF = True

        if 'format' in param_dict:
            parm_format = param_dict['format']

        if 'apix' in param_dict:
            parm_apix = float(param_dict['apix'])

        boxsize = 64
        if 'boxsize' in param_dict:
            boxsize = int(param_dict['boxsize'])

        print "pixel size: ", parm_apix, " format: ", parm_format, " add CTF: ", parm_CTF, " box size: ", boxsize

        scale_mult = 2500
        sigma_add = 1.5
        sigma_proj = 30.0
        sigma2_proj = 17.5
        sigma_gauss = 0.3
        sigma_mic = 30.0
        sigma2_mic = 17.5
        sigma_gauss_mic = 0.3

        if 'scale_mult' in param_dict:
            scale_mult = float(param_dict['scale_mult'])
        if 'sigma_add' in param_dict:
            sigma_add = float(param_dict['sigma_add'])
        if 'sigma_proj' in param_dict:
            sigma_proj = float(param_dict['sigma_proj'])
        if 'sigma2_proj' in param_dict:
            sigma2_proj = float(param_dict['sigma2_proj'])
        if 'sigma_gauss' in param_dict:
            sigma_gauss = float(param_dict['sigma_gauss'])
        if 'sigma_mic' in param_dict:
            sigma_mic = float(param_dict['sigma_mic'])
        if 'sigma2_mic' in param_dict:
            sigma2_mic = float(param_dict['sigma2_mic'])
        if 'sigma_gauss_mic' in param_dict:
            sigma_gauss_mic = float(param_dict['sigma_gauss_mic'])

        from filter import filt_gaussl, filt_ctf
        from utilities import drop_spider_doc, even_angles, model_gauss, delete_bdb, model_blank, pad, model_gauss_noise, set_params2D, set_params_proj
        from projection import prep_vol, prgs
        seed(14567)
        delta = 29
        angles = even_angles(delta, 0.0, 89.9, 0.0, 359.9, "S")
        nangle = len(angles)

        modelvol = []
        nvlms = EMUtil.get_image_count(inpstr)
        from utilities import get_im
        for k in xrange(nvlms):
            modelvol.append(get_im(inpstr, k))

        nx = modelvol[0].get_xsize()

        if nx != boxsize:
            ERROR("Requested box dimension does not match dimension of the input model.", \
            "sxprocess - generate projections",1)
        nvol = 10
        volfts = [[] for k in xrange(nvlms)]
        for k in xrange(nvlms):
            for i in xrange(nvol):
                sigma = sigma_add + random()  # 1.5-2.5
                addon = model_gauss(sigma, boxsize, boxsize, boxsize, sigma,
                                    sigma, 38, 38, 40)
                scale = scale_mult * (0.5 + random())
                vf, kb = prep_vol(modelvol[k] + scale * addon)
                volfts[k].append(vf)
        del vf, modelvol

        if parm_format == "bdb":
            stack_data = "bdb:" + outstk
            delete_bdb(stack_data)
        else:
            stack_data = outstk + ".hdf"
        Cs = 2.0
        pixel = parm_apix
        voltage = 120.0
        ampcont = 10.0
        ibd = 4096 / 2 - boxsize
        iprj = 0

        width = 240
        xstart = 8 + boxsize / 2
        ystart = 8 + boxsize / 2
        rowlen = 17
        from random import randint
        params = []
        for idef in xrange(3, 8):

            irow = 0
            icol = 0

            mic = model_blank(4096, 4096)
            defocus = idef * 0.5  #0.2
            if parm_CTF:
                astampl = defocus * 0.15
                astangl = 50.0
                ctf = generate_ctf([
                    defocus, Cs, voltage, pixel, ampcont, 0.0, astampl, astangl
                ])

            for i in xrange(nangle):
                for k in xrange(12):
                    dphi = 8.0 * (random() - 0.5)
                    dtht = 8.0 * (random() - 0.5)
                    psi = 360.0 * random()

                    phi = angles[i][0] + dphi
                    tht = angles[i][1] + dtht

                    s2x = 4.0 * (random() - 0.5)
                    s2y = 4.0 * (random() - 0.5)

                    params.append([phi, tht, psi, s2x, s2y])

                    ivol = iprj % nvol
                    #imgsrc = randint(0,nvlms-1)
                    imgsrc = iprj % nvlms
                    proj = prgs(volfts[imgsrc][ivol], kb,
                                [phi, tht, psi, -s2x, -s2y])

                    x = xstart + irow * width
                    y = ystart + icol * width

                    mic += pad(proj, 4096, 4096, 1, 0.0, x - 2048, y - 2048, 0)

                    proj = proj + model_gauss_noise(sigma_proj, nx, nx)
                    if parm_CTF:
                        proj = filt_ctf(proj, ctf)
                        proj.set_attr_dict({"ctf": ctf, "ctf_applied": 0})

                    proj = proj + filt_gaussl(
                        model_gauss_noise(sigma2_proj, nx, nx), sigma_gauss)
                    proj.set_attr("origimgsrc", imgsrc)
                    proj.set_attr("test_id", iprj)
                    # flags describing the status of the image (1 = true, 0 = false)
                    set_params2D(proj, [0.0, 0.0, 0.0, 0, 1.0])
                    set_params_proj(proj, [phi, tht, psi, s2x, s2y])

                    proj.write_image(stack_data, iprj)

                    icol += 1
                    if icol == rowlen:
                        icol = 0
                        irow += 1

                    iprj += 1

            mic += model_gauss_noise(sigma_mic, 4096, 4096)
            if parm_CTF:
                #apply CTF
                mic = filt_ctf(mic, ctf)
            mic += filt_gaussl(model_gauss_noise(sigma2_mic, 4096, 4096),
                               sigma_gauss_mic)

            mic.write_image(micpref + "%1d.hdf" % (idef - 3), 0)

        drop_spider_doc("params.txt", params)

    elif options.importctf != None:
        print ' IMPORTCTF  '
        from utilities import read_text_row, write_text_row
        from random import randint
        import subprocess
        grpfile = 'groupid%04d' % randint(1000, 9999)
        ctfpfile = 'ctfpfile%04d' % randint(1000, 9999)
        cterr = [options.defocuserror / 100.0, options.astigmatismerror]
        ctfs = read_text_row(options.importctf)
        for kk in xrange(len(ctfs)):
            root, name = os.path.split(ctfs[kk][-1])
            ctfs[kk][-1] = name[:-4]
        if (options.input[:4] != 'bdb:'):
            ERROR('Sorry, only bdb files implemented', 'importctf', 1)
        d = options.input[4:]
        #try:     str = d.index('*')
        #except:  str = -1
        from string import split
        import glob
        uu = os.path.split(d)
        uu = os.path.join(uu[0], 'EMAN2DB', uu[1] + '.bdb')
        flist = glob.glob(uu)
        for i in xrange(len(flist)):
            root, name = os.path.split(flist[i])
            root = root[:-7]
            name = name[:-4]
            fil = 'bdb:' + os.path.join(root, name)
            sourcemic = EMUtil.get_all_attributes(fil, 'ptcl_source_image')
            nn = len(sourcemic)
            gctfp = []
            groupid = []
            for kk in xrange(nn):
                junk, name2 = os.path.split(sourcemic[kk])
                name2 = name2[:-4]
                ctfp = [-1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
                for ll in xrange(len(ctfs)):
                    if (name2 == ctfs[ll][-1]):
                        #  found correct
                        if (ctfs[ll][8] / ctfs[ll][0] <= cterr[0]):
                            #  acceptable defocus error
                            ctfp = ctfs[ll][:8]
                            if (ctfs[ll][10] > cterr[1]):
                                # error of astigmatism exceed the threshold, set astigmatism to zero.
                                ctfp[6] = 0.0
                                ctfp[7] = 0.0
                            gctfp.append(ctfp)
                            groupid.append(kk)
                        break
            if (len(groupid) > 0):
                write_text_row(groupid, grpfile)
                write_text_row(gctfp, ctfpfile)
                cmd = "{} {} {} {}".format(
                    'e2bdb.py', fil, '--makevstack=bdb:' + root + 'G' + name,
                    '--list=' + grpfile)
                #print cmd
                subprocess.call(cmd, shell=True)
                cmd = "{} {} {} {}".format('sxheader.py',
                                           'bdb:' + root + 'G' + name,
                                           '--params=ctf',
                                           '--import=' + ctfpfile)
                #print cmd
                subprocess.call(cmd, shell=True)
            else:
                print ' >>>  Group ', name, '  skipped.'

        cmd = "{} {} {}".format("rm -f", grpfile, ctfpfile)
        subprocess.call(cmd, shell=True)

    elif options.scale > 0.0:
        from utilities import read_text_row, write_text_row
        scale = options.scale
        nargs = len(args)
        if nargs != 2:
            print "Please provide names of input and output file!"
            return
        p = read_text_row(args[0])
        for i in xrange(len(p)):
            p[i][3] /= scale
            p[i][4] /= scale
        write_text_row(p, args[1])

    elif options.adaptive_mask:
        from utilities import get_im
        from morphology import adaptive_mask
        nsigma = options.nsigma
        ndilation = options.ndilation
        kernel_size = options.kernel_size
        gauss_standard_dev = options.gauss_standard_dev
        nargs = len(args)
        if nargs > 2:
            print "Too many inputs are given, try again!"
            return
        else:
            inputvol = get_im(args[0])
            input_path, input_file_name = os.path.split(args[0])
            input_file_name_root, ext = os.path.splitext(input_file_name)
            if nargs == 2: mask_file_name = args[1]
            else:
                mask_file_name = "adaptive_mask_for" + input_file_name_root + ".hdf"  # Only hdf file is output.
            mask3d = adaptive_mask(inputvol, nsigma, ndilation, kernel_size,
                                   gauss_standard_dev)
            mask3d.write_image(mask_file_name)

    else:
        ERROR("Please provide option name", "sxprocess.py", 1)