Exemplo n.º 1
0
	def __new__(cls, file_name):
		if not file_exists(file_name): raise
		splt = file_name.split(".")
		if len(file_name) > 4 and file_name[:4] == "bdb:": pass
		elif EMUtil.get_image_ext_type(splt[-1]) == EMUtil.ImageType.IMAGE_UNKNOWN:
			raise
		
		if splt[-1] in ["img","hed"]: return EMImagicTmpFileHandle(file_name)
		elif file_name[:4] == "bdb:": return EMDBTmpFileHandle(file_name)
		else: return EMGeneralTmpFileHandle(file_name)
Exemplo n.º 2
0
	def __new__(cls, file_name):
		if not file_exists(file_name): raise
		splt = file_name.split(".")
		if len(file_name) > 4 and file_name[:4] == "bdb:": pass
		elif EMUtil.get_image_ext_type(splt[-1]) == EMUtil.ImageType.IMAGE_UNKNOWN:
			raise
		
		if splt[-1] in ["img","hed"]: return EMImagicTmpFileHandle(file_name)
		elif file_name[:4] == "bdb:": return EMDBTmpFileHandle(file_name)
		else: return EMGeneralTmpFileHandle(file_name)
Exemplo n.º 3
0
def main():
	progname = os.path.basename(sys.argv[0])
	usage = """%prog [options] <image1> <image2> <image3> <image4> ....
	
	WARNING: Experimental program. Contact [email protected] for more info.
	
	Currently only supports bootstrapping an initial probe doing all versus all alignment of the input images

"""

	parser = EMArgumentParser(usage=usage,version=EMANVERSION)

	parser.add_argument("--align",type=str,help="The aligner and its parameters. e.g. --align=rt.3d.grid:ralt=180:dalt=10:dphi=10:rphi=180:search=5", default="rt.3d.grid")
	parser.add_argument("--aligncmp",type=str,help="The comparator used for determing the best initial alignment", default="dot.tomo:threshold=0")
	parser.add_argument("--cmp",type=str,help="The comparator used to obtain the final similarity", default="dot.tomo:threshold=0")
	parser.add_argument("--ralign",type=str,help="This is the second stage aligner used to refine the first alignment. This is usually the \'refine\' aligner.", default=None)
	parser.add_argument("--raligncmp",type=str,help="The comparator used for determing the refined alignment", default="dot.tomo:threshold=0")
	parser.add_argument("--bootstrap",action="store_true",default=True,help="Boot strap alignment")
	parser.add_argument("--output",type=str,default="e2tomoave.hdf",help="The output image which will store the results matrix")
	parser.add_argument("--parallel",type=str,default=None,help="Use parallelism")
	parser.add_argument("--path", default=None, type=str,help="The name of a directory where results are placed. If unspecified will generate one automatically of type tomo_ave_??.")
	parser.add_argument("--nsoln", default=1, type=int,help="If supplied and greater than 1, the nsoln-best alignments will be written to a text file. This is useful for debug but may be left unspecified")
	parser.add_argument("--dbls",type=str,help="data base list storage, used by the workflow. You can ignore this argument.",default=None)
	parser.add_argument("--shrink",type=int,help="Shrink the data as part of the alignment - for speed purposes but at the potential loss of accuracy",default=None)
	parser.add_argument("--filter",type=str,help="The name and parameters of an EMAN2 processor. Will be applied prior to shrinking.",default=None)
	parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
	parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higner number means higher level of verboseness")
	if EMUtil.cuda_available():
		parser.add_argument("--cuda",action="store_true",help="GPU acceleration using CUDA. Experimental", default=False)

	(options, args) = parser.parse_args()
	
	print options.shrink
	
	error_messages = check_options(options,args)
	if len(error_messages) != 0:
		msg = "\n"
		for error in error_messages:
			msg += error +"\n"
		parser.error(msg)
		exit(1)
	
	logger=E2init(sys.argv,options.ppid)
	
	if options.bootstrap:
		module = EMBootStrappedAverages(args,options,logger)
		module.execute()
	else:
		print "boot strap only supported technique"
	E2end(logger)
Exemplo n.º 4
0
	def __init__(self,files,options,logger):
		'''
		@param options - the options returned by the call to (options, args) = parser.parse_args() 
		@param args - a list of image names - args is that which is returned by the call to (options, args) = parser.parse_args()
		'''
		self.options = options
		self.files = files
		self.logger = logger
		self.images = None
		self.using_cuda = EMUtil.cuda_available() and options.cuda
		self.current_files = None
		self.jobs_completed = 0
		self.total_jobs = 0 # you can never know how many jobs will need to be completed
		self.current_progress = 0.0
Exemplo n.º 5
0
    def __init__(self, files, options, logger):
        '''
		@param options - the options returned by the call to (options, args) = parser.parse_args() 
		@param args - a list of image names - args is that which is returned by the call to (options, args) = parser.parse_args()
		'''
        self.options = options
        self.files = files
        self.logger = logger
        self.images = None
        self.using_cuda = EMUtil.cuda_available() and options.cuda
        self.current_files = None
        self.jobs_completed = 0
        self.total_jobs = 0  # you can never know how many jobs will need to be completed
        self.current_progress = 0.0
def stackToHDF(infile,outfile,apix,pinfo=None):
        """
        convert stack to an hdf stack
        pinfo contains CTF information from apFrealign.getPerParticleCTF
        pinfo may also contain helical information
        """
        from utilities import generate_ctf

        a = EMAN2.EMData()
        imn = EMUtil.get_image_count(infile)

        if pinfo is not None:
                if len(pinfo) != imn:
                        apDisplay.printError("insufficient particle info for stack")

        # output must end with hdf
        outf,ext = os.path.splitext(outfile)
        if ext != '.hdf':
                outstack = outf+".hdf"

        apDisplay.printMsg("Generating '%s' with %i particles"%(outstack,imn))

        for i in xrange(imn):
                a.read_image(infile,i)
                a.set_attr_dict({'active':1})
                t2 = EMAN2.Transform({"type":"spider","phi":0,"theta":0,"psi":0})
                a.set_attr("xform.projection", t2)
                a.set_attr("apix_x",apix)

                if pinfo is not None:
                        pdata = pinfo[i+1]
                        df1 = pdata['df1']
                        df2 = pdata['df2']
                        astig = pdata['angastig']
                        kv = pdata['kev']
                        cs = pdata['cs']
                        ampc = pdata['ampc']*100

                        # save CTF dict (bfactor is hard coded to 0.0)
                        df=(float(df1)+float(df2))/2
                        ctfgen = generate_ctf([df,cs,kv,apix,0.0,ampc])
                        a.set_attr("ctf",ctfgen)

                        # if helical info is present
                        if pdata['hnum'] is not None:
                                a.set_attr("h_angle",pdata['hangle'])
                a.write_image(outstack,i)

        return outstack
Exemplo n.º 7
0
	def __init__(self,file_name):
		'''
		@param file_name the name of the file that which is being overwritten
		@exception raised if the file_name does not exist on the file system
		@exception raised if the file_name has a unrecognized (or no) type
		'''
		if not os.path.exists(file_name): raise
		splt = file_name.split(".")
		if len(splt) < 2: raise
		if EMUtil.get_image_ext_type(splt[-1]) == EMUtil.ImageType.IMAGE_UNKNOWN:
			raise
		self.__file_ext = "."+splt[-1]
		
		EMTmpFileHandleBase.__init__(self)
		self.__file_name = get_file_tag(file_name)
		self.__full_file_name = file_name
		self.__tmp_file_name = None
		self.__establish_tmp_file_name()
Exemplo n.º 8
0
	def __init__(self,file_name):
		'''
		@param file_name the name of the file that which is being overwritten
		@exception raised if the file_name does not exist on the file system
		@exception raised if the file_name has a unrecognized (or no) type
		'''
		if not os.path.exists(file_name): raise
		splt = file_name.split(".")
		if len(splt) < 2: raise
		if EMUtil.get_image_ext_type(splt[-1]) == EMUtil.ImageType.IMAGE_UNKNOWN:
			raise
		self.__file_ext = "."+splt[-1]
		
		EMTmpFileHandleBase.__init__(self)
		self.__file_name = base_name(file_name)
		self.__full_file_name = file_name
		self.__tmp_file_name = None
		self.__establish_tmp_file_name()
Exemplo n.º 9
0
def main(args):
    if os.path.exists(args.output):
        os.remove(args.output)
    for fn in args.input:
        if fn.endswith(".star"):
            star = parse_star(fn, keep_index=False)
            for p in star["rlnImageName"]:
                stack = p.split("@")[1]
                idx = int(p.split("@")[0]) - 1
                try:
                    img = EMData(stack, idx)
                    img.append_image(args.output)
                except Exception:
                    print("Error at %s" % p)
        else:
            n = EMUtil.get_image_count(fn)
            for i in range(n):
                img = EMData(fn, i)
                img.append_image(args.output)
    return 0
Exemplo n.º 10
0
def main():
	arglist = []
	for arg in sys.argv:
		arglist.append( arg )

	progname = os.path.basename( arglist[0] )
	usage = progname + " prj_stack volume [begin end step] --CTF --npad=ntimes_padding --list=file --group=ID --snr=SNR --sym=symmetry --verbose=(0|1) --xysize --MPI"
	parser = OptionParser(usage, version=SPARXVERSION)

	parser.add_option("--CTF",     action="store_true", default=False, help="apply CTF correction")
	parser.add_option("--snr",     type="float",	    default=1.0,   help="Signal-to-Noise Ratio" )
	parser.add_option("--sym",     type="string",	    default="c1",  help="symmetry" )
	parser.add_option("--list",    type="string",                      help="file with list of images to be used in the first column" )
	parser.add_option("--group",   type="int",          default=-1,    help="perform reconstruction using images for a given group number (group is attribute in the header)" )
	parser.add_option("--MPI",     action="store_true", default=False, help="use MPI version ")
	parser.add_option("--npad",    type="int",	        default=2,     help="number of times padding (default 2)" )
	parser.add_option("--verbose", type="int",          default=0,     help="verbose level: 0 no verbose, 1 verbose" )
	parser.add_option("--xysize",  type="int",	        default=-1,    help="user expected size at xy direction" )
	parser.add_option("--zsize",   type="int",	        default=-1,    help="user expected size at z direction" )
	parser.add_option("--smearstep",   type="float",	default=0.0,   help="Rotational smear step (default 0.0, no smear)" )

	(options,args) = parser.parse_args(arglist[1:])


	if options.MPI:
		from mpi import mpi_init
		sys.argv = mpi_init(len(sys.argv), sys.argv)

	if global_def.CACHE_DISABLE:
		from utilities import disable_bdb_cache
		disable_bdb_cache()

	if len(args) == 2:
		prj_stack = args[0]
		vol_stack = args[1]
		nimage = EMUtil.get_image_count( prj_stack )
		pid_list = range(0, nimage)
	elif len(args) == 5:
		prj_stack = args[0]
		vol_stack = args[1]
		begin = atoi( args[2] )
		end   = atoi( args[3] )
		step  = atoi( args[4] )
		pid_list = range(begin, end, step)
	else:
		ERROR("incomplete list of arguments","recon3d_n",1)
		exit()

	if(options.list and options.group > -1):
		ERROR("options group and list cannot be used together","recon3d_n",1)
		sys.exit()

	from applications import recons3d_n

	global_def.BATCH = True
	recons3d_n(prj_stack, pid_list, vol_stack, options.CTF, options.snr, 1, options.npad,\
		 options.sym, options.list, options.group, options.verbose, options.MPI,options.xysize, options.zsize, options.smearstep)
	global_def.BATCH = False

	if options.MPI:
		from mpi import mpi_finalize
		mpi_finalize()
Exemplo n.º 11
0
def main():

	def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
		# the final ali2d parameters already combine shifts operation first and rotation operation second for parameters converted from 3D
		if mirror:
			m = 1
			alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 540.0-psi, 0, 0, 1.0)
		else:
			m = 0
			alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 360.0-psi, 0, 0, 1.0)
		return  alpha, sx, sy, m
	
	progname = os.path.basename(sys.argv[0])
	usage = progname + " prj_stack  --ave2D= --var2D=  --ave3D= --var3D= --img_per_grp= --fl=  --aa=   --sym=symmetry --CTF"
	parser = OptionParser(usage, version=SPARXVERSION)
	
	parser.add_option("--output_dir",   type="string"	   ,	default="./",				    help="Output directory")
	parser.add_option("--ave2D",		type="string"	   ,	default=False,				help="Write to the disk a stack of 2D averages")
	parser.add_option("--var2D",		type="string"	   ,	default=False,				help="Write to the disk a stack of 2D variances")
	parser.add_option("--ave3D",		type="string"	   ,	default=False,				help="Write to the disk reconstructed 3D average")
	parser.add_option("--var3D",		type="string"	   ,	default=False,				help="Compute 3D variability (time consuming!)")
	parser.add_option("--img_per_grp",	type="int"         ,	default=100,	     	    help="Number of neighbouring projections.(Default is 100)")
	parser.add_option("--no_norm",		action="store_true",	default=False,				help="Do not use normalization.(Default is to apply normalization)")
	#parser.add_option("--radius", 	    type="int"         ,	default=-1   ,				help="radius for 3D variability" )
	parser.add_option("--npad",			type="int"         ,	default=2    ,				help="Number of time to pad the original images.(Default is 2 times padding)")
	parser.add_option("--sym" , 		type="string"      ,	default="c1",				help="Symmetry. (Default is no symmetry)")
	parser.add_option("--fl",			type="float"       ,	default=0.0,				help="Low pass filter cutoff in absolute frequency (0.0 - 0.5) and is applied to decimated images. (Default - no filtration)")
	parser.add_option("--aa",			type="float"       ,	default=0.02 ,				help="Fall off of the filter. Use default value if user has no clue about falloff (Default value is 0.02)")
	parser.add_option("--CTF",			action="store_true",	default=False,				help="Use CFT correction.(Default is no CTF correction)")
	#parser.add_option("--MPI" , 		action="store_true",	default=False,				help="use MPI version")
	#parser.add_option("--radiuspca", 	type="int"         ,	default=-1   ,				help="radius for PCA" )
	#parser.add_option("--iter", 		type="int"         ,	default=40   ,				help="maximum number of iterations (stop criterion of reconstruction process)" )
	#parser.add_option("--abs", 		type="float"   ,        default=0.0  ,				help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
	#parser.add_option("--squ", 		type="float"   ,	    default=0.0  ,				help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
	parser.add_option("--VAR" , 		action="store_true",	default=False,				help="Stack of input consists of 2D variances (Default False)")
	parser.add_option("--decimate",     type  ="float",         default=0.25,               help="Image decimate rate, a number less than 1. (Default is 0.25)")
	parser.add_option("--window",       type  ="int",           default=0,                  help="Target image size relative to original image size. (Default value is zero.)")
	#parser.add_option("--SND",			action="store_true",	default=False,				help="compute squared normalized differences (Default False)")
	#parser.add_option("--nvec",			type="int"         ,	default=0    ,				help="Number of eigenvectors, (Default = 0 meaning no PCA calculated)")
	parser.add_option("--symmetrize",	action="store_true",	default=False,				help="Prepare input stack for handling symmetry (Default False)")
	parser.add_option("--overhead",     type  ="float",         default=0.5,                help="python overhead per CPU.")

	(options,args) = parser.parse_args()
	#####
	from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD
	from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
	#from mpi import *
	from applications   import MPI_start_end
	from reconstruction import recons3d_em, recons3d_em_MPI
	from reconstruction	import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
	from utilities      import print_begin_msg, print_end_msg, print_msg
	from utilities      import read_text_row, get_image, get_im, wrap_mpi_send, wrap_mpi_recv
	from utilities      import bcast_EMData_to_all, bcast_number_to_all
	from utilities      import get_symt

	#  This is code for handling symmetries by the above program.  To be incorporated. PAP 01/27/2015

	from EMAN2db import db_open_dict

	# Set up global variables related to bdb cache 
	if global_def.CACHE_DISABLE:
		from utilities import disable_bdb_cache
		disable_bdb_cache()
	
	# Set up global variables related to ERROR function
	global_def.BATCH = True
	
	# detect if program is running under MPI
	RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ
	if RUNNING_UNDER_MPI: global_def.MPI = True
	if options.output_dir =="./": current_output_dir = os.path.abspath(options.output_dir)
	else: current_output_dir = options.output_dir
	if options.symmetrize :
		if RUNNING_UNDER_MPI:
			try:
				sys.argv = mpi_init(len(sys.argv), sys.argv)
				try:	
					number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
					if( number_of_proc > 1 ):
						ERROR("Cannot use more than one CPU for symmetry preparation","sx3dvariability",1)
				except:
					pass
			except:
				pass
		if not os.path.exists(current_output_dir): os.mkdir(current_output_dir)
		
		#  Input
		#instack = "Clean_NORM_CTF_start_wparams.hdf"
		#instack = "bdb:data"
		
		
		from logger import Logger,BaseLogger_Files
		if os.path.exists(os.path.join(current_output_dir, "log.txt")): os.remove(os.path.join(current_output_dir, "log.txt"))
		log_main=Logger(BaseLogger_Files())
		log_main.prefix = os.path.join(current_output_dir, "./")
		
		instack = args[0]
		sym = options.sym.lower()
		if( sym == "c1" ):
			ERROR("There is no need to symmetrize stack for C1 symmetry","sx3dvariability",1)
		
		line =""
		for a in sys.argv:
			line +=" "+a
		log_main.add(line)
	
		if(instack[:4] !="bdb:"):
			#if output_dir =="./": stack = "bdb:data"
			stack = "bdb:"+current_output_dir+"/data"
			delete_bdb(stack)
			junk = cmdexecute("sxcpy.py  "+instack+"  "+stack)
		else: stack = instack
		
		qt = EMUtil.get_all_attributes(stack,'xform.projection')

		na = len(qt)
		ts = get_symt(sym)
		ks = len(ts)
		angsa = [None]*na
		
		for k in range(ks):
			#Qfile = "Q%1d"%k
			#if options.output_dir!="./": Qfile = os.path.join(options.output_dir,"Q%1d"%k)
			Qfile = os.path.join(current_output_dir, "Q%1d"%k)
			#delete_bdb("bdb:Q%1d"%k)
			delete_bdb("bdb:"+Qfile)
			#junk = cmdexecute("e2bdb.py  "+stack+"  --makevstack=bdb:Q%1d"%k)
			junk = cmdexecute("e2bdb.py  "+stack+"  --makevstack=bdb:"+Qfile)
			#DB = db_open_dict("bdb:Q%1d"%k)
			DB = db_open_dict("bdb:"+Qfile)
			for i in range(na):
				ut = qt[i]*ts[k]
				DB.set_attr(i, "xform.projection", ut)
				#bt = ut.get_params("spider")
				#angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
			#write_text_row(angsa, 'ptsma%1d.txt'%k)
			#junk = cmdexecute("e2bdb.py  "+stack+"  --makevstack=bdb:Q%1d"%k)
			#junk = cmdexecute("sxheader.py  bdb:Q%1d  --params=xform.projection  --import=ptsma%1d.txt"%(k,k))
			DB.close()
		#if options.output_dir =="./": delete_bdb("bdb:sdata")
		delete_bdb("bdb:" + current_output_dir + "/"+"sdata")
		#junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
		sdata = "bdb:"+current_output_dir+"/"+"sdata"
		print(sdata)
		junk = cmdexecute("e2bdb.py   " + current_output_dir +"  --makevstack="+sdata +" --filt=Q")
		#junk = cmdexecute("ls  EMAN2DB/sdata*")
		#a = get_im("bdb:sdata")
		a = get_im(sdata)
		a.set_attr("variabilitysymmetry",sym)
		#a.write_image("bdb:sdata")
		a.write_image(sdata)

	else:

		from fundamentals import window2d
		sys.argv       = mpi_init(len(sys.argv), sys.argv)
		myid           = mpi_comm_rank(MPI_COMM_WORLD)
		number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
		main_node      = 0
		shared_comm  = mpi_comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED,  0, MPI_INFO_NULL)
		myid_on_node = mpi_comm_rank(shared_comm)
		no_of_processes_per_group = mpi_comm_size(shared_comm)
		masters_from_groups_vs_everything_else_comm = mpi_comm_split(MPI_COMM_WORLD, main_node == myid_on_node, myid_on_node)
		color, no_of_groups, balanced_processor_load_on_nodes = get_colors_and_subsets(main_node, MPI_COMM_WORLD, myid, \
		    shared_comm, myid_on_node, masters_from_groups_vs_everything_else_comm)
		overhead_loading = options.overhead*number_of_proc
		#memory_per_node  = options.memory_per_node
		#if memory_per_node == -1.: memory_per_node = 2.*no_of_processes_per_group
		keepgoing = 1
		
		current_window   = options.window
		current_decimate = options.decimate
		
		if len(args) == 1: stack = args[0]
		else:
			print(( "usage: " + usage))
			print(( "Please run '" + progname + " -h' for detailed options"))
			return 1

		t0 = time()	
		# obsolete flags
		options.MPI  = True
		#options.nvec = 0
		options.radiuspca = -1
		options.iter = 40
		options.abs  = 0.0
		options.squ  = 0.0

		if options.fl > 0.0 and options.aa == 0.0:
			ERROR("Fall off has to be given for the low-pass filter", "sx3dvariability", 1, myid)
			
		#if options.VAR and options.SND:
		#	ERROR("Only one of var and SND can be set!", "sx3dvariability", myid)
			
		if options.VAR and (options.ave2D or options.ave3D or options.var2D): 
			ERROR("When VAR is set, the program cannot output ave2D, ave3D or var2D", "sx3dvariability", 1, myid)
			
		#if options.SND and (options.ave2D or options.ave3D):
		#	ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid)
		
		#if options.nvec > 0 :
		#	ERROR("PCA option not implemented", "sx3dvariability", 1, myid)
			
		#if options.nvec > 0 and options.ave3D == None:
		#	ERROR("When doing PCA analysis, one must set ave3D", "sx3dvariability", 1, myid)
		
		if current_decimate>1.0 or current_decimate<0.0:
			ERROR("Decimate rate should be a value between 0.0 and 1.0", "sx3dvariability", 1, myid)
		
		if current_window < 0.0:
			ERROR("Target window size should be always larger than zero", "sx3dvariability", 1, myid)
			
		if myid == main_node:
			img  = get_image(stack, 0)
			nx   = img.get_xsize()
			ny   = img.get_ysize()
			if(min(nx, ny) < current_window):   keepgoing = 0
		keepgoing = bcast_number_to_all(keepgoing, main_node, MPI_COMM_WORLD)
		if keepgoing == 0: ERROR("The target window size cannot be larger than the size of decimated image", "sx3dvariability", 1, myid)

		import string
		options.sym = options.sym.lower()
		# if global_def.CACHE_DISABLE:
		# 	from utilities import disable_bdb_cache
		# 	disable_bdb_cache()
		# global_def.BATCH = True
		
		if myid == main_node:
			if not os.path.exists(current_output_dir): os.mkdir(current_output_dir)# Never delete output_dir in the program!
	
		img_per_grp = options.img_per_grp
		#nvec        = options.nvec
		radiuspca   = options.radiuspca
		from logger import Logger,BaseLogger_Files
		#if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt"))
		log_main=Logger(BaseLogger_Files())
		log_main.prefix = os.path.join(current_output_dir, "./")

		if myid == main_node:
			line = ""
			for a in sys.argv: line +=" "+a
			log_main.add(line)
			log_main.add("-------->>>Settings given by all options<<<-------")
			log_main.add("Symmetry             : %s"%options.sym)
			log_main.add("Input stack          : %s"%stack)
			log_main.add("Output_dir           : %s"%current_output_dir)
			
			if options.ave3D: log_main.add("Ave3d                : %s"%options.ave3D)
			if options.var3D: log_main.add("Var3d                : %s"%options.var3D)
			if options.ave2D: log_main.add("Ave2D                : %s"%options.ave2D)
			if options.var2D: log_main.add("Var2D                : %s"%options.var2D)
			if options.VAR:   log_main.add("VAR                  : True")
			else:             log_main.add("VAR                  : False")
			if options.CTF:   log_main.add("CTF correction       : True  ")
			else:             log_main.add("CTF correction       : False ")
			
			log_main.add("Image per group      : %5d"%options.img_per_grp)
			log_main.add("Image decimate rate  : %4.3f"%current_decimate)
			log_main.add("Low pass filter      : %4.3f"%options.fl)
			current_fl = options.fl
			if current_fl == 0.0: current_fl = 0.5
			log_main.add("Current low pass filter is equivalent to cutoff frequency %4.3f for original image size"%round((current_fl*current_decimate),3))
			log_main.add("Window size          : %5d "%current_window)
			log_main.add("sx3dvariability begins")
	
		symbaselen = 0
		if myid == main_node:
			nima = EMUtil.get_image_count(stack)
			img  = get_image(stack)
			nx   = img.get_xsize()
			ny   = img.get_ysize()
			nnxo = nx
			nnyo = ny
			if options.sym != "c1" :
				imgdata = get_im(stack)
				try:
					i = imgdata.get_attr("variabilitysymmetry").lower()
					if(i != options.sym):
						ERROR("The symmetry provided does not agree with the symmetry of the input stack", "sx3dvariability", 1, myid)
				except:
					ERROR("Input stack is not prepared for symmetry, please follow instructions", "sx3dvariability", 1, myid)
				from utilities import get_symt
				i = len(get_symt(options.sym))
				if((nima/i)*i != nima):
					ERROR("The length of the input stack is incorrect for symmetry processing", "sx3dvariability", 1, myid)
				symbaselen = nima/i
			else:  symbaselen = nima
		else:
			nima = 0
			nx = 0
			ny = 0
			nnxo = 0
			nnyo = 0
		nima    = bcast_number_to_all(nima)
		nx      = bcast_number_to_all(nx)
		ny      = bcast_number_to_all(ny)
		nnxo    = bcast_number_to_all(nnxo)
		nnyo    = bcast_number_to_all(nnyo)
		if current_window > max(nx, ny):
			ERROR("Window size is larger than the original image size", "sx3dvariability", 1)
		
		if current_decimate == 1.:
			if current_window !=0:
				nx = current_window
				ny = current_window
		else:
			if current_window == 0:
				nx = int(nx*current_decimate+0.5)
				ny = int(ny*current_decimate+0.5)
			else:
				nx = int(current_window*current_decimate+0.5)
				ny = nx
		symbaselen = bcast_number_to_all(symbaselen)
		
		# check FFT prime number
		from fundamentals import smallprime
		is_fft_friendly = (nx == smallprime(nx))
		
		if not is_fft_friendly:
			if myid == main_node:
				log_main.add("The target image size is not a product of small prime numbers")
				log_main.add("Program adjusts the input settings!")
			### two cases
			if current_decimate == 1.:
				nx = smallprime(nx)
				ny = nx
				current_window = nx # update
				if myid == main_node:
					log_main.add("The window size is updated to %d."%current_window)
			else:
				if current_window == 0:
					nx = smallprime(int(nx*current_decimate+0.5))
					current_decimate = float(nx)/nnxo
					ny = nx
					if (myid == main_node):
						log_main.add("The decimate rate is updated to %f."%current_decimate)
				else:
					nx = smallprime(int(current_window*current_decimate+0.5))
					ny = nx
					current_window = int(nx/current_decimate+0.5)
					if (myid == main_node):
						log_main.add("The window size is updated to %d."%current_window)
						
		if myid == main_node:
			log_main.add("The target image size is %d"%nx)
						
		if radiuspca == -1: radiuspca = nx/2-2
		if myid == main_node: log_main.add("%-70s:  %d\n"%("Number of projection", nima))
		img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
		
		"""
		if options.SND:
			from projection		import prep_vol, prgs
			from statistics		import im_diff
			from utilities		import get_im, model_circle, get_params_proj, set_params_proj
			from utilities		import get_ctf, generate_ctf
			from filter			import filt_ctf
		
			imgdata = EMData.read_images(stack, range(img_begin, img_end))

			if options.CTF:
				vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
			else:
				vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)

			bcast_EMData_to_all(vol, myid)
			volft, kb = prep_vol(vol)

			mask = model_circle(nx/2-2, nx, ny)
			varList = []
			for i in xrange(img_begin, img_end):
				phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
				ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
				if options.CTF:
					ctf_params = get_ctf(imgdata[i-img_begin])
					ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
				diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
				diff2 = diff*diff
				set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
				varList.append(diff2)
			mpi_barrier(MPI_COMM_WORLD)
		"""
		
		if options.VAR: # 2D variance images have no shifts
			#varList   = EMData.read_images(stack, range(img_begin, img_end))
			from EMAN2 import Region
			for index_of_particle in range(img_begin,img_end):
				image = get_im(stack, index_of_proj)
				if current_window > 0: varList.append(fdecimate(window2d(image,current_window,current_window), nx,ny))
				else:   varList.append(fdecimate(image, nx,ny))
				
		else:
			from utilities		import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
			from utilities		import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
			from utilities		import model_blank, nearest_proj, model_circle, write_text_row, wrap_mpi_gatherv
			from applications	import pca
			from statistics		import avgvar, avgvar_ctf, ccc
			from filter		    import filt_tanl
			from morphology		import threshold, square_root
			from projection 	import project, prep_vol, prgs
			from sets		    import Set
			from utilities      import wrap_mpi_recv, wrap_mpi_bcast, wrap_mpi_send
			import numpy as np
			if myid == main_node:
				t1          = time()
				proj_angles = []
				aveList     = []
				tab = EMUtil.get_all_attributes(stack, 'xform.projection')	
				for i in range(nima):
					t     = tab[i].get_params('spider')
					phi   = t['phi']
					theta = t['theta']
					psi   = t['psi']
					x     = theta
					if x > 90.0: x = 180.0 - x
					x = x*10000+psi
					proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
				t2 = time()
				log_main.add( "%-70s:  %d\n"%("Number of neighboring projections", img_per_grp))
				log_main.add("...... Finding neighboring projections\n")
				log_main.add( "Number of images per group: %d"%img_per_grp)
				log_main.add( "Now grouping projections")
				proj_angles.sort()
				proj_angles_list = np.full((nima, 4), 0.0, dtype=np.float32)	
				for i in range(nima):
					proj_angles_list[i][0] = proj_angles[i][1]
					proj_angles_list[i][1] = proj_angles[i][2]
					proj_angles_list[i][2] = proj_angles[i][3]
					proj_angles_list[i][3] = proj_angles[i][4]
			else: proj_angles_list = 0
			proj_angles_list = wrap_mpi_bcast(proj_angles_list, main_node, MPI_COMM_WORLD)
			proj_angles      = []
			for i in range(nima):
				proj_angles.append([proj_angles_list[i][0], proj_angles_list[i][1], proj_angles_list[i][2], int(proj_angles_list[i][3])])
			del proj_angles_list
			proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp, range(img_begin, img_end))
			all_proj = Set()
			for im in proj_list:
				for jm in im:
					all_proj.add(proj_angles[jm][3])
			all_proj = list(all_proj)
			index = {}
			for i in range(len(all_proj)): index[all_proj[i]] = i
			mpi_barrier(MPI_COMM_WORLD)
			if myid == main_node:
				log_main.add("%-70s:  %.2f\n"%("Finding neighboring projections lasted [s]", time()-t2))
				log_main.add("%-70s:  %d\n"%("Number of groups processed on the main node", len(proj_list)))
				log_main.add("Grouping projections took:  %12.1f [m]"%((time()-t2)/60.))
				log_main.add("Number of groups on main node: ", len(proj_list))
			mpi_barrier(MPI_COMM_WORLD)

			if myid == main_node:
				log_main.add("...... Calculating the stack of 2D variances \n")
			# Memory estimation. There are two memory consumption peaks
			# peak 1. Compute ave, var; 
			# peak 2. Var volume reconstruction;
			# proj_params = [0.0]*(nima*5)
			aveList = []
			varList = []				
			#if nvec > 0: eigList = [[] for i in range(nvec)]
			dnumber   = len(all_proj)# all neighborhood set for assigned to myid
			pnumber   = len(proj_list)*2. + img_per_grp # aveList and varList 
			tnumber   = dnumber+pnumber
			vol_size2 = nx**3*4.*8/1.e9
			vol_size1 = 2.*nnxo**3*4.*8/1.e9
			proj_size         = nnxo*nnyo*len(proj_list)*4.*2./1.e9 # both aveList and varList
			orig_data_size    = nnxo*nnyo*4.*tnumber/1.e9
			reduced_data_size = nx*nx*4.*tnumber/1.e9
			full_data         = np.full((number_of_proc, 2), -1., dtype=np.float16)
			full_data[myid]   = orig_data_size, reduced_data_size
			if myid != main_node: wrap_mpi_send(full_data, main_node, MPI_COMM_WORLD)
			if myid == main_node:
				for iproc in range(number_of_proc):
					if iproc != main_node:
						dummy = wrap_mpi_recv(iproc, MPI_COMM_WORLD)
						full_data[np.where(dummy>-1)] = dummy[np.where(dummy>-1)]
				del dummy
			mpi_barrier(MPI_COMM_WORLD)
			full_data = wrap_mpi_bcast(full_data, main_node, MPI_COMM_WORLD)
			# find the CPU with heaviest load
			minindx         = np.argsort(full_data, 0)
			heavy_load_myid = minindx[-1][1]
			total_mem       = sum(full_data)
			if myid == main_node:
				if current_window == 0:
					log_main.add("Nx:   current image size = %d. Decimated by %f from %d"%(nx, current_decimate, nnxo))
				else:
					log_main.add("Nx:   current image size = %d. Windowed to %d, and decimated by %f from %d"%(nx, current_window, current_decimate, nnxo))
				log_main.add("Nproj:       number of particle images.")
				log_main.add("Navg:        number of 2D average images.")
				log_main.add("Nvar:        number of 2D variance images.")
				log_main.add("Img_per_grp: user defined image per group for averaging = %d"%img_per_grp)
				log_main.add("Overhead:    total python overhead memory consumption   = %f"%overhead_loading)
				log_main.add("Total memory) = 4.0*nx^2*(nproj + navg +nvar+ img_per_grp)/1.0e9 + overhead: %12.3f [GB]"%\
				   (total_mem[1] + overhead_loading))
			del full_data
			mpi_barrier(MPI_COMM_WORLD)
			if myid == heavy_load_myid:
				log_main.add("Begin reading and preprocessing images on processor. Wait... ")
				ttt = time()
			#imgdata = EMData.read_images(stack, all_proj)			
			imgdata = [ None for im in range(len(all_proj))]
			for index_of_proj in range(len(all_proj)):
				#image = get_im(stack, all_proj[index_of_proj])
				if( current_window > 0): imgdata[index_of_proj] = fdecimate(window2d(get_im(stack, all_proj[index_of_proj]),current_window,current_window), nx, ny)
				else:                    imgdata[index_of_proj] = fdecimate(get_im(stack, all_proj[index_of_proj]), nx, ny)
				
				if (current_decimate> 0.0 and options.CTF):
					ctf = imgdata[index_of_proj].get_attr("ctf")
					ctf.apix = ctf.apix/current_decimate
					imgdata[index_of_proj].set_attr("ctf", ctf)
					
				if myid == heavy_load_myid and index_of_proj%100 == 0:
					log_main.add(" ...... %6.2f%% "%(index_of_proj/float(len(all_proj))*100.))
			mpi_barrier(MPI_COMM_WORLD)
			if myid == heavy_load_myid:
				log_main.add("All_proj preprocessing cost %7.2f m"%((time()-ttt)/60.))
				log_main.add("Wait untill reading on all CPUs done...")
			'''	
			imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
			if options.fl > 0.0:
				for k in xrange(len(imgdata2)):
					imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
			if options.CTF:
				vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
			else:
				vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
			if myid == main_node:
				vol.write_image("vol_ctf.hdf")
				print_msg("Writing to the disk volume reconstructed from averages as		:  %s\n"%("vol_ctf.hdf"))
			del vol, imgdata2
			mpi_barrier(MPI_COMM_WORLD)
			'''
			from applications import prepare_2d_forPCA
			from utilities    import model_blank
			from EMAN2        import Transform
			if not options.no_norm: 
				mask = model_circle(nx/2-2, nx, nx)
			if options.CTF: 
				from utilities import pad
				from filter import filt_ctf
			from filter import filt_tanl
			if myid == heavy_load_myid:
				log_main.add("Start computing 2D aveList and varList. Wait...")
				ttt = time()
			inner=nx//2-4
			outer=inner+2
			xform_proj_for_2D = [ None for i in range(len(proj_list))]
			for i in range(len(proj_list)):
				ki = proj_angles[proj_list[i][0]][3]
				if ki >= symbaselen:  continue
				mi = index[ki]
				dpar = Util.get_transform_params(imgdata[mi], "xform.projection", "spider")
				phiM, thetaM, psiM, s2xM, s2yM  = dpar["phi"],dpar["theta"],dpar["psi"],-dpar["tx"]*current_decimate,-dpar["ty"]*current_decimate
				grp_imgdata = []
				for j in range(img_per_grp):
					mj = index[proj_angles[proj_list[i][j]][3]]
					cpar = Util.get_transform_params(imgdata[mj], "xform.projection", "spider")
					alpha, sx, sy, mirror = params_3D_2D_NEW(cpar["phi"], cpar["theta"],cpar["psi"], -cpar["tx"]*current_decimate, -cpar["ty"]*current_decimate, mirror_list[i][j])
					if thetaM <= 90:
						if mirror == 0:  alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, phiM - cpar["phi"], 0.0, 0.0, 1.0)
						else:            alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, 180-(phiM - cpar["phi"]), 0.0, 0.0, 1.0)
					else:
						if mirror == 0:  alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(phiM- cpar["phi"]), 0.0, 0.0, 1.0)
						else:            alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(180-(phiM - cpar["phi"])), 0.0, 0.0, 1.0)
					imgdata[mj].set_attr("xform.align2d", Transform({"type":"2D","alpha":alpha,"tx":sx,"ty":sy,"mirror":mirror,"scale":1.0}))
					grp_imgdata.append(imgdata[mj])
				if not options.no_norm:
					for k in range(img_per_grp):
						ave, std, minn, maxx = Util.infomask(grp_imgdata[k], mask, False)
						grp_imgdata[k] -= ave
						grp_imgdata[k] /= std
				if options.fl > 0.0:
					for k in range(img_per_grp):
						grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)

				#  Because of background issues, only linear option works.
				if options.CTF:  ave, var = aves_wiener(grp_imgdata, SNR = 1.0e5, interpolation_method = "linear")
				else:  ave, var = ave_var(grp_imgdata)
				# Switch to std dev
				# threshold is not really needed,it is just in case due to numerical accuracy something turns out negative.
				var = square_root(threshold(var))

				set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
				set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])

				aveList.append(ave)
				varList.append(var)
				xform_proj_for_2D[i] = [phiM, thetaM, 0.0, 0.0, 0.0]

				'''
				if nvec > 0:
					eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True)
					for k in range(nvec):
						set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
						eigList[k].append(eig[k])
					"""
					if myid == 0 and i == 0:
						for k in xrange(nvec):
							eig[k].write_image("eig.hdf", k)
					"""
				'''
				if (myid == heavy_load_myid) and (i%100 == 0):
					log_main.add(" ......%6.2f%%  "%(i/float(len(proj_list))*100.))		
			del imgdata, grp_imgdata, cpar, dpar, all_proj, proj_angles, index
			if not options.no_norm: del mask
			if myid == main_node: del tab
			#  At this point, all averages and variances are computed
			mpi_barrier(MPI_COMM_WORLD)
			
			if (myid == heavy_load_myid):
				log_main.add("Computing aveList and varList took %12.1f [m]"%((time()-ttt)/60.))
			
			xform_proj_for_2D = wrap_mpi_gatherv(xform_proj_for_2D, main_node, MPI_COMM_WORLD)
			if (myid == main_node):
				write_text_row(xform_proj_for_2D, os.path.join(current_output_dir, "params.txt"))
			del xform_proj_for_2D
			mpi_barrier(MPI_COMM_WORLD)
			if options.ave2D:
				from fundamentals import fpol
				from applications import header
				if myid == main_node:
					log_main.add("Compute ave2D ... ")
					km = 0
					for i in range(number_of_proc):
						if i == main_node :
							for im in range(len(aveList)):
								aveList[im].write_image(os.path.join(current_output_dir, options.ave2D), km)
								km += 1
						else:
							nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
							nl = int(nl[0])
							for im in range(nl):
								ave = recv_EMData(i, im+i+70000)
								"""
								nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
								nm = int(nm[0])
								members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
								ave.set_attr('members', map(int, members))
								members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
								ave.set_attr('pix_err', map(float, members))
								members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
								ave.set_attr('refprojdir', map(float, members))
								"""
								tmpvol=fpol(ave, nx, nx,1)								
								tmpvol.write_image(os.path.join(current_output_dir, options.ave2D), km)
								km += 1
				else:
					mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
					for im in range(len(aveList)):
						send_EMData(aveList[im], main_node,im+myid+70000)
						"""
						members = aveList[im].get_attr('members')
						mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						members = aveList[im].get_attr('pix_err')
						mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						try:
							members = aveList[im].get_attr('refprojdir')
							mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						except:
							mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						"""
				if myid == main_node:
					header(os.path.join(current_output_dir, options.ave2D), params='xform.projection', fimport = os.path.join(current_output_dir, "params.txt"))
				mpi_barrier(MPI_COMM_WORLD)	
			if options.ave3D:
				from fundamentals import fpol
				t5 = time()
				if myid == main_node: log_main.add("Reconstruct ave3D ... ")
				ave3D = recons3d_4nn_MPI(myid, aveList, symmetry=options.sym, npad=options.npad)
				bcast_EMData_to_all(ave3D, myid)
				if myid == main_node:
					if current_decimate != 1.0: ave3D = resample(ave3D, 1./current_decimate)
					ave3D = fpol(ave3D, nnxo, nnxo, nnxo) # always to the orignal image size
					set_pixel_size(ave3D, 1.0)
					ave3D.write_image(os.path.join(current_output_dir, options.ave3D))
					log_main.add("Ave3D reconstruction took %12.1f [m]"%((time()-t5)/60.0))
					log_main.add("%-70s:  %s\n"%("The reconstructed ave3D is saved as ", options.ave3D))
					
			mpi_barrier(MPI_COMM_WORLD)		
			del ave, var, proj_list, stack, alpha, sx, sy, mirror, aveList
			'''
			if nvec > 0:
				for k in range(nvec):
					if myid == main_node:log_main.add("Reconstruction eigenvolumes", k)
					cont = True
					ITER = 0
					mask2d = model_circle(radiuspca, nx, nx)
					while cont:
						#print "On node %d, iteration %d"%(myid, ITER)
						eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad)
						bcast_EMData_to_all(eig3D, myid, main_node)
						if options.fl > 0.0:
							eig3D = filt_tanl(eig3D, options.fl, options.aa)
						if myid == main_node:
							eig3D.write_image(os.path.join(options.outpout_dir, "eig3d_%03d.hdf"%(k, ITER)))
						Util.mul_img( eig3D, model_circle(radiuspca, nx, nx, nx) )
						eig3Df, kb = prep_vol(eig3D)
						del eig3D
						cont = False
						icont = 0
						for l in range(len(eigList[k])):
							phi, theta, psi, s2x, s2y = get_params_proj(eigList[k][l])
							proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y])
							cl = ccc(proj, eigList[k][l], mask2d)
							if cl < 0.0:
								icont += 1
								cont = True
								eigList[k][l] *= -1.0
						u = int(cont)
						u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD)
						icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)

						if myid == main_node:
							u = int(u[0])
							log_main.add(" Eigenvector: ",k," number changed ",int(icont[0]))
						else: u = 0
						u = bcast_number_to_all(u, main_node)
						cont = bool(u)
						ITER += 1

					del eig3Df, kb
					mpi_barrier(MPI_COMM_WORLD)
				del eigList, mask2d
			'''
			if options.ave3D: del ave3D
			if options.var2D:
				from fundamentals import fpol 
				from applications import header
				if myid == main_node:
					log_main.add("Compute var2D...")
					km = 0
					for i in range(number_of_proc):
						if i == main_node :
							for im in range(len(varList)):
								tmpvol=fpol(varList[im], nx, nx,1)
								tmpvol.write_image(os.path.join(current_output_dir, options.var2D), km)
								km += 1
						else:
							nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
							nl = int(nl[0])
							for im in range(nl):
								ave = recv_EMData(i, im+i+70000)
								tmpvol=fpol(ave, nx, nx,1)
								tmpvol.write_image(os.path.join(current_output_dir, options.var2D), km)
								km += 1
				else:
					mpi_send(len(varList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
					for im in range(len(varList)):
						send_EMData(varList[im], main_node, im+myid+70000)#  What with the attributes??
				mpi_barrier(MPI_COMM_WORLD)
				if myid == main_node:
					from applications import header
					header(os.path.join(current_output_dir, options.var2D), params = 'xform.projection',fimport = os.path.join(current_output_dir, "params.txt"))
				mpi_barrier(MPI_COMM_WORLD)
		if options.var3D:
			if myid == main_node: log_main.add("Reconstruct var3D ...")
			t6 = time()
			# radiusvar = options.radius
			# if( radiusvar < 0 ):  radiusvar = nx//2 -3
			res = recons3d_4nn_MPI(myid, varList, symmetry = options.sym, npad=options.npad)
			#res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
			if myid == main_node:
				from fundamentals import fpol
				if current_decimate != 1.0: res	= resample(res, 1./current_decimate)
				res = fpol(res, nnxo, nnxo, nnxo)
				set_pixel_size(res, 1.0)
				res.write_image(os.path.join(current_output_dir, options.var3D))
				log_main.add("%-70s:  %s\n"%("The reconstructed var3D is saved as ", options.var3D))
				log_main.add("Var3D reconstruction took %f12.1 [m]"%((time()-t6)/60.0))
				log_main.add("Total computation time %f12.1 [m]"%((time()-t0)/60.0))
				log_main.add("sx3dvariability finishes")
		from mpi import mpi_finalize
		mpi_finalize()
		
		if RUNNING_UNDER_MPI: global_def.MPI = False

		global_def.BATCH = False
Exemplo n.º 12
0
def main():
	global debug
	progname = os.path.basename(sys.argv[0])
	usage = """prog [options] <path or db> ...
	
Various utilities related to BDB databases.

examples :
e2bdb.py -c   Is perhaps the most critical function, as it cleans up the database cache. See the Wiki for more.
e2bdb.py <path> -s    will list the contents of the database in a directory in bdb: notation
e2bdb.py <path> -l    Will give useful summary info about stacks in a directory
e2bdb.py <database> --dump    Gives a mechanism to dump all of the metadata in a database, even if the database contains no images
"""

	parser = EMArgumentParser(usage=usage,version=EMANVERSION)

	parser.add_argument("--cleanup","-c",action="store_true",default=False,help="This option will clean up the database cache so files can safely be moved or accessed on another computer via NFS.")
	parser.add_argument("--force","-F",action="store_true",default=False,help="This will force an action that would normally fail due to failed checks.")
	parser.add_argument("--delete",action="store_true",default=False,help="This will delete (or at least empty) the named database(s)")
	parser.add_argument("--all","-a",action="store_true",help="List per-particle info",default=False)
	parser.add_argument("--long","-l",action="store_true",help="Long listing",default=False)
	parser.add_argument("--short","-s",action="store_true",help="Dense listing of names only",default=False)
	parser.add_argument("--filt",type=str,help="Only include dictionary names containing the specified string",default=None)
	parser.add_argument("--filtexclude",type=str,help="Exclude dictionary names containing the specified string",default=None)
	parser.add_argument("--match",type=str,help="Only include dictionaries matching the provided Python regular expression",default=None)
	parser.add_argument("--exclude",type=str,help="The name of a database containing a list of exclusion keys",default=None)
	parser.add_argument("--dump","-D",action="store_true",help="List contents of an entire database, eg 'e2bdb.py -D refine_01#register",default=False)
	parser.add_argument("--smalldump",action="store_true",help="Lists contents of an entire database, but only list 2 items per dictionary to better see headers",default=False)
	parser.add_argument("--extractplots",action="store_true",help="If a database contains sets of plots, such as bdb:refine_xx#convergence.results, this will extract the plots as text files.")
	parser.add_argument("--check",action="store_true",help="Check for self-consistency and errors in the structure of specified databases",default=False)
	parser.add_argument("--nocache",action="store_true",help="Don't use the database cache for this operation",default=False)
	parser.add_argument("--merge",action="store_true",help="This will merge the contents of BDB 2-N into BDB 1 (including BDB 1's contents)",default=False)


	parser.add_argument("--makevstack",type=str,help="Creates a 'virtual' BDB stack with its own metadata, but the binary data taken from the (filtered) list of stacks",default=None)
	parser.add_argument("--appendvstack",type=str,help="Appends to/creates a 'virtual' BDB stack with its own metadata, but the binary data taken from the (filtered) list of stacks",default=None)
	parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higher number means higher level of verboseness")
	parser.add_argument("--list",type=str,help="Specify the name of a file with a list of images to use in creation of virtual stacks. Please see source for details.",default=None)
	parser.add_argument("--exlist",type=str,help="Specify the name of a file with a list of images to exclude in creation of virtual stacks. Please see source for details.",default=None)
	parser.add_argument("--restore",type=str,help="Write changes in the derived virtual stack back to the original stack",default=None)
	parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
	parser.add_argument("--checkctf",action="store_true",help="Verfies that all images in the file contain CTF information, and gives some basic statistics",default=False)

	parser.add_argument("--step",type=str,default="0,1",help="Specify <init>,<step>[,<max>]. Processes only a subset of the input data. For example, 0,2 would process only the even numbered particles")
	(options, args) = parser.parse_args()

	if options.nocache : EMAN2db.BDB_CACHE_DISABLE=True

	if options.cleanup : 
		db_cleanup(options.force)
		sys.exit(0)

	try : options.step=int(options.step.split(",")[0]),int(options.step.split(",")[1]),int(options.step.split(",")[2])		# convert strings to tuple
	except:
		try:
			options.step=int(options.step.split(",")[0]),int(options.step.split(",")[1])
		except:
			print "Invalid --step specification"
			sys.exit(1)

	if options.all : options.long=1
	if len(args)==0 : args.append("bdb:.")
	
	logid=0
	if options.makevstack : 
		logid=E2init(sys.argv)
		vstack=db_open_dict(options.makevstack)
		vstackn=0
	elif options.appendvstack :
		logid=E2init(sys.argv)
		vstack=db_open_dict(options.appendvstack)
		vstackn=len(vstack)
	else : vstack=None
		
	if options.merge :
		print "WARNING: Merge mode\nCombining contents of: ",", ".join(args[1:])
		print "into ",args[0]
		
		if raw_input("Proceed (y/n) :").lower() != "y" :
			print "Aborting"
			sys.exit(1)
		
		
		for i,path in enumerate(args):
			if path.lower()[:4]=="bdb:" and not "#" in path : path="bdb:.#"+path[4:]
			if path.lower()[:4]!="bdb:" : path="bdb:"+path
			
			if i==0 :
				outdb=db_open_dict(path)
				continue
			
			indb=db_open_dict(path,True)
			for k in indb.keys():
				outdb[k]=indb[k]
				
		print "Merging complete"
		sys.exit(0)
	
	for path in args:
		if path.lower()[:4]=="bdb:" and not "#" in path :
			uu = os.path.split(path)
			if(uu[0] == ''):    path="bdb:.#"+path[4:]
			else:               path=uu[0]+"#"+uu[1]
		if path.lower()[:4]!="bdb:" : path="bdb:"+path
		if '#' in path :
			if len(args)>1 : print "\n",path,":"
			path,dbs=path.rsplit("#",1)
			path+="#"
			dbs=[dbs]
		else:
			if not '#' in path and path[-1]!='/' : path+='#'			
			if len(args)>1 : print "\n",path[:-1],":"
			dbs=db_list_dicts(path)
			

		dbs.sort()
		if options.filt:
			dbs=[db for db in dbs if options.filt in db]
			
		if options.filtexclude:
			dbs=[db for db in dbs if options.filtexclude not in db]

		if options.match!=None:
			dbs=[db for db in dbs if re.match(options.match,db)]
		
		
		if options.list :
			if options.makevstack==None and options.appendvstack==None :
				print "ERROR, this option is used for virtual stack creation, please add makevstack or appendvstack options, and restart"
				sys.exit(1)
			vdata=open(options.list,'r').readlines()
			n=len(vdata[0].split())
			slist=[]
			for line in vdata:
				line=line.split()
				for i in xrange(n):
					val=int(line[i])
					slist.append(val)     		
			del n,val,vdata
		
		if options.exlist :
			if options.makevstack==None:
				print "ERROR, this option is used for virtual stack creation, please add makevstack or appendvstack options, and restart"
				sys.exit(1)
			vdata=open(options.exlist,'r').readlines()
			n=len(vdata[0].split())
			slist=[]
			for line in vdata:
				line=line.split()
				for i in xrange(n):
					val=int(line[i])
					slist.append(val)     
			n = EMUtil.get_image_count(args[0])
			good = set(range(n)) - set(slist)
			slist = [i for i in good]
			slist.sort()
			del n,val,vdata,good
			
		if options.makevstack!=None or options.appendvstack!=None :
			
			vspath=os.path.realpath(vstack.path)+"/"
			if options.verbose>2 : print "vspath: ",vspath
			for db in dbs:
				dct,keys=db_open_dict(path+db,ro=True,with_keys=True)
				if dct==vstack : continue
				if len(options.step)==2 :
					if keys == None: vals = xrange(options.step[0],len(dct),options.step[1])
					else: vals = keys[options.step[0]::options.step[1]]		# we apply --step even if we have a list of keys
				else:
					if keys == None: vals = xrange(options.step[0],options.step[2],options.step[1])
					else: vals = keys[options.step[0]:options.step[2]:options.step[1]]		# we apply --step even if we have a list of keys

				if options.list !=None or options.exlist != None: vals=slist
				for n in vals:
					try: d=dct.get(n,nodata=1).get_attr_dict()
					except:
						traceback.print_exc()
						print "---\nerror reading ",db,n 
						continue
					# This block converts an absolute path to the actual data to a relative path
					try:
						dpath=os.path.realpath(dct.get_data_path(n))
						if options.verbose>2 : print "dpath: ",dpath
						if os.name == 'nt':
							vspath=vspath.replace("\\", '/')
							dpath=dpath.replace('\\', '/')
						rpath=makerelpath(vspath,dpath)
						if options.verbose>2 : print "rpath: ",rpath
					except:
						print "error with data_path ",db,n
						continue
					d["data_path"]=rpath
					d["data_n"]=n
					d["data_source"]= path+db
					if d["data_path"]==None :
						print "error with data_path ",db,n
						continue
					vstack[vstackn]=d
					vstackn+=1
					if vstackn%100==0:
						try:
							print "\r  ",vstackn,"     ",
							sys.stdout.flush()
						except: pass	
				print "\r  ",vstackn,"     "
				dct.close()

		try: maxname=max([len(s) for s in dbs])
		except:
			print "Error reading ",path

		if options.restore :
			nima = EMUtil.get_image_count(options.restore)
			IB = db_open_dict(options.restore)
			source_old = None
			if len(options.step)==3 : nima=min(options.step[2],nima)
			for i in xrange(options.step[0],nima,options.step[1]):
				source = IB.get_header(i)
				source_path = source["source_path"]
				ID = source["source_n"]
				if( source_old != source_path):
					if( source_old != None):  DB.close()
					DB = db_open_dict(source_path,ro=True)
					source_old = source_path
				target = DB.get_header( ID )
				try:
					source["data_path"] = target["data_path"]
					source["data_n"]    = target["data_n"]
					source["source_path"] = target["source_path"]
					source["source_n"]    = target["source_n"]
				except:
					#  top level does not have data_path
					del source['data_path']
					del source['data_n']
					source["source_path"] = target["source_path"]
					source["source_n"]    = target["source_n"]
				DB.set_header(ID, source)
			DB.close()

		if options.extractplots :
			for db in dbs:
				print "####  Extracting plots from ",db
				dct=db_open_dict(path+db,ro=True)
				
				#### Dump
				keys=dct.keys()
				keys.sort()
				for k in keys:
					v=dct[k]
					try :
						ns=[len(i) for i in v]
						fsp=db+"-"+k+".txt"
						print "%s  (%d columns)"%(fsp,len(ns))
						out=file(fsp,"w")
						for i in range(ns[0]):
							for j in range(len(ns)):
								out.write(str(v[j][i]))
								if j<len(ns)-1 : out.write("\t")
							out.write("\n")
						out.close()
					except: continue
				dct.close()
			
		if options.smalldump :
			for db in dbs:
				print "##### ",db
				dct=db_open_dict(path+db,ro=True)
				
				#### Dump
				keys=dct.keys()
				keys.sort()
				if len(options.step)==3 : keys=keys[:options.step[2]]
				for k in keys[options.step[0]::options.step[1]]:
					v=dct[k]
					print "%s : "%k,
					if isinstance (v,list) or isinstance(v,tuple)  :
						for i in v: print "\n\t%s"%str(i),
						print ""
					elif isinstance(v,dict) :
						ks2=v.keys()
						ks2.sort()
						kc=0
						for i in ks2:
							if kc>=2 :
								print "..."
								break
							print "\n\t%s : %s"%(i,v[i]),
							kc+=1
						print ""
					else : print str(v)
				dct.close()
		if options.checkctf:
			for db in dbs:
				print "##### CTF -> ",db
				dct=db_open_dict(path+db,ro=True)
				keys=dct.keys()
				if len(options.step)==3 : keys=keys[:options.step[2]]
				defocus=set()
				for k in keys[options.step[0]::options.step[1]]:
					v=dct.get_header(k)
					try:
						ctf=v["ctf"]
					except:
						if k!="maxrec" : print "CTF missing on image %s"%k
						continue
					
					defocus.add(ctf.defocus)

				defocus=list(defocus)
				print "Defocuses found: ",
				for i in defocus: print "%1.3f, "%i,
				print "\n\nRange: %1.3f - %1.3f  (%d unique values)"%(min(defocus),max(defocus),len(defocus))

					

		if options.dump :
			for db in dbs:
				print "##### ",db
				dct=db_open_dict(path+db,ro=True)
				
				#### Dump
				keys=dct.keys()
				if len(options.step)==3 : keys=keys[:options.step[2]]
				keys.sort()
				for k in keys[options.step[0]::options.step[1]]:
					v=dct[k]
					print "%s : "%k,
					if isinstance (v,list) or isinstance(v,tuple)  :
						for i in v: print "\n\t%s"%str(i),
						print ""
					elif isinstance(v,dict) :
						ks2=v.keys()
						ks2.sort()
						for i in ks2:
							print "\n\t%s : %s"%(i,v[i]),
						print ""
					else : print str(v)
				dct.close()
			
		# long listing, one db per line
		elif options.long :
			width=maxname+3
			fmt="%%-%ds %%-07d %%14s  %%s"%width
			fmt2="%%-%ds (not an image stack)"%width
			total=[0,0]
			for db in dbs:
				dct=db_open_dict(path+db,True)
								
				### Info on all particles
				if options.all :
					mx=len(dct)
					if len(options.step)==3 : mx=min(mx,options.step[2])
					for i in range(options.step[0],mx,options.step[1]):
						try: 
							im=dct[i]
							if im==None : raise Exception
						except: continue
						print "%d. %d x %d x %d\tA/pix=%1.2f\tM=%1.4f\tS=%1.4f\tSk=%1.4f"%(i,im["nx"],im["ny"],im["nz"],im["apix_x"],im["mean"],im["sigma"],im["skewness"]),
						try: print "\t%s"%str(im["model_id"])
						except: pass
						try:
							print "\tdf=%1.3f\tB=%1.1f"%(im["ctf"].defocus,im["ctf"].bfactor)
						except: print " "

				first=EMData()
				try: 
					first.read_image(path+db,0,True)
					size=first.get_xsize()*first.get_ysize()*first.get_zsize()*len(dct)*4;
					total[0]+=len(dct)
					total[1]+=size
					print fmt%(db,len(dct),"%dx%dx%d   apix: %1.2f"%(first.get_xsize(),first.get_ysize(),first.get_zsize(),first["apix_x"]),human_size(size)),
				except:
					print fmt2%db
				try: print "\tdf: %1.3f\tB: %1.0f"%(first["ctf"].defocus,first["ctf"].bfactor)
				except: print ""
				dct.close()
			print fmt%("TOTAL",total[0],"",human_size(total[1]))
		elif options.check :
			from cPickle import loads
			for db in dbs:
				dct=db_open_dict(path+db,ro=True)
				dct.realopen()
				keys=dct.bdb.keys()
				allkvp={}
				for k in keys:
					s1,s2=k.split("\x80",1)		# start of a pickled string. 
					s2=loads("\x80"+s2)			# the pickled part
					if len(s1)>0 :				# If anything unpickled, then it is an axbxc prefix identifying the location of a binary
						st=allkvp.setdefault(s1,set()) # set of all positions seen so far
						v=loads(dct.bdb.get(k))	# position in binary file
						if v in st : print "Error: value %d seen multiple times in %s (%s,%s)"%(v,db,s1,s2)
						st.add(v)
				print "%s : "%db,
				for i in allkvp.keys(): 
					if options.verbose>0 : print "%s %d/%d\t"%(i,len(allkvp[i]),int(max(allkvp[i]))+1),
					if len(allkvp[i])!=int(max(allkvp[i])+1) : print "\nMismatch found in %s. Could be normal if file has been rewritten multiple times, but is unusual"%db
				if options.verbose>0 : print ""
				else : print " done"
				dct.close()

		elif options.short :
			for db in dbs:
				print path+db,
			print " "
Exemplo n.º 13
0
def main():

    progname = os.path.basename(sys.argv[0])
    usage = progname + " proj_stack output_averages --MPI"
    parser = OptionParser(usage, version=SPARXVERSION)

    parser.add_option("--img_per_group",
                      type="int",
                      default=100,
                      help="number of images per group")
    parser.add_option("--radius",
                      type="int",
                      default=-1,
                      help="radius for alignment")
    parser.add_option(
        "--xr",
        type="string",
        default="2 1",
        help="range for translation search in x direction, search is +/xr")
    parser.add_option(
        "--yr",
        type="string",
        default="-1",
        help=
        "range for translation search in y direction, search is +/yr (default = same as xr)"
    )
    parser.add_option(
        "--ts",
        type="string",
        default="1 0.5",
        help=
        "step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional"
    )
    parser.add_option(
        "--iter",
        type="int",
        default=30,
        help="number of iterations within alignment (default = 30)")
    parser.add_option(
        "--num_ali",
        type="int",
        default=5,
        help="number of alignments performed for stability (default = 5)")
    parser.add_option("--thld_err",
                      type="float",
                      default=1.0,
                      help="threshold of pixel error (default = 1.732)")
    parser.add_option(
        "--grouping",
        type="string",
        default="GRP",
        help=
        "do grouping of projections: PPR - per projection, GRP - different size groups, exclusive (default), GEV - grouping equal size"
    )
    parser.add_option(
        "--delta",
        type="float",
        default=-1.0,
        help="angular step for reference projections (required for GEV method)"
    )
    parser.add_option(
        "--fl",
        type="float",
        default=0.3,
        help="cut-off frequency of hyperbolic tangent low-pass Fourier filter")
    parser.add_option(
        "--aa",
        type="float",
        default=0.2,
        help="fall-off of hyperbolic tangent low-pass Fourier filter")
    parser.add_option("--CTF",
                      action="store_true",
                      default=False,
                      help="Consider CTF correction during the alignment ")
    parser.add_option("--MPI",
                      action="store_true",
                      default=False,
                      help="use MPI version")

    (options, args) = parser.parse_args()

    myid = mpi.mpi_comm_rank(MPI_COMM_WORLD)
    number_of_proc = mpi.mpi_comm_size(MPI_COMM_WORLD)
    main_node = 0

    if len(args) == 2:
        stack = args[0]
        outdir = args[1]
    else:
        sp_global_def.ERROR("Incomplete list of arguments",
                            "sxproj_stability.main",
                            1,
                            myid=myid)
        return
    if not options.MPI:
        sp_global_def.ERROR("Non-MPI not supported!",
                            "sxproj_stability.main",
                            1,
                            myid=myid)
        return

    if sp_global_def.CACHE_DISABLE:
        from sp_utilities import disable_bdb_cache
        disable_bdb_cache()
    sp_global_def.BATCH = True

    img_per_grp = options.img_per_group
    radius = options.radius
    ite = options.iter
    num_ali = options.num_ali
    thld_err = options.thld_err

    xrng = get_input_from_string(options.xr)
    if options.yr == "-1":
        yrng = xrng
    else:
        yrng = get_input_from_string(options.yr)

    step = get_input_from_string(options.ts)

    if myid == main_node:
        nima = EMUtil.get_image_count(stack)
        img = get_image(stack)
        nx = img.get_xsize()
        ny = img.get_ysize()
    else:
        nima = 0
        nx = 0
        ny = 0
    nima = bcast_number_to_all(nima)
    nx = bcast_number_to_all(nx)
    ny = bcast_number_to_all(ny)
    if radius == -1: radius = nx / 2 - 2
    mask = model_circle(radius, nx, nx)

    st = time()
    if options.grouping == "GRP":
        if myid == main_node:
            sxprint("  A  ", myid, "  ", time() - st)
            proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
            proj_params = []
            for i in range(nima):
                dp = proj_attr[i].get_params("spider")
                phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[
                    "psi"], -dp["tx"], -dp["ty"]
                proj_params.append([phi, theta, psi, s2x, s2y])

            # Here is where the grouping is done, I didn't put enough annotation in the group_proj_by_phitheta,
            # So I will briefly explain it here
            # proj_list  : Returns a list of list of particle numbers, each list contains img_per_grp particle numbers
            #              except for the last one. Depending on the number of particles left, they will either form a
            #              group or append themselves to the last group
            # angle_list : Also returns a list of list, each list contains three numbers (phi, theta, delta), (phi,
            #              theta) is the projection angle of the center of the group, delta is the range of this group
            # mirror_list: Also returns a list of list, each list contains img_per_grp True or False, which indicates
            #              whether it should take mirror position.
            # In this program angle_list and mirror list are not of interest.

            proj_list_all, angle_list, mirror_list = group_proj_by_phitheta(
                proj_params, img_per_grp=img_per_grp)
            del proj_params
            sxprint("  B  number of groups  ", myid, "  ", len(proj_list_all),
                    time() - st)
        mpi_barrier(MPI_COMM_WORLD)

        # Number of groups, actually there could be one or two more groups, since the size of the remaining group varies
        # we will simply assign them to main node.
        n_grp = nima / img_per_grp - 1

        # Divide proj_list_all equally to all nodes, and becomes proj_list
        proj_list = []
        for i in range(n_grp):
            proc_to_stay = i % number_of_proc
            if proc_to_stay == main_node:
                if myid == main_node: proj_list.append(proj_list_all[i])
            elif myid == main_node:
                mpi_send(len(proj_list_all[i]), 1, MPI_INT, proc_to_stay,
                         SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
                mpi_send(proj_list_all[i], len(proj_list_all[i]), MPI_INT,
                         proc_to_stay, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
            elif myid == proc_to_stay:
                img_per_grp = mpi_recv(1, MPI_INT, main_node,
                                       SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
                img_per_grp = int(img_per_grp[0])
                temp = mpi_recv(img_per_grp, MPI_INT, main_node,
                                SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
                proj_list.append(list(map(int, temp)))
                del temp
            mpi_barrier(MPI_COMM_WORLD)
        sxprint("  C  ", myid, "  ", time() - st)
        if myid == main_node:
            # Assign the remaining groups to main_node
            for i in range(n_grp, len(proj_list_all)):
                proj_list.append(proj_list_all[i])
            del proj_list_all, angle_list, mirror_list

    #   Compute stability per projection projection direction, equal number assigned, thus overlaps
    elif options.grouping == "GEV":

        if options.delta == -1.0:
            ERROR(
                "Angular step for reference projections is required for GEV method"
            )
            return

        from sp_utilities import even_angles, nearestk_to_refdir, getvec
        refproj = even_angles(options.delta)
        img_begin, img_end = MPI_start_end(len(refproj), number_of_proc, myid)
        # Now each processor keeps its own share of reference projections
        refprojdir = refproj[img_begin:img_end]
        del refproj

        ref_ang = [0.0] * (len(refprojdir) * 2)
        for i in range(len(refprojdir)):
            ref_ang[i * 2] = refprojdir[0][0]
            ref_ang[i * 2 + 1] = refprojdir[0][1] + i * 0.1

        sxprint("  A  ", myid, "  ", time() - st)
        proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
        #  the solution below is very slow, do not use it unless there is a problem with the i/O
        """
		for i in xrange(number_of_proc):
			if myid == i:
				proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
			mpi_barrier(MPI_COMM_WORLD)
		"""
        sxprint("  B  ", myid, "  ", time() - st)

        proj_ang = [0.0] * (nima * 2)
        for i in range(nima):
            dp = proj_attr[i].get_params("spider")
            proj_ang[i * 2] = dp["phi"]
            proj_ang[i * 2 + 1] = dp["theta"]
        sxprint("  C  ", myid, "  ", time() - st)
        asi = Util.nearestk_to_refdir(proj_ang, ref_ang, img_per_grp)
        del proj_ang, ref_ang
        proj_list = []
        for i in range(len(refprojdir)):
            proj_list.append(asi[i * img_per_grp:(i + 1) * img_per_grp])
        del asi
        sxprint("  D  ", myid, "  ", time() - st)
        #from sys import exit
        #exit()

    #   Compute stability per projection
    elif options.grouping == "PPR":
        sxprint("  A  ", myid, "  ", time() - st)
        proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
        sxprint("  B  ", myid, "  ", time() - st)
        proj_params = []
        for i in range(nima):
            dp = proj_attr[i].get_params("spider")
            phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[
                "psi"], -dp["tx"], -dp["ty"]
            proj_params.append([phi, theta, psi, s2x, s2y])
        img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
        sxprint("  C  ", myid, "  ", time() - st)
        from sp_utilities import nearest_proj
        proj_list, mirror_list = nearest_proj(
            proj_params, img_per_grp,
            list(range(img_begin, img_begin + 1)))  #range(img_begin, img_end))
        refprojdir = proj_params[img_begin:img_end]
        del proj_params, mirror_list
        sxprint("  D  ", myid, "  ", time() - st)

    else:
        ERROR("Incorrect projection grouping option")
        return

    ###########################################################################################################
    # Begin stability test
    from sp_utilities import get_params_proj, read_text_file
    #if myid == 0:
    #	from utilities import read_text_file
    #	proj_list[0] = map(int, read_text_file("lggrpp0.txt"))

    from sp_utilities import model_blank
    aveList = [model_blank(nx, ny)] * len(proj_list)
    if options.grouping == "GRP":
        refprojdir = [[0.0, 0.0, -1.0]] * len(proj_list)
    for i in range(len(proj_list)):
        sxprint("  E  ", myid, "  ", time() - st)
        class_data = EMData.read_images(stack, proj_list[i])
        #print "  R  ",myid,"  ",time()-st
        if options.CTF:
            from sp_filter import filt_ctf
            for im in range(len(class_data)):  #  MEM LEAK!!
                atemp = class_data[im].copy()
                btemp = filt_ctf(atemp, atemp.get_attr("ctf"), binary=1)
                class_data[im] = btemp
                #class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1)
        for im in class_data:
            try:
                t = im.get_attr(
                    "xform.align2d")  # if they are there, no need to set them!
            except:
                try:
                    t = im.get_attr("xform.projection")
                    d = t.get_params("spider")
                    set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0])
                except:
                    set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0])
        #print "  F  ",myid,"  ",time()-st
        # Here, we perform realignment num_ali times
        all_ali_params = []
        for j in range(num_ali):
            if (xrng[0] == 0.0 and yrng[0] == 0.0):
                avet = ali2d_ras(class_data,
                                 randomize=True,
                                 ir=1,
                                 ou=radius,
                                 rs=1,
                                 step=1.0,
                                 dst=90.0,
                                 maxit=ite,
                                 check_mirror=True,
                                 FH=options.fl,
                                 FF=options.aa)
            else:
                avet = within_group_refinement(class_data, mask, True, 1,
                                               radius, 1, xrng, yrng, step,
                                               90.0, ite, options.fl,
                                               options.aa)
            ali_params = []
            for im in range(len(class_data)):
                alpha, sx, sy, mirror, scale = get_params2D(class_data[im])
                ali_params.extend([alpha, sx, sy, mirror])
            all_ali_params.append(ali_params)
        #aveList[i] = avet
        #print "  G  ",myid,"  ",time()-st
        del ali_params
        # We determine the stability of this group here.
        # stable_set contains all particles deemed stable, it is a list of list
        # each list has two elements, the first is the pixel error, the second is the image number
        # stable_set is sorted based on pixel error
        #from utilities import write_text_file
        #write_text_file(all_ali_params, "all_ali_params%03d.txt"%myid)
        stable_set, mir_stab_rate, average_pix_err = multi_align_stability(
            all_ali_params, 0.0, 10000.0, thld_err, False, 2 * radius + 1)
        #print "  H  ",myid,"  ",time()-st
        if (len(stable_set) > 5):
            stable_set_id = []
            members = []
            pix_err = []
            # First put the stable members into attr 'members' and 'pix_err'
            for s in stable_set:
                # s[1] - number in this subset
                stable_set_id.append(s[1])
                # the original image number
                members.append(proj_list[i][s[1]])
                pix_err.append(s[0])
            # Then put the unstable members into attr 'members' and 'pix_err'
            from sp_fundamentals import rot_shift2D
            avet.to_zero()
            if options.grouping == "GRP":
                aphi = 0.0
                atht = 0.0
                vphi = 0.0
                vtht = 0.0
            l = -1
            for j in range(len(proj_list[i])):
                #  Here it will only work if stable_set_id is sorted in the increasing number, see how l progresses
                if j in stable_set_id:
                    l += 1
                    avet += rot_shift2D(class_data[j], stable_set[l][2][0],
                                        stable_set[l][2][1],
                                        stable_set[l][2][2],
                                        stable_set[l][2][3])
                    if options.grouping == "GRP":
                        phi, theta, psi, sxs, sy_s = get_params_proj(
                            class_data[j])
                        if (theta > 90.0):
                            phi = (phi + 540.0) % 360.0
                            theta = 180.0 - theta
                        aphi += phi
                        atht += theta
                        vphi += phi * phi
                        vtht += theta * theta
                else:
                    members.append(proj_list[i][j])
                    pix_err.append(99999.99)
            aveList[i] = avet.copy()
            if l > 1:
                l += 1
                aveList[i] /= l
                if options.grouping == "GRP":
                    aphi /= l
                    atht /= l
                    vphi = (vphi - l * aphi * aphi) / l
                    vtht = (vtht - l * atht * atht) / l
                    from math import sqrt
                    refprojdir[i] = [
                        aphi, atht,
                        (sqrt(max(vphi, 0.0)) + sqrt(max(vtht, 0.0))) / 2.0
                    ]

            # Here more information has to be stored, PARTICULARLY WHAT IS THE REFERENCE DIRECTION
            aveList[i].set_attr('members', members)
            aveList[i].set_attr('refprojdir', refprojdir[i])
            aveList[i].set_attr('pixerr', pix_err)
        else:
            sxprint(" empty group ", i, refprojdir[i])
            aveList[i].set_attr('members', [-1])
            aveList[i].set_attr('refprojdir', refprojdir[i])
            aveList[i].set_attr('pixerr', [99999.])

    del class_data

    if myid == main_node:
        km = 0
        for i in range(number_of_proc):
            if i == main_node:
                for im in range(len(aveList)):
                    aveList[im].write_image(args[1], km)
                    km += 1
            else:
                nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL,
                              MPI_COMM_WORLD)
                nl = int(nl[0])
                for im in range(nl):
                    ave = recv_EMData(i, im + i + 70000)
                    nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL,
                                  MPI_COMM_WORLD)
                    nm = int(nm[0])
                    members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL,
                                       MPI_COMM_WORLD)
                    ave.set_attr('members', list(map(int, members)))
                    members = mpi_recv(nm, MPI_FLOAT, i,
                                       SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
                    ave.set_attr('pixerr', list(map(float, members)))
                    members = mpi_recv(3, MPI_FLOAT, i,
                                       SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
                    ave.set_attr('refprojdir', list(map(float, members)))
                    ave.write_image(args[1], km)
                    km += 1
    else:
        mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL,
                 MPI_COMM_WORLD)
        for im in range(len(aveList)):
            send_EMData(aveList[im], main_node, im + myid + 70000)
            members = aveList[im].get_attr('members')
            mpi_send(len(members), 1, MPI_INT, main_node,
                     SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
            mpi_send(members, len(members), MPI_INT, main_node,
                     SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
            members = aveList[im].get_attr('pixerr')
            mpi_send(members, len(members), MPI_FLOAT, main_node,
                     SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
            try:
                members = aveList[im].get_attr('refprojdir')
                mpi_send(members, 3, MPI_FLOAT, main_node,
                         SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
            except:
                mpi_send([-999.0, -999.0, -999.0], 3, MPI_FLOAT, main_node,
                         SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)

    sp_global_def.BATCH = False
    mpi_barrier(MPI_COMM_WORLD)
Exemplo n.º 14
0
	def __new__(cls,filename,application,force_plot=False,force_2d=False,old=None):
		
		file_type = Util.get_filename_ext(filename)
		em_file_type = EMUtil.get_image_ext_type(file_type)
		if not file_exists(filename): return None
		
		if force_plot and force_2d:
			# ok this sucks but it suffices for the time being
			print "Error, the force_plot and force_2d options are mutually exclusive"
			return None
		
		if force_plot:
			from emplot2d import EMPlot2DWidget
			if isinstance(old,EMPlot2DWidget): widget = old
			else: widget = EMPlot2DWidget(application=application)
			widget.set_data_from_file(filename)
			return widget
		
		if em_file_type != IMAGE_UNKNOWN or filename[:4] == "bdb:":
			n = EMUtil.get_image_count(filename)
			nx,ny,nz = gimme_image_dimensions3D(filename)
			if n > 1 and nz == 1: 
				if force_2d:
					a = EMData()
					data=a.read_images(filename)
				else:
					data = None # This is like a flag - the ImageMXWidget only needs the file name
			elif nz == 1:
				data = [EMData(filename,0)]
			else:
				data = EMData()
				data.read_image(filename,0,not force_2d)		# This should be 3-D. We read the header-only here
				data = [data]
				
			if data != None and len(data) == 1: data = data[0]
			
			if force_2d or isinstance(data,EMData) and data.get_zsize()==1:
				if isinstance(data,list) or data.get_ysize() != 1:
					from emimage2d import EMImage2DWidget
					if isinstance(old,EMImage2DWidget): widget = old
					else: widget= EMImage2DWidget(application=application)
				else:
					from emplot2d import EMPlot2DWidget
					if isinstance(old,EMPlot2DWidget): widget = old
					else: widget = EMPlot2DWidget(application=application)
					widget.set_data_from_file(filename)
					return widget
			elif isinstance(data,EMData):
				if isinstance(old,EMScene3D): widget = old
				else: widget = EMScene3D()
#				print n,data
				for ii in xrange(n):
					data=EMData(filename,ii)
					datai = EMDataItem3D(data, transform=Transform())
					widget.insertNewNode(os.path.basename(filename), datai, parentnode=widget)
					isosurface = EMIsosurface(datai, transform=Transform())
					widget.insertNewNode("Iso", isosurface, parentnode=datai)
				return widget
				
			elif data == None or isinstance(data,list):
				from emimagemx import EMImageMXWidget
				if isinstance(old,EMImageMXWidget): widget = old
				else: widget = EMImageMXWidget(application=application)
				data = filename
			else: 
				print filename
				raise # weirdness, this should never happen
			widget.set_data(data,filename)
			return widget
		else:
			from emplot2d import EMPlot2DWidget
			if isinstance(old,EMPlot2DWidget): widget = old
			else: widget = EMPlot2DWidget(application=application)
			widget.set_data_from_file(filename)
			return widget
Exemplo n.º 15
0
def main():
    def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
        if mirror:
            m = 1
            alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
                                                       540.0 - psi, 0, 0, 1.0)
        else:
            m = 0
            alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
                                                       360.0 - psi, 0, 0, 1.0)
        return alpha, sx, sy, m

    progname = os.path.basename(sys.argv[0])
    usage = progname + " prj_stack  --ave2D= --var2D=  --ave3D= --var3D= --img_per_grp= --fl=15. --aa=0.01  --sym=symmetry --CTF"
    parser = OptionParser(usage, version=SPARXVERSION)

    parser.add_option("--output_dir",
                      type="string",
                      default="./",
                      help="output directory")
    parser.add_option("--ave2D",
                      type="string",
                      default=False,
                      help="write to the disk a stack of 2D averages")
    parser.add_option("--var2D",
                      type="string",
                      default=False,
                      help="write to the disk a stack of 2D variances")
    parser.add_option("--ave3D",
                      type="string",
                      default=False,
                      help="write to the disk reconstructed 3D average")
    parser.add_option("--var3D",
                      type="string",
                      default=False,
                      help="compute 3D variability (time consuming!)")
    parser.add_option("--img_per_grp",
                      type="int",
                      default=10,
                      help="number of neighbouring projections")
    parser.add_option("--no_norm",
                      action="store_true",
                      default=False,
                      help="do not use normalization")
    #parser.add_option("--radius", 	    type="int"         ,	default=-1   ,				help="radius for 3D variability" )
    parser.add_option("--npad",
                      type="int",
                      default=2,
                      help="number of time to pad the original images")
    parser.add_option("--sym", type="string", default="c1", help="symmetry")
    parser.add_option(
        "--fl",
        type="float",
        default=0.0,
        help=
        "cutoff freqency in absolute frequency (0.0-0.5). (Default - no filtration)"
    )
    parser.add_option(
        "--aa",
        type="float",
        default=0.0,
        help=
        "fall off of the filter. Put 0.01 if user has no clue about falloff (Default - no filtration)"
    )
    parser.add_option("--CTF",
                      action="store_true",
                      default=False,
                      help="use CFT correction")
    parser.add_option("--VERBOSE",
                      action="store_true",
                      default=False,
                      help="Long output for debugging")
    #parser.add_option("--MPI" , 		action="store_true",	default=False,				help="use MPI version")
    #parser.add_option("--radiuspca", 	type="int"         ,	default=-1   ,				help="radius for PCA" )
    #parser.add_option("--iter", 		type="int"         ,	default=40   ,				help="maximum number of iterations (stop criterion of reconstruction process)" )
    #parser.add_option("--abs", 		type="float"   ,        default=0.0  ,				help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
    #parser.add_option("--squ", 		type="float"   ,	    default=0.0  ,				help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
    parser.add_option(
        "--VAR",
        action="store_true",
        default=False,
        help="stack on input consists of 2D variances (Default False)")
    parser.add_option(
        "--decimate",
        type="float",
        default=1.0,
        help=
        "image decimate rate, a number larger (expand image) or less (shrink image) than 1. default is 1"
    )
    parser.add_option(
        "--window",
        type="int",
        default=0,
        help=
        "reduce images to a small image size without changing pixel_size. Default value is zero."
    )
    #parser.add_option("--SND",			action="store_true",	default=False,				help="compute squared normalized differences (Default False)")
    parser.add_option(
        "--nvec",
        type="int",
        default=0,
        help="number of eigenvectors, default = 0 meaning no PCA calculated")
    parser.add_option(
        "--symmetrize",
        action="store_true",
        default=False,
        help="Prepare input stack for handling symmetry (Default False)")

    (options, args) = parser.parse_args()
    #####
    from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD
    from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
    from applications import MPI_start_end
    from reconstruction import recons3d_em, recons3d_em_MPI
    from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
    from utilities import print_begin_msg, print_end_msg, print_msg
    from utilities import read_text_row, get_image, get_im
    from utilities import bcast_EMData_to_all, bcast_number_to_all
    from utilities import get_symt

    #  This is code for handling symmetries by the above program.  To be incorporated. PAP 01/27/2015

    from EMAN2db import db_open_dict

    # Set up global variables related to bdb cache
    if global_def.CACHE_DISABLE:
        from utilities import disable_bdb_cache
        disable_bdb_cache()

    # Set up global variables related to ERROR function
    global_def.BATCH = True

    # detect if program is running under MPI
    RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ
    if RUNNING_UNDER_MPI:
        global_def.MPI = True

    if options.symmetrize:
        if RUNNING_UNDER_MPI:
            try:
                sys.argv = mpi_init(len(sys.argv), sys.argv)
                try:
                    number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
                    if (number_of_proc > 1):
                        ERROR(
                            "Cannot use more than one CPU for symmetry prepration",
                            "sx3dvariability", 1)
                except:
                    pass
            except:
                pass
        if options.output_dir != "./" and not os.path.exists(
                options.output_dir):
            os.mkdir(options.output_dir)
        #  Input
        #instack = "Clean_NORM_CTF_start_wparams.hdf"
        #instack = "bdb:data"

        from logger import Logger, BaseLogger_Files
        if os.path.exists(os.path.join(options.output_dir, "log.txt")):
            os.remove(os.path.join(options.output_dir, "log.txt"))
        log_main = Logger(BaseLogger_Files())
        log_main.prefix = os.path.join(options.output_dir, "./")

        instack = args[0]
        sym = options.sym.lower()
        if (sym == "c1"):
            ERROR("There is no need to symmetrize stack for C1 symmetry",
                  "sx3dvariability", 1)

        line = ""
        for a in sys.argv:
            line += " " + a
        log_main.add(line)

        if (instack[:4] != "bdb:"):
            if output_dir == "./": stack = "bdb:data"
            else: stack = "bdb:" + options.output_dir + "/data"
            delete_bdb(stack)
            junk = cmdexecute("sxcpy.py  " + instack + "  " + stack)
        else:
            stack = instack

        qt = EMUtil.get_all_attributes(stack, 'xform.projection')

        na = len(qt)
        ts = get_symt(sym)
        ks = len(ts)
        angsa = [None] * na

        for k in xrange(ks):
            #Qfile = "Q%1d"%k
            if options.output_dir != "./":
                Qfile = os.path.join(options.output_dir, "Q%1d" % k)
            else:
                Qfile = os.path.join(options.output_dir, "Q%1d" % k)
            #delete_bdb("bdb:Q%1d"%k)
            delete_bdb("bdb:" + Qfile)
            #junk = cmdexecute("e2bdb.py  "+stack+"  --makevstack=bdb:Q%1d"%k)
            junk = cmdexecute("e2bdb.py  " + stack + "  --makevstack=bdb:" +
                              Qfile)
            #DB = db_open_dict("bdb:Q%1d"%k)
            DB = db_open_dict("bdb:" + Qfile)
            for i in xrange(na):
                ut = qt[i] * ts[k]
                DB.set_attr(i, "xform.projection", ut)
                #bt = ut.get_params("spider")
                #angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
            #write_text_row(angsa, 'ptsma%1d.txt'%k)
            #junk = cmdexecute("e2bdb.py  "+stack+"  --makevstack=bdb:Q%1d"%k)
            #junk = cmdexecute("sxheader.py  bdb:Q%1d  --params=xform.projection  --import=ptsma%1d.txt"%(k,k))
            DB.close()
        if options.output_dir == "./": delete_bdb("bdb:sdata")
        else: delete_bdb("bdb:" + options.output_dir + "/" + "sdata")
        #junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
        sdata = "bdb:" + options.output_dir + "/" + "sdata"
        print(sdata)
        junk = cmdexecute("e2bdb.py   " + options.output_dir +
                          "  --makevstack=" + sdata + " --filt=Q")
        #junk = cmdexecute("ls  EMAN2DB/sdata*")
        #a = get_im("bdb:sdata")
        a = get_im(sdata)
        a.set_attr("variabilitysymmetry", sym)
        #a.write_image("bdb:sdata")
        a.write_image(sdata)

    else:

        sys.argv = mpi_init(len(sys.argv), sys.argv)
        myid = mpi_comm_rank(MPI_COMM_WORLD)
        number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
        main_node = 0

        if len(args) == 1:
            stack = args[0]
        else:
            print(("usage: " + usage))
            print(("Please run '" + progname + " -h' for detailed options"))
            return 1

        t0 = time()
        # obsolete flags
        options.MPI = True
        options.nvec = 0
        options.radiuspca = -1
        options.iter = 40
        options.abs = 0.0
        options.squ = 0.0

        if options.fl > 0.0 and options.aa == 0.0:
            ERROR("Fall off has to be given for the low-pass filter",
                  "sx3dvariability", 1, myid)
        if options.VAR and options.SND:
            ERROR("Only one of var and SND can be set!", "sx3dvariability",
                  myid)
            exit()
        if options.VAR and (options.ave2D or options.ave3D or options.var2D):
            ERROR(
                "When VAR is set, the program cannot output ave2D, ave3D or var2D",
                "sx3dvariability", 1, myid)
            exit()
        #if options.SND and (options.ave2D or options.ave3D):
        #	ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid)
        #	exit()
        if options.nvec > 0:
            ERROR("PCA option not implemented", "sx3dvariability", 1, myid)
            exit()
        if options.nvec > 0 and options.ave3D == None:
            ERROR("When doing PCA analysis, one must set ave3D",
                  "sx3dvariability",
                  myid=myid)
            exit()
        import string
        options.sym = options.sym.lower()

        # if global_def.CACHE_DISABLE:
        # 	from utilities import disable_bdb_cache
        # 	disable_bdb_cache()
        # global_def.BATCH = True

        if myid == main_node:
            if options.output_dir != "./" and not os.path.exists(
                    options.output_dir):
                os.mkdir(options.output_dir)

        img_per_grp = options.img_per_grp
        nvec = options.nvec
        radiuspca = options.radiuspca

        from logger import Logger, BaseLogger_Files
        #if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt"))
        log_main = Logger(BaseLogger_Files())
        log_main.prefix = os.path.join(options.output_dir, "./")

        if myid == main_node:
            line = ""
            for a in sys.argv:
                line += " " + a
            log_main.add(line)
            log_main.add("-------->>>Settings given by all options<<<-------")
            log_main.add("instack  		    :" + stack)
            log_main.add("output_dir        :" + options.output_dir)
            log_main.add("var3d   		    :" + options.var3D)

        if myid == main_node:
            line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
            #print_begin_msg("sx3dvariability")
            msg = "sx3dvariability"
            log_main.add(msg)
            print(line, msg)
            msg = ("%-70s:  %s\n" % ("Input stack", stack))
            log_main.add(msg)
            print(line, msg)

        symbaselen = 0
        if myid == main_node:
            nima = EMUtil.get_image_count(stack)
            img = get_image(stack)
            nx = img.get_xsize()
            ny = img.get_ysize()
            if options.sym != "c1":
                imgdata = get_im(stack)
                try:
                    i = imgdata.get_attr("variabilitysymmetry").lower()
                    if (i != options.sym):
                        ERROR(
                            "The symmetry provided does not agree with the symmetry of the input stack",
                            "sx3dvariability",
                            myid=myid)
                except:
                    ERROR(
                        "Input stack is not prepared for symmetry, please follow instructions",
                        "sx3dvariability",
                        myid=myid)
                from utilities import get_symt
                i = len(get_symt(options.sym))
                if ((nima / i) * i != nima):
                    ERROR(
                        "The length of the input stack is incorrect for symmetry processing",
                        "sx3dvariability",
                        myid=myid)
                symbaselen = nima / i
            else:
                symbaselen = nima
        else:
            nima = 0
            nx = 0
            ny = 0
        nima = bcast_number_to_all(nima)
        nx = bcast_number_to_all(nx)
        ny = bcast_number_to_all(ny)
        Tracker = {}
        Tracker["total_stack"] = nima
        if options.decimate == 1.:
            if options.window != 0:
                nx = options.window
                ny = options.window
        else:
            if options.window == 0:
                nx = int(nx * options.decimate)
                ny = int(ny * options.decimate)
            else:
                nx = int(options.window * options.decimate)
                ny = nx
        Tracker["nx"] = nx
        Tracker["ny"] = ny
        Tracker["nz"] = nx
        symbaselen = bcast_number_to_all(symbaselen)
        if radiuspca == -1: radiuspca = nx / 2 - 2

        if myid == main_node:
            line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
            msg = "%-70s:  %d\n" % ("Number of projection", nima)
            log_main.add(msg)
            print(line, msg)
        img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
        """
		if options.SND:
			from projection		import prep_vol, prgs
			from statistics		import im_diff
			from utilities		import get_im, model_circle, get_params_proj, set_params_proj
			from utilities		import get_ctf, generate_ctf
			from filter			import filt_ctf
		
			imgdata = EMData.read_images(stack, range(img_begin, img_end))

			if options.CTF:
				vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
			else:
				vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)

			bcast_EMData_to_all(vol, myid)
			volft, kb = prep_vol(vol)

			mask = model_circle(nx/2-2, nx, ny)
			varList = []
			for i in xrange(img_begin, img_end):
				phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
				ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
				if options.CTF:
					ctf_params = get_ctf(imgdata[i-img_begin])
					ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
				diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
				diff2 = diff*diff
				set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
				varList.append(diff2)
			mpi_barrier(MPI_COMM_WORLD)
		"""
        if options.VAR:
            #varList   = EMData.read_images(stack, range(img_begin, img_end))
            varList = []
            this_image = EMData()
            for index_of_particle in xrange(img_begin, img_end):
                this_image.read_image(stack, index_of_particle)
                varList.append(
                    image_decimate_window_xform_ctf(this_image,
                                                    options.decimate,
                                                    options.window,
                                                    options.CTF))
        else:
            from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
            from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
            from utilities import model_blank, nearest_proj, model_circle
            from applications import pca
            from statistics import avgvar, avgvar_ctf, ccc
            from filter import filt_tanl
            from morphology import threshold, square_root
            from projection import project, prep_vol, prgs
            from sets import Set

            if myid == main_node:
                t1 = time()
                proj_angles = []
                aveList = []
                tab = EMUtil.get_all_attributes(stack, 'xform.projection')
                for i in xrange(nima):
                    t = tab[i].get_params('spider')
                    phi = t['phi']
                    theta = t['theta']
                    psi = t['psi']
                    x = theta
                    if x > 90.0: x = 180.0 - x
                    x = x * 10000 + psi
                    proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
                t2 = time()
                line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
                msg = "%-70s:  %d\n" % ("Number of neighboring projections",
                                        img_per_grp)
                log_main.add(msg)
                print(line, msg)
                msg = "...... Finding neighboring projections\n"
                log_main.add(msg)
                print(line, msg)
                if options.VERBOSE:
                    msg = "Number of images per group: %d" % img_per_grp
                    log_main.add(msg)
                    print(line, msg)
                    msg = "Now grouping projections"
                    log_main.add(msg)
                    print(line, msg)
                proj_angles.sort()
            proj_angles_list = [0.0] * (nima * 4)
            if myid == main_node:
                for i in xrange(nima):
                    proj_angles_list[i * 4] = proj_angles[i][1]
                    proj_angles_list[i * 4 + 1] = proj_angles[i][2]
                    proj_angles_list[i * 4 + 2] = proj_angles[i][3]
                    proj_angles_list[i * 4 + 3] = proj_angles[i][4]
            proj_angles_list = bcast_list_to_all(proj_angles_list, myid,
                                                 main_node)
            proj_angles = []
            for i in xrange(nima):
                proj_angles.append([
                    proj_angles_list[i * 4], proj_angles_list[i * 4 + 1],
                    proj_angles_list[i * 4 + 2],
                    int(proj_angles_list[i * 4 + 3])
                ])
            del proj_angles_list
            proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp,
                                                  range(img_begin, img_end))

            all_proj = Set()
            for im in proj_list:
                for jm in im:
                    all_proj.add(proj_angles[jm][3])

            all_proj = list(all_proj)
            if options.VERBOSE:
                print("On node %2d, number of images needed to be read = %5d" %
                      (myid, len(all_proj)))

            index = {}
            for i in xrange(len(all_proj)):
                index[all_proj[i]] = i
            mpi_barrier(MPI_COMM_WORLD)

            if myid == main_node:
                line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
                msg = ("%-70s:  %.2f\n" %
                       ("Finding neighboring projections lasted [s]",
                        time() - t2))
                log_main.add(msg)
                print(msg)
                msg = ("%-70s:  %d\n" %
                       ("Number of groups processed on the main node",
                        len(proj_list)))
                log_main.add(msg)
                print(line, msg)
                if options.VERBOSE:
                    print("Grouping projections took: ", (time() - t2) / 60,
                          "[min]")
                    print("Number of groups on main node: ", len(proj_list))
            mpi_barrier(MPI_COMM_WORLD)

            if myid == main_node:
                line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
                msg = ("...... calculating the stack of 2D variances \n")
                log_main.add(msg)
                print(line, msg)
                if options.VERBOSE:
                    print("Now calculating the stack of 2D variances")

            proj_params = [0.0] * (nima * 5)
            aveList = []
            varList = []
            if nvec > 0:
                eigList = [[] for i in xrange(nvec)]

            if options.VERBOSE:
                print("Begin to read images on processor %d" % (myid))
            ttt = time()
            #imgdata = EMData.read_images(stack, all_proj)
            imgdata = []
            for index_of_proj in xrange(len(all_proj)):
                #img     = EMData()
                #img.read_image(stack, all_proj[index_of_proj])
                dmg = image_decimate_window_xform_ctf(
                    get_im(stack, all_proj[index_of_proj]), options.decimate,
                    options.window, options.CTF)
                #print dmg.get_xsize(), "init"
                imgdata.append(dmg)
            if options.VERBOSE:
                print("Reading images on processor %d done, time = %.2f" %
                      (myid, time() - ttt))
                print("On processor %d, we got %d images" %
                      (myid, len(imgdata)))
            mpi_barrier(MPI_COMM_WORLD)
            '''	
			imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
			if options.fl > 0.0:
				for k in xrange(len(imgdata2)):
					imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
			if options.CTF:
				vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
			else:
				vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
			if myid == main_node:
				vol.write_image("vol_ctf.hdf")
				print_msg("Writing to the disk volume reconstructed from averages as		:  %s\n"%("vol_ctf.hdf"))
			del vol, imgdata2
			mpi_barrier(MPI_COMM_WORLD)
			'''
            from applications import prepare_2d_forPCA
            from utilities import model_blank
            for i in xrange(len(proj_list)):
                ki = proj_angles[proj_list[i][0]][3]
                if ki >= symbaselen: continue
                mi = index[ki]
                phiM, thetaM, psiM, s2xM, s2yM = get_params_proj(imgdata[mi])

                grp_imgdata = []
                for j in xrange(img_per_grp):
                    mj = index[proj_angles[proj_list[i][j]][3]]
                    phi, theta, psi, s2x, s2y = get_params_proj(imgdata[mj])
                    alpha, sx, sy, mirror = params_3D_2D_NEW(
                        phi, theta, psi, s2x, s2y, mirror_list[i][j])
                    if thetaM <= 90:
                        if mirror == 0:
                            alpha, sx, sy, scale = compose_transform2(
                                alpha, sx, sy, 1.0, phiM - phi, 0.0, 0.0, 1.0)
                        else:
                            alpha, sx, sy, scale = compose_transform2(
                                alpha, sx, sy, 1.0, 180 - (phiM - phi), 0.0,
                                0.0, 1.0)
                    else:
                        if mirror == 0:
                            alpha, sx, sy, scale = compose_transform2(
                                alpha, sx, sy, 1.0, -(phiM - phi), 0.0, 0.0,
                                1.0)
                        else:
                            alpha, sx, sy, scale = compose_transform2(
                                alpha, sx, sy, 1.0, -(180 - (phiM - phi)), 0.0,
                                0.0, 1.0)
                    set_params2D(imgdata[mj], [alpha, sx, sy, mirror, 1.0])
                    grp_imgdata.append(imgdata[mj])
                    #print grp_imgdata[j].get_xsize(), imgdata[mj].get_xsize()

                if not options.no_norm:
                    #print grp_imgdata[j].get_xsize()
                    mask = model_circle(nx / 2 - 2, nx, nx)
                    for k in xrange(img_per_grp):
                        ave, std, minn, maxx = Util.infomask(
                            grp_imgdata[k], mask, False)
                        grp_imgdata[k] -= ave
                        grp_imgdata[k] /= std
                    del mask

                if options.fl > 0.0:
                    from filter import filt_ctf, filt_table
                    from fundamentals import fft, window2d
                    nx2 = 2 * nx
                    ny2 = 2 * ny
                    if options.CTF:
                        from utilities import pad
                        for k in xrange(img_per_grp):
                            grp_imgdata[k] = window2d(
                                fft(
                                    filt_tanl(
                                        filt_ctf(
                                            fft(
                                                pad(grp_imgdata[k], nx2, ny2,
                                                    1, 0.0)),
                                            grp_imgdata[k].get_attr("ctf"),
                                            binary=1), options.fl,
                                        options.aa)), nx, ny)
                            #grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
                            #grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
                    else:
                        for k in xrange(img_per_grp):
                            grp_imgdata[k] = filt_tanl(grp_imgdata[k],
                                                       options.fl, options.aa)
                            #grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
                            #grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
                else:
                    from utilities import pad, read_text_file
                    from filter import filt_ctf, filt_table
                    from fundamentals import fft, window2d
                    nx2 = 2 * nx
                    ny2 = 2 * ny
                    if options.CTF:
                        from utilities import pad
                        for k in xrange(img_per_grp):
                            grp_imgdata[k] = window2d(
                                fft(
                                    filt_ctf(fft(
                                        pad(grp_imgdata[k], nx2, ny2, 1, 0.0)),
                                             grp_imgdata[k].get_attr("ctf"),
                                             binary=1)), nx, ny)
                            #grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
                            #grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
                '''
				if i < 10 and myid == main_node:
					for k in xrange(10):
						grp_imgdata[k].write_image("grp%03d.hdf"%i, k)
				'''
                """
				if myid == main_node and i==0:
					for pp in xrange(len(grp_imgdata)):
						grp_imgdata[pp].write_image("pp.hdf", pp)
				"""
                ave, grp_imgdata = prepare_2d_forPCA(grp_imgdata)
                """
				if myid == main_node and i==0:
					for pp in xrange(len(grp_imgdata)):
						grp_imgdata[pp].write_image("qq.hdf", pp)
				"""

                var = model_blank(nx, ny)
                for q in grp_imgdata:
                    Util.add_img2(var, q)
                Util.mul_scalar(var, 1.0 / (len(grp_imgdata) - 1))
                # Switch to std dev
                var = square_root(threshold(var))
                #if options.CTF:	ave, var = avgvar_ctf(grp_imgdata, mode="a")
                #else:	            ave, var = avgvar(grp_imgdata, mode="a")
                """
				if myid == main_node:
					ave.write_image("avgv.hdf",i)
					var.write_image("varv.hdf",i)
				"""

                set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
                set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])

                aveList.append(ave)
                varList.append(var)

                if options.VERBOSE:
                    print("%5.2f%% done on processor %d" %
                          (i * 100.0 / len(proj_list), myid))
                if nvec > 0:
                    eig = pca(input_stacks=grp_imgdata,
                              subavg="",
                              mask_radius=radiuspca,
                              nvec=nvec,
                              incore=True,
                              shuffle=False,
                              genbuf=True)
                    for k in xrange(nvec):
                        set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
                        eigList[k].append(eig[k])
                    """
					if myid == 0 and i == 0:
						for k in xrange(nvec):
							eig[k].write_image("eig.hdf", k)
					"""

            del imgdata
            #  To this point, all averages, variances, and eigenvectors are computed

            if options.ave2D:
                from fundamentals import fpol
                if myid == main_node:
                    km = 0
                    for i in xrange(number_of_proc):
                        if i == main_node:
                            for im in xrange(len(aveList)):
                                aveList[im].write_image(
                                    os.path.join(options.output_dir,
                                                 options.ave2D), km)
                                km += 1
                        else:
                            nl = mpi_recv(1, MPI_INT, i,
                                          SPARX_MPI_TAG_UNIVERSAL,
                                          MPI_COMM_WORLD)
                            nl = int(nl[0])
                            for im in xrange(nl):
                                ave = recv_EMData(i, im + i + 70000)
                                """
								nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
								nm = int(nm[0])
								members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
								ave.set_attr('members', map(int, members))
								members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
								ave.set_attr('pix_err', map(float, members))
								members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
								ave.set_attr('refprojdir', map(float, members))
								"""
                                tmpvol = fpol(ave, Tracker["nx"],
                                              Tracker["nx"], 1)
                                tmpvol.write_image(
                                    os.path.join(options.output_dir,
                                                 options.ave2D), km)
                                km += 1
                else:
                    mpi_send(len(aveList), 1, MPI_INT, main_node,
                             SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
                    for im in xrange(len(aveList)):
                        send_EMData(aveList[im], main_node, im + myid + 70000)
                        """
						members = aveList[im].get_attr('members')
						mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						members = aveList[im].get_attr('pix_err')
						mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						try:
							members = aveList[im].get_attr('refprojdir')
							mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						except:
							mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						"""

            if options.ave3D:
                from fundamentals import fpol
                if options.VERBOSE:
                    print("Reconstructing 3D average volume")
                ave3D = recons3d_4nn_MPI(myid,
                                         aveList,
                                         symmetry=options.sym,
                                         npad=options.npad)
                bcast_EMData_to_all(ave3D, myid)
                if myid == main_node:
                    line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
                    ave3D = fpol(ave3D, Tracker["nx"], Tracker["nx"],
                                 Tracker["nx"])
                    ave3D.write_image(
                        os.path.join(options.output_dir, options.ave3D))
                    msg = ("%-70s:  %s\n" % (
                        "Writing to the disk volume reconstructed from averages as",
                        options.ave3D))
                    log_main.add(msg)
                    print(line, msg)
            del ave, var, proj_list, stack, phi, theta, psi, s2x, s2y, alpha, sx, sy, mirror, aveList

            if nvec > 0:
                for k in xrange(nvec):
                    if options.VERBOSE:
                        print("Reconstruction eigenvolumes", k)
                    cont = True
                    ITER = 0
                    mask2d = model_circle(radiuspca, nx, nx)
                    while cont:
                        #print "On node %d, iteration %d"%(myid, ITER)
                        eig3D = recons3d_4nn_MPI(myid,
                                                 eigList[k],
                                                 symmetry=options.sym,
                                                 npad=options.npad)
                        bcast_EMData_to_all(eig3D, myid, main_node)
                        if options.fl > 0.0:
                            eig3D = filt_tanl(eig3D, options.fl, options.aa)
                        if myid == main_node:
                            eig3D.write_image(
                                os.path.join(options.outpout_dir,
                                             "eig3d_%03d.hdf" % (k, ITER)))
                        Util.mul_img(eig3D,
                                     model_circle(radiuspca, nx, nx, nx))
                        eig3Df, kb = prep_vol(eig3D)
                        del eig3D
                        cont = False
                        icont = 0
                        for l in xrange(len(eigList[k])):
                            phi, theta, psi, s2x, s2y = get_params_proj(
                                eigList[k][l])
                            proj = prgs(eig3Df, kb,
                                        [phi, theta, psi, s2x, s2y])
                            cl = ccc(proj, eigList[k][l], mask2d)
                            if cl < 0.0:
                                icont += 1
                                cont = True
                                eigList[k][l] *= -1.0
                        u = int(cont)
                        u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node,
                                       MPI_COMM_WORLD)
                        icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM,
                                           main_node, MPI_COMM_WORLD)

                        if myid == main_node:
                            line = strftime("%Y-%m-%d_%H:%M:%S",
                                            localtime()) + " =>"
                            u = int(u[0])
                            msg = (" Eigenvector: ", k, " number changed ",
                                   int(icont[0]))
                            log_main.add(msg)
                            print(line, msg)
                        else:
                            u = 0
                        u = bcast_number_to_all(u, main_node)
                        cont = bool(u)
                        ITER += 1

                    del eig3Df, kb
                    mpi_barrier(MPI_COMM_WORLD)
                del eigList, mask2d

            if options.ave3D: del ave3D
            if options.var2D:
                from fundamentals import fpol
                if myid == main_node:
                    km = 0
                    for i in xrange(number_of_proc):
                        if i == main_node:
                            for im in xrange(len(varList)):
                                tmpvol = fpol(varList[im], Tracker["nx"],
                                              Tracker["nx"], 1)
                                tmpvol.write_image(
                                    os.path.join(options.output_dir,
                                                 options.var2D), km)
                                km += 1
                        else:
                            nl = mpi_recv(1, MPI_INT, i,
                                          SPARX_MPI_TAG_UNIVERSAL,
                                          MPI_COMM_WORLD)
                            nl = int(nl[0])
                            for im in xrange(nl):
                                ave = recv_EMData(i, im + i + 70000)
                                tmpvol = fpol(ave, Tracker["nx"],
                                              Tracker["nx"], 1)
                                tmpvol.write_image(
                                    os.path.join(options.output_dir,
                                                 options.var2D, km))
                                km += 1
                else:
                    mpi_send(len(varList), 1, MPI_INT, main_node,
                             SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
                    for im in xrange(len(varList)):
                        send_EMData(varList[im], main_node, im + myid +
                                    70000)  #  What with the attributes??

            mpi_barrier(MPI_COMM_WORLD)

        if options.var3D:
            if myid == main_node and options.VERBOSE:
                line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
                msg = ("Reconstructing 3D variability volume")
                log_main.add(msg)
                print(line, msg)
            t6 = time()
            # radiusvar = options.radius
            # if( radiusvar < 0 ):  radiusvar = nx//2 -3
            res = recons3d_4nn_MPI(myid,
                                   varList,
                                   symmetry=options.sym,
                                   npad=options.npad)
            #res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
            if myid == main_node:
                from fundamentals import fpol
                res = fpol(res, Tracker["nx"], Tracker["nx"], Tracker["nx"])
                res.write_image(os.path.join(options.output_dir,
                                             options.var3D))

            if myid == main_node:
                line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
                msg = ("%-70s:  %.2f\n" %
                       ("Reconstructing 3D variability took [s]", time() - t6))
                log_main.add(msg)
                print(line, msg)
                if options.VERBOSE:
                    print("Reconstruction took: %.2f [min]" %
                          ((time() - t6) / 60))

            if myid == main_node:
                line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
                msg = ("%-70s:  %.2f\n" %
                       ("Total time for these computations [s]", time() - t0))
                print(line, msg)
                log_main.add(msg)
                if options.VERBOSE:
                    print("Total time for these computations: %.2f [min]" %
                          ((time() - t0) / 60))
                line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
                msg = ("sx3dvariability")
                print(line, msg)
                log_main.add(msg)

        from mpi import mpi_finalize
        mpi_finalize()

        if RUNNING_UNDER_MPI:
            global_def.MPI = False

        global_def.BATCH = False
Exemplo n.º 16
0
def main():
    arglist = []
    for arg in sys.argv:
        arglist.append(arg)

    progname = os.path.basename(arglist[0])
    usage = progname + " prj_stack volume [begin end step] --CTF --npad=ntimes_padding --list=file --group=ID --snr=SNR --sym=symmetry --verbose=(0|1) --xysize --MPI"
    parser = OptionParser(usage, version=SPARXVERSION)

    parser.add_option("--CTF",
                      action="store_true",
                      default=False,
                      help="apply CTF correction")
    parser.add_option("--snr",
                      type="float",
                      default=1.0,
                      help="Signal-to-Noise Ratio")
    parser.add_option("--sym", type="string", default="c1", help="symmetry")
    parser.add_option(
        "--list",
        type="string",
        help="file with list of images to be used in the first column")
    parser.add_option(
        "--group",
        type="int",
        default=-1,
        help=
        "perform reconstruction using images for a given group number (group is attribute in the header)"
    )
    parser.add_option("--MPI",
                      action="store_true",
                      default=False,
                      help="use MPI version ")
    parser.add_option("--npad",
                      type="int",
                      default=2,
                      help="number of times padding (default 2)")
    parser.add_option("--verbose",
                      type="int",
                      default=0,
                      help="verbose level: 0 no verbose, 1 verbose")
    parser.add_option("--xysize",
                      type="int",
                      default=-1,
                      help="user expected size at xy direction")
    parser.add_option("--zsize",
                      type="int",
                      default=-1,
                      help="user expected size at z direction")
    parser.add_option("--smearstep",
                      type="float",
                      default=0.0,
                      help="Rotational smear step (default 0.0, no smear)")

    (options, args) = parser.parse_args(arglist[1:])

    if options.MPI:
        from mpi import mpi_init
        sys.argv = mpi_init(len(sys.argv), sys.argv)

    if global_def.CACHE_DISABLE:
        from utilities import disable_bdb_cache
        disable_bdb_cache()

    if len(args) == 2:
        prj_stack = args[0]
        vol_stack = args[1]
        nimage = EMUtil.get_image_count(prj_stack)
        pid_list = range(0, nimage)
    elif len(args) == 5:
        prj_stack = args[0]
        vol_stack = args[1]
        begin = atoi(args[2])
        end = atoi(args[3])
        step = atoi(args[4])
        pid_list = range(begin, end, step)
    else:
        ERROR("incomplete list of arguments", "recon3d_n", 1)
        exit()

    if (options.list and options.group > -1):
        ERROR("options group and list cannot be used together", "recon3d_n", 1)
        sys.exit()

    from applications import recons3d_n

    global_def.BATCH = True
    recons3d_n(prj_stack, pid_list, vol_stack, options.CTF, options.snr, 1, options.npad,\
      options.sym, options.list, options.group, options.verbose, options.MPI,options.xysize, options.zsize, options.smearstep)
    global_def.BATCH = False

    if options.MPI:
        from mpi import mpi_finalize
        mpi_finalize()
Exemplo n.º 17
0
def main():
	import	global_def
	from	optparse 	import OptionParser
	from	EMAN2 		import EMUtil
	import	os
	import	sys
	from time import time

	progname = os.path.basename(sys.argv[0])
	usage = progname + " proj_stack output_averages --MPI"
	parser = OptionParser(usage, version=SPARXVERSION)

	parser.add_option("--img_per_group",type="int"         ,	default=100  ,				help="number of images per group" )
	parser.add_option("--radius", 		type="int"         ,	default=-1   ,				help="radius for alignment" )
	parser.add_option("--xr",           type="string"      ,    default="2 1",              help="range for translation search in x direction, search is +/xr")
	parser.add_option("--yr",           type="string"      ,    default="-1",               help="range for translation search in y direction, search is +/yr (default = same as xr)")
	parser.add_option("--ts",           type="string"      ,    default="1 0.5",            help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional")
	parser.add_option("--iter", 		type="int"         ,	default=30,                 help="number of iterations within alignment (default = 30)" )
	parser.add_option("--num_ali",      type="int"     	   ,    default=5,         			help="number of alignments performed for stability (default = 5)" )
	parser.add_option("--thld_err",     type="float"       ,    default=1.0,         		help="threshold of pixel error (default = 1.732)" )
	parser.add_option("--grouping" , 	type="string"      ,	default="GRP",				help="do grouping of projections: PPR - per projection, GRP - different size groups, exclusive (default), GEV - grouping equal size")
	parser.add_option("--delta",        type="float"       ,    default=-1.0,         		help="angular step for reference projections (required for GEV method)")
	parser.add_option("--fl",           type="float"       ,    default=0.3,                help="cut-off frequency of hyperbolic tangent low-pass Fourier filter")
	parser.add_option("--aa",           type="float"       ,    default=0.2,                help="fall-off of hyperbolic tangent low-pass Fourier filter")
	parser.add_option("--CTF",          action="store_true",    default=False,              help="Consider CTF correction during the alignment ")
	parser.add_option("--MPI" , 		action="store_true",	default=False,				help="use MPI version")

	(options,args) = parser.parse_args()
	
	from mpi          import mpi_init, mpi_comm_rank, mpi_comm_size, MPI_COMM_WORLD, MPI_TAG_UB
	from mpi          import mpi_barrier, mpi_send, mpi_recv, mpi_bcast, MPI_INT, mpi_finalize, MPI_FLOAT
	from applications import MPI_start_end, within_group_refinement, ali2d_ras
	from pixel_error  import multi_align_stability
	from utilities    import send_EMData, recv_EMData
	from utilities    import get_image, bcast_number_to_all, set_params2D, get_params2D
	from utilities    import group_proj_by_phitheta, model_circle, get_input_from_string

	sys.argv = mpi_init(len(sys.argv), sys.argv)
	myid = mpi_comm_rank(MPI_COMM_WORLD)
	number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
	main_node = 0

	if len(args) == 2:
		stack  = args[0]
		outdir = args[1]
	else:
		ERROR("incomplete list of arguments", "sxproj_stability", 1, myid=myid)
		exit()
	if not options.MPI:
		ERROR("Non-MPI not supported!", "sxproj_stability", myid=myid)
		exit()		 

	if global_def.CACHE_DISABLE:
		from utilities import disable_bdb_cache
		disable_bdb_cache()
	global_def.BATCH = True

	#if os.path.exists(outdir):  ERROR('Output directory exists, please change the name and restart the program', "sxproj_stability", 1, myid)
	#mpi_barrier(MPI_COMM_WORLD)

	
	img_per_grp = options.img_per_group
	radius = options.radius
	ite = options.iter
	num_ali = options.num_ali
	thld_err = options.thld_err

	xrng        = get_input_from_string(options.xr)
	if  options.yr == "-1":  yrng = xrng
	else          :  yrng = get_input_from_string(options.yr)
	step        = get_input_from_string(options.ts)


	if myid == main_node:
		nima = EMUtil.get_image_count(stack)
		img  = get_image(stack)
		nx   = img.get_xsize()
		ny   = img.get_ysize()
	else:
		nima = 0
		nx = 0
		ny = 0
	nima = bcast_number_to_all(nima)
	nx   = bcast_number_to_all(nx)
	ny   = bcast_number_to_all(ny)
	if radius == -1: radius = nx/2-2
	mask = model_circle(radius, nx, nx)

	st = time()
	if options.grouping == "GRP":
		if myid == main_node:
			print "  A  ",myid,"  ",time()-st
			proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
			proj_params = []
			for i in xrange(nima):
				dp = proj_attr[i].get_params("spider")
				phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp["psi"], -dp["tx"], -dp["ty"]
				proj_params.append([phi, theta, psi, s2x, s2y])

			# Here is where the grouping is done, I didn't put enough annotation in the group_proj_by_phitheta,
			# So I will briefly explain it here
			# proj_list  : Returns a list of list of particle numbers, each list contains img_per_grp particle numbers
			#              except for the last one. Depending on the number of particles left, they will either form a
			#              group or append themselves to the last group
			# angle_list : Also returns a list of list, each list contains three numbers (phi, theta, delta), (phi, 
			#              theta) is the projection angle of the center of the group, delta is the range of this group
			# mirror_list: Also returns a list of list, each list contains img_per_grp True or False, which indicates
			#              whether it should take mirror position.
			# In this program angle_list and mirror list are not of interest.

			proj_list_all, angle_list, mirror_list = group_proj_by_phitheta(proj_params, img_per_grp=img_per_grp)
			del proj_params
			print "  B  number of groups  ",myid,"  ",len(proj_list_all),time()-st
		mpi_barrier(MPI_COMM_WORLD)

		# Number of groups, actually there could be one or two more groups, since the size of the remaining group varies
		# we will simply assign them to main node.
		n_grp = nima/img_per_grp-1

		# Divide proj_list_all equally to all nodes, and becomes proj_list
		proj_list = []
		for i in xrange(n_grp):
			proc_to_stay = i%number_of_proc
			if proc_to_stay == main_node:
				if myid == main_node: 	proj_list.append(proj_list_all[i])
			elif myid == main_node:
				mpi_send(len(proj_list_all[i]), 1, MPI_INT, proc_to_stay, MPI_TAG_UB, MPI_COMM_WORLD)
				mpi_send(proj_list_all[i], len(proj_list_all[i]), MPI_INT, proc_to_stay, MPI_TAG_UB, MPI_COMM_WORLD)
			elif myid == proc_to_stay:
				img_per_grp = mpi_recv(1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
				img_per_grp = int(img_per_grp[0])
				temp = mpi_recv(img_per_grp, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
				proj_list.append(map(int, temp))
				del temp
			mpi_barrier(MPI_COMM_WORLD)
		print "  C  ",myid,"  ",time()-st
		if myid == main_node:
			# Assign the remaining groups to main_node
			for i in xrange(n_grp, len(proj_list_all)):
				proj_list.append(proj_list_all[i])
			del proj_list_all, angle_list, mirror_list


	#   Compute stability per projection projection direction, equal number assigned, thus overlaps
	elif options.grouping == "GEV":
		if options.delta == -1.0: ERROR("Angular step for reference projections is required for GEV method","sxproj_stability",1)
		from utilities import even_angles, nearestk_to_refdir, getvec
		refproj = even_angles(options.delta)
		img_begin, img_end = MPI_start_end(len(refproj), number_of_proc, myid)
		# Now each processor keeps its own share of reference projections
		refprojdir = refproj[img_begin: img_end]
		del refproj

		ref_ang = [0.0]*(len(refprojdir)*2)
		for i in xrange(len(refprojdir)):
			ref_ang[i*2]   = refprojdir[0][0]
			ref_ang[i*2+1] = refprojdir[0][1]+i*0.1

		print "  A  ",myid,"  ",time()-st
		proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
		#  the solution below is very slow, do not use it unless there is a problem with the i/O
		"""
		for i in xrange(number_of_proc):
			if myid == i:
				proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
			mpi_barrier(MPI_COMM_WORLD)
		"""
		print "  B  ",myid,"  ",time()-st

		proj_ang = [0.0]*(nima*2)
		for i in xrange(nima):
			dp = proj_attr[i].get_params("spider")
			proj_ang[i*2]   = dp["phi"]
			proj_ang[i*2+1] = dp["theta"]
		print "  C  ",myid,"  ",time()-st
		asi = Util.nearestk_to_refdir(proj_ang, ref_ang, img_per_grp)
		del proj_ang, ref_ang
		proj_list = []
		for i in xrange(len(refprojdir)):
			proj_list.append(asi[i*img_per_grp:(i+1)*img_per_grp])
		del asi
		print "  D  ",myid,"  ",time()-st
		#from sys import exit
		#exit()


	#   Compute stability per projection
	elif options.grouping == "PPR":
		print "  A  ",myid,"  ",time()-st
		proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
		print "  B  ",myid,"  ",time()-st
		proj_params = []
		for i in xrange(nima):
			dp = proj_attr[i].get_params("spider")
			phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp["psi"], -dp["tx"], -dp["ty"]
			proj_params.append([phi, theta, psi, s2x, s2y])
		img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
		print "  C  ",myid,"  ",time()-st
		from utilities import nearest_proj
		proj_list, mirror_list = nearest_proj(proj_params, img_per_grp, range(img_begin, img_begin+1))#range(img_begin, img_end))
		refprojdir = proj_params[img_begin: img_end]
		del proj_params, mirror_list
		print "  D  ",myid,"  ",time()-st
	else:  ERROR("Incorrect projection grouping option","sxproj_stability",1)
	"""
	from utilities import write_text_file
	for i in xrange(len(proj_list)):
		write_text_file(proj_list[i],"projlist%06d_%04d"%(i,myid))
	"""

	###########################################################################################################
	# Begin stability test
	from utilities import get_params_proj, read_text_file
	#if myid == 0:
	#	from utilities import read_text_file
	#	proj_list[0] = map(int, read_text_file("lggrpp0.txt"))


	from utilities import model_blank
	aveList = [model_blank(nx,ny)]*len(proj_list)
	if options.grouping == "GRP":  refprojdir = [[0.0,0.0,-1.0]]*len(proj_list)
	for i in xrange(len(proj_list)):
		print "  E  ",myid,"  ",time()-st
		class_data = EMData.read_images(stack, proj_list[i])
		#print "  R  ",myid,"  ",time()-st
		if options.CTF :
			from filter import filt_ctf
			for im in xrange(len(class_data)):  #  MEM LEAK!!
				atemp = class_data[im].copy()
				btemp = filt_ctf(atemp, atemp.get_attr("ctf"), binary=1)
				class_data[im] = btemp
				#class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1)
		for im in class_data:
			try:
				t = im.get_attr("xform.align2d") # if they are there, no need to set them!
			except:
				try:
					t = im.get_attr("xform.projection")
					d = t.get_params("spider")
					set_params2D(im, [0.0,-d["tx"],-d["ty"],0,1.0])
				except:
					set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0])
		#print "  F  ",myid,"  ",time()-st
		# Here, we perform realignment num_ali times
		all_ali_params = []
		for j in xrange(num_ali):
			if( xrng[0] == 0.0 and yrng[0] == 0.0 ):
				avet = ali2d_ras(class_data, randomize = True, ir = 1, ou = radius, rs = 1, step = 1.0, dst = 90.0, maxit = ite, check_mirror = True, FH=options.fl, FF=options.aa)
			else:
				avet = within_group_refinement(class_data, mask, True, 1, radius, 1, xrng, yrng, step, 90.0, ite, options.fl, options.aa)
			ali_params = []
			for im in xrange(len(class_data)):
				alpha, sx, sy, mirror, scale = get_params2D(class_data[im])
				ali_params.extend( [alpha, sx, sy, mirror] )
			all_ali_params.append(ali_params)
		#aveList[i] = avet
		#print "  G  ",myid,"  ",time()-st
		del ali_params
		# We determine the stability of this group here.
		# stable_set contains all particles deemed stable, it is a list of list
		# each list has two elements, the first is the pixel error, the second is the image number
		# stable_set is sorted based on pixel error
		#from utilities import write_text_file
		#write_text_file(all_ali_params, "all_ali_params%03d.txt"%myid)
		stable_set, mir_stab_rate, average_pix_err = multi_align_stability(all_ali_params, 0.0, 10000.0, thld_err, False, 2*radius+1)
		#print "  H  ",myid,"  ",time()-st
		if(len(stable_set) > 5):
			stable_set_id = []
			members = []
			pix_err = []
			# First put the stable members into attr 'members' and 'pix_err'
			for s in stable_set:
				# s[1] - number in this subset
				stable_set_id.append(s[1])
				# the original image number
				members.append(proj_list[i][s[1]])
				pix_err.append(s[0])
			# Then put the unstable members into attr 'members' and 'pix_err'
			from fundamentals import rot_shift2D
			avet.to_zero()
			if options.grouping == "GRP":
				aphi = 0.0
				atht = 0.0
				vphi = 0.0
				vtht = 0.0
			l = -1
			for j in xrange(len(proj_list[i])):
				#  Here it will only work if stable_set_id is sorted in the increasing number, see how l progresses
				if j in stable_set_id:
					l += 1
					avet += rot_shift2D(class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3] )
					if options.grouping == "GRP":
						phi, theta, psi, sxs, sys = get_params_proj(class_data[j])
						if( theta > 90.0):
							phi = (phi+540.0)%360.0
							theta = 180.0 - theta
						aphi += phi
						atht += theta
						vphi += phi*phi
						vtht += theta*theta
				else:
					members.append(proj_list[i][j])
					pix_err.append(99999.99)
			aveList[i] = avet.copy()
			if l>1 :
				l += 1
				aveList[i] /= l
				if options.grouping == "GRP":
					aphi /= l
					atht /= l
					vphi = (vphi - l*aphi*aphi)/l
					vtht = (vtht - l*atht*atht)/l
					from math import sqrt
					refprojdir[i] = [aphi, atht, (sqrt(max(vphi,0.0))+sqrt(max(vtht,0.0)))/2.0]

			# Here more information has to be stored, PARTICULARLY WHAT IS THE REFERENCE DIRECTION
			aveList[i].set_attr('members', members)
			aveList[i].set_attr('refprojdir',refprojdir[i])
			aveList[i].set_attr('pixerr', pix_err)
		else:
			print  " empty group ",i, refprojdir[i]
			aveList[i].set_attr('members',[-1])
			aveList[i].set_attr('refprojdir',refprojdir[i])
			aveList[i].set_attr('pixerr', [99999.])

	del class_data

	if myid == main_node:
		km = 0
		for i in xrange(number_of_proc):
			if i == main_node :
				for im in xrange(len(aveList)):
					aveList[im].write_image(args[1], km)
					km += 1
			else:
				nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
				nl = int(nl[0])
				for im in xrange(nl):
					ave = recv_EMData(i, im+i+70000)
					nm = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
					nm = int(nm[0])
					members = mpi_recv(nm, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
					ave.set_attr('members', map(int, members))
					members = mpi_recv(nm, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
					ave.set_attr('pixerr', map(float, members))
					members = mpi_recv(3, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
					ave.set_attr('refprojdir', map(float, members))
					ave.write_image(args[1], km)
					km += 1
	else:
		mpi_send(len(aveList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
		for im in xrange(len(aveList)):
			send_EMData(aveList[im], main_node,im+myid+70000)
			members = aveList[im].get_attr('members')
			mpi_send(len(members), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
			mpi_send(members, len(members), MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
			members = aveList[im].get_attr('pixerr')
			mpi_send(members, len(members), MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
			try:
				members = aveList[im].get_attr('refprojdir')
				mpi_send(members, 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
			except:
				mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)

	global_def.BATCH = False
	mpi_barrier(MPI_COMM_WORLD)
	from mpi import mpi_finalize
	mpi_finalize()
Exemplo n.º 18
0
def main():
	import os
	import sys
	from optparse import OptionParser
	from global_def import SPARXVERSION
	import global_def
        arglist = []
        for arg in sys.argv:
        	arglist.append( arg )
	progname = os.path.basename(arglist[0])
	usage2 = progname + """ inputfile outputfile [options]
        Functionalities:

        1. Helicise input volume and save the result to output volume:
            sxhelicon_utils.py input_vol.hdf output_vol.hdf --helicise --dp=27.6 --dphi=166.5 --fract=0.65 --rmax=70 --rmin=1 --apix=1.84 --sym=D1        

        2. Helicise pdb file and save the result to a new pdb file:
            sxhelicon_utils.py input.pdb output.pdb --helicisepdb --dp=27.6 --dphi=166.5 --nrepeats --apix=1.84         

        3. Generate two lists of image indices used to split segment stack into halves for helical fsc calculation.			
            sxhelicon_utils.py bdb:big_stack --hfsc='flst' --filament_attr=filament

        4. Map of filament distribution in the stack
            sxhelicon_utils.py bdb:big_stack --filinfo=info.txt
            The output file will contain four columns:
                     1                    2                     3                         4
            first image number     last image number      number of images         in the filament name

        5. Predict segments' orientation parameters based on distances between segments and known helical symmetry
            sxhelicon_utils.py bdb:big_stack --predict_helical=helical_params.txt --dp=27.6 --dphi=166.5 --apix=1.84
            
        6. Generate disks from filament based reconstructions:		
            sxheader.py stk.hdf --params=xform.projection --import=params.txt

			# horatio active_refactoring Jy51i1EwmLD4tWZ9_00000_1
            # sxheader.py stk.hdf --params=active --one

            mpirun -np 2 sxhelicon_utils.py stk.hdf --gendisk='bdb:disk' --ref_nx=100 --ref_ny=100 --ref_nz=200 --apix=1.84 --dp=27.6 --dphi=166.715 --fract=0.67 --rmin=0 --rmax=64 --function="[.,nofunc,helical3c]" --sym="c1" --MPI

        7. Stack disks based on helical symmetry parameters
            sxhelicon_utils.py disk_to_stack.hdf --stackdisk=stacked_disks.hdf --dphi=166.5 --dp=27.6 --ref_nx=160 --ref_ny=160 --ref_nz=225 --apix=1.84
		
        8. Helical symmetry search:
            mpirun -np 3 sxhelicon_utils.py volf0010.hdf outsymsearch --symsearch --dp=27.6 --dphi=166.715 --apix=1.84 --fract=0.65 --rmin=0 --rmax=92.0 --datasym=datasym.txt  --dp_step=0.92 --ndp=3 --dphi_step=1.0 --ndphi=10 --MPI
"""
	parser = OptionParser(usage2,version=SPARXVERSION)
	#parser.add_option("--ir",                 type="float", 	     default= -1,                 help="inner radius for rotational correlation > 0 (set to 1) (Angstroms)")
	parser.add_option("--ou",                 type="float", 	     default= -1,                 help="outer radius for rotational 2D correlation < int(nx/2)-1 (set to the radius of the particle) (Angstroms)")
	parser.add_option("--rs",                 type="int",   		 default= 1,                  help="step between rings in rotational correlation >0  (set to 1)" ) 
	parser.add_option("--xr",                 type="string",		 default= "4 2 1 1 1",        help="range for translation search in x direction, search is +/-xr (Angstroms) ")
	parser.add_option("--txs",                type="string",		 default= "1 1 1 0.5 0.25",   help="step size of the translation search in x directions, search is -xr, -xr+ts, 0, xr-ts, xr (Angstroms)")
	parser.add_option("--delta",              type="string",		 default= "10 6 4 3 2",       help="angular step of reference projections")
	parser.add_option("--an",                 type="string",		 default= "-1",               help="angular neighborhood for local searches")
	parser.add_option("--maxit",              type="int",            default= 30,                 help="maximum number of iterations performed for each angular step (set to 30) ")
	parser.add_option("--CTF",                action="store_true",   default=False,      		  help="CTF correction")
	parser.add_option("--snr",                type="float",          default= 1.0,                help="Signal-to-Noise Ratio of the data")	
	parser.add_option("--MPI",                action="store_true",   default=False,               help="use MPI version")
	#parser.add_option("--fourvar",           action="store_true",   default=False,               help="compute Fourier variance")
	parser.add_option("--apix",               type="float",			 default= -1.0,               help="pixel size in Angstroms")   
	parser.add_option("--dp",                 type="float",			 default= -1.0,               help="delta z - translation in Angstroms")   
	parser.add_option("--dphi",               type="float",			 default= -1.0,               help="delta phi - rotation in degrees")  
		  
	parser.add_option("--rmin",               type="float", 		 default= 0.0,                help="minimal radius for hsearch (Angstroms)")   
	parser.add_option("--rmax",               type="float", 		 default= 80.0,               help="maximal radius for hsearch (Angstroms)")
	parser.add_option("--fract",              type="float", 		 default= 0.7,                help="fraction of the volume used for helical search")
	parser.add_option("--sym",                type="string",		 default= "c1",               help="symmetry of the structure")
	parser.add_option("--function",           type="string",		 default="helical",  	      help="name of the reference preparation function")
	parser.add_option("--npad",               type="int",   		 default= 2,                  help="padding size for 3D reconstruction")
	parser.add_option("--debug",              action="store_true",   default=False,               help="debug")
	
	parser.add_option("--volalixshift",       action="store_true",   default=False,               help="Use volalixshift refinement")
	parser.add_option("--searchxshift",       type="float",		     default= 0.0,                help="search range for x-shift determination: +/- searchxshift (Angstroms)")
	parser.add_option("--nearby",             type="float",		     default= 6.0,                help="neighborhood within which to search for peaks in 1D ccf for x-shift search (Angstroms)")

	# filinfo
	parser.add_option( "--filinfo",            type="string",      	 default="",                  help="Store in an output text file infomration about distribution of filaments in the stack." )


	# diskali
	parser.add_option("--diskali",            action="store_true",   default=False,               help="volume alignment")
	parser.add_option("--zstep",              type="float",          default= 1,                  help="Step size for translational search along z (Angstroms)")   

	# helicise
	parser.add_option("--helicise",           action="store_true",	 default=False,               help="helicise input volume and save results to output volume")
	parser.add_option("--hfsc",               type="string",      	 default="",                  help="Generate two lists of image indices used to split segment stack into halves for helical fsc calculation. The lists will be stored in two text files named using file_prefix with '_even' and '_odd' suffixes, respectively." )
	parser.add_option("--filament_attr",      type="string",      	 default="filament",          help="attribute under which filament identification is stored" )
	parser.add_option("--predict_helical",    type="string",      	 default="",                  help="Generate projection parameters consistent with helical symmetry")

	# helicise pdb
	parser.add_option("--helicisepdb",        action="store_true",	 default=False,               help="Helicise pdb file and save the result to a new pdb file")
	parser.add_option("--nrepeats",           type="int",   		 default= 50,                  help="Number of time the helical symmetry will be applied to the input file")


	# input options for generating disks
	parser.add_option("--gendisk",            type="string",		 default="",                  help="Name of file under which generated disks will be saved to") 
	parser.add_option("--ref_nx",             type="int",   		 default= -1,                 help="nx=ny volume size" ) 
	parser.add_option("--ref_nz",             type="int",   		 default= -1,                 help="nz volume size - computed disks will be nx x ny x rise/apix" ) 
	parser.add_option("--new_pixel_size",     type="float", 		 default= -1,                 help="desired pixel size of the output disks. The default is -1, in which case there is no resampling (unless --match_pixel_rise flag is True).")
	parser.add_option("--maxerror",           type="float", 		 default= 0.1,                help="proportional to the maximum amount of error to tolerate between (dp/new_pixel_size) and int(dp/new_pixel_size ), where new_pixel_size is the pixel size calculated when the option --match_pixel_rise flag is True.")
	parser.add_option("--match_pixel_rise",   action="store_true",	 default=False,               help="calculate new pixel size such that the rise is approximately integer number of pixels given the new pixel size. This will be the pixel size of the output disks.")

	# get consistency
	parser.add_option("--consistency",        type="string",		 default="",                  help="Name of parameters to get consistency statistics for") 
	parser.add_option("--phithr",             type="float", 		 default= 2.0,                help="phi threshold for consistency check")  
	parser.add_option("--ythr",               type="float", 		 default= 2.0,                help="y threshold (in Angstroms) for consistency check")  
	parser.add_option("--segthr",             type="int", 		     default= 3,                  help="minimum number of segments/filament for consistency check")  

	# stack disks
	parser.add_option("--stackdisk",          type="string",		 default="",                  help="Name of file under which output volume will be saved to.")
	parser.add_option("--ref_ny",             type="int",   		 default=-1,                  help="ny of output volume size. Default is ref_nx" ) 

	# symmetry search
	parser.add_option("--symsearch",          action="store_true",	 default=False, 	  	      help="Do helical symmetry search." ) 
	parser.add_option("--ndp",                type="int",            default= 12,                 help="In symmetrization search, number of delta z steps equals to 2*ndp+1") 
	parser.add_option("--ndphi",              type="int",            default= 12,                 help="In symmetrization search, number of dphi steps equals to 2*ndphi+1")  
	parser.add_option("--dp_step",            type="float",          default= 0.1,                help="delta z step  for symmetrization [Angstroms] (default 0.1)")
	parser.add_option("--dphi_step",          type="float",          default= 0.1,                help="dphi step for symmetrization [degrees] (default 0.1)")
	parser.add_option("--datasym",            type="string",		 default="datasym.txt",       help="symdoc")
	parser.add_option("--symdoc",             type="string",		 default="",      	    	  help="text file containing helical symmetry parameters dp and dphi")

	# filament statistics in the stack

	(options, args) = parser.parse_args(arglist[1:])
	if len(args) < 1 or len(args) > 5:
		print "Various helical reconstruction related functionalities: " + usage2
		print "Please run '" + progname + " -h' for detailed options"
	else:

		if len(options.hfsc) > 0:
			if len(args) != 1:
				print  "Incorrect number of parameters"
				sys.exit()
			from applications import imgstat_hfsc
			imgstat_hfsc( args[0], options.hfsc, options.filament_attr)
			sys.exit()
		elif len(options.filinfo) > 0:
			if len(args) != 1:
				print  "Incorrect number of parameters"
				sys.exit()
			from EMAN2 import EMUtil
			filams =  EMUtil.get_all_attributes(args[0], "filament")
			ibeg = 0
			filcur = filams[0]
			n = len(filams)
			inf = []
			i = 1
			while( i <= n):
				if(i < n): fis = filams[i]
				else: fis = ""
				if( fis != filcur ):
					iend = i-1
					inf.append([ibeg,iend,iend-ibeg+1,filcur])
					ibeg = i
					filcur = fis
				i += 1
			from utilities import write_text_row
			write_text_row(inf, options.filinfo)
			sys.exit()
		
		if len(options.stackdisk) > 0:
			if len(args) != 1:
				print  "Incorrect number of parameters"
				sys.exit()
			dpp = (float(options.dp)/options.apix)
			rise = int(dpp)
			if(abs(float(rise) - dpp)>1.0e-3):
				print "  dpp has to be integer multiplicity of the pixel size"
				sys.exit()
			from utilities import get_im
			v = get_im(args[0])
			from applications import stack_disks
			ref_ny = options.ref_ny
			if ref_ny < 0:
				ref_ny = options.ref_nx
			sv = stack_disks(v, options.ref_nx, ref_ny, options.ref_nz, options.dphi, rise)
			sv.write_image(options.stackdisk)
			sys.exit()

		if len(options.consistency) > 0:
			if len(args) != 1:
				print  "Incorrect number of parameters"
				sys.exit()
			from development import consistency_params	
			consistency_params(args[0], options.consistency, options.dphi, options.dp, options.apix,phithr=options.phithr, ythr=options.ythr, THR=options.segthr)
			sys.exit()

		rminp = int((float(options.rmin)/options.apix) + 0.5)
		rmaxp = int((float(options.rmax)/options.apix) + 0.5)
		
		from utilities import get_input_from_string, get_im

		xr = get_input_from_string(options.xr)
		txs = get_input_from_string(options.txs)

		irp = 1
		if options.ou < 0:  oup = -1
		else:               oup = int( (options.ou/options.apix) + 0.5)
		xrp = ''
		txsp = ''
		
		for i in xrange(len(xr)):
			xrp += " "+str(float(xr[i])/options.apix)
		for i in xrange(len(txs)):
			txsp += " "+str(float(txs[i])/options.apix)

		searchxshiftp = int( (options.searchxshift/options.apix) + 0.5)
		nearbyp = int( (options.nearby/options.apix) + 0.5)
		zstepp = int( (options.zstep/options.apix) + 0.5)

		if options.MPI:
			from mpi import mpi_init, mpi_finalize
			sys.argv = mpi_init(len(sys.argv), sys.argv)

		if len(options.predict_helical) > 0:
			if len(args) != 1:
				print  "Incorrect number of parameters"
				sys.exit()
			if options.dp < 0:
				print "Helical symmetry paramter rise --dp should not be negative"
				sys.exit()
			from applications import predict_helical_params
			predict_helical_params(args[0], options.dp, options.dphi, options.apix, options.predict_helical)
			sys.exit()

		if options.helicise:	
			if len(args) != 2:
				print "Incorrect number of parameters"
				sys.exit()
			if options.dp < 0:
				print "Helical symmetry paramter rise --dp should not be negative"
				sys.exit()
			from utilities import get_im, sym_vol
			vol = get_im(args[0])
			vol = sym_vol(vol, options.sym)
			hvol = vol.helicise(options.apix, options.dp, options.dphi, options.fract, rmaxp, rminp)
			hvol = sym_vol(hvol, options.sym)
			hvol.write_image(args[1])
			sys.exit()


		if options.helicisepdb:	
			if len(args) != 2:
				print "Incorrect number of parameters"
				sys.exit()
			if options.dp < 0:
				print "Helical symmetry paramter rise --dp should not be negative"
				sys.exit()
			from math import cos, sin, radians
			from copy import deepcopy
			import numpy
			from numpy import zeros,dot,float32

			dp   = options.dp
			dphi = options.dphi
			nperiod = options.nrepeats

			infile =open(args[0],"r")
			pall = infile.readlines()
			infile.close()

			p = []

			pos = []
			lkl = -1
			for i in xrange( len(pall) ):
				if( (pall[i])[:4] == 'ATOM'):
					if( lkl == -1 ):  lkl = i
					p.append( pall[i] )
					pos.append(i)
			n = len(p)

			X = zeros( (3,len(p) ), dtype=float32 )
			X_new = zeros( (3,len(p) ), dtype=float32 )

			for i in xrange( len(p) ):
				element = deepcopy( p[i] )
				X[0,i]=float(element[30:38])
				X[1,i]=float(element[38:46])	
				X[2,i]=float(element[46:54])

			pnew = []
			for j in xrange(-nperiod, nperiod+1):
				for i in xrange( n ):
					pnew.append( deepcopy(p[i]) )

			dphi = radians(dphi)
			m = zeros( (3,3 ), dtype=float32 )
			t = zeros( (3,1 ), dtype=float32 )
			m[2][2] = 1.0
			t[0,0]  = 0.0
			t[1,0]  = 0.0

			for j in xrange(-nperiod, nperiod+1):
				if j != 0:
					rd = j*dphi
					m[0][0] =  cos(rd)
					m[0][1] =  sin(rd)
					m[1][0] = -m[0][1]
					m[1][1] =  m[0][0]
					t[2,0]  = j*dp
					X_new = dot(m, X) + t
					for i in xrange( n ):
						pnew[j*n+i] = pnew[j*n+i][:30] + "%8.3f"%( float(X_new[0,i]) )+"%8.3f"%( float(X_new[1,i]) )+"%8.3f"%( float(X_new[2,i]) ) + pnew[j*n+i][54:]


			outfile=open(args[1],"w")
			outfile.writelines(pall[0:lkl])
			outfile.writelines(pnew)
			outfile.writelines("END\n")
			outfile.close()
			sys.exit()

		if options.volalixshift:
			if options.maxit > 1:
				print "Inner iteration for x-shift determinatin is restricted to 1"
				sys.exit()
			if len(args) < 4:  mask = None
			else:               mask = args[3]
			from applications import volalixshift_MPI
			global_def.BATCH = True
			volalixshift_MPI(args[0], args[1], args[2], searchxshiftp, options.apix, options.dp, options.dphi, options.fract, rmaxp, rminp, mask, options.maxit, options.CTF, options.snr, options.sym,  options.function, options.npad, options.debug, nearbyp)
			global_def.BATCH = False

		if options.diskali:
			#if options.maxit > 1:
			#	print "Inner iteration for disk alignment is restricted to 1"
			#	sys.exit()
			if len(args) < 4:  mask = None
			else:               mask = args[3]
			global_def.BATCH = True
			if(options.sym[:1] == "d" or options.sym[:1] == "D" ):
				from development import diskaliD_MPI
				diskaliD_MPI(args[0], args[1], args[2], mask, options.dp, options.dphi, options.apix, options.function, zstepp, options.fract, rmaxp, rminp, options.CTF, options.maxit, options.sym)
			else:
				from applications import diskali_MPI
				diskali_MPI(args[0], args[1], args[2], mask, options.dp, options.dphi, options.apix, options.function, zstepp, options.fract, rmaxp, rminp, options.CTF, options.maxit, options.sym)
			global_def.BATCH = False
		
		if options.symsearch:
		
			if len(options.symdoc) < 1:
				if options.dp < 0 or options.dphi < 0:
					print "Enter helical symmetry parameters either using --symdoc or --dp and --dphi"
					sys.exit()
			
			if options.dp < 0 or options.dphi < 0:
				# read helical symmetry parameters from symdoc
				from utilities import read_text_row
				hparams=read_text_row(options.symdoc)
				dp = hparams[0][0]
				dphi = hparams[0][1]
			else:
				dp   = options.dp
				dphi = options.dphi
			
			from applications import symsearch_MPI
			if len(args) < 3:	
				mask = None
			else:
				mask= args[2]
			global_def.BATCH = True
			symsearch_MPI(args[0], args[1], mask, dp, options.ndp, options.dp_step, dphi, options.ndphi, options.dphi_step, rminp, rmaxp, options.fract, options.sym, options.function, options.datasym, options.apix, options.debug)
			global_def.BATCH = False
			
		elif len(options.gendisk)> 0:
			from applications import gendisks_MPI
			global_def.BATCH = True
			if len(args) == 1:  mask3d = None
			else:               mask3d = args[1]
			if options.dp < 0:
				print "Helical symmetry paramter rise --dp must be explictly set!"
				sys.exit()
			gendisks_MPI(args[0], mask3d, options.ref_nx, options.apix, options.dp, options.dphi, options.fract, rmaxp, rminp, options.CTF, options.function, options.sym, options.gendisk, options.maxerror, options.new_pixel_size, options.match_pixel_rise)
			global_def.BATCH = False
		
		if options.MPI:
			from mpi import mpi_finalize
			mpi_finalize()
Exemplo n.º 19
0
def main():
	import sys
	import os
	import math
	import random
	import pyemtbx.options
	import time
	from   random   import random, seed, randint
	from   optparse import OptionParser

	progname = os.path.basename(sys.argv[0])
	usage = progname + """ [options] <inputfile> <outputfile>

	Generic 2-D image processing programs.

	Functionality:

	1.  Phase flip a stack of images and write output to new file:
		sxprocess.py input_stack.hdf output_stack.hdf --phase_flip
	
	2.  Resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.
	    The window size will change accordingly.
		sxprocess input.hdf output.hdf  --changesize --ratio=0.5

	3.  Compute average power spectrum of a stack of 2D images with optional padding (option wn) with zeroes or a 3-D volume.
		sxprocess.py input_stack.hdf powerspectrum.hdf --pw [--wn=1024]

	4.  Generate a stack of projections bdb:data and micrographs with prefix mic (i.e., mic0.hdf, mic1.hdf etc) from structure input_structure.hdf, with CTF applied to both projections and micrographs:
		sxprocess.py input_structure.hdf data mic --generate_projections format="bdb":apix=5.2:CTF=True:boxsize=64

    5.  Retrieve original image numbers in the selected ISAC group (here group 12 from generation 3):
    	sxprocess.py  bdb:test3 class_averages_generation_3.hdf  list3_12.txt --isacgroup=12 --params=originalid

    6.  Retrieve original image numbers of images listed in ISAC output stack of averages:
    	sxprocess.py  select1.hdf  ohk.txt

    7.  Adjust rotationally averaged power spectrum of an image to that of a reference image or a reference 1D power spectrum stored in an ASCII file.
    	Optionally use a tangent low-pass filter.  Also works for a stack of images, in which case the output is also a stack.
    	sxprocess.py  vol.hdf ref.hdf  avol.hdf < 0.25 0.2> --adjpw
   	 	sxprocess.py  vol.hdf pw.txt   avol.hdf < 0.25 0.2> --adjpw

    8.  Generate a 1D rotationally averaged power spectrum of an image.
		sxprocess.py  vol.hdf --rotwp=rotpw.txt
    	# Output will contain three columns:
       (1) rotationally averaged power spectrum
       (2) logarithm of the rotationally averaged power spectrum
       (3) integer line number (from zero to approximately to half the image size)

    9.  Apply 3D transformation (rotation and/or shift) to a set of orientation parameters associated with projection data.
    	sxprocess.py  --transfromparams=phi,theta,psi,tx,ty,tz      input.txt  output.txt
    	The output file is then imported and 3D transformed volume computed:
    	sxheader.py  bdb:p  --params=xform.projection  --import=output.txt
    	mpirun -np 2 sxrecons3d_n.py  bdb:p tvol.hdf --MPI
    	The reconstructed volume is in the position of the volume computed using the input.txt parameters and then
    	transformed with rot_shift3D(vol, phi,theta,psi,tx,ty,tz)

   10.  Import ctf parameters from the output of sxcter into windowed particle headers.
	    There are three possible input files formats:  (1) all particles are in one stack, (2 aor 3) particles are in stacks, each stack corresponds to a single micrograph.
	    In each case the particles should contain a name of the micrograph of origin stores using attribute name 'ptcl_source_image'.
        Normally this is done by e2boxer.py during windowing.
	    Particles whose defocus or astigmatism error exceed set thresholds will be skipped, otherwise, virtual stacks with the original way preceded by G will be created.
		sxprocess.py  --input=bdb:data  --importctf=outdir/partres  --defocuserror=10.0  --astigmatismerror=5.0
		#  Output will be a vritual stack bdb:Gdata
		sxprocess.py  --input="bdb:directory/stacks*"  --importctf=outdir/partres  --defocuserror=10.0  --astigmatismerror=5.0
		To concatenate output files:
		cd directory
		e2bdb.py . --makevstack=bdb:allparticles  --filt=G
		IMPORTANT:  Please do not move (or remove!) any input/intermediate EMAN2DB files as the information is linked between them.

   11. Scale 3D shifts.  The shifts in the input five columns text file with 3D orientation parameters will be DIVIDED by the scale factor
		sxprocess.py  orientationparams.txt  scaledparams.txt  scale=0.5
   
   12. Generate 3D mask from a given 3-D volume automatically or using threshold provided by user.
   
   13. Postprocess 3-D or 2-D images: 
   			for 3-D volumes: calculate FSC with provided mask; weight summed volume with FSC; estimate B-factor from FSC weighted summed two volumes; apply negative B-factor to the weighted volume. 
   			for 2-D images:  calculate B-factor and apply negative B-factor to 2-D images.
   14. Winow stack file -reduce size of images without changing the pixel size. 


"""

	parser = OptionParser(usage,version=SPARXVERSION)
	parser.add_option("--order", 				action="store_true", help="Two arguments are required: name of input stack and desired name of output stack. The output stack is the input stack sorted by similarity in terms of cross-correlation coefficent.", default=False)
	parser.add_option("--order_lookup", 		action="store_true", help="Test/Debug.", default=False)
	parser.add_option("--order_metropolis", 	action="store_true", help="Test/Debug.", default=False)
	parser.add_option("--order_pca", 			action="store_true", help="Test/Debug.", default=False)
	parser.add_option("--initial",				type="int", 		default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)")
	parser.add_option("--circular", 			action="store_true", help="Select circular ordering (fisr image has to be similar to the last", default=False)
	parser.add_option("--radius", 				type="int", 		default=-1, help="Radius of a circular mask for similarity based ordering")
	parser.add_option("--changesize", 			action="store_true", help="resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.", default=False)
	parser.add_option("--ratio", 				type="float", 		default=1.0, help="The ratio of new to old image size (if <1 the pixel size will increase and image size decrease, if>1, the other way round")
	parser.add_option("--pw", 					action="store_true", help="compute average power spectrum of a stack of 2-D images with optional padding (option wn) with zeroes", default=False)
	parser.add_option("--wn", 					type="int", 		default=-1, help="Size of window to use (should be larger/equal than particle box size, default padding to max(nx,ny))")
	parser.add_option("--phase_flip", 			action="store_true", help="Phase flip the input stack", default=False)
	parser.add_option("--makedb", 				metavar="param1=value1:param2=value2", type="string",
					action="append",  help="One argument is required: name of key with which the database will be created. Fill in database with parameters specified as follows: --makedb param1=value1:param2=value2, e.g. 'gauss_width'=1.0:'pixel_input'=5.2:'pixel_output'=5.2:'thr_low'=1.0")
	parser.add_option("--generate_projections", metavar="param1=value1:param2=value2", type="string",
					action="append", help="Three arguments are required: name of input structure from which to generate projections, desired name of output projection stack, and desired prefix for micrographs (e.g. if prefix is 'mic', then micrographs mic0.hdf, mic1.hdf etc will be generated). Optional arguments specifying format, apix, box size and whether to add CTF effects can be entered as follows after --generate_projections: format='bdb':apix=5.2:CTF=True:boxsize=100, or format='hdf', etc., where format is bdb or hdf, apix (pixel size) is a float, CTF is True or False, and boxsize denotes the dimension of the box (assumed to be a square). If an optional parameter is not specified, it will default as follows: format='bdb', apix=2.5, CTF=False, boxsize=64.")
	parser.add_option("--isacgroup", 			type="int", 		help="Retrieve original image numbers in the selected ISAC group. See ISAC documentation for details.", default=-1)
	parser.add_option("--isacselect", 			action="store_true", 		help="Retrieve original image numbers of images listed in ISAC output stack of averages. See ISAC documentation for details.", default=False)
	parser.add_option("--params",	   			type="string",      default=None,    help="Name of header of parameter, which one depends on specific option")
	parser.add_option("--adjpw", 				action="store_true",	help="Adjust rotationally averaged power spectrum of an image", default=False)
	parser.add_option("--rotpw", 				type="string",   	default=None,    help="Name of the text file to contain rotationally averaged power spectrum of the input image.")
	parser.add_option("--transformparams",		type="string",   	default=None,    help="Transform 3D projection orientation parameters using six 3D parameters (phi, theta,psi,sx,sy,sz).  Input: --transformparams=45.,66.,12.,-2,3,-5.5 desired six transformation of the reconstructed structure. Output: file with modified orientation parameters.")

	
	# import ctf estimates done using cter
	parser.add_option("--input",              	type="string",		default= None,     		  help="Input particles.")
	parser.add_option("--importctf",          	type="string",		default= None,     		  help="Name of the file containing CTF parameters produced by sxcter.")
	parser.add_option("--defocuserror",       	type="float",  		default=1000000.0,        help="Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent.  The error is computed as (std dev defocus)/defocus*100%")
	parser.add_option("--astigmatismerror",   	type="float",  		default=360.0,            help="Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees.")

	# import ctf estimates done using cter
	parser.add_option("--scale",              	type="float", 		default=-1.0,      		  help="Divide shifts in the input 3D orientation parameters text file by the scale factor.")
	
	# generate adaptive mask from an given 3-D volume
	parser.add_option("--adaptive_mask",        action="store_true",                      help="create adavptive 3-D mask from a given volume", default=False)
	parser.add_option("--nsigma",              	type="float",	default= 1.,     	      help="number of times of sigma of the input volume to obtain the the large density cluster")
	parser.add_option("--ndilation",            type="int",		default= 3,     		  help="number of times of dilation applied to the largest cluster of density")
	parser.add_option("--kernel_size",          type="int",		default= 11,     		  help="convolution kernel for smoothing the edge of the mask")
	parser.add_option("--gauss_standard_dev",   type="int",		default= 9,     		  help="stanadard deviation value to generate Gaussian edge")
	parser.add_option("--threshold",            type="float",	default= 9999.,           help="threshold provided by user to binarize input volume")
	parser.add_option("--ne",                   type="int",		default= 0,     		  help="number of times to erode the binarized  input image")
	parser.add_option("--nd",                   type="int",		default= 0,     		  help="number of times to dilate the binarized input image")
	parser.add_option("--postprocess",          action="store_true",                      help="postprocess unfiltered odd, even 3-D volumes",default=False)
	parser.add_option("--fsc_weighted",         action="store_true",                      help="postprocess unfiltered odd, even 3-D volumes")
	parser.add_option("--low_pass_filter",      action="store_true",      default=False,  help="postprocess unfiltered odd, even 3-D volumes")
	parser.add_option("--ff",                   type="float", default=.25,                help="low pass filter stop band frequency in absolute unit")
	parser.add_option("--aa",                   type="float", default=.1,                 help="low pass filter falloff" )
	parser.add_option("--mask",           type="string",                                  help="input mask file",  default=None)
	parser.add_option("--output",         type="string",                                  help="output file name", default="postprocessed.hdf")
	parser.add_option("--pixel_size",     type="float",                                   help="pixel size of the data", default=1.0)
	parser.add_option("--B_start",     type="float",                                      help="starting frequency in Angstrom for B-factor estimation", default=10.)
	parser.add_option("--FSC_cutoff",     type="float",                                   help="stop frequency in Angstrom for B-factor estimation", default=0.143)
	parser.add_option("--2d",          action="store_true",                      help="postprocess isac 2-D averaged images",default=False)
	parser.add_option("--window_stack",                     action="store_true",          help="window stack images using a smaller window size", default=False)
	parser.add_option("--box",           type="int",		default= 0,                   help="the new window size ") 
 	(options, args) = parser.parse_args()

	global_def.BATCH = True
		
	if options.phase_flip:
		nargs = len(args)
		if nargs != 2:
			print "must provide name of input and output file!"
			return
		from EMAN2 import Processor
		instack = args[0]
		outstack = args[1]
		nima = EMUtil.get_image_count(instack)
		from filter import filt_ctf
		for i in xrange(nima):
			img = EMData()
			img.read_image(instack, i)
			try:
				ctf = img.get_attr('ctf')
			except:
				print "no ctf information in input stack! Exiting..."
				return
			
			dopad = True
			sign = 1
			binary = 1  # phase flip
				
			assert img.get_ysize() > 1	
			dict = ctf.to_dict()
			dz = dict["defocus"]
			cs = dict["cs"]
			voltage = dict["voltage"]
			pixel_size = dict["apix"]
			b_factor = dict["bfactor"]
			ampcont = dict["ampcont"]
			dza = dict["dfdiff"]
			azz = dict["dfang"]
			
			if dopad and not img.is_complex(): ip = 1
			else:                             ip = 0
	
	
			params = {"filter_type": Processor.fourier_filter_types.CTF_,
	 			"defocus" : dz,
				"Cs": cs,
				"voltage": voltage,
				"Pixel_size": pixel_size,
				"B_factor": b_factor,
				"amp_contrast": ampcont,
				"dopad": ip,
				"binary": binary,
				"sign": sign,
				"dza": dza,
				"azz":azz}
			
			tmp = Processor.EMFourierFilter(img, params)
			tmp.set_attr_dict({"ctf": ctf})
			
			tmp.write_image(outstack, i)

	elif options.changesize:
		nargs = len(args)
		if nargs != 2:
			ERROR("must provide name of input and output file!", "change size", 1)
			return
		from utilities import get_im
		instack = args[0]
		outstack = args[1]
		sub_rate = float(options.ratio)
			
		nima = EMUtil.get_image_count(instack)
		from fundamentals import resample
		for i in xrange(nima):
			resample(get_im(instack, i), sub_rate).write_image(outstack, i)

	elif options.isacgroup>-1:
		nargs = len(args)
		if nargs != 3:
			ERROR("Three files needed on input!", "isacgroup", 1)
			return
		from utilities import get_im
		instack = args[0]
		m=get_im(args[1],int(options.isacgroup)).get_attr("members")
		l = []
		for k in m:
			l.append(int(get_im(args[0],k).get_attr(options.params)))
		from utilities import write_text_file
		write_text_file(l, args[2])

	elif options.isacselect:
		nargs = len(args)
		if nargs != 2:
			ERROR("Two files needed on input!", "isacgroup", 1)
			return
		from utilities import get_im
		nima = EMUtil.get_image_count(args[0])
		m = []
		for k in xrange(nima):
			m += get_im(args[0],k).get_attr("members")
		m.sort()
		from utilities import write_text_file
		write_text_file(m, args[1])

	elif options.pw:
		nargs = len(args)
		if nargs < 2:
			ERROR("must provide name of input and output file!", "pw", 1)
			return
		from utilities import get_im, write_text_file
		from fundamentals import rops_table
		d = get_im(args[0])
		ndim = d.get_ndim()
		if ndim ==3:
			pw = rops_table(d)
			write_text_file(pw, args[1])			
		else:
			nx = d.get_xsize()
			ny = d.get_ysize()
			if nargs ==3: mask = get_im(args[2])
			wn = int(options.wn)
			if wn == -1:
				wn = max(nx, ny)
			else:
				if( (wn<nx) or (wn<ny) ):  ERROR("window size cannot be smaller than the image size","pw",1)
			n = EMUtil.get_image_count(args[0])
			from utilities import model_blank, model_circle, pad
			from EMAN2 import periodogram
			p = model_blank(wn,wn)
		
			for i in xrange(n):
				d = get_im(args[0], i)
				if nargs==3:
					d *=mask
				st = Util.infomask(d, None, True)
				d -= st[0]
				p += periodogram(pad(d, wn, wn, 1, 0.))
			p /= n
			p.write_image(args[1])

	elif options.adjpw:

		if len(args) < 3:
			ERROR("filt_by_rops input target output fl aa (the last two are optional parameters of a low-pass filter)","adjpw",1)
			return
		img_stack = args[0]
		from math         import sqrt
		from fundamentals import rops_table, fft
		from utilities    import read_text_file, get_im
		from filter       import  filt_tanl, filt_table
		if(  args[1][-3:] == 'txt'):
			rops_dst = read_text_file( args[1] )
		else:
			rops_dst = rops_table(get_im( args[1] ))

		out_stack = args[2]
		if(len(args) >4):
			fl = float(args[3])
			aa = float(args[4])
		else:
			fl = -1.0
			aa = 0.0

		nimage = EMUtil.get_image_count( img_stack )

		for i in xrange(nimage):
			img = fft(get_im(img_stack, i) )
			rops_src = rops_table(img)

			assert len(rops_dst) == len(rops_src)

			table = [0.0]*len(rops_dst)
			for j in xrange( len(rops_dst) ):
				table[j] = sqrt( rops_dst[j]/rops_src[j] )

			if( fl > 0.0):
				img = filt_tanl(img, fl, aa)
			img = fft(filt_table(img, table))
			img.write_image(out_stack, i)

	elif options.rotpw != None:

		if len(args) != 1:
			ERROR("Only one input permitted","rotpw",1)
			return
		from utilities import write_text_file, get_im
		from fundamentals import rops_table
		from math import log10
		t = rops_table(get_im(args[0]))
		x = range(len(t))
		r = [0.0]*len(x)
		for i in x:  r[i] = log10(t[i])
		write_text_file([t,r,x],options.rotpw)

	elif options.transformparams != None:
		if len(args) != 2:
			ERROR("Please provide names of input and output files with orientation parameters","transformparams",1)
			return
		from utilities import read_text_row, write_text_row
		transf = [0.0]*6
		spl=options.transformparams.split(',')
		for i in xrange(len(spl)):  transf[i] = float(spl[i])

		write_text_row( rotate_shift_params(read_text_row(args[0]), transf)	, args[1])

	elif options.makedb != None:
		nargs = len(args)
		if nargs != 1:
			print "must provide exactly one argument denoting database key under which the input params will be stored"
			return
		dbkey = args[0]
		print "database key under which params will be stored: ", dbkey
		gbdb = js_open_dict("e2boxercache/gauss_box_DB.json")
				
		parmstr = 'dummy:'+options.makedb[0]
		(processorname, param_dict) = parsemodopt(parmstr)
		dbdict = {}
		for pkey in param_dict:
			if (pkey == 'invert_contrast') or (pkey == 'use_variance'):
				if param_dict[pkey] == 'True':
					dbdict[pkey] = True
				else:
					dbdict[pkey] = False
			else:		
				dbdict[pkey] = param_dict[pkey]
		gbdb[dbkey] = dbdict

	elif options.generate_projections:
		nargs = len(args)
		if nargs != 3:
			ERROR("Must provide name of input structure(s) from which to generate projections, name of output projection stack, and prefix for output micrographs."\
			"sxprocess - generate projections",1)
			return
		inpstr  = args[0]
		outstk  = args[1]
		micpref = args[2]

		parmstr = 'dummy:'+options.generate_projections[0]
		(processorname, param_dict) = parsemodopt(parmstr)

		parm_CTF    = False
		parm_format = 'bdb'
		parm_apix   = 2.5

		if 'CTF' in param_dict:
			if param_dict['CTF'] == 'True':
				parm_CTF = True

		if 'format' in param_dict:
			parm_format = param_dict['format']

		if 'apix' in param_dict:
			parm_apix = float(param_dict['apix'])

		boxsize = 64
		if 'boxsize' in param_dict:
			boxsize = int(param_dict['boxsize'])

		print "pixel size: ", parm_apix, " format: ", parm_format, " add CTF: ", parm_CTF, " box size: ", boxsize

		scale_mult      = 2500
		sigma_add       = 1.5
		sigma_proj      = 30.0
		sigma2_proj     = 17.5
		sigma_gauss     = 0.3
		sigma_mic       = 30.0
		sigma2_mic      = 17.5
		sigma_gauss_mic = 0.3
		
		if 'scale_mult' in param_dict:
			scale_mult = float(param_dict['scale_mult'])
		if 'sigma_add' in param_dict:
			sigma_add = float(param_dict['sigma_add'])
		if 'sigma_proj' in param_dict:
			sigma_proj = float(param_dict['sigma_proj'])
		if 'sigma2_proj' in param_dict:
			sigma2_proj = float(param_dict['sigma2_proj'])
		if 'sigma_gauss' in param_dict:
			sigma_gauss = float(param_dict['sigma_gauss'])	
		if 'sigma_mic' in param_dict:
			sigma_mic = float(param_dict['sigma_mic'])
		if 'sigma2_mic' in param_dict:
			sigma2_mic = float(param_dict['sigma2_mic'])
		if 'sigma_gauss_mic' in param_dict:
			sigma_gauss_mic = float(param_dict['sigma_gauss_mic'])	
			
		from filter import filt_gaussl, filt_ctf
		from utilities import drop_spider_doc, even_angles, model_gauss, delete_bdb, model_blank,pad,model_gauss_noise,set_params2D, set_params_proj
		from projection import prep_vol,prgs
		seed(14567)
		delta = 29
		angles = even_angles(delta, 0.0, 89.9, 0.0, 359.9, "S")
		nangle = len(angles)
		
		modelvol = []
		nvlms = EMUtil.get_image_count(inpstr)
		from utilities import get_im
		for k in xrange(nvlms):  modelvol.append(get_im(inpstr,k))
		
		nx = modelvol[0].get_xsize()
		
		if nx != boxsize:
			ERROR("Requested box dimension does not match dimension of the input model.", \
			"sxprocess - generate projections",1)
		nvol = 10
		volfts = [[] for k in xrange(nvlms)]
		for k in xrange(nvlms):
			for i in xrange(nvol):
				sigma = sigma_add + random()  # 1.5-2.5
				addon = model_gauss(sigma, boxsize, boxsize, boxsize, sigma, sigma, 38, 38, 40 )
				scale = scale_mult * (0.5+random())
				vf, kb = prep_vol(modelvol[k] + scale*addon)
				volfts[k].append(vf)
		del vf, modelvol

		if parm_format == "bdb":
			stack_data = "bdb:"+outstk
			delete_bdb(stack_data)
		else:
			stack_data = outstk + ".hdf"
		Cs      = 2.0
		pixel   = parm_apix
		voltage = 120.0
		ampcont = 10.0
		ibd     = 4096/2-boxsize
		iprj    = 0

		width = 240
		xstart = 8 + boxsize/2
		ystart = 8 + boxsize/2
		rowlen = 17
		from random import randint
		params = []
		for idef in xrange(3, 8):

			irow = 0
			icol = 0

			mic = model_blank(4096, 4096)
			defocus = idef * 0.5#0.2
			if parm_CTF:
				astampl=defocus*0.15
				astangl=50.0
				ctf = generate_ctf([defocus, Cs, voltage,  pixel, ampcont, 0.0, astampl, astangl])

			for i in xrange(nangle):
				for k in xrange(12):
					dphi = 8.0*(random()-0.5)
					dtht = 8.0*(random()-0.5)
					psi  = 360.0*random()

					phi = angles[i][0]+dphi
					tht = angles[i][1]+dtht

					s2x = 4.0*(random()-0.5)
					s2y = 4.0*(random()-0.5)

					params.append([phi, tht, psi, s2x, s2y])

					ivol = iprj % nvol
					#imgsrc = randint(0,nvlms-1)
					imgsrc = iprj % nvlms
					proj = prgs(volfts[imgsrc][ivol], kb, [phi, tht, psi, -s2x, -s2y])

					x = xstart + irow * width
					y = ystart + icol * width

					mic += pad(proj, 4096, 4096, 1, 0.0, x-2048, y-2048, 0)

					proj = proj + model_gauss_noise( sigma_proj, nx, nx )
					if parm_CTF:
						proj = filt_ctf(proj, ctf)
						proj.set_attr_dict({"ctf":ctf, "ctf_applied":0})

					proj = proj + filt_gaussl(model_gauss_noise(sigma2_proj, nx, nx), sigma_gauss)
					proj.set_attr("origimgsrc",imgsrc)
					proj.set_attr("test_id", iprj)
					# flags describing the status of the image (1 = true, 0 = false)
					set_params2D(proj, [0.0, 0.0, 0.0, 0, 1.0])
					set_params_proj(proj, [phi, tht, psi, s2x, s2y])

					proj.write_image(stack_data, iprj)
			
					icol += 1
					if icol == rowlen:
						icol = 0
						irow += 1

					iprj += 1

			mic += model_gauss_noise(sigma_mic,4096,4096)
			if parm_CTF:
				#apply CTF
				mic = filt_ctf(mic, ctf)
			mic += filt_gaussl(model_gauss_noise(sigma2_mic, 4096, 4096), sigma_gauss_mic)
	
			mic.write_image(micpref + "%1d.hdf" % (idef-3), 0)
		
		drop_spider_doc("params.txt", params)

	elif options.importctf != None:
		print ' IMPORTCTF  '
		from utilities import read_text_row,write_text_row
		from random import randint
		import subprocess
		grpfile = 'groupid%04d'%randint(1000,9999)
		ctfpfile = 'ctfpfile%04d'%randint(1000,9999)
		cterr = [options.defocuserror/100.0, options.astigmatismerror]
		ctfs = read_text_row(options.importctf)
		for kk in xrange(len(ctfs)):
			root,name = os.path.split(ctfs[kk][-1])
			ctfs[kk][-1] = name[:-4]
		if(options.input[:4] != 'bdb:'):
			ERROR('Sorry, only bdb files implemented','importctf',1)
		d = options.input[4:]
		#try:     str = d.index('*')
		#except:  str = -1
		from string import split
		import glob
		uu = os.path.split(d)
		uu = os.path.join(uu[0],'EMAN2DB',uu[1]+'.bdb')
		flist = glob.glob(uu)
		for i in xrange(len(flist)):
			root,name = os.path.split(flist[i])
			root = root[:-7]
			name = name[:-4]
			fil = 'bdb:'+os.path.join(root,name)
			sourcemic = EMUtil.get_all_attributes(fil,'ptcl_source_image')
			nn = len(sourcemic)
			gctfp = []
			groupid = []
			for kk in xrange(nn):
				junk,name2 = os.path.split(sourcemic[kk])
				name2 = name2[:-4]
				ctfp = [-1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]
				for ll in xrange(len(ctfs)):
					if(name2 == ctfs[ll][-1]):
						#  found correct
						if(ctfs[ll][8]/ctfs[ll][0] <= cterr[0]):
							#  acceptable defocus error
							ctfp = ctfs[ll][:8]
							if(ctfs[ll][10] > cterr[1] ):
								# error of astigmatism exceed the threshold, set astigmatism to zero.
								ctfp[6] = 0.0
								ctfp[7] = 0.0
							gctfp.append(ctfp)
							groupid.append(kk)
						break
			if(len(groupid) > 0):
				write_text_row(groupid, grpfile)
				write_text_row(gctfp, ctfpfile)
				cmd = "{} {} {} {}".format('e2bdb.py',fil,'--makevstack=bdb:'+root+'G'+name,'--list='+grpfile)
				#print cmd
				subprocess.call(cmd, shell=True)
				cmd = "{} {} {} {}".format('sxheader.py','bdb:'+root+'G'+name,'--params=ctf','--import='+ctfpfile)
				#print cmd
				subprocess.call(cmd, shell=True)
			else:
				print  ' >>>  Group ',name,'  skipped.'
				
		cmd = "{} {} {}".format("rm -f",grpfile,ctfpfile)
		subprocess.call(cmd, shell=True)

	elif options.scale > 0.0:
		from utilities import read_text_row,write_text_row
		scale = options.scale
		nargs = len(args)
		if nargs != 2:
			print "Please provide names of input and output file!"
			return
		p = read_text_row(args[0])
		for i in xrange(len(p)):
			p[i][3] /= scale
			p[i][4] /= scale
		write_text_row(p, args[1])
		
	elif options.adaptive_mask:
		from utilities import get_im
		from morphology import adaptive_mask, binarize, erosion, dilation
		nsigma             = options.nsigma
		ndilation          = options.ndilation
		kernel_size        = options.kernel_size
		gauss_standard_dev = options.gauss_standard_dev
		nargs = len(args)
		if nargs ==0:
			print " Create 3D mask from a given volume, either automatically or from the user provided threshold."
		elif nargs > 2:
			print "Too many inputs are given, try again!"
			return
		else:
			inputvol = get_im(args[0])
			input_path, input_file_name = os.path.split(args[0])
			input_file_name_root,ext=os.path.splitext(input_file_name)
			if nargs == 2:  mask_file_name = args[1]
			else:           mask_file_name = "adaptive_mask_for_"+input_file_name_root+".hdf" # Only hdf file is output.
			if options.threshold !=9999.:
				mask3d = binarize(inputvol, options.threshold)
				for i in xrange(options.ne): mask3d = erosion(mask3d)
				for i in xrange(options.nd): mask3d = dilation(mask3d)
			else: 
				mask3d = adaptive_mask(inputvol, nsigma, ndilation, kernel_size, gauss_standard_dev)
			mask3d.write_image(mask_file_name)
			
	elif options.postprocess:
		from utilities    import get_im
		from fundamentals import rot_avg_table
		from morphology   import compute_bfactor,power
		from statistics   import fsc
		from filter       import filt_table, filt_gaussinv
		from EMAN2 import periodogram
		e1   = get_im(args[0],0)
		if e1.get_zsize()==1:
			nimage = EMUtil.get_image_count(args[0])
			if options.mask !=None: m = get_im(options.mask)
			else: m = None
			for i in xrange(nimage):
				e1 = get_im(args[0],i)
				if m: e1 *=m
				guinerline = rot_avg_table(power(periodogram(e1),.5))
				freq_max   =  1/(2.*pixel_size)
				freq_min   =  1./options.B_start
				b,junk=compute_bfactor(guinerline, freq_min, freq_max, pixel_size)
				tmp = b/pixel_size**2
				sigma_of_inverse=sqrt(2./tmp)
				e1 = filt_gaussinv(e1,sigma_of_inverse)
				if options.low_pass_filter:
					from filter import filt_tanl
					e1 =filt_tanl(e1,options.ff, options.aa)
				e1.write_image(options.output)							
		else:
			nargs = len(args)
			e1    = get_im(args[0])
			if nargs >1: e2 = get_im(args[1])
			if options.mask !=None: m = get_im(options.mask)
			else: m =None
			pixel_size = options.pixel_size
			from math import sqrt
			if m !=None:
				e1 *=m
				if nargs >1 :e2 *=m
			if options.fsc_weighted:
				frc = fsc(e1,e2,1)
				## FSC is done on masked two images
				#### FSC weighting sqrt((2.*fsc)/(1+fsc));
				fil = len(frc[1])*[None]
				for i in xrange(len(fil)):
					if frc[1][i]>=options.FSC_cutoff: tmp = frc[1][i]
					else: tmp = 0.0
					fil[i] = sqrt(2.*tmp/(1.+tmp))
			if nargs>1: e1 +=e2
			if options.fsc_weighted: e1=filt_table(e1,fil) 
			guinerline = rot_avg_table(power(periodogram(e1),.5))
			freq_max   = 1/(2.*pixel_size)
			freq_min   = 1./options.B_start
			b,junk     = compute_bfactor(guinerline, freq_min, freq_max, pixel_size)
			tmp        = b/pixel_size**2
			sigma_of_inverse=sqrt(2./tmp)
			e1  = filt_gaussinv(e1,sigma_of_inverse)
			if options.low_pass_filter:
				from filter       import filt_tanl
				e1 =filt_tanl(e1,options.ff, options.aa)
			e1.write_image(options.output)
		 
	elif options.window_stack:
		nargs = len(args)
		if nargs ==0:
			print "  Reduce image size of a stack"
			return
		else:
			output_stack_name = None
			inputstack = args[0]
			if nargs ==2:output_stack_name = args[1]
			input_path,input_file_name=os.path.split(inputstack)
			input_file_name_root,ext=os.path.splitext(input_file_name)
			if input_file_name_root[0:3]=="bdb":stack_is_bdb= True
			else: stack_is_bdb= False
			if output_stack_name is None:
				if stack_is_bdb: output_stack_name ="bdb:reduced_"+input_file_name_root[4:]
				else:output_stack_name = "reduced_"+input_file_name_root+".hdf" # Only hdf file is output.
			nimage = EMUtil.get_image_count(inputstack)
			from fundamentals import window2d
			for i in xrange(nimage):
				image = EMData()
				image.read_image(inputstack,i)
				w = window2d(image,options.box,options.box)
				w.write_image(output_stack_name,i)
	else:  ERROR("Please provide option name","sxprocess.py",1)	
Exemplo n.º 20
0
def main():

	def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
		if mirror:
			m = 1
			alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 540.0-psi, 0, 0, 1.0)
		else:
			m = 0
			alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 360.0-psi, 0, 0, 1.0)
		return  alpha, sx, sy, m
	
	progname = os.path.basename(sys.argv[0])
	usage = progname + " prj_stack  --ave2D= --var2D=  --ave3D= --var3D= --img_per_grp= --fl=0.2 --aa=0.1  --sym=symmetry --CTF"
	parser = OptionParser(usage, version=SPARXVERSION)

	parser.add_option("--ave2D",		type="string"	   ,	default=False,				help="write to the disk a stack of 2D averages")
	parser.add_option("--var2D",		type="string"	   ,	default=False,				help="write to the disk a stack of 2D variances")
	parser.add_option("--ave3D",		type="string"	   ,	default=False,				help="write to the disk reconstructed 3D average")
	parser.add_option("--var3D",		type="string"	   ,	default=False,				help="compute 3D variability (time consuming!)")
	parser.add_option("--img_per_grp",	type="int"         ,	default=10   ,				help="number of neighbouring projections")
	parser.add_option("--no_norm",		action="store_true",	default=False,				help="do not use normalization")
	parser.add_option("--radiusvar", 	type="int"         ,	default=-1   ,				help="radius for 3D var" )
	parser.add_option("--npad",			type="int"         ,	default=2    ,				help="number of time to pad the original images")
	parser.add_option("--sym" , 		type="string"      ,	default="c1" ,				help="symmetry")
	parser.add_option("--fl",			type="float"       ,	default=0.0  ,				help="stop-band frequency (Default - no filtration)")
	parser.add_option("--aa",			type="float"       ,	default=0.0  ,				help="fall off of the filter (Default - no filtration)")
	parser.add_option("--CTF",			action="store_true",	default=False,				help="use CFT correction")
	parser.add_option("--VERBOSE",		action="store_true",	default=False,				help="Long output for debugging")
	#parser.add_option("--MPI" , 		action="store_true",	default=False,				help="use MPI version")
	#parser.add_option("--radiuspca", 	type="int"         ,	default=-1   ,				help="radius for PCA" )
	#parser.add_option("--iter", 		type="int"         ,	default=40   ,				help="maximum number of iterations (stop criterion of reconstruction process)" )
	#parser.add_option("--abs", 			type="float"       ,	default=0.0  ,				help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
	#parser.add_option("--squ", 			type="float"       ,	default=0.0  ,				help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
	parser.add_option("--VAR" , 		action="store_true",	default=False,				help="stack on input consists of 2D variances (Default False)")
	parser.add_option("--decimate",     type="float",           default=1.0,                 help="image decimate rate, a number large than 1. default is 1")
	parser.add_option("--window",       type="int",             default=0,                   help="reduce images to a small image size without changing pixel_size. Default value is zero.")
	#parser.add_option("--SND",			action="store_true",	default=False,				help="compute squared normalized differences (Default False)")
	parser.add_option("--nvec",			type="int"         ,	default=0    ,				help="number of eigenvectors, default = 0 meaning no PCA calculated")
	parser.add_option("--symmetrize",	action="store_true",	default=False,				help="Prepare input stack for handling symmetry (Default False)")
	
	(options,args) = parser.parse_args()
	#####
	from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD, MPI_TAG_UB
	from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
	from applications import MPI_start_end
	from reconstruction import recons3d_em, recons3d_em_MPI
	from reconstruction	import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
	from utilities import print_begin_msg, print_end_msg, print_msg
	from utilities import read_text_row, get_image, get_im
	from utilities import bcast_EMData_to_all, bcast_number_to_all
	from utilities import get_symt

	#  This is code for handling symmetries by the above program.  To be incorporated. PAP 01/27/2015

	from EMAN2db import db_open_dict
	
	if options.symmetrize :
		try:
			sys.argv = mpi_init(len(sys.argv), sys.argv)
			try:	
				number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
				if( number_of_proc > 1 ):
					ERROR("Cannot use more than one CPU for symmetry prepration","sx3dvariability",1)
			except:
				pass
		except:
			pass

		#  Input
		#instack = "Clean_NORM_CTF_start_wparams.hdf"
		#instack = "bdb:data"
		instack = args[0]
		sym = options.sym
		if( sym == "c1" ):
			ERROR("Thre is no need to symmetrize stack for C1 symmetry","sx3dvariability",1)

		if(instack[:4] !="bdb:"):
			stack = "bdb:data"
			delete_bdb(stack)
			cmdexecute("sxcpy.py  "+instack+"  "+stack)
		else:
			stack = instack

		qt = EMUtil.get_all_attributes(stack,'xform.projection')

		na = len(qt)
		ts = get_symt(sym)
		ks = len(ts)
		angsa = [None]*na
		for k in xrange(ks):
			delete_bdb("bdb:Q%1d"%k)
			cmdexecute("e2bdb.py  "+stack+"  --makevstack=bdb:Q%1d"%k)
			DB = db_open_dict("bdb:Q%1d"%k)
			for i in xrange(na):
				ut = qt[i]*ts[k]
				DB.set_attr(i, "xform.projection", ut)
				#bt = ut.get_params("spider")
				#angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
			#write_text_row(angsa, 'ptsma%1d.txt'%k)
			#cmdexecute("e2bdb.py  "+stack+"  --makevstack=bdb:Q%1d"%k)
			#cmdexecute("sxheader.py  bdb:Q%1d  --params=xform.projection  --import=ptsma%1d.txt"%(k,k))
			DB.close()
		delete_bdb("bdb:sdata")
		cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
		#cmdexecute("ls  EMAN2DB/sdata*")
		a = get_im("bdb:sdata")
		a.set_attr("variabilitysymmetry",sym)
		a.write_image("bdb:sdata")


	else:

		sys.argv = mpi_init(len(sys.argv), sys.argv)
		myid     = mpi_comm_rank(MPI_COMM_WORLD)
		number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
		main_node = 0

		if len(args) == 1:
			stack = args[0]
		else:
			print( "usage: " + usage)
			print( "Please run '" + progname + " -h' for detailed options")
			return 1

		t0 = time()
	
		# obsolete flags
		options.MPI = True
		options.nvec = 0
		options.radiuspca = -1
		options.iter = 40
		options.abs = 0.0
		options.squ = 0.0

		if options.fl > 0.0 and options.aa == 0.0:
			ERROR("Fall off has to be given for the low-pass filter", "sx3dvariability", 1, myid)
		if options.VAR and options.SND:
			ERROR("Only one of var and SND can be set!", "sx3dvariability", myid)
			exit()
		if options.VAR and (options.ave2D or options.ave3D or options.var2D): 
			ERROR("When VAR is set, the program cannot output ave2D, ave3D or var2D", "sx3dvariability", 1, myid)
			exit()
		#if options.SND and (options.ave2D or options.ave3D):
		#	ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid)
		#	exit()
		if options.nvec > 0 :
			ERROR("PCA option not implemented", "sx3dvariability", 1, myid)
			exit()
		if options.nvec > 0 and options.ave3D == None:
			ERROR("When doing PCA analysis, one must set ave3D", "sx3dvariability", myid=myid)
			exit()
		import string
		options.sym = options.sym.lower()
		 
		if global_def.CACHE_DISABLE:
			from utilities import disable_bdb_cache
			disable_bdb_cache()
		global_def.BATCH = True

		if myid == main_node:
			print_begin_msg("sx3dvariability")
			print_msg("%-70s:  %s\n"%("Input stack", stack))
	
		img_per_grp = options.img_per_grp
		nvec = options.nvec
		radiuspca = options.radiuspca

		symbaselen = 0
		if myid == main_node:
			nima = EMUtil.get_image_count(stack)
			img  = get_image(stack)
			nx   = img.get_xsize()
			ny   = img.get_ysize()
			if options.sym != "c1" :
				imgdata = get_im(stack)
				try:
					i = imgdata.get_attr("variabilitysymmetry")
					if(i != options.sym):
						ERROR("The symmetry provided does not agree with the symmetry of the input stack", "sx3dvariability", myid=myid)
				except:
					ERROR("Input stack is not prepared for symmetry, please follow instructions", "sx3dvariability", myid=myid)
				from utilities import get_symt
				i = len(get_symt(options.sym))
				if((nima/i)*i != nima):
					ERROR("The length of the input stack is incorrect for symmetry processing", "sx3dvariability", myid=myid)
				symbaselen = nima/i
			else:  symbaselen = nima
		else:
			nima = 0
			nx = 0
			ny = 0
		nima = bcast_number_to_all(nima)
		nx   = bcast_number_to_all(nx)
		ny   = bcast_number_to_all(ny)
		Tracker ={}
		Tracker["nx"]  =nx
		Tracker["ny"]  =ny
		Tracker["total_stack"]=nima
		if options.decimate==1.:
			if options.window !=0:
				nx = options.window
				ny = options.window
		else:
			if options.window ==0:
				nx = int(nx/options.decimate)
				ny = int(ny/options.decimate)
			else:
				nx = int(options.window/options.decimate)
				ny = nx
		symbaselen = bcast_number_to_all(symbaselen)
		if radiuspca == -1: radiuspca = nx/2-2

		if myid == main_node:
			print_msg("%-70s:  %d\n"%("Number of projection", nima))
		
		img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
		"""
		if options.SND:
			from projection		import prep_vol, prgs
			from statistics		import im_diff
			from utilities		import get_im, model_circle, get_params_proj, set_params_proj
			from utilities		import get_ctf, generate_ctf
			from filter			import filt_ctf
		
			imgdata = EMData.read_images(stack, range(img_begin, img_end))

			if options.CTF:
				vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
			else:
				vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)

			bcast_EMData_to_all(vol, myid)
			volft, kb = prep_vol(vol)

			mask = model_circle(nx/2-2, nx, ny)
			varList = []
			for i in xrange(img_begin, img_end):
				phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
				ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
				if options.CTF:
					ctf_params = get_ctf(imgdata[i-img_begin])
					ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
				diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
				diff2 = diff*diff
				set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
				varList.append(diff2)
			mpi_barrier(MPI_COMM_WORLD)
		"""
		if options.VAR:
			#varList = EMData.read_images(stack, range(img_begin, img_end))
			varList = []
			this_image = EMData()
			for index_of_particle in xrange(img_begin,img_end):
				this_image.read_image(stack,index_of_particle)
				varList.append(image_decimate_window_xform_ctf(img,options.decimate,options.window,options.CTF))
		else:
			from utilities		import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
			from utilities		import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
			from utilities		import model_blank, nearest_proj, model_circle
			from applications	import pca
			from statistics		import avgvar, avgvar_ctf, ccc
			from filter		    import filt_tanl
			from morphology		import threshold, square_root
			from projection 	import project, prep_vol, prgs
			from sets		    import Set

			if myid == main_node:
				t1 = time()
				proj_angles = []
				aveList = []
				tab = EMUtil.get_all_attributes(stack, 'xform.projection')
				for i in xrange(nima):
					t     = tab[i].get_params('spider')
					phi   = t['phi']
					theta = t['theta']
					psi   = t['psi']
					x     = theta
					if x > 90.0: x = 180.0 - x
					x = x*10000+psi
					proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
				t2 = time()
				print_msg("%-70s:  %d\n"%("Number of neighboring projections", img_per_grp))
				print_msg("...... Finding neighboring projections\n")
				if options.VERBOSE:
					print "Number of images per group: ", img_per_grp
					print "Now grouping projections"
				proj_angles.sort()

			proj_angles_list = [0.0]*(nima*4)
			if myid == main_node:
				for i in xrange(nima):
					proj_angles_list[i*4]   = proj_angles[i][1]
					proj_angles_list[i*4+1] = proj_angles[i][2]
					proj_angles_list[i*4+2] = proj_angles[i][3]
					proj_angles_list[i*4+3] = proj_angles[i][4]
			proj_angles_list = bcast_list_to_all(proj_angles_list, myid, main_node)
			proj_angles = []
			for i in xrange(nima):
				proj_angles.append([proj_angles_list[i*4], proj_angles_list[i*4+1], proj_angles_list[i*4+2], int(proj_angles_list[i*4+3])])
			del proj_angles_list

			proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp, range(img_begin, img_end))

			all_proj = Set()
			for im in proj_list:
				for jm in im:
					all_proj.add(proj_angles[jm][3])

			all_proj = list(all_proj)
			if options.VERBOSE:
				print "On node %2d, number of images needed to be read = %5d"%(myid, len(all_proj))

			index = {}
			for i in xrange(len(all_proj)): index[all_proj[i]] = i
			mpi_barrier(MPI_COMM_WORLD)

			if myid == main_node:
				print_msg("%-70s:  %.2f\n"%("Finding neighboring projections lasted [s]", time()-t2))
				print_msg("%-70s:  %d\n"%("Number of groups processed on the main node", len(proj_list)))
				if options.VERBOSE:
					print "Grouping projections took: ", (time()-t2)/60	, "[min]"
					print "Number of groups on main node: ", len(proj_list)
			mpi_barrier(MPI_COMM_WORLD)

			if myid == main_node:
				print_msg("...... calculating the stack of 2D variances \n")
				if options.VERBOSE:
					print "Now calculating the stack of 2D variances"

			proj_params = [0.0]*(nima*5)
			aveList = []
			varList = []				
			if nvec > 0:
				eigList = [[] for i in xrange(nvec)]

			if options.VERBOSE: 	print "Begin to read images on processor %d"%(myid)
			ttt = time()
			#imgdata = EMData.read_images(stack, all_proj)
			img     = EMData()
			imgdata = []
			for index_of_proj in xrange(len(all_proj)):
				img.read_image(stack, all_proj[index_of_proj])
				dmg = image_decimate_window_xform_ctf(img,options.decimate,options.window,options.CTF)
				#print dmg.get_xsize(), "init"
				imgdata.append(dmg)
			if options.VERBOSE:
				print "Reading images on processor %d done, time = %.2f"%(myid, time()-ttt)
				print "On processor %d, we got %d images"%(myid, len(imgdata))
			mpi_barrier(MPI_COMM_WORLD)

			'''	
			imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
			if options.fl > 0.0:
				for k in xrange(len(imgdata2)):
					imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
			if options.CTF:
				vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
			else:
				vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
			if myid == main_node:
				vol.write_image("vol_ctf.hdf")
				print_msg("Writing to the disk volume reconstructed from averages as		:  %s\n"%("vol_ctf.hdf"))
			del vol, imgdata2
			mpi_barrier(MPI_COMM_WORLD)
			'''
			from applications import prepare_2d_forPCA
			from utilities import model_blank
			for i in xrange(len(proj_list)):
				ki = proj_angles[proj_list[i][0]][3]
				if ki >= symbaselen:  continue
				mi = index[ki]
				phiM, thetaM, psiM, s2xM, s2yM = get_params_proj(imgdata[mi])

				grp_imgdata = []
				for j in xrange(img_per_grp):
					mj = index[proj_angles[proj_list[i][j]][3]]
					phi, theta, psi, s2x, s2y = get_params_proj(imgdata[mj])
					alpha, sx, sy, mirror = params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror_list[i][j])
					if thetaM <= 90:
						if mirror == 0:  alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, phiM-phi, 0.0, 0.0, 1.0)
						else:            alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, 180-(phiM-phi), 0.0, 0.0, 1.0)
					else:
						if mirror == 0:  alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(phiM-phi), 0.0, 0.0, 1.0)
						else:            alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(180-(phiM-phi)), 0.0, 0.0, 1.0)
					set_params2D(imgdata[mj], [alpha, sx, sy, mirror, 1.0])
					grp_imgdata.append(imgdata[mj])
					#print grp_imgdata[j].get_xsize(), imgdata[mj].get_xsize()

				if not options.no_norm:
					#print grp_imgdata[j].get_xsize()
					mask = model_circle(nx/2-2, nx, nx)
					for k in xrange(img_per_grp):
						ave, std, minn, maxx = Util.infomask(grp_imgdata[k], mask, False)
						grp_imgdata[k] -= ave
						grp_imgdata[k] /= std
					del mask

				if options.fl > 0.0:
					from filter import filt_ctf, filt_table
					from fundamentals import fft, window2d
					nx2 = 2*nx
					ny2 = 2*ny
					if options.CTF:
						from utilities import pad
						for k in xrange(img_per_grp):
							grp_imgdata[k] = window2d(fft( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa) ),nx,ny)
							#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
							#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
					else:
						for k in xrange(img_per_grp):
							grp_imgdata[k] = filt_tanl( grp_imgdata[k], options.fl, options.aa)
							#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
							#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
				else:
					from utilities import pad, read_text_file
					from filter import filt_ctf, filt_table
					from fundamentals import fft, window2d
					nx2 = 2*nx
					ny2 = 2*ny
					if options.CTF:
						from utilities import pad
						for k in xrange(img_per_grp):
							grp_imgdata[k] = window2d( fft( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1) ) , nx,ny)
							#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
							#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)

				'''
				if i < 10 and myid == main_node:
					for k in xrange(10):
						grp_imgdata[k].write_image("grp%03d.hdf"%i, k)
				'''
				"""
				if myid == main_node and i==0:
					for pp in xrange(len(grp_imgdata)):
						grp_imgdata[pp].write_image("pp.hdf", pp)
				"""
				ave, grp_imgdata = prepare_2d_forPCA(grp_imgdata)
				"""
				if myid == main_node and i==0:
					for pp in xrange(len(grp_imgdata)):
						grp_imgdata[pp].write_image("qq.hdf", pp)
				"""

				var = model_blank(nx,ny)
				for q in grp_imgdata:  Util.add_img2( var, q )
				Util.mul_scalar( var, 1.0/(len(grp_imgdata)-1))
				# Switch to std dev
				var = square_root(threshold(var))
				#if options.CTF:	ave, var = avgvar_ctf(grp_imgdata, mode="a")
				#else:	            ave, var = avgvar(grp_imgdata, mode="a")
				"""
				if myid == main_node:
					ave.write_image("avgv.hdf",i)
					var.write_image("varv.hdf",i)
				"""
			
				set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
				set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])

				aveList.append(ave)
				varList.append(var)

				if options.VERBOSE:
					print "%5.2f%% done on processor %d"%(i*100.0/len(proj_list), myid)
				if nvec > 0:
					eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True)
					for k in xrange(nvec):
						set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
						eigList[k].append(eig[k])
					"""
					if myid == 0 and i == 0:
						for k in xrange(nvec):
							eig[k].write_image("eig.hdf", k)
					"""

			del imgdata
			#  To this point, all averages, variances, and eigenvectors are computed

			if options.ave2D:
				from fundamentals import fpol
				if myid == main_node:
					km = 0
					for i in xrange(number_of_proc):
						if i == main_node :
							for im in xrange(len(aveList)):
								aveList[im].write_image(options.ave2D, km)
								km += 1
						else:
							nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
							nl = int(nl[0])
							for im in xrange(nl):
								ave = recv_EMData(i, im+i+70000)
								"""
								nm = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
								nm = int(nm[0])
								members = mpi_recv(nm, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
								ave.set_attr('members', map(int, members))
								members = mpi_recv(nm, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
								ave.set_attr('pix_err', map(float, members))
								members = mpi_recv(3, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
								ave.set_attr('refprojdir', map(float, members))
								"""
								tmpvol=fpol(ave, Tracker["nx"],Tracker["nx"],Tracker["nx"])								
								tmpvol.write_image(options.ave2D, km)
								km += 1
				else:
					mpi_send(len(aveList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
					for im in xrange(len(aveList)):
						send_EMData(aveList[im], main_node,im+myid+70000)
						"""
						members = aveList[im].get_attr('members')
						mpi_send(len(members), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
						mpi_send(members, len(members), MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
						members = aveList[im].get_attr('pix_err')
						mpi_send(members, len(members), MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
						try:
							members = aveList[im].get_attr('refprojdir')
							mpi_send(members, 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
						except:
							mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
						"""

			if options.ave3D:
				from fundamentals import fpol
				if options.VERBOSE:
					print "Reconstructing 3D average volume"
				ave3D = recons3d_4nn_MPI(myid, aveList, symmetry=options.sym, npad=options.npad)
				bcast_EMData_to_all(ave3D, myid)
				if myid == main_node:
					ave3D=fpol(ave3D,Tracker["nx"],Tracker["nx"],Tracker["nx"])
					ave3D.write_image(options.ave3D)
					print_msg("%-70s:  %s\n"%("Writing to the disk volume reconstructed from averages as", options.ave3D))
			del ave, var, proj_list, stack, phi, theta, psi, s2x, s2y, alpha, sx, sy, mirror, aveList

			if nvec > 0:
				for k in xrange(nvec):
					if options.VERBOSE:
						print "Reconstruction eigenvolumes", k
					cont = True
					ITER = 0
					mask2d = model_circle(radiuspca, nx, nx)
					while cont:
						#print "On node %d, iteration %d"%(myid, ITER)
						eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad)
						bcast_EMData_to_all(eig3D, myid, main_node)
						if options.fl > 0.0:
							eig3D = filt_tanl(eig3D, options.fl, options.aa)
						if myid == main_node:
							eig3D.write_image("eig3d_%03d.hdf"%k, ITER)
						Util.mul_img( eig3D, model_circle(radiuspca, nx, nx, nx) )
						eig3Df, kb = prep_vol(eig3D)
						del eig3D
						cont = False
						icont = 0
						for l in xrange(len(eigList[k])):
							phi, theta, psi, s2x, s2y = get_params_proj(eigList[k][l])
							proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y])
							cl = ccc(proj, eigList[k][l], mask2d)
							if cl < 0.0:
								icont += 1
								cont = True
								eigList[k][l] *= -1.0
						u = int(cont)
						u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD)
						icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)

						if myid == main_node:
							u = int(u[0])
							print " Eigenvector: ",k," number changed ",int(icont[0])
						else: u = 0
						u = bcast_number_to_all(u, main_node)
						cont = bool(u)
						ITER += 1

					del eig3Df, kb
					mpi_barrier(MPI_COMM_WORLD)
				del eigList, mask2d

			if options.ave3D: del ave3D
			if options.var2D:
				from fundamentals import fpol 
				if myid == main_node:
					km = 0
					for i in xrange(number_of_proc):
						if i == main_node :
							for im in xrange(len(varList)):
								tmpvol=fpol(varList[im], Tracker["nx"], Tracker["nx"],1)
								tmpvol.write_image(options.var2D, km)
								km += 1
						else:
							nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
							nl = int(nl[0])
							for im in xrange(nl):
								ave = recv_EMData(i, im+i+70000)
								tmpvol=fpol(ave, Tracker["nx"], Tracker["nx"],1)
								tmpvol.write_image(options.var2D, km)
								km += 1
				else:
					mpi_send(len(varList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
					for im in xrange(len(varList)):
						send_EMData(varList[im], main_node, im+myid+70000)#  What with the attributes??

			mpi_barrier(MPI_COMM_WORLD)

		if  options.var3D:
			if myid == main_node and options.VERBOSE:
				print "Reconstructing 3D variability volume"

			t6 = time()
			radiusvar = options.radiusvar
			if( radiusvar < 0 ):  radiusvar = nx//2 -3
			res = recons3d_4nn_MPI(myid, varList, symmetry=options.sym, npad=options.npad)
			#res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
			if myid == main_node:
				from fundamentals import fpol
				res =fpol(res, Tracker["nx"], Tracker["nx"], Tracker["nx"])
				res.write_image(options.var3D)

			if myid == main_node:
				print_msg("%-70s:  %.2f\n"%("Reconstructing 3D variability took [s]", time()-t6))
				if options.VERBOSE:
					print "Reconstruction took: %.2f [min]"%((time()-t6)/60)

			if myid == main_node:
				print_msg("%-70s:  %.2f\n"%("Total time for these computations [s]", time()-t0))
				if options.VERBOSE:
					print "Total time for these computations: %.2f [min]"%((time()-t0)/60)
				print_end_msg("sx3dvariability")

		global_def.BATCH = False

		from mpi import mpi_finalize
		mpi_finalize()
Exemplo n.º 21
0
def separate_class(classavgstack, instack, options, outdir='.', verbose=False):
    """
	Main function overseeing various projection-comparison modes.
	
	Arguments:
		classavgstack : Input image stack
		classmap : Class-to-particle lookup table. Each (long) line contains particles assigned to a class, one file for all classes
		options : (list) Command-line options, run 'sxproj_compare.py -h' for an exhaustive list
		outdir : Output directory
		verbose : (boolean) Whether to write additional information to screen
	"""

    # Set output directory and log file name
    prepare_outdir(outdir, verbose)
    log, verbose = prepare_log(outdir, verbose)

    # Expand paths for outputs
    classmap = os.path.join(outdir, CLASSMAPFILE)
    classdoc = os.path.join(outdir, CLASSDOCPREFIX + '{0:03d}.txt')
    outbdb = os.path.join(outdir, CLASSSTACKPREFIX + '{0:03d}')
    outflt = os.path.join(outdir, FILTSTACKPREFIX + '{0:03d}')
    num_classes = EMUtil.get_image_count(classavgstack)

    # Generate class-to-particle lookup table and class-selection lists
    vomq(classavgstack, classmap, classdoc, log=log, verbose=verbose)

    print_log_msg("Writing each class to a separate stack", log, verbose)
    if options.shrink:
        print_log_msg(
            "Will downsample stacks by a factor of %s" % options.shrink, log,
            verbose)

    if options.filtrad:
        if options.pxsz:
            filtrad = options.pxsz / options.filtrad
        else:
            filtrad = options.filtrad
        print_log_msg("Will low-pass filter to %s px^-1" % options.shrink, log,
                      verbose)

    tot_parts = 0

    # Loop through classes
    for class_num in xrange(num_classes):
        # Write BDB stacks
        cmd = "e2bdb.py %s --makevstack bdb:%s --list %s" % (
            instack, outbdb.format(class_num), classdoc.format(class_num))
        print_log_msg(cmd, log, verbose)
        os.system(cmd)
        num_class_imgs = EMUtil.get_image_count('bdb:' +
                                                outbdb.format(class_num))
        tot_parts += num_class_imgs

        # Optional filtered stack
        if options.filtrad or options.shrink:
            cmd = "e2proc2d.py bdb:%s bdb:%s --inplace" % (
                outbdb.format(class_num), outflt.format(class_num))
            # --inplace overwrites existing images

            if options.filtrad:
                cmd = cmd + " --process=filter.lowpass.gauss:cutoff_freq=%s" % filtrad

            if options.shrink:
                cmd = cmd + " --meanshrink=%s" % options.shrink

            print_log_msg(cmd, log, verbose)
            os.system(cmd)

    print_log_msg("\nDone! Separated %s particles from %s classes\n" %
                  (tot_parts, num_classes),
                  log,
                  verbose=True)
Exemplo n.º 22
0
def main():
	import sys
	import os
	import math
	import random
	import pyemtbx.options
	import time
	from   random   import random, seed, randint
	from   optparse import OptionParser

	progname = os.path.basename(sys.argv[0])
	usage = progname + """ [options] <inputfile> <outputfile>

	Forms chains of 2D images based on their similarities.

	Functionality:


	4.  Order a 2-D stack of image based on pair-wise similarity (computed as a cross-correlation coefficent).
		Options 1-3 require image stack to be aligned.  The program will apply orientation parameters if present in headers.
	    The ways to use the program:
	   4.1  Use option initial to specify which image will be used as an initial seed to form the chain.
	        sxprocess.py input_stack.hdf output_stack.hdf --initial=23 --radius=25
	   4.2  If options initial is omitted, the program will determine which image best serves as initial seed to form the chain
	        sxprocess.py input_stack.hdf output_stack.hdf --radius=25
	   4.3  Use option circular to form a circular chain.
	        sxprocess.py input_stack.hdf output_stack.hdf --circular--radius=25
	   4.4  New circular code based on pairwise alignments
			sxprocess.py aclf.hdf chain.hdf circle.hdf --align  --radius=25 --xr=2 --pairwiseccc=lcc.txt

	   4.5  Circular ordering based on pairwise alignments
			sxprocess.py vols.hdf chain.hdf mask.hdf --dd  --radius=25


"""

	parser = OptionParser(usage,version=SPARXVERSION)
	parser.add_option("--dd", action="store_true", help="Circular ordering without adjustment of orientations", default=False)
	parser.add_option("--circular", action="store_true", help="Select circular ordering (first image has to be similar to the last)", default=False)
	parser.add_option("--align", action="store_true", help="Compute all pairwise alignments and for the table of their similarities find the best chain", default=False)
	parser.add_option("--initial", type="int", default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)")
	parser.add_option("--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering")
	#  import params for 2D alignment
	parser.add_option("--ou",           type="int",    default=-1,          help="outer radius for 2D alignment < nx/2-1 (set to the radius of the particle)")
	parser.add_option("--xr",           type="int",    default=0,     		help="range for translation search in x direction, search is +/xr (0)")
	parser.add_option("--yr",           type="int",    default=0,          	help="range for translation search in y direction, search is +/yr (0)")
	#parser.add_option("--nomirror",     action="store_true", default=False,   help="Disable checking mirror orientations of images (default False)")
	parser.add_option("--pairwiseccc",  type="string",	default= None,      help="Input/output pairwise ccc file")


 	(options, args) = parser.parse_args()

	global_def.BATCH = True

					
	if options.dd:
		nargs = len(args)
		if nargs != 3:
			print "must provide name of input and two output files!"
			return
		stack = args[0]
		new_stack = args[1]


		from utilities import model_circle
		from statistics import ccc
		from statistics import mono
		lend = EMUtil.get_image_count(stack)
		lccc = [None]*(lend*(lend-1)/2)

		for i in xrange(lend-1):
			v1 = get_im( stack, i )
			if( i == 0 and nargs == 2):
				nx = v1.get_xsize()
				ny = v1.get_ysize()
				nz = v1.get_ysize()
				if options.ou < 1 : radius = nx//2-2
				else:  radius = options.ou
				mask = model_circle(radius, nx, ny, nz)
			else:
				mask = get_im(args[2])
				
			for j in xrange(i+1, lend):
				lccc[mono(i,j)] = [ccc(v1, get_im( stack, j ), mask), 0]


		order = tsp(lccc)
		if(len(order) != lend):
			print  " problem with data length"
			from sys import exit
			exit()
		print  "Total sum of cccs :",TotalDistance(order, lccc)
		print "ordering :",order
		for i in xrange(lend):  get_im(stack, order[i]).write_image( new_stack, i )

	elif options.align:
		nargs = len(args)
		if nargs != 3:
			print "must provide name of input and two output files!"
			return

		from utilities import get_params2D, model_circle
		from fundamentals import rot_shift2D
		from statistics import ccc
		from time import time
		from alignment import align2d, align2d_scf
		from multi_shc import mult_transform 
		
		stack = args[0]
		new_stack = args[1]
		
		d = EMData.read_images(stack)

		"""
		# will align anyway
		try:
			ttt = d[0].get_attr('xform.params2d')
			for i in xrange(len(d)):
				alpha, sx, sy, mirror, scale = get_params2D(d[i])
				d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror)
		except:
			pass
		"""

		nx = d[0].get_xsize()
		ny = d[0].get_ysize()
		if options.ou < 1 : radius = nx//2-2
		else:  radius = options.ou
		mask = model_circle(radius, nx, ny)

		if(options.xr < 0):	xrng = 0
		else:					xrng = options.xr
		if(options.yr < 0):	yrng = xrng
		else:					yrng = options.yr
			
		initial = max(options.initial, 0)

		from statistics import mono
		lend = len(d)
		lccc = [None]*(lend*(lend-1)/2)
		from utilities import read_text_row

		if  options.pairwiseccc == None or not os.path.exists(options.pairwiseccc) :
			st = time()
			for i in xrange(lend-1):
				for j in xrange(i+1, lend):
					#  j>i meaning mono entry (i,j) or (j,i) indicates T i->j (from smaller index to larger)
					#alpha, sx, sy, mir, peak = align2d(d[i],d[j], xrng, yrng, step=options.ts, first_ring=options.ir, last_ring=radius, mode = "F")
					alpha, sx, sy, mir, peak = align2d_scf(d[i],d[j], xrng, yrng, ou=radius)
					lccc[mono(i,j)] = [ccc(d[j], rot_shift2D(d[i], alpha, sx, sy, mir, 1.0), mask), alpha, sx, sy, mir]
				#print "  %4d   %10.1f"%(i,time()-st)

			if(not os.path.exists(options.pairwiseccc)):
				from utilities import write_text_row
				write_text_row([[initial,0,0,0,0]]+lccc,options.pairwiseccc)
		elif(os.path.exists(options.pairwiseccc)):
			lccc = read_text_row(options.pairwiseccc)
			initial = int(lccc[0][0] + 0.1)
			del lccc[0]


		for i in xrange(len(lccc)):
			T = Transform({"type":"2D","alpha":lccc[i][1],"tx":lccc[i][2],"ty":lccc[i][3],"mirror":int(lccc[i][4]+0.1)})
			lccc[i] = [lccc[i][0],T]

		tdummy = Transform({"type":"2D"})
		maxsum = -1.023
		for m in xrange(0,lend):#initial, initial+1):
			indc = range( lend )
			lsnake = [[m, tdummy, 0.0]]
			del indc[m]

			lsum = 0.0
			while len(indc) > 1:
				maxcit = -111.
				for i in xrange(len(indc)):
						cuc = lccc[mono(indc[i], lsnake[-1][0])][0]
						if cuc > maxcit:
								maxcit = cuc
								qi = indc[i]
								#  Here we need transformation from the current to the previous,
								#     meaning indc[i] -> lsnake[-1][0]
								T = lccc[mono(indc[i], lsnake[-1][0])][1]
								#  If direction is from larger to smaller index, the transformation has to be inverted
								if( indc[i] > lsnake[-1][0] ):  T = T.inverse()
								
								
				lsnake.append([qi,T, maxcit])
				lsum += maxcit

				del indc[indc.index(qi)]

			T = lccc[mono(indc[-1], lsnake[-1][0])][1]
			if( indc[-1] > lsnake[-1][0]):  T = T.inverse()
			lsnake.append([indc[-1], T, lccc[mono(indc[-1], lsnake[-1][0])][0]])
			print  " initial image and lsum  ",m,lsum
			#print lsnake
			if(lsum > maxsum):
				maxsum = lsum
				init = m
				snake = [lsnake[i] for i in xrange(lend)]
		print  "  Initial image selected : ",init,maxsum,"    ",TotalDistance([snake[m][0] for m in xrange(lend)], lccc)
		#for q in snake: print q

		from copy import deepcopy
		trans=deepcopy([snake[i][1] for i in xrange(len(snake))])
		print  [snake[i][0] for i in xrange(len(snake))]
Exemplo n.º 23
0
def main():
	import sys
	import os
	import math
	import random
	import pyemtbx.options
	import time
	from   random   import random, seed, randint
	from   optparse import OptionParser

	progname = os.path.basename(sys.argv[0])
	usage = progname + """ [options] <inputfile> <outputfile>

	Forms chains of 2D images based on their similarities.

	Functionality:


	4.  Order a 2-D stack of image based on pair-wise similarity (computed as a cross-correlation coefficent).
		Options 1-3 require image stack to be aligned.  The program will apply orientation parameters if present in headers.
	    The ways to use the program:
	   4.1  Use option initial to specify which image will be used as an initial seed to form the chain.
	        sxprocess.py input_stack.hdf output_stack.hdf --initial=23 --radius=25
	   4.2  If options initial is omitted, the program will determine which image best serves as initial seed to form the chain
	        sxprocess.py input_stack.hdf output_stack.hdf --radius=25
	   4.3  Use option circular to form a circular chain.
	        sxprocess.py input_stack.hdf output_stack.hdf --circular--radius=25
	   4.4  New circular code based on pairwise alignments
			sxprocess.py aclf.hdf chain.hdf circle.hdf --align  --radius=25 --xr=2 --pairwiseccc=lcc.txt

	   4.5  Circular ordering based on pairwise alignments
			sxprocess.py vols.hdf chain.hdf mask.hdf --dd  --radius=25


"""

	parser = OptionParser(usage,version=SPARXVERSION)
	parser.add_option("--dd", action="store_true", help="Circular ordering without adjustment of orientations", default=False)
	parser.add_option("--circular", action="store_true", help="Select circular ordering (first image has to be similar to the last)", default=False)
	parser.add_option("--align", action="store_true", help="Compute all pairwise alignments and for the table of their similarities find the best chain", default=False)
	parser.add_option("--initial", type="int", default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)")
	parser.add_option("--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering")
	#  import params for 2D alignment
	parser.add_option("--ou",           type="int",    default=-1,          help="outer radius for 2D alignment < nx/2-1 (set to the radius of the particle)")
	parser.add_option("--xr",           type="int",    default=0,     		help="range for translation search in x direction, search is +/xr (0)")
	parser.add_option("--yr",           type="int",    default=0,          	help="range for translation search in y direction, search is +/yr (0)")
	#parser.add_option("--nomirror",     action="store_true", default=False,   help="Disable checking mirror orientations of images (default False)")
	parser.add_option("--pairwiseccc",  type="string",	default= None,      help="Input/output pairwise ccc file")


 	(options, args) = parser.parse_args()

	global_def.BATCH = True

					
	if options.dd:
		nargs = len(args)
		if nargs != 3:
			print "must provide name of input and two output files!"
			return
		stack = args[0]
		new_stack = args[1]


		from utilities import model_circle
		from statistics import ccc
		from statistics import mono
		lend = EMUtil.get_image_count(stack)
		lccc = [None]*(lend*(lend-1)/2)

		for i in xrange(lend-1):
			v1 = get_im( stack, i )
			if( i == 0 and nargs == 2):
				nx = v1.get_xsize()
				ny = v1.get_ysize()
				nz = v1.get_ysize()
				if options.ou < 1 : radius = nx//2-2
				else:  radius = options.ou
				mask = model_circle(radius, nx, ny, nz)
			else:
				mask = get_im(args[2])
				
			for j in xrange(i+1, lend):
				lccc[mono(i,j)] = [ccc(v1, get_im( stack, j ), mask), 0]


		order = tsp(lccc)
		if(len(order) != lend):
			print  " problem with data length"
			from sys import exit
			exit()
		print  "Total sum of cccs :",TotalDistance(order, lccc)
		print "ordering :",order
		for i in xrange(lend):  get_im(stack, order[i]).write_image( new_stack, i )

	elif options.align:
		nargs = len(args)
		if nargs != 3:
			print "must provide name of input and two output files!"
			return

		from utilities import get_params2D, model_circle
		from fundamentals import rot_shift2D
		from statistics import ccc
		from time import time
		from alignment import align2d, align2d_scf
		from multi_shc import mult_transform 
		
		stack = args[0]
		new_stack = args[1]
		
		d = EMData.read_images(stack)

		"""
		# will align anyway
		try:
			ttt = d[0].get_attr('xform.params2d')
			for i in xrange(len(d)):
				alpha, sx, sy, mirror, scale = get_params2D(d[i])
				d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror)
		except:
			pass
		"""

		nx = d[0].get_xsize()
		ny = d[0].get_ysize()
		if options.ou < 1 : radius = nx//2-2
		else:  radius = options.ou
		mask = model_circle(radius, nx, ny)

		if(options.xr < 0):	xrng = 0
		else:					xrng = options.xr
		if(options.yr < 0):	yrng = xrng
		else:					yrng = options.yr
			
		initial = max(options.initial, 0)

		from statistics import mono
		lend = len(d)
		lccc = [None]*(lend*(lend-1)/2)
		from utilities import read_text_row

		if  options.pairwiseccc == None or not os.path.exists(options.pairwiseccc) :
			st = time()
			for i in xrange(lend-1):
				for j in xrange(i+1, lend):
					#  j>i meaning mono entry (i,j) or (j,i) indicates T i->j (from smaller index to larger)
					#alpha, sx, sy, mir, peak = align2d(d[i],d[j], xrng, yrng, step=options.ts, first_ring=options.ir, last_ring=radius, mode = "F")
					alpha, sx, sy, mir, peak = align2d_scf(d[i],d[j], xrng, yrng, ou=radius)
					lccc[mono(i,j)] = [ccc(d[j], rot_shift2D(d[i], alpha, sx, sy, mir, 1.0), mask), alpha, sx, sy, mir]
				#print "  %4d   %10.1f"%(i,time()-st)

			if(not os.path.exists(options.pairwiseccc)):
				from utilities import write_text_row
				write_text_row([[initial,0,0,0,0]]+lccc,options.pairwiseccc)
		elif(os.path.exists(options.pairwiseccc)):
			lccc = read_text_row(options.pairwiseccc)
			initial = int(lccc[0][0] + 0.1)
			del lccc[0]


		for i in xrange(len(lccc)):
			T = Transform({"type":"2D","alpha":lccc[i][1],"tx":lccc[i][2],"ty":lccc[i][3],"mirror":int(lccc[i][4]+0.1)})
			lccc[i] = [lccc[i][0],T]

		tdummy = Transform({"type":"2D"})
		maxsum = -1.023
		for m in xrange(0,lend):#initial, initial+1):
			indc = range( lend )
			lsnake = [[m, tdummy, 0.0]]
			del indc[m]

			lsum = 0.0
			while len(indc) > 1:
				maxcit = -111.
				for i in xrange(len(indc)):
						cuc = lccc[mono(indc[i], lsnake[-1][0])][0]
						if cuc > maxcit:
								maxcit = cuc
								qi = indc[i]
								#  Here we need transformation from the current to the previous,
								#     meaning indc[i] -> lsnake[-1][0]
								T = lccc[mono(indc[i], lsnake[-1][0])][1]
								#  If direction is from larger to smaller index, the transformation has to be inverted
								if( indc[i] > lsnake[-1][0] ):  T = T.inverse()
								
								
				lsnake.append([qi,T, maxcit])
				lsum += maxcit

				del indc[indc.index(qi)]

			T = lccc[mono(indc[-1], lsnake[-1][0])][1]
			if( indc[-1] > lsnake[-1][0]):  T = T.inverse()
			lsnake.append([indc[-1], T, lccc[mono(indc[-1], lsnake[-1][0])][0]])
			print  " initial image and lsum  ",m,lsum
			#print lsnake
			if(lsum > maxsum):
				maxsum = lsum
				init = m
				snake = [lsnake[i] for i in xrange(lend)]
		print  "  Initial image selected : ",init,maxsum,"    ",TotalDistance([snake[m][0] for m in xrange(lend)], lccc)
		#for q in snake: print q

		from copy import deepcopy
		trans=deepcopy([snake[i][1] for i in xrange(len(snake))])
		print  [snake[i][0] for i in xrange(len(snake))]
		"""
		for m in xrange(lend):
			prms = trans[m].get_params("2D")
			print " %3d   %7.1f   %7.1f   %7.1f   %2d  %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2])
		"""
		for k in xrange(lend-2,0,-1):
			T = snake[k][1]
			for i in xrange(k+1, lend):
 					trans[i] = T*trans[i]
		#  To add - apply all transformations and do the overall centering.
		for m in xrange(lend):
			prms = trans[m].get_params("2D")
			#print " %3d   %7.1f   %7.1f   %7.1f   %2d  %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2])
			#rot_shift2D(d[snake[m][0]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image(new_stack, m)
			rot_shift2D(d[snake[m][0]], prms["alpha"], 0.0,0.0, prms["mirror"]).write_image(new_stack, m)

		order = tsp(lccc)
		if(len(order) != lend):
			print  " problem with data length"
			from sys import exit
			exit()
		print  TotalDistance(order, lccc)
		print order
		ibeg = order.index(init)
		order = [order[(i+ibeg)%lend] for i in xrange(lend)]
		print  TotalDistance(order, lccc)
		print order


		snake = [tdummy]
		for i in xrange(1,lend):
			#  Here we need transformation from the current to the previous,
			#     meaning order[i] -> order[i-1]]
			T = lccc[mono(order[i], order[i-1])][1]
			#  If direction is from larger to smaller index, the transformation has to be inverted
			if( order[i] > order[i-1] ):  T = T.inverse()
			snake.append(T)
		assert(len(snake) == lend)
		from copy import deepcopy
		trans = deepcopy(snake)
		for k in xrange(lend-2,0,-1):
			T = snake[k]
			for i in xrange(k+1, lend):
 					trans[i] = T*trans[i]

		#  Try to smooth the angles - complicated, I am afraid one would have to use angles forward and backwards
		#     and find their average??
		#  In addition, one would have to recenter them
		"""
		trms = []
		for m in xrange(lend):
			prms = trans[m].get_params("2D")
			trms.append([prms["alpha"], prms["mirror"]])
		for i in xrange(3):
			for m in xrange(lend):
				mb = (m-1)%lend
				me = (m+1)%lend
				#  angles order mb,m,me
				# calculate predicted angles mb->m 
		"""

		for m in xrange(lend):
			prms = trans[m].get_params("2D")
			#rot_shift2D(d[order[m]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image("metro.hdf", m)
			rot_shift2D(d[order[m]], prms["alpha"], 0.0,0.0, prms["mirror"]).write_image(args[2], m)

		"""
		#  This was an effort to get number of loops, inconclusive, to say the least
		from numpy import outer, zeros, float32, sqrt
		lend = len(d)
 		cor = zeros(lend,float32)
 		cor = outer(cor, cor)
		for i in xrange(lend):  cor[i][i] = 1.0
		for i in xrange(lend-1):
			for j in xrange(i+1, lend):
				cor[i,j] = lccc[mono(i,j)][0]
				cor[j,i] = cor[i,j]

		lmbd, eigvec = pca(cor)

		from utilities import write_text_file

		nvec=20
		print  [lmbd[j] for j in xrange(nvec)]
		print  " G"
		mm = [-1]*lend
		for i in xrange(lend):  # row
			mi = -1.0e23
			for j in xrange(nvec):
				qt = eigvec[j][i]
				if(abs(qt)>mi):
					mi = abs(qt)
					mm[i] = j
			for j in xrange(nvec):
				qt = eigvec[j][i]
				print  round(qt,3),   #  eigenvector
			print  mm[i]
		print
		for j in xrange(nvec):
			qt = []
			for i in xrange(lend):
				if(mm[i] == j):  qt.append(i)
			if(len(qt)>0):  write_text_file(qt,"loop%02d.txt"%j)
		"""
		"""
		print  [lmbd[j] for j in xrange(nvec)]
		print  " B"
		mm = [-1]*lend
		for i in xrange(lend):  # row
			mi = -1.0e23
			for j in xrange(nvec):
				qt = eigvec[j][i]/sqrt(lmbd[j])
				if(abs(qt)>mi):
					mi = abs(qt)
					mm[i] = j
			for j in xrange(nvec):
				qt = eigvec[j][i]/sqrt(lmbd[j])
				print  round(qt,3),   #  eigenvector
			print  mm[i]
		print
		"""

		"""
		lend=3
 		cor = zeros(lend,float32)
 		
 		cor = outer(cor, cor)
 		
 		
 		cor[0][0] =136.77
 		cor[0][1] = 79.15
 		cor[0][2] = 37.13
 		
 		cor[1][0] = 79.15
 		cor[2][0] = 37.13
 		
 		
 		cor[1][1] = 50.04
 		cor[1][2] = 21.65
 		
 		cor[2][1] = 21.65
 		
 		
 		cor[2][2] = 13.26

		lmbd, eigvec = pca(cor)
		print  lmbd
		print  eigvec
		for i in xrange(lend):  # row
			for j in xrange(lend):  print  eigvec[j][i],   #  eigenvector
			print
		print  " B"
		for i in xrange(lend):  # row
			for j in xrange(lend):  print  eigvec[j][i]/sqrt(lmbd[j]),   #  eigenvector
			print
		print  " G"
		for i in xrange(lend):  # row
			for j in xrange(lend):  print  eigvec[j][i]*sqrt(lmbd[j]),   #  eigenvector
			print
		"""
	else:
		nargs = len(args)
		if nargs != 2:
			print "must provide name of input and output file!"
			return
		
		from utilities import get_params2D, model_circle
		from fundamentals import rot_shift2D
		from statistics import ccc
		from time import time
		from alignment import align2d
		from multi_shc import mult_transform 
		
		stack = args[0]
		new_stack = args[1]
		
		d = EMData.read_images(stack)
		try:
			ttt = d[0].get_attr('xform.params2d')
			for i in xrange(len(d)):
				alpha, sx, sy, mirror, scale = get_params2D(d[i])
				d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror)
		except:
			pass

		nx = d[0].get_xsize()
		ny = d[0].get_ysize()
		if options.radius < 1 : radius = nx//2-2
		else:  radius = options.radius
		mask = model_circle(radius, nx, ny)

		init = options.initial
		
		if init > -1 :
			print "      initial image: %d" % init
			temp = d[init].copy()
			temp.write_image(new_stack, 0)
			del d[init]
			k = 1
			lsum = 0.0
			while len(d) > 1:
				maxcit = -111.
				for i in xrange(len(d)):
						cuc = ccc(d[i], temp, mask)
						if cuc > maxcit:
								maxcit = cuc
								qi = i
				# 	print k, maxcit
				lsum += maxcit
				temp = d[qi].copy()
				del d[qi]
				temp.write_image(new_stack, k)
				k += 1
			print  lsum
			d[0].write_image(new_stack, k)
		else:			
			if options.circular :
				print "Using options.circular"
				#  figure the "best circular" starting image
				maxsum = -1.023
				for m in xrange(len(d)):
					indc = range(len(d) )
					lsnake = [-1]*(len(d)+1)
					lsnake[0]  = m
					lsnake[-1] = m
					del indc[m]
					temp = d[m].copy()
					lsum = 0.0
					direction = +1
					k = 1
					while len(indc) > 1:
						maxcit = -111.
						for i in xrange(len(indc)):
								cuc = ccc(d[indc[i]], temp, mask)
								if cuc > maxcit:
										maxcit = cuc
										qi = indc[i]
						lsnake[k] = qi
						lsum += maxcit
						del indc[indc.index(qi)]
						direction = -direction
						for i in xrange( 1,len(d) ):
							if( direction > 0 ):
								if(lsnake[i] == -1):
									temp = d[lsnake[i-1]].copy()
									#print  "  forw  ",lsnake[i-1]
									k = i
									break
							else:
								if(lsnake[len(d) - i] == -1):
									temp = d[lsnake[len(d) - i +1]].copy()
									#print  "  back  ",lsnake[len(d) - i +1]
									k = len(d) - i
									break

					lsnake[lsnake.index(-1)] = indc[-1]
					#print  " initial image and lsum  ",m,lsum
					#print lsnake
					if(lsum > maxsum):
						maxsum = lsum
						init = m
						snake = [lsnake[i] for i in xrange(len(d))]
				print  "  Initial image selected : ",init,maxsum
				print lsnake
				for m in xrange(len(d)):  d[snake[m]].write_image(new_stack, m)
			else:
				#  figure the "best" starting image
				maxsum = -1.023
				for m in xrange(len(d)):
					indc = range(len(d) )
					lsnake = [m]
					del indc[m]
					temp = d[m].copy()
					lsum = 0.0
					while len(indc) > 1:
						maxcit = -111.
						for i in xrange(len(indc)):
								cuc = ccc(d[indc[i]], temp, mask)
								if cuc > maxcit:
										maxcit = cuc
										qi = indc[i]
						lsnake.append(qi)
						lsum += maxcit
						temp = d[qi].copy()
						del indc[indc.index(qi)]

					lsnake.append(indc[-1])
					#print  " initial image and lsum  ",m,lsum
					#print lsnake
					if(lsum > maxsum):
						maxsum = lsum
						init = m
						snake = [lsnake[i] for i in xrange(len(d))]
				print  "  Initial image selected : ",init,maxsum
				print lsnake
				for m in xrange(len(d)):  d[snake[m]].write_image(new_stack, m)
Exemplo n.º 24
0
def runcheck(classavgstack, recon, outdir, inangles=None, selectdoc=None, displayYN=False, 
			 projstack='proj.hdf', outangles='angles.txt', outstack='comp-proj-reproj.hdf', normstack='comp-proj-reproj-norm.hdf'):
	
	print
	
	# Check if inputs exist
	check(classavgstack)
	check(recon)
	
	# Create directory if it doesn't exist
	if not os.path.isdir(outdir):
		os.makedirs(outdir)  # os.mkdir() can only operate one directory deep
		print("mkdir -p %s" % outdir)

	# Expand path for outputs
	projstack = os.path.join(outdir, projstack)
	outangles = os.path.join(outdir, outangles)
	outstack  = os.path.join(outdir, outstack)
	normstack = os.path.join(outdir, normstack)
	
	# Get number of images
	nimg0 = EMUtil.get_image_count(classavgstack)
	#print("nimg0: %s" % nimg0)
	
	# In case class averages include discarded images, apply selection file
	if selectdoc:
		goodavgs, extension = os.path.splitext(classavgstack)
		newclasses = goodavgs + "_kept" + extension
		
		# e2proc2d appends to existing files, so rename existing output
		if os.path.exists(newclasses):
			renamefile = newclasses + '.bak'
			os.rename(newclasses, renamefile)
			print("mv %s %s" % (newclasses, renamefile))
		
		cmd7="e2proc2d.py %s %s --list=%s" % (classavgstack, newclasses, selectdoc)
		print cmd7
		os.system(cmd7)
		
		# Update class-averages
		classavgstack = newclasses
	
	# Import Euler angles
	if inangles:
		cmd6="sxheader.py %s --params=xform.projection --import=%s" % (classavgstack, inangles)
		print cmd6
		header(classavgstack, 'xform.projection', fimport=inangles)
	
	cmd1="sxheader.py %s --params=xform.projection --export=%s" % (classavgstack, outangles) 
	print cmd1
	#os.system(cmd1)
	try:
		header(classavgstack, 'xform.projection', fexport=outangles)
	except RuntimeError:
		print("\nERROR!! No projection angles found in class-average stack header!\n")
		exit()
	
	cmd2="sxproject3d.py %s %s --angles=%s" % (recon, projstack, outangles)
	print cmd2
	#os.system(cmd2)
	project3d(recon, stack=projstack, listagls=outangles)
	
	imgcounter = 0  # montage will have double the number of images as number of class-averages
	result=[]
	
	# Number of images may have changed
	nimg1   = EMUtil.get_image_count(classavgstack)
	
	for imgnum in xrange(nimg1):
		#print imgnum
		classimg = get_im(classavgstack, imgnum)
		ccc1 = classimg.get_attr_default('cross-corr', -1.0)
		prjimg = get_im(projstack,imgnum)
		ccc1 = prjimg.get_attr_default('cross-corr', -1.0)
		cccoeff = ccc(prjimg,classimg)
		#print imgnum, cccoeff
		classimg.set_attr_dict({'cross-corr':cccoeff})
		prjimg.set_attr_dict({'cross-corr':cccoeff})
		prjimg.write_image(outstack,imgcounter)
		imgcounter += 1
		classimg.write_image(outstack, imgcounter)
		imgcounter += 1
		result.append(cccoeff)
	result1 = sum(result)
	#print result1

	nimg2   = EMUtil.get_image_count(outstack)
	meanccc = result1/nimg1
	print("Mean CCC is %s" % meanccc)
	
	for imgnum in xrange(nimg2):
		if (imgnum % 2 ==0):
			prjimg = get_im(outstack,imgnum)
			meanccc1 = prjimg.get_attr_default('mean-cross-corr', -1.0)
			prjimg.set_attr_dict({'mean-cross-corr':meanccc})
			write_header(outstack,prjimg,imgnum)
		if (imgnum % 100) == 0:
			print imgnum
	
	# e2proc2d appends to existing files, so delete existing output
	if os.path.exists(normstack):
		os.remove(normstack)
		print("rm %s" % normstack)
		
	cmd5="e2proc2d.py %s %s --process=normalize" % (outstack, normstack)
	print cmd5
	os.system(cmd5)
	
	# Optionally pop up e2display
	if displayYN:
		cmd8 = "e2display.py %s" % normstack
		print cmd8
		os.system(cmd8)
	
	print("Done!")
Exemplo n.º 25
0
	def __new__(cls,filename,application,force_plot=False,force_2d=False,old=None):
		
		file_type = Util.get_filename_ext(filename)
		em_file_type = EMUtil.get_image_ext_type(file_type)
		if not file_exists(filename): return None
		
		if force_plot and force_2d:
			# ok this sucks but it suffices for the time being
			print("Error, the force_plot and force_2d options are mutually exclusive")
			return None
		
		if force_plot:
			from .emplot2d import EMPlot2DWidget
			if isinstance(old,EMPlot2DWidget): widget = old
			else: widget = EMPlot2DWidget(application=application)
			widget.set_data_from_file(filename)
			return widget
		
		if em_file_type != IMAGE_UNKNOWN or filename[:4] == "bdb:":
			n = EMUtil.get_image_count(filename)
			nx,ny,nz = gimme_image_dimensions3D(filename)
			if n > 1 and nz == 1: 
				if force_2d:
					a = EMData()
					data=a.read_images(filename)
				else:
					data = None # This is like a flag - the ImageMXWidget only needs the file name
			elif nz == 1:
				data = [EMData(filename,0)]
			else:
				data = EMData()
				data.read_image(filename,0,not force_2d)		# This should be 3-D. We read the header-only here
				data = [data]
				
			if data != None and len(data) == 1: data = data[0]
			
			if force_2d or isinstance(data,EMData) and data.get_zsize()==1:
				if isinstance(data,list) or data.get_ysize() != 1:
					from .emimage2d import EMImage2DWidget
					if isinstance(old,EMImage2DWidget): widget = old
					else: widget= EMImage2DWidget(application=application)
				else:
					from .emplot2d import EMPlot2DWidget
					if isinstance(old,EMPlot2DWidget): widget = old
					else: widget = EMPlot2DWidget(application=application)
					widget.set_data_from_file(filename)
					return widget
			elif isinstance(data,EMData):
				if isinstance(old,EMScene3D): widget = old
				else: widget = EMScene3D()
#				print n,data
				for ii in range(n):
					data=EMData(filename,ii)
					datai = EMDataItem3D(data, transform=Transform())
					widget.insertNewNode(os.path.basename(filename), datai, parentnode=widget)
					isosurface = EMIsosurface(datai, transform=Transform())
					widget.insertNewNode("Iso", isosurface, parentnode=datai)
				return widget
				
			elif data == None or isinstance(data,list):
				from .emimagemx import EMImageMXWidget
				if isinstance(old,EMImageMXWidget): widget = old
				else: widget = EMImageMXWidget(application=application)
				data = filename
			else: 
				print(filename)
				raise # weirdness, this should never happen
			widget.set_data(data,filename)
			return widget
		else:
			from .emplot2d import EMPlot2DWidget
			if isinstance(old,EMPlot2DWidget): widget = old
			else: widget = EMPlot2DWidget(application=application)
			widget.set_data_from_file(filename)
			return widget
Exemplo n.º 26
0
def main():
	arglist = []
	for arg in sys.argv:
		arglist.append( arg )

	progname = os.path.basename( arglist[0] )
	usage = progname + " prj_stack volume [begin end step] --CTF --npad=ntimes_padding --list=file --group=ID --snr=SNR --sym=symmetry --verbose=(0|1) --xysize --MPI"
	parser = OptionParser(usage, version=SPARXVERSION)

	parser.add_option("--CTF",                    action="store_true",   default=False, help="apply CTF correction")
	parser.add_option("--snr",                    type="float",	         default=1.0,   help="Signal-to-Noise Ratio" )
	parser.add_option("--sym",                    type="string",	     default="c1",  help="symmetry" )
	parser.add_option("--list",                   type="string",                        help="file with list of images to be used in the first column" )
	parser.add_option("--group",                  type="int",           default=-1,     help="perform reconstruction using images for a given group number (group is attribute in the header)" )
	parser.add_option("--MPI",                    action="store_true",  default=False,  help="use MPI version ")
	parser.add_option("--npad",                   type="int",	        default=2,      help="number of times padding (default 2)" )
	parser.add_option("--verbose",                type="int",           default=0,      help="verbose level: 0 no verbose, 1 verbose" )
	parser.add_option("--xysize",                 type="int",	        default=-1,     help="user expected size at xy direction" )
	parser.add_option("--zsize",                  type="int",	        default=-1,     help="user expected size at z direction" )
	parser.add_option("--smearstep",              type="float",	        default=0.0,    help="Rotational smear step (default 0.0, no smear)" )
	parser.add_option("--interpolation_method",   type ="string",	    default="4nn",  help="4nn, or tril: nearest neighbor, or trillinear interpolation" )
	parser.add_option("--niter",                  type="int",	        default=10,     help="number of iterations for iterative reconstruction" )
	parser.add_option("--upweighted",             action="store_true",  default=False,  help="apply background noise")
	parser.add_option("--compensate",             action="store_true",  default=False,  help="compensate in reconstruction")
	parser.add_option("--chunk_id",               type="int",           default=-1,     help="reconstruct both odd and even groups of particles")
	parser.add_option("--target_window_size",               type="int",           default=-1,     help=" size of the targeted reconstruction ")
	(options,args) = parser.parse_args(arglist[1:])


	if options.MPI:
		from mpi import mpi_init
		sys.argv = mpi_init(len(sys.argv), sys.argv)

	if global_def.CACHE_DISABLE:
		from utilities import disable_bdb_cache
		disable_bdb_cache()

	if len(args) == 2:
		prj_stack = args[0]
		vol_stack = args[1]
		nimage = EMUtil.get_image_count( prj_stack )
		pid_list = range(0, nimage)
	elif len(args) == 5:
		prj_stack = args[0]
		vol_stack = args[1]
		begin = atoi( args[2] )
		end   = atoi( args[3] )
		step  = atoi( args[4] )
		pid_list = range(begin, end, step)
	else:
		ERROR("incomplete list of arguments","recon3d_n",1)
		exit()

	if(options.list and options.group > -1):
		ERROR("options group and list cannot be used together","recon3d_n",1)
		sys.exit()

	from applications import recons3d_n, recons3d_trl_MPI

	global_def.BATCH = True
	if options.interpolation_method == "4nn":
		recons3d_n(prj_stack, pid_list, vol_stack, options.CTF, options.snr, 1, options.npad,\
		 options.sym, options.list, options.group, options.verbose, options.MPI,options.xysize, options.zsize, options.smearstep, options.upweighted, options.compensate,options.chunk_id)
	elif options.interpolation_method == "tril":
		if options.MPI is False:
			ERROR(" trillinear interpolation reconstruction has MPI version only!")
			sys.exit()
		recons3d_trl_MPI(prj_stack, pid_list, vol_stack, options.CTF, options.snr, 1, options.npad,\
		 options.sym, options.verbose, options.niter, options.compensate, options.target_window_size)
		
	else:
		ERROR(" Wrong interpolation method. The current options are 4nn, and tril. 4nn is the defalut one. ")
		
	global_def.BATCH = False

	if options.MPI:
		from mpi import mpi_finalize
		mpi_finalize()
Exemplo n.º 27
0
def main():
    global debug
    progname = os.path.basename(sys.argv[0])
    usage = """prog [options] <path or db> ...
	
Various utilities related to BDB databases.

examples :
e2bdb.py -c   Is perhaps the most critical function, as it cleans up the database cache. See the Wiki for more.
e2bdb.py <path> -s    will list the contents of the database in a directory in bdb: notation
e2bdb.py <path> -l    Will give useful summary info about stacks in a directory
e2bdb.py <database> --dump    Gives a mechanism to dump all of the metadata in a database, even if the database contains no images
"""

    parser = EMArgumentParser(usage=usage, version=EMANVERSION)

    parser.add_argument(
        "--cleanup",
        "-c",
        action="store_true",
        default=False,
        help=
        "This option will clean up the database cache so files can safely be moved or accessed on another computer via NFS."
    )
    parser.add_argument(
        "--force",
        "-F",
        action="store_true",
        default=False,
        help=
        "This will force an action that would normally fail due to failed checks."
    )
    parser.add_argument(
        "--delete",
        action="store_true",
        default=False,
        help="This will delete (or at least empty) the named database(s)")
    parser.add_argument("--all",
                        "-a",
                        action="store_true",
                        help="List per-particle info",
                        default=False)
    parser.add_argument("--long",
                        "-l",
                        action="store_true",
                        help="Long listing",
                        default=False)
    parser.add_argument("--short",
                        "-s",
                        action="store_true",
                        help="Dense listing of names only",
                        default=False)
    parser.add_argument(
        "--filt",
        type=str,
        help="Only include dictionary names containing the specified string",
        default=None)
    parser.add_argument(
        "--filtexclude",
        type=str,
        help="Exclude dictionary names containing the specified string",
        default=None)
    parser.add_argument(
        "--match",
        type=str,
        help=
        "Only include dictionaries matching the provided Python regular expression",
        default=None)
    parser.add_argument(
        "--exclude",
        type=str,
        help="The name of a database containing a list of exclusion keys",
        default=None)
    parser.add_argument(
        "--dump",
        "-D",
        action="store_true",
        help=
        "List contents of an entire database, eg 'e2bdb.py -D refine_01#register",
        default=False)
    parser.add_argument(
        "--smalldump",
        action="store_true",
        help=
        "Lists contents of an entire database, but only list 2 items per dictionary to better see headers",
        default=False)
    parser.add_argument(
        "--extractplots",
        action="store_true",
        help=
        "If a database contains sets of plots, such as bdb:refine_xx#convergence.results, this will extract the plots as text files."
    )
    parser.add_argument(
        "--check",
        action="store_true",
        help=
        "Check for self-consistency and errors in the structure of specified databases",
        default=False)
    parser.add_argument("--nocache",
                        action="store_true",
                        help="Don't use the database cache for this operation",
                        default=False)
    parser.add_argument(
        "--merge",
        action="store_true",
        help=
        "This will merge the contents of BDB 2-N into BDB 1 (including BDB 1's contents)",
        default=False)

    parser.add_argument(
        "--makevstack",
        type=str,
        help=
        "Creates a 'virtual' BDB stack with its own metadata, but the binary data taken from the (filtered) list of stacks",
        default=None)
    parser.add_argument(
        "--appendvstack",
        type=str,
        help=
        "Appends to/creates a 'virtual' BDB stack with its own metadata, but the binary data taken from the (filtered) list of stacks",
        default=None)
    parser.add_argument(
        "--verbose",
        "-v",
        dest="verbose",
        action="store",
        metavar="n",
        type=int,
        default=0,
        help=
        "verbose level [0-9], higher number means higher level of verboseness")
    parser.add_argument(
        "--list",
        type=str,
        help=
        "Specify the name of a file with a list of images to use in creation of virtual stacks. Please see source for details.",
        default=None)
    parser.add_argument(
        "--exlist",
        type=str,
        help=
        "Specify the name of a file with a list of images to exclude in creation of virtual stacks. Please see source for details.",
        default=None)
    parser.add_argument(
        "--restore",
        type=str,
        help=
        "Write changes in the derived virtual stack back to the original stack",
        default=None)
    parser.add_argument(
        "--ppid",
        type=int,
        help="Set the PID of the parent process, used for cross platform PPID",
        default=-1)
    parser.add_argument(
        "--checkctf",
        action="store_true",
        help=
        "Verfies that all images in the file contain CTF information, and gives some basic statistics",
        default=False)

    parser.add_argument(
        "--step",
        type=str,
        default="0,1",
        help=
        "Specify <init>,<step>[,<max>]. Processes only a subset of the input data. For example, 0,2 would process only the even numbered particles"
    )
    (options, args) = parser.parse_args()

    if options.nocache: EMAN2db.BDB_CACHE_DISABLE = True

    if options.cleanup:
        db_cleanup(options.force)
        sys.exit(0)

    try:
        options.step = int(options.step.split(",")[0]), int(
            options.step.split(",")[1]), int(
                options.step.split(",")[2])  # convert strings to tuple
    except:
        try:
            options.step = int(options.step.split(",")[0]), int(
                options.step.split(",")[1])
        except:
            print("Invalid --step specification")
            sys.exit(1)

    if options.all: options.long = 1
    if len(args) == 0: args.append("bdb:.")

    logid = 0
    if options.makevstack:
        logid = E2init(sys.argv)
        vstack = db_open_dict(options.makevstack)
        vstackn = 0
    elif options.appendvstack:
        logid = E2init(sys.argv)
        vstack = db_open_dict(options.appendvstack)
        vstackn = len(vstack)
    else:
        vstack = None

    if options.merge:
        print("WARNING: Merge mode\nCombining contents of: ",
              ", ".join(args[1:]))
        print("into ", args[0])

        if input("Proceed (y/n) :").lower() != "y":
            print("Aborting")
            sys.exit(1)

        for i, path in enumerate(args):
            if path.lower()[:4] == "bdb:" and not "#" in path:
                path = "bdb:.#" + path[4:]
            if path.lower()[:4] != "bdb:": path = "bdb:" + path

            if i == 0:
                outdb = db_open_dict(path)
                continue

            indb = db_open_dict(path, True)
            for k in list(indb.keys()):
                outdb[k] = indb[k]

        print("Merging complete")
        sys.exit(0)

    for path in args:
        if path.lower()[:4] == "bdb:" and not "#" in path:
            uu = os.path.split(path)
            if (uu[0] == ''): path = "bdb:.#" + path[4:]
            else: path = uu[0] + "#" + uu[1]
        if path.lower()[:4] != "bdb:": path = "bdb:" + path
        if '#' in path:
            if len(args) > 1: print("\n", path, ":")
            path, dbs = path.rsplit("#", 1)
            path += "#"
            dbs = [dbs]
        else:
            if not '#' in path and path[-1] != '/': path += '#'
            if len(args) > 1: print("\n", path[:-1], ":")
            dbs = db_list_dicts(path)

        dbs.sort()
        if options.filt:
            dbs = [db for db in dbs if options.filt in db]

        if options.filtexclude:
            dbs = [db for db in dbs if options.filtexclude not in db]

        if options.match != None:
            dbs = [db for db in dbs if re.match(options.match, db)]

        if options.list:
            if options.makevstack == None and options.appendvstack == None:
                print(
                    "ERROR, this option is used for virtual stack creation, please add makevstack or appendvstack options, and restart"
                )
                sys.exit(1)
            vdata = open(options.list, 'r').readlines()
            n = len(vdata[0].split())
            slist = []
            for line in vdata:
                line = line.split()
                for i in range(n):
                    val = int(line[i])
                    slist.append(val)
            del n, val, vdata

        if options.exlist:
            if options.makevstack == None:
                print(
                    "ERROR, this option is used for virtual stack creation, please add makevstack or appendvstack options, and restart"
                )
                sys.exit(1)
            vdata = open(options.exlist, 'r').readlines()
            n = len(vdata[0].split())
            slist = []
            for line in vdata:
                line = line.split()
                for i in range(n):
                    val = int(line[i])
                    slist.append(val)
            n = EMUtil.get_image_count(args[0])
            good = set(range(n)) - set(slist)
            slist = [i for i in good]
            slist.sort()
            del n, val, vdata, good

        if options.makevstack != None or options.appendvstack != None:

            vspath = os.path.realpath(vstack.path) + "/"
            if options.verbose > 2: print("vspath: ", vspath)
            for db in dbs:
                dct, keys = db_open_dict(path + db, ro=True, with_keys=True)
                if dct == vstack: continue
                if len(options.step) == 2:
                    if keys == None:
                        vals = list(
                            range(options.step[0], len(dct), options.step[1]))
                    else:
                        vals = keys[options.step[0]::options.step[
                            1]]  # we apply --step even if we have a list of keys
                else:
                    if keys == None:
                        vals = list(
                            range(options.step[0], options.step[2],
                                  options.step[1]))
                    else:
                        vals = keys[options.step[0]:options.step[2]:options.step[
                            1]]  # we apply --step even if we have a list of keys

                if options.list != None or options.exlist != None: vals = slist
                for n in vals:
                    try:
                        d = dct.get(n, nodata=1).get_attr_dict()
                    except:
                        traceback.print_exc()
                        print("---\nerror reading ", db, n)
                        continue
                    # This block converts an absolute path to the actual data to a relative path
                    try:
                        dpath = os.path.realpath(dct.get_data_path(n))
                        if options.verbose > 2: print("dpath: ", dpath)
                        if os.name == 'nt':
                            vspath = vspath.replace("\\", '/')
                            dpath = dpath.replace('\\', '/')
                        rpath = makerelpath(vspath, dpath)
                        if options.verbose > 2: print("rpath: ", rpath)
                    except:
                        print("error with data_path ", db, n)
                        continue
                    d["data_path"] = rpath
                    d["data_n"] = n
                    d["data_source"] = path + db
                    if d["data_path"] == None:
                        print("error with data_path ", db, n)
                        continue
                    vstack[vstackn] = d
                    vstackn += 1
                    if vstackn % 100 == 0:
                        try:
                            print("\r  ", vstackn, "     ", end=' ')
                            sys.stdout.flush()
                        except:
                            pass
                print("\r  ", vstackn, "     ")
                dct.close()

        try:
            maxname = max([len(s) for s in dbs])
        except:
            print("Error reading ", path)

        if options.restore:
            nima = EMUtil.get_image_count(options.restore)
            IB = db_open_dict(options.restore)
            source_old = None
            if len(options.step) == 3: nima = min(options.step[2], nima)
            for i in range(options.step[0], nima, options.step[1]):
                source = IB.get_header(i)
                source_path = source["source_path"]
                ID = source["source_n"]
                if (source_old != source_path):
                    if (source_old != None): DB.close()
                    DB = db_open_dict(source_path, ro=True)
                    source_old = source_path
                target = DB.get_header(ID)
                try:
                    source["data_path"] = target["data_path"]
                    source["data_n"] = target["data_n"]
                    source["source_path"] = target["source_path"]
                    source["source_n"] = target["source_n"]
                except:
                    #  top level does not have data_path
                    del source['data_path']
                    del source['data_n']
                    source["source_path"] = target["source_path"]
                    source["source_n"] = target["source_n"]
                DB.set_header(ID, source)
            DB.close()

        if options.extractplots:
            for db in dbs:
                print("####  Extracting plots from ", db)
                dct = db_open_dict(path + db, ro=True)

                #### Dump
                keys = list(dct.keys())
                keys.sort()
                for k in keys:
                    v = dct[k]
                    try:
                        ns = [len(i) for i in v]
                        fsp = db + "-" + k + ".txt"
                        print("%s  (%d columns)" % (fsp, len(ns)))
                        out = open(fsp, "w")
                        for i in range(ns[0]):
                            for j in range(len(ns)):
                                out.write(str(v[j][i]))
                                if j < len(ns) - 1: out.write("\t")
                            out.write("\n")
                        out.close()
                    except:
                        continue
                dct.close()

        if options.smalldump:
            for db in dbs:
                print("##### ", db)
                dct = db_open_dict(path + db, ro=True)

                #### Dump
                keys = list(dct.keys())
                keys.sort()
                if len(options.step) == 3: keys = keys[:options.step[2]]
                for k in keys[options.step[0]::options.step[1]]:
                    v = dct[k]
                    print("%s : " % k, end=' ')
                    if isinstance(v, list) or isinstance(v, tuple):
                        for i in v:
                            print("\n\t%s" % str(i), end=' ')
                        print("")
                    elif isinstance(v, dict):
                        ks2 = list(v.keys())
                        ks2.sort()
                        kc = 0
                        for i in ks2:
                            if kc >= 2:
                                print("...")
                                break
                            print("\n\t%s : %s" % (i, v[i]), end=' ')
                            kc += 1
                        print("")
                    else:
                        print(str(v))
                dct.close()
        if options.checkctf:
            for db in dbs:
                print("##### CTF -> ", db)
                dct = db_open_dict(path + db, ro=True)
                keys = list(dct.keys())
                if len(options.step) == 3: keys = keys[:options.step[2]]
                defocus = set()
                for k in keys[options.step[0]::options.step[1]]:
                    v = dct.get_header(k)
                    try:
                        ctf = v["ctf"]
                    except:
                        if k != "maxrec": print("CTF missing on image %s" % k)
                        continue

                    defocus.add(ctf.defocus)

                defocus = list(defocus)
                print("Defocuses found: ", end=' ')
                for i in defocus:
                    print("%1.3f, " % i, end=' ')
                print("\n\nRange: %1.3f - %1.3f  (%d unique values)" %
                      (min(defocus), max(defocus), len(defocus)))

        if options.dump:
            for db in dbs:
                print("##### ", db)
                dct = db_open_dict(path + db, ro=True)

                #### Dump
                keys = list(dct.keys())
                if len(options.step) == 3: keys = keys[:options.step[2]]
                keys.sort()
                for k in keys[options.step[0]::options.step[1]]:
                    v = dct[k]
                    print("%s : " % k, end=' ')
                    if isinstance(v, list) or isinstance(v, tuple):
                        for i in v:
                            print("\n\t%s" % str(i), end=' ')
                        print("")
                    elif isinstance(v, dict):
                        ks2 = list(v.keys())
                        ks2.sort()
                        for i in ks2:
                            print("\n\t%s : %s" % (i, v[i]), end=' ')
                        print("")
                    else:
                        print(str(v))
                dct.close()

        # long listing, one db per line
        elif options.long:
            width = maxname + 3
            fmt = "%%-%ds %%-07d %%14s  %%s" % width
            fmt2 = "%%-%ds (not an image stack)" % width
            total = [0, 0]
            for db in dbs:
                dct = db_open_dict(path + db, True)

                ### Info on all particles
                if options.all:
                    mx = len(dct)
                    if len(options.step) == 3: mx = min(mx, options.step[2])
                    for i in range(options.step[0], mx, options.step[1]):
                        try:
                            im = dct[i]
                            if im == None: raise Exception
                        except:
                            continue
                        print(
                            "%d. %d x %d x %d\tA/pix=%1.2f\tM=%1.4f\tS=%1.4f\tSk=%1.4f"
                            % (i, im["nx"], im["ny"], im["nz"], im["apix_x"],
                               im["mean"], im["sigma"], im["skewness"]),
                            end=' ')
                        try:
                            print("\t%s" % str(im["model_id"]))
                        except:
                            pass
                        try:
                            print("\tdf=%1.3f\tB=%1.1f" %
                                  (im["ctf"].defocus, im["ctf"].bfactor))
                        except:
                            print(" ")

                first = EMData()
                try:
                    first.read_image(path + db, 0, True)
                    size = first.get_xsize() * first.get_ysize(
                    ) * first.get_zsize() * len(dct) * 4
                    total[0] += len(dct)
                    total[1] += size
                    print(fmt % (db, len(dct), "%dx%dx%d   apix: %1.2f" %
                                 (first.get_xsize(), first.get_ysize(),
                                  first.get_zsize(), first["apix_x"]),
                                 human_size(size)),
                          end=' ')
                except:
                    print(fmt2 % db)
                try:
                    print("\tdf: %1.3f\tB: %1.0f" %
                          (first["ctf"].defocus, first["ctf"].bfactor))
                except:
                    print("")
                dct.close()
            print(fmt % ("TOTAL", total[0], "", human_size(total[1])))
        elif options.check:
            from pickle import loads
            for db in dbs:
                dct = db_open_dict(path + db, ro=True)
                dct.realopen()
                keys = list(dct.bdb.keys())
                allkvp = {}
                for k in keys:
                    s1, s2 = k.split("\x80", 1)  # start of a pickled string.
                    s2 = loads("\x80" + s2)  # the pickled part
                    if len(
                            s1
                    ) > 0:  # If anything unpickled, then it is an axbxc prefix identifying the location of a binary
                        st = allkvp.setdefault(
                            s1, set())  # set of all positions seen so far
                        v = loads(dct.bdb.get(k))  # position in binary file
                        if v in st:
                            print(
                                "Error: value %d seen multiple times in %s (%s,%s)"
                                % (v, db, s1, s2))
                        st.add(v)
                print("%s : " % db, end=' ')
                for i in list(allkvp.keys()):
                    if options.verbose > 0:
                        print("%s %d/%d\t" %
                              (i, len(allkvp[i]), int(max(allkvp[i])) + 1),
                              end=' ')
                    if len(allkvp[i]) != int(max(allkvp[i]) + 1):
                        print(
                            "\nMismatch found in %s. Could be normal if file has been rewritten multiple times, but is unusual"
                            % db)
                if options.verbose > 0: print("")
                else: print(" done")
                dct.close()

        elif options.short:
            for db in dbs:
                print(path + db, end=' ')
            print(" ")

        elif not options.makevstack and not options.appendvstack:
            # Nicely formatted 'ls' style display
            cols = int(floor(old_div(80.0, (maxname + 3))))
            width = old_div(80, cols)
            rows = int(ceil(old_div(float(len(dbs)), cols)))

            fmt = "%%-%ds" % width
            for r in range(rows):
                for c in range(cols):
                    try:
                        print(fmt % dbs[r + c * rows], end=' ')
                    except:
                        pass
                print(" ")

        if options.delete:
            if not options.force:
                print("You are requesting to delete the following databases:")
                for db in dbs:
                    print(db, " ", end=' ')
                if input("\nAre you sure (y/n) ? ")[0].lower() != 'y':
                    print("Aborted")
                    sys.exit(1)

            for db in dbs:
                db_remove_dict(path + db)

    if logid: E2end(logid)
Exemplo n.º 28
0
def main():
    import sys
    import os
    import math
    import random
    import pyemtbx.options
    import time
    from random import random, seed, randint
    from optparse import OptionParser

    progname = os.path.basename(sys.argv[0])
    usage = progname + """ [options] <inputfile> <outputfile>

	Generic 2-D image processing programs.

	Functionality:

	1.  Phase flip a stack of images and write output to new file:
		sxprocess.py input_stack.hdf output_stack.hdf --phase_flip
	
	2.  Resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.
	    The window size will change accordingly.
		sxprocess input.hdf output.hdf  --changesize --ratio=0.5

	3.  Compute average power spectrum of a stack of 2D images with optional padding (option wn) with zeroes.
		sxprocess.py input_stack.hdf powerspectrum.hdf --pw [--wn=1024]

	4.  Generate a stack of projections bdb:data and micrographs with prefix mic (i.e., mic0.hdf, mic1.hdf etc) from structure input_structure.hdf, with CTF applied to both projections and micrographs:
		sxprocess.py input_structure.hdf data mic --generate_projections format="bdb":apix=5.2:CTF=True:boxsize=64

    5.  Retrieve original image numbers in the selected ISAC group (here group 12 from generation 3):
    	sxprocess.py  bdb:test3 class_averages_generation_3.hdf  list3_12.txt --isacgroup=12 --params=originalid

    6.  Retrieve original image numbers of images listed in ISAC output stack of averages:
    	sxprocess.py  select1.hdf  ohk.txt

    7.  Adjust rotationally averaged power spectrum of an image to that of a reference image or a reference 1D power spectrum stored in an ASCII file.
    	Optionally use a tangent low-pass filter.  Also works for a stack of images, in which case the output is also a stack.
    	sxprocess.py  vol.hdf ref.hdf  avol.hdf < 0.25 0.2> --adjpw
   	 	sxprocess.py  vol.hdf pw.txt   avol.hdf < 0.25 0.2> --adjpw

    8.  Generate a 1D rotationally averaged power spectrum of an image.
		sxprocess.py  vol.hdf --rotwp=rotpw.txt
    	# Output will contain three columns:
       (1) rotationally averaged power spectrum
       (2) logarithm of the rotationally averaged power spectrum
       (3) integer line number (from zero to approximately to half the image size)

    9.  Apply 3D transformation (rotation and/or shift) to a set of orientation parameters associated with projection data.
    	sxprocess.py  --transfromparams=phi,theta,psi,tx,ty,tz      input.txt  output.txt
    	The output file is then imported and 3D transformed volume computed:
    	sxheader.py  bdb:p  --params=xform.projection  --import=output.txt
    	mpirun -np 2 sxrecons3d_n.py  bdb:p tvol.hdf --MPI
    	The reconstructed volume is in the position of the volume computed using the input.txt parameters and then
    	transformed with rot_shift3D(vol, phi,theta,psi,tx,ty,tz)

   10.  Import ctf parameters from the output of sxcter into windowed particle headers.
	    There are three possible input files formats:  (1) all particles are in one stack, (2 aor 3) particles are in stacks, each stack corresponds to a single micrograph.
	    In each case the particles should contain a name of the micrograph of origin stores using attribute name 'ptcl_source_image'.
        Normally this is done by e2boxer.py during windowing.
	    Particles whose defocus or astigmatism error exceed set thresholds will be skipped, otherwise, virtual stacks with the original way preceded by G will be created.
		sxprocess.py  --input=bdb:data  --importctf=outdir/partres  --defocuserror=10.0  --astigmatismerror=5.0
		#  Output will be a vritual stack bdb:Gdata
		sxprocess.py  --input="bdb:directory/stacks*"  --importctf=outdir/partres  --defocuserror=10.0  --astigmatismerror=5.0
		To concatenate output files:
		cd directory
		e2bdb.py . --makevstack=bdb:allparticles  --filt=G
		IMPORTANT:  Please do not move (or remove!) any input/intermediate EMAN2DB files as the information is linked between them.

   11. Scale 3D shifts.  The shifts in the input five columns text file with 3D orientation parameters will be DIVIDED by the scale factor
		sxprocess.py  orientationparams.txt  scaledparams.txt  scale=0.5
   
   12. Generate adaptive mask from a given 3-D volume. 


"""

    parser = OptionParser(usage, version=SPARXVERSION)
    parser.add_option(
        "--order",
        action="store_true",
        help=
        "Two arguments are required: name of input stack and desired name of output stack. The output stack is the input stack sorted by similarity in terms of cross-correlation coefficent.",
        default=False)
    parser.add_option("--order_lookup",
                      action="store_true",
                      help="Test/Debug.",
                      default=False)
    parser.add_option("--order_metropolis",
                      action="store_true",
                      help="Test/Debug.",
                      default=False)
    parser.add_option("--order_pca",
                      action="store_true",
                      help="Test/Debug.",
                      default=False)
    parser.add_option(
        "--initial",
        type="int",
        default=-1,
        help=
        "Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)"
    )
    parser.add_option(
        "--circular",
        action="store_true",
        help=
        "Select circular ordering (fisr image has to be similar to the last",
        default=False)
    parser.add_option(
        "--radius",
        type="int",
        default=-1,
        help="Radius of a circular mask for similarity based ordering")
    parser.add_option(
        "--changesize",
        action="store_true",
        help=
        "resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.",
        default=False)
    parser.add_option(
        "--ratio",
        type="float",
        default=1.0,
        help=
        "The ratio of new to old image size (if <1 the pixel size will increase and image size decrease, if>1, the other way round"
    )
    parser.add_option(
        "--pw",
        action="store_true",
        help=
        "compute average power spectrum of a stack of 2-D images with optional padding (option wn) with zeroes",
        default=False)
    parser.add_option(
        "--wn",
        type="int",
        default=-1,
        help=
        "Size of window to use (should be larger/equal than particle box size, default padding to max(nx,ny))"
    )
    parser.add_option("--phase_flip",
                      action="store_true",
                      help="Phase flip the input stack",
                      default=False)
    parser.add_option(
        "--makedb",
        metavar="param1=value1:param2=value2",
        type="string",
        action="append",
        help=
        "One argument is required: name of key with which the database will be created. Fill in database with parameters specified as follows: --makedb param1=value1:param2=value2, e.g. 'gauss_width'=1.0:'pixel_input'=5.2:'pixel_output'=5.2:'thr_low'=1.0"
    )
    parser.add_option(
        "--generate_projections",
        metavar="param1=value1:param2=value2",
        type="string",
        action="append",
        help=
        "Three arguments are required: name of input structure from which to generate projections, desired name of output projection stack, and desired prefix for micrographs (e.g. if prefix is 'mic', then micrographs mic0.hdf, mic1.hdf etc will be generated). Optional arguments specifying format, apix, box size and whether to add CTF effects can be entered as follows after --generate_projections: format='bdb':apix=5.2:CTF=True:boxsize=100, or format='hdf', etc., where format is bdb or hdf, apix (pixel size) is a float, CTF is True or False, and boxsize denotes the dimension of the box (assumed to be a square). If an optional parameter is not specified, it will default as follows: format='bdb', apix=2.5, CTF=False, boxsize=64."
    )
    parser.add_option(
        "--isacgroup",
        type="int",
        help=
        "Retrieve original image numbers in the selected ISAC group. See ISAC documentation for details.",
        default=-1)
    parser.add_option(
        "--isacselect",
        action="store_true",
        help=
        "Retrieve original image numbers of images listed in ISAC output stack of averages. See ISAC documentation for details.",
        default=False)
    parser.add_option(
        "--params",
        type="string",
        default=None,
        help="Name of header of parameter, which one depends on specific option"
    )
    parser.add_option(
        "--adjpw",
        action="store_true",
        help="Adjust rotationally averaged power spectrum of an image",
        default=False)
    parser.add_option(
        "--rotpw",
        type="string",
        default=None,
        help=
        "Name of the text file to contain rotationally averaged power spectrum of the input image."
    )
    parser.add_option(
        "--transformparams",
        type="string",
        default=None,
        help=
        "Transform 3D projection orientation parameters using six 3D parameters (phi, theta,psi,sx,sy,sz).  Input: --transformparams=45.,66.,12.,-2,3,-5.5 desired six transformation of the reconstructed structure. Output: file with modified orientation parameters."
    )

    # import ctf estimates done using cter
    parser.add_option("--input",
                      type="string",
                      default=None,
                      help="Input particles.")
    parser.add_option(
        "--importctf",
        type="string",
        default=None,
        help="Name of the file containing CTF parameters produced by sxcter.")
    parser.add_option(
        "--defocuserror",
        type="float",
        default=1000000.0,
        help=
        "Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent.  The error is computed as (std dev defocus)/defocus*100%"
    )
    parser.add_option(
        "--astigmatismerror",
        type="float",
        default=360.0,
        help=
        "Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees."
    )

    # import ctf estimates done using cter
    parser.add_option(
        "--scale",
        type="float",
        default=-1.0,
        help=
        "Divide shifts in the input 3D orientation parameters text file by the scale factor."
    )

    # generate adaptive mask from an given 3-Db volue
    parser.add_option("--adaptive_mask",
                      action="store_true",
                      help="create adavptive 3-D mask from a given volume",
                      default=False)
    parser.add_option(
        "--nsigma",
        type="float",
        default=1.,
        help=
        "number of times of sigma of the input volume to obtain the the large density cluster"
    )
    parser.add_option(
        "--ndilation",
        type="int",
        default=3,
        help=
        "number of times of dilation applied to the largest cluster of density"
    )
    parser.add_option(
        "--kernel_size",
        type="int",
        default=11,
        help="convolution kernel for smoothing the edge of the mask")
    parser.add_option(
        "--gauss_standard_dev",
        type="int",
        default=9,
        help="stanadard deviation value to generate Gaussian edge")

    (options, args) = parser.parse_args()

    global_def.BATCH = True

    if options.phase_flip:
        nargs = len(args)
        if nargs != 2:
            print "must provide name of input and output file!"
            return
        from EMAN2 import Processor
        instack = args[0]
        outstack = args[1]
        nima = EMUtil.get_image_count(instack)
        from filter import filt_ctf
        for i in xrange(nima):
            img = EMData()
            img.read_image(instack, i)
            try:
                ctf = img.get_attr('ctf')
            except:
                print "no ctf information in input stack! Exiting..."
                return

            dopad = True
            sign = 1
            binary = 1  # phase flip

            assert img.get_ysize() > 1
            dict = ctf.to_dict()
            dz = dict["defocus"]
            cs = dict["cs"]
            voltage = dict["voltage"]
            pixel_size = dict["apix"]
            b_factor = dict["bfactor"]
            ampcont = dict["ampcont"]
            dza = dict["dfdiff"]
            azz = dict["dfang"]

            if dopad and not img.is_complex(): ip = 1
            else: ip = 0

            params = {
                "filter_type": Processor.fourier_filter_types.CTF_,
                "defocus": dz,
                "Cs": cs,
                "voltage": voltage,
                "Pixel_size": pixel_size,
                "B_factor": b_factor,
                "amp_contrast": ampcont,
                "dopad": ip,
                "binary": binary,
                "sign": sign,
                "dza": dza,
                "azz": azz
            }

            tmp = Processor.EMFourierFilter(img, params)
            tmp.set_attr_dict({"ctf": ctf})

            tmp.write_image(outstack, i)

    elif options.changesize:
        nargs = len(args)
        if nargs != 2:
            ERROR("must provide name of input and output file!", "change size",
                  1)
            return
        from utilities import get_im
        instack = args[0]
        outstack = args[1]
        sub_rate = float(options.ratio)

        nima = EMUtil.get_image_count(instack)
        from fundamentals import resample
        for i in xrange(nima):
            resample(get_im(instack, i), sub_rate).write_image(outstack, i)

    elif options.isacgroup > -1:
        nargs = len(args)
        if nargs != 3:
            ERROR("Three files needed on input!", "isacgroup", 1)
            return
        from utilities import get_im
        instack = args[0]
        m = get_im(args[1], int(options.isacgroup)).get_attr("members")
        l = []
        for k in m:
            l.append(int(get_im(args[0], k).get_attr(options.params)))
        from utilities import write_text_file
        write_text_file(l, args[2])

    elif options.isacselect:
        nargs = len(args)
        if nargs != 2:
            ERROR("Two files needed on input!", "isacgroup", 1)
            return
        from utilities import get_im
        nima = EMUtil.get_image_count(args[0])
        m = []
        for k in xrange(nima):
            m += get_im(args[0], k).get_attr("members")
        m.sort()
        from utilities import write_text_file
        write_text_file(m, args[1])

    elif options.pw:
        nargs = len(args)
        if nargs < 2:
            ERROR("must provide name of input and output file!", "pw", 1)
            return
        from utilities import get_im
        d = get_im(args[0])
        nx = d.get_xsize()
        ny = d.get_ysize()
        if nargs == 3: mask = get_im(args[2])
        wn = int(options.wn)
        if wn == -1:
            wn = max(nx, ny)
        else:
            if ((wn < nx) or (wn < ny)):
                ERROR("window size cannot be smaller than the image size",
                      "pw", 1)
        n = EMUtil.get_image_count(args[0])
        from utilities import model_blank, model_circle, pad
        from EMAN2 import periodogram
        p = model_blank(wn, wn)

        for i in xrange(n):
            d = get_im(args[0], i)
            if nargs == 3:
                d *= mask
            st = Util.infomask(d, None, True)
            d -= st[0]
            p += periodogram(pad(d, wn, wn, 1, 0.))
        p /= n
        p.write_image(args[1])

    elif options.adjpw:

        if len(args) < 3:
            ERROR(
                "filt_by_rops input target output fl aa (the last two are optional parameters of a low-pass filter)",
                "adjpw", 1)
            return
        img_stack = args[0]
        from math import sqrt
        from fundamentals import rops_table, fft
        from utilities import read_text_file, get_im
        from filter import filt_tanl, filt_table
        if (args[1][-3:] == 'txt'):
            rops_dst = read_text_file(args[1])
        else:
            rops_dst = rops_table(get_im(args[1]))

        out_stack = args[2]
        if (len(args) > 4):
            fl = float(args[3])
            aa = float(args[4])
        else:
            fl = -1.0
            aa = 0.0

        nimage = EMUtil.get_image_count(img_stack)

        for i in xrange(nimage):
            img = fft(get_im(img_stack, i))
            rops_src = rops_table(img)

            assert len(rops_dst) == len(rops_src)

            table = [0.0] * len(rops_dst)
            for j in xrange(len(rops_dst)):
                table[j] = sqrt(rops_dst[j] / rops_src[j])

            if (fl > 0.0):
                img = filt_tanl(img, fl, aa)
            img = fft(filt_table(img, table))
            img.write_image(out_stack, i)

    elif options.rotpw != None:

        if len(args) != 1:
            ERROR("Only one input permitted", "rotpw", 1)
            return
        from utilities import write_text_file, get_im
        from fundamentals import rops_table
        from math import log10
        t = rops_table(get_im(args[0]))
        x = range(len(t))
        r = [0.0] * len(x)
        for i in x:
            r[i] = log10(t[i])
        write_text_file([t, r, x], options.rotpw)

    elif options.transformparams != None:
        if len(args) != 2:
            ERROR(
                "Please provide names of input and output files with orientation parameters",
                "transformparams", 1)
            return
        from utilities import read_text_row, write_text_row
        transf = [0.0] * 6
        spl = options.transformparams.split(',')
        for i in xrange(len(spl)):
            transf[i] = float(spl[i])

        write_text_row(rotate_shift_params(read_text_row(args[0]), transf),
                       args[1])

    elif options.makedb != None:
        nargs = len(args)
        if nargs != 1:
            print "must provide exactly one argument denoting database key under which the input params will be stored"
            return
        dbkey = args[0]
        print "database key under which params will be stored: ", dbkey
        gbdb = js_open_dict("e2boxercache/gauss_box_DB.json")

        parmstr = 'dummy:' + options.makedb[0]
        (processorname, param_dict) = parsemodopt(parmstr)
        dbdict = {}
        for pkey in param_dict:
            if (pkey == 'invert_contrast') or (pkey == 'use_variance'):
                if param_dict[pkey] == 'True':
                    dbdict[pkey] = True
                else:
                    dbdict[pkey] = False
            else:
                dbdict[pkey] = param_dict[pkey]
        gbdb[dbkey] = dbdict

    elif options.generate_projections:
        nargs = len(args)
        if nargs != 3:
            ERROR("Must provide name of input structure(s) from which to generate projections, name of output projection stack, and prefix for output micrographs."\
            "sxprocess - generate projections",1)
            return
        inpstr = args[0]
        outstk = args[1]
        micpref = args[2]

        parmstr = 'dummy:' + options.generate_projections[0]
        (processorname, param_dict) = parsemodopt(parmstr)

        parm_CTF = False
        parm_format = 'bdb'
        parm_apix = 2.5

        if 'CTF' in param_dict:
            if param_dict['CTF'] == 'True':
                parm_CTF = True

        if 'format' in param_dict:
            parm_format = param_dict['format']

        if 'apix' in param_dict:
            parm_apix = float(param_dict['apix'])

        boxsize = 64
        if 'boxsize' in param_dict:
            boxsize = int(param_dict['boxsize'])

        print "pixel size: ", parm_apix, " format: ", parm_format, " add CTF: ", parm_CTF, " box size: ", boxsize

        scale_mult = 2500
        sigma_add = 1.5
        sigma_proj = 30.0
        sigma2_proj = 17.5
        sigma_gauss = 0.3
        sigma_mic = 30.0
        sigma2_mic = 17.5
        sigma_gauss_mic = 0.3

        if 'scale_mult' in param_dict:
            scale_mult = float(param_dict['scale_mult'])
        if 'sigma_add' in param_dict:
            sigma_add = float(param_dict['sigma_add'])
        if 'sigma_proj' in param_dict:
            sigma_proj = float(param_dict['sigma_proj'])
        if 'sigma2_proj' in param_dict:
            sigma2_proj = float(param_dict['sigma2_proj'])
        if 'sigma_gauss' in param_dict:
            sigma_gauss = float(param_dict['sigma_gauss'])
        if 'sigma_mic' in param_dict:
            sigma_mic = float(param_dict['sigma_mic'])
        if 'sigma2_mic' in param_dict:
            sigma2_mic = float(param_dict['sigma2_mic'])
        if 'sigma_gauss_mic' in param_dict:
            sigma_gauss_mic = float(param_dict['sigma_gauss_mic'])

        from filter import filt_gaussl, filt_ctf
        from utilities import drop_spider_doc, even_angles, model_gauss, delete_bdb, model_blank, pad, model_gauss_noise, set_params2D, set_params_proj
        from projection import prep_vol, prgs
        seed(14567)
        delta = 29
        angles = even_angles(delta, 0.0, 89.9, 0.0, 359.9, "S")
        nangle = len(angles)

        modelvol = []
        nvlms = EMUtil.get_image_count(inpstr)
        from utilities import get_im
        for k in xrange(nvlms):
            modelvol.append(get_im(inpstr, k))

        nx = modelvol[0].get_xsize()

        if nx != boxsize:
            ERROR("Requested box dimension does not match dimension of the input model.", \
            "sxprocess - generate projections",1)
        nvol = 10
        volfts = [[] for k in xrange(nvlms)]
        for k in xrange(nvlms):
            for i in xrange(nvol):
                sigma = sigma_add + random()  # 1.5-2.5
                addon = model_gauss(sigma, boxsize, boxsize, boxsize, sigma,
                                    sigma, 38, 38, 40)
                scale = scale_mult * (0.5 + random())
                vf, kb = prep_vol(modelvol[k] + scale * addon)
                volfts[k].append(vf)
        del vf, modelvol

        if parm_format == "bdb":
            stack_data = "bdb:" + outstk
            delete_bdb(stack_data)
        else:
            stack_data = outstk + ".hdf"
        Cs = 2.0
        pixel = parm_apix
        voltage = 120.0
        ampcont = 10.0
        ibd = 4096 / 2 - boxsize
        iprj = 0

        width = 240
        xstart = 8 + boxsize / 2
        ystart = 8 + boxsize / 2
        rowlen = 17
        from random import randint
        params = []
        for idef in xrange(3, 8):

            irow = 0
            icol = 0

            mic = model_blank(4096, 4096)
            defocus = idef * 0.5  #0.2
            if parm_CTF:
                astampl = defocus * 0.15
                astangl = 50.0
                ctf = generate_ctf([
                    defocus, Cs, voltage, pixel, ampcont, 0.0, astampl, astangl
                ])

            for i in xrange(nangle):
                for k in xrange(12):
                    dphi = 8.0 * (random() - 0.5)
                    dtht = 8.0 * (random() - 0.5)
                    psi = 360.0 * random()

                    phi = angles[i][0] + dphi
                    tht = angles[i][1] + dtht

                    s2x = 4.0 * (random() - 0.5)
                    s2y = 4.0 * (random() - 0.5)

                    params.append([phi, tht, psi, s2x, s2y])

                    ivol = iprj % nvol
                    #imgsrc = randint(0,nvlms-1)
                    imgsrc = iprj % nvlms
                    proj = prgs(volfts[imgsrc][ivol], kb,
                                [phi, tht, psi, -s2x, -s2y])

                    x = xstart + irow * width
                    y = ystart + icol * width

                    mic += pad(proj, 4096, 4096, 1, 0.0, x - 2048, y - 2048, 0)

                    proj = proj + model_gauss_noise(sigma_proj, nx, nx)
                    if parm_CTF:
                        proj = filt_ctf(proj, ctf)
                        proj.set_attr_dict({"ctf": ctf, "ctf_applied": 0})

                    proj = proj + filt_gaussl(
                        model_gauss_noise(sigma2_proj, nx, nx), sigma_gauss)
                    proj.set_attr("origimgsrc", imgsrc)
                    proj.set_attr("test_id", iprj)
                    # flags describing the status of the image (1 = true, 0 = false)
                    set_params2D(proj, [0.0, 0.0, 0.0, 0, 1.0])
                    set_params_proj(proj, [phi, tht, psi, s2x, s2y])

                    proj.write_image(stack_data, iprj)

                    icol += 1
                    if icol == rowlen:
                        icol = 0
                        irow += 1

                    iprj += 1

            mic += model_gauss_noise(sigma_mic, 4096, 4096)
            if parm_CTF:
                #apply CTF
                mic = filt_ctf(mic, ctf)
            mic += filt_gaussl(model_gauss_noise(sigma2_mic, 4096, 4096),
                               sigma_gauss_mic)

            mic.write_image(micpref + "%1d.hdf" % (idef - 3), 0)

        drop_spider_doc("params.txt", params)

    elif options.importctf != None:
        print ' IMPORTCTF  '
        from utilities import read_text_row, write_text_row
        from random import randint
        import subprocess
        grpfile = 'groupid%04d' % randint(1000, 9999)
        ctfpfile = 'ctfpfile%04d' % randint(1000, 9999)
        cterr = [options.defocuserror / 100.0, options.astigmatismerror]
        ctfs = read_text_row(options.importctf)
        for kk in xrange(len(ctfs)):
            root, name = os.path.split(ctfs[kk][-1])
            ctfs[kk][-1] = name[:-4]
        if (options.input[:4] != 'bdb:'):
            ERROR('Sorry, only bdb files implemented', 'importctf', 1)
        d = options.input[4:]
        #try:     str = d.index('*')
        #except:  str = -1
        from string import split
        import glob
        uu = os.path.split(d)
        uu = os.path.join(uu[0], 'EMAN2DB', uu[1] + '.bdb')
        flist = glob.glob(uu)
        for i in xrange(len(flist)):
            root, name = os.path.split(flist[i])
            root = root[:-7]
            name = name[:-4]
            fil = 'bdb:' + os.path.join(root, name)
            sourcemic = EMUtil.get_all_attributes(fil, 'ptcl_source_image')
            nn = len(sourcemic)
            gctfp = []
            groupid = []
            for kk in xrange(nn):
                junk, name2 = os.path.split(sourcemic[kk])
                name2 = name2[:-4]
                ctfp = [-1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
                for ll in xrange(len(ctfs)):
                    if (name2 == ctfs[ll][-1]):
                        #  found correct
                        if (ctfs[ll][8] / ctfs[ll][0] <= cterr[0]):
                            #  acceptable defocus error
                            ctfp = ctfs[ll][:8]
                            if (ctfs[ll][10] > cterr[1]):
                                # error of astigmatism exceed the threshold, set astigmatism to zero.
                                ctfp[6] = 0.0
                                ctfp[7] = 0.0
                            gctfp.append(ctfp)
                            groupid.append(kk)
                        break
            if (len(groupid) > 0):
                write_text_row(groupid, grpfile)
                write_text_row(gctfp, ctfpfile)
                cmd = "{} {} {} {}".format(
                    'e2bdb.py', fil, '--makevstack=bdb:' + root + 'G' + name,
                    '--list=' + grpfile)
                #print cmd
                subprocess.call(cmd, shell=True)
                cmd = "{} {} {} {}".format('sxheader.py',
                                           'bdb:' + root + 'G' + name,
                                           '--params=ctf',
                                           '--import=' + ctfpfile)
                #print cmd
                subprocess.call(cmd, shell=True)
            else:
                print ' >>>  Group ', name, '  skipped.'

        cmd = "{} {} {}".format("rm -f", grpfile, ctfpfile)
        subprocess.call(cmd, shell=True)

    elif options.scale > 0.0:
        from utilities import read_text_row, write_text_row
        scale = options.scale
        nargs = len(args)
        if nargs != 2:
            print "Please provide names of input and output file!"
            return
        p = read_text_row(args[0])
        for i in xrange(len(p)):
            p[i][3] /= scale
            p[i][4] /= scale
        write_text_row(p, args[1])

    elif options.adaptive_mask:
        from utilities import get_im
        from morphology import adaptive_mask
        nsigma = options.nsigma
        ndilation = options.ndilation
        kernel_size = options.kernel_size
        gauss_standard_dev = options.gauss_standard_dev
        nargs = len(args)
        if nargs > 2:
            print "Too many inputs are given, try again!"
            return
        else:
            inputvol = get_im(args[0])
            input_path, input_file_name = os.path.split(args[0])
            input_file_name_root, ext = os.path.splitext(input_file_name)
            if nargs == 2: mask_file_name = args[1]
            else:
                mask_file_name = "adaptive_mask_for" + input_file_name_root + ".hdf"  # Only hdf file is output.
            mask3d = adaptive_mask(inputvol, nsigma, ndilation, kernel_size,
                                   gauss_standard_dev)
            mask3d.write_image(mask_file_name)

    else:
        ERROR("Please provide option name", "sxprocess.py", 1)
Exemplo n.º 29
0
def main():
    arglist = []
    for arg in sys.argv:
        arglist.append(arg)

    progname = os.path.basename(arglist[0])
    usage = progname + " prj_stack volume [begin end step] --CTF --npad=ntimes_padding --list=file --group=ID --snr=SNR --sym=symmetry --verbose=(0|1) --xysize --MPI"
    parser = OptionParser(usage, version=SPARXVERSION)

    parser.add_option("--CTF",
                      action="store_true",
                      default=False,
                      help="apply CTF correction")
    parser.add_option("--snr",
                      type="float",
                      default=1.0,
                      help="Signal-to-Noise Ratio")
    parser.add_option("--sym", type="string", default="c1", help="symmetry")
    parser.add_option(
        "--list",
        type="string",
        help="file with list of images to be used in the first column")
    parser.add_option(
        "--group",
        type="int",
        default=-1,
        help=
        "perform reconstruction using images for a given group number (group is attribute in the header)"
    )
    parser.add_option("--MPI",
                      action="store_true",
                      default=False,
                      help="use MPI version ")
    parser.add_option("--npad",
                      type="int",
                      default=2,
                      help="number of times padding (default 2)")
    parser.add_option("--verbose",
                      type="int",
                      default=0,
                      help="verbose level: 0 no verbose, 1 verbose")
    parser.add_option("--xysize",
                      type="int",
                      default=-1,
                      help="user expected size at xy direction")
    parser.add_option("--zsize",
                      type="int",
                      default=-1,
                      help="user expected size at z direction")
    parser.add_option("--smearstep",
                      type="float",
                      default=0.0,
                      help="Rotational smear step (default 0.0, no smear)")
    parser.add_option(
        "--interpolation_method",
        type="string",
        default="4nn",
        help="4nn, or tril: nearest neighbor, or trillinear interpolation")
    parser.add_option("--niter",
                      type="int",
                      default=10,
                      help="number of iterations for iterative reconstruction")
    parser.add_option("--upweighted",
                      action="store_true",
                      default=False,
                      help="apply background noise")
    parser.add_option("--compensate",
                      action="store_true",
                      default=False,
                      help="compensate in reconstruction")
    parser.add_option("--chunk_id",
                      type="int",
                      default=-1,
                      help="reconstruct both odd and even groups of particles")
    parser.add_option("--target_window_size",
                      type="int",
                      default=-1,
                      help=" size of the targeted reconstruction ")
    (options, args) = parser.parse_args(arglist[1:])

    if options.MPI:
        from mpi import mpi_init
        sys.argv = mpi_init(len(sys.argv), sys.argv)

    if global_def.CACHE_DISABLE:
        from utilities import disable_bdb_cache
        disable_bdb_cache()

    if len(args) == 2:
        prj_stack = args[0]
        vol_stack = args[1]
        nimage = EMUtil.get_image_count(prj_stack)
        pid_list = list(range(0, nimage))
    elif len(args) == 5:
        prj_stack = args[0]
        vol_stack = args[1]
        begin = atoi(args[2])
        end = atoi(args[3])
        step = atoi(args[4])
        pid_list = list(range(begin, end, step))
    else:
        ERROR("incomplete list of arguments", "recon3d_n", 1)
        exit()

    if (options.list and options.group > -1):
        ERROR("options group and list cannot be used together", "recon3d_n", 1)
        sys.exit()

    from applications import recons3d_n, recons3d_trl_MPI

    global_def.BATCH = True
    if options.interpolation_method == "4nn":
        recons3d_n(prj_stack, pid_list, vol_stack, options.CTF, options.snr, 1, options.npad,\
         options.sym, options.list, options.group, options.verbose, options.MPI,options.xysize, options.zsize, options.smearstep, options.upweighted, options.compensate,options.chunk_id)
    elif options.interpolation_method == "tril":
        if options.MPI is False:
            ERROR(
                " trillinear interpolation reconstruction has MPI version only!"
            )
            sys.exit()
        recons3d_trl_MPI(prj_stack, pid_list, vol_stack, options.CTF, options.snr, 1, options.npad,\
         options.sym, options.verbose, options.niter, options.compensate, options.target_window_size)

    else:
        ERROR(
            " Wrong interpolation method. The current options are 4nn, and tril. 4nn is the defalut one. "
        )

    global_def.BATCH = False

    if options.MPI:
        from mpi import mpi_finalize
        mpi_finalize()
Exemplo n.º 30
0
	def __init__(self,options):
		self.options = options
		self.using_cuda = EMUtil.cuda_available() and options.cuda
		self.nsoln = options.nsoln
Exemplo n.º 31
0
def main():
    import os
    import sys
    from optparse import OptionParser
    from global_def import SPARXVERSION
    import global_def
    arglist = []
    for arg in sys.argv:
        arglist.append(arg)
    progname = os.path.basename(arglist[0])
    usage2 = progname + """ inputfile outputfile [options]
        Functionalities:

        1. Helicise input volume and save the result to output volume:
            sxhelicon_utils.py input_vol.hdf output_vol.hdf --helicise --dp=27.6 --dphi=166.5 --fract=0.65 --rmax=70 --rmin=1 --apix=1.84 --sym=D1        

        2. Helicise pdb file and save the result to a new pdb file:
            sxhelicon_utils.py input.pdb output.pdb --helicisepdb --dp=27.6 --dphi=166.5 --nrepeats --apix=1.84         

        3. Generate two lists of image indices used to split segment stack into halves for helical fsc calculation.			
            sxhelicon_utils.py bdb:big_stack --hfsc='flst' --filament_attr=filament

        4. Map of filament distribution in the stack
            sxhelicon_utils.py bdb:big_stack --filinfo=info.txt
            The output file will contain four columns:
                     1                    2                     3                         4
            first image number     last image number      number of images         in the filament name

        5. Predict segments' orientation parameters based on distances between segments and known helical symmetry
            sxhelicon_utils.py bdb:big_stack --predict_helical=helical_params.txt --dp=27.6 --dphi=166.5 --apix=1.84
            
        6. Generate disks from filament based reconstructions:		
            sxheader.py stk.hdf --params=xform.projection --import=params.txt
            mpirun -np 2 sxhelicon_utils.py stk.hdf --gendisk='bdb:disk' --ref_nx=100 --ref_ny=100 --ref_nz=200 --apix=1.84 --dp=27.6 --dphi=166.715 --fract=0.67 --rmin=0 --rmax=64 --function="[.,nofunc,helical3c]" --sym="c1" --MPI

        7. Stack disks based on helical symmetry parameters
            sxhelicon_utils.py disk_to_stack.hdf --stackdisk=stacked_disks.hdf --dphi=166.5 --dp=27.6 --ref_nx=160 --ref_ny=160 --ref_nz=225 --apix=1.84
		
        8. Helical symmetry search:
            mpirun -np 3 sxhelicon_utils.py volf0010.hdf outsymsearch --symsearch --dp=27.6 --dphi=166.715 --apix=1.84 --fract=0.65 --rmin=0 --rmax=92.0 --datasym=datasym.txt  --dp_step=0.92 --ndp=3 --dphi_step=1.0 --ndphi=10 --MPI
"""
    parser = OptionParser(usage2, version=SPARXVERSION)
    #parser.add_option("--ir",                 type="float", 	     default= -1,                 help="inner radius for rotational correlation > 0 (set to 1) (Angstroms)")
    parser.add_option(
        "--ou",
        type="float",
        default=-1,
        help=
        "outer radius for rotational 2D correlation < int(nx/2)-1 (set to the radius of the particle) (Angstroms)"
    )
    parser.add_option(
        "--rs",
        type="int",
        default=1,
        help="step between rings in rotational correlation >0  (set to 1)")
    parser.add_option(
        "--xr",
        type="string",
        default="4 2 1 1 1",
        help=
        "range for translation search in x direction, search is +/-xr (Angstroms) "
    )
    parser.add_option(
        "--txs",
        type="string",
        default="1 1 1 0.5 0.25",
        help=
        "step size of the translation search in x directions, search is -xr, -xr+ts, 0, xr-ts, xr (Angstroms)"
    )
    parser.add_option("--delta",
                      type="string",
                      default="10 6 4 3 2",
                      help="angular step of reference projections")
    parser.add_option("--an",
                      type="string",
                      default="-1",
                      help="angular neighborhood for local searches")
    parser.add_option(
        "--maxit",
        type="int",
        default=30,
        help=
        "maximum number of iterations performed for each angular step (set to 30) "
    )
    parser.add_option("--CTF",
                      action="store_true",
                      default=False,
                      help="CTF correction")
    parser.add_option("--snr",
                      type="float",
                      default=1.0,
                      help="Signal-to-Noise Ratio of the data")
    parser.add_option("--MPI",
                      action="store_true",
                      default=False,
                      help="use MPI version")
    #parser.add_option("--fourvar",           action="store_true",   default=False,               help="compute Fourier variance")
    parser.add_option("--apix",
                      type="float",
                      default=-1.0,
                      help="pixel size in Angstroms")
    parser.add_option("--dp",
                      type="float",
                      default=-1.0,
                      help="delta z - translation in Angstroms")
    parser.add_option("--dphi",
                      type="float",
                      default=-1.0,
                      help="delta phi - rotation in degrees")

    parser.add_option("--rmin",
                      type="float",
                      default=0.0,
                      help="minimal radius for hsearch (Angstroms)")
    parser.add_option("--rmax",
                      type="float",
                      default=80.0,
                      help="maximal radius for hsearch (Angstroms)")
    parser.add_option("--fract",
                      type="float",
                      default=0.7,
                      help="fraction of the volume used for helical search")
    parser.add_option("--sym",
                      type="string",
                      default="c1",
                      help="symmetry of the structure")
    parser.add_option("--function",
                      type="string",
                      default="helical",
                      help="name of the reference preparation function")
    parser.add_option("--npad",
                      type="int",
                      default=2,
                      help="padding size for 3D reconstruction")
    parser.add_option("--debug",
                      action="store_true",
                      default=False,
                      help="debug")

    parser.add_option("--volalixshift",
                      action="store_true",
                      default=False,
                      help="Use volalixshift refinement")
    parser.add_option(
        "--searchxshift",
        type="float",
        default=0.0,
        help=
        "search range for x-shift determination: +/- searchxshift (Angstroms)")
    parser.add_option(
        "--nearby",
        type="float",
        default=6.0,
        help=
        "neighborhood within which to search for peaks in 1D ccf for x-shift search (Angstroms)"
    )

    # filinfo
    parser.add_option(
        "--filinfo",
        type="string",
        default="",
        help=
        "Store in an output text file infomration about distribution of filaments in the stack."
    )

    # diskali
    parser.add_option("--diskali",
                      action="store_true",
                      default=False,
                      help="volume alignment")
    parser.add_option(
        "--zstep",
        type="float",
        default=1,
        help="Step size for translational search along z (Angstroms)")

    # helicise
    parser.add_option(
        "--helicise",
        action="store_true",
        default=False,
        help="helicise input volume and save results to output volume")
    parser.add_option(
        "--hfsc",
        type="string",
        default="",
        help=
        "Generate two lists of image indices used to split segment stack into halves for helical fsc calculation. The lists will be stored in two text files named using file_prefix with '_even' and '_odd' suffixes, respectively."
    )
    parser.add_option(
        "--filament_attr",
        type="string",
        default="filament",
        help="attribute under which filament identification is stored")
    parser.add_option(
        "--predict_helical",
        type="string",
        default="",
        help="Generate projection parameters consistent with helical symmetry")

    # helicise pdb
    parser.add_option(
        "--helicisepdb",
        action="store_true",
        default=False,
        help="Helicise pdb file and save the result to a new pdb file")
    parser.add_option(
        "--nrepeats",
        type="int",
        default=50,
        help=
        "Number of time the helical symmetry will be applied to the input file"
    )

    # input options for generating disks
    parser.add_option(
        "--gendisk",
        type="string",
        default="",
        help="Name of file under which generated disks will be saved to")
    parser.add_option("--ref_nx",
                      type="int",
                      default=-1,
                      help="nx=ny volume size")
    parser.add_option(
        "--ref_nz",
        type="int",
        default=-1,
        help="nz volume size - computed disks will be nx x ny x rise/apix")
    parser.add_option(
        "--new_pixel_size",
        type="float",
        default=-1,
        help=
        "desired pixel size of the output disks. The default is -1, in which case there is no resampling (unless --match_pixel_rise flag is True)."
    )
    parser.add_option(
        "--maxerror",
        type="float",
        default=0.1,
        help=
        "proportional to the maximum amount of error to tolerate between (dp/new_pixel_size) and int(dp/new_pixel_size ), where new_pixel_size is the pixel size calculated when the option --match_pixel_rise flag is True."
    )
    parser.add_option(
        "--match_pixel_rise",
        action="store_true",
        default=False,
        help=
        "calculate new pixel size such that the rise is approximately integer number of pixels given the new pixel size. This will be the pixel size of the output disks."
    )

    # get consistency
    parser.add_option(
        "--consistency",
        type="string",
        default="",
        help="Name of parameters to get consistency statistics for")
    parser.add_option("--phithr",
                      type="float",
                      default=2.0,
                      help="phi threshold for consistency check")
    parser.add_option("--ythr",
                      type="float",
                      default=2.0,
                      help="y threshold (in Angstroms) for consistency check")
    parser.add_option(
        "--segthr",
        type="int",
        default=3,
        help="minimum number of segments/filament for consistency check")

    # stack disks
    parser.add_option(
        "--stackdisk",
        type="string",
        default="",
        help="Name of file under which output volume will be saved to.")
    parser.add_option("--ref_ny",
                      type="int",
                      default=-1,
                      help="ny of output volume size. Default is ref_nx")

    # symmetry search
    parser.add_option("--symsearch",
                      action="store_true",
                      default=False,
                      help="Do helical symmetry search.")
    parser.add_option(
        "--ndp",
        type="int",
        default=12,
        help=
        "In symmetrization search, number of delta z steps equals to 2*ndp+1")
    parser.add_option(
        "--ndphi",
        type="int",
        default=12,
        help=
        "In symmetrization search, number of dphi steps equals to 2*ndphi+1")
    parser.add_option(
        "--dp_step",
        type="float",
        default=0.1,
        help="delta z step  for symmetrization [Angstroms] (default 0.1)")
    parser.add_option(
        "--dphi_step",
        type="float",
        default=0.1,
        help="dphi step for symmetrization [degrees] (default 0.1)")
    parser.add_option("--datasym",
                      type="string",
                      default="datasym.txt",
                      help="symdoc")
    parser.add_option(
        "--symdoc",
        type="string",
        default="",
        help="text file containing helical symmetry parameters dp and dphi")

    # filament statistics in the stack

    (options, args) = parser.parse_args(arglist[1:])
    if len(args) < 1 or len(args) > 5:
        print("Various helical reconstruction related functionalities: " +
              usage2)
        print("Please run '" + progname + " -h' for detailed options")
    else:

        if len(options.hfsc) > 0:
            if len(args) != 1:
                print("Incorrect number of parameters")
                sys.exit()
            from applications import imgstat_hfsc
            imgstat_hfsc(args[0], options.hfsc, options.filament_attr)
            sys.exit()
        elif len(options.filinfo) > 0:
            if len(args) != 1:
                print("Incorrect number of parameters")
                sys.exit()
            from EMAN2 import EMUtil
            filams = EMUtil.get_all_attributes(args[0], "filament")
            ibeg = 0
            filcur = filams[0]
            n = len(filams)
            inf = []
            i = 1
            while (i <= n):
                if (i < n): fis = filams[i]
                else: fis = ""
                if (fis != filcur):
                    iend = i - 1
                    inf.append([ibeg, iend, iend - ibeg + 1, filcur])
                    ibeg = i
                    filcur = fis
                i += 1
            from utilities import write_text_row
            write_text_row(inf, options.filinfo)
            sys.exit()

        if len(options.stackdisk) > 0:
            if len(args) != 1:
                print("Incorrect number of parameters")
                sys.exit()
            dpp = (float(options.dp) / options.apix)
            rise = int(dpp)
            if (abs(float(rise) - dpp) > 1.0e-3):
                print("  dpp has to be integer multiplicity of the pixel size")
                sys.exit()
            from utilities import get_im
            v = get_im(args[0])
            from applications import stack_disks
            ref_ny = options.ref_ny
            if ref_ny < 0:
                ref_ny = options.ref_nx
            sv = stack_disks(v, options.ref_nx, ref_ny, options.ref_nz,
                             options.dphi, rise)
            sv.write_image(options.stackdisk)
            sys.exit()

        if len(options.consistency) > 0:
            if len(args) != 1:
                print("Incorrect number of parameters")
                sys.exit()
            from development import consistency_params
            consistency_params(args[0],
                               options.consistency,
                               options.dphi,
                               options.dp,
                               options.apix,
                               phithr=options.phithr,
                               ythr=options.ythr,
                               THR=options.segthr)
            sys.exit()

        rminp = int((float(options.rmin) / options.apix) + 0.5)
        rmaxp = int((float(options.rmax) / options.apix) + 0.5)

        from utilities import get_input_from_string, get_im

        xr = get_input_from_string(options.xr)
        txs = get_input_from_string(options.txs)

        irp = 1
        if options.ou < 0: oup = -1
        else: oup = int((options.ou / options.apix) + 0.5)
        xrp = ''
        txsp = ''

        for i in xrange(len(xr)):
            xrp += " " + str(float(xr[i]) / options.apix)
        for i in xrange(len(txs)):
            txsp += " " + str(float(txs[i]) / options.apix)

        searchxshiftp = int((options.searchxshift / options.apix) + 0.5)
        nearbyp = int((options.nearby / options.apix) + 0.5)
        zstepp = int((options.zstep / options.apix) + 0.5)

        if options.MPI:
            from mpi import mpi_init, mpi_finalize
            sys.argv = mpi_init(len(sys.argv), sys.argv)

        if len(options.predict_helical) > 0:
            if len(args) != 1:
                print("Incorrect number of parameters")
                sys.exit()
            if options.dp < 0:
                print(
                    "Helical symmetry paramter rise --dp should not be negative"
                )
                sys.exit()
            from applications import predict_helical_params
            predict_helical_params(args[0], options.dp, options.dphi,
                                   options.apix, options.predict_helical)
            sys.exit()

        if options.helicise:
            if len(args) != 2:
                print("Incorrect number of parameters")
                sys.exit()
            if options.dp < 0:
                print(
                    "Helical symmetry paramter rise --dp should not be negative"
                )
                sys.exit()
            from utilities import get_im, sym_vol
            vol = get_im(args[0])
            vol = sym_vol(vol, options.sym)
            hvol = vol.helicise(options.apix, options.dp, options.dphi,
                                options.fract, rmaxp, rminp)
            hvol = sym_vol(hvol, options.sym)
            hvol.write_image(args[1])
            sys.exit()

        if options.helicisepdb:
            if len(args) != 2:
                print("Incorrect number of parameters")
                sys.exit()
            if options.dp < 0:
                print(
                    "Helical symmetry paramter rise --dp should not be negative"
                )
                sys.exit()
            from math import cos, sin, radians
            from copy import deepcopy
            import numpy
            from numpy import zeros, dot, float32

            dp = options.dp
            dphi = options.dphi
            nperiod = options.nrepeats

            infile = open(args[0], "r")
            pall = infile.readlines()
            infile.close()

            p = []

            pos = []
            lkl = -1
            for i in xrange(len(pall)):
                if ((pall[i])[:4] == 'ATOM'):
                    if (lkl == -1): lkl = i
                    p.append(pall[i])
                    pos.append(i)
            n = len(p)

            X = zeros((3, len(p)), dtype=float32)
            X_new = zeros((3, len(p)), dtype=float32)

            for i in xrange(len(p)):
                element = deepcopy(p[i])
                X[0, i] = float(element[30:38])
                X[1, i] = float(element[38:46])
                X[2, i] = float(element[46:54])

            pnew = []
            for j in xrange(-nperiod, nperiod + 1):
                for i in xrange(n):
                    pnew.append(deepcopy(p[i]))

            dphi = radians(dphi)
            m = zeros((3, 3), dtype=float32)
            t = zeros((3, 1), dtype=float32)
            m[2][2] = 1.0
            t[0, 0] = 0.0
            t[1, 0] = 0.0

            for j in xrange(-nperiod, nperiod + 1):
                if j != 0:
                    rd = j * dphi
                    m[0][0] = cos(rd)
                    m[0][1] = sin(rd)
                    m[1][0] = -m[0][1]
                    m[1][1] = m[0][0]
                    t[2, 0] = j * dp
                    X_new = dot(m, X) + t
                    for i in xrange(n):
                        pnew[j * n +
                             i] = pnew[j * n + i][:30] + "%8.3f" % (float(
                                 X_new[0, i])) + "%8.3f" % (float(
                                     X_new[1, i])) + "%8.3f" % (float(
                                         X_new[2, i])) + pnew[j * n + i][54:]

            outfile = open(args[1], "w")
            outfile.writelines(pall[0:lkl])
            outfile.writelines(pnew)
            outfile.writelines("END\n")
            outfile.close()
            sys.exit()

        if options.volalixshift:
            if options.maxit > 1:
                print(
                    "Inner iteration for x-shift determinatin is restricted to 1"
                )
                sys.exit()
            if len(args) < 4: mask = None
            else: mask = args[3]
            from applications import volalixshift_MPI
            global_def.BATCH = True
            volalixshift_MPI(args[0], args[1], args[2], searchxshiftp,
                             options.apix, options.dp, options.dphi,
                             options.fract, rmaxp, rminp, mask, options.maxit,
                             options.CTF, options.snr, options.sym,
                             options.function, options.npad, options.debug,
                             nearbyp)
            global_def.BATCH = False

        if options.diskali:
            #if options.maxit > 1:
            #	print "Inner iteration for disk alignment is restricted to 1"
            #	sys.exit()
            if len(args) < 4: mask = None
            else: mask = args[3]
            global_def.BATCH = True
            if (options.sym[:1] == "d" or options.sym[:1] == "D"):
                from development import diskaliD_MPI
                diskaliD_MPI(args[0], args[1], args[2], mask, options.dp,
                             options.dphi, options.apix, options.function,
                             zstepp, options.fract, rmaxp, rminp, options.CTF,
                             options.maxit, options.sym)
            else:
                from applications import diskali_MPI
                diskali_MPI(args[0], args[1], args[2], mask, options.dp,
                            options.dphi, options.apix, options.function,
                            zstepp, options.fract, rmaxp, rminp, options.CTF,
                            options.maxit, options.sym)
            global_def.BATCH = False

        if options.symsearch:

            if len(options.symdoc) < 1:
                if options.dp < 0 or options.dphi < 0:
                    print(
                        "Enter helical symmetry parameters either using --symdoc or --dp and --dphi"
                    )
                    sys.exit()

            if options.dp < 0 or options.dphi < 0:
                # read helical symmetry parameters from symdoc
                from utilities import read_text_row
                hparams = read_text_row(options.symdoc)
                dp = hparams[0][0]
                dphi = hparams[0][1]
            else:
                dp = options.dp
                dphi = options.dphi

            from applications import symsearch_MPI
            if len(args) < 3:
                mask = None
            else:
                mask = args[2]
            global_def.BATCH = True
            symsearch_MPI(args[0], args[1], mask, dp, options.ndp,
                          options.dp_step, dphi, options.ndphi,
                          options.dphi_step, rminp, rmaxp, options.fract,
                          options.sym, options.function, options.datasym,
                          options.apix, options.debug)
            global_def.BATCH = False

        elif len(options.gendisk) > 0:
            from applications import gendisks_MPI
            global_def.BATCH = True
            if len(args) == 1: mask3d = None
            else: mask3d = args[1]
            if options.dp < 0:
                print(
                    "Helical symmetry paramter rise --dp must be explictly set!"
                )
                sys.exit()
            gendisks_MPI(args[0], mask3d, options.ref_nx, options.apix,
                         options.dp, options.dphi, options.fract, rmaxp, rminp,
                         options.CTF, options.function, options.sym,
                         options.gendisk, options.maxerror,
                         options.new_pixel_size, options.match_pixel_rise)
            global_def.BATCH = False

        if options.MPI:
            from mpi import mpi_finalize
            mpi_finalize()
Exemplo n.º 32
0
 def __init__(self, options):
     self.options = options
     self.using_cuda = EMUtil.cuda_available() and options.cuda
     self.nsoln = options.nsoln