예제 #1
0
def main():
    usage = """
	Alternative to e2spt_align.py, but uses 2D subtilt images instead of 3D particle volume for alignment. Also a modified scipy minimizer is used in place of EMAN2 aligners. 
	e2spt_align_subtlt.py sets/ptcls.lst reference.hdf --path spt_xx --iter x
"""

    parser = EMArgumentParser(usage=usage, version=EMANVERSION)

    parser.add_argument(
        "--path",
        type=str,
        default=None,
        help=
        "Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)"
    )
    parser.add_argument("--iter",
                        type=int,
                        help="Iteration number within path. Default = 0",
                        default=0)
    parser.add_argument(
        "--goldcontinue",
        action="store_true",
        help=
        "Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",
        default=False)
    parser.add_argument(
        "--sym",
        type=str,
        default="c1",
        help=
        "Symmetry of the input. Must be aligned in standard orientation to work properly."
    )
    parser.add_argument(
        "--maxres",
        type=float,
        help="Maximum resolution to consider in alignment (in A, not 1/A)",
        default=0)
    parser.add_argument(
        "--minres",
        type=float,
        help="Minimum resolution to consider in alignment (in A, not 1/A)",
        default=0)
    #parser.add_argument("--maxtilt",type=float,help="Excluding tilt images beyond the angle",default=-1)
    parser.add_argument("--parallel",
                        type=str,
                        help="Thread/mpi parallelism to use",
                        default="thread:4")
    parser.add_argument(
        "--fromscratch",
        action="store_true",
        help=
        "Start from exhaustive coarse alignment. Otherwise will use alignment from the previous rounds and do local search only.",
        default=False)
    parser.add_argument(
        "--use3d",
        action="store_true",
        help="use projection of 3d particles instead of 2d sub tilt series",
        default=False)
    parser.add_argument(
        "--preprocess",
        metavar="processor_name:param1=value1:param2=value2",
        type=str,
        default=None,
        help="Preprocess each 2-D subtilt while loading (alignment only)")
    parser.add_argument(
        "--debug",
        action="store_true",
        help=
        "Debug mode. Will run a small number of particles directly without parallelism with lots of print out. ",
        default=False)
    parser.add_argument(
        "--plst",
        type=str,
        default=None,
        help=
        "list of 2d particle with alignment parameters. The program will reconstruct before alignment so it can be slower."
    )

    parser.add_argument(
        "--maxshift",
        type=int,
        help=
        "Maximum shift from the center of the box or the previous alignment. default box size//6",
        default=-1)
    parser.add_argument(
        "--maxang",
        type=int,
        help=
        "Maximum angle difference from starting point. Ignored when --fromscratch is on.",
        default=30)
    parser.add_argument(
        "--curve",
        action="store_true",
        help=
        "Mode for filament structure refinement. Still under testing. Ignored when --fromscratch is on.",
        default=False)
    parser.add_argument(
        "--skipali",
        action="store_true",
        help=
        "Skip alignment and only calculate the score. Incompatible with --fromscratch, but --breaksym will still be considered.",
        default=False)
    parser.add_argument(
        "--breaksym",
        type=str,
        default=None,
        help="Specify symmetry to break. Ignored when --fromscratch is on.")

    parser.add_argument(
        "--verbose",
        "-v",
        dest="verbose",
        action="store",
        metavar="n",
        type=int,
        default=0,
        help=
        "verbose level [0-9], higher number means higher level of verboseness")
    parser.add_argument(
        "--ppid",
        type=int,
        help="Set the PID of the parent process, used for cross platform PPID",
        default=-1)

    (options, args) = parser.parse_args()
    logid = E2init(sys.argv, options.ppid)

    options.ref = args[1]
    #### we assume the particle_info files are in the right place so the bookkeeping is easier
    ##   they should be created by e2spt_refine_new or e2spt_refine_multi_new
    options.info3dname = "{}/particle_info_3d.lst".format(options.path)
    options.info2dname = "{}/particle_info_2d.lst".format(options.path)
    if options.preprocess != None:
        options.preprocess = parsemodopt(options.preprocess)

    #### this should generate a list of dictionaries (one dictionary per 3d particle)
    tasks = load_lst_params(args[0])

    #### keep a copy of the particle index here, so we can make sure the output list has the same order as the input list
    for i, t in enumerate(tasks):
        t["ii"] = i

    from EMAN2PAR import EMTaskCustomer
    etc = EMTaskCustomer(options.parallel,
                         module="e2spt_align_subtlt.SptAlignTask")
    num_cpus = etc.cpu_est()
    options.nowtime = time.time()

    #### only run a few particles for debugging
    if options.debug:
        tasks = tasks[:num_cpus * 4]

    print("{} jobs on {} CPUs".format(len(tasks), num_cpus))
    njob = num_cpus

    #### send out jobs
    tids = []
    for i in range(njob):
        t = tasks[i::njob]
        task = SptAlignTask(t, options)
        if options.debug:
            ret = task.execute(print)
            return
        tid = etc.send_task(task)
        tids.append(tid)

    while 1:
        st_vals = etc.check_task(tids)
        if -100 in st_vals:
            print("Error occurs in parallelism. Exit")
            return
        E2progress(logid, np.mean(st_vals) / 100.)

        if np.min(st_vals) == 100: break
        time.sleep(5)

    #### each job returns a 3-object tuple:
    ##     (index of 3d particle, 3d particle align info dict, list of dict for 2d particles align info)
    output3d = [None] * len(tasks)
    output2d = [None] * len(tasks)
    for i in tids:
        rets = etc.get_results(i)[1]
        for r in rets:
            output3d[r[0]] = r[1]
            output2d[r[0]] = r[2]

    del etc

    #### merge the 2d particle alignment info into one list
    output2d = sum(output2d, [])

    #### just dump the list of dictionaries to list files
    fm3d = f"{options.path}/aliptcls3d_{options.iter:02d}.lst"
    save_lst_params(output3d, fm3d)

    fm2d = f"{options.path}/aliptcls2d_{options.iter:02d}.lst"
    save_lst_params(output2d, fm2d)

    E2end(logid)
예제 #2
0
def main():
	progname = os.path.basename(sys.argv[0])
	usage = """prog <output> [options]

	The goal of this program is to reduce the heterogeneity of a reconstruction by splitting a single map
	into two maps, each more homogeneous. You must run e2refine_easy to completion before using this program.
	It will take the class-averaging results from the final iteration, and split the particles from each 
	class-average into 2 groups, producing 2 class-averages for each. The program then attempts to construct
	a maximally self-consistent grouping of these pairs of class averages into 2 3-D maps. 
	"""

	parser = EMArgumentParser(usage=usage,version=EMANVERSION)

	parser.add_argument("--path", default=None, type=str,help="The name of an existing refine_xx folder, where e2refine_easy ran to completion",guitype='filebox', filecheck=False,browser="EMBrowserWidget(withmodal=True,multiselect=False)", row=3, col=0, rowspan=1, colspan=3)
	parser.add_argument("--basisn", default=1,type=int,help="Select which Eigenimage to use for separation. 1 = highest energy. max = 5", guitype='intbox', row=4, col=0, rowspan=1, colspan=1)
	parser.add_argument("--parallel", default="thread:2", help="Standard parallelism option. Default=thread:2", guitype='strbox', row=5, col=0, rowspan=1, colspan=2)
	parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n",type=int, default=0, help="verbose level [0-9], higner number means higher level of verboseness")
	parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)

	(options, args) = parser.parse_args()

	if options.basisn<1 or options.basisn>5 : 
		print "Error: basisn must be in the 1-5 range"
		sys.exit(1)

	if options.path==None:
		paths=[i for i in os.listdir(".") if "refine_" in i and len(i)==9]
		paths.sort()
		options.path=paths[-1]

	pathnum=options.path[-2:]

	# check the specified path for the files we need
	try:
		olddb = js_open_dict(options.path+"/0_refine_parms.json")
		last_map=olddb["last_map"]
		targetres=olddb["targetres"]
		last_iter=int(last_map.split("_")[-1][:2])
		try: 
			ptcls=olddb["inputavg"]
			if ptcls==None : raise Exception
		except: ptcls=olddb["input"]
		
		sym=olddb["sym"]
		if options.verbose : print "Found iteration {} in {}, using {}".format(last_iter,options.path," & ".join(ptcls))
	except:
		traceback.print_exc()
		print "Error: Cannot find necessary files in ",options.path
		sys.exit(1)
		
	logger=E2init(sys.argv,options.ppid)

	# classmx is a list with 2 elements. Each element is a list of EMData from the corresponding cls_result file
	classmx=[]
	classmx.append(EMData.read_images("{}/cls_result_{:02d}_even.hdf".format(options.path,last_iter)))
	classmx.append(EMData.read_images("{}/cls_result_{:02d}_odd.hdf".format(options.path,last_iter)))
	ncls=max(int(classmx[0][0]["maximum"])+1,int(classmx[1][0]["maximum"])+1)

	# Rearrange the info in classmx
	classlists=[[] for i in xrange(ncls)]	# empty list for each class
	
	# This will produce a list of particles with Transforms for each class
	for eo in (0,1):
		for y in xrange(classmx[eo][0]["ny"]):
			ptcl=[eo,y,Transform({"type":"2d","tx":classmx[eo][2][0,y],"ty":classmx[eo][3][0,y],"alpha":classmx[eo][4][0,y],"mirror":int(classmx[eo][5][0,y])})]
			#print ptcl, 
			#print int(classmx[eo][0][0,y])
			classlists[int(classmx[eo][0][0,y])].append(ptcl)
	
	#if len(classlists[0])>100 :
		#print "Warning: this program is normally intended for use with downsampled data and fairly coarse angular sampling. If you try to use it with a large number of class-averages you may have a variety of problems, and should insure that your machine has sufficient RAM."
		

	# Initialize parallelism
	from EMAN2PAR import EMTaskCustomer
	etc=EMTaskCustomer(options.parallel)

	# Empty image to pad classes file
	zero=EMData(str(ptcls[0]),0)
	zero.to_zero()
	zero["ptcl_repr"]=0
	
	# Euler angles for averages
	projin="{}/projections_{:02d}_even.hdf".format(options.path,last_iter)
	eulers=[EMData(projin,i,True)["xform.projection"] for i in xrange(ncls)]
	
	# prepare tasks
	tasks=[]
	gc=0
	ns=[classmx[eo][0]["ny"] for eo in (0,1)]
	for c,cl in enumerate(classlists):
		if len(cl)<20 : 							# we require at least 20 particles in a class to make the attempt
#			zero.write_image(classout[0],c)
#			zero.write_image(classout[1],c)
			continue
		tasks.append(ClassSplitTask(ptcls,ns,cl,c,eulers[c],options.basisn,options.verbose-1))
		gc+=1
	
#	for t in tasks: t.execute()

	# execute task list
	taskids=etc.send_tasks(tasks)
	alltaskids=taskids[:]

	classes=[]
	while len(taskids)>0 :
		curstat=etc.check_task(taskids)
		for i,j in enumerate(curstat):
			if j==100 :
				rslt=etc.get_results(taskids[i])
				rsltd=rslt[1]
				cls=rslt[0].options["classnum"]
				if rsltd.has_key("failed") :
					print "Bad average in ",cls
				else:
					#rsltd["avg1"].write_image(classout[0],cls)
					#rsltd["avg2"].write_image(classout[1],cls)
					ncls=rsltd["avg1"]["ptcl_repr"]+rsltd["avg2"]["ptcl_repr"]
					# note that the 2 results we get back are in arbitrary order!
					# the next section of code with 3D reconstruction is designed to sort out
					# which average should be paired with which
					classes.append([ncls,rsltd["avg1"]["xform.projection"],rsltd["avg1"],rsltd["avg2"],rsltd["basis"]])	# list of (ptcl_repr,xform,avg1,avg2)
				
		taskids=[j for i,j in enumerate(taskids) if curstat[i]!=100]

		if options.verbose and 100 in curstat :
			print "%d/%d tasks remain"%(len(taskids),len(alltaskids))
		if 100 in curstat :
			E2progress(logger,1.0-(float(len(taskids))/len(alltaskids)))

	if options.verbose : print "Completed all tasks\nGrouping consistent averages"

	classes.sort(reverse=True)		# we want to start with the largest number of particles
	apix=classes[0][2]["apix_x"]

	boxsize=classes[0][2]["ny"]
	pad=good_size(boxsize*1.5)
	if options.verbose: print "Boxsize -> {}, padding to {}".format(boxsize,pad)
		
	# a pair of reconstructors. we will then simultaneously reconstruct in the pair, and use each to decide on the best target for each particle
	recon=[Reconstructors.get("fourier",{"size":[pad,pad,pad],"sym":sym,"mode":"gauss_5"}) for i in (0,1)]
	for r in recon: r.setup()
	
	# We insert the first class-average (with the most particles) randomly into reconstructor 1 or 2
	p2=classes[0][2].get_clip(Region(-(pad-boxsize)/2,-(pad-boxsize)/2,pad,pad))
	p3=recon[0].preprocess_slice(p2,classes[0][1])
	recon[0].insert_slice(p3,classes[0][1],classes[0][2].get_attr_default("ptcl_repr",1.0))

	p2=classes[0][3].get_clip(Region(-(pad-boxsize)/2,-(pad-boxsize)/2,pad,pad))
	p3=recon[1].preprocess_slice(p2,classes[0][1])
	recon[1].insert_slice(p3,classes[0][1],classes[0][3].get_attr_default("ptcl_repr",1.0))
	
	classes[0].append(0)

	if options.verbose : print "Reconstruction: pass 1"
	for i,c in enumerate(classes[1:]):
		a2=c[2].get_clip(Region(-(pad-boxsize)/2,-(pad-boxsize)/2,pad,pad))		# first class-average
		a3=recon[0].preprocess_slice(a2,classes[0][1])
		a3n=c[2].get_attr_default("ptcl_repr",1.0)
		
		b2=c[3].get_clip(Region(-(pad-boxsize)/2,-(pad-boxsize)/2,pad,pad))
		b3=recon[1].preprocess_slice(b2,classes[0][1])						# I don't believe it matters if we use recon[0] or 1 here, but haven't checked
		b3n=c[3].get_attr_default("ptcl_repr",1.0)
		
		recon[0].determine_slice_agreement(a3,c[1],a3n,False)
		q0a=a3["reconstruct_absqual"]		# quality for average a in reconstruction0
		n0a=a3["reconstruct_norm"]			# normalization for same
		
		recon[1].determine_slice_agreement(a3,c[1],a3n,False)
		q1a=a3["reconstruct_absqual"]		# quality for average a in reconstruction0
		n1a=a3["reconstruct_norm"]			# normalization for same
		
		recon[0].determine_slice_agreement(b3,c[1],b3n,False)
		q0b=b3["reconstruct_absqual"]		# quality for average a in reconstruction0
		n0b=b3["reconstruct_norm"]			# normalization for same
		
		recon[1].determine_slice_agreement(b3,c[1],b3n,False)
		q1b=b3["reconstruct_absqual"]		# quality for average a in reconstruction0
		n1b=b3["reconstruct_norm"]			# normalization for same
		
		if options.verbose>1 : print i,q0a,q1a,q0b,q1b,q0a+q1b,q1a+q0b
			
		if q0a+q1b>q1a+q0b :		# if true, a -> recon0 and b -> recon1 
			c.append(0)				# we put a 0 at the end of the classes element if we use a->0,b->1 ordering, 1 if swapped
			a3.mult(n0a)
			recon[0].insert_slice(a3,c[1],a3n)
			b3.mult(n1b)
			recon[1].insert_slice(b3,c[1],b3n)
		else:
			c.append(1)
			a3.mult(n1a)
			recon[1].insert_slice(a3,c[1],a3n)
			b3.mult(n0b)
			recon[0].insert_slice(b3,c[1],b3n)

	if options.verbose : print "Reconstruction: pass 2"
	
	# another pass with the filled reconstruction to make sure our initial assignments were ok
#	for i,c in enumerate(classes[1:]):
#		a2=c[2].get_clip(Region(-(pad-boxsize)/2,-(pad-boxsize)/2,pad,pad))		# first class-average
#		a3=recon[0].preprocess_slice(a2,classes[0][1])
#		a3n=c[2].get_attr_default("ptcl_repr",1.0)
#		
#		b2=c[3].get_clip(Region(-(pad-boxsize)/2,-(pad-boxsize)/2,pad,pad))
#		b3=recon[1].preprocess_slice(b2,classes[0][1])						# I don't believe it matters if we use recon[0] or 1 here, but haven't checked
#		b3n=c[3].get_attr_default("ptcl_repr",1.0)
#		
#		recon[0].determine_slice_agreement(a3,c[1],a3n,0) # c[-1]==0
#		q0a=a3["reconstruct_absqual"]			# quality for average a in reconstruction0
#		n0a=a3["reconstruct_norm"]			# normalization for same
#		
#		recon[1].determine_slice_agreement(a3,c[1],a3n,0) # c[-1]==1
#		q1a=a3["reconstruct_absqual"]			# quality for average a in reconstruction0
#		n1a=a3["reconstruct_norm"]			# normalization for same
#		
#		recon[0].determine_slice_agreement(b3,c[1],b3n,0) # c[-1]==1
#		q0b=b3["reconstruct_absqual"]			# quality for average a in reconstruction0
#		n0b=b3["reconstruct_norm"]			# normalization for same
#		
#		recon[1].determine_slice_agreement(b3,c[1],b3n,0) # c[-1]==0
#		q1b=b3["reconstruct_absqual"]			# quality for average a in reconstruction0
#		n1b=b3["reconstruct_norm"]			# normalization for same
#		
#		if options.verbose>1 : print i,q0a,q1a,q0b,q1b,q0a+q1b,q1a+q0b
#			
#		if q0a+q1b>q1a+q0b :		# if true, a -> recon0 and b -> recon1 
#			if c[-1]==1 :
#				c[-1]=0
#				print i," 1->0"
#			
#			c.append(0)				# we put a 0 at the end of the classes element if we use a->0,b->1 ordering, 1 if swapped
#			a3.mult(n0a)
#			recon[0].insert_slice(a3,c[1],a3n)
#			b3.mult(n1b)
#			recon[1].insert_slice(b3,c[1],b3n)
#		else:
#			if c[-1]==0 :
#				c[-1]=1
#				print i," 0->1"
#
#			c.append(1)
#			a3.mult(n1a)
#			recon[1].insert_slice(a3,c[1],a3n)
#			b3.mult(n0b)
#	
#		
	if options.verbose : print "All done, writing output"

	classout=["{}/classes_{:02d}_bas{}_split0.hdf".format(options.path,last_iter,options.basisn),"{}/classes_{:02d}_bas{}_split1.hdf".format(options.path,last_iter,options.basisn)]
	basisout="{}/classes_{:02d}_basis".format(options.path,last_iter)
	threedout="{}/threed_{:02d}_split.hdf".format(options.path,last_iter)
	threedout2="{}/threed_{:02d}_split_filt_bas{}.hdf".format(options.path,last_iter,options.basisn)
	setout=["sets/split_{}_0.lst".format(pathnum),"sets/split_{}_1.lst".format(pathnum)]
	split=[r.finish(True).get_clip(Region((pad-boxsize)/2,(pad-boxsize)/2,(pad-boxsize)/2,boxsize,boxsize,boxsize)) for r in recon]
	split[0]["apix_x"]=apix
	split[0]["apix_y"]=apix
	split[0]["apix_z"]=apix
	split[1]["apix_x"]=apix
	split[1]["apix_y"]=apix
	split[1]["apix_z"]=apix
	split[0].process_inplace("mask.soft",{"outer_radius":-8,"width":4})
	split[1].process_inplace("mask.soft",{"outer_radius":-8,"width":4})
	split[0].write_image(threedout,0)
	split[1].write_image(threedout,1)

	# now we write the class-averages and the new (split) particle files
	lstin =[LSXFile(ptcls[0],True),LSXFile(ptcls[1],True)]
	try:
		os.unlink("sets/split0.lst")
		os.unlink("sets/split1.lst")
	except: pass
	lstout=[LSXFile("sets/split0.lst"),LSXFile("sets/split1.lst")]
	for i,c in enumerate(classes):
		c[2].write_image(classout[c[-1]],i)	# class-average
		ptcln=c[2]["class_eoidxs"]		# eofile/ptcl# pairs
		for p in xrange(0,len(ptcln),2):
			lstout[0][-1]=lstin[ptcln[p]][ptcln[p+1]]		# wierd syntax, but the -1 here appends
			
		c[3].write_image(classout[c[-1]^1],i)
		ptcln=c[3]["class_eoidxs"]		# eofile/ptcl# pairs
		for p in xrange(0,len(ptcln),2):
			lstout[1][-1]=lstin[ptcln[p]][ptcln[p+1]]		# wierd syntax, but the -1 here appends

		if options.verbose>2:
			c[4][0].write_image(basisout+"1.hdf",i)
			c[4][1].write_image(basisout+"2.hdf",i)
			c[4][2].write_image(basisout+"3.hdf",i)

	launch_childprocess("e2proclst.py sets/split0.lst --mergesort {}".format(setout[0]))
	launch_childprocess("e2proclst.py sets/split1.lst --mergesort {}".format(setout[1]))

	try:
		os.unlink("sets/split0.lst")
		os.unlink("sets/split1.lst")
	except:
		pass

	if os.path.exists("strucfac.txt"):
		launch_childprocess("e2proc3d.py {} {} --setsf strucfac.txt --process filter.wiener.byfsc:fscfile={}/fsc_masked_{:02d}.txt:snrmult=2:sscale=1.1:maxfreq={} --process mask.soft:outer_radius=-9:width=4".format(threedout,threedout2,options.path,last_iter,1.0/targetres))
	else:
		print "Missing structure factor, cannot filter properly"
		launch_childprocess("e2proc3d.py {} {} --process filter.wiener.byfsc:fscfile={}/fsc_masked_{:02d}.txt:snrmult=2:sscale=1.1:maxfreq={} --process mask.soft:outer_radius=-9:width=4".format(threedout,threedout2,options.path,last_iter,1.0/targetres))

	E2end(logger)
예제 #3
0
def main():

    usage = """e2classifytree.py <projection> <particle> [options]
	
	Classify particles using a binary tree. Can be used as an alternative for e2simmx2stage.py + e2classify.py.
	"""
    parser = EMArgumentParser(usage=usage, version=EMANVERSION)
    parser.add_argument("--threads", type=int, help="", default=12)
    parser.add_argument("--nodes", type=str, help="", default="nodes.hdf")
    #parser.add_argument("--clsmx", type=str,help="", default="clsmx.hdf")
    parser.add_argument("--output", type=str, help="", default="clsmx.hdf")
    parser.add_argument(
        "--align",
        type=str,
        help="The name of an 'aligner' to use prior to comparing the images",
        default=None)
    parser.add_argument(
        "--aligncmp",
        type=str,
        help="Name of the aligner along with its construction arguments",
        default="dot")
    parser.add_argument(
        "--ralign",
        type=str,
        help=
        "The name and parameters of the second stage aligner which refines the results of the first alignment",
        default=None)
    parser.add_argument(
        "--raligncmp",
        type=str,
        help=
        "The name and parameters of the comparitor used by the second stage aligner. Default is dot.",
        default="dot")
    parser.add_argument(
        "--cmp",
        type=str,
        help="The name of a 'cmp' to be used in comparing the aligned images",
        default="dot:normalize=1")
    parser.add_argument(
        "--cmpdiff",
        action="store_true",
        default=False,
        help="Compare using the difference of the two children")
    parser.add_argument(
        "--incomplete",
        type=int,
        help="The degree of incomplete allowed in the tree on each level",
        default=0)
    parser.add_argument(
        "--ppid",
        type=int,
        help="Set the PID of the parent process, used for cross platform PPID",
        default=-1)
    parser.add_argument("--parallel",
                        default=None,
                        help="parallelism argument")
    parser.add_argument(
        "--verbose",
        "-v",
        dest="verbose",
        action="store",
        metavar="n",
        type=int,
        default=0,
        help=
        "verbose level [0-9], higher number means higher level of verboseness")

    (options, args) = parser.parse_args()
    E2n = E2init(sys.argv, options.ppid)

    options.align = parsemodopt(options.align)
    options.aligncmp = parsemodopt(options.aligncmp)
    options.ralign = parsemodopt(options.ralign)
    options.raligncmp = parsemodopt(options.raligncmp)
    options.cmp = parsemodopt(options.cmp)

    projs = args[0]
    #projsimmx=args[1]
    ptcl = args[1]
    npj = EMUtil.get_image_count(projs)
    npt = EMUtil.get_image_count(ptcl)
    if options.parallel == None:
        par = "thread:{:d}".format(options.threads)
    else:
        par = options.parallel

    ### Build tree
    ### always overwrite the tree here now
    #if not os.path.isfile(options.nodes):
    print("Building binary tree...")
    buildtree(projs, par, options.nodes, options.incomplete, options.verbose)
    #else:
    #print "Using existing tree..."

    ## Generate children pairs for comparison
    print("Generating children pairs for comparison...")
    if options.cmpdiff:
        nodepath = os.path.dirname(options.nodes)
        masktmp = '/'.join([nodepath, "tmp_msk.hdf"])
        if os.path.isfile(masktmp): os.remove(masktmp)
        cmptmp = '/'.join([nodepath, "tmp_cmp.hdf"])
        if os.path.isfile(cmptmp):
            os.remove(cmptmp)
        makechildpair(options.nodes, cmptmp, masktmp)
    else:
        masktmp = None
        cmptmp = None

    E2progress(E2n, 0.5)
    #exit()
    print("Starting classification...")
    ### Classify particles

    clsmx = [EMData(1, npt) for i in range(7)]
    nnod = EMUtil.get_image_count(options.nodes)
    if options.parallel:
        from EMAN2PAR import EMTaskCustomer
        etc = EMTaskCustomer(options.parallel,
                             module="e2classifytree.TreeClassifyTask")
        tasks = []
        step = 50
        tt = [list(range(i, i + step)) for i in range(0, npt - step, step)]
        tt.append(list(range(tt[-1][-1] + 1, npt)))

        for it in tt:
            tasks.append(
                TreeClassifyTask(ptcl, it, options.nodes, options.align,
                                 options.aligncmp, options.cmp, options.ralign,
                                 options.raligncmp, cmptmp, masktmp))

        taskids = etc.send_tasks(tasks)
        ptclpernode = [0 for i in range(nnod)]
        nfinished = 0
        while len(taskids) > 0:
            haveprogress = False
            time.sleep(3)
            curstat = etc.check_task(taskids)
            for i, j in enumerate(curstat):
                if j == 100:
                    haveprogress = True
                    rslt = etc.get_results(taskids[i])
                    rslt = rslt[1]
                    for r in rslt:
                        nfinished += 1
                        if options.verbose > 0:
                            print("Particle:", r["id"], "\tnodes:",
                                  r["choice"])
                        for c in r["choice"]:
                            ptclpernode[c] += 1
                        clsmx[0].set_value_at(0, r["id"], r["cls"])
                        for nt in range(1, 7):
                            clsmx[nt].set_value_at(0, r["id"], r["simmx"][nt])

            taskids = [j for i, j in enumerate(taskids) if curstat[i] != 100]
            if haveprogress: print("{:d}/{:d} finished".format(nfinished, npt))
            E2progress(E2n, 0.5 + old_div(float(nfinished), npt))

        for i in range(nnod):
            ndtmp = EMData(options.nodes, i, True)
            ndtmp["tree_nptls"] = ptclpernode[i]
            ndtmp.write_image(options.nodes, i)

    else:

        ### To record the number of particles in each branch of the tree
        for i in range(nnod):
            ndtmp = EMData(options.nodes, i, True)
            ndtmp["tree_nptls"] = 0
            ndtmp.write_image(options.nodes, i)
        t = {}
        clsmx = [EMData(1, npt) for i in range(7)]
        for i in range(options.threads):
            ai = [x for x in range(npt) if x % options.threads == i]
            t[i] = threading.Thread(target=classify,
                                    args=(ptcl, ai, options.nodes, clsmx,
                                          options.align, options.aligncmp,
                                          options.cmp, options.ralign,
                                          options.raligncmp, cmptmp, masktmp))
            t[i].start()
        for i in range(options.threads):
            t[i].join()

    if os.path.isfile(options.output):
        os.remove(options.output)
    for i in clsmx:
        i.write_image(options.output, -1)

    if options.cmpdiff:
        os.remove(cmptmp)
        os.remove(masktmp)
    print("Finished~")
    E2progress(E2n, 1.0)
    E2end(E2n)
예제 #4
0
def main():
    progname = os.path.basename(sys.argv[0])
    usage = """prog [options]
	This program will take a set of reference-free class-averages (or other projections) and generate a set of possible
	3-D initial models. It does this by heavily downsampling the data, then running a number of very fast, full iterative
	refinements, each seeded with a random starting model. The results are sorted in order of apparent agreement with the
	data, such that at the end, the first numbered model should be the best result. Ideally the top few answers will all
	qualtitatively agree on the overall structure. If they do not, the results should be thoroughly assessed manually to
	insure a sensible result. By default this routine will generate 10 initial models, but this may be fewer or more than
	is strictly necessary depending on a number of factors. If the data is highly structurally heterogeneous, particularly
	if combined with a strongly preferred orientation, a correct solution using this technique may not be possible, but
	for most situations it will work well. For other situations, single particle tomography presents a good alternative
	for generating initial models."""
    parser = EMArgumentParser(usage=usage, version=EMANVERSION)

    parser.add_header(
        name="initialmodelheader",
        help='Options below this label are specific to e2initialmodel',
        title="### e2initialmodel options ###",
        row=1,
        col=0,
        rowspan=1,
        colspan=3)
    parser.add_argument(
        "--input",
        dest="input",
        default=None,
        type=str,
        help=
        "This file should contain good class-averages to use in constructing the initial model",
        browser='EMBrowserWidget(withmodal=True,multiselect=False)',
        guitype='filebox',
        row=0,
        col=0,
        rowspan=1,
        colspan=3)
    parser.add_argument(
        "--iter",
        type=int,
        default=8,
        help=
        "The total number of refinement iterations to perform, typically 5-10",
        guitype='intbox',
        row=2,
        col=0,
        rowspan=1,
        colspan=1)
    parser.add_argument(
        "--tries",
        type=int,
        default=10,
        help=
        "The number of different initial models to generate in search of a good one",
        guitype='intbox',
        row=2,
        col=1,
        rowspan=1,
        colspan=1)
    parser.add_argument(
        "--shrink",
        dest="shrink",
        type=int,
        default=0,
        help=
        "Optionally shrink the input particles by an integer factor prior to reconstruction. Default=0, no shrinking",
        guitype='shrinkbox',
        row=2,
        col=2,
        rowspan=1,
        colspan=1)
    parser.add_argument(
        "--sym",
        dest="sym",
        help="Specify symmetry - choices are: c<n>, d<n>, h<n>, tet, oct, icos",
        default="c1",
        guitype='symbox',
        row=4,
        col=0,
        rowspan=1,
        colspan=2)
    parser.add_argument(
        "--automaskexpand",
        default=-1,
        type=int,
        help=
        "Number of voxels of post-threshold expansion in the mask, for use when peripheral features are truncated. (default=shrunk boxsize/20)",
        guitype='intbox',
        row=6,
        col=2,
        rowspan=1,
        colspan=1)
    parser.add_argument(
        "--randorient",
        action="store_true",
        help=
        "Instead of seeding with a random volume, seeds by randomizing input orientations",
        default=False,
        guitype='boolbox',
        row=4,
        col=2,
        rowspan=1,
        colspan=1)
    parser.add_argument(
        "--maskproc",
        default=None,
        type=str,
        help=
        "Default=none. If specified, this mask will be performed after the built-in automask, eg - mask.soft to remove the core of a virus",
    )
    #	parser.add_argument("--savemore",action="store_true",help="Will cause intermediate results to be written to flat files",default=False, guitype='boolbox', expert=True, row=5, col=0, rowspan=1, colspan=1)
    parser.add_argument(
        "--verbose",
        "-v",
        dest="verbose",
        action="store",
        metavar="n",
        type=int,
        default=0,
        help=
        "verbose level [0-9], higher number means higher level of verboseness")
    parser.add_argument(
        "--orientgen",
        type=str,
        default="eman:delta=9.0:inc_mirror=0:perturb=1",
        help=
        "The type of orientation generator. Default is eman:delta=9.0:inc_mirror=0:perturb=1. See e2help.py orientgens",
        guitype='strbox',
        expert=True,
        row=4,
        col=2,
        rowspan=1,
        colspan=1)
    parser.add_argument(
        "--parallel",
        "-P",
        type=str,
        help=
        "Run in parallel, specify type:<option>=<value>:<option>=<value>. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel",
        default="thread:1",
        guitype='strbox',
        row=6,
        col=0,
        rowspan=1,
        colspan=2)
    parser.add_argument(
        "--ppid",
        type=int,
        help="Set the PID of the parent process, used for cross platform PPID",
        default=-1)

    # Database Metadata storage
    #parser.add_argument("--dbls",type=str,default=None,help="data base list storage, used by the workflow. You can ignore this argument.")

    (options, args) = parser.parse_args()
    verbose = options.verbose

    try:
        ptcls = EMData.read_images(options.input)
    except:
        print("Error: bad input file")
        exit(1)
    apix = ptcls[0]["apix_x"]
    if options.shrink > 1: apix *= options.shrink

    if options.tries < 10:
        print(
            "Warning: suggest using --tries >=10. The first 8 starting maps are generated deterministically, and it is good to have several random seeds as well to increase the liklihood of a good outcome."
        )

    for i in range(len(ptcls)):
        ptcls[i].process_inplace("normalize.edgemean", {})
        if options.shrink > 1:
            ptcls[i] = ptcls[i].process("math.meanshrink",
                                        {"n": options.shrink})
    if ptcls[0]["nx"] > 160:
        print(
            "WARNING: using a large box size may be slow. Suggest trying --shrink="
        )
    if not ptcls or len(ptcls) == 0: parser.error("Bad input file")
    boxsize = ptcls[0].get_xsize()
    if verbose > 0:
        print("%d particles %dx%d" % (len(ptcls), boxsize, boxsize))
    print("Models will be %1.3f A/pix" % apix)

    [og_name, og_args] = parsemodopt(options.orientgen)

    try:
        sfcurve = XYData()
        sfcurve.read_file("strucfac.txt")

        sfcurve.update()
    except:
        sfcurve = None

    if options.maskproc != None:
        mask2 = EMData(boxsize, boxsize, boxsize)
        mask2.to_one()
        parms = parsemodopt(options.maskproc)
        if parms[0] == "mask.auto3d":
            print(
                "Error, maskproc may not be mask.auto3d, it must be a processor that does not rely on the input map density to function"
            )
            sys.exit(1)
        mask2.process_inplace(parms[0], parms[1])
    else:
        mask2 = None

    # angles to use for refinement
    sym_object = parsesym(options.sym)
    orts = sym_object.gen_orientations(og_name, og_args)

    logid = E2init(sys.argv, options.ppid)
    results = []

    try:
        os.mkdir("initial_models")
    except:
        pass
    iters = [
        int(i[10:12]) for i in os.listdir("initial_models")
        if i[:10] == "particles_"
    ]
    try:
        newiter = max(iters) + 1
    except:
        newiter = 0
    results_name = "initial_models/model_%02d" % newiter
    particles_name = "initial_models/particles_%02d.hdf" % newiter

    # we write the pre-processed "particles" (usually class-averages) to disk, both as a record and to prevent collisions
    for i, p in enumerate(ptcls):
        p.write_image(particles_name, i)

    # parallelism
    from EMAN2PAR import EMTaskCustomer  # we need to put this here to avoid a circular reference

    etc = EMTaskCustomer(options.parallel, module="e2initialmodel.InitMdlTask")
    pclist = [particles_name]

    etc.precache(
        pclist
    )  # make sure the input particles are precached on the compute nodes

    tasks = []
    for t in range(options.tries):
        tasks.append(
            InitMdlTask(particles_name, len(ptcls), orts, t, sfcurve,
                        options.iter, options.sym, mask2, options.randorient,
                        options.automaskexpand, options.verbose))

    taskids = etc.send_tasks(tasks)
    alltaskids = taskids[:]  # we keep a copy for monitoring progress

    # This loop runs until all subtasks are complete (via the parallelism system
    ltime = 0
    while len(taskids) > 0:
        time.sleep(0.1)
        curstat = etc.check_task(
            taskids)  # a list of the progress on each task
        if options.verbose > 1:
            if time.time() - ltime > 1:
                print("progress: ", curstat)
                ltime = time.time()
        for i, j in enumerate(curstat):
            if j == 100:
                rslt = etc.get_results(
                    taskids[i]
                )  # read the results back from a completed task as a one item dict
                results.append(rslt[1]["result"])
                if options.verbose == 1:
                    print("Task {} ({}) complete".format(i, taskids[i]))

        # filter out completed tasks. We can't do this until after the previous loop completes
        taskids = [
            taskids[i] for i in range(len(taskids)) if curstat[i] != 100
        ]

    # Write out the final results
    results.sort()
    for i, j in enumerate(results):
        out_name = results_name + "_%02d.hdf" % (i + 1)
        j[1].write_image(out_name, 0)
        j[4].write_image(results_name + "_%02d_init.hdf" % (i + 1), 0)
        print(out_name, j[1]["quality"], j[0], j[1]["apix_x"])
        for k, l in enumerate(j[3]):
            l[0].write_image(results_name + "_%02d_proj.hdf" % (i + 1),
                             k)  # set of projection images
        for k, l in enumerate(j[2]):
            l.process("normalize").write_image(
                results_name + "_%02d_aptcl.hdf" % (i + 1),
                k * 2)  # set of aligned particles
            j[3][l["match_n"]][0].process("normalize").write_image(
                results_name + "_%02d_aptcl.hdf" % (i + 1),
                k * 2 + 1)  # set of projections matching aligned particles

    E2end(logid)
예제 #5
0
def main():
    progname = os.path.basename(sys.argv[0])
    usage = """prog <output> [options]

	The goal of this program is to reduce the heterogeneity of a reconstruction by splitting a single map
	into two maps, each more homogeneous. You must run e2refine_easy to completion before using this program.
	It will take the class-averaging results from the final iteration, and split the particles from each 
	class-average into 2 groups, producing 2 class-averages for each. The program then attempts to construct
	a maximally self-consistent grouping of these pairs of class averages into 2 3-D maps. 
	"""

    parser = EMArgumentParser(usage=usage, version=EMANVERSION)

    parser.add_argument(
        "--path",
        default=None,
        type=str,
        help=
        "The name of an existing refine_xx folder, where e2refine_easy ran to completion",
        guitype='filebox',
        filecheck=False,
        browser="EMBrowserWidget(withmodal=True,multiselect=False)",
        row=3,
        col=0,
        rowspan=1,
        colspan=3)
    parser.add_argument(
        "--usebasis",
        default=0,
        type=int,
        help=
        "Select which Eigenimage to use for separation. With novarimax, n=0 is highest energy.",
        guitype='intbox',
        row=5,
        col=0,
        rowspan=1,
        colspan=1)
    parser.add_argument(
        "--nbasis",
        default=-1,
        type=int,
        help=
        "Number of basis vectors to compute. Must be at least usebasis+1. Default 6 or usebasis+1.",
        guitype='intbox',
        row=4,
        col=0,
        rowspan=1,
        colspan=1)
    parser.add_argument(
        "--novarimax",
        action="store_true",
        default=False,
        help="Disable varimax rotation among computed basis vectors.",
        guitype='boolbox',
        row=7,
        col=0,
        rowspan=1,
        colspan=1)
    parser.add_argument(
        "--mask",
        default=None,
        help="Optional 3D mask to focus the classification",
        guitype='filebox',
        browser='EMSetsTable(withmodal=True,multiselect=False)',
        filecheck=False,
        row=6,
        col=0,
        rowspan=1,
        colspan=3,
        mode="refinement")
    parser.add_argument("--parallel",
                        default="thread:2",
                        help="Standard parallelism option. Default=thread:2",
                        guitype='strbox',
                        row=8,
                        col=0,
                        rowspan=1,
                        colspan=2)
    parser.add_argument(
        "--verbose",
        "-v",
        dest="verbose",
        action="store",
        metavar="n",
        type=int,
        default=0,
        help=
        "verbose level [0-9], higner number means higher level of verboseness")
    parser.add_argument(
        "--ppid",
        type=int,
        help="Set the PID of the parent process, used for cross platform PPID",
        default=-1)

    (options, args) = parser.parse_args()

    if options.nbasis <= 1:
        options.nbasis = 6
        if options.nbasis <= options.usebasis + 1:
            options.nbasis = options.usebasis + 1
            print "--nbasis adjusted to ", options.nbasis

    if options.path == None:
        paths = [i for i in os.listdir(".") if "refine_" in i and len(i) == 9]
        paths.sort()
        options.path = paths[-1]

    pathnum = options.path[-2:]

    # check the specified path for the files we need
    try:
        olddb = js_open_dict(options.path + "/0_refine_parms.json")
        last_map = olddb["last_map"]
        targetres = olddb["targetres"]
        last_iter = int(last_map.split("_")[-1][:2])
        try:
            ptcls = olddb["inputavg"]
            if ptcls == None: raise Exception
        except:
            ptcls = olddb["input"]

        sym = olddb["sym"]
        if options.verbose:
            print "Found iteration {} in {}, using {}".format(
                last_iter, options.path, " & ".join(ptcls))
    except:
        traceback.print_exc()
        print "Error: Cannot find necessary files in ", options.path
        sys.exit(1)

    logger = E2init(sys.argv, options.ppid)

    # classmx is a list with 2 elements. Each element is a list of EMData from the corresponding cls_result file
    classmx = []
    classmx.append(
        EMData.read_images("{}/cls_result_{:02d}_even.hdf".format(
            options.path, last_iter)))
    classmx.append(
        EMData.read_images("{}/cls_result_{:02d}_odd.hdf".format(
            options.path, last_iter)))
    ncls = max(
        int(classmx[0][0]["maximum"]) + 1,
        int(classmx[1][0]["maximum"]) + 1)

    # Rearrange the info in classmx
    classlists = [[] for i in xrange(ncls)]  # empty list for each class

    # This will produce a list of particles with Transforms for each class
    for eo in (0, 1):
        for y in xrange(classmx[eo][0]["ny"]):
            ptcl = [
                eo, y,
                Transform({
                    "type": "2d",
                    "tx": classmx[eo][2][0, y],
                    "ty": classmx[eo][3][0, y],
                    "alpha": classmx[eo][4][0, y],
                    "mirror": int(classmx[eo][5][0, y])
                })
            ]
            #print ptcl,
            #print int(classmx[eo][0][0,y])
            classlists[int(classmx[eo][0][0, y])].append(ptcl)

    #if len(classlists[0])>100 :
    #print "Warning: this program is normally intended for use with downsampled data and fairly coarse angular sampling. If you try to use it with a large number of class-averages you may have a variety of problems, and should insure that your machine has sufficient RAM."

    # Initialize parallelism
    from EMAN2PAR import EMTaskCustomer
    etc = EMTaskCustomer(options.parallel)

    # Empty image to pad classes file
    zero = EMData(str(ptcls[0]), 0)
    zero.to_zero()
    zero["ptcl_repr"] = 0

    # Euler angles for averages
    projin = "{}/projections_{:02d}_even.hdf".format(options.path, last_iter)
    eulers = [
        EMData(projin, i, True)["xform.projection"] for i in xrange(ncls)
    ]

    # Prepare mask if specified
    if options.mask != None:
        mask = EMData(options.mask)

    else:
        mask = None

    # prepare tasks
    tasks = []
    gc = 0
    ns = [classmx[eo][0]["ny"] for eo in (0, 1)]
    for c, cl in enumerate(classlists):
        if len(
                cl
        ) < 20:  # we require at least 20 particles in a class to make the attempt
            #			zero.write_image(classout[0],c)
            #			zero.write_image(classout[1],c)
            continue
        if mask != None:
            maskp = mask.project("standard", eulers[c])
        else:
            maskp = None
        tasks.append(
            ClassSplitTask(ptcls, ns, cl, c, eulers[c], maskp,
                           options.usebasis, options.nbasis, options.novarimax,
                           options.verbose - 1))
        gc += 1


#	for t in tasks: t.execute()

# execute task list
    taskids = etc.send_tasks(tasks)
    alltaskids = taskids[:]

    classes = []
    while len(taskids) > 0:
        curstat = etc.check_task(taskids)
        for i, j in enumerate(curstat):
            if j == 100:
                rslt = etc.get_results(taskids[i])
                rsltd = rslt[1]
                cls = rslt[0].options["classnum"]
                if rsltd.has_key("failed"):
                    print "Bad average in ", cls
                else:
                    #rsltd["avg1"].write_image(classout[0],cls)
                    #rsltd["avg2"].write_image(classout[1],cls)
                    ncls = rsltd["avg1"]["ptcl_repr"] + rsltd["avg2"][
                        "ptcl_repr"]
                    # note that the 2 results we get back are in arbitrary order!
                    # the next section of code with 3D reconstruction is designed to sort out
                    # which average should be paired with which
                    classes.append([
                        ncls, rsltd["avg1"]["xform.projection"], rsltd["avg1"],
                        rsltd["avg2"], rsltd["basis"], cls
                    ])  # list of (ptcl_repr,xform,avg1,avg2)

        taskids = [j for i, j in enumerate(taskids) if curstat[i] != 100]

        if options.verbose and 100 in curstat:
            print "%d/%d tasks remain" % (len(taskids), len(alltaskids))
        if 100 in curstat:
            E2progress(logger, 1.0 - (float(len(taskids)) / len(alltaskids)))

    if options.verbose:
        print "Completed all tasks\nGrouping consistent averages"

    classes.sort(
        reverse=True)  # we want to start with the largest number of particles
    apix = classes[0][2]["apix_x"]

    boxsize = classes[0][2]["ny"]
    pad = good_size(boxsize * 1.5)
    if options.verbose:
        print "Boxsize -> {}, padding to {}".format(boxsize, pad)

    # a pair of reconstructors. we will then simultaneously reconstruct in the pair, and use each to decide on the best target for each particle
    recon = [
        Reconstructors.get("fourier", {
            "size": [pad, pad, pad],
            "sym": sym,
            "mode": "gauss_5"
        }) for i in (0, 1)
    ]
    for r in recon:
        r.setup()

    # We insert the first class-average (with the most particles) randomly into reconstructor 1 or 2
    p2 = classes[0][2].get_clip(
        Region(-(pad - boxsize) / 2, -(pad - boxsize) / 2, pad, pad))
    p3 = recon[0].preprocess_slice(p2, classes[0][1])
    recon[0].insert_slice(p3, classes[0][1],
                          classes[0][2].get_attr_default("ptcl_repr", 1.0))

    p2 = classes[0][3].get_clip(
        Region(-(pad - boxsize) / 2, -(pad - boxsize) / 2, pad, pad))
    p3 = recon[1].preprocess_slice(p2, classes[0][1])
    recon[1].insert_slice(p3, classes[0][1],
                          classes[0][3].get_attr_default("ptcl_repr", 1.0))

    classes[0].append(0)

    if options.verbose: print "Reconstruction: pass 1"
    for i, c in enumerate(classes[1:]):
        proj = EMData(projin,
                      c[5])  # the projection corresponding to this average
        # while this does cost us a final interpolation, high resolution isn't the primary aim anyway, and getting the alignment consistent is important
        # also gives us a chance to normalize
        c[2]["xform.align2d"] = Transform()
        ali2 = c[2].align("refine", proj)
        ali2.process_inplace("normalize.toimage", {
            "to": proj,
            "ignore_zero": 1
        })
        c[3]["xform.align2d"] = Transform()
        ali3 = c[3].align("refine", proj)
        ali3.process_inplace("normalize.toimage", {
            "to": proj,
            "ignore_zero": 1
        })

        #		print "ROT:\t",ali2["xform.align2d"].get_params("2d"),"\t",ali3["xform.align2d"].get_params("2d")

        # note that ali2 and c[2] are the same except for a final alignment
        a2 = ali2.get_clip(
            Region(-(pad - boxsize) / 2, -(pad - boxsize) / 2, pad,
                   pad))  # first class-average
        a3 = recon[0].preprocess_slice(a2, classes[0][1])
        a3n = c[2].get_attr_default("ptcl_repr", 1.0)

        # similarly ali3 and c[3] are the same
        b2 = ali3.get_clip(
            Region(-(pad - boxsize) / 2, -(pad - boxsize) / 2, pad, pad))
        b3 = recon[1].preprocess_slice(
            b2, classes[0][1]
        )  # I don't believe it matters if we use recon[0] or 1 here, but haven't checked
        b3n = c[3].get_attr_default("ptcl_repr", 1.0)

        recon[0].determine_slice_agreement(a3, c[1], a3n, False)
        #		print a3.get_attr_dict()
        q0a = a3[
            "reconstruct_absqual_lowres"]  # quality for average a in reconstruction0
        #		n0a=a3["reconstruct_norm"]			# normalization for same

        recon[1].determine_slice_agreement(a3, c[1], a3n, False)
        q1a = a3[
            "reconstruct_absqual_lowres"]  # quality for average a in reconstruction0
        #		n1a=a3["reconstruct_norm"]			# normalization for same

        recon[0].determine_slice_agreement(b3, c[1], b3n, False)
        q0b = b3[
            "reconstruct_absqual_lowres"]  # quality for average a in reconstruction0
        #		n0b=b3["reconstruct_norm"]			# normalization for same

        recon[1].determine_slice_agreement(b3, c[1], b3n, False)
        q1b = b3[
            "reconstruct_absqual_lowres"]  # quality for average a in reconstruction0
        #		n1b=b3["reconstruct_norm"]			# normalization for same

        if options.verbose > 1:
            print i, q0a, q1a, q0b, q1b, q0a + q1b, q1a + q0b
        if options.verbose > 2: print "\t\t", n0a, n1a, n0b, n1b

        if q0a + q1b > q1a + q0b:  # if true, a -> recon0 and b -> recon1
            c.append(
                0
            )  # we put a 0 at the end of the classes element if we use a->0,b->1 ordering, 1 if swapped
            #			a3.mult(n0a)
            recon[0].insert_slice(a3, c[1], a3n)
            #			b3.mult(n1b)
            recon[1].insert_slice(b3, c[1], b3n)
        else:
            c.append(1)
            #			a3.mult(n1a)
            recon[1].insert_slice(a3, c[1], a3n)
            #			b3.mult(n0b)
            recon[0].insert_slice(b3, c[1], b3n)

    if options.verbose: print "Reconstruction: pass 2"

    # another pass with the filled reconstruction to make sure our initial assignments were ok
    #	for i,c in enumerate(classes[1:]):
    #		a2=c[2].get_clip(Region(-(pad-boxsize)/2,-(pad-boxsize)/2,pad,pad))		# first class-average
    #		a3=recon[0].preprocess_slice(a2,classes[0][1])
    #		a3n=c[2].get_attr_default("ptcl_repr",1.0)
    #
    #		b2=c[3].get_clip(Region(-(pad-boxsize)/2,-(pad-boxsize)/2,pad,pad))
    #		b3=recon[1].preprocess_slice(b2,classes[0][1])						# I don't believe it matters if we use recon[0] or 1 here, but haven't checked
    #		b3n=c[3].get_attr_default("ptcl_repr",1.0)
    #
    #		recon[0].determine_slice_agreement(a3,c[1],a3n,0) # c[-1]==0
    #		q0a=a3["reconstruct_absqual"]			# quality for average a in reconstruction0
    #		n0a=a3["reconstruct_norm"]			# normalization for same
    #
    #		recon[1].determine_slice_agreement(a3,c[1],a3n,0) # c[-1]==1
    #		q1a=a3["reconstruct_absqual"]			# quality for average a in reconstruction0
    #		n1a=a3["reconstruct_norm"]			# normalization for same
    #
    #		recon[0].determine_slice_agreement(b3,c[1],b3n,0) # c[-1]==1
    #		q0b=b3["reconstruct_absqual"]			# quality for average a in reconstruction0
    #		n0b=b3["reconstruct_norm"]			# normalization for same
    #
    #		recon[1].determine_slice_agreement(b3,c[1],b3n,0) # c[-1]==0
    #		q1b=b3["reconstruct_absqual"]			# quality for average a in reconstruction0
    #		n1b=b3["reconstruct_norm"]			# normalization for same
    #
    #		if options.verbose>1 : print i,q0a,q1a,q0b,q1b,q0a+q1b,q1a+q0b
    #
    #		if q0a+q1b>q1a+q0b :		# if true, a -> recon0 and b -> recon1
    #			if c[-1]==1 :
    #				c[-1]=0
    #				print i," 1->0"
    #
    #			c.append(0)				# we put a 0 at the end of the classes element if we use a->0,b->1 ordering, 1 if swapped
    #			a3.mult(n0a)
    #			recon[0].insert_slice(a3,c[1],a3n)
    #			b3.mult(n1b)
    #			recon[1].insert_slice(b3,c[1],b3n)
    #		else:
    #			if c[-1]==0 :
    #				c[-1]=1
    #				print i," 0->1"
    #
    #			c.append(1)
    #			a3.mult(n1a)
    #			recon[1].insert_slice(a3,c[1],a3n)
    #			b3.mult(n0b)
    #
    #
    if options.verbose: print "All done, writing output"

    if mask != None: msk = "_msk"
    else: msk = ""
    classout = [
        "{}/classes_{:02d}_bas{}{}_split0.hdf".format(options.path, last_iter,
                                                      options.usebasis, msk),
        "{}/classes_{:02d}_bas{}{}_split1.hdf".format(options.path, last_iter,
                                                      options.usebasis, msk)
    ]
    basisout = "{}/classes_{:02d}{}_basis".format(options.path, last_iter, msk)
    threedout = "{}/threed_{:02d}{}_split.hdf".format(options.path, last_iter,
                                                      msk)
    threedout2 = "{}/threed_{:02d}{}_split_filt_bas{}.hdf".format(
        options.path, last_iter, msk, options.usebasis)
    setout = [
        "sets/split_{}{}_bas{}_0.lst".format(pathnum, msk, options.usebasis),
        "sets/split_{}{}_bas{}_1.lst".format(pathnum, msk, options.usebasis)
    ]
    split = [
        r.finish(True).get_clip(
            Region((pad - boxsize) / 2, (pad - boxsize) / 2,
                   (pad - boxsize) / 2, boxsize, boxsize, boxsize))
        for r in recon
    ]
    split[0]["apix_x"] = apix
    split[0]["apix_y"] = apix
    split[0]["apix_z"] = apix
    split[1]["apix_x"] = apix
    split[1]["apix_y"] = apix
    split[1]["apix_z"] = apix
    split[0].process_inplace("mask.soft", {"outer_radius": -8, "width": 4})
    split[1].process_inplace("mask.soft", {"outer_radius": -8, "width": 4})
    split[0].write_image(threedout, 0)
    split[1].write_image(threedout, 1)

    # now we write the class-averages and the new (split) particle files
    lstin = [LSXFile(ptcls[0], True), LSXFile(ptcls[1], True)]
    try:
        os.unlink("sets/split0.lst")
        os.unlink("sets/split1.lst")
    except:
        pass
    lstout = [LSXFile("sets/split0.lst"), LSXFile("sets/split1.lst")]
    for i, c in enumerate(classes):
        c[2].write_image(classout[c[-1]], i)  # class-average
        ptcln = c[2]["class_eoidxs"]  # eofile/ptcl# pairs
        for p in xrange(0, len(ptcln), 2):
            lstout[0][-1] = lstin[ptcln[p]][ptcln[
                p + 1]]  # wierd syntax, but the -1 here appends

        c[3].write_image(classout[c[-1] ^ 1], i)
        ptcln = c[3]["class_eoidxs"]  # eofile/ptcl# pairs
        for p in xrange(0, len(ptcln), 2):
            lstout[1][-1] = lstin[ptcln[p]][ptcln[
                p + 1]]  # wierd syntax, but the -1 here appends

        if options.verbose > 2:
            c[4][0].write_image(basisout + "1.hdf", i)
            c[4][1].write_image(basisout + "2.hdf", i)
            c[4][2].write_image(basisout + "3.hdf", i)

    launch_childprocess("e2proclst.py sets/split0.lst --mergesort {}".format(
        setout[0]))
    launch_childprocess("e2proclst.py sets/split1.lst --mergesort {}".format(
        setout[1]))

    try:
        os.unlink("sets/split0.lst")
        os.unlink("sets/split1.lst")
    except:
        pass

    if os.path.exists("strucfac.txt"):
        launch_childprocess(
            "e2proc3d.py {} {} --setsf strucfac.txt --process filter.wiener.byfsc:fscfile={}/fsc_masked_{:02d}.txt:snrmult=2:sscale=1.1:maxfreq={} --process mask.soft:outer_radius=-9:width=4"
            .format(threedout, threedout2, options.path, last_iter,
                    1.0 / targetres))
    else:
        print "Missing structure factor, cannot filter properly"
        launch_childprocess(
            "e2proc3d.py {} {} --process filter.wiener.byfsc:fscfile={}/fsc_masked_{:02d}.txt:snrmult=2:sscale=1.1:maxfreq={} --process mask.soft:outer_radius=-9:width=4"
            .format(threedout, threedout2, options.path, last_iter,
                    1.0 / targetres))

    E2end(logger)
예제 #6
0
def main():

    usage = " "
    parser = EMArgumentParser(usage=usage, version=EMANVERSION)

    parser.add_argument("--ptclin",
                        type=str,
                        help="particle input",
                        default=None)
    parser.add_argument("--ptclout",
                        type=str,
                        help="particle output",
                        default=None)
    parser.add_argument("--ref",
                        type=str,
                        help="reference input",
                        default=None)
    parser.add_argument(
        "--parallel",
        type=str,
        help="Thread/mpi parallelism to use. Default is thread:12",
        default="thread:12")

    parser.add_argument(
        "--debug",
        action="store_true",
        default=False,
        help=
        "Turn on debug mode. This will only process a small subset of the data"
    )
    #parser.add_argument("--maxshift", type=int,help="maximum shift allowed", default=-1)
    parser.add_argument(
        "--localrefine",
        type=int,
        default=-1,
        help="local refinement. larger value correspond to smaller local region"
    )
    parser.add_argument("--goldcontinue",
                        action="store_true",
                        default=False,
                        help="split even/odd subset and references.")
    #parser.add_argument("--ctfweight", action="store_true", default=False ,help="weight by ctf. not used yet...")
    #parser.add_argument("--slow", action="store_true", default=False ,help="slow but finer search")
    parser.add_argument("--maxres",
                        type=float,
                        default=-1,
                        help="max resolution for cmp")
    parser.add_argument("--minrespx",
                        type=int,
                        default=4,
                        help="skip the first x pixel in fourier space")
    parser.add_argument("--sym", type=str, help="symmetry. ", default="c1")
    parser.add_argument("--ppid", type=int, help="ppid...", default=-1)
    parser.add_argument("--verbose", "-v", type=int, help="Verbose", default=0)

    (options, args) = parser.parse_args()
    logid = E2init(sys.argv)

    #m=EMData(options.ref)
    #bxsz=m["nx"]
    #apix=m["apix_x"]

    options.shrink = 1
    pinfo = load_lst_params(options.ptclin)
    nptcl = len(pinfo)
    #if options.maxshift<0:
    #options.maxshift=bxsz//2

    print("Initializing parallelism...")
    etc = EMTaskCustomer(options.parallel, module="e2spa_align.SpaAlignTask")
    num_cpus = etc.cpu_est()

    print("{} particles".format(nptcl))
    print("{} total CPUs available".format(num_cpus))
    if options.debug:
        nptcl = min(4 * num_cpus, nptcl)
        print("Debugging mode. running on one thread with 8 particles")

    nbatch = min(nptcl // 4, num_cpus)

    infos = [[] for i in range(nbatch)]
    for i, info in enumerate(pinfo):
        infos[i % nbatch].append([i, info])

    print("{} jobs, each with {:.1f} particles".format(
        len(infos), np.mean([len(i) for i in infos])))

    tids = []
    for info in infos:
        task = SpaAlignTask(info, options)

        if options.debug:
            task.execute(print)
            return
        tid = etc.send_task(task)
        tids.append(tid)

    while 1:
        st_vals = etc.check_task(tids)
        if -100 in st_vals:
            print("Error occurs in parallelism. Exit")
            return
        E2progress(logid, np.mean(st_vals) / 100.)

        if np.min(st_vals) == 100: break
        time.sleep(5)

    output = [None] * nptcl
    for i in tids:
        ret = etc.get_results(i)[1]
        for r in ret:
            output[r[0]] = r[1]

    del etc

    fm = options.ptclout
    save_lst_params(output, fm)

    E2end(logid)
예제 #7
0
def main():
    progname = os.path.basename(sys.argv[0])
    usage = """Usage: e2spt_align.py [options] <subvolume_stack> <reference>
This program is part of the 'new' hierarchy of e2spt_ programs. It performs one iteration of a classical subtomogram refinement, ie -  aligning particles with missing wedge to a reference in 3-D

The reference may be <volume> or <volume>,<n>

If --goldstandard is specified, then even and odd particles will be aligned to different perturbed versions of the reference volume, phase-randomized past the specified resolution."""

    parser = EMArgumentParser(usage=usage, version=EMANVERSION)

    parser.add_argument(
        "--threads",
        default=4,
        type=int,
        help=
        "Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present.",
        guitype='intbox',
        row=24,
        col=2,
        rowspan=1,
        colspan=1,
        mode="refinement")
    parser.add_argument(
        "--iter",
        type=int,
        help="Iteration number within path. Default = start a new iteration",
        default=0)
    parser.add_argument(
        "--goldstandard",
        type=float,
        help=
        "If specified, will phase randomize the even and odd references past the specified resolution (in A, not 1/A)",
        default=0)
    parser.add_argument(
        "--goldcontinue",
        action="store_true",
        help=
        "Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",
        default=False)
    #parser.add_argument("--saveali",action="store_true",help="Save a stack file (aliptcls.hdf) containing the aligned subtomograms.",default=False)
    #parser.add_argument("--savealibin",type=int,help="shrink aligned particles before saving",default=1)
    parser.add_argument(
        "--path",
        type=str,
        default=None,
        help=
        "Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)"
    )
    parser.add_argument(
        "--sym",
        type=str,
        default="c1",
        help=
        "Symmetry of the input. Must be aligned in standard orientation to work properly."
    )
    parser.add_argument(
        "--maxres",
        type=float,
        help="Maximum resolution to consider in alignment (in A, not 1/A)",
        default=0)
    #parser.add_argument("--wtori",type=float,help="Weight for using the prior orientation in the particle header. default is -1, i.e. not used.",default=-1)
    parser.add_argument(
        "--nsoln",
        type=int,
        help="number of solutions to keep at low resolution for the aligner",
        default=1)
    parser.add_argument(
        "--verbose",
        "-v",
        dest="verbose",
        action="store",
        metavar="n",
        type=int,
        default=0,
        help=
        "verbose level [0-9], higher number means higher level of verboseness")
    parser.add_argument(
        "--ppid",
        type=int,
        help="Set the PID of the parent process, used for cross platform PPID",
        default=-1)
    parser.add_argument("--parallel",
                        type=str,
                        help="Thread/mpi parallelism to use",
                        default=None)
    parser.add_argument("--refine",
                        action="store_true",
                        help="local refinement from xform.align3d in header.",
                        default=False)

    parser.add_argument("--refinentry",
                        type=int,
                        help="number of tests for refine mode. default is 8",
                        default=8)
    parser.add_argument("--randphi",
                        action="store_true",
                        help="randomize phi during refine alignment",
                        default=False)
    parser.add_argument("--breaksym",
                        action="store_true",
                        help="symmetry breaking.",
                        default=False)
    parser.add_argument(
        "--breaksymsym",
        type=str,
        help=
        "the symmetry to use for breaksym. setting sym to c6 and this to c2 results in a c3 structure. default is the same as sym",
        default=None)
    parser.add_argument(
        "--rand180",
        action="store_true",
        help="randomly add a 180 degree rotation during refine alignment",
        default=False)
    parser.add_argument(
        "--skipali",
        action="store_true",
        help=
        "skip alignment. the program will do nothing. mostly for testing...",
        default=False)
    parser.add_argument(
        "--maxang",
        type=float,
        help="Maximum angular difference for the refine mode. default is 30",
        default=30)
    parser.add_argument(
        "--maxshift",
        type=float,
        help="Maximum shift for the refine mode. default is 16",
        default=-1)

    (options, args) = parser.parse_args()

    #task=SptAlignTask(0,1,2,options)
    #from pickle import dumps,loads,dump,load
    #f=open("task.tmp",'w')
    #dump(task,f)
    #f.close()
    #print(task)

    #return

    if options.path == None:
        fls = [
            int(i[-2:]) for i in os.listdir(".")
            if i[:4] == "spt_" and len(i) == 6 and str.isdigit(i[-2:])
        ]
        if len(fls) == 0: fls = [0]
        options.path = "spt_{:02d}".format(max(fls) + 1)
        try:
            os.mkdir(options.path)
        except:
            pass

    if options.iter <= 0:
        fls = [
            int(i[15:17]) for i in os.listdir(options.path)
            if i[:15] == "particle_parms_" and str.isdigit(i[15:17])
        ]
        if len(fls) == 0: options.iter = 1
        else: options.iter = max(fls) + 1

    if options.parallel == None:
        options.parallel = "thread:{}".format(options.threads)

    if options.breaksym:
        if options.breaksymsym == None:
            if options.sym == "c1":
                print("cannot break a c1 symmetry. breaksym disabled.")
                options.breaksym = False
            else:
                options.breaksymsym = options.sym

    # file may be "name" or "name,#"
    reffile = args[1].split(",")[0]
    try:
        refn = int(args[1].split(",")[1])
    except:
        refn = 0

    NTHREADS = max(options.threads + 1,
                   2)  # we have one thread just writing results

    logid = E2init(sys.argv, options.ppid)
    refnames = []

    if options.goldcontinue:
        ref = []
        try:
            refnames = [reffile[:-4] + "_even.hdf", reffile[:-4] + "_odd.hdf"]
            ref.append(EMData(refnames[0], 0))
            ref.append(EMData(refnames[1], 0))

        except:
            print("Error: cannot find one of reference files, eg: ",
                  EMData(reffile[:-4] + "_even.hdf", 0))
    else:
        ref = []
        ref.append(EMData(reffile, refn))
        ref.append(EMData(reffile, refn))

        if options.goldstandard > 0:
            ref[0].process_inplace(
                "filter.lowpass.randomphase",
                {"cutoff_freq": old_div(1.0, options.goldstandard)})
            ref[0].process_inplace(
                "filter.lowpass.gauss",
                {"cutoff_freq": old_div(1.0, options.goldstandard)})
            ref[1].process_inplace(
                "filter.lowpass.randomphase",
                {"cutoff_freq": old_div(1.0, options.goldstandard)})
            ref[1].process_inplace(
                "filter.lowpass.gauss",
                {"cutoff_freq": old_div(1.0, options.goldstandard)})
            refnames = [
                "{}/align_ref_even.hdf".format(options.path),
                "{}/align_ref_odd.hdf".format(options.path)
            ]
            ref[0].write_image(refnames[0], 0)
            ref[1].write_image(refnames[1], 0)

        else:
            refnames = [reffile, reffile]

    #ref[0]=ref[0].do_fft()
    #ref[0].process_inplace("xform.phaseorigin.tocorner")
    #ref[1]=ref[1].do_fft()
    #ref[1].process_inplace("xform.phaseorigin.tocorner")

    #jsd=queue.Queue(0)

    n = -1
    tasks = []
    if args[0].endswith(".lst") or args[0].endswith(".hdf"):
        #### check if even/odd split exists
        fsps = [args[0][:-4] + "__even.lst", args[0][:-4] + "__odd.lst"]

        if os.path.isfile(fsps[0]) and os.path.isfile(fsps[1]):
            print("Using particle list: \n\t {} \n\t {}".format(
                fsps[0], fsps[1]))
            for eo, f in enumerate(fsps):
                N = EMUtil.get_image_count(f)
                tasks.extend([(f, i, refnames, eo) for i in range(N)])

        #### split by even/odd by default
        else:
            N = EMUtil.get_image_count(args[0])
            tasks.extend([(args[0], i, refnames, i % 2) for i in range(N)])
            #thrds=[threading.Thread(target=alifn,args=(jsd,args[0],i,ref[i%2],options)) for i in range(N)]

    elif args[0].endswith(".json"):
        print("Reading particles from json. This is experimental...")
        js = js_open_dict(args[0])
        keys = sorted(js.keys())
        for k in keys:
            src, ii = eval(k)
            dic = js[k]
            xf = dic["xform.align3d"]
            tasks.append([src, ii, refnames, ii % 2, xf])

    from EMAN2PAR import EMTaskCustomer
    etc = EMTaskCustomer(options.parallel, module="e2spt_align.SptAlignTask")
    num_cpus = etc.cpu_est()
    options.nowtime = time.time()
    print("{} jobs on {} CPUs".format(len(tasks), num_cpus))
    njob = num_cpus  #*4

    tids = []
    for i in range(njob):
        t = tasks[i::njob]
        task = SptAlignTask(t, options)
        tid = etc.send_task(task)
        tids.append(tid)

    #print("starting...")
    #for t in tasks:
    #if len(t)>3:
    #task = SptAlignTask(t[0], t[1], t[2], options, t[3])
    #else:
    #task = SptAlignTask(t[0], t[1], t[2], options)
    #tid=etc.send_task(task)
    #tids.append(tid)

    while 1:
        st_vals = etc.check_task(tids)
        if -100 in st_vals:
            print("Error occurs in parallelism. Exit")
            return
        E2progress(logid, np.mean(st_vals) / 100.)

        if np.min(st_vals) == 100: break
        time.sleep(5)

    #dics=[0]*nptcl

    angs = {}
    for i in tids:
        rets = etc.get_results(i)[1]
        for ret in rets:
            fsp, n, dic = ret
            if len(dic) == 1:
                angs[(fsp, n)] = dic[0]
            else:
                angs[(fsp, n)] = dic

    out = "{}/particle_parms_{:02d}.json".format(options.path, options.iter)
    if os.path.isfile(out):
        os.remove(out)
    js = js_open_dict(out)
    js.update(angs)
    js.close()

    del etc

    ## here we run the threads and save the results, no actual alignment done here
    #print(len(thrds)," threads")
    #thrtolaunch=0
    #while thrtolaunch<len(thrds) or threading.active_count()>1:
    ## If we haven't launched all threads yet, then we wait for an empty slot, and launch another
    ## note that it's ok that we wait here forever, since there can't be new results if an existing
    ## thread hasn't finished.
    #if thrtolaunch<len(thrds) :
    #while (threading.active_count()==NTHREADS ) : time.sleep(.1)
    #if options.verbose : print("Starting thread {}/{}".format(thrtolaunch,len(thrds)))
    #thrds[thrtolaunch].start()
    #thrtolaunch+=1
    #else: time.sleep(1)

    #while not jsd.empty():
    #fsp,n,d=jsd.get()
    #angs[(fsp,n)]=d
    #if options.saveali:
    #v=EMData(fsp,n)
    #v.transform(d["xform.align3d"])
    #if options.savealibin>1:
    #v.process_inplace("math.meanshrink",{"n":options.savealibin})
    #v.write_image("{}/aliptcls_{:02d}.hdf".format(options.path, options.iter),n)

    #for t in thrds:
    #t.join()

    E2end(logid)
예제 #8
0
def main():

    usage = " "
    parser = EMArgumentParser(usage=usage, version=EMANVERSION)

    parser.add_argument("--ptclin",
                        type=str,
                        help="particle input",
                        default=None)
    parser.add_argument("--ptclout",
                        type=str,
                        help="particle output",
                        default=None)
    parser.add_argument("--ref",
                        type=str,
                        help="reference input",
                        default=None)
    parser.add_argument("--threedout",
                        type=str,
                        help="map output",
                        default=None)

    parser.add_argument("--keep",
                        type=float,
                        help="propotion of tilts to keep. default is 0.5",
                        default=0.5)

    parser.add_argument("--threads",
                        type=int,
                        help="Number of CPU threads to use. Default is 12.",
                        default=12)
    parser.add_argument(
        "--parallel",
        type=str,
        help="Thread/mpi parallelism to use. Default is thread:12",
        default="thread:12")

    parser.add_argument(
        "--debug",
        action="store_true",
        default=False,
        help=
        "Turn on debug mode. This will only process a small subset of the data (threads * 8 particles)"
    )

    parser.add_argument("--transonly",
                        action="store_true",
                        default=False,
                        help="only refine translation")
    parser.add_argument("--savepath",
                        action="store_true",
                        default=False,
                        help="save alignment path in a json file for testing.")
    #parser.add_argument("--scipytest", action="store_true", default=False ,help="test scipy optimizer")
    parser.add_argument(
        "--fromscratch",
        action="store_true",
        default=False,
        help=
        "align from scratch and ignore previous particle transforms. for spt mostly. will include mirror"
    )
    parser.add_argument("--refineastep",
                        type=float,
                        help="Mean angular variation for refine alignment",
                        default=2.)
    parser.add_argument("--refinentry",
                        type=int,
                        help="number of starting points for refine alignment",
                        default=4)
    parser.add_argument("--maxshift",
                        type=int,
                        help="maximum shift allowed",
                        default=8)
    parser.add_argument("--localrefine",
                        action="store_true",
                        default=False,
                        help="local refinement")
    parser.add_argument("--defocus",
                        action="store_true",
                        default=False,
                        help="refine defocus. Still under development")
    parser.add_argument("--seedmap",
                        action="store_true",
                        default=False,
                        help="seed")
    parser.add_argument("--ctfweight",
                        action="store_true",
                        default=False,
                        help="weight by ctf")
    parser.add_argument("--skipm3d",
                        action="store_true",
                        default=False,
                        help="skip make3d. only output aligned list")

    parser.add_argument("--padby",
                        type=float,
                        default=2.0,
                        help="pad by factor. default is 2")
    parser.add_argument("--maxres",
                        type=float,
                        default=-1,
                        help="max resolution for cmp")
    parser.add_argument("--minres",
                        type=float,
                        default=-1,
                        help="min resolution for cmp")
    parser.add_argument(
        "--sym",
        type=str,
        help="symmetry. will use symmetry from spt refinement by default",
        default="c1")
    parser.add_argument(
        "--smooth",
        type=int,
        help=
        "Smooth trajectory per image based on nearby particles. Still under development",
        default=-1)
    parser.add_argument("--ppid", type=int, help="ppid...", default=-1)
    parser.add_argument("--nkeep", type=int, help="", default=1)
    parser.add_argument("--verbose", "-v", type=int, help="Verbose", default=0)

    (options, args) = parser.parse_args()
    logid = E2init(sys.argv)

    lstname = options.ptclin
    threedname = options.ref
    lname = options.ptclout
    threedout = options.threedout

    lst = LSXFile(lstname, True)
    m = EMData(threedname)
    bxsz = m["nx"]
    apix = m["apix_x"]

    options.shrink = 1
    pinfo = []
    if options.debug: nptcl = options.threads * 8
    else: nptcl = lst.n
    for i in range(nptcl):
        pinfo.append(lst.read(i))
    lst = None

    print("Initializing parallelism...")
    etc = EMTaskCustomer(options.parallel,
                         module="e2spt_tiltrefine_oneiter.SptTltRefineTask")

    num_cpus = etc.cpu_est()

    print("{} total CPUs available".format(num_cpus))
    print("{} jobs".format(nptcl))
    infos = [[] for i in range(num_cpus)]
    for i, info in enumerate(pinfo):
        infos[i % num_cpus].append([i, info])

    tids = []
    for info in infos:
        task = SptTltRefineTask(info, threedname, options)

        if options.debug:
            task.execute(print)
            return
        tid = etc.send_task(task)
        tids.append(tid)

    while 1:
        st_vals = etc.check_task(tids)
        if -100 in st_vals:
            print("Error occurs in parallelism. Exit")
            return
        E2progress(logid, np.mean(st_vals) / 100.)

        if np.min(st_vals) == 100: break
        time.sleep(5)

    dics = [0] * nptcl
    for i in tids:
        ret = etc.get_results(i)[1]
        for r in ret:
            #print(r)
            ii = r.pop("idx")
            dics[ii] = r

    del etc

    allscr = [d["score"] for d in dics]
    if options.smooth > 0 or options.defocus:
        ### need to add per tilt smoothing later...
        s = np.array(allscr)
        oname = lname.replace(".lst", "_score.hdf")
        ss = from_numpy(s).copy()
        ss.write_image(oname)
        return

    maxl = np.max([len(s) for s in allscr])
    maxv = np.max(np.concatenate(allscr))
    for s in allscr:
        s.extend([maxv] * (maxl - len(s)))
    allscr = np.array(allscr)

    #print(np.min(allscr), np.mean(allscr), np.max(allscr), np.std(allscr))
    if options.skipm3d:
        pass
    else:
        allscr = 2 - allscr
        allscr -= np.min(allscr)
        allscr /= np.max(allscr)

        if maxl > 1:
            mx = np.max(allscr, axis=1)[:, None]
            allscr = np.exp(allscr * 20)
            allscr = allscr / np.sum(allscr, axis=1)[:, None]
            allscr *= mx

    try:
        os.remove(lname)
    except:
        pass
    lout = LSXFile(lname, False)
    for i, dc in enumerate(dics):
        lc = ""

        if isinstance(dc["xform.align3d"], list):
            alilist = dc["xform.align3d"]
            scorelist = dc["score"]
        else:
            alilist = [dc["xform.align3d"]]
            scorelist = [dc["score"]]
        for j, xf in enumerate(alilist):
            d = xf.get_params("eman")
            d["score"] = float(allscr[i, j])
            if d["score"] > .05 or j == 0:
                lc = lc + str(d) + ';'

        l = pinfo[i]
        lout.write(-1, l[0], l[1], lc[:-1])

    lout = None

    pb = options.padby

    if options.parallel.startswith("mpi") and len(dics) > 10000:
        m3dpar = "--parallel {}".format(options.parallel)
    else:
        m3dpar = ""

    if options.seedmap:
        seed = "--seedmap " + threedname
    else:
        seed = ""

    cmd = "e2make3dpar.py --input {inp} --output {out} --pad {pd} --padvol {pdv} --threads {trd} --outsize {bx} --apix {apx} --mode trilinear --keep {kp} --sym {sm} {seed} {par}".format(
        inp=lname,
        out=threedout,
        bx=bxsz,
        pd=int(bxsz * pb),
        pdv=int(bxsz * pb),
        apx=apix,
        kp=options.keep,
        sm=options.sym,
        trd=options.threads,
        par=m3dpar,
        seed=seed)

    if options.skipm3d:
        print("Skipping 3D reconstruction")
    else:
        run(cmd)

    E2end(logid)
예제 #9
0
파일: e2project3d.py 프로젝트: cryoem/test
class EMParallelProject3D:
    def __init__(self, options, fsp, sym, start, modeln=0, logger=None):
        """
		@param options the options produced by (options, args) = parser.parse_args()
		@param args the options produced by (options, args) = parser.parse_args()
		@param logger and EMAN2 logger, i.e. logger=E2init(sys.argv)
		assumes you have already called the check function.
		"""
        self.options = options
        self.args = fsp
        self.sym = sym
        self.logger = logger
        self.start = start
        self.modeln = modeln

        from EMAN2PAR import EMTaskCustomer

        self.etc = EMTaskCustomer(options.parallel)
        print "Precache ", fsp
        self.etc.precache([fsp])

        self.num_cpus = self.etc.cpu_est()
        print self.num_cpus, " total CPUs available"
        if self.num_cpus > 64:  # upper limit
            self.num_cpus = 64

        self.__task_options = None

    def __init_memory(self, options):
        """

		"""
        sym_object = parsesym(self.sym)
        [og_name, og_args] = parsemodopt(options.orientgen)
        self.eulers = sym_object.gen_orientations(og_name, og_args)

    def __get_task_options(self, options):

        if self.__task_options == None:
            d = {}
            d["projector"] = parsemodopt(options.projector)
            d["prethreshold"] = options.prethreshold
            self.__task_options = d

        return self.__task_options

    def execute(self):
        # 		from EMAN2PAR import EMTaskCustomer

        if len(self.options.parallel) > 1:
            self.__init_memory(self.options)

            num_tasks = self.num_cpus
            # In the worst case we can only spawn as many tasks as there are eulers
            if self.num_cpus > len(self.eulers):
                num_tasks = len(self.eulers)

            eulers_per_task = len(self.eulers) / num_tasks
            resid_eulers = len(self.eulers) - eulers_per_task * num_tasks  # we can distribute the residual evenly

            first = 0
            task_customers = []
            tids = []
            # 			self.etc=EMTaskCustomer(self.options.parallel)
            for i in xrange(0, num_tasks):
                last = first + eulers_per_task
                if resid_eulers > 0:
                    last += 1
                    resid_eulers -= 1

                tmp_eulers = self.eulers[first:last]
                indices = range(first, last)

                data = {}
                data["input"] = ("cache", self.args, 0)
                data["eulers"] = tmp_eulers
                data["indices"] = indices

                task = EMProject3DTaskDC(data=data, options=self.__get_task_options(self.options))

                # print "Est %d CPUs"%etc.cpu_est()
                tid = self.etc.send_task(task)
                # print "Task submitted tid=",tid

                tids.append(tid)

                first = last

            print "Task ids are", tids

            while 1:

                print len(tids), "projection tasks left in main loop"
                st_vals = self.etc.check_task(tids)
                for i in xrange(len(tids) - 1, -1, -1):
                    st = st_vals[i]
                    if st == 100:
                        tid = tids[i]

                        rslts = self.etc.get_results(tid)

                        if not self.__write_output_data(rslts[1]):
                            print "There was a problem with the task of id", tid

                        if self.logger != None:
                            E2progress(self.logger, 1.0 - len(tids) / float(num_tasks))
                            if self.options.verbose > 0:
                                print "%d/%d\r" % (num_tasks - len(tids), num_tasks)
                                sys.stdout.flush()

                        print "Task", tids.pop(i), "completed"
                        print "These tasks are remaining:", tids

                if len(tids) == 0:
                    break
                time.sleep(5)

            return len(self.eulers)
        else:
            raise NotImplementedError(
                "The parallelism option you specified (%s) is not suppored" % self.options.parallel
            )

    def __write_output_data(self, rslts):
        for idx, image in rslts.items():
            if not isinstance(image, EMData):
                continue  # this is here because we get the dimensions of the database as a key (e.g. '40x40x1').
            image["model_id"] = self.modeln
            if self.options.append:
                image.write_image(self.options.outfile, -1)
            else:
                image.write_image(self.options.outfile, idx + self.start)

        return True
예제 #10
0
def main():

    usage = " "
    parser = EMArgumentParser(usage=usage, version=EMANVERSION)

    parser.add_argument("--ptclin",
                        type=str,
                        help="particle input",
                        default=None)
    parser.add_argument("--ptclout",
                        type=str,
                        help="particle output",
                        default=None)
    parser.add_argument("--ref",
                        type=str,
                        help="reference input",
                        default=None)
    parser.add_argument("--threedout",
                        type=str,
                        help="map output",
                        default=None)

    parser.add_argument("--keep",
                        type=float,
                        help="propotion of tilts to keep. default is 0.5",
                        default=0.5)

    parser.add_argument("--threads",
                        type=int,
                        help="Number of CPU threads to use. Default is 12.",
                        default=12)
    parser.add_argument(
        "--parallel",
        type=str,
        help="Thread/mpi parallelism to use. Default is thread:12",
        default="thread:12")

    parser.add_argument(
        "--debug",
        action="store_true",
        default=False,
        help=
        "Turn on debug mode. This will only process a small subset of the data (threads * 8 particles)"
    )

    parser.add_argument("--transonly",
                        action="store_true",
                        default=False,
                        help="only refine translation")
    parser.add_argument(
        "--fromscratch",
        action="store_true",
        default=False,
        help=
        "align from scratch and ignore previous particle transforms. for spt mostly. will include mirror"
    )
    parser.add_argument("--refineastep",
                        type=float,
                        help="Mean angular variation for refine alignment",
                        default=2.)
    parser.add_argument("--refinentry",
                        type=int,
                        help="number of starting points for refine alignment",
                        default=4)
    parser.add_argument("--maxshift",
                        type=int,
                        help="maximum shift allowed",
                        default=8)

    parser.add_argument("--padby",
                        type=float,
                        default=2.0,
                        help="pad by factor. default is 2")
    parser.add_argument("--maxres",
                        type=float,
                        default=-1,
                        help="max resolution for cmp")
    parser.add_argument(
        "--sym",
        type=str,
        help="symmetry. will use symmetry from spt refinement by default",
        default="c1")
    parser.add_argument("--ppid", type=int, help="ppid...", default=-1)

    (options, args) = parser.parse_args()
    logid = E2init(sys.argv)

    lstname = options.ptclin
    threedname = options.ref
    lname = options.ptclout
    threedout = options.threedout

    lst = LSXFile(lstname, True)
    m = EMData(threedname)
    bxsz = m["nx"]
    apix = m["apix_x"]
    if options.maxres > 0:
        options.shrink = max(1, int(options.maxres / apix * .3))
        options.shrink = min(options.shrink, bxsz // 48)
        print("Will shrink by {} and filter to {:.0f} A. Box size {}".format(
            options.shrink, options.maxres, bxsz // options.shrink))
    else:
        options.shrink = 1
    #m.process_inplace('normalize.edgemean')

    pinfo = []
    if options.debug: nptcl = options.threads * 8
    else: nptcl = lst.n
    for i in range(nptcl):
        pinfo.append(lst.read(i))
    lst = None

    print("Initializing parallelism...")
    etc = EMTaskCustomer(options.parallel,
                         module="e2spt_tiltrefine_oneiter.SptTltRefineTask")
    num_cpus = etc.cpu_est()

    print("{} total CPUs available".format(num_cpus))
    print("{} jobs".format(nptcl))

    infos = [[] for i in range(num_cpus)]
    for i, info in enumerate(pinfo):
        infos[i % num_cpus].append([i, info])

    tids = []
    for info in infos:
        task = SptTltRefineTask(info, threedname, options)
        tid = etc.send_task(task)
        tids.append(tid)

    while 1:
        st_vals = etc.check_task(tids)
        #print(st_vals)
        if -100 in st_vals:
            print("Error occurs in parallelism. Exit")
            return
        E2progress(logid, np.mean(st_vals) / 100.)

        if np.min(st_vals) == 100: break
        time.sleep(5)

    dics = [0] * nptcl
    for i in tids:
        ret = etc.get_results(i)[1]
        for r in ret:
            #print(r)
            ii = r.pop("idx")
            dics[ii] = r

    del etc

    allscr = np.array([d["score"] for d in dics])
    print(np.min(allscr), np.mean(allscr), np.max(allscr), np.std(allscr))
    allscr = 2 - allscr
    s = allscr.copy()
    s -= np.mean(s)
    s /= np.std(s)
    clp = 2
    ol = abs(s) > clp
    print("Removing {} outliers from {} particles..".format(
        np.sum(ol), len(s)))
    s = (s + clp) / clp / 2
    s[ol] = 0
    allscr = s
    allscr -= np.min(allscr) - 1e-5
    allscr /= np.max(allscr)

    try:
        os.remove(lname)
    except:
        pass
    lout = LSXFile(lname, False)
    for i, dc in enumerate(dics):
        d = dc["xform.align3d"].get_params("eman")
        d["score"] = float(allscr[i])
        l = pinfo[i]
        lout.write(-1, l[0], l[1], str(d))

    lout = None

    pb = options.padby

    if options.parallel.startswith("mpi") and len(dics) > 10000:
        m3dpar = "--parallel {}".format(options.parallel)
    else:
        m3dpar = ""
    cmd = "e2make3dpar.py --input {inp} --output {out} --pad {pd} --padvol {pdv} --threads {trd} --outsize {bx} --apix {apx} --mode gauss_2 --keep {kp} --sym {sm} {par}".format(
        inp=lname,
        out=threedout,
        bx=bxsz,
        pd=int(bxsz * pb),
        pdv=int(bxsz * pb),
        apx=apix,
        kp=options.keep,
        sm=options.sym,
        trd=options.threads,
        par=m3dpar)

    run(cmd)

    E2end(logid)
예제 #11
0
def main():
	progname = os.path.basename(sys.argv[0])
	usage = """prog <output> [options]

	This program produces iterative class-averages, one of the secrets to EMAN's rapid convergence.
	Normal usage is to provide a stack of particle images and a classification matrix file defining
	class membership. Members of each class are then iteratively aligned to each other and averaged
	together with (optional) CTF correction.  It is also possible to use this program on all of the
	images in a single stack.

	"""

	parser = EMArgumentParser(usage=usage,version=EMANVERSION)

	parser.add_argument("--input", type=str, help="The name of the input particle stack", default=None)
	parser.add_argument("--output", type=str, help="The name of the output class-average stack", default=None)
	parser.add_argument("--oneclass", type=int, help="Create only a single class-average. Specify the number.",default=None)
	parser.add_argument("--classmx", type=str, help="The name of the classification matrix specifying how particles in 'input' should be grouped. If omitted, all particles will be averaged.", default=None)
	parser.add_argument("--ref", type=str, help="Reference image(s). Used as an initial alignment reference and for final orientation adjustment if present. Also used to assign euler angles to the generated classes. This is typically the projections that were used for classification.", default=None)
	parser.add_argument("--storebad", action="store_true", help="Even if a class-average fails, write to the output. Forces 1->1 numbering in output",default=False)
	parser.add_argument("--decayedge", action="store_true", help="Applies an edge decay to zero on the output class-averages. A very good idea if you plan on 3-D reconstruction.",default=False)
	parser.add_argument("--resultmx",type=str,help="Specify an output image to store the result matrix. This contains 5 images where row is particle number. Rows in the first image contain the class numbers and in the second image consist of 1s or 0s indicating whether or not the particle was included in the class. The corresponding rows in the third, fourth and fifth images are the refined x, y and angle (respectively) used in the final alignment, these are updated and accurate, even if the particle was excluded from the class.", default=None)
	parser.add_argument("--iter", type=int, help="The number of iterations to perform. Default is 1.", default=1)
	parser.add_argument("--prefilt",action="store_true",help="Filter each reference (c) to match the power spectrum of each particle (r) before alignment and comparison",default=False)
	parser.add_argument("--align",type=str,help="This is the aligner used to align particles to the previous class average. Default is None.", default=None)
	parser.add_argument("--aligncmp",type=str,help="The comparitor used for the --align aligner. Default is ccc.",default="ccc")
	parser.add_argument("--ralign",type=str,help="This is the second stage aligner used to refine the first alignment. This is usually the \'refine\' aligner.", default=None)
	parser.add_argument("--raligncmp",type=str,help="The comparitor used by the second stage aligner.",default="ccc")
	parser.add_argument("--averager",type=str,help="The type of averager used to produce the class average.",default="mean")
	parser.add_argument("--setsfref",action="store_true",help="This will impose the 1-D structure factor of the reference on the class-average (recommended when a reference is available)",default=False)
	parser.add_argument("--cmp",type=str,help="The comparitor used to generate quality scores for the purpose of particle exclusion in classes, strongly linked to the keep argument.", default="ccc")
	parser.add_argument("--keep",type=float,help="The fraction of particles to keep in each class.",default=1.0)
	parser.add_argument("--keepsig", action="store_true", help="Causes the keep argument to be interpreted in standard deviations.",default=False)
	parser.add_argument("--automask",action="store_true",help="Applies a 2-D automask before centering. Can help with negative stain data, and other cases where centering is poor.")
	parser.add_argument("--center",type=str,default="xform.center",help="If the default centering algorithm (xform.center) doesn't work well, you can specify one of the others here (e2help.py processor center)")
	parser.add_argument("--bootstrap",action="store_true",help="Ignored. Present for historical reasons only.")
	parser.add_argument("--normproc",type=str,help="Normalization processor applied to particles before alignment. Default is normalize.edgemean. If you want to turn this option off specify \'None\'", default="normalize.edgemean")
	parser.add_argument("--usefilt", dest="usefilt", default=None, help="Specify a particle data file that has been low pass or Wiener filtered. Has a one to one correspondence with your particle data. If specified will be used to align particles to the running class average, however the original particle will be used to generate the actual final class average")
	parser.add_argument("--idxcache", default=False, action="store_true", help="Ignored. Present for historical reasons.")
	parser.add_argument("--dbpath", help="Ignored. Present for historical reasons.", default=".")
	parser.add_argument("--resample",action="store_true",help="If set, will perform bootstrap resampling on the particle data for use in making variance maps.",default=False)
	parser.add_argument("--odd", default=False, help="Used by EMAN2 when running eotests. Includes only odd numbered particles in class averages.", action="store_true")
	parser.add_argument("--even", default=False, help="Used by EMAN2 when running eotests. Includes only even numbered particles in class averages.", action="store_true")
	parser.add_argument("--parallel", default=None, help="parallelism argument")
	parser.add_argument("--force", "-f",dest="force",default=False, action="store_true",help="Force overwrite the output file if it exists.")
	parser.add_argument("--saveali",action="store_true",help="Writes aligned particle images to aligned.hdf. Normally resultmx produces more useful informtation. This can be used for debugging.",default=False)
	parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n",type=int, default=0, help="verbose level [0-9], higner number means higher level of verboseness")
	parser.add_argument("--debug","-d",action="store_true",help="Print debugging infromation while the program is running. Default is off.",default=False)
	parser.add_argument("--nofilecheck",action="store_true",help="Turns file checking off in the check functionality - used by e2refine.py.",default=False)
	parser.add_argument("--check","-c",action="store_true",help="Performs a command line argument check only.",default=False)
	parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)

	(options, args) = parser.parse_args()

	if (options.check): options.verbose = 9 # turn verbose on if the user is only checking...

	error = check(options,True)

	if options.align : options.align=parsemodopt(options.align)
	if options.ralign : options.ralign=parsemodopt(options.ralign)
	if options.aligncmp : options.aligncmp=parsemodopt(options.aligncmp)
	if options.raligncmp : options.raligncmp=parsemodopt(options.raligncmp)
	if options.averager : options.averager=parsemodopt(options.averager)
	if options.cmp : options.cmp=parsemodopt(options.cmp)
	if options.normproc : options.normproc=parsemodopt(options.normproc)

	if options.resultmx!=None : options.storebad=True

	if (options.verbose>0):
		if (error):
			print "e2classaverage.py command line arguments test.... FAILED"
		else:
			print "e2classaverage.py command line arguments test.... PASSED"

	# returning a different error code is currently important to e2refine.py - returning 0 tells e2refine.py that it has enough
	# information to execute this script
	if error : exit(1)
	if options.check: exit(0)

	logger=E2init(sys.argv,options.ppid)
	print "Class averaging beginning"

	try:
		classmx=EMData.read_images(options.classmx)		# we keep the entire classification matrix in memory, since we need to update it in most cases
		ncls=int(classmx[0]["maximum"])+1
	except:
		ncls=1
		if options.resultmx!=None :
			print "resultmx can only be specified in conjunction with a valid classmx input."
			sys.exit(1)

	nptcl=EMUtil.get_image_count(options.input)

	try: apix=EMData(options.input,0,True)["apix_x"]
	except:
		apix=1.0
		print "WARNING: could not get apix from first image. Setting to 1.0. May impact results !"

	# Initialize parallelism
	if options.parallel :
		from EMAN2PAR import EMTaskCustomer
		etc=EMTaskCustomer(options.parallel)
		pclist=[options.input]
		if options.ref: pclist.append(options.ref)
		if options.usefilt: pclist.append(options.usefilt)
		etc.precache(pclist)

	# prepare tasks
	tasks=[]
	if ncls>1:
		if options.oneclass==None : clslst=range(ncls)
		else : clslst=[options.oneclass]

		for cl in clslst:
			ptcls=classmx_ptcls(classmx[0],cl)
			if options.resample : ptcls=[random.choice(ptcls) for i in ptcls]	# this implements bootstrap resampling of the class-average
			if options.odd : ptcls=[i for i in ptcls if i%2==1]
			if options.even: ptcls=[i for i in ptcls if i%2==0]
			tasks.append(ClassAvTask(options.input,ptcls,options.usefilt,options.ref,options.iter,options.normproc,options.prefilt,
			  options.align,options.aligncmp,options.ralign,options.raligncmp,options.averager,options.cmp,options.keep,options.keepsig,
			  options.automask,options.saveali,options.setsfref,options.verbose,cl,options.center))

	else:
		ptcls=range(nptcl)
		if options.resample : ptcls=[random.choice(ptcls) for i in ptcls]
		if options.odd : ptcls=[i for i in ptcls if i%2==1]
		if options.even: ptcls=[i for i in ptcls if i%2==0]
		tasks.append(ClassAvTask(options.input,range(nptcl),options.usefilt,options.ref,options.iter,options.normproc,options.prefilt,
			  options.align,options.aligncmp,options.ralign,options.raligncmp,options.averager,options.cmp,options.keep,options.keepsig,
			  options.automask,options.saveali,options.setsfref,options.verbose,0,options.center))

	# execute task list
	if options.parallel:				# run in parallel
		taskids=etc.send_tasks(tasks)
		alltaskids=taskids[:]

		while len(taskids)>0 :
			curstat=etc.check_task(taskids)
			for i,j in enumerate(curstat):
				if j==100 :
					rslt=etc.get_results(taskids[i])
					if rslt[1]["average"]!=None:
						rslt[1]["average"]["class_ptcl_src"]=options.input
						if options.decayedge:
							nx=rslt[1]["average"]["nx"]
							rslt[1]["average"].process_inplace("normalize.circlemean",{"radius":nx/2-nx/15})
							rslt[1]["average"].process_inplace("mask.gaussian",{"inner_radius":nx/2-nx/15,"outer_radius":nx/20})
							#rslt[1]["average"].process_inplace("mask.decayedge2d",{"width":nx/15})

						if options.ref!=None : rslt[1]["average"]["projection_image"]=options.ref
						if options.storebad : rslt[1]["average"].write_image(options.output,rslt[1]["n"])
						else: rslt[1]["average"].write_image(options.output,-1)


						# Update the resultsmx if requested
						if options.resultmx!=None:
							allinfo=rslt[1]["info"]				# the info result array list of (qual,xform,used) tuples
							pnums=rslt[0].data["images"][2]		# list of image numbers corresponding to information

							for n,info in enumerate(allinfo):
								y=pnums[n]		# actual particle number

								# find the matching class in the existing classification matrix
								for x in range(classmx[0]["nx"]):
									if classmx[0][x,y]==rslt[1]["n"] :		# if the class number in the classmx matches the current class-average number
										break
								else :
									print "Resultmx error: no match found ! (%d %d %d)"%(x,y,rslt[1]["n"])
									continue
								xform=info[1].get_params("2d")
								classmx[1][x,y]=info[2]					# used
								classmx[2][x,y]=xform["tx"]				# dx
								classmx[3][x,y]=xform["ty"]				# dy
								classmx[4][x,y]=xform["alpha"]			# da
								classmx[5][x,y]=xform["mirror"]			# flip
								try: classmx[6][x,y]=xform["scale"]
								except: pass
					# failed average
					elif options.storebad :
						blk=EMData(options.ref,0)
						apix=blk["apix_x"]
						blk=EMData(blk["nx"],blk["ny"],1)
						blk["apix_x"]=apix
						blk.to_zero()
						blk.set_attr("ptcl_repr", 0)
						blk.set_attr("apix_x",apix)
						blk.write_image(options.output,rslt[1]["n"])

			taskids=[j for i,j in enumerate(taskids) if curstat[i]!=100]

			if options.verbose and 100 in curstat :
				print "%d/%d tasks remain"%(len(taskids),len(alltaskids))
			if 100 in curstat :
				E2progress(logger,1.0-(float(len(taskids))/len(alltaskids)))

			time.sleep(3)


		if options.verbose : print "Completed all tasks"

	# single thread
	else:
		for t in tasks:
			rslt=t.execute()
			if rslt==None : sys.exit(1)

			if rslt["average"]!=None :
				rslt["average"]["class_ptcl_src"]=options.input
				if options.decayedge:
					nx=rslt["average"]["nx"]
					rslt["average"].process_inplace("normalize.circlemean",{"radius":nx/2-nx/15})
					rslt["average"].process_inplace("mask.gaussian",{"inner_radius":nx/2-nx/15,"outer_radius":nx/20})
					#rslt["average"].process_inplace("mask.decayedge2d",{"width":nx/15})
				if options.ref!=None : rslt["average"]["projection_image"]=options.ref
				if options.storebad : rslt["average"].write_image(options.output,t.options["n"])
				else: rslt["average"].write_image(options.output,-1)

				# Update the resultsmx if requested
				if options.resultmx!=None:
					allinfo=rslt["info"]				# the info result array list of (qual,xform,used) tuples
					pnums=t.data["images"][2]		# list of image numbers corresponding to information
					for n,info in enumerate(allinfo):
						y=pnums[n]		# actual particle number

						# find the matching class in the existing classification matrix
						for x in range(classmx[0]["nx"]):
							if classmx[0][x,y]==rslt["n"] :		# if the class number in the classmx matches the current class-average number
								break
						else :
							print "Resultmx error: no match found ! (%d %d %d)"%(x,y,rslt[1]["n"])
							continue
						xform=info[1].get_params("2d")
						classmx[1][x,y]=info[2]					# used
						classmx[2][x,y]=xform["tx"]				# dx
						classmx[3][x,y]=xform["ty"]				# dy
						classmx[4][x,y]=xform["alpha"]			# da
						classmx[5][x,y]=xform["mirror"]			# flip
						try: classmx[6][x,y]=xform["scale"]
						except: pass

			# Failed average
			elif options.storebad :
				blk=EMData(options.ref,0)
				apix=blk["apix_x"]
				blk=EMData(blk["nx"],blk["ny"],1)
				blk["apix_x"]=apix
				blk.to_zero()
				blk.set_attr("ptcl_repr", 0)
				blk.set_attr("apix_x",apix)
				blk.write_image(options.output,t.options["n"])

	if options.resultmx!=None:
		if options.verbose : print "Writing results matrix"
		for i,j in enumerate(classmx) : j.write_image(options.resultmx,i)

	print "Class averaging complete"
	E2end(logger)
예제 #12
0
def main():
    progname = os.path.basename(sys.argv[0])
    usage = """Usage: e2spt_average.py [options] 
Note that this program is not part of the original e2spt hierarchy, but is part of an experimental refactoring.

Will read metadata from the specified spt_XX directory, as produced by e2spt_align.py, and average a selected subset of subtomograms in the predetermined orientation.
"""

    parser = EMArgumentParser(usage=usage, version=EMANVERSION)

    parser.add_argument(
        "--threads",
        default=4,
        type=int,
        help=
        "Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present."
    )
    parser.add_argument(
        "--iter",
        type=int,
        help="Iteration number within path. Default = start a new iteration",
        default=-1)
    parser.add_argument(
        "--simthr",
        default=-0.1,
        type=float,
        help=
        "Similarity is smaller for better 'quality' particles. Specify the highest value to include from e2spt_hist.py. Default -0.1"
    )
    parser.add_argument(
        "--keep",
        default=-1,
        type=float,
        help="fraction of particles to keep. will overwrite simthr if set.")
    parser.add_argument(
        "--replace",
        type=str,
        default=None,
        help=
        "Replace the input subtomograms used for alignment with the specified file (used when the aligned particles were masked or filtered)"
    )
    parser.add_argument(
        "--outfile",
        type=str,
        default=None,
        help=
        "Normally even/odd and overall outputs automatically generated. If specified, only overall file written with this specified filename. Suppresses postprocessing."
    )
    parser.add_argument(
        "--wedgesigma",
        type=float,
        help=
        "Threshold for identifying missing data in Fourier space in terms of standard deviation of each Fourier shell. Default 3.0",
        default=3.0)
    parser.add_argument(
        "--minalt",
        type=float,
        help="Minimum alignment altitude to include. Default=0",
        default=0)
    parser.add_argument(
        "--maxalt",
        type=float,
        help="Maximum alignment altitude to include. Deafult=180",
        default=180)
    parser.add_argument(
        "--maxtilt",
        type=float,
        help=
        "Explicitly zeroes data beyond specified tilt angle. Assumes tilt axis exactly on Y and zero tilt in X-Y plane. Default 90 (no limit).",
        default=90.0)
    parser.add_argument(
        "--listfile",
        type=str,
        help=
        "Specify a filename containing a list of integer particle numbers to include in the average, one per line, first is 0. Additional exclusions may apply.",
        default=None)
    parser.add_argument(
        "--automaskexpand",
        default=-1,
        type=int,
        help=
        "Default=boxsize/20. Specify number of voxels to expand mask before soft edge. Use this if low density peripheral features are cut off by the mask.",
        guitype='intbox',
        row=12,
        col=1,
        rowspan=1,
        colspan=1,
        mode="refinement[-1]")
    parser.add_argument(
        "--symalimasked",
        type=str,
        default=None,
        help=
        "This will translationally realign each asymmetric unit to the specified (usually masked) reference "
    )
    parser.add_argument(
        "--sym",
        type=str,
        default="c1",
        help=
        "Symmetry of the input. Must be aligned in standard orientation to work properly."
    )
    parser.add_argument(
        "--path",
        type=str,
        default=None,
        help=
        "Path to a folder containing current results (default = highest spt_XX)"
    )
    parser.add_argument("--skippostp",
                        action="store_true",
                        default=False,
                        help="Skip post process steps (fsc, mask and filters)")
    parser.add_argument(
        "--verbose",
        "-v",
        dest="verbose",
        action="store",
        metavar="n",
        type=int,
        default=0,
        help=
        "verbose level [0-9], higher number means higher level of verboseness")
    parser.add_argument(
        "--ppid",
        type=int,
        help="Set the PID of the parent process, used for cross platform PPID",
        default=-1)
    parser.add_argument("--parallel",
                        type=str,
                        help="Thread/mpi parallelism to use",
                        default=None)

    (options, args) = parser.parse_args()

    if options.path == None:
        fls = [
            int(i[-2:]) for i in os.listdir(".")
            if i[:4] == "spt_" and len(i) == 6 and str.isdigit(i[-2:])
        ]
        if len(fls) == 0:
            print("Error, cannot find any spt_XX folders")
            sys.exit(2)
        options.path = "spt_{:02d}".format(max(fls))
        if options.verbose: print("Working in : ", options.path)

    if options.iter < 0:
        fls = [
            int(i[15:17]) for i in os.listdir(options.path)
            if i[:15] == "particle_parms_" and str.isdigit(i[15:17])
        ]
        if len(fls) == 0:
            print("Cannot find a {}/particle_parms* file".format(options.path))
            sys.exit(2)
        options.iter = max(fls)
        if options.verbose: print("Using iteration ", options.iter)
        angs = js_open_dict("{}/particle_parms_{:02d}.json".format(
            options.path, options.iter))
    else:
        fls = [
            int(i[15:17]) for i in os.listdir(options.path)
            if i[:15] == "particle_parms_" and str.isdigit(i[15:17])
        ]
        if len(fls) == 0:
            print("Cannot find a {}/particle_parms* file".format(options.path))
            sys.exit(2)
        mit = max(fls)
        if options.iter > mit:
            angs = js_open_dict("{}/particle_parms_{:02d}.json".format(
                options.path, mit))
            print(
                "WARNING: no particle_parms found for iter {}, using parms from {}"
                .format(options.iter, mit))
        else:
            angs = js_open_dict("{}/particle_parms_{:02d}.json".format(
                options.path, options.iter))

    if options.listfile != None:
        plist = set([int(i) for i in open(options.listfile, "r")])

    NTHREADS = max(options.threads + 1,
                   2)  # we have one thread just writing results

    logid = E2init(sys.argv, options.ppid)

    #	jsd=Queue.Queue(0)

    # filter the list of particles to include
    keys = list(angs.keys())
    if options.listfile != None:
        keys = [i for i in keys if eval(i)[1] in plist]
        if options.verbose:
            print("{}/{} particles based on list file".format(
                len(keys), len(list(angs.keys()))))

    newkey = []
    newang = {}
    if options.keep > 0 and options.keep <= 1:
        score = [float(angs[k]["score"]) for k in keys]
        options.simthr = np.sort(score)[int(len(score) * options.keep) - 1]
        print("Keeping {:.0f}% particles with score below {:.2f}".format(
            options.keep * 100, options.simthr))

    for k in keys:
        val = angs[k]
        if type(val) == list:
            val = val[0]

        if val["score"] <= options.simthr and inrange(
                options.minalt, val["xform.align3d"].get_params("eman")["alt"],
                options.maxalt):
            newkey.append(k)
            newang[k] = val

    if options.verbose:
        print("{}/{} particles after filters".format(len(newkey),
                                                     len(list(angs.keys()))))
    keys = newkey
    angs = newang

    if options.parallel and options.symalimasked == None:
        #print("running in mpi mode. This is experimental, so please switch back to threading if anything goes wrong...")

        from EMAN2PAR import EMTaskCustomer
        etc = EMTaskCustomer(options.parallel,
                             module="e2spt_average.SptavgTask")
        num_cpus = etc.cpu_est()

        print("{} total CPUs available".format(num_cpus))
        data = [[], []]  ## even/odd
        if "eo" in angs[keys[0]]:
            print("Reading even/odd subset from json file...")

            for i, k in enumerate(keys):
                src, ii = eval(k)[0], eval(k)[1]
                eo = int(angs[k]["eo"])
                data[eo].append([src, ii, angs[k]["xform.align3d"]])
        else:
            for i, k in enumerate(keys):
                src, ii = eval(k)[0], eval(k)[1]
                data[ii % 2].append([src, ii, angs[k]["xform.align3d"]])

        #### check and save size of particle
        fsp, i, xf = data[0][0]
        b = EMData(fsp, i, True)
        sz = options.boxsz = b["ny"]
        avgs = []
        for ieo, eo in enumerate(["even", "odd"]):
            print("Averaging {}...".format(eo))
            nbatch = min(len(data[ieo]) // 4, num_cpus)
            tasks = [data[ieo][i::nbatch] for i in range(nbatch)]

            print("{} particles in {} jobs".format(len(data[ieo]), len(tasks)))

            tids = []
            for t in tasks:
                task = SptavgTask(t, options)
                tid = etc.send_task(task)
                tids.append(tid)

            while 1:
                st_vals = etc.check_task(tids)
                if np.min(st_vals) == 100: break
                time.sleep(5)

            output = EMData(sz, sz, sz)
            normvol = EMData((sz // 2 + 1) * 2, sz, sz)
            output.to_zero()
            output.do_fft_inplace()
            avg = Averagers.get("mean")
            normvol.to_zero()
            for i in tids:
                threed, norm = etc.get_results(i)[1]
                #print(i, threed["mean"], threed["sigma"], norm["mean"], norm["sigma"])
                #norm.div(len(tids))
                threed.process_inplace("math.multamplitude", {"amp": norm})
                avg.add_image(threed)
                normvol.add(norm)

            output = avg.finish()
            normvol.process_inplace("math.reciprocal")
            output.process_inplace("math.multamplitude", {"amp": normvol})
            output.process_inplace("xform.phaseorigin.tocenter")

            output.do_ift_inplace()
            output.depad()
            avgs.append(output)

        ave, avo = avgs

    else:
        avg = [0, 0]
        avg[0] = Averagers.get(
            "mean.tomo",
            {"thresh_sigma": options.wedgesigma})  #,{"save_norm":1})
        avg[1] = Averagers.get("mean.tomo",
                               {"thresh_sigma": options.wedgesigma})

        # Rotation and insertion are slow, so we do it with threads.
        if options.symalimasked != None:
            if options.replace != None:
                print("Error: --replace cannot be used with --symalimasked")
                sys.exit(1)
            alimask = EMData(options.symalimasked)
            thrds = [
                threading.Thread(target=rotfnsym,
                                 args=(avg[eval(k)[1] % 2], eval(k)[0],
                                       eval(k)[1], angs[k]["xform.align3d"],
                                       options.sym, alimask, options.maxtilt,
                                       options.verbose))
                for i, k in enumerate(keys)
            ]
        else:
            # Averager isn't strictly threadsafe, so possibility of slight numerical errors with a lot of threads
            if options.replace != None:
                thrds = [
                    threading.Thread(target=rotfn,
                                     args=(avg[eval(k)[1] % 2],
                                           options.replace, eval(k)[1],
                                           angs[k]["xform.align3d"],
                                           options.maxtilt, options.verbose))
                    for i, k in enumerate(keys)
                ]

            else:
                thrds = [
                    threading.Thread(target=rotfn,
                                     args=(avg[eval(k)[1] % 2], eval(k)[0],
                                           eval(k)[1],
                                           angs[k]["xform.align3d"],
                                           options.maxtilt, options.verbose))
                    for i, k in enumerate(keys)
                ]

        print(len(thrds), " threads")
        thrtolaunch = 0
        while thrtolaunch < len(thrds) or threading.active_count() > 1:
            # If we haven't launched all threads yet, then we wait for an empty slot, and launch another
            # note that it's ok that we wait here forever, since there can't be new results if an existing
            # thread hasn't finished.
            if thrtolaunch < len(thrds):
                while (threading.active_count() == NTHREADS):
                    time.sleep(.1)
                if options.verbose:
                    print("Starting thread {}/{}".format(
                        thrtolaunch, len(thrds)))
                thrds[thrtolaunch].start()
                thrtolaunch += 1
            else:
                time.sleep(1)

            #while not jsd.empty():
            #fsp,n,ptcl=jsd.get()
            #avg[n%2].add_image(ptcl)

        for t in thrds:
            t.join()

        ave = avg[0].finish()  #.process("xform.phaseorigin.tocenter").do_ift()
        avo = avg[1].finish()  #.process("xform.phaseorigin.tocenter").do_ift()

        # impose symmetry on even and odd halves if appropriate
        if options.sym != None and options.sym.lower(
        ) != "c1" and options.symalimasked == None:
            ave.process_inplace("xform.applysym", {
                "averager": "mean.tomo",
                "sym": options.sym
            })
            avo.process_inplace("xform.applysym", {
                "averager": "mean.tomo",
                "sym": options.sym
            })

    av = ave + avo
    av.mult(0.5)

    if options.outfile:
        av.write_image(options.outfile)
        sys.exit(0)

    evenfile = "{}/threed_{:02d}_even.hdf".format(options.path, options.iter)
    oddfile = "{}/threed_{:02d}_odd.hdf".format(options.path, options.iter)
    combfile = "{}/threed_{:02d}.hdf".format(options.path, options.iter)
    ave.write_image(evenfile, 0)
    avo.write_image(oddfile, 0)
    av.write_image(combfile, 0)

    cmd = "e2proc3d.py {evenfile} {path}/fsc_unmasked_{itr:02d}.txt --calcfsc={oddfile}".format(
        path=options.path,
        itr=options.iter,
        evenfile=evenfile,
        oddfile=oddfile)
    launch_childprocess(cmd)

    #### skip post process in case we want to do this elsewhere...
    if options.skippostp:
        E2end(logid)
        return

    # final volume at this point is Wiener filtered
    launch_childprocess(
        "e2proc3d.py {combfile} {combfile} --process=filter.wiener.byfsc:fscfile={path}/fsc_unmasked_{itr:02d}.txt:snrmult=2"
        .format(path=options.path, itr=options.iter, combfile=combfile))

    # New version of automasking based on a more intelligent interrogation of the volume
    vol = EMData(combfile)
    nx = vol["nx"]
    apix = vol["apix_x"]
    md = vol.calc_radial_dist(old_div(nx, 2), 0, 1,
                              3)  # radial max value per shell in real space

    rmax = int(old_div(nx, 2.2))  # we demand at least 10% padding
    vmax = max(md[:rmax])  # max value within permitted radius

    # this finds the first radius where the max value @ r falls below overall max/4
    # this becomes the new maximum mask radius
    act = 0
    mv = 0, 0
    for i in range(rmax):
        if md[i] > mv[0]:
            mv = md[i], i  # find the radius of the  max val in range
        if not act and md[i] < 0.9 * vmax: continue
        act = True
        if md[i] < 0.2 * vmax:
            rmax = i
            break

    rmaxval = mv[1]
    vmax = mv[0]

    # excludes any spurious high values at large radius
    vol.process_inplace("mask.sharp", {"outer_radius": rmax})

    # automask
    mask = vol.process(
        "mask.auto3d", {
            "threshold": vmax * .15,
            "radius": 0,
            "nshells":
            int(nx * 0.05 + 0.5 + old_div(20, apix)) + options.automaskexpand,
            "nmaxseed": 24,
            "return_mask": 1
        })

    mask.process_inplace("filter.lowpass.gauss",
                         {"cutoff_freq": old_div(1.0, (40.0))})
    mask.write_image("{path}/mask.hdf".format(path=options.path), 0)

    # compute masked fsc and refilter
    ave.mult(mask)
    ave.write_image("{path}/tmp_even.hdf".format(path=options.path), 0)
    avo.mult(mask)
    avo.write_image("{path}/tmp_odd.hdf".format(path=options.path), 0)
    av.mult(mask)
    av.write_image(combfile, 0)

    cmd = "e2proc3d.py {path}/tmp_even.hdf {path}/fsc_masked_{itr:02d}.txt --calcfsc={path}/tmp_odd.hdf".format(
        path=options.path, itr=options.iter)
    launch_childprocess(cmd)

    # final volume is premasked and Wiener filtered based on the masked FSC
    launch_childprocess(
        "e2proc3d.py {combfile} {combfile} --process=filter.wiener.byfsc:fscfile={path}/fsc_masked_{itr:02d}.txt:snrmult=2"
        .format(path=options.path, itr=options.iter, combfile=combfile))

    E2end(logid)
예제 #13
0
def main():
	parser = EMArgumentParser(usage="")

	parser.add_argument("--output", default="threed.hdf", help="Output reconstructed volume file name.")
	parser.add_argument("--input", default=None, help="The input projections. Project should usually have the xform.projection header attribute, which is used for slice insertion")
	parser.add_argument("--sym", dest="sym", default="c1", help="Set the symmetry; if no value is given then the model is assumed to have no symmetry.\nChoices are: i, c, d, tet, icos, or oct.")

	parser.add_argument("--pad", default=-1,type=int, help="Will zero-pad images to the specifed size. ")
	
	parser.add_argument("--outsize", default=-1, type=int, help="Defines the dimensions of the final volume written to disk")
	
	parser.add_argument("--keep", type=str, dest="keep", help="The fraction of slices to keep, based on quality scores (1.0 = use all slices).",default=".9")
	
	parser.add_argument("--no_wt", action="store_true", dest="no_wt", default=False, help="This argument turns automatic weighting off causing all images to be weighted by 1.")
	
	parser.add_argument("--mode", type=str, default="trilinear", help="Fourier reconstruction 'mode' to use. The default should not normally be changed.")
	
	parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higher number means higher level of verboseness")

	parser.add_argument("--apix",metavar="A/pix",type=float,help="A/pix value for output, overrides automatic values",default=None)
	parser.add_argument("--tidrange", type=str,help="Range of tilt id to include for particles from tilt series. Specify two integers separated by ','.", default="-1,-1")
	
	parser.add_argument("--ref", type=str,help="Weight each particle using a specified reference map.", default=None)
	parser.add_argument("--minres", type=float,help="minimum resolution to compare when weighting by a reference map.", default=50)
	parser.add_argument("--maxres", type=float,help="maximum resolution to compare when weighting by a reference map.", default=-1)
	parser.add_argument("--parallel", type=str,help="Thread/mpi parallelism to use without shared memory. Each worker will reconstruct a map with a subset of particles and the results from workers will be averaged together with the corresponding Fourier weighting. Along with --threads, this allows having one worker per node using multiple threads.", default="thread:1")
	parser.add_argument("--threads", type=int,help="Number of threads using shared memory.", default=1)
	parser.add_argument("--setsf", type=str,help="Set structure factor from text file", default=None)
	
	parser.add_argument("--debug", action="store_true", default=False, help="Turn on debug mode. This will only process a small subset of the data.")
	parser.add_argument("--clsid", default=None, type=str, help="Only reconstruct a class of particles. Also take even/odd to reconstruct subsets of particles.")
	parser.add_argument("--listsel", default=None, type=str, help="only reconstruct particles of indices from the given list in a text file.")

	parser.add_argument("--p3did", type=int, help="only reconstruct images for one 3d particle of the given ID.",default=-1)
	parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)

	(options, args) = parser.parse_args()

	logger=E2init(sys.argv,options.ppid)
	time0=time.time()

	# so it recognize even/odd or 0/1
	if options.clsid:
		options.clsid=options.clsid.replace("even","0").replace("odd","1")
		try: options.clsid=int(options.clsid)
		except:options.clsid=-1
	else:
		options.clsid=-1
		
	if options.sym.startswith("h"):
		options.sym="c1"
		
	options.keep=[float(k) for k in options.keep.split(',')]
	if len(options.keep)<3: options.keep=[options.keep[0]]*3
	
	# get basic image parameters
	tmp=EMData(options.input,0,True)
	boxsz=tmp["nx"]
	if options.apix!=None : apix=options.apix
	else : apix=tmp["apix_x"]

	if options.pad<0:
		options.pad=good_size(boxsz*1.5)
		
	if options.outsize<0:
		options.outsize=boxsz
		
	options.tidrange=[int(i) for i in options.tidrange.split(',')]
	if options.tidrange[0]>=0:
		print("including tilt ids from {} to {}".format(options.tidrange[0], options.tidrange[1]))
	
	data=initialize_data(options.input, options)
	padvol=options.pad

	from EMAN2PAR import EMTaskCustomer
	if options.ref:
		print("weighting by reference...")
		ref=EMData(options.ref)
		ny=options.pad
		by=ref["ny"]
		ref=ref.get_clip(Region((by-ny)/2, (by-ny)/2,(by-ny)/2, ny, ny,ny))
		ref=ref.do_fft()
		ref.process_inplace("xform.phaseorigin.tocenter")
		ref.process_inplace("xform.fourierorigin.tocenter")
		
		etc=EMTaskCustomer(options.parallel, module="e2spa_make3d.WeightptclTask")
		num_cpus = etc.cpu_est()
		print("{} total CPUs available".format(num_cpus))
		tasks=[data[i::num_cpus] for i in range(num_cpus)]
		print("{} jobs".format(len(tasks)))
		
		tids=[]
		for t in tasks:
			task = WeightptclTask(t, ref, options)
			if options.debug:
				task.execute(print)
				return
			tid=etc.send_task(task)
			tids.append(tid)

		while 1:
			st_vals = etc.check_task(tids)
			E2progress(logger, .5*np.mean(st_vals)/100.)
			if np.min(st_vals) == 100: break
			time.sleep(5)
			
		wts=[0]*len(data)
		for i in tids:
			wt=etc.get_results(i)[1]
			for w in wt:
				try:
					wts[w[0]]=w[1]
				except:
					print(w)
				
		
		wts=np.array(wts)
		wts[wts<0]=0
		if options.minres>0:
			r0=int(apix*ny/options.minres)
			wts[:,:r0]=np.mean(wts[:,:r0], axis=0)
		
		if options.maxres>0:
			r1=int(apix*ny/options.maxres)
			wts[:,r1:]=np.mean(wts[:,r1:], axis=0)
			
		
		
		print(wts.shape)
		del etc
		
		#print(r0,r1)
		#scrs=np.mean(wts[:,r0:r1], axis=1)
		#if options.keep<1:
			#thr=np.sort(scrs)[int(len(scrs)*(1-options.keep))-1]
			#scrs[scrs<thr]=-1
		
		for i,d in enumerate(data):
			d["curve"]=wts[i]
			#d["weight"]=float(scrs[i])
			
			
	etc=EMTaskCustomer(options.parallel, module="e2spa_make3d.Make3dTask")
	num_cpus = etc.cpu_est()
	print("{} total CPUs available".format(num_cpus))
	tasks=[data[i::num_cpus] for i in range(num_cpus)]
	print("{} jobs".format(len(tasks)))

	tids=[]
	for t in tasks:
		task = Make3dTask(t, options)
		if options.debug:
			task.execute(print)
			return
		tid=etc.send_task(task)
		tids.append(tid)

	while 1:
		st_vals = etc.check_task(tids)
		E2progress(logger, .5+.5*np.mean(st_vals)/100.)
		if np.min(st_vals) == 100: break
		time.sleep(5)
		
	output=EMData(padvol, padvol, padvol)
	normvol=EMData(padvol//2+1, padvol, padvol)
	output.to_zero()
	output.do_fft_inplace()
	normvol.to_zero()
	
	for i in tids:
		threed, norm=etc.get_results(i)[1]
		threed.process_inplace("math.multamplitude", {"amp":norm})
		output.add(threed)
		normvol.add(norm)
		
	normvol.process_inplace("math.reciprocal")
	output.process_inplace("math.multamplitude", {"amp":normvol})
	
	output.do_ift_inplace()
	output.depad()
	output.process_inplace("xform.phaseorigin.tocenter")
	
	del etc

	if options.verbose>0 : print("Finished Reconstruction")
	output["apix_x"]=output["apix_y"]=output["apix_z"]=apix
	sz=options.outsize
	output.clip_inplace(Region((padvol-sz)//2,(padvol-sz)//2,(padvol-sz)//2,sz,sz,sz))
	
	if os.path.isfile(options.output):
		os.remove(options.output)
		
	output.write_image(options.output,0)
	if options.setsf:
		launch_childprocess("e2proc3d.py {} {} --setsf {}".format(options.output,options.output,options.setsf))
		
	if options.verbose>0:
			print("Output File: "+options.output)

	E2end(logger)

	print("Reconstruction finishend ({:.1f} s)".format(time.time()-time0))
예제 #14
0
def main():
    usage = """
"""

    parser = EMArgumentParser(usage=usage, version=EMANVERSION)

    parser.add_argument(
        "--path",
        type=str,
        default=None,
        help=
        "Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)"
    )
    parser.add_argument(
        "--iter",
        type=int,
        help="Iteration number within path. Default = start a new iteration",
        default=0)
    parser.add_argument(
        "--goldcontinue",
        action="store_true",
        help=
        "Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",
        default=False)
    parser.add_argument(
        "--sym",
        type=str,
        default="c1",
        help=
        "Symmetry of the input. Must be aligned in standard orientation to work properly."
    )
    parser.add_argument(
        "--maxres",
        type=float,
        help="Maximum resolution to consider in alignment (in A, not 1/A)",
        default=0)
    parser.add_argument(
        "--minres",
        type=float,
        help="Minimum resolution to consider in alignment (in A, not 1/A)",
        default=0)
    parser.add_argument("--parallel",
                        type=str,
                        help="Thread/mpi parallelism to use",
                        default="thread:4")
    parser.add_argument("--fromscratch",
                        action="store_true",
                        help=".",
                        default=False)
    parser.add_argument(
        "--use3d",
        action="store_true",
        help="use projection of 3d particles instead of 2d sub tilt series",
        default=False)
    parser.add_argument("--debug",
                        action="store_true",
                        help=".",
                        default=False)
    parser.add_argument(
        "--plst",
        type=str,
        default=None,
        help=
        "list of 2d particle with alignment parameters. will reconstruct before alignment."
    )

    parser.add_argument("--maxshift",
                        type=int,
                        help="maximum shift. default box size/6",
                        default=-1)
    parser.add_argument(
        "--maxang",
        type=int,
        help=
        "maximum angle difference from starting point. ignored when fromscratch is on",
        default=30)

    parser.add_argument(
        "--verbose",
        "-v",
        dest="verbose",
        action="store",
        metavar="n",
        type=int,
        default=0,
        help=
        "verbose level [0-9], higher number means higher level of verboseness")
    parser.add_argument(
        "--ppid",
        type=int,
        help="Set the PID of the parent process, used for cross platform PPID",
        default=-1)

    (options, args) = parser.parse_args()
    logid = E2init(sys.argv, options.ppid)

    options.ref = args[1]
    options.info3dname = "{}/particle_info_3d.lst".format(options.path)
    options.info2dname = "{}/particle_info_2d.lst".format(options.path)

    tasks = load_lst_params(args[0])
    for i, t in enumerate(tasks):
        t["ii"] = i

    from EMAN2PAR import EMTaskCustomer
    etc = EMTaskCustomer(options.parallel,
                         module="e2spt_align_subtlt.SptAlignTask")

    num_cpus = etc.cpu_est()
    options.nowtime = time.time()
    if options.debug:
        tasks = tasks[:num_cpus * 4]
    print("{} jobs on {} CPUs".format(len(tasks), num_cpus))
    njob = num_cpus

    tids = []
    for i in range(njob):
        t = tasks[i::njob]
        task = SptAlignTask(t, options)
        if options.debug:
            ret = task.execute(print)
            return
        tid = etc.send_task(task)
        tids.append(tid)

    while 1:
        st_vals = etc.check_task(tids)
        if -100 in st_vals:
            print("Error occurs in parallelism. Exit")
            return
        E2progress(logid, np.mean(st_vals) / 100.)

        if np.min(st_vals) == 100: break
        time.sleep(5)

    output3d = [None] * len(tasks)
    output2d = [None] * len(tasks)
    for i in tids:
        rets = etc.get_results(i)[1]
        for r in rets:
            output3d[r[0]] = r[1]
            output2d[r[0]] = r[2]

    del etc

    output2d = sum(output2d, [])

    fm3d = f"{options.path}/aliptcls3d_{options.iter:02d}.lst"
    save_lst_params(output3d, fm3d)

    fm2d = f"{options.path}/aliptcls2d_{options.iter:02d}.lst"
    save_lst_params(output2d, fm2d)

    E2end(logid)
예제 #15
0
def main():
    progname = os.path.basename(sys.argv[0])
    usage = """prog <output> [options]

	This program produces iterative class-averages, one of the secrets to EMAN's rapid convergence.
	Normal usage is to provide a stack of particle images and a classification matrix file defining
	class membership. Members of each class are then iteratively aligned to each other and averaged
	together with (optional) CTF correction.  It is also possible to use this program on all of the
	images in a single stack.

	"""

    parser = EMArgumentParser(usage=usage, version=EMANVERSION)

    parser.add_argument("--input",
                        type=str,
                        help="The name of the input particle stack",
                        default=None)
    parser.add_argument("--output",
                        type=str,
                        help="The name of the output class-average stack",
                        default=None)
    parser.add_argument(
        "--oneclass",
        type=int,
        help="Create only a single class-average. Specify the number.",
        default=None)
    parser.add_argument(
        "--classmx",
        type=str,
        help=
        "The name of the classification matrix specifying how particles in 'input' should be grouped. If omitted, all particles will be averaged.",
        default=None)
    parser.add_argument(
        "--ref",
        type=str,
        help=
        "Reference image(s). Used as an initial alignment reference and for final orientation adjustment if present. Also used to assign euler angles to the generated classes. This is typically the projections that were used for classification.",
        default=None)
    parser.add_argument(
        "--storebad",
        action="store_true",
        help=
        "Even if a class-average fails, write to the output. Forces 1->1 numbering in output",
        default=False)
    parser.add_argument(
        "--decayedge",
        action="store_true",
        help=
        "Applies an edge decay to zero on the output class-averages. A very good idea if you plan on 3-D reconstruction.",
        default=False)
    parser.add_argument(
        "--resultmx",
        type=str,
        help=
        "Specify an output image to store the result matrix. This contains 5 images where row is particle number. Rows in the first image contain the class numbers and in the second image consist of 1s or 0s indicating whether or not the particle was included in the class. The corresponding rows in the third, fourth and fifth images are the refined x, y and angle (respectively) used in the final alignment, these are updated and accurate, even if the particle was excluded from the class.",
        default=None)
    parser.add_argument(
        "--iter",
        type=int,
        help="The number of iterations to perform. Default is 1.",
        default=1)
    parser.add_argument(
        "--prefilt",
        action="store_true",
        help=
        "Filter each reference (c) to match the power spectrum of each particle (r) before alignment and comparison",
        default=False)
    parser.add_argument(
        "--align",
        type=str,
        help=
        "This is the aligner used to align particles to the previous class average. Default is None.",
        default=None)
    parser.add_argument(
        "--aligncmp",
        type=str,
        help="The comparitor used for the --align aligner. Default is ccc.",
        default="ccc")
    parser.add_argument(
        "--ralign",
        type=str,
        help=
        "This is the second stage aligner used to refine the first alignment. This is usually the \'refine\' aligner.",
        default=None)
    parser.add_argument(
        "--raligncmp",
        type=str,
        help="The comparitor used by the second stage aligner.",
        default="ccc")
    parser.add_argument(
        "--averager",
        type=str,
        help="The type of averager used to produce the class average.",
        default="mean")
    parser.add_argument(
        "--setsfref",
        action="store_true",
        help=
        "This will impose the 1-D structure factor of the reference on the class-average (recommended when a reference is available)",
        default=False)
    parser.add_argument(
        "--cmp",
        type=str,
        help=
        "The comparitor used to generate quality scores for the purpose of particle exclusion in classes, strongly linked to the keep argument.",
        default="ccc")
    parser.add_argument(
        "--keep",
        type=float,
        help="The fraction of particles to keep in each class.",
        default=1.0)
    parser.add_argument(
        "--keepsig",
        action="store_true",
        help=
        "Causes the keep argument to be interpreted in standard deviations.",
        default=False)
    parser.add_argument(
        "--automask",
        action="store_true",
        help=
        "Applies a 2-D automask before centering. Can help with negative stain data, and other cases where centering is poor."
    )
    parser.add_argument(
        "--center",
        type=str,
        default="xform.center",
        help=
        "If the default centering algorithm (xform.center) doesn't work well, you can specify one of the others here (e2help.py processor center)"
    )
    parser.add_argument("--bootstrap",
                        action="store_true",
                        help="Ignored. Present for historical reasons only.")
    parser.add_argument(
        "--normproc",
        type=str,
        help=
        "Normalization processor applied to particles before alignment. Default is normalize.edgemean. If you want to turn this option off specify \'None\'",
        default="normalize.edgemean")
    parser.add_argument(
        "--usefilt",
        dest="usefilt",
        default=None,
        help=
        "Specify a particle data file that has been low pass or Wiener filtered. Has a one to one correspondence with your particle data. If specified will be used to align particles to the running class average, however the original particle will be used to generate the actual final class average"
    )
    parser.add_argument("--idxcache",
                        default=False,
                        action="store_true",
                        help="Ignored. Present for historical reasons.")
    parser.add_argument("--dbpath",
                        help="Ignored. Present for historical reasons.",
                        default=".")
    parser.add_argument(
        "--resample",
        action="store_true",
        help=
        "If set, will perform bootstrap resampling on the particle data for use in making variance maps.",
        default=False)
    parser.add_argument(
        "--odd",
        default=False,
        help=
        "Used by EMAN2 when running eotests. Includes only odd numbered particles in class averages.",
        action="store_true")
    parser.add_argument(
        "--even",
        default=False,
        help=
        "Used by EMAN2 when running eotests. Includes only even numbered particles in class averages.",
        action="store_true")
    parser.add_argument("--parallel",
                        default=None,
                        help="parallelism argument")
    parser.add_argument("--force",
                        "-f",
                        dest="force",
                        default=False,
                        action="store_true",
                        help="Force overwrite the output file if it exists.")
    parser.add_argument(
        "--saveali",
        action="store_true",
        help=
        "Writes aligned particle images to aligned.hdf. Normally resultmx produces more useful informtation. This can be used for debugging.",
        default=False)
    parser.add_argument(
        "--verbose",
        "-v",
        dest="verbose",
        action="store",
        metavar="n",
        type=int,
        default=0,
        help=
        "verbose level [0-9], higner number means higher level of verboseness")
    parser.add_argument(
        "--debug",
        "-d",
        action="store_true",
        help=
        "Print debugging infromation while the program is running. Default is off.",
        default=False)
    parser.add_argument(
        "--nofilecheck",
        action="store_true",
        help=
        "Turns file checking off in the check functionality - used by e2refine.py.",
        default=False)
    parser.add_argument("--check",
                        "-c",
                        action="store_true",
                        help="Performs a command line argument check only.",
                        default=False)
    parser.add_argument(
        "--ppid",
        type=int,
        help="Set the PID of the parent process, used for cross platform PPID",
        default=-1)

    (options, args) = parser.parse_args()

    if (options.check):
        options.verbose = 9  # turn verbose on if the user is only checking...

    error = check(options, True)

    if options.align: options.align = parsemodopt(options.align)
    if options.ralign: options.ralign = parsemodopt(options.ralign)
    if options.aligncmp: options.aligncmp = parsemodopt(options.aligncmp)
    if options.raligncmp: options.raligncmp = parsemodopt(options.raligncmp)
    if options.averager: options.averager = parsemodopt(options.averager)
    if options.cmp: options.cmp = parsemodopt(options.cmp)
    if options.normproc: options.normproc = parsemodopt(options.normproc)

    if options.resultmx != None: options.storebad = True

    if (options.verbose > 0):
        if (error):
            print "e2classaverage.py command line arguments test.... FAILED"
        else:
            print "e2classaverage.py command line arguments test.... PASSED"

    # returning a different error code is currently important to e2refine.py - returning 0 tells e2refine.py that it has enough
    # information to execute this script
    if error: exit(1)
    if options.check: exit(0)

    logger = E2init(sys.argv, options.ppid)
    print "Class averaging beginning"

    try:
        classmx = EMData.read_images(
            options.classmx
        )  # we keep the entire classification matrix in memory, since we need to update it in most cases
        ncls = int(classmx[0]["maximum"]) + 1
    except:
        ncls = 1
        if options.resultmx != None:
            print "resultmx can only be specified in conjunction with a valid classmx input."
            sys.exit(1)

    nptcl = EMUtil.get_image_count(options.input)

    try:
        apix = EMData(options.input, 0, True)["apix_x"]
    except:
        apix = 1.0
        print "WARNING: could not get apix from first image. Setting to 1.0. May impact results !"

    # Initialize parallelism
    if options.parallel:
        from EMAN2PAR import EMTaskCustomer
        etc = EMTaskCustomer(options.parallel)
        pclist = [options.input]
        if options.ref: pclist.append(options.ref)
        if options.usefilt: pclist.append(options.usefilt)
        etc.precache(pclist)

    # prepare tasks
    tasks = []
    if ncls > 1:
        if options.oneclass == None: clslst = range(ncls)
        else: clslst = [options.oneclass]

        for cl in clslst:
            ptcls = classmx_ptcls(classmx[0], cl)
            if options.resample:
                ptcls = [
                    random.choice(ptcls) for i in ptcls
                ]  # this implements bootstrap resampling of the class-average
            if options.odd: ptcls = [i for i in ptcls if i % 2 == 1]
            if options.even: ptcls = [i for i in ptcls if i % 2 == 0]
            tasks.append(
                ClassAvTask(options.input, ptcls, options.usefilt, options.ref,
                            options.iter, options.normproc, options.prefilt,
                            options.align, options.aligncmp, options.ralign,
                            options.raligncmp, options.averager, options.cmp,
                            options.keep, options.keepsig, options.automask,
                            options.saveali, options.setsfref, options.verbose,
                            cl, options.center))

    else:
        ptcls = range(nptcl)
        if options.resample: ptcls = [random.choice(ptcls) for i in ptcls]
        if options.odd: ptcls = [i for i in ptcls if i % 2 == 1]
        if options.even: ptcls = [i for i in ptcls if i % 2 == 0]
        tasks.append(
            ClassAvTask(options.input, range(nptcl), options.usefilt,
                        options.ref, options.iter, options.normproc,
                        options.prefilt, options.align, options.aligncmp,
                        options.ralign, options.raligncmp, options.averager,
                        options.cmp, options.keep, options.keepsig,
                        options.automask, options.saveali, options.setsfref,
                        options.verbose, 0, options.center))

    # execute task list
    if options.parallel:  # run in parallel
        taskids = etc.send_tasks(tasks)
        alltaskids = taskids[:]

        while len(taskids) > 0:
            curstat = etc.check_task(taskids)
            for i, j in enumerate(curstat):
                if j == 100:
                    rslt = etc.get_results(taskids[i])
                    if rslt[1]["average"] != None:
                        rslt[1]["average"]["class_ptcl_src"] = options.input
                        if options.decayedge:
                            nx = rslt[1]["average"]["nx"]
                            rslt[1]["average"].process_inplace(
                                "normalize.circlemean",
                                {"radius": nx / 2 - nx / 15})
                            rslt[1]["average"].process_inplace(
                                "mask.gaussian", {
                                    "inner_radius": nx / 2 - nx / 15,
                                    "outer_radius": nx / 20
                                })
                            #rslt[1]["average"].process_inplace("mask.decayedge2d",{"width":nx/15})

                        if options.ref != None:
                            rslt[1]["average"][
                                "projection_image"] = options.ref
                        if options.storebad:
                            rslt[1]["average"].write_image(
                                options.output, rslt[1]["n"])
                        else:
                            rslt[1]["average"].write_image(options.output, -1)

                        # Update the resultsmx if requested
                        if options.resultmx != None:
                            allinfo = rslt[1][
                                "info"]  # the info result array list of (qual,xform,used) tuples
                            pnums = rslt[0].data["images"][
                                2]  # list of image numbers corresponding to information

                            for n, info in enumerate(allinfo):
                                y = pnums[n]  # actual particle number

                                # find the matching class in the existing classification matrix
                                for x in range(classmx[0]["nx"]):
                                    if classmx[0][x, y] == rslt[1][
                                            "n"]:  # if the class number in the classmx matches the current class-average number
                                        break
                                else:
                                    print "Resultmx error: no match found ! (%d %d %d)" % (
                                        x, y, rslt[1]["n"])
                                    continue
                                xform = info[1].get_params("2d")
                                classmx[1][x, y] = info[2]  # used
                                classmx[2][x, y] = xform["tx"]  # dx
                                classmx[3][x, y] = xform["ty"]  # dy
                                classmx[4][x, y] = xform["alpha"]  # da
                                classmx[5][x, y] = xform["mirror"]  # flip
                                try:
                                    classmx[6][x, y] = xform["scale"]
                                except:
                                    pass
                    # failed average
                    elif options.storebad:
                        blk = EMData(options.ref, 0)
                        apix = blk["apix_x"]
                        blk = EMData(blk["nx"], blk["ny"], 1)
                        blk["apix_x"] = apix
                        blk.to_zero()
                        blk.set_attr("ptcl_repr", 0)
                        blk.set_attr("apix_x", apix)
                        blk.write_image(options.output, rslt[1]["n"])

            taskids = [j for i, j in enumerate(taskids) if curstat[i] != 100]

            if options.verbose and 100 in curstat:
                print "%d/%d tasks remain" % (len(taskids), len(alltaskids))
            if 100 in curstat:
                E2progress(logger,
                           1.0 - (float(len(taskids)) / len(alltaskids)))

            time.sleep(3)

        if options.verbose: print "Completed all tasks"

    # single thread
    else:
        for t in tasks:
            rslt = t.execute()
            if rslt == None: sys.exit(1)

            if rslt["average"] != None:
                rslt["average"]["class_ptcl_src"] = options.input
                if options.decayedge:
                    nx = rslt["average"]["nx"]
                    rslt["average"].process_inplace(
                        "normalize.circlemean", {"radius": nx / 2 - nx / 15})
                    rslt["average"].process_inplace(
                        "mask.gaussian", {
                            "inner_radius": nx / 2 - nx / 15,
                            "outer_radius": nx / 20
                        })
                    #rslt["average"].process_inplace("mask.decayedge2d",{"width":nx/15})
                if options.ref != None:
                    rslt["average"]["projection_image"] = options.ref
                try:
                    if options.storebad:
                        rslt["average"].write_image(options.output,
                                                    t.options["n"])
                    else:
                        rslt["average"].write_image(options.output, -1)
                except:
                    traceback.print_exc()
                    print "Error writing class average {} to {}".format(
                        t.options["n"], options.output)
                    print "Image attr: ", rslt["average"].get_attr_dict()
                    display(rslt["average"])
                    sys.exit(1)

                # Update the resultsmx if requested
                if options.resultmx != None:
                    allinfo = rslt[
                        "info"]  # the info result array list of (qual,xform,used) tuples
                    pnums = t.data["images"][
                        2]  # list of image numbers corresponding to information
                    for n, info in enumerate(allinfo):
                        y = pnums[n]  # actual particle number

                        # find the matching class in the existing classification matrix
                        for x in range(classmx[0]["nx"]):
                            if classmx[0][x, y] == rslt[
                                    "n"]:  # if the class number in the classmx matches the current class-average number
                                break
                        else:
                            print "Resultmx error: no match found ! (%d %d %d)" % (
                                x, y, rslt[1]["n"])
                            continue
                        xform = info[1].get_params("2d")
                        classmx[1][x, y] = info[2]  # used
                        classmx[2][x, y] = xform["tx"]  # dx
                        classmx[3][x, y] = xform["ty"]  # dy
                        classmx[4][x, y] = xform["alpha"]  # da
                        classmx[5][x, y] = xform["mirror"]  # flip
                        try:
                            classmx[6][x, y] = xform["scale"]
                        except:
                            pass

            # Failed average
            elif options.storebad:
                blk = EMData(options.ref, 0)
                apix = blk["apix_x"]
                blk = EMData(blk["nx"], blk["ny"], 1)
                blk["apix_x"] = apix
                blk.to_zero()
                blk.set_attr("ptcl_repr", 0)
                blk.set_attr("apix_x", apix)
                blk.write_image(options.output, t.options["n"])

    if options.resultmx != None:
        if options.verbose: print "Writing results matrix"
        for i, j in enumerate(classmx):
            j.write_image(options.resultmx, i)

    print "Class averaging complete"
    E2end(logger)
예제 #16
0
def main():
    progname = os.path.basename(sys.argv[0])
    usage = """Usage: e2spt_average.py <ref1> <ref2> ... [options] 
Note that this program is not part of the original e2spt hierarchy, but is part of an experimental refactoring.

Will read metadata from the specified spt_XX directory, as produced by e2spt_align.py, and average a selected subset of subtomograms in the predetermined orientation.
This version of the program competes each particle against N reference volumes, and only averages it with the best match. Alignment parameters from a previous
e2spt_align run are used to define the coarse orientation, so the references must be similar and in the same orientation. Alignments are translationally adjusted only.
If --sym is specified, each possible symmetric orientation is tested starting with the exisiting alignment parameters, and only the best is kept.
"""

    parser = EMArgumentParser(usage=usage, version=EMANVERSION)

    parser.add_argument(
        "--iter",
        type=int,
        help="Iteration number within path. Default = start a new iteration",
        default=0)
    parser.add_argument(
        "--simthr",
        default=-0.1,
        type=float,
        help=
        "Similarity is smaller for better 'quality' particles. Specify the highest value to include from e2spt_hist.py. Default -0.1"
    )
    parser.add_argument(
        "--simthr2",
        default=0,
        type=float,
        help=
        "Simlarity score for the best matching final alignment. Scaling may be different due to resolution limit. Default 0"
    )
    parser.add_argument(
        "--replace",
        type=str,
        default=None,
        help=
        "Replace the input subtomograms used for alignment with the specified file (used when the aligned particles were masked or filtered)"
    )
    parser.add_argument(
        "--wedgesigma",
        type=float,
        help=
        "Threshold for identifying missing data in Fourier space in terms of standard deviation of each Fourier shell. Default 3.0",
        default=3.0)
    parser.add_argument(
        "--minalt",
        type=float,
        help="Minimum alignment altitude to include. Default=0",
        default=0)
    parser.add_argument(
        "--maxalt",
        type=float,
        help="Maximum alignment altitude to include. Deafult=180",
        default=180)
    parser.add_argument(
        "--maxtilt",
        type=float,
        help=
        "Explicitly zeroes data beyond specified tilt angle. Assumes tilt axis exactly on Y and zero tilt in X-Y plane. Default 90 (no limit).",
        default=90.0)
    parser.add_argument(
        "--maxres",
        type=float,
        help=
        "Lowpass filter applied to particles prior to alignment/averaging, resolution in A. Default disabled",
        default=-1)
    parser.add_argument(
        "--listfile",
        type=str,
        help=
        "Specify a filename containing a list of integer particle numbers to include in the average, one per line, first is 0. Additional exclusions may apply.",
        default=None)
    parser.add_argument(
        "--shrinkcompare",
        type=int,
        help="Shrink factor for classification only (for speed)",
        default=0)
    parser.add_argument(
        "--sym",
        type=str,
        help=
        "Symmetry of the input. Must be aligned in standard orientation to work properly. The structure will be expanded from this symmetry to c1.",
        default="c1")
    parser.add_argument(
        "--applysym",
        type=str,
        help="Symmetry to apply to the structure after classification.",
        default="c1")
    parser.add_argument(
        "--path",
        type=str,
        default=None,
        help=
        "Path to a folder containing current results (default = highest spt_XX)"
    )
    parser.add_argument(
        "--parallel",
        type=str,
        default=None,
        help="parallel mode. Not all functions are implemented yet..")
    parser.add_argument(
        "--threads",
        default=4,
        type=int,
        help=
        "Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present."
    )
    parser.add_argument("--maskclass",
                        type=str,
                        default=None,
                        help="Mask each reference before classification")
    parser.add_argument("--mask",
                        type=str,
                        default=None,
                        help="Mask applied to final averages")
    parser.add_argument(
        "--verbose",
        "-v",
        dest="verbose",
        action="store",
        metavar="n",
        type=int,
        default=0,
        help=
        "verbose level [0-9], higher number means higher level of verboseness")
    parser.add_argument("--noali",
                        action="store_true",
                        default=False,
                        help="Skip translational alignment.")
    parser.add_argument(
        "--symcopy",
        action="store_true",
        default=False,
        help=
        "Copy each particle for each asymetrical unit. need a maskclass to focus on one unit. do not work with applysym"
    )
    parser.add_argument("--nolstout",
                        action="store_true",
                        default=False,
                        help="Skip writting lst output.")
    parser.add_argument("--sample",
                        type=int,
                        help="use only N samples.",
                        default=-1)
    parser.add_argument("--randnclass",
                        type=int,
                        help="split into N random classes. ignore refs",
                        default=-1)
    parser.add_argument(
        "--ppid",
        type=int,
        help="Set the PID of the parent process, used for cross platform PPID",
        default=-1)

    (options, args) = parser.parse_args()

    if options.parallel != None and options.parallel[:6] == "thread":
        options.threads = int(options.parallel.split(":")[1])
        options.parallel = None
        print("--parallel converted to --threads for efficiency")

    if options.path == None:
        options.path = num_path_last("spt_")
        if options.verbose: print("Working in : ", options.path)

    options.path = options.path.strip('/\\')

    if options.iter <= 0:
        fls = [
            int(i[15:17]) for i in os.listdir(options.path)
            if i[:15] == "particle_parms_" and str.isdigit(i[15:17])
        ]
        if len(fls) == 0:
            print("Cannot find a {}/particle_parms* file".format(options.path))
            sys.exit(2)
        options.iter = max(fls)
        if options.verbose: print("Using iteration ", options.iter)
        angs = js_open_dict("{}/particle_parms_{:02d}.json".format(
            options.path, options.iter))
    else:
        fls = [
            int(i[15:17]) for i in os.listdir(options.path)
            if i[:15] == "particle_parms_" and str.isdigit(i[15:17])
        ]
        if len(fls) == 0:
            print("Cannot find a {}/particle_parms* file".format(options.path))
            sys.exit(2)
        mit = max(fls)
        if options.iter > mit:
            angs = js_open_dict("{}/particle_parms_{:02d}.json".format(
                options.path, mit))
            print(
                "WARNING: no particle_parms found for iter {}, using parms from {}"
                .format(options.iter, mit))
        else:
            angs = js_open_dict("{}/particle_parms_{:02d}.json".format(
                options.path, options.iter))

    if options.listfile != None:
        plist = set([int(i) for i in open(options.listfile, "r")])

    NTHREADS = max(options.threads + 1,
                   2)  # we have one thread just writing results

    logid = E2init(sys.argv, options.ppid)

    # filter the list of particles to include
    keys = list(angs.keys())
    if options.listfile != None:
        keys = [i for i in keys if eval(i)[1] in plist]
        if options.verbose:
            print("{}/{} particles based on list file".format(
                len(keys), len(list(angs.keys()))))

    keys = [
        k for k in keys if angs[k]["score"] <= options.simthr
        and inrange(options.minalt, angs[k]["xform.align3d"].get_params("eman")
                    ["alt"], options.maxalt)
    ]
    if options.verbose:
        print("{}/{} particles after filters".format(len(keys),
                                                     len(list(angs.keys()))))

    if options.sample > 0:
        rnd = np.arange(len(keys))
        np.random.shuffle(rnd)
        rnd = rnd[:options.sample]
        keys = [keys[i] for i in rnd.tolist()]
        print("using {} samples...".format(len(keys)))

    if options.parallel:
        print(
            "Running in parallel mode. Warning, memory consumption may be high!"
        )
        if options.randnclass > 0:
            nref = options.randnclass
        else:
            nref = len(args)
        from EMAN2PAR import EMTaskCustomer
        etc = EMTaskCustomer(options.parallel,
                             module="e2spt_average_multi.SptavgmultTask")
        num_cpus = etc.cpu_est()

        print("{} total CPUs available".format(num_cpus))
        data = []  ## even/odd
        for i, k in enumerate(keys):
            src, ii = eval(k)[0], eval(k)[1]
            data.append([src, ii, angs[k]["xform.align3d"]])

        #### check and save size of particle
        fsp, i, xf = data[0]
        b = EMData(fsp, i, True)
        sz = options.boxsz = b["ny"]
        avgs = []

        tasks = [data[i::num_cpus] for i in range(num_cpus)]

        print("{} particles in {} jobs".format(len(data), len(tasks)))

        tids = []
        for t in tasks:
            task = SptavgmultTask(t, args, options)
            #task.execute(print)
            #exit()
            tid = etc.send_task(task)
            tids.append(tid)

        while 1:
            st_vals = etc.check_task(tids)
            #print("{:.1f}/{} finished".format(np.mean(st_vals), 100))
            #print(tids)
            if np.min(st_vals) == 100: break
            time.sleep(5)

        #dics=[0]*nptcl
        threeds = []
        normvols = []
        stats = []
        print("collecting from workers...")
        for i in tids:
            threed, norm, stat = etc.get_results(i)[1]
            #print(len(threed), len(norm), threed[0]["sigma"])
            threeds.append(threed)
            normvols.append(norm)
            stats.append(stat)

        stats = np.vstack(stats)
        stats = stats[np.argsort(stats[:, 0]), :]
        np.savetxt(
            "{}/avg_multi_{:02d}.txt".format(options.path, options.iter),
            stats)
        if not options.nolstout:
            lsts = [
                LSXFile(f"sets/{options.path}_{options.iter:02d}_{i:02d}.lst")
                for i in range(nref)
            ]
            for n, score, cls, x in stats:
                lsts[int(cls)].write(-1, int(n), data[0][0])
            lsts = None

        avs = []
        for r in range(nref):
            output = EMData(sz, sz, sz)
            normvol = EMData((sz // 2 + 1) * 2, sz, sz)
            output.to_zero()
            output.do_fft_inplace()

            avg = Averagers.get("mean")
            normvol.to_zero()

            for k, thd in enumerate(threeds):
                threed = thd[r]
                norm = normvols[k][r]
                threed.process_inplace("math.multamplitude", {"amp": norm})
                avg.add_image(threed)
                normvol.add(norm)

            output = avg.finish()
            normvol.process_inplace("math.reciprocal")
            output.process_inplace("math.multamplitude", {"amp": normvol})
            output.process_inplace("xform.phaseorigin.tocenter")

            output.do_ift_inplace()
            output.depad()
            avs.append(output)
    else:
        print("Running in thread mode")
        n = len(args)
        args = [comma(i) for i in args]
        refs = [EMData(i[0], i[1]) for i in args]
        if options.maxres > 0:
            for r in refs:
                r.process_inplace(
                    "filter.lowpass.gauss",
                    {"cutoff_freq": old_div(1.0, options.maxres)})
        if options.maskclass != None:
            mask = EMData(options.maskclass)
            for r in refs:
                r.mult(mask)

        jsd = queue.Queue(0)

        avgs = [
            Averagers.get("mean.tomo", {"thresh_sigma": options.wedgesigma})
            for i in range(n)
        ]

        if options.shrinkcompare > 1:
            shrinkrefs = [
                r.process("math.meanshrink", {"n": options.shrinkcompare})
                for r in refs
            ]
        else:
            shrinkrefs = refs

        # Rotation and insertion are slow, so we do it with threads.
        # Averager isn't strictly threadsafe, so possibility of slight numerical errors with a lot of threads
        if options.replace != None:
            thrds = [
                threading.Thread(target=rotfncompete,
                                 args=(jsd, avgs, options.replace, eval(k)[1],
                                       angs[k]["xform.align3d"], options.sym,
                                       refs, shrinkrefs, options.maxtilt,
                                       options.wedgesigma,
                                       options.shrinkcompare, options.maxres,
                                       options.simthr2, options.verbose))
                for i, k in enumerate(keys)
            ]

        else:
            #thrds=[threading.Thread(target=rotfncompete,args=(jsd,avgs,eval(k)[0],eval(k)[1],angs[k]["xform.align3d"],options.sym,refs,shrinkrefs,options.maxtilt,options.wedgesigma,options.shrinkcompare,options.maxres,options.simthr2,options.verbose)) for i,k in enumerate(keys)]
            thrds = [
                threading.Thread(target=rotfncompete,
                                 args=(jsd, avgs, eval(k)[0], eval(k)[1],
                                       angs[k]["xform.align3d"], refs,
                                       shrinkrefs, options))
                for i, k in enumerate(keys)
            ]

        print(len(thrds), " threads")
        thrtolaunch = 0
        out = open(
            "{}/avg_multi_{:02d}.txt".format(options.path, options.iter), "w")
        while thrtolaunch < len(
                thrds) or threading.active_count() > 1 or not jsd.empty():
            # If we haven't launched all threads yet, then we wait for an empty slot, and launch another
            # note that it's ok that we wait here forever, since there can't be new results if an existing
            # thread hasn't finished.
            if thrtolaunch < len(thrds):
                while (threading.active_count() == NTHREADS):
                    time.sleep(.1)
                if options.verbose:
                    print("Starting thread {}/{}".format(
                        thrtolaunch, len(thrds)))
                thrds[thrtolaunch].start()
                thrtolaunch += 1
            else:
                time.sleep(1)

            while not jsd.empty():
                fspn, score, ref, sym = jsd.get()
                out.write(
                    "{}\t{}\t{}\t{}\n".format(fspn, score, ref, sym)
                )  # Output columns are img #, best score, # of best ref, # of best sym
                #avg[n%2].add_image(ptcl)
            out.flush()

        for t in thrds:
            t.join()

        avs = [i.finish() for i in avgs]

    if options.mask: mask = EMData(options.mask)
    for i, v in enumerate(avs):
        if options.mask: v.mult(mask)
        #		v.write_image("{}/threed_{:02d}_{:02d}.hdf".format(options.path,options.iter,i),0)
        v.write_compressed(
            "{}/threed_{:02d}_{:02d}.hdf".format(options.path, options.iter,
                                                 i), 0, 12)

    print("Done")
    E2end(logid)
예제 #17
0
class EMParallelProject3D:
    def __init__(self, options, fsp, sym, start, modeln=0, logger=None):
        '''
		@param options the options produced by (options, args) = parser.parse_args()
		@param args the options produced by (options, args) = parser.parse_args()
		@param logger and EMAN2 logger, i.e. logger=E2init(sys.argv)
		assumes you have already called the check function.
		'''
        self.options = options
        self.args = fsp
        self.sym = sym
        self.logger = logger
        self.start = start
        self.modeln = modeln

        from EMAN2PAR import EMTaskCustomer
        self.etc = EMTaskCustomer(options.parallel)
        print "Precache ", fsp
        self.etc.precache([fsp])

        self.num_cpus = self.etc.cpu_est()
        print self.num_cpus, " total CPUs available"
        if self.num_cpus > 64:  # upper limit
            self.num_cpus = 64

        self.__task_options = None

    def __init_memory(self, options):
        '''

		'''
        sym_object = parsesym(self.sym)
        [og_name, og_args] = parsemodopt(options.orientgen)
        self.eulers = sym_object.gen_orientations(og_name, og_args)

    def __get_task_options(self, options):

        if self.__task_options == None:
            d = {}
            d["projector"] = parsemodopt(options.projector)
            d["prethreshold"] = options.prethreshold
            self.__task_options = d

        return self.__task_options

    def execute(self):
        #		from EMAN2PAR import EMTaskCustomer

        if len(self.options.parallel) > 1:
            self.__init_memory(self.options)

            num_tasks = self.num_cpus
            # In the worst case we can only spawn as many tasks as there are eulers
            if self.num_cpus > len(self.eulers): num_tasks = len(self.eulers)

            eulers_per_task = len(self.eulers) / num_tasks
            resid_eulers = len(
                self.eulers
            ) - eulers_per_task * num_tasks  # we can distribute the residual evenly

            first = 0
            task_customers = []
            tids = []
            #			self.etc=EMTaskCustomer(self.options.parallel)
            for i in xrange(0, num_tasks):
                last = first + eulers_per_task
                if resid_eulers > 0:
                    last += 1
                    resid_eulers -= 1

                tmp_eulers = self.eulers[first:last]
                indices = range(first, last)

                data = {}
                data["input"] = ("cache", self.args, 0)
                data["eulers"] = tmp_eulers
                data["indices"] = indices

                task = EMProject3DTaskDC(data=data,
                                         options=self.__get_task_options(
                                             self.options))

                #print "Est %d CPUs"%etc.cpu_est()
                tid = self.etc.send_task(task)
                #print "Task submitted tid=",tid

                tids.append(tid)

                first = last

            print "Task ids are", tids

            while 1:

                print len(tids), "projection tasks left in main loop"
                st_vals = self.etc.check_task(tids)
                for i in xrange(len(tids) - 1, -1, -1):
                    st = st_vals[i]
                    if st == 100:
                        tid = tids[i]

                        rslts = self.etc.get_results(tid)

                        if not self.__write_output_data(rslts[1]):
                            print "There was a problem with the task of id", tid

                        if self.logger != None:
                            E2progress(self.logger,
                                       1.0 - len(tids) / float(num_tasks))
                            if self.options.verbose > 0:
                                print "%d/%d\r" % (num_tasks - len(tids),
                                                   num_tasks)
                                sys.stdout.flush()

                        print "Task", tids.pop(i), "completed"
                        print "These tasks are remaining:", tids

                if len(tids) == 0: break
                time.sleep(5)

            return len(self.eulers)
        else:
            raise NotImplementedError(
                "The parallelism option you specified (%s) is not suppored" %
                self.options.parallel)

    def __write_output_data(self, rslts):
        for idx, image in rslts.items():
            if not isinstance(image, EMData):
                continue  # this is here because we get the dimensions of the database as a key (e.g. '40x40x1').
            image["model_id"] = self.modeln
            if self.options.append: image.write_image(self.options.outfile, -1)
            else: image.write_image(self.options.outfile, idx + self.start)

        return True
예제 #18
0
def main():
	
	usage="""e2classifytree.py <projection> <particle> [options]
	
	Classify particles using a binary tree. Can be used as an alternative for e2simmx2stage.py + e2classify.py.
	"""
	parser = EMArgumentParser(usage=usage,version=EMANVERSION)
	parser.add_argument("--threads", type=int,help="", default=12)
	parser.add_argument("--nodes", type=str,help="", default="nodes.hdf")
	#parser.add_argument("--clsmx", type=str,help="", default="clsmx.hdf")
	parser.add_argument("--output", type=str,help="", default="clsmx.hdf")
	parser.add_argument("--align",type=str,help="The name of an 'aligner' to use prior to comparing the images", default=None)
	parser.add_argument("--aligncmp",type=str,help="Name of the aligner along with its construction arguments",default="dot")
	parser.add_argument("--ralign",type=str,help="The name and parameters of the second stage aligner which refines the results of the first alignment", default=None)
	parser.add_argument("--raligncmp",type=str,help="The name and parameters of the comparitor used by the second stage aligner. Default is dot.",default="dot")
	parser.add_argument("--cmp",type=str,help="The name of a 'cmp' to be used in comparing the aligned images", default="dot:normalize=1")
	parser.add_argument("--cmpdiff", action="store_true", default=False ,help="Compare using the difference of the two children")
	parser.add_argument("--incomplete", type=int,help="The degree of incomplete allowed in the tree on each level", default=0)
	parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
	parser.add_argument("--parallel", default=None, help="parallelism argument")
	parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higner number means higher level of verboseness")

	(options, args) = parser.parse_args()
	E2n=E2init(sys.argv,options.ppid)
	
	options.align=parsemodopt(options.align)
	options.aligncmp=parsemodopt(options.aligncmp)
	options.ralign=parsemodopt(options.ralign)
	options.raligncmp=parsemodopt(options.raligncmp)
	options.cmp=parsemodopt(options.cmp)
	
	projs=args[0]
	#projsimmx=args[1]
	ptcl=args[1]
	npj=EMUtil.get_image_count(projs)
	npt=EMUtil.get_image_count(ptcl)
	if options.parallel==None:
		par="thread:{:d}".format(options.threads)
	else:
		par=options.parallel
		
	### Build tree
	### always overwrite the tree here now
	#if not os.path.isfile(options.nodes):
	print "Building binary tree..."
	buildtree(projs,par,options.nodes,options.incomplete,options.verbose)
	#else:
		#print "Using existing tree..."
	
	## Generate children pairs for comparison
	print "Generating children pairs for comparison..."
	if options.cmpdiff:
		nodepath= os.path.dirname(options.nodes)
		masktmp='/'.join([nodepath,"tmp_msk.hdf"])
		if os.path.isfile(masktmp): os.remove(masktmp)
		cmptmp='/'.join([nodepath,"tmp_cmp.hdf"])
		if os.path.isfile(cmptmp):
			os.remove(cmptmp)
		makechildpair(options.nodes, cmptmp, masktmp)
	else:
		masktmp=None
		cmptmp=None
	
	E2progress(E2n,0.5)
	#exit()
	print "Starting classification..."
	### Classify particles
	
		
	clsmx=[EMData(1,npt) for i in range(7)]
	nnod=EMUtil.get_image_count(options.nodes)
	if options.parallel :
		from EMAN2PAR import EMTaskCustomer
		etc=EMTaskCustomer(options.parallel)
		tasks=[]
		step=50
		tt=[range(i,i+step) for i in range(0,npt-step,step)]
		tt.append(range(tt[-1][-1]+1,npt))
		
		for it in tt:
			tasks.append(TreeClassifyTask(ptcl, it, options.nodes, options.align, options.aligncmp, options.cmp, options.ralign, options.raligncmp, cmptmp, masktmp))
		
		taskids=etc.send_tasks(tasks)
		ptclpernode=[0 for i in range(nnod)]
		nfinished=0
		while len(taskids)>0 :
			haveprogress=False
			time.sleep(3)
			curstat=etc.check_task(taskids)
			for i,j in enumerate(curstat):
				if j==100 :
					haveprogress=True
					rslt=etc.get_results(taskids[i])
					rslt= rslt[1]
					for r in rslt:
						nfinished+=1
						if options.verbose>0: print "Particle:",r["id"],"\tnodes:",r["choice"]
						for c in r["choice"]:
							ptclpernode[c]+=1
						clsmx[0].set_value_at(0,r["id"],r["cls"])
						for nt in range(1,7):
							clsmx[nt].set_value_at(0,r["id"],r["simmx"][nt])
			
			taskids=[j for i,j in enumerate(taskids) if curstat[i]!=100]
			if haveprogress: print "{:d}/{:d} finished".format(nfinished,npt)
			E2progress(E2n, 0.5 + float(nfinished)/npt)
			
		for i in range(nnod):
			ndtmp=EMData(options.nodes,i,True)
			ndtmp["tree_nptls"]=ptclpernode[i]
			ndtmp.write_image(options.nodes,i)
	
	else:
		
		### To record the number of particles in each branch of the tree
		for i in range(nnod):
			ndtmp=EMData(options.nodes,i,True)
			ndtmp["tree_nptls"]=0
			ndtmp.write_image(options.nodes,i)
		t={}
		clsmx=[EMData(1,npt) for i in range(7)]
		for i in range(options.threads):
			ai=[x for x in range(npt) if x%options.threads==i]
			t[i]=threading.Thread(target=classify,args=(ptcl,ai,options.nodes,clsmx,options.align,options.aligncmp,options.cmp,options.ralign,options.raligncmp,cmptmp,masktmp))
			t[i].start()
		for i in range(options.threads):
			t[i].join()
		
	if os.path.isfile(options.output):
		os.remove(options.output)
	for  i in clsmx:
		i.write_image(options.output,-1)
	
	if options.cmpdiff:	
		os.remove(cmptmp)
		os.remove(masktmp)
	print "Finished~"
	E2progress(E2n,1.0)
	E2end(E2n)
예제 #19
0
def main():
	progname = os.path.basename(sys.argv[0])
	usage = """Usage: e2spt_align.py [options] <subvolume_stack> <reference>
This program is part of the 'new' hierarchy of e2spt_ programs. It performs one iteration of a classical subtomogram refinement, ie -  aligning particles with missing wedge to a reference in 3-D

The reference may be <volume> or <volume>,<n>
"""

	parser = EMArgumentParser(usage=usage,version=EMANVERSION)

	parser.add_argument("--threads", default=4,type=int,help="Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present.", guitype='intbox', row=24, col=2, rowspan=1, colspan=1, mode="refinement")
	parser.add_argument("--iter",type=int,help="Iteration number within path. Default = start a new iteration",default=0)
	parser.add_argument("--goldcontinue",action="store_true",help="Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",default=False)
	parser.add_argument("--path",type=str,default=None,help="Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)")
	parser.add_argument("--sym",type=str,default="c1",help="Symmetry of the input. Must be aligned in standard orientation to work properly.")
	parser.add_argument("--maxres",type=float,help="Maximum resolution to consider in alignment (in A, not 1/A)",default=0)
	parser.add_argument("--minres",type=float,help="Minimum resolution to consider in alignment (in A, not 1/A)",default=0)
	parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higher number means higher level of verboseness")
	parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
	parser.add_argument("--maxshift", type=int, help="maximum shift for subtilt refine",default=8)
	parser.add_argument("--parallel", type=str,help="Thread/mpi parallelism to use", default=None)
	parser.add_argument("--fromscratch",action="store_true",help=".",default=False)
	parser.add_argument("--skipali3d",action="store_true",help="",default=False)
	parser.add_argument("--skipali2d",action="store_true",help="",default=False)
	parser.add_argument("--debug",action="store_true",help=".",default=False)
	parser.add_argument("--plst",type=str,default=None,help="list of 2d particle with alignment parameters. will reconstruct before alignment.")
	parser.add_argument("--smooth",type=float,help="smooth local motion by this factor. smoother local motion with larger numbers",default=-1)
	parser.add_argument("--smoothN",type=float,help="number of neighboring particles used for smoothing. default 15",default=15)

	(options, args) = parser.parse_args()
	
	if options.path == None:
		fls=[int(i[-2:]) for i in os.listdir(".") if i[:4]=="spt_" and len(i)==6 and str.isdigit(i[-2:])]
		if len(fls)==0 : fls=[0]
		options.path = "spt_{:02d}".format(max(fls)+1)
		try: os.mkdir(options.path)
		except: pass

	if options.iter<=0 :
		fls=[int(i[15:17]) for i in os.listdir(options.path) if i[:15]=="particle_parms_" and str.isdigit(i[15:17])]
		if len(fls)==0 : options.iter=1
		else: options.iter=max(fls)+1
		
	if options.parallel==None:
		options.parallel="thread:{}".format(options.threads)
		

	# file may be "name" or "name,#"
	reffile=args[1].split(",")[0]
	try: refn=int(args[1].split(",")[1])
	except: refn=0
	
	NTHREADS=max(options.threads+1,2)		# we have one thread just writing results

	logid=E2init(sys.argv, options.ppid)
	refnames=[reffile, reffile]

	if options.goldcontinue:
		ref=[]
		try:
			refnames=[reffile[:-4]+"_even.hdf", reffile[:-4]+"_odd.hdf"]
			ref.append(EMData(refnames[0],0))
			ref.append(EMData(refnames[1],0))
			
		except:
			print("Error: cannot find one of reference files, eg: ",EMData(reffile[:-4]+"_even.hdf",0))
			
	n=-1
	tasks=[]
	readjson=False
	if args[0].endswith(".lst") or args[0].endswith(".hdf"):
		nptcl=EMUtil.get_image_count(args[0])
		tasks.extend([(args[0],i,refnames, i%2) for i in range(nptcl)])
	
	elif args[0].endswith(".json"):
		js=js_open_dict(args[0])
		readjson=True
		jsinput=dict(js)
		keys=sorted(js.keys())
		nptcl=len(keys)
		for k in keys:
			src, ii=eval(k)
			dic=js[k]
			xf=dic["xform.align3d"]
			tasks.append([src, ii, refnames, ii%2, xf])
	
	
	if options.plst:
		xinfo=[[] for i in range(nptcl)]
		xfkey=["type","alt","az","phi","tx","ty","tz","alpha","scale"]
		lst=LSXFile(options.plst)
		for i in range(lst.n):
			l=lst.read(i)
			dc=eval(l[2])
			pid=dc["pid"]
			dxf=Transform({k:dc[k] for k in dc.keys() if k in xfkey})
			xinfo[pid].append([dc["tid"],i, dxf])
		
		lst=None
		for i,xn in enumerate(xinfo):
			sid=np.argsort([x[0] for x in xn])
			xinfo[i]=[[xn[s][1],xn[s][2]] for s in sid]
			
		for i in range(nptcl):
			ii=tasks[i][1]
			tasks[i].append(xinfo[ii])

	from EMAN2PAR import EMTaskCustomer
	etc=EMTaskCustomer(options.parallel, module="e2spt_align_subtlt.SptAlignTask")
	
	num_cpus = etc.cpu_est()
	options.nowtime=time.time()
	if options.debug:
		tasks=tasks[:num_cpus*4]
	print("{} jobs on {} CPUs".format(len(tasks), num_cpus))
	njob=num_cpus
	
	tids=[]
	for i in range(njob):
		t=tasks[i::njob]
		task=SptAlignTask(t, options)
		if options.debug:
			ret=task.execute(print)
			return 
		tid=etc.send_task(task)
		tids.append(tid)

	while 1:
		st_vals = etc.check_task(tids)
		if -100 in st_vals:
			print("Error occurs in parallelism. Exit")
			return
		E2progress(logid, np.mean(st_vals)/100.)
		
		if np.min(st_vals) == 100: break
		time.sleep(5)
	
	output=[]
	for i in tids:
		rets=etc.get_results(i)[1]
		for ret in rets:
			fsp,n,dic=ret
			output.append([fsp, n, dic])
		
	
	angs={}
	data=[]
	
	for out in output:
		fsp,n,dc=out
		angs[(fsp, n)]={
			"xform.align3d":dc["xform.align3d"], 
			"score":dc["score"]}
		
		ts=dc.pop("transscr")
		for i, ii in enumerate(dc["imgidx"]):
			dxf=dc["imgxfs"][i].get_params("eman")
			#dxf["score"]=float(dc["imgscore"][i])
			dxf["pid"]=n; dxf["tid"]=i; dxf["class"]=n%2
			tsmult=dc["tsmult"]
			data.append({
				"src":dc["imgsrc"], "srci":ii, "pid":n, "tid":i,
				"xf":dxf, "coord":dc["coord"], "tsscr":ts[i]
				})
	
	m=options.maxshift
	trans=np.indices((m*2+1, m*2+1)).reshape((2,-1)).T-m
	alldtx=np.zeros((len(data), 3))
	fnames=np.unique([d["src"] for d in data])
	maxtid=np.max([d["tid"] for d in data])+1
	print("Smoothing trajectories...")
	for fname in fnames:
		print(fname)

		for tid in range(maxtid):
			idx=[i for i,d in enumerate(data) if d["src"]==fname and d["tid"]==tid]
			if len(idx)==0:
				continue
			
			coord=np.array([data[i]["coord"] for i in idx])
			scrs=np.array([data[i]["tsscr"] for i in idx])
			
			if options.smooth<=0:
				s=np.argmin(scrs, 1)
				sv=np.min(scrs, axis=1)
				newdtx=np.hstack([trans[s], sv[:,None]])
				if tid==0:print("  Skip smoothing")
			else:	
				newdtx=[]
				
				for i, crd in enumerate(coord):
					dst=np.linalg.norm(coord-crd, axis=1)
					srtid=np.argsort(dst)[:options.smoothN]
					dst=dst[srtid]
					wt=np.exp(-dst/options.smooth)

					scr=scrs[srtid].copy()
					scr=np.sum(scr*wt[:,None], axis=0)
					pos=trans[np.argmin(scr)].copy()
					
					pos=pos.astype(float)*tsmult


					newdtx.append([pos[0], pos[1], np.min(scr)])

				sys.stdout.write("\r   {}: {:.4f}".format(tid, np.mean(scr)))
				sys.stdout.flush()
				newdtx=np.array(newdtx)
			
			alldtx[idx]=newdtx
		print()
		 
	for i,d in enumerate(data):
		dxf=d["xf"]
		dxf["tx"]+=alldtx[i][0]
		dxf["ty"]+=alldtx[i][1]
		dxf["score"]=alldtx[i][2]
		d["xf"]=dxf
	
	fm="{}/aliptcls_{:02d}.lst".format(options.path, options.iter)
	if os.path.isfile(fm): os.remove(fm)
	lout=LSXFile(fm, False)
	for i,d in enumerate(data):
		lout.write(-1, d["srci"], d["src"], d["xf"])
	lout=None
	
	f="{}/aliptcls_ts_{:02d}.hdf".format(options.path,options.iter)
	if os.path.isfile(f): os.remove(f)
	t=np.array([d["tsscr"] for d in data])
	m=from_numpy(t).copy()
	m.write_image(f)
	#print(f, t.shape)
	
	out="{}/particle_parms_{:02d}.json".format(options.path,options.iter)
	if os.path.isfile(out):
		os.remove(out)
	js=js_open_dict(out)
	js.update(angs)
	js.close()

	del etc

	E2end(logid)
예제 #20
0
def main():
    progname = os.path.basename(sys.argv[0])
    usage = """Usage: e2spt_align.py [options] <subvolume_stack> <reference>
Note that this program is not part of the original e2spt hierarchy, but is part of an experimental refactoring.

This program will take an input stack of subtomograms and a reference volume, and perform a missing-wedge aware alignment of each particle to the reference. If --goldstandard is specified, then even and odd particles will be aligned to different perturbed versions of the reference volume, phase-randomized past the specified resolution."""

    parser = EMArgumentParser(usage=usage, version=EMANVERSION)

    parser.add_argument(
        "--threads",
        default=4,
        type=int,
        help=
        "Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present.",
        guitype='intbox',
        row=24,
        col=2,
        rowspan=1,
        colspan=1,
        mode="refinement")
    parser.add_argument(
        "--iter",
        type=int,
        help="Iteration number within path. Default = start a new iteration",
        default=0)
    parser.add_argument(
        "--goldstandard",
        type=float,
        help=
        "If specified, will phase randomize the even and odd references past the specified resolution (in A, not 1/A)",
        default=0)
    parser.add_argument(
        "--goldcontinue",
        action="store_true",
        help=
        "Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",
        default=False)
    parser.add_argument(
        "--saveali",
        action="store_true",
        help=
        "Save a stack file (aliptcls.hdf) containing the aligned subtomograms.",
        default=False)
    parser.add_argument("--savealibin",
                        type=int,
                        help="shrink aligned particles before saving",
                        default=1)
    parser.add_argument(
        "--path",
        type=str,
        default=None,
        help=
        "Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)"
    )
    parser.add_argument(
        "--sym",
        type=str,
        default="c1",
        help=
        "Symmetry of the input. Must be aligned in standard orientation to work properly."
    )
    parser.add_argument(
        "--maxres",
        type=float,
        help="Maximum resolution to consider in alignment (in A, not 1/A)",
        default=0)
    #parser.add_argument("--wtori",type=float,help="Weight for using the prior orientation in the particle header. default is -1, i.e. not used.",default=-1)
    parser.add_argument(
        "--nsoln",
        type=int,
        help="number of solutions to keep at low resolution for the aligner",
        default=1)
    parser.add_argument(
        "--verbose",
        "-v",
        dest="verbose",
        action="store",
        metavar="n",
        type=int,
        default=0,
        help=
        "verbose level [0-9], higher number means higher level of verboseness")
    parser.add_argument(
        "--ppid",
        type=int,
        help="Set the PID of the parent process, used for cross platform PPID",
        default=-1)
    parser.add_argument("--parallel",
                        type=str,
                        help="Thread/mpi parallelism to use",
                        default=None)
    parser.add_argument("--refine",
                        action="store_true",
                        help="local refinement from xform.init in header.",
                        default=False)
    parser.add_argument(
        "--maxang",
        type=float,
        help="Maximum angular difference for the refine mode. default is 30",
        default=30)

    (options, args) = parser.parse_args()

    #task=SptAlignTask(0,1,2,options)
    #from pickle import dumps,loads,dump,load
    #f=open("task.tmp",'w')
    #dump(task,f)
    #f.close()
    #print(task)

    #return

    if options.path == None:
        fls = [
            int(i[-2:]) for i in os.listdir(".")
            if i[:4] == "spt_" and len(i) == 6 and str.isdigit(i[-2:])
        ]
        if len(fls) == 0: fls = [0]
        options.path = "spt_{:02d}".format(max(fls) + 1)
        try:
            os.mkdir(options.path)
        except:
            pass

    if options.iter <= 0:
        fls = [
            int(i[15:17]) for i in os.listdir(options.path)
            if i[:15] == "particle_parms_" and str.isdigit(i[15:17])
        ]
        if len(fls) == 0: options.iter = 1
        else: options.iter = max(fls) + 1

    if options.parallel == None:
        options.parallel = "thread:{}".format(options.threads)

    reffile = args[1]
    NTHREADS = max(options.threads + 1,
                   2)  # we have one thread just writing results

    logid = E2init(sys.argv, options.ppid)

    if options.goldcontinue:
        ref = []
        try:
            ref.append(EMData(reffile[:-4] + "_even.hdf", 0))
            ref.append(EMData(reffile[:-4] + "_odd.hdf", 0))
        except:
            print("Error: cannot find one of reference files, eg: ",
                  EMData(reffile[:-4] + "_even.hdf", 0))
    else:
        ref = []
        ref.append(EMData(reffile, 0))
        ref.append(EMData(reffile, 0))

        if options.goldstandard > 0:
            ref[0].process_inplace(
                "filter.lowpass.randomphase",
                {"cutoff_freq": old_div(1.0, options.goldstandard)})
            ref[0].process_inplace(
                "filter.lowpass.gauss",
                {"cutoff_freq": old_div(1.0, options.goldstandard)})
            ref[1].process_inplace(
                "filter.lowpass.randomphase",
                {"cutoff_freq": old_div(1.0, options.goldstandard)})
            ref[1].process_inplace(
                "filter.lowpass.gauss",
                {"cutoff_freq": old_div(1.0, options.goldstandard)})
            ref[0].write_image("{}/align_ref.hdf".format(options.path), 0)
            ref[1].write_image("{}/align_ref.hdf".format(options.path), 1)

    ref[0] = ref[0].do_fft()
    ref[0].process_inplace("xform.phaseorigin.tocorner")
    ref[1] = ref[1].do_fft()
    ref[1].process_inplace("xform.phaseorigin.tocorner")

    jsd = queue.Queue(0)

    n = -1
    #### check if even/odd split exists
    fsps = [args[0][:-4] + "__even.lst", args[0][:-4] + "__odd.lst"]
    tasks = []
    if os.path.isfile(fsps[0]) and os.path.isfile(fsps[1]):
        print("Using particle list: \n\t {} \n\t {}".format(fsps[0], fsps[1]))
        for eo, f in enumerate(fsps):
            N = EMUtil.get_image_count(f)
            tasks.extend([(f, i, ref[eo]) for i in range(N)])

    #### split by even/odd by default
    else:
        N = EMUtil.get_image_count(args[0])
        tasks.extend([(args[0], i, ref[i % 2]) for i in range(N)])
        #thrds=[threading.Thread(target=alifn,args=(jsd,args[0],i,ref[i%2],options)) for i in range(N)]

    from EMAN2PAR import EMTaskCustomer
    etc = EMTaskCustomer(options.parallel, module="e2spt_align.SptAlignTask")
    num_cpus = etc.cpu_est()

    #tasks=tasks[:24]
    print("{} total CPUs available".format(num_cpus))
    print("{} jobs".format(len(tasks)))

    tids = []
    for t in tasks:
        task = SptAlignTask(t[0], t[1], t[2], options)
        tid = etc.send_task(task)
        tids.append(tid)

    while 1:
        st_vals = etc.check_task(tids)
        #print("{:.1f}/{} finished".format(np.mean(st_vals), 100))
        #print(tids)
        if np.min(st_vals) == 100: break
        time.sleep(5)

    #dics=[0]*nptcl

    angs = {}
    for i in tids:
        ret = etc.get_results(i)[1]
        fsp, n, dic = ret
        if len(dic) == 1:
            angs[(fsp, n)] = dic[0]
        else:
            angs[(fsp, n)] = dic

    js = js_open_dict("{}/particle_parms_{:02d}.json".format(
        options.path, options.iter))
    js.update(angs)

    del etc

    ## here we run the threads and save the results, no actual alignment done here
    #print(len(thrds)," threads")
    #thrtolaunch=0
    #while thrtolaunch<len(thrds) or threading.active_count()>1:
    ## If we haven't launched all threads yet, then we wait for an empty slot, and launch another
    ## note that it's ok that we wait here forever, since there can't be new results if an existing
    ## thread hasn't finished.
    #if thrtolaunch<len(thrds) :
    #while (threading.active_count()==NTHREADS ) : time.sleep(.1)
    #if options.verbose : print("Starting thread {}/{}".format(thrtolaunch,len(thrds)))
    #thrds[thrtolaunch].start()
    #thrtolaunch+=1
    #else: time.sleep(1)

    #while not jsd.empty():
    #fsp,n,d=jsd.get()
    #angs[(fsp,n)]=d
    #if options.saveali:
    #v=EMData(fsp,n)
    #v.transform(d["xform.align3d"])
    #if options.savealibin>1:
    #v.process_inplace("math.meanshrink",{"n":options.savealibin})
    #v.write_image("{}/aliptcls_{:02d}.hdf".format(options.path, options.iter),n)

    #for t in thrds:
    #t.join()

    E2end(logid)
예제 #21
0
class EMParallelSimMX(object):
    def __init__(self, options, args, logger=None):
        '''
		@param options the options produced by (options, args) = parser.parse_args()
		@param args the options produced by (options, args) = parser.parse_args()
		@param logger and EMAN2 logger, i.e. logger=E2init(sys.argv)
		assumes you have already called the check function.
		'''
        self.options = options
        self.args = args
        self.logger = logger

        from EMAN2PAR import EMTaskCustomer
        self.etc = EMTaskCustomer(options.parallel,
                                  module="e2simmx.EMSimTaskDC")
        if options.colmasks != None:
            self.etc.precache([args[0], args[1], options.colmasks])
        else:
            self.etc.precache([args[0], args[1]])
        self.num_cpus = self.etc.cpu_est()
        if self.num_cpus < 32:  # lower limit
            self.num_cpus = 32

        self.__task_options = None

    def __get_task_options(self, options):
        '''
		Get the options required by each task as a dict
		@param options is always self.options - the initialization argument. Could be changed.
		'''
        if self.__task_options == None:
            d = {}
            d["align"] = parsemodopt(options.align)
            d["aligncmp"] = parsemodopt(options.aligncmp)
            d["cmp"] = parsemodopt(options.cmp)

            if hasattr(options, "ralign") and options.ralign != None:
                d["ralign"] = parsemodopt(options.ralign)
                d["raligncmp"] = parsemodopt(
                    options.raligncmp
                )  # raligncmp must be specified if using ralign
            else:
                d["ralign"] = None
                d["raligncmp"] = None
            d["prefilt"] = options.prefilt

            if hasattr(options, "shrink") and options.shrink != None:
                d["shrink"] = options.shrink
            else:
                d["shrink"] = None

            self.__task_options = d

        return self.__task_options

    def __init_memory(self, options):
        '''
		@param options is always self.options - the initialization argument. Could be changed.
		Establishes several important attributes they are:
		----
		self.clen - the number of images in the image defined by args[0], the number of columns in the similarity matrix
		self.rlen - the number of images in the image defined by args[1], the number of rows in the similarity matrix
		----
		Also, since we adopted region output writing as our preferred approach, this function makes sure the output
		image(s) exists on disk and has the correct dimensions - seeing as this is the way region writing works (the image
		has to exist on disk and have its full dimensions)
		'''
        self.clen = EMUtil.get_image_count(self.args[0])
        self.rlen = EMUtil.get_image_count(self.args[1])

        output = self.args[2]

        if file_exists(output) and not options.fillzero:
            remove_file(output)

        e = EMData(self.clen, self.rlen)
        e.to_zero()
        e.set_attr(PROJ_FILE_ATTR, self.args[0])
        e.set_attr(PART_FILE_ATTR, self.args[1])
        n = 1
        if self.options.saveali:
            n = 6  # the total number of images written to disk
        if not options.fillzero: e.write_image(output, 0)
        for i in range(1, n):
            e.write_image(output, i)

    def __get_blocks(self):
        '''
		Gets the blocks that will be processed in parallel, these are essentially ranges
		'''

        steve_factor = 3  # increase number of jobs a bit for better distribution
        total_jobs = steve_factor * self.num_cpus

        [col_div,
         row_div] = opt_rectangular_subdivision(self.clen, self.rlen,
                                                total_jobs)

        block_c = old_div(self.clen, col_div)
        block_r = old_div(self.rlen, row_div)

        residual_c = self.clen - block_c * col_div  # residual left over by integer division

        blocks = []

        current_c = 0
        for c in range(0, col_div):
            last_c = current_c + block_c
            if residual_c > 0:
                last_c += 1
                residual_c -= 1

            current_r = 0
            residual_r = self.rlen - block_r * row_div  # residual left over by integer division
            for r in range(0, row_div):
                last_r = current_r + block_r
                if residual_r > 0:
                    last_r += 1
                    residual_r -= 1

                blocks.append([current_c, last_c, current_r, last_r])
                current_r = last_r

            current_c = last_c

#		print col_div,row_div,col_div*row_div
#		print self.clen,self.rlen,residual_c,residual_r
        return blocks

    def execute(self):
        '''
		The main function to be called
		'''
        if len(self.options.parallel) > 1:
            self.__init_memory(self.options)
            blocks = self.__get_blocks()
            #			print blocks

            #			self.check_blocks(blocks) # testing function can be removed at some point

            tasks = []
            for bn, block in enumerate(blocks):

                data = {}
                data["references"] = ("cache", self.args[0], block[0],
                                      block[1])
                data["particles"] = ("cache", self.args[1], block[2], block[3])
                if self.options.colmasks != None:
                    data["colmasks"] = ("cache", self.options.colmasks,
                                        block[0], block[1])
                if self.options.mask != None:
                    data["mask"] = ("cache", self.options.mask, 0, 1)
                if self.options.fillzero:
                    # for each particle check to see which portion of the matrix we need to fill
                    if (bn % 10 == 0):
                        print("%d/%d     \r" % (bn, len(blocks)), end=' ')
                    sys.stdout.flush()
                    rng = []
                    for i in range(block[2], block[3]):
                        c = EMData()
                        c.read_image(
                            self.args[2], 0, False,
                            Region(block[0], i, block[1] - block[0] + 1, 1))
                        inr = 0
                        st = 0
                        for j in range(c["nx"]):
                            if c[j] == 0 and not inr:
                                st = j
                                inr = 1
                            if c[j] != 0 and inr:
                                rng.append(
                                    (i, st + block[0], j - 1 + block[0]))
                                inr = 0
                        if inr:
                            rng.append((i, st + block[0], j + block[0]))
                    data["partial"] = rng


#					print "%d) %s\t"%(bn,str(block)),rng

                if self.options.fillzero and len(data["partial"]) == 0:
                    continue  # nothing to compute in this block, skip it completely
                else:
                    task = EMSimTaskDC(data=data,
                                       options=self.__get_task_options(
                                           self.options))
                    #print "Est %d CPUs"%etc.cpu_est()
                    tasks.append(task)

            # This just verifies that all particles have at least one class
            #a=set()
            #for i in tasks:
            #for k in i.data["partial"] : a.add(k[0])

            #b=set(range(self.rlen))
            #b-=a
            #print b

            print("%d/%d         " % (bn, len(blocks)))
            self.tids = self.etc.send_tasks(tasks)
            print(len(self.tids), " tasks submitted")
            #
            while 1:
                if len(self.tids) == 0: break
                print(len(self.tids),
                      "simmx tasks left in main loop   \r",
                      end=' ')
                sys.stdout.flush()
                st_vals = self.etc.check_task(self.tids)
                for i in range(len(self.tids) - 1, -1, -1):
                    st = st_vals[i]
                    if st == 100:
                        tid = self.tids[i]

                        try:
                            rslts = self.etc.get_results(tid)
                            #							display(rslts[1]["rslt_data"][0])
                            self.__store_output_data(rslts[1])
                        except:
                            traceback.print_exc()
                            print(
                                "ERROR storing results for task %d. Rerunning."
                                % tid)
                            self.etc.rerun_task(tid)
                            continue
                        if self.logger != None:
                            E2progress(
                                self.logger, 1.0 -
                                old_div(len(self.tids), float(len(blocks))))
                            if self.options.verbose > 0:
                                print("%d/%d\r" %
                                      (len(self.tids), len(blocks)))
                                sys.stdout.flush()

                        self.tids.pop(i)
                    print(len(self.tids),
                          "simmx tasks left in main loop   \r",
                          end=' ')
                    sys.stdout.flush()

                time.sleep(10)
            print("\nAll simmx tasks complete ")

            # if using fillzero, we must fix the -1.0e38 values placed into empty cells
            if self.options.fillzero:
                l = EMData(self.args[2], 0, True)
                rlen = l["ny"]
                clen = l["nx"]
                #				launch_childprocess("e2proc2d.py %s %s"%(self.args[2],self.args[2]+"_x"))
                print(
                    "Filling noncomputed regions in similarity matrix (%dx%d)"
                    % (clen, rlen))
                l = EMData()
                for r in range(rlen):
                    l.read_image(self.args[2], 0, False, Region(0, r, clen, 1))
                    fill = l["maximum"] + .0001
                    l.process_inplace("threshold.belowtominval", {
                        "minval": -1.0e37,
                        "newval": fill
                    })
                    l.write_image(self.args[2], 0,
                                  EMUtil.ImageType.IMAGE_UNKNOWN, False,
                                  Region(0, r, clen, 1))

                print("Filling complete")

        else:
            raise NotImplementedError(
                "The parallelism option you specified (%s) is not supported" %
                self.options.parallel)

    def __store_output_data(self, rslts):
        '''
		Store output data to internal images (matrices)
		@param a dictionary return by the EMSimTaskDC
		'''

        result_data = rslts["rslt_data"]
        output = self.args[2]

        insertion_c = rslts["min_ref_idx"]
        insertion_r = rslts["min_ptcl_idx"]
        result_mx = result_data[0]
        r = Region(insertion_c, insertion_r, result_mx.get_xsize(),
                   result_mx.get_ysize())

        # Note this is region io - the init_memory function made sure the images exist and are the right dimensions (on disk)
        for i, mxout in enumerate(result_data):
            mxout.write_image(output, i, EMUtil.ImageType.IMAGE_UNKNOWN, False,
                              r)
예제 #22
0
def main():
	progname = os.path.basename(sys.argv[0])
	usage = """Usage: e2spt_align.py [options] <subvolume_stack> <reference>
This program is part of the 'new' hierarchy of e2spt_ programs. It performs one iteration of a classical subtomogram refinement, ie -  aligning particles with missing wedge to a reference in 3-D

The reference may be <volume> or <volume>,<n>

If --goldstandard is specified, even and odd variants of the alignment reference must be provided, and even and odd particles will be aligned separately"""

	parser = EMArgumentParser(usage=usage,version=EMANVERSION)

	parser.add_argument("--threads", default=4,type=int,help="Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present.", guitype='intbox', row=24, col=2, rowspan=1, colspan=1, mode="refinement")
	parser.add_argument("--iter",type=int,help="Iteration number within path. Default = start a new iteration",default=0)
	parser.add_argument("--goldstandard",type=float,help="If specified, will phase randomize the even and odd references past the specified resolution (in A, not 1/A)",default=0)
	parser.add_argument("--goldcontinue",action="store_true",help="Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",default=False)
	#parser.add_argument("--saveali",action="store_true",help="Save a stack file (aliptcls.hdf) containing the aligned subtomograms.",default=False)
	#parser.add_argument("--savealibin",type=int,help="shrink aligned particles before saving",default=1)
	parser.add_argument("--mask",type=str,default=None,help="Mask file aligned to the input reference. Alignment occurs under this mask.")
	parser.add_argument("--path",type=str,default=None,help="Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)")
	parser.add_argument("--sym",type=str,default="c1",help="Symmetry of the input. Must be aligned in standard orientation to work properly.")
	parser.add_argument("--maxres",type=float,help="Maximum resolution (the smaller number) to consider in alignment (in A, not 1/A)",default=0)
	parser.add_argument("--minres",type=float,help="Minimum resolution (the larger number) to consider in alignment (in A, not 1/A)",default=0)
	#parser.add_argument("--wtori",type=float,help="Weight for using the prior orientation in the particle header. default is -1, i.e. not used.",default=-1)
	parser.add_argument("--nsoln",type=int,help="number of solutions to keep at low resolution for the aligner",default=1)
	parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higher number means higher level of verboseness")
	parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
	parser.add_argument("--parallel", type=str,help="Thread/mpi parallelism to use", default=None)
	parser.add_argument("--transonly",action="store_true",help="translational alignment only, for prealigned particles",default=False)
	parser.add_argument("--refine",action="store_true",help="local refinement from xform.align3d in header.",default=False)
	parser.add_argument("--flcf",action="store_true",help="use slower aligner (experimental)",default=False)
	
	parser.add_argument("--refinentry", type=int, help="number of tests for refine mode. default is 8",default=8)
	parser.add_argument("--randphi",action="store_true",help="randomize phi during refine alignment",default=False)
	parser.add_argument("--breaksym",action="store_true",help="symmetry breaking.",default=False)
	parser.add_argument("--breaksymsym",type=str,help="the symmetry to use for breaksym. setting sym to c6 and this to c2 results in a c3 structure. default is the same as sym",default=None)
	parser.add_argument("--rand180",action="store_true",help="randomly add a 180 degree rotation during refine alignment",default=False)
	parser.add_argument("--test180",action="store_true",help="Test for improved alignment with 180 degree rotations even during refine alignment",default=False)
	parser.add_argument("--skipali",action="store_true",help="skip alignment. the program will do nothing. mostly for testing...",default=False)
	parser.add_argument("--maxang",type=float,help="Maximum angular difference for the refine mode. default is 30",default=30)
	parser.add_argument("--maxshift",type=float,help="Maximum shift for the refine mode. default is 16",default=-1)
	parser.add_argument("--scipytest",action="store_true",help="test scipy optimizer.",default=False)
	parser.add_argument("--debug",action="store_true",help=".",default=False)



	(options, args) = parser.parse_args()
	
	if options.path == None:
		options.path=num_path_new("spt_")

	if options.iter<=0 :
		fls=[int(i[15:17]) for i in os.listdir(options.path) if i[:15]=="particle_parms_" and str.isdigit(i[15:17])]
		if len(fls)==0 : options.iter=1
		else: options.iter=max(fls)+1
		
	if options.parallel==None:
		options.parallel="thread:{}".format(options.threads)
		
	if options.breaksym:
		if options.breaksymsym==None:
			if options.sym=="c1":
				print("cannot break a c1 symmetry. breaksym disabled.")
				options.breaksym=False
			else:
				options.breaksymsym=options.sym
		

	# file may be "name" or "name,#"
	reffile=args[1].split(",")[0]
	try: refn=int(args[1].split(",")[1])
	except: refn=0
	
	NTHREADS=max(options.threads+1,2)		# we have one thread just writing results

	logid=E2init(sys.argv, options.ppid)
	refnames=[reffile,reffile]

	if options.goldcontinue or options.goldstandard>0:
		ref=[]
		try:
			refnames=[reffile[:-4]+"_even.hdf", reffile[:-4]+"_odd.hdf"]
			ref.append(EMData(refnames[0],0))
			ref.append(EMData(refnames[1],0))
			
		except:
			print("Error: cannot find one of reference files, eg: ",EMData(reffile[:-4]+"_even.hdf",0))
#	else:
#		ref=[]
#		ref.append(EMData(reffile,refn))
#		ref.append(EMData(reffile,refn))
#
#		if options.goldstandard>0 :
#			ref[0].process_inplace("filter.lowpass.randomphase",{"cutoff_freq":old_div(1.0,options.goldstandard)})
#			ref[0].process_inplace("filter.lowpass.gauss",{"cutoff_freq":old_div(1.0,options.goldstandard)})
#			ref[1].process_inplace("filter.lowpass.randomphase",{"cutoff_freq":old_div(1.0,options.goldstandard)})
#			ref[1].process_inplace("filter.lowpass.gauss",{"cutoff_freq":old_div(1.0,options.goldstandard)})
#			refnames=["{}/align_ref_even.hdf".format(options.path), "{}/align_ref_odd.hdf".format(options.path)]
#			ref[0].write_image(refnames[0],0)
#			ref[1].write_image(refnames[1],0)
#			
#		else:
#			refnames=[reffile, reffile]

	n=-1
	tasks=[]
	readjson=False
	if args[0].endswith(".lst") or args[0].endswith(".hdf"):
		#### check if even/odd split exists
		fsps=[args[0][:-4]+"__even.lst",args[0][:-4]+"__odd.lst"]
		
		if os.path.isfile(fsps[0]) and os.path.isfile(fsps[1]):
			print("Using particle list: \n\t {} \n\t {}".format(fsps[0], fsps[1]))
			for eo, f in enumerate(fsps):
				N=EMUtil.get_image_count(f)
				tasks.extend([(f,i,refnames, eo) for i in range(N)])
				
		#### split by even/odd by default
		else:
			N=EMUtil.get_image_count(args[0])
			tasks.extend([(args[0],i,refnames, i%2) for i in range(N)])
			#thrds=[threading.Thread(target=alifn,args=(jsd,args[0],i,ref[i%2],options)) for i in range(N)]
	
	elif args[0].endswith(".json"):
		#print("Reading particles from json. This is experimental...")
		js=js_open_dict(args[0])
		readjson=True
		jsinput=dict(js)
		keys=sorted(js.keys())
		for k in keys:
			src, ii=eval(k)
			dic=js[k]
			xf=dic["xform.align3d"]
			tasks.append([src, ii, refnames, ii%2, xf])
			


	from EMAN2PAR import EMTaskCustomer
	if options.scipytest:
		etc=EMTaskCustomer(options.parallel, module="e2spt_align.ScipySptAlignTask")
	else:
		etc=EMTaskCustomer(options.parallel, module="e2spt_align.SptAlignTask")
	num_cpus = etc.cpu_est()
	options.nowtime=time.time()
	print("{} jobs on {} CPUs".format(len(tasks), num_cpus))
	njob=num_cpus#*4
	
	tids=[]
	for i in range(njob):
		t=tasks[i::njob]
		if options.scipytest:
			task=ScipySptAlignTask(t, options)
		else:
			task=SptAlignTask(t, options)
		if options.debug:
			ret=task.execute(print)
			print(ret)
			return 
		tid=etc.send_task(task)
		tids.append(tid)

	while 1:
		st_vals = etc.check_task(tids)
		#print(st_vals)
		if -100 in st_vals:
			print("Error occurs in parallelism. Exit")
			return
		E2progress(logid, np.mean(st_vals)/100.)
		
		if np.min(st_vals) == 100: break
		time.sleep(5)

	#dics=[0]*nptcl
	
	angs={}
	for i in tids:
		rets=etc.get_results(i)[1]
		for ret in rets:
			fsp,n,dic=ret
			if len(dic)==1:
				dic=dic[0]
				
			if readjson:
				k=str((fsp,n))
				if "eo" in js[k]:
					dic["eo"]=jsinput[k]["eo"]
			angs[(fsp,n)]=dic
		
	out="{}/particle_parms_{:02d}.json".format(options.path,options.iter)
	if os.path.isfile(out):
		os.remove(out)
	js=js_open_dict(out)
	js.update(angs)
	js.close()

	del etc

	E2end(logid)
예제 #23
0
def main():
	progname = os.path.basename(sys.argv[0])
	usage = """prog [options]
	This program will take a set of reference-free class-averages (or other projections) and generate a set of possible
	3-D initial models. It does this by heavily downsampling the data, then running a number of very fast, full iterative
	refinements, each seeded with a random starting model. The results are sorted in order of apparent agreement with the
	data, such that at the end, the first numbered model should be the best result. Ideally the top few answers will all
	qualtitatively agree on the overall structure. If they do not, the results should be thoroughly assessed manually to
	insure a sensible result. By default this routine will generate 10 initial models, but this may be fewer or more than
	is strictly necessary depending on a number of factors. If the data is highly structurally heterogeneous, particularly
	if combined with a strongly preferred orientation, a correct solution using this technique may not be possible, but
	for most situations it will work well. For other situations, single particle tomography presents a good alternative
	for generating initial models."""
	parser = EMArgumentParser(usage=usage,version=EMANVERSION)

	parser.add_header(name="initialmodelheader", help='Options below this label are specific to e2initialmodel', title="### e2initialmodel options ###", row=1, col=0, rowspan=1, colspan=3)
	parser.add_argument("--input", dest="input", default=None,type=str, help="This file should contain good class-averages to use in constructing the initial model", browser='EMBrowserWidget(withmodal=True,multiselect=False)', guitype='filebox', row=0, col=0, rowspan=1, colspan=3)
	parser.add_argument("--iter", type = int, default=8, help = "The total number of refinement iterations to perform, typically 5-10", guitype='intbox', row=2, col=0, rowspan=1, colspan=1)
	parser.add_argument("--tries", type=int, default=10, help="The number of different initial models to generate in search of a good one", guitype='intbox', row=2, col=1, rowspan=1, colspan=1)
	parser.add_argument("--shrink", dest="shrink", type = int, default=0, help="Optionally shrink the input particles by an integer factor prior to reconstruction. Default=0, no shrinking", guitype='shrinkbox', row=2, col=2, rowspan=1, colspan=1)
	parser.add_argument("--sym", dest = "sym", help = "Specify symmetry - choices are: c<n>, d<n>, h<n>, tet, oct, icos",default="c1", guitype='symbox', row=4, col=0, rowspan=1, colspan=2)
	parser.add_argument("--randorient",action="store_true",help="Instead of seeding with a random volume, seeds by randomizing input orientations",default=False, guitype='boolbox', row=4, col=2, rowspan=1, colspan=1)
	parser.add_argument("--maskproc", default=None, type=str,help="Default=none. If specified, this mask will be performed after the built-in automask, eg - mask.soft to remove the core of a virus", )
#	parser.add_argument("--savemore",action="store_true",help="Will cause intermediate results to be written to flat files",default=False, guitype='boolbox', expert=True, row=5, col=0, rowspan=1, colspan=1)
	parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higner number means higher level of verboseness")
	parser.add_argument("--orientgen",type=str, default="eman:delta=9.0:inc_mirror=0:perturb=1",help="The type of orientation generator. Default is eman:delta=9.0:inc_mirror=0:perturb=1. See e2help.py orientgens", guitype='strbox', expert=True, row=4, col=2, rowspan=1, colspan=1)
	parser.add_argument("--parallel","-P",type=str,help="Run in parallel, specify type:<option>=<value>:<option>=<value>. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel",default="thread:1", guitype='strbox', row=6, col=0, rowspan=1, colspan=2)
	parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)

	# Database Metadata storage
	#parser.add_argument("--dbls",type=str,default=None,help="data base list storage, used by the workflow. You can ignore this argument.")

	(options, args) = parser.parse_args()
	verbose=options.verbose

	try: ptcls=EMData.read_images(options.input)
	except:
		print "Error: bad input file"
		exit(1)
	apix=ptcls[0]["apix_x"]
	if options.shrink>1 : apix*=options.shrink

	for i in range(len(ptcls)):
		ptcls[i].process_inplace("normalize.edgemean",{})
		if options.shrink>1 :
			ptcls[i]=ptcls[i].process("math.meanshrink",{"n":options.shrink})
	if ptcls[0]["nx"]>160 : print "WARNING: using a large box size may be slow. Suggest trying --shrink="
	if not ptcls or len(ptcls)==0 : parser.error("Bad input file")
	boxsize=ptcls[0].get_xsize()
	if verbose>0 : print "%d particles %dx%d"%(len(ptcls),boxsize,boxsize)
	print "Models will be %1.3f A/pix"%apix

	[og_name,og_args] = parsemodopt(options.orientgen)

	try:
			sfcurve=XYData()
			sfcurve.read_file("strucfac.txt")

			sfcurve.update()
	except : sfcurve=None

	if options.maskproc!=None :
		mask2=EMData(boxsize,boxsize,boxsize)
		mask2.to_one()
		parms=parsemodopt(options.maskproc)
		if parms[0]=="mask.auto3d":
			print "Error, maskproc may not be mask.auto3d, it must be a processor that does not rely on the input map density to function"
			sys.exit(1)
		mask2.process_inplace(parms[0],parms[1])
	else: mask2=None

	# angles to use for refinement
	sym_object = parsesym(options.sym)
	orts = sym_object.gen_orientations(og_name,og_args)

	logid=E2init(sys.argv,options.ppid)
	results=[]

	try: os.mkdir("initial_models")
	except: pass
	iters=[int(i[10:12]) for i in os.listdir("initial_models") if i[:10]=="particles_"]
	try : newiter=max(iters)+1
	except : newiter=0
	results_name="initial_models/model_%02d"%newiter
	particles_name="initial_models/particles_%02d.hdf"%newiter

	# we write the pre-processed "particles" (usually class-averages) to disk, both as a record and to prevent collisions
	for i,p in enumerate(ptcls):
		p.write_image(particles_name,i)

	# parallelism
	from EMAN2PAR import EMTaskCustomer			# we need to put this here to avoid a circular reference

	etc=EMTaskCustomer(options.parallel)
	pclist=[particles_name]

	etc.precache(pclist)		# make sure the input particles are precached on the compute nodes

	tasks=[]
	for t in xrange(options.tries):
		tasks.append(InitMdlTask(particles_name,len(ptcls),orts,t,sfcurve,options.iter,options.sym,mask2,options.randorient,options.verbose))

	taskids=etc.send_tasks(tasks)
	alltaskids=taskids[:]			# we keep a copy for monitoring progress

	# This loop runs until all subtasks are complete (via the parallelism system
	ltime=0
	while len(taskids)>0 :
		time.sleep(0.1)
		curstat=etc.check_task(taskids)			# a list of the progress on each task
		if options.verbose>1 :
			if time.time()-ltime>1 :
				print "progress: ",curstat
				ltime=time.time()
		for i,j in enumerate(curstat):
			if j==100 :
				rslt=etc.get_results(taskids[i])		# read the results back from a completed task as a one item dict
				results.append(rslt[1]["result"])
				if options.verbose==1 : print "Task {} ({}) complete".format(i,taskids[i])

		# filter out completed tasks. We can't do this until after the previous loop completes
		taskids=[taskids[i] for i in xrange(len(taskids)) if curstat[i]!=100]


	# Write out the final results
	results.sort()
	for i,j in enumerate(results):
		out_name = results_name+"_%02d.hdf"%(i+1)
		j[1].write_image(out_name,0)
		j[4].write_image(results_name+"_%02d_init.hdf"%(i+1),0)
		print out_name,j[1]["quality"],j[0],j[1]["apix_x"]
		for k,l in enumerate(j[3]): l[0].write_image(results_name+"_%02d_proj.hdf"%(i+1),k)	# set of projection images
		for k,l in enumerate(j[2]):
			l.process("normalize").write_image(results_name+"_%02d_aptcl.hdf"%(i+1),k*2)						# set of aligned particles
			j[3][l["match_n"]][0].process("normalize").write_image(results_name+"_%02d_aptcl.hdf"%(i+1),k*2+1)	# set of projections matching aligned particles


	E2end(logid)
예제 #24
0
def main():
    progname = os.path.basename(sys.argv[0])
    usage = """
"""

    parser = EMArgumentParser(usage=usage, version=EMANVERSION)

    parser.add_argument(
        "--path",
        type=str,
        help=
        "Path to a folder where results should be stored, following standard naming conventions",
        default="subtlt_00")
    parser.add_argument("--ref", type=str, help="reference map", default=None)
    parser.add_argument(
        "--iter",
        type=int,
        help="Iteration number within path. Default = start a new iteration",
        default=0)
    parser.add_argument(
        "--goldcontinue",
        action="store_true",
        help=
        "Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",
        default=False)
    parser.add_argument(
        "--maxres",
        type=float,
        help="Maximum resolution to consider in alignment (in A, not 1/A)",
        default=0)
    parser.add_argument(
        "--minres",
        type=float,
        help="Minimum resolution to consider in alignment (in A, not 1/A)",
        default=0)
    parser.add_argument(
        "--smooth",
        type=float,
        help=
        "smooth local motion by this factor. smoother local motion with larger numbers. default 100",
        default=100)
    parser.add_argument(
        "--smoothN",
        type=int,
        help="number of neighboring particles used for smoothing. default 15",
        default=15)
    parser.add_argument("--maxshift",
                        type=float,
                        help="max shift in pixel. default default box size/6",
                        default=-1)
    parser.add_argument("--refine_trans",
                        action="store_true",
                        help="do translational alignment.",
                        default=False)
    parser.add_argument("--refine_trans_ccf",
                        action="store_true",
                        help="do translational alignment using simple ccf.",
                        default=False)
    parser.add_argument(
        "--refine_rot",
        action="store_true",
        help=
        "do translational-rotational alignment. better to start from an existing translational alignment.",
        default=False)
    parser.add_argument(
        "--refine_defocus",
        action="store_true",
        help=
        "do defocus refinement. need aliptcls input. doesn't work with refine_trans or rot yet..",
        default=False)
    parser.add_argument(
        "--use3d",
        action="store_true",
        help="use projection of 3d particles instead of 2d ones..",
        default=False)
    parser.add_argument(
        "--preprocess",
        metavar="processor_name:param1=value1:param2=value2",
        type=str,
        default=None,
        help="Preprocess each 2-D subtilt while loading (alignment only)")

    parser.add_argument(
        "--aliptcls2d",
        type=str,
        help=
        "optional aliptcls input. the program can start search from the position from last run.",
        default="")
    parser.add_argument("--aliptcls3d",
                        type=str,
                        help="optional aliptcls input.",
                        default="")

    parser.add_argument("--parallel",
                        type=str,
                        help="Thread/mpi parallelism to use",
                        default="thread:4")
    parser.add_argument("--debug",
                        action="store_true",
                        help="for testing.",
                        default=False)
    parser.add_argument(
        "--verbose",
        "-v",
        dest="verbose",
        action="store",
        metavar="n",
        type=int,
        default=0,
        help=
        "verbose level [0-9], higher number means higher level of verboseness")
    parser.add_argument(
        "--ppid",
        type=int,
        help="Set the PID of the parent process, used for cross platform PPID",
        default=-1)

    (options, args) = parser.parse_args()
    logid = E2init(sys.argv, options.ppid)

    options.info2dname = "{}/particle_info_2d.lst".format(options.path)
    options.info3dname = "{}/particle_info_3d.lst".format(options.path)
    n = EMUtil.get_image_count(options.info2dname)
    tasks = list(range(n))

    if options.preprocess != None:
        options.preprocess = parsemodopt(options.preprocess)

    from EMAN2PAR import EMTaskCustomer
    etc = EMTaskCustomer(options.parallel,
                         module="e2spt_subtlt_local.SptAlignTask")

    num_cpus = etc.cpu_est()
    if options.debug:
        tasks = tasks[:num_cpus * 4]
    print("{} jobs on {} CPUs".format(len(tasks), num_cpus))
    njob = num_cpus

    tids = []
    for i in range(njob):
        t = tasks[i::njob]
        task = SptAlignTask(t, options)
        if options.debug:
            ret = task.execute(print)
            return
        tid = etc.send_task(task)
        tids.append(tid)

    while 1:
        st_vals = etc.check_task(tids)
        if -100 in st_vals:
            print("Error occurs in parallelism. Exit")
            return
        E2progress(logid, np.mean(st_vals) / 100.)

        if np.min(st_vals) == 100: break
        time.sleep(5)

    output = [None] * len(tasks)
    for i in tids:
        rets = etc.get_results(i)[1]
        for r in rets:
            output[r[0]] = r[1]

    del etc

    fm = "{}/aliptcls2d_{:02d}.lst".format(options.path, options.iter)
    save_lst_params(output, fm)

    E2end(logid)
예제 #25
0
def main():
	parser = EMArgumentParser(usage=get_usage())

	parser.add_argument("--output", default="threed.hdf", help="Output reconstructed volume file name.")
	parser.add_argument("--input", default=None, help="The input projections. Project should usually have the xform.projection header attribute, which is used for slice insertion")
	parser.add_argument("--input_model", default=None, help="If the class-averages have the model_id parameter (produced by e2refinemulti.py), this will use only class-averages with the specified model_id for the reconstruction.")
	parser.add_argument("--tlt", help="An imod tlt file containing alignment angles. If specified slices will be inserted using these angles in the IMOD convention", type=str, default=None)
	parser.add_argument("--sym", dest="sym", default="c1", help="Set the symmetry; if no value is given then the model is assumed to have no symmetry.\nChoices are: i, c, d, tet, icos, or oct.")

	parser.add_argument("--fillangle", type=float, dest="fillangle", help="An angular range used for both alt & az over which the projection should be averaged. Generally the angular step used when making projections.",default=0)
	parser.add_argument("--pad", metavar="x or x,y", default=None,type=str, help="Will zero-pad images to the specifed size (x,y) or (x,x) prior to reconstruction. If not specified or 0 no padding occurs. If a negative value is specified automatic padding is performed. ")
	parser.add_argument("--padvol", metavar="x or x,y,z", default=None,type=str, help="Defines the dimensions (x,y,z) or (x,x,x) of the reconstructed volume. If ommitted, implied value based on padded 2D images is used.")
	parser.add_argument("--outsize", metavar="x or x,y,z", default=None, type=str, help="Defines the dimensions (x,y,z) or (x,x,x) of the final volume written to disk, if ommitted, size will be based on unpadded input size")
	#parser.add_argument("--clipz", default=None, type=int, help="Extract a specified number of central z-slices. Option disabled by default.")

	parser.add_argument("--savenorm", default=None, type=str, help="If set, will save the normalization volume showing Fourier space filling to the specified file")

	parser.add_argument("--keep", type=float, dest="keep", help="The fraction of slices to keep, based on quality scores (1.0 = use all slices). See keepsig.",default=1.0)
	parser.add_argument("--keepsig",action="store_true",default=False, dest="keepsig", help="If set, keep will be interpreted as a standard deviation coefficient instead of as a percentage.")
	parser.add_argument("--keepabs",action="store_true",default=False, dest="keepabs", help="If set, keep will refer to the absolute quality of the class-average, not a local quality relative to other similar sized classes.")
	parser.add_argument("--altedgemask",action="store_true",default=False, dest="altedgemask", help="If set, assumes tomographic data with a thin specimen, and masks out the +-X edges based on the alt Euler angle.")
	parser.add_argument("--no_wt", action="store_true", dest="no_wt", default=False, help="This argument turns automatic weighting off causing all images to be weighted by 1. If this argument is not specified images inserted into the reconstructed volume are weighted by the number of particles that contributed to them (i.e. as in class averages), which is extracted from the image header (as the ptcl_repr attribute).")
	parser.add_argument("--sqrt_wt", action="store_true", default=False, help="Normally class-averages are weighted into the reconstruction based on the number of particles in the average. This option causes the sqrt of the number of particles to be used instead.")
	parser.add_argument("--iterative", action="store_true", default=False, help="Uses iterative interpolation in Fourier space instead of single step gridding or interpolation. --mode and --usessnr are ignored with this option.")
	parser.add_argument("--itermask",type=str, default = None, help="Used in iterative mode to define a mask to apply between iterations. The resulting reconstruction won't be masked per-se, but will be biased towards lower densities outside the mask.")
	parser.add_argument("--usessnr", action="store_true", default=False, help="Makes use of the class_ssnr header data to weight each slice during insertion, instead of the default behavior of just using the number of particles in the average as a global weight.")
	parser.add_argument("--mode", type=str, default="gauss_var", help="Fourier reconstruction 'mode' to use. The default should not normally be changed. default='gauss_var'")
	parser.add_argument("--noradcor", action="store_true",default=False, help="Normally a radial correction will be applied based on the --mode used. This option disables that correction.")
	parser.add_argument("--seedmap",type=str, default = None, help="If specified this volume will be used as a starting point for the reconstruction, filling any missing values in Fourier space. experimental.")
	parser.add_argument("--seedweight", type=float, default=1.0, help="If seedmap specified, this is how strongly the seedmap will bias existing values. 1 is default, and is equivalent to a one particle weight.")
	parser.add_argument("--seedweightmap", type=str, default=None, help="Specify a full map of weights for the seed. This must be in the same format as the --savenorm output map.")

	parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higher number means higher level of verboseness")

	parser.add_argument("--threads", default=4,type=int,help="Number of threads to run in parallel on a single computer. This is the only parallelism supported by e2make3dpar", guitype='intbox', row=24, col=2, rowspan=1, colspan=1, mode="refinement")
	parser.add_argument("--preprocess", metavar="processor_name(param1=value1:param2=value2)", type=str, action="append", help="preprocessor to be applied to the projections prior to 3D insertion. There can be more than one preprocessor and they are applied in the order in which they are specifed. Applied before padding occurs. See e2help.py processors for a complete list of available processors.")
	parser.add_argument("--setsf",type=str,help="Force the structure factor to match a 'known' curve prior to postprocessing (<filename>, auto or none). default=none",default="none")
	parser.add_argument("--postprocess", metavar="processor_name(param1=value1:param2=value2)", type=str, action="append", help="postprocessor to be applied to the 3D volume once the reconstruction is completed. There can be more than one postprocessor, and they are applied in the order in which they are specified. See e2help.py processors for a complete list of available processors.")
	parser.add_argument("--apix",metavar="A/pix",type=float,help="A/pix value for output, overrides automatic values",default=None)

	parser.add_argument("--parallel", type=str,help="Thread/mpi parallelism to use", default=None)
	
	# Database Metadata storage
	parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)

	(options, args) = parser.parse_args()

#	options.fillangle=options.fillangle*pi/180.0
	# make sure that the user has atleast specified an input file
#	if len(args) <1:
#		parser.error("No input file specified")
#	options.input = args[0]

	if (not options.keep and not options.keepsig):
		print("Warning, neither the keep nor the keepsig argument was specified. Setting keep=1 (keeping 100% of inserted slices)")
		options.keep=1

	if options.input_model!=None : options.input_model=int(options.input_model)

	print("e2make3dpar.py")
	logger=E2init(sys.argv,options.ppid)

	# update the no_wt parameter
	if options.sqrt_wt : options.no_wt=2
	elif options.no_wt : options.no_wt=1
	else : options.no_wt=0

	# get basic image parameters
	tmp=EMData()
	tmp.read_image(options.input,0,True)

	if options.no_wt!=1 :
		try:
			n=1
			while tmp["ptcl_repr"]==0 :
				tmp.read_image(options.input,n,True)
				n+=1
		except: pass

	nx=tmp["nx"]
	ny=tmp["ny"]
	nslice=tmp["nz"]
	if options.apix!=None : apix=options.apix
	else : apix=tmp["apix_x"]

	if options.verbose>0: print("Image dimensions %d x %d"%(nx,ny))

	# parse the padding options, to make sure we have a 2 or 3 tuple for each
	try :
		if options.pad==None or options.pad==0: options.pad=(max(nx,ny),max(nx,ny))
		elif options.pad[0]=='-':
			sz=max(nx,ny)
			sz=good_size(sz*1.25)
			options.pad=(sz,sz)
			print("padding to {}x{}".format(sz,sz))
		elif "," in options.pad :
			s=options.pad.split(",")
			options.pad=(int(s[0]),int(s[1]))
		else : options.pad=(int(options.pad),int(options.pad))
		if options.verbose>0 : print("Pad to %d x %d"%(options.pad[0],options.pad[1]))
	except:
		print("Couldn't parse pad option :",options.pad)
		exit(1)

	try :
		if options.padvol==None : padvol=(max(options.pad),max(options.pad),max(options.pad))
		elif "," in options.padvol :
			s=options.padvol.split(",")
			padvol=(int(s[0]),int(s[1]),int(s[2]))
		else : padvol=(int(options.padvol),int(options.padvol),int(options.padvol))
		if options.verbose>0 : print("Padded volume to reconstruct %d x %d x %d"%(padvol[0],padvol[1],padvol[2]))
	except:
		print("Couldn't parse padvol option :",options.padvol)
		exit(1)

	try :
		if options.outsize==None : outsize=(max(nx,ny),max(nx,ny),max(nx,ny))
		elif "," in options.outsize :
			s=options.outsize.split(",")
			outsize=(int(s[0]),int(s[1]),int(s[2]))
		else : outsize=(int(options.outsize),int(options.outsize),int(options.outsize))
		if options.verbose>0 : print("Final output volume %d x %d x %d"%(outsize[0],outsize[1],outsize[2]))
	except:
		print("Couldn't parse outsize option :",options.outsize)
		exit(1)

	data=initialize_data(options.input,options.input_model,options.tlt,options.pad,options.no_wt,options.preprocess)
	
			

	# Filter out averages/images which aren't good enough
	if options.verbose: print("Filtering data, %d images ->"%len(data), end=' ')

	# we have an absolute threshold
	if options.keepabs: thr=options.keep
	else :
		quals=np.array(sorted([i["quality"] for i in data]))
		if np.std(quals)==0 : thr=max(quals)+1.0
		else :
			if options.keepsig:
				thr=np.mean(quals)+options.keep*np.std(quals)
			else:
				try:
					thr=quals[int(floor(len(quals)*options.keep))]
				except: thr=max(quals)+1.0

	excluded=[max(i["fileslice"],i["filenum"]) for i in data if i["quality"]>thr]
	included=[max(i["fileslice"],i["filenum"]) for i in data if i["quality"]<=thr]
	data=[i for i in data if i["quality"]<=thr]
	ptclcount=sum([i["weight"] for i in data])

	if options.verbose: print("After filter, %d images"%len(data))

	if options.itermask:
		itmask=EMData(options.itermask)
		itmask=itmask.get_clip(Region(old_div((itmask["nx"]-options.pad[1]),2),old_div((itmask["ny"]-options.pad[1]),2),old_div((itmask["nz"]-options.pad[1]),2),options.pad[1],options.pad[1],options.pad[1]))

	# Get the reconstructor and initialize it correctly
	if options.iterative :
		a = {"size":padvol,"sym":options.sym,"verbose":options.verbose-1}
		if options.savenorm!=None : a["savenorm"]=options.savenorm
		recon=Reconstructors.get("fourier_iter", a)
		niter=4
	else :
		a = {"size":padvol,"sym":options.sym,"mode":options.mode,"usessnr":options.usessnr,"verbose":options.verbose-1}
		if options.savenorm!=None : a["savenorm"]=options.savenorm
		recon=Reconstructors.get("fourier", a)
		niter=1
	#########################################################
	# The actual reconstruction
	options.padvol3=padvol
	if options.parallel!=None:
		par=options.parallel.split(':')
		if par[0].startswith("thread"):
			#options.parallel=None
			options.threads=int(par[1])
		elif par[0]=="mpi":
			nthr=int(par[1])
			ppt=len(data)/nthr
			if ppt>16:
				print("Inserting {:.1f} images per thread...".format(ppt))
				### prepare some options
				
				
			else:
				print("Too few images in input ({:.1f} images per thread). Switching back to threading".format(ppt))
				options.parallel=None
				options.threads=nthr
		else:
			
			options.parallel=None
			
	
	if options.seedmap!=None :
		seed=EMData(options.seedmap)
		seed.clip_inplace(Region(old_div((nx-padvol[0]),2),old_div((ny-padvol[1]),2),old_div((nslice-padvol[2]),2),padvol[0],padvol[1],padvol[2]))
		seed.do_fft_inplace()
	else:
		seed=None
		


	for it in range(niter):
		
		if options.parallel:
			print("running in mpi mode. This is experimental, so please switch back to threading if anything goes wrong...")
			
			if it>0:
				seed=output
		
			from EMAN2PAR import EMTaskCustomer
			etc=EMTaskCustomer(options.parallel, module="e2make3dpar.Make3dTask")
			num_cpus = etc.cpu_est()
			
			print("{} total CPUs available".format(num_cpus))
			
			
			tasks=[data[i::num_cpus] for i in range(num_cpus)]
			
			
			print("{} jobs".format(len(tasks)))

			tids=[]
			for t in tasks:
				task = Make3dTask(t, seed, options)
				tid=etc.send_task(task)
				tids.append(tid)

			while 1:
				st_vals = etc.check_task(tids)
				#print("{:.1f}/{} finished".format(np.mean(st_vals), 100))
				#print(tids)
				if np.min(st_vals) == 100: break
				time.sleep(5)

			#dics=[0]*nptcl
			
			output=EMData(padvol[0], padvol[1], padvol[2])
			normvol=EMData(padvol[0]//2+1, padvol[1], padvol[2])
			output.to_zero()
			output.do_fft_inplace()
			
			normvol.to_zero()
			for i in tids:
				threed, norm=etc.get_results(i)[1]
				threed.process_inplace("math.multamplitude", {"amp":norm})
				output.add(threed)
				normvol.add(norm)
				
			normvol.process_inplace("math.reciprocal")
			output.process_inplace("math.multamplitude", {"amp":normvol})
			
			output.do_ift_inplace()
			output.depad()
			output.process_inplace("xform.phaseorigin.tocenter")
			
			del etc
			
		else:
			threads=[threading.Thread(target=reconstruct,args=(data[i::options.threads],recon,options.preprocess,options.pad,
					options.fillangle,options.altedgemask,max(options.verbose-1,0),options.input.endswith(".lst"))) for i in range(options.threads)]

			if it==0:
				if options.seedmap!=None :
					#seed=EMData(options.seedmap)
			#		seed.process_inplace("normalize.edgemean")
					#seed.clip_inplace(Region(old_div((nx-padvol[0]),2),old_div((ny-padvol[1]),2),old_div((nslice-padvol[2]),2),padvol[0],padvol[1],padvol[2]))
					#seed.do_fft_inplace()
					if options.seedweightmap==None:  
						recon.setup_seed(seed,options.seedweight)
					else:
						seedweightmap=EMData(seedweightmap,0)
						recon.setup_seedandweights(seed,seedweightmap)
				else : recon.setup()
			else:
				recon.setup_seed(output,1.0)

			for i,t in enumerate(threads):
				if options.verbose>1: print("started thread ",i)
				t.start()

			for t in threads: t.join()

	#		output = recon.finish(it==niter-1)		# only return real-space on the final pass
			output = recon.finish(True)

		if options.verbose:
			print("Iteration ",it)
			if options.verbose>2: output.write_image("it{}.hdf".format(it))
			#if options.verbose>2 and it!=niter-1: 
				#output.process("xform.phaseorigin.tocenter").do_ift().write_image("it{}.hdf".format(it))

		if it<niter-1 :
			if options.itermask:
				output.mult(itmask)
			else:
				output.process_inplace("threshold.compress",{"range":output["sigma"],"value":output["mean"],"clamponly":True})
		
		if options.verbose:
			print("Iteration ",it)
			output.write_image("it{}.hdf".format(it))
			#if options.verbose>2 and it!=niter-1: 
				#output.process("xform.phaseorigin.tocenter").do_ift().write_image("it{}.hdf".format(it))
			

	if options.verbose>0 : print("Finished Reconstruction")

	try:
		output.set_attr("ptcl_repr",ptclcount)
		if len(included)>0 : output.set_attr("threed_ptcl_idxs",included)
		if len(excluded)>0 : output.set_attr("threed_excl_ptcl_idxs",excluded)
		output.set_attr("threed_ptcl_src",data[0]["filename"])
	except:
		print("Warning, error setting reconstruction attributes")
#	progress += 10
#	E2progress(logid,float(progress)/total_progress)

	#
	########################################################3

	# Correct for Gaussian falloff
	if not options.noradcor :
		cor={"gauss_2":4.0,"gauss_3":6.4,"gauss_4":8.8,"gauss_5":10.4,"gauss_5_slow":10.4}		# Gaussian widths for different reconstruction modes
		if options.mode in cor :
			output.process_inplace("math.gausskernelfix",{"gauss_width":cor[options.mode]})
		elif options.mode in ["gridding_5","gridding_7"]:
			output.process_inplace("math.gridkernelfix",{"mode":options.mode})
		else:
			print("Warning: no radial correction applied for this mode")

	# clip to the requested final dimensions
	if output["nx"]!=outsize[0] or output["ny"]!=outsize[1] or output["nz"]!=outsize[2] :
		output.clip_inplace(Region(old_div((output["nx"]-outsize[0]),2),old_div((output["ny"]-outsize[1]),2),old_div((output["nz"]-outsize[2]),2), outsize[0],outsize[1],outsize[2]))

	#if options.clipz != None:
	#	output.clip_inplace(Region(0,0,(output["nz"]-options.clipz)//2, output["nx"], output["ny"], options.clipz))

	if options.apix!=None : apix=options.apix
	output["apix_x"]=apix
	output["apix_y"]=apix
	output["apix_z"]=apix

	# Structure factor setting
	if options.setsf.lower() != "none" :
		if options.setsf.lower()!="auto" :
			try:
				sfcurve=XYData()
				sfcurve.read_file(options.setsf)
				for i in range(sfcurve.get_size()):
					v=sfcurve.get_y(i)
					if v<=0 :
						print("Warning values <=0 found in structure factor file. Please remove.")
				sfcurve.update()
			except:
				print("ERROR: Specified structure factor ({}) not found.".format(options.setsf))
				sys.exit(1)
		else:
			try:
				sfcurve=XYData()
				sfcurve.read_file("strucfac.txt")
				for i in range(sfcurve.get_size()):
					v=sfcurve.get_y(i)
					if v<=0 :
						print("Warning values <=0 found in structure factor file. Please remove.")
				sfcurve.update()
			except : sfcurve=None

		if sfcurve==None:
			print("ERROR : Structure factor read failed. Not applying structure factor")
		else:
			# need to be really careful about the corners
#			for i in range(sfcurve.get_size()):
#				if sfcurve.get_x(i)>1.0/(2.0*apix) : sfcurve.set_y(i,0.0)
			output.process_inplace("filter.setstrucfac",{"apix":apix,"strucfac":sfcurve})
#			output.process_inplace("filter.lowpass.tophat",{"apix":apix,"cutoff_abs":0.49})


	if options.postprocess != None:
		for p in options.postprocess:
			try:
				(processorname, param_dict) = parsemodopt(p)
				if not param_dict : param_dict={}
				output.process_inplace(str(processorname), param_dict)
			except:
				print("warning - application of the post processor",p," failed. Continuing anyway")

#	output.process_inplace("normalize.circlemean")

	# just remove the output file
	if file_exists(options.output):
		remove_file(options.output)

	# write the reconstruction to disk
	output.write_image(options.output,0)
	if options.verbose>0:
			print("Output File: "+options.output)

	E2end(logger)

	print("Exiting")
예제 #26
0
파일: e2simmx.py 프로젝트: cpsemmens/eman2
class EMParallelSimMX:
	def __init__(self,options,args,logger=None):
		'''
		@param options the options produced by (options, args) = parser.parse_args()
		@param args the options produced by (options, args) = parser.parse_args()
		@param logger and EMAN2 logger, i.e. logger=E2init(sys.argv)
		assumes you have already called the check function.
		'''
		self.options = options
		self.args = args
		self.logger = logger


		from EMAN2PAR import EMTaskCustomer
		self.etc=EMTaskCustomer(options.parallel)
		if options.colmasks!=None : self.etc.precache([args[0],args[1],options.colmasks])
		else : self.etc.precache([args[0],args[1]])
		self.num_cpus = self.etc.cpu_est()
		if self.num_cpus < 32: # lower limit
			self.num_cpus = 32

		self.__task_options = None

	def __get_task_options(self,options):
		'''
		Get the options required by each task as a dict
		@param options is always self.options - the initialization argument. Could be changed.
		'''
		if self.__task_options == None:
			d = {}
			d["align"] = parsemodopt(options.align)
			d["aligncmp"] = parsemodopt(options.aligncmp)
			d["cmp"] = parsemodopt(options.cmp)

			if hasattr(options,"ralign") and options.ralign != None:
				d["ralign"] = parsemodopt(options.ralign)
				d["raligncmp"] = parsemodopt(options.raligncmp)  # raligncmp must be specified if using ralign
			else:
				d["ralign"] = None
				d["raligncmp"] = None
			d["prefilt"]=options.prefilt

			if hasattr(options,"shrink") and options.shrink != None: d["shrink"] = options.shrink
			else: d["shrink"] = None


			self.__task_options = d

		return self.__task_options

	def __init_memory(self,options):
		'''
		@param options is always self.options - the initialization argument. Could be changed.
		Establishes several important attributes they are:
		----
		self.clen - the number of images in the image defined by args[0], the number of columns in the similarity matrix
		self.rlen - the number of images in the image defined by args[1], the number of rows in the similarity matrix
		----
		Also, since we adopted region output writing as our preferred approach, this function makes sure the output
		image(s) exists on disk and has the correct dimensions - seeing as this is the way region writing works (the image
		has to exist on disk and have its full dimensions)
		'''
		self.clen=EMUtil.get_image_count(self.args[0])
		self.rlen=EMUtil.get_image_count(self.args[1])

		output = self.args[2]

		if file_exists(output) and not options.fillzero:
			if options.force: remove_file(output)
			else: raise RuntimeError("The output file exists. Please remove it or specify the force option")

		e = EMData(self.clen,self.rlen)
		e.to_zero()
		e.set_attr(PROJ_FILE_ATTR,self.args[0])
		e.set_attr(PART_FILE_ATTR,self.args[1])
		n = 1
		if self.options.saveali: n = 6 # the total number of images written to disk
		if not options.fillzero : e.write_image(output,0)
		for i in range(1,n):
			e.write_image(output,i)

	def __get_blocks(self):
		'''
		Gets the blocks that will be processed in parallel, these are essentially ranges
		'''

		steve_factor = 3 # increase number of jobs a bit for better distribution
		total_jobs = steve_factor*self.num_cpus

		[col_div,row_div] = opt_rectangular_subdivision(self.clen,self.rlen,total_jobs)


		block_c = self.clen/col_div
		block_r = self.rlen/row_div

		residual_c = self.clen-block_c*col_div # residual left over by integer division

		blocks = []

		current_c = 0
		for c in xrange(0,col_div):
			last_c = current_c + block_c
			if residual_c > 0:
				last_c += 1
				residual_c -= 1

			current_r = 0
			residual_r = self.rlen-block_r*row_div # residual left over by integer division
			for r in xrange(0,row_div) :
				last_r = current_r + block_r
				if residual_r > 0:
					last_r += 1
					residual_r -= 1


				blocks.append([current_c,last_c,current_r,last_r])
				current_r = last_r

			current_c = last_c

#		print col_div,row_div,col_div*row_div
#		print self.clen,self.rlen,residual_c,residual_r
		return blocks

	def execute(self):
		'''
		The main function to be called
		'''
		if len(self.options.parallel) > 1 :
			self.__init_memory(self.options)
			blocks = self.__get_blocks()
#			print blocks

#			self.check_blocks(blocks) # testing function can be removed at some point

			tasks=[]
			for bn,block in enumerate(blocks):

				data = {}
				data["references"] = ("cache",self.args[0],block[0],block[1])
				data["particles"] = ("cache",self.args[1],block[2],block[3])
				if self.options.colmasks!=None : data["colmasks"] = ("cache",self.options.colmasks,block[0],block[1])
				if self.options.mask!=None : data["mask"] = ("cache",self.options.mask,0,1)
				if self.options.fillzero :
					# for each particle check to see which portion of the matrix we need to fill
					if (bn%10==0) : print "%d/%d     \r"%(bn,len(blocks)),
					sys.stdout.flush()
					rng=[]
					for i in range(block[2],block[3]):
						c=EMData()
						c.read_image(self.args[2],0,False,Region(block[0],i,block[1]-block[0]+1,1))
						inr=0
						st=0
						for j in range(c["nx"]):
							if c[j]==0 and not inr:
								st=j
								inr=1
							if c[j]!=0 and inr:
								rng.append((i,st+block[0],j-1+block[0]))
								inr=0
						if inr :
							rng.append((i,st+block[0],j+block[0]))
					data["partial"]=rng
#					print "%d) %s\t"%(bn,str(block)),rng

				if self.options.fillzero and len(data["partial"])==0 : continue		# nothing to compute in this block, skip it completely
				else :
					task = EMSimTaskDC(data=data,options=self.__get_task_options(self.options))
					#print "Est %d CPUs"%etc.cpu_est()
					tasks.append(task)

			# This just verifies that all particles have at least one class
			#a=set()
			#for i in tasks:
				#for k in i.data["partial"] : a.add(k[0])

			#b=set(range(self.rlen))
			#b-=a
			#print b

			print "%d/%d         "%(bn,len(blocks))
			self.tids=self.etc.send_tasks(tasks)
			print len(self.tids)," tasks submitted"
#
			while 1:
				if len(self.tids) == 0: break
				print len(self.tids),"simmx tasks left in main loop   \r",
				sys.stdout.flush()
				st_vals = self.etc.check_task(self.tids)
				for i in xrange(len(self.tids)-1,-1,-1):
					st = st_vals[i]
					if st==100:
						tid = self.tids[i]

						try:
							rslts = self.etc.get_results(tid)
#							display(rslts[1]["rslt_data"][0])
							self.__store_output_data(rslts[1])
						except:
							traceback.print_exc()
							print "ERROR storing results for task %d. Rerunning."%tid
							self.etc.rerun_task(tid)
							continue
						if self.logger != None:
							E2progress(self.logger,1.0-len(self.tids)/float(len(blocks)))
							if self.options.verbose>0:
								print "%d/%d\r"%(len(self.tids),len(blocks))
								sys.stdout.flush()

						self.tids.pop(i)
					print len(self.tids),"simmx tasks left in main loop   \r",
					sys.stdout.flush()


				time.sleep(10)
			print "\nAll simmx tasks complete "

			# if using fillzero, we must fix the -1.0e38 values placed into empty cells
			if self.options.fillzero :
				l=EMData(self.args[2],0,True)
				rlen=l["ny"]
				clen=l["nx"]
#				launch_childprocess("e2proc2d.py %s %s"%(self.args[2],self.args[2]+"_x"))
				print "Filling noncomputed regions in similarity matrix (%dx%d)"%(clen,rlen)
				l=EMData()
				for r in range(rlen):
					l.read_image(self.args[2],0,False,Region(0,r,clen,1))
					fill=l["maximum"]+.0001
					l.process_inplace("threshold.belowtominval",{"minval":-1.0e37,"newval":fill})
					l.write_image(self.args[2],0,EMUtil.ImageType.IMAGE_UNKNOWN,False,Region(0,r,clen,1))

				print "Filling complete"



		else: raise NotImplementedError("The parallelism option you specified (%s) is not supported" %self.options.parallel )

	def __store_output_data(self,rslts):
		'''
		Store output data to internal images (matrices)
		@param a dictionary return by the EMSimTaskDC
		'''

		result_data = rslts["rslt_data"]
		output = self.args[2]

		insertion_c = rslts["min_ref_idx"]
		insertion_r = rslts["min_ptcl_idx"]
		result_mx = result_data[0]
		r = Region(insertion_c,insertion_r,result_mx.get_xsize(),result_mx.get_ysize())

		# Note this is region io - the init_memory function made sure the images exist and are the right dimensions (on disk)
		for i,mxout in enumerate(result_data):
			mxout.write_image(output,i,EMUtil.ImageType.IMAGE_UNKNOWN,False,r)
예제 #27
0
def main():
	
	usage=" "
	parser = EMArgumentParser(usage=usage,version=EMANVERSION)

	parser.add_argument("--path", type=str,help="path", default=None, guitype='strbox',row=0, col=0,rowspan=1, colspan=1)
	parser.add_argument("--iter", type=int,help="start from iteration X", default=-1, guitype='intbox',row=0, col=1,rowspan=1, colspan=1)
	parser.add_argument("--niters", type=int,help="run this many iterations. Default is 4.", default=4, guitype='intbox',row=0, col=2,rowspan=1, colspan=1)

	parser.add_argument("--sym", type=str,help="symmetry. will use symmetry from spt refinement by default", default="", guitype='strbox',row=2, col=0,rowspan=1, colspan=1)
	parser.add_argument("--padby", type=float,help="pad by factor. default is 2", default=2., guitype='floatbox',row=1, col=1,rowspan=1, colspan=1)
	parser.add_argument("--keep", type=float,help="propotion of tilts to keep. default is 0.5", default=0.5, guitype='floatbox',row=1, col=2,rowspan=1, colspan=1)

	parser.add_argument("--maxalt", type=float,help="max altitude to insert to volume", default=90.0, guitype='floatbox',row=1, col=0,rowspan=1, colspan=1)	
	parser.add_argument("--nogs", action="store_true", default=False ,help="skip gold standard...", guitype='boolbox',row=2, col=1,rowspan=1, colspan=1)
	parser.add_argument("--localfilter", type=int, default=-1 ,help="use tophat local. specify 0 or 1 to overwrite the setting in the spt refinement")
	parser.add_argument("--mask", type=str, default="None" ,help="Refinement masking. default is the same as the spt refinement. Leave this empty for automasking",guitype='strbox',row=3, col=0,rowspan=1, colspan=2)	

	parser.add_argument("--threads", type=int,help="Number of CPU threads to use. Default is 12.", default=12, guitype='intbox',row=2, col=2,rowspan=1, colspan=1)
	parser.add_argument("--parallel", type=str,help="Thread/mpi parallelism to use. Default is thread:12", default="thread:12", guitype='strbox',row=4, col=0,rowspan=1, colspan=3)

	parser.add_argument("--refineastep", type=float,help="angular variation for refine alignment (gauss std)", default=8.)
	parser.add_argument("--refinentry", type=int,help="number of starting points for refine alignment", default=32)
	parser.add_argument("--maxshift", type=int,help="maximum shift allowed", default=10)

	parser.add_argument("--buildsetonly", action="store_true", default=False ,help="build sets only")
	parser.add_argument("--output", type=str,help="Write results to this directory. We do not recommend changing this.", default="subtlt")#, guitype='intbox',row=2, col=1,rowspan=1, colspan=1)

	parser.add_argument("--debug", action="store_true", default=False ,help="Turn on debug mode. This will only process a small subset of the data (threads * 8 particles)")
	parser.add_argument("--ppid", type=int,help="ppid...", default=-1)

	(options, args) = parser.parse_args()
	logid=E2init(sys.argv)

	itr = options.iter

	oldpath = options.path	
	if not oldpath:
		print("No input path. Exit.")
		return
	

	if options.iter != -1:
		itr = options.iter
	elif "spt" in oldpath:
		for f in sorted(os.listdir(oldpath)):
			if "particle_parms" in f:
				itrstr = f[15:].split(".")[0]
				if os.path.isfile("{}/threed_{}.hdf".format(oldpath,itrstr)):
					itr = int(itrstr)
	else:
		for f in sorted(os.listdir(oldpath)):
			if re.match("threed_[0-9][0-9].hdf",f):
				itr = int(f[7:].split(".")[0])

	# print(oldpath)
	fromspt=True
	if "0_subtlt_params.json" in os.listdir(oldpath):
		print("Continuing from a subtilt refinement...")
		fromspt=False
		
	
	path = make_path(options.output)
	
	if not os.path.isfile("{}/threed_{:02d}.hdf".format(oldpath,itr)):
		print("Could not locate {}/threed_{:02d}.hdf".format(oldpath,itr))
		print("Please specify the iteration number (--iter) of a completed subtomogram refinement.")
		sys.exit(1)
	#elif not os.path.isfile("{}/particle_parms_{:02d}.json".format(oldpath,itr)):
		#print("Could not locate {}/particle_parms_{:02d}.json".format(oldpath,itr))
		#print("Please specify the iteration number (--iter) of a completed subtomogram refinement.")
		#sys.exit(1)
	else:
		#copy2("{}/0_spt_params.json".format(oldpath),"{}/0_subtlt_params.json".format(path))

		oldmap = os.path.join(oldpath,"threed_{:02d}.hdf".format(itr))
		oem = os.path.join(oldpath,"threed_{:02d}_even.hdf".format(itr))
		oom = os.path.join(oldpath,"threed_{:02d}_odd.hdf".format(itr))
		oldfsc = os.path.join(oldpath, "fsc_masked_{:02d}.txt".format(itr))

		copy2(oldmap,os.path.join(path,"threed_00.hdf"))
		copy2(oldfsc, os.path.join(path, "fsc_masked_00.txt"))
		copy2(oem,os.path.join(path,"threed_00_even.hdf"))
		copy2(oom,os.path.join(path,"threed_00_odd.hdf"))
		
		if fromspt:
			oldparm = os.path.join(oldpath,"particle_parms_{:02d}.json".format(itr))
			copy2(oldparm,os.path.join(path,"particle_parms_00.json"))
		else:
			for eo in ["even", "odd"]:
				oali = os.path.join(oldpath,"ali_ptcls_{:02d}_{}.lst".format(itr, eo))
				copy2(oali,os.path.join(path,"ali_ptcls_00_{}.lst".format(eo)))


	e=EMData(os.path.join(path,"threed_00.hdf"))
	
	bxsz=e["nx"]
	apix=e["apix_x"]
	jd = js_open_dict("{}/0_subtlt_params.json".format(path))
	jd.update(vars(options))
	jd["cmd"] = " ".join(sys.argv)
	jd["path"] = oldpath
	jd["iter"] = itr
	jd["output"] = path

	if fromspt:
		sptparms = os.path.join(oldpath,"0_spt_params.json")
	else:
		sptparms = os.path.join(oldpath,"0_subtlt_params.json")
	if os.path.isfile(sptparms):
		oldjd = js_open_dict(sptparms)
		#print(oldjd.keys())
		jd["mass"] = oldjd["mass"]
		jd["setsf"] = oldjd["setsf"]
		jd["sym"] = oldjd["sym"]
		jd["localfilter"]=oldjd["localfilter"]
		jd["mask"]=oldjd["mask"]
		oldjd.close()
	else:
		print("Cannot find {}. exit.".format(sptparms))
	
	if options.mask.lower()!="none":
		print("Overwritting masking")
		jd["mask"]=options.mask
	
	if options.localfilter==0:
		jd["localfilter"]=False
	elif options.localfilter==1:
		jd["localfilter"]=True
		
	if len(options.sym)>0:
		jd["sym"]=options.sym
		
	jsparams=jd.data
	jd.close()
	jd = jsparams

	if fromspt:
		js=js_open_dict(os.path.join(path,"particle_parms_00.json"))
		k=list(js.keys())[0]
		src=eval(k)[0]
		
		print("loading 3D particles from {}".format(src))
		print("box size {}, apix {:.2f}".format(bxsz, apix))

		lname=[os.path.join(path, "ali_ptcls_00_{}.lst".format(eo)) for eo in ["even", "odd"]]
		for l in lname:
			try: os.remove(l)
			except:pass
		
		lst=[LSXFile(m, False) for m in lname]
		n3d=len(list(js.keys()))
		for ii in range(n3d):
			e=EMData(src, ii, True)
			fname=e["class_ptcl_src"]
			ids=e["class_ptcl_idxs"]
			ky="('{}', {})".format(src, ii)
			dic=js[ky]
			xali=dic["xform.align3d"]
			for i in ids:
				try:
					m=EMData(fname, i, True)
				except:
					continue
				xf=m["xform.projection"]
				dc=xf.get_params("xyz")
				if abs(dc["ytilt"])>options.maxalt:
					continue
				rot=xf*xali.inverse()
				lst[ii%2].write(-1, i, fname, str(rot.get_params("eman")))
		for l in lst:
			l.close()
		js=None

	if options.buildsetonly: return

	for itr in range(0,options.niters):

		from EMAN2PAR import EMTaskCustomer

		for eo in ["even", "odd"]:
			
			if options.nogs:
				threedname="{}/threed_{:02d}.hdf".format(path, itr)
			else:
				threedname="{}/threed_{:02d}_{}.hdf".format(path, itr, eo)
			
			lstname="{}/ali_ptcls_{:02d}_{}.lst".format(path, itr, eo)
			lst=LSXFile(lstname, True)
			m=EMData(threedname)
			
			m.process_inplace('normalize.edgemean')
			
			pinfo=[]
			if options.debug: nptcl=options.threads*8
			else: nptcl=lst.n
			for i in range(nptcl):
				pinfo.append(lst.read(i))
			lst=None
			
			etc=EMTaskCustomer(options.parallel)
			num_cpus = etc.cpu_est()
			
			print("{} total CPUs available".format(num_cpus))
			print("{} jobs".format(nptcl))
			
			infos=[[] for i in range(num_cpus)]
			for i,info in enumerate(pinfo):
				infos[i%num_cpus].append([i, info])
			
			tids=[]
			for info in infos:
				task = SptTltRefineTask(info, m, options)
				tid=etc.send_task(task)
				tids.append(tid)
			
			while 1:
				st_vals = etc.check_task(tids)
				#print("{:.1f}/{} finished".format(np.mean(st_vals), 100))
				#print(tids)
				if np.min(st_vals) == 100: break
				time.sleep(5)
			
			dics=[0]*nptcl
			for i in tids:
				ret=etc.get_results(i)[1]
				for r in ret:
					#print(r)
					ii=r.pop("idx")
					dics[ii]=r
			
			del etc
			
			allscr=np.array([d["score"] for d in dics])
			print(np.min(allscr), np.mean(allscr), np.max(allscr), np.std(allscr))
			allscr*=-1
			s=allscr.copy()
			s-=np.mean(s)
			s/=np.std(s)
			clp=2
			ol=abs(s)>clp
			print("Removing {} outliers from {} particles..".format(np.sum(ol), len(s)))
			s=old_div(old_div((s+clp),clp),2)
			s[ol]=0
			allscr=s
			#allscr-=np.min(allscr)-1e-5
			#allscr/=np.max(allscr)

			lname="{}/ali_ptcls_{:02d}_{}.lst".format(path, itr+1, eo)
			try: os.remove(lname)
			except: pass
			lout=LSXFile(lname, False)
			for i, dc in enumerate(dics):
				d=dc["xform.align3d"].get_params("eman")
				d["score"]=float(allscr[i])
				l=pinfo[i]
				lout.write(-1, l[0], l[1], str(d))

			lout=None

			pb=options.padby
			threedout="{}/threed_{:02d}_{}.hdf".format(path, itr+1, eo)
			cmd="e2make3dpar.py --input {inp} --output {out} --pad {pd} --padvol {pdv} --threads {trd} --outsize {bx} --apix {apx} --mode gauss_var --keep {kp} --sym {sm}".format(
				inp=lname, 
				out=threedout,
				bx=bxsz, pd=int(bxsz*pb), pdv=int(bxsz*pb), apx=apix, kp=options.keep, sm=jd["sym"], trd=options.threads)
			
			run(cmd)
			run("e2proc3d.py {} {}".format(threedout, "{}/threed_raw_{}.hdf".format(path, eo)))

		s = ""
		
		if jd.has_key("goldstandard"): 
			if jd["goldstandard"] > 0: 
				s += " --align"
		if jd.has_key("setsf"):
			s += " --setsf {}".format(jd['setsf']) #options.setsf)
		if jd.has_key("localfilter"):
			s += " --tophat local"
		msk = jd["mask"] #{}/mask_tight.hdf".format(path)
		if len(msk)>0:
			if os.path.isfile(msk):
				msk=" --automask3d mask.fromfile:filename={}".format(msk)
			else:
				msk=" --automask3d {}".format(msk)

		# get target resolution from last iteration map
		ref=os.path.join(path, "threed_{:02d}.hdf".format(itr))
		fsc=np.loadtxt(os.path.join(path, "fsc_masked_{:02d}.txt".format(itr)))
		rs=1./fsc[fsc[:,1]<0.3, 0][0]
		curres=rs*.5

		#os.system("rm {}/mask*.hdf {}/*unmasked.hdf".format(path, path))
		ppcmd="e2refine_postprocess.py --even {} --odd {} --output {} --iter {:d} --restarget {} --threads {} --sym {} --mass {} {}".format(
			os.path.join(path, "threed_{:02d}_even.hdf".format(itr+1)), 
			os.path.join(path, "threed_{:02d}_odd.hdf".format(itr+1)), 
			os.path.join(path, "threed_{:02d}.hdf".format(itr+1)), itr+1, curres, options.threads, jd["sym"], jd["mass"], s)
		run(ppcmd)

		fsc=np.loadtxt(os.path.join(path, "fsc_masked_{:02d}.txt".format(itr+1)))
		rs=1./fsc[fsc[:,1]<0.3, 0][0]
		print("Resolution (FSC<0.3) is ~{:.1f} A".format(rs))
				
	E2end(logid)
예제 #28
0
def main():
	parser = EMArgumentParser(usage="")

	parser.add_argument("--output", default="threed.hdf", help="Output reconstructed volume file name.")
	parser.add_argument("--input", default=None, help="The input projections. Project should usually have the xform.projection header attribute, which is used for slice insertion")
	parser.add_argument("--sym", dest="sym", default="c1", help="Set the symmetry; if no value is given then the model is assumed to have no symmetry.\nChoices are: i, c, d, tet, icos, or oct.")

	parser.add_argument("--pad", default=-1,type=int, help="Will zero-pad images to the specifed size. ")
	
	parser.add_argument("--outsize", default=-1, type=int, help="Defines the dimensions of the final volume written to disk")
	
	parser.add_argument("--keep", type=float, dest="keep", help="The fraction of slices to keep, based on quality scores (1.0 = use all slices). See keepsig.",default=.9)
	
	parser.add_argument("--no_wt", action="store_true", dest="no_wt", default=False, help="This argument turns automatic weighting off causing all images to be weighted by 1.")
	
	parser.add_argument("--mode", type=str, default="trilinear", help="Fourier reconstruction 'mode' to use. The default should not normally be changed.")
	
	parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higher number means higher level of verboseness")

	parser.add_argument("--apix",metavar="A/pix",type=float,help="A/pix value for output, overrides automatic values",default=None)
	
	parser.add_argument("--ref", type=str,help="ref", default=None)
	parser.add_argument("--tidrange", type=str,help="range of tilt id to include", default="-1,-1")
	parser.add_argument("--minres", type=float,help="", default=200)
	parser.add_argument("--maxres", type=float,help="", default=5)
	parser.add_argument("--parallel", type=str,help="Thread/mpi parallelism to use", default=None)
	
	parser.add_argument("--debug", action="store_true", default=False, help="")
	parser.add_argument("--clsid", default=-1, type=int, help="only reconstruct a class of particles")

	parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)

	(options, args) = parser.parse_args()

	logger=E2init(sys.argv,options.ppid)
	
	# get basic image parameters
	tmp=EMData(options.input,0,True)
	boxsz=tmp["nx"]
	if options.apix!=None : apix=options.apix
	else : apix=tmp["apix_x"]

	if options.pad<0:
		options.pad=good_size(boxsz*1.5)
		
	if options.outsize<0:
		options.outsize=boxsz
		
	options.tidrange=[int(i) for i in options.tidrange.split(',')]
	if options.tidrange[0]>=0:
		print("including tilt ids from {} to {}".format(options.tidrange[0], options.tidrange[1]))
	
	data=initialize_data(options.input, options)
	padvol=options.pad
		

	from EMAN2PAR import EMTaskCustomer
	if options.ref:
		print("weighting by reference...")
		ref=EMData(options.ref)
		ref=ref.do_fft()
		ref.process_inplace("xform.phaseorigin.tocenter")
		ref.process_inplace("xform.fourierorigin.tocenter")
		
		etc=EMTaskCustomer(options.parallel, module="e2spa_make3d.WeightptclTask")
		num_cpus = etc.cpu_est()
		print("{} total CPUs available".format(num_cpus))
		tasks=[data[i::num_cpus] for i in range(num_cpus)]
		print("{} jobs".format(len(tasks)))
		
		
		tids=[]
		for t in tasks:
			task = WeightptclTask(t, ref, options)
			if options.debug:
				task.execute(print)
				return
			tid=etc.send_task(task)
			tids.append(tid)

		while 1:
			st_vals = etc.check_task(tids)
			E2progress(logger, .5*np.mean(st_vals)/100.)
			if np.min(st_vals) == 100: break
			time.sleep(5)
			
		wts=[0]*len(data)
		for i in tids:
			wt=etc.get_results(i)[1]
			for w in wt:
				try:
					wts[w[0]]=w[1]
				except:
					print(len(wts), w)
				
		wts=np.array(wts)
		del etc
		
		r0=int(apix*boxsz/options.minres)
		r1=int(apix*boxsz/options.maxres)
		print(r0,r1)
		scrs=np.mean(wts[:,r0:r1], axis=1)
		if options.keep<1:
			thr=np.sort(scrs)[int(len(scrs)*(1-options.keep))-1]
			scrs[scrs<thr]=-1
		
		for i,d in enumerate(data):
			d["curve"]=wts[i]
			d["weight"]=float(scrs[i])
			
			
	etc=EMTaskCustomer(options.parallel, module="e2spa_make3d.Make3dTask")
	num_cpus = etc.cpu_est()
	print("{} total CPUs available".format(num_cpus))
	tasks=[data[i::num_cpus] for i in range(num_cpus)]
	print("{} jobs".format(len(tasks)))

	tids=[]
	for t in tasks:
		task = Make3dTask(t, options)
		if options.debug:
			task.execute(print)
			return
		tid=etc.send_task(task)
		tids.append(tid)

	while 1:
		st_vals = etc.check_task(tids)
		E2progress(logger, .5+.5*np.mean(st_vals)/100.)
		if np.min(st_vals) == 100: break
		time.sleep(5)
		
	output=EMData(padvol, padvol, padvol)
	normvol=EMData(padvol//2+1, padvol, padvol)
	output.to_zero()
	output.do_fft_inplace()
	normvol.to_zero()
	
	for i in tids:
		threed, norm=etc.get_results(i)[1]
		threed.process_inplace("math.multamplitude", {"amp":norm})
		output.add(threed)
		normvol.add(norm)
		
	normvol.process_inplace("math.reciprocal")
	output.process_inplace("math.multamplitude", {"amp":normvol})
	
	output.do_ift_inplace()
	output.depad()
	output.process_inplace("xform.phaseorigin.tocenter")
	
	del etc

	if options.verbose>0 : print("Finished Reconstruction")
	output["apix_x"]=output["apix_y"]=output["apix_z"]=apix
	sz=options.outsize
	output.clip_inplace(Region((padvol-sz)//2,(padvol-sz)//2,(padvol-sz)//2,sz,sz,sz))
	
	if os.path.isfile(options.output):
		os.remove(options.output)
		
	output.write_image(options.output,0)
	if options.verbose>0:
			print("Output File: "+options.output)

	E2end(logger)

	print("Exiting")
예제 #29
0
def main():

    usage = " "
    parser = EMArgumentParser(usage=usage, version=EMANVERSION)

    parser.add_argument("--ptclin",
                        type=str,
                        help="particle input",
                        default=None)
    parser.add_argument("--output",
                        type=str,
                        help="score output",
                        default=None)

    parser.add_argument(
        "--parallel",
        type=str,
        help="Thread/mpi parallelism to use. Default is thread:12",
        default="thread:12")

    parser.add_argument(
        "--debug",
        action="store_true",
        default=False,
        help=
        "Turn on debug mode. This will only process a small subset of the data (threads * 8 particles)"
    )

    parser.add_argument("--maxres",
                        type=float,
                        default=15,
                        help="max resolution for cmp")
    parser.add_argument("--minres",
                        type=float,
                        default=300,
                        help="min resolution for cmp")
    parser.add_argument("--sym", type=str, help="symmetry. ", default="c1")
    parser.add_argument("--mask", type=str, help="mask. ", default=None)
    parser.add_argument("--maxshift", type=int, help="max shift.", default=0)

    parser.add_argument("--ppid", type=int, help="ppid...", default=-1)
    parser.add_argument("--verbose", "-v", type=int, help="Verbose", default=0)

    (options, args) = parser.parse_args()
    logid = E2init(sys.argv)

    threedname = args
    lst = LSXFile(options.ptclin, True)
    pinfo = []
    nptcl = lst.n
    for i in range(nptcl):
        pinfo.append(lst.read(i))
    lst = None

    e = EMData(options.ptclin, 0, True)
    nx = e["nx"]
    apix = e["apix_x"]
    b = good_size(nx * apix / options.maxres * 2)
    options.shrink = max(1, nx / b)

    print("Initializing parallelism...")
    etc = EMTaskCustomer(options.parallel,
                         module="e2spa_classify.SpaClassifyTask")

    num_cpus = etc.cpu_est()

    print("{} total CPUs available".format(num_cpus))
    print("{} jobs".format(nptcl))
    infos = [[] for i in range(num_cpus)]
    for i, info in enumerate(pinfo):
        infos[i % num_cpus].append([i, info])

    tids = []
    for info in infos:
        task = SpaClassifyTask(info, threedname, options)

        if options.debug:
            task.execute(print)
            return
        tid = etc.send_task(task)
        tids.append(tid)

    while 1:
        st_vals = etc.check_task(tids)
        if -100 in st_vals:
            print("Error occurs in parallelism. Exit")
            return
        E2progress(logid, np.mean(st_vals) / 100.)

        if np.min(st_vals) == 100: break
        time.sleep(5)

    dics = np.zeros((nptcl, len(threedname)))
    for i in tids:
        ret = etc.get_results(i)[1]
        for r in ret:
            #print(r)
            ii = r.pop("idx")
            dics[ii] = r["score"]

    del etc
    np.savetxt(options.output, dics)

    E2end(logid)
예제 #30
0
def main():

    usage = " "
    parser = EMArgumentParser(usage=usage, version=EMANVERSION)

    parser.add_argument("--ptclin",
                        type=str,
                        help="particle input",
                        default=None)
    parser.add_argument("--ptclout",
                        type=str,
                        help="particle output",
                        default=None)
    parser.add_argument("--ref",
                        type=str,
                        help="reference input",
                        default=None)
    parser.add_argument("--keep",
                        type=float,
                        help="propotion of tilts to keep. default is 0.8",
                        default=0.8)
    parser.add_argument(
        "--parallel",
        type=str,
        help="Thread/mpi parallelism to use. Default is thread:12",
        default="thread:12")

    parser.add_argument(
        "--debug",
        action="store_true",
        default=False,
        help=
        "Turn on debug mode. This will only process a small subset of the data"
    )
    parser.add_argument("--maxshift",
                        type=int,
                        help="maximum shift allowed",
                        default=-1)
    parser.add_argument("--localrefine",
                        action="store_true",
                        default=False,
                        help="local refinement")
    #parser.add_argument("--ctfweight", action="store_true", default=False ,help="weight by ctf. not used yet...")
    parser.add_argument("--slow",
                        action="store_true",
                        default=False,
                        help="slow but finer search")
    parser.add_argument("--maxres",
                        type=float,
                        default=-1,
                        help="max resolution for cmp")
    parser.add_argument("--minrespx",
                        type=int,
                        default=4,
                        help="skip the first x pixel in fourier space")
    parser.add_argument("--sym", type=str, help="symmetry. ", default="c1")
    parser.add_argument("--ppid", type=int, help="ppid...", default=-1)
    #parser.add_argument("--nkeep", type=int,help="", default=1)
    parser.add_argument("--verbose", "-v", type=int, help="Verbose", default=0)

    (options, args) = parser.parse_args()
    logid = E2init(sys.argv)

    lstname = options.ptclin
    threedname = options.ref
    lname = options.ptclout

    lst = LSXFile(lstname, True)
    m = EMData(threedname)
    bxsz = m["nx"]
    apix = m["apix_x"]

    options.shrink = 1
    pinfo = []
    nptcl = lst.n
    if options.maxshift < 0:
        options.maxshift = bxsz // 2

    print("Initializing parallelism...")
    etc = EMTaskCustomer(options.parallel, module="e2spa_align.SpaAlignTask")
    num_cpus = etc.cpu_est()

    print("{} particles".format(nptcl))
    print("{} total CPUs available".format(num_cpus))
    if options.debug:
        nptcl = min(4 * num_cpus, nptcl)
        print("Debugging mode. running on one thread with 8 particles")

    for i in range(nptcl):
        pinfo.append(lst.read(i))
    lst = None

    nbatch = min(nptcl // 4, num_cpus)

    infos = [[] for i in range(nbatch)]
    for i, info in enumerate(pinfo):
        infos[i % nbatch].append([i, info])

    print("{} jobs, each with {:.1f} particles".format(
        len(infos), np.mean([len(i) for i in infos])))

    tids = []
    for info in infos:
        task = SpaAlignTask(info, threedname, options)

        if options.debug:
            task.execute(print)
            return
        tid = etc.send_task(task)
        tids.append(tid)

    while 1:
        st_vals = etc.check_task(tids)
        if -100 in st_vals:
            print("Error occurs in parallelism. Exit")
            return
        E2progress(logid, np.mean(st_vals) / 100.)

        if np.min(st_vals) == 100: break
        time.sleep(5)

    dics = [0] * nptcl
    for i in tids:
        ret = etc.get_results(i)[1]
        for r in ret:
            ii = r.pop("idx")
            dics[ii] = r

    del etc

    allscr = [d["score"] for d in dics]
    maxl = np.max([len(s) for s in allscr])
    maxv = np.max(np.concatenate(allscr))
    for s in allscr:
        s.extend([maxv] * (maxl - len(s)))
    allscr = np.array(allscr)

    try:
        os.remove(lname)
    except:
        pass
    lout = LSXFile(lname, False)
    for i, dc in enumerate(dics):
        lc = ""
        for j, xf in enumerate(dc["xform.align3d"]):
            d = xf.get_params("eman")
            d["score"] = float(allscr[i, j])
            lc = lc + str(d) + ';'

        l = pinfo[i]
        lout.write(-1, l[0], l[1], lc[:-1])

    lout = None
    E2end(logid)