def runFFT(images, apix=3.7, do_avg=True):
"""Run radially averaged FFT on an EMAN image.
Args:
images:
apix: angstroms per pixel
do_avg: compute average
Rets:
curve: radially-averaged FFT
sf_dx: corresponding x values.
"""
# This code by Steven Ludtke
nx = images[0]["nx"] + 2
ny = images[0]["ny"]
fftavg = EMData(nx, ny, 1)
fftavg.to_zero()
fftavg.set_complex(1)
for d in images:
d.process_inplace("mask.ringmean")
d.process_inplace("normalize")
df = d.do_fft()
df.mult(df.get_ysize())
fftavg.add_incoherent(df)
if do_avg:
fftavg.mult(1.0 / len(images))
curve = fftavg.calc_radial_dist(ny, 0, 0.5, 1)
sf_dx = 1.0 / (apix * 2.0 * ny)
return (curve, sf_dx)
def addNoiseToMapSolvent(map, varNoise):
#*********************************
#****** add Noise To Map *********
#*********************************
mapData = np.copy(map)
mapSize = mapData.shape
noiseMap = np.random.randn(mapSize[0], mapSize[1],
mapSize[2]) * math.sqrt(varNoise)
mask = EMData()
mask.set_size(mapSize[0], mapSize[1], mapSize[2])
mask.to_zero()
sphere_radius = (np.min(mapSize) / 2.0 - 60)
mask.process_inplace("testimage.circlesphere", {"radius": sphere_radius})
maskData = np.copy(EMNumPy.em2numpy(mask))
maskData[maskData > 0] = 10
maskData[maskData <= 0] = 0
maskData[maskData == 0] = 1
maskData[maskData == 10] = 0
noiseMap = noiseMap * maskData
mapData = mapData + noiseMap
return mapData
def main(args):
dens = EMData(args.map)
star = parse_star(args.input, keep_index=False)
star[["ImageNumber", "ImageName"]] = star['rlnImageName'].str.split("@", expand=True)
grouped = star.groupby("ImageName")
pool = None
if args.nproc > 1:
pool = Pool(processes=args.nproc)
results = pool.imap(lambda x: project_stack(x, dens, args.dest), (group for name, group in grouped))
else:
results = (project_stack(group, dens, args.dest) for name, group in grouped)
i = 0
t = 0
for r in results:
i += 1
t += r
sys.stdout.write("\rProjected %d particles in %d stacks" % (t, i))
sys.stdout.flush()
if pool is not None:
pool.close()
pool.join()
sys.stdout.write('\n')
sys.stdout.flush()
return 0
def update_iso_file(self):
iso_file_path = str(self.volume_line_edit.text())
data = EMData(iso_file_path)
self.viewer_window.set_data(data, iso_file_path)
self.iso_model = self.viewer_window.viewables[0]
# if self.iso_model == None:
# self. __init_iso_model()
# self.iso_model.set_data(data)
self.viewer_window.updateGL()
def main(args):
star = parse_star(args.input, keep_index=False)
if args.class_2d is not None:
refs = glob.glob(args.class_2d)
elif args.class_3d is not None:
refs = glob.glob(args.class_3d)
else:
refs = []
shifts = []
for r in refs:
if args.class_3d is not None:
refmap = EMData(r)
com = Vec3f(*refmap.phase_cog()[:3])
shifts.append(com)
else:
stack = EMData.read_images(r)
for im in stack:
com = Vec2f(*im.phase_cog()[:2])
shifts.append(com)
if args.class_2d is None and args.class_3d is None:
for ptcl in star.rows:
im = EMData.read_image(ptcl)
com = im.phase_cog()
ptcl["rlnOriginX"] += com[0]
ptcl["rlnOriginY"] += com[1]
else:
for ptcl in star.rows:
com = shifts[ptcl["rlnClassNumber"]]
xshift, yshift = transform_com(com, ptcl)
ptcl["rlnOriginX"] += xshift
ptcl["rlnOriginY"] += yshift
if args.zero_origin:
star["rlnCoordinateX"] = star["rlnCoordinateX"] - star["rlnOriginX"]
star["rlnCoordinateY"] = star["rlnCoordinateY"] - star["rlnOriginY"]
star["rlnOriginX"] = 0
star["rlnOriginY"] = 0
write_star(args.output, star, reindex=True)
return 0
def getmxim(fsp,fsp2,clsnum): """reads the raw particles associated with a particular class. fsp2 is the matrix file and fsp is the raw particle file""" mx=EMData(fsp2,0) dx=EMData(fsp2,2) dy=EMData(fsp2,3) da=EMData(fsp2,4) imgs=[(EMData(fsp,i),dx.get(0,i),dy.get(0,i),da.get(0,i)) for i in range(mx.get_ysize()) if mx.get(0,i)==clsnum] for i in imgs : print i i[0].rotate_translate(i[3],0,0,i[1],i[2],0) imgs=[i[0] for i in imgs] return imgs
def compute_average_noctf(mlist, radius):
from fundamentals import fft
params_list = [None] * len(mlist)
orig_image_size = mlist[0].get_xsize()
avgo = EMData(orig_image_size, orig_image_size, 1, False) #
avge = EMData(orig_image_size, orig_image_size, 1, False) #
for im in xrange(len(mlist)):
alpha, sx, sy, mr, scale = get_params2D(mlist[im],
xform="xform.align2d")
params_list[im] = [alpha, sx, sy, mr, scale]
tmp = cosinemask(rot_shift2D(mlist[im], alpha, sx, sy, mr), radius)
tmp = fft(tmp)
if im % 2 == 0: Util.add_img(avge, tmp)
else: Util.add_img(avgo, tmp)
frc = fsc(fft(avge), fft(avgo))
frc[1][0] = 1.0
for ifreq in xrange(1, len(frc[0])):
frc[1][ifreq] = max(0.0, frc[1][ifreq])
frc[1][ifreq] = 2. * frc[1][ifreq] / (1. + frc[1][ifreq])
sumavg = Util.addn_img(avgo, avge)
sumavg = fft(sumavg)
return sumavg, frc, params_list
def particles(star):
"""
Generator function using StarFile object to produce particle EMData and MetaData objects.
:param star: StarFile object
:return: Tuple holding (particle EMData, particle MetaData)
"""
if isinstance(star, DataFrame):
star = star.to_dict(orient="list")
npart = len(star['rlnImageName'])
for i in range(npart):
meta = MetaData(star, i)
ptcl = EMData(meta.name, meta.number)
yield ptcl, meta
def compute_average_ctf(mlist, radius):
from morphology import ctf_img
from filter import filt_ctf, filt_table
from fundamentals import fft, fftip
params_list = [None] * len(mlist)
orig_image_size = mlist[0].get_xsize()
avgo = EMData(orig_image_size, orig_image_size, 1, False) #
avge = EMData(orig_image_size, orig_image_size, 1, False) #
ctf_2_sumo = EMData(orig_image_size, orig_image_size, 1, False)
ctf_2_sume = EMData(orig_image_size, orig_image_size, 1, False)
for im in xrange(len(mlist)):
ctt = ctf_img(orig_image_size, mlist[im].get_attr("ctf"))
alpha, sx, sy, mr, scale = get_params2D(mlist[im],
xform="xform.align2d")
tmp = cosinemask(rot_shift2D(mlist[im], alpha, sx, sy, mr), radius)
params_list[im] = [alpha, sx, sy, mr, scale]
tmp = fft(tmp)
Util.mul_img(tmp, ctt)
#ima_filt = filt_ctf(tmp, ctf_params, dopad=False)
if im % 2 == 0:
Util.add_img2(ctf_2_sume, ctt)
Util.add_img(avge, tmp)
else:
Util.add_img2(ctf_2_sumo, ctt)
Util.add_img(avgo, tmp)
sumavg = Util.divn_img(avge, ctf_2_sume)
sumctf2 = Util.divn_img(avgo, ctf_2_sumo)
frc = fsc(fft(sumavg), fft(sumctf2))
frc[1][0] = 1.0
for ifreq in xrange(1, len(frc[0])):
frc[1][ifreq] = max(0.0, frc[1][ifreq])
frc[1][ifreq] = 2. * frc[1][ifreq] / (1. + frc[1][ifreq])
sumavg = Util.addn_img(avgo, avge)
sumctf2 = Util.addn_img(ctf_2_sume, ctf_2_sumo)
Util.div_img(sumavg, sumctf2)
sumavg = fft(sumavg)
return sumavg, frc, params_list
def setup_test_data(voldim=30, size=10):
from sparx import model_gauss
emmap = model_gauss(size, voldim, voldim, voldim)
modmap = EMData()
modmap.set_size(voldim, voldim, voldim)
modmap.process_inplace("testimage.noise.gauss", {"sigma": 1, "seed": 99})
mask = model_square(size, voldim, voldim, voldim)
return emmap, modmap, mask
def internal_test_image2(self, nx, ny=1, nz=1):
from EMAN2 import EMData, display
from fundamentals import cyclic_shift, mirror
e = EMData()
e.set_size(nx, ny, nz)
e.process_inplace("testimage.tomo.objects")
e = cyclic_shift(e, nx/2, ny/3, nz/5)
e = mirror(e)
return e
def getalldata(stack, myid, nproc):
if (myid == 0): ndata = EMUtil.get_image_count(stack)
else: ndata = 0
ndata = bcast_number_to_all(ndata)
if (ndata < nproc):
if (myid < ndata):
image_start = myid
image_end = myid + 1
else:
image_start = 0
image_end = 1
else:
image_start, image_end = MPI_start_end(ndata, nproc, myid)
data = EMData.read_images(stack, list(range(image_start, image_end)))
return data
def montage2(inputstack, ncol, marginwidth=0, bkgd=0, outfile=None):
"""
Generates montage of images into one image.
Adapted from sxmontage.py
Arguments:
inputstack : Stack of input images to merge into montage
ncol : Number of images per row
marginwidth : Margin width, pixels
bkgd : Background value of montage
outfile : Optional output file with montage output
Returns:
montage : EMData object of image montage
"""
if isinstance(inputstack, str): inputstack = EMData.read_images(inputstack)
# Get single-image dimensions
nx = inputstack[0].get_xsize()
ny = inputstack[0].get_ysize()
# Get number of images and calculate montage dimensions
numimgs = len(inputstack)
numrows = (numimgs - 1) / ncol + 1
# Create blank image
montage_xdim = (nx + marginwidth) * ncol
montage_ydim = (ny + marginwidth) * numrows
montage = model_blank(montage_xdim, montage_ydim, 1, bkgd)
# Loop through images
for imgnum in range(numimgs):
# Horizontal grid position is image# modulo NCOL
colnum = imgnum % ncol
# Montage is numbered from the top down
rownum = numrows - 1 - imgnum / ncol
xoffset = colnum * (nx + marginwidth)
yoffset = rownum * (ny + marginwidth)
insert_image(inputstack[imgnum], montage, xoffset, yoffset)
if outfile: montage.write_image(outfile)
return montage
def main(args):
if os.path.exists(args.output):
os.remove(args.output)
for fn in args.input:
if fn.endswith(".star"):
star = parse_star(fn, keep_index=False)
for p in star["rlnImageName"]:
stack = p.split("@")[1]
idx = int(p.split("@")[0]) - 1
try:
img = EMData(stack, idx)
img.append_image(args.output)
except Exception:
print("Error at %s" % p)
else:
n = EMUtil.get_image_count(fn)
for i in range(n):
img = EMData(fn, i)
img.append_image(args.output)
return 0
def prepare_mask_and_maps_for_scaling(args):
emmap = get_image(args.em_map)
modmap = get_image(args.model_map)
if args.mask is None:
mask = EMData()
xsize, ysize, zsize = emmap.get_xsize(), emmap.get_ysize(
), emmap.get_zsize()
mask.set_size(xsize, ysize, zsize)
mask.to_zero()
if xsize == ysize and xsize == zsize and ysize == zsize:
sphere_radius = xsize // 2
mask.process_inplace("testimage.circlesphere",
{"radius": sphere_radius})
else:
mask += 1
mask = Util.window(mask, xsize - 1, ysize - 1, zsize - 1)
mask = Util.pad(mask, xsize, ysize, zsize, 0, 0, 0, '0')
elif args.mask is not None:
mask = binarize(get_image(args.mask), 0.5)
if args.window_size is None:
wn = int(math.ceil(round((7 * 3 * args.apix)) / 2.) * 2)
elif args.window_size is not None:
wn = int(math.ceil(args.window_size / 2.) * 2)
window_bleed_and_pad = check_for_window_bleeding(mask, wn)
if window_bleed_and_pad:
pad_int_emmap = compute_padding_average(emmap, mask)
pad_int_modmap = compute_padding_average(modmap, mask)
map_shape = [(emmap.get_xsize() + wn), (emmap.get_ysize() + wn),
(emmap.get_zsize() + wn)]
emmap = Util.pad(emmap, map_shape[0], map_shape[1], map_shape[2], 0, 0,
0, 'pad_int_emmap')
modmap = Util.pad(modmap, map_shape[0], map_shape[1], map_shape[2], 0,
0, 0, 'pad_int_modmap')
mask = Util.pad(mask, map_shape[0], map_shape[1], map_shape[2], 0, 0,
0, '0')
return emmap, modmap, mask, wn, window_bleed_and_pad
def getindexdata(stack, partids, partstack, myid, nproc): # The function will read from stack a subset of images specified in partids # and assign to them parameters from partstack # So, the lengths of partids and partstack are the same. # The read data is properly distributed among MPI threads. lpartids = list(map(int, read_text_file(partids) )) ndata = len(lpartids) partstack = read_text_row(partstack) if( ndata < nproc): if(myid<ndata): image_start = myid image_end = myid+1 else: image_start = 0 image_end = 1 else: image_start, image_end = MPI_start_end(ndata, nproc, myid) lpartids = lpartids[image_start:image_end] partstack = partstack[image_start:image_end] data = EMData.read_images(stack, lpartids) for i in range(len(partstack)): set_params_proj(data[i], partstack[i]) return data
def readAndFlattenImageStack(filename):
#***********************************************
#* read imageStack and return Nx(nX*nY) array **
#***********************************************
imageStack = EMData()
imageStack.read_image(filename)
nx, ny, numImages = imageStack.get_xsize(), imageStack.get_ysize(
), imageStack.get_zsize()
imageStackData = EMNumPy.em2numpy(imageStack)
imageStack = []
#free memory
#flatten each image and append to array of NxD, with N the number of images and D the size of the flat image
imageStackData = np.reshape(imageStackData, (nx * ny, numImages),
order='F')
imageStackData = np.transpose(imageStackData)
print(imageStackData.shape)
return imageStackData
def main():
from logger import Logger, BaseLogger_Files
import user_functions
from optparse import OptionParser, SUPPRESS_HELP
from global_def import SPARXVERSION
from EMAN2 import EMData
main_node = 0
mpi_init(0, [])
mpi_comm = MPI_COMM_WORLD
myid = mpi_comm_rank(MPI_COMM_WORLD)
mpi_size = mpi_comm_size(MPI_COMM_WORLD) # Total number of processes, passed by --np option.
# mpi_barrier(mpi_comm)
# from mpi import mpi_finalize
# mpi_finalize()
# print "mpi finalize"
# from sys import exit
# exit()
progname = os.path.basename(sys.argv[0])
usage = progname + " stack [output_directory] --ir=inner_radius --radius=outer_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translational_search_step --delta=angular_step --an=angular_neighborhood --center=center_type --maxit1=max_iter1 --maxit2=max_iter2 --L2threshold=0.1 --fl --aa --ref_a=S --sym=c1"
usage += """
stack 2D images in a stack file: (default required string)
output_directory: directory name into which the output files will be written. If it does not exist, the directory will be created. If it does exist, the program will continue executing from where it stopped (if it did not already reach the end). The "--use_latest_master_directory" option can be used to choose the most recent directory that starts with "master".
"""
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--radius", type="int", help="radius of the particle: has to be less than < int(nx/2)-1 (default required int)")
parser.add_option("--ir", type="int", default=1, help="inner radius for rotational search: > 0 (default 1)")
parser.add_option("--rs", type="int", default=1, help="step between rings in rotational search: >0 (default 1)")
parser.add_option("--xr", type="string", default='0', help="range for translation search in x direction: search is +/xr in pixels (default '0')")
parser.add_option("--yr", type="string", default='0', help="range for translation search in y direction: if omitted will be set to xr, search is +/yr in pixels (default '0')")
parser.add_option("--ts", type="string", default='1.0', help="step size of the translation search in x-y directions: search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional (default '1.0')")
parser.add_option("--delta", type="string", default='2.0', help="angular step of reference projections: (default '2.0')")
#parser.add_option("--an", type="string", default= "-1", help="angular neighborhood for local searches (phi and theta)")
parser.add_option("--center", type="float", default=-1.0, help="centering of 3D template: average shift method; 0: no centering; 1: center of gravity (default -1.0)")
parser.add_option("--maxit1", type="int", default=400, help="maximum number of iterations performed for the GA part: (default 400)")
parser.add_option("--maxit2", type="int", default=50, help="maximum number of iterations performed for the finishing up part: (default 50)")
parser.add_option("--L2threshold", type="float", default=0.03, help="stopping criterion of GA: given as a maximum relative dispersion of volumes' L2 norms: (default 0.03)")
parser.add_option("--doga", type="float", default=0.1, help="do GA when fraction of orientation changes less than 1.0 degrees is at least doga: (default 0.1)")
parser.add_option("--n_shc_runs", type="int", default=4, help="number of quasi-independent shc runs (same as '--nruns' parameter from sxviper.py): (default 4)")
parser.add_option("--n_rv_runs", type="int", default=10, help="number of rviper iterations: (default 10)")
parser.add_option("--n_v_runs", type="int", default=3, help="number of viper runs for each r_viper cycle: (default 3)")
parser.add_option("--outlier_percentile", type="float", default=95.0, help="percentile above which outliers are removed every rviper iteration: (default 95.0)")
parser.add_option("--iteration_start", type="int", default=0, help="starting iteration for rviper: 0 means go to the most recent one (default 0)")
#parser.add_option("--CTF", action="store_true", default=False, help="NOT IMPLEMENTED Consider CTF correction during the alignment ")
#parser.add_option("--snr", type="float", default= 1.0, help="Signal-to-Noise Ratio of the data (default 1.0)")
parser.add_option("--ref_a", type="string", default='S', help="method for generating the quasi-uniformly distributed projection directions: (default S)")
parser.add_option("--sym", type="string", default='c1', help="point-group symmetry of the structure: (default c1)")
# parser.add_option("--function", type="string", default="ref_ali3d", help="name of the reference preparation function (ref_ali3d by default)")
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--function", type="string", default="ref_ali3d", help=SUPPRESS_HELP)
parser.add_option("--npad", type="int", default=2, help="padding size for 3D reconstruction: (default 2)")
# parser.add_option("--npad", type="int", default= 2, help="padding size for 3D reconstruction (default 2)")
#options introduced for the do_volume function
parser.add_option("--fl", type="float", default=0.25, help="cut-off frequency applied to the template volume: using a hyperbolic tangent low-pass filter (default 0.25)")
parser.add_option("--aa", type="float", default=0.1, help="fall-off of hyperbolic tangent low-pass filter: (default 0.1)")
parser.add_option("--pwreference", type="string", default='', help="text file with a reference power spectrum: (default none)")
parser.add_option("--mask3D", type="string", default=None, help="3D mask file: (default sphere)")
parser.add_option("--moon_elimination", type="string", default='', help="elimination of disconnected pieces: two arguments: mass in KDa and pixel size in px/A separated by comma, no space (default none)")
# used for debugging, help is supressed with SUPPRESS_HELP
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--my_random_seed", type="int", default=123, help = SUPPRESS_HELP)
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--run_get_already_processed_viper_runs", action="store_true", dest="run_get_already_processed_viper_runs", default=False, help = SUPPRESS_HELP)
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--use_latest_master_directory", action="store_true", dest="use_latest_master_directory", default=False, help = SUPPRESS_HELP)
parser.add_option("--criterion_name", type="string", default='80th percentile',help="criterion deciding if volumes have a core set of stable projections: '80th percentile', other options:'fastest increase in the last quartile' (default '80th percentile')")
parser.add_option("--outlier_index_threshold_method",type="string", default='discontinuity_in_derivative',help="method that decides which images to keep: discontinuity_in_derivative, other options:percentile, angle_measure (default discontinuity_in_derivative)")
parser.add_option("--angle_threshold", type="int", default=30, help="angle threshold for projection removal if using 'angle_measure': (default 30)")
required_option_list = ['radius']
(options, args) = parser.parse_args(sys.argv[1:])
options.CTF = False
options.snr = 1.0
options.an = -1
if options.moon_elimination == "":
options.moon_elimination = []
else:
options.moon_elimination = map(float, options.moon_elimination.split(","))
# Making sure all required options appeared.
for required_option in required_option_list:
if not options.__dict__[required_option]:
print "\n ==%s== mandatory option is missing.\n"%required_option
print "Please run '" + progname + " -h' for detailed options"
return 1
mpi_barrier(MPI_COMM_WORLD)
if(myid == main_node):
print "****************************************************************"
Util.version()
print "****************************************************************"
sys.stdout.flush()
mpi_barrier(MPI_COMM_WORLD)
# this is just for benefiting from a user friendly parameter name
options.ou = options.radius
my_random_seed = options.my_random_seed
criterion_name = options.criterion_name
outlier_index_threshold_method = options.outlier_index_threshold_method
use_latest_master_directory = options.use_latest_master_directory
iteration_start_default = options.iteration_start
number_of_rrr_viper_runs = options.n_rv_runs
no_of_viper_runs_analyzed_together_from_user_options = options.n_v_runs
no_of_shc_runs_analyzed_together = options.n_shc_runs
outlier_percentile = options.outlier_percentile
angle_threshold = options.angle_threshold
run_get_already_processed_viper_runs = options.run_get_already_processed_viper_runs
get_already_processed_viper_runs(run_get_already_processed_viper_runs)
import random
random.seed(my_random_seed)
if len(args) < 1 or len(args) > 3:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
return 1
# if len(args) > 2:
# ref_vol = get_im(args[2])
# else:
ref_vol = None
# error_status = None
# if myid == 0:
# number_of_images = EMUtil.get_image_count(args[0])
# if mpi_size > number_of_images:
# error_status = ('Number of processes supplied by --np in mpirun needs to be less than or equal to %d (total number of images) ' % number_of_images, getframeinfo(currentframe()))
# if_error_then_all_processes_exit_program(error_status)
bdb_stack_location = ""
masterdir = ""
if len(args) == 2:
masterdir = args[1]
if masterdir[-1] != DIR_DELIM:
masterdir += DIR_DELIM
elif len(args) == 1:
if use_latest_master_directory:
all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)]
import re; r = re.compile("^master.*$")
all_dirs = filter(r.match, all_dirs)
if len(all_dirs)>0:
# all_dirs = max(all_dirs, key=os.path.getctime)
masterdir = max(all_dirs, key=os.path.getmtime)
masterdir += DIR_DELIM
log = Logger(BaseLogger_Files())
error_status = 0
if mpi_size % no_of_shc_runs_analyzed_together != 0:
ERROR('Number of processes needs to be a multiple of the number of quasi-independent runs (shc) within each viper run. '
'Total quasi-independent runs by default are 3, you can change it by specifying '
'--n_shc_runs option (in sxviper this option is called --nruns). Also, to improve communication time it is recommended that '
'the number of processes divided by the number of quasi-independent runs is a power '
'of 2 (e.g. 2, 4, 8 or 16 depending on how many physical cores each node has).', 'sxviper', 1)
error_status = 1
if_error_then_all_processes_exit_program(error_status)
#Create folder for all results or check if there is one created already
if(myid == main_node):
#cmd = "{}".format("Rmycounter ccc")
#cmdexecute(cmd)
if( masterdir == ""):
timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime())
masterdir = "master"+timestring
if not os.path.exists(masterdir):
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
if ':' in args[0]:
bdb_stack_location = args[0].split(":")[0] + ":" + masterdir + args[0].split(":")[1]
org_stack_location = args[0]
if(not os.path.exists(os.path.join(masterdir,"EMAN2DB" + DIR_DELIM))):
# cmd = "{} {}".format("cp -rp EMAN2DB", masterdir, "EMAN2DB" DIR_DELIM)
# cmdexecute(cmd)
cmd = "{} {} {}".format("e2bdb.py", org_stack_location,"--makevstack=" + bdb_stack_location + "_000")
cmdexecute(cmd)
from applications import header
try:
header(bdb_stack_location + "_000", params='original_image_index', fprint=True)
print "Images were already indexed!"
except KeyError:
print "Indexing images"
header(bdb_stack_location + "_000", params='original_image_index', consecutive=True)
else:
filename = os.path.basename(args[0])
bdb_stack_location = "bdb:" + masterdir + os.path.splitext(filename)[0]
if(not os.path.exists(os.path.join(masterdir,"EMAN2DB" + DIR_DELIM))):
cmd = "{} {} {}".format("sxcpy.py ", args[0], bdb_stack_location + "_000")
cmdexecute(cmd)
from applications import header
try:
header(bdb_stack_location + "_000", params='original_image_index', fprint=True)
print "Images were already indexed!"
except KeyError:
print "Indexing images"
header(bdb_stack_location + "_000", params='original_image_index', consecutive=True)
# send masterdir to all processes
dir_len = len(masterdir)*int(myid == main_node)
dir_len = mpi_bcast(dir_len,1,MPI_INT,0,MPI_COMM_WORLD)[0]
masterdir = mpi_bcast(masterdir,dir_len,MPI_CHAR,main_node,MPI_COMM_WORLD)
masterdir = string.join(masterdir,"")
if masterdir[-1] != DIR_DELIM:
masterdir += DIR_DELIM
global_def.LOGFILE = os.path.join(masterdir, global_def.LOGFILE)
print_program_start_information()
# mpi_barrier(mpi_comm)
# from mpi import mpi_finalize
# mpi_finalize()
# print "mpi finalize"
# from sys import exit
# exit()
# send bdb_stack_location to all processes
dir_len = len(bdb_stack_location)*int(myid == main_node)
dir_len = mpi_bcast(dir_len,1,MPI_INT,0,MPI_COMM_WORLD)[0]
bdb_stack_location = mpi_bcast(bdb_stack_location,dir_len,MPI_CHAR,main_node,MPI_COMM_WORLD)
bdb_stack_location = string.join(bdb_stack_location,"")
iteration_start = get_latest_directory_increment_value(masterdir, "main")
if (myid == main_node):
if (iteration_start < iteration_start_default):
ERROR('Starting iteration provided is greater than last iteration performed. Quiting program', 'sxviper', 1)
error_status = 1
if iteration_start_default!=0:
iteration_start = iteration_start_default
if (myid == main_node):
if (number_of_rrr_viper_runs < iteration_start):
ERROR('Please provide number of rviper runs (--n_rv_runs) greater than number of iterations already performed.', 'sxviper', 1)
error_status = 1
if_error_then_all_processes_exit_program(error_status)
for rviper_iter in range(iteration_start, number_of_rrr_viper_runs + 1):
if(myid == main_node):
all_projs = EMData.read_images(bdb_stack_location + "_%03d"%(rviper_iter - 1))
print "XXXXXXXXXXXXXXXXX"
print "Number of projections (in loop): " + str(len(all_projs))
print "XXXXXXXXXXXXXXXXX"
subset = range(len(all_projs))
else:
all_projs = None
subset = None
runs_iter = get_latest_directory_increment_value(masterdir + NAME_OF_MAIN_DIR + "%03d"%rviper_iter, DIR_DELIM + NAME_OF_RUN_DIR, start_value=0) - 1
no_of_viper_runs_analyzed_together = max(runs_iter + 2, no_of_viper_runs_analyzed_together_from_user_options)
first_time_entering_the_loop_need_to_do_full_check_up = True
while True:
runs_iter += 1
if not first_time_entering_the_loop_need_to_do_full_check_up:
if runs_iter >= no_of_viper_runs_analyzed_together:
break
first_time_entering_the_loop_need_to_do_full_check_up = False
this_run_is_NOT_complete = 0
if (myid == main_node):
independent_run_dir = masterdir + DIR_DELIM + NAME_OF_MAIN_DIR + ('%03d' + DIR_DELIM + NAME_OF_RUN_DIR + "%03d" + DIR_DELIM)%(rviper_iter, runs_iter)
if run_get_already_processed_viper_runs:
cmd = "{} {}".format("mkdir -p", masterdir + DIR_DELIM + NAME_OF_MAIN_DIR + ('%03d' + DIR_DELIM)%(rviper_iter)); cmdexecute(cmd)
cmd = "{} {}".format("rm -rf", independent_run_dir); cmdexecute(cmd)
cmd = "{} {}".format("cp -r", get_already_processed_viper_runs() + " " + independent_run_dir); cmdexecute(cmd)
if os.path.exists(independent_run_dir + "log.txt") and (string_found_in_file("Finish VIPER2", independent_run_dir + "log.txt")):
this_run_is_NOT_complete = 0
else:
this_run_is_NOT_complete = 1
cmd = "{} {}".format("rm -rf", independent_run_dir); cmdexecute(cmd)
cmd = "{} {}".format("mkdir -p", independent_run_dir); cmdexecute(cmd)
this_run_is_NOT_complete = mpi_bcast(this_run_is_NOT_complete,1,MPI_INT,main_node,MPI_COMM_WORLD)[0]
dir_len = len(independent_run_dir)
dir_len = mpi_bcast(dir_len,1,MPI_INT,main_node,MPI_COMM_WORLD)[0]
independent_run_dir = mpi_bcast(independent_run_dir,dir_len,MPI_CHAR,main_node,MPI_COMM_WORLD)
independent_run_dir = string.join(independent_run_dir,"")
else:
this_run_is_NOT_complete = mpi_bcast(this_run_is_NOT_complete,1,MPI_INT,main_node,MPI_COMM_WORLD)[0]
dir_len = 0
independent_run_dir = ""
dir_len = mpi_bcast(dir_len,1,MPI_INT,main_node,MPI_COMM_WORLD)[0]
independent_run_dir = mpi_bcast(independent_run_dir,dir_len,MPI_CHAR,main_node,MPI_COMM_WORLD)
independent_run_dir = string.join(independent_run_dir,"")
if this_run_is_NOT_complete:
mpi_barrier(MPI_COMM_WORLD)
if independent_run_dir[-1] != DIR_DELIM:
independent_run_dir += DIR_DELIM
log.prefix = independent_run_dir
options.user_func = user_functions.factory[options.function]
# for debugging purposes
#if (myid == main_node):
#cmd = "{} {}".format("cp ~/log.txt ", independent_run_dir)
#cmdexecute(cmd)
#cmd = "{} {}{}".format("cp ~/paramdir/params$(mycounter ccc).txt ", independent_run_dir, "param%03d.txt"%runs_iter)
#cmd = "{} {}{}".format("cp ~/paramdir/params$(mycounter ccc).txt ", independent_run_dir, "params.txt")
#cmdexecute(cmd)
if (myid == main_node):
store_value_of_simple_vars_in_json_file(masterdir + 'program_state_stack.json', locals(), exclude_list_of_vars=["usage"],
vars_that_will_show_only_size = ["subset"])
store_value_of_simple_vars_in_json_file(masterdir + 'program_state_stack.json', options.__dict__, write_or_append='a')
# mpi_barrier(mpi_comm)
# from mpi import mpi_finalize
# mpi_finalize()
# print "mpi finalize"
# from sys import exit
# exit()
out_params, out_vol, out_peaks = multi_shc(all_projs, subset, no_of_shc_runs_analyzed_together, options,
mpi_comm=mpi_comm, log=log, ref_vol=ref_vol)
# end of: if this_run_is_NOT_complete:
if runs_iter >= (no_of_viper_runs_analyzed_together_from_user_options - 1):
increment_for_current_iteration = identify_outliers(myid, main_node, rviper_iter,
no_of_viper_runs_analyzed_together, no_of_viper_runs_analyzed_together_from_user_options, masterdir,
bdb_stack_location, outlier_percentile, criterion_name, outlier_index_threshold_method, angle_threshold)
if increment_for_current_iteration == MUST_END_PROGRAM_THIS_ITERATION:
break
no_of_viper_runs_analyzed_together += increment_for_current_iteration
# end of independent viper loop
calculate_volumes_after_rotation_and_save_them(options, rviper_iter, masterdir, bdb_stack_location, myid,
mpi_size, no_of_viper_runs_analyzed_together, no_of_viper_runs_analyzed_together_from_user_options)
if increment_for_current_iteration == MUST_END_PROGRAM_THIS_ITERATION:
if (myid == main_node):
print "RVIPER found a core set of stable projections for the current RVIPER iteration (%d), the maximum angle difference between corresponding projections from different VIPER volumes is less than %.2f. Finishing."%(rviper_iter, ANGLE_ERROR_THRESHOLD)
break
else:
if (myid == main_node):
print "After running the last iteration (%d), RVIPER did not find a set of projections with the maximum angle difference between corresponding projections from different VIPER volumes less than %.2f Finishing."%(rviper_iter, ANGLE_ERROR_THRESHOLD)
# end of RVIPER loop
#mpi_finalize()
#sys.exit()
mpi_barrier(MPI_COMM_WORLD)
mpi_finalize()
def main(args):
from utilities import if_error_then_all_processes_exit_program, write_text_row, drop_image, model_gauss_noise, get_im, set_params_proj, wrap_mpi_bcast, model_circle
from logger import Logger, BaseLogger_Files
from mpi import mpi_init, mpi_finalize, MPI_COMM_WORLD, mpi_comm_rank, mpi_comm_size, mpi_barrier
import user_functions
import sys
import os
from applications import MPI_start_end
from optparse import OptionParser, SUPPRESS_HELP
from global_def import SPARXVERSION
from EMAN2 import EMData
from multi_shc import multi_shc
progname = os.path.basename(sys.argv[0])
usage = progname + " stack [output_directory] --ir=inner_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translational_search_step --delta=angular_step --center=center_type --maxit1=max_iter1 --maxit2=max_iter2 --L2threshold=0.1 --ref_a=S --sym=c1"
usage += """
stack 2D images in a stack file: (default required string)
directory output directory name: into which the results will be written (if it does not exist, it will be created, if it does exist, the results will be written possibly overwriting previous results) (default required string)
"""
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option(
"--radius",
type="int",
help=
"radius of the particle: has to be less than < int(nx/2)-1 (default required int)"
)
parser.add_option(
"--xr",
type="string",
default='0',
help=
"range for translation search in x direction: search is +/xr in pixels (default '0')"
)
parser.add_option(
"--yr",
type="string",
default='0',
help=
"range for translation search in y direction: if omitted will be set to xr, search is +/yr in pixels (default '0')"
)
parser.add_option("--mask3D",
type="string",
default=None,
help="3D mask file: (default sphere)")
parser.add_option(
"--moon_elimination",
type="string",
default='',
help=
"elimination of disconnected pieces: two arguments: mass in KDa and pixel size in px/A separated by comma, no space (default none)"
)
parser.add_option(
"--ir",
type="int",
default=1,
help="inner radius for rotational search: > 0 (default 1)")
# 'radius' and 'ou' are the same as per Pawel's request; 'ou' is hidden from the user
# the 'ou' variable is not changed to 'radius' in the 'sparx' program. This change is at interface level only for sxviper.
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--ou", type="int", default=-1, help=SUPPRESS_HELP)
parser.add_option(
"--rs",
type="int",
default=1,
help="step between rings in rotational search: >0 (default 1)")
parser.add_option(
"--ts",
type="string",
default='1.0',
help=
"step size of the translation search in x-y directions: search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional (default '1.0')"
)
parser.add_option(
"--delta",
type="string",
default='2.0',
help="angular step of reference projections: (default '2.0')")
parser.add_option(
"--center",
type="float",
default=-1.0,
help=
"centering of 3D template: average shift method; 0: no centering; 1: center of gravity (default -1.0)"
)
parser.add_option(
"--maxit1",
type="int",
default=400,
help=
"maximum number of iterations performed for the GA part: (default 400)"
)
parser.add_option(
"--maxit2",
type="int",
default=50,
help=
"maximum number of iterations performed for the finishing up part: (default 50)"
)
parser.add_option(
"--L2threshold",
type="float",
default=0.03,
help=
"stopping criterion of GA: given as a maximum relative dispersion of volumes' L2 norms: (default 0.03)"
)
parser.add_option(
"--ref_a",
type="string",
default='S',
help=
"method for generating the quasi-uniformly distributed projection directions: (default S)"
)
parser.add_option(
"--sym",
type="string",
default='c1',
help="point-group symmetry of the structure: (default c1)")
# parser.add_option("--function", type="string", default="ref_ali3d", help="name of the reference preparation function (ref_ali3d by default)")
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--function",
type="string",
default="ref_ali3d",
help=SUPPRESS_HELP)
parser.add_option(
"--nruns",
type="int",
default=6,
help=
"GA population: aka number of quasi-independent volumes (default 6)")
parser.add_option(
"--doga",
type="float",
default=0.1,
help=
"do GA when fraction of orientation changes less than 1.0 degrees is at least doga: (default 0.1)"
)
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--npad",
type="int",
default=2,
help="padding size for 3D reconstruction (default=2)")
parser.add_option(
"--fl",
type="float",
default=0.25,
help=
"cut-off frequency applied to the template volume: using a hyperbolic tangent low-pass filter (default 0.25)"
)
parser.add_option(
"--aa",
type="float",
default=0.1,
help="fall-off of hyperbolic tangent low-pass filter: (default 0.1)")
parser.add_option(
"--pwreference",
type="string",
default='',
help="text file with a reference power spectrum: (default none)")
parser.add_option("--debug",
action="store_true",
default=False,
help="debug info printout: (default False)")
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--return_options",
action="store_true",
dest="return_options",
default=False,
help=SUPPRESS_HELP)
#parser.add_option("--an", type="string", default= "-1", help="NOT USED angular neighborhood for local searches (phi and theta)")
#parser.add_option("--CTF", action="store_true", default=False, help="NOT USED Consider CTF correction during the alignment ")
#parser.add_option("--snr", type="float", default= 1.0, help="NOT USED Signal-to-Noise Ratio of the data (default 1.0)")
# (options, args) = parser.parse_args(sys.argv[1:])
required_option_list = ['radius']
(options, args) = parser.parse_args(args)
# option_dict = vars(options)
# print parser
if options.return_options:
return parser
if options.moon_elimination == "":
options.moon_elimination = []
else:
options.moon_elimination = map(float,
options.moon_elimination.split(","))
# Making sure all required options appeared.
for required_option in required_option_list:
if not options.__dict__[required_option]:
print "\n ==%s== mandatory option is missing.\n" % required_option
print "Please run '" + progname + " -h' for detailed options"
return 1
if len(args) < 2 or len(args) > 3:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
return 1
mpi_init(0, [])
log = Logger(BaseLogger_Files())
# 'radius' and 'ou' are the same as per Pawel's request; 'ou' is hidden from the user
# the 'ou' variable is not changed to 'radius' in the 'sparx' program. This change is at interface level only for sxviper.
options.ou = options.radius
runs_count = options.nruns
mpi_rank = mpi_comm_rank(MPI_COMM_WORLD)
mpi_size = mpi_comm_size(
MPI_COMM_WORLD) # Total number of processes, passed by --np option.
if mpi_rank == 0:
all_projs = EMData.read_images(args[0])
subset = range(len(all_projs))
# if mpi_size > len(all_projs):
# ERROR('Number of processes supplied by --np needs to be less than or equal to %d (total number of images) ' % len(all_projs), 'sxviper', 1)
# mpi_finalize()
# return
else:
all_projs = None
subset = None
outdir = args[1]
if mpi_rank == 0:
if mpi_size % options.nruns != 0:
ERROR(
'Number of processes needs to be a multiple of total number of runs. Total runs by default are 3, you can change it by specifying --nruns option.',
'sxviper', 1)
mpi_finalize()
return
if os.path.exists(outdir):
ERROR(
'Output directory exists, please change the name and restart the program',
"sxviper", 1)
mpi_finalize()
return
os.mkdir(outdir)
import global_def
global_def.LOGFILE = os.path.join(outdir, global_def.LOGFILE)
mpi_barrier(MPI_COMM_WORLD)
if outdir[-1] != "/":
outdir += "/"
log.prefix = outdir
# if len(args) > 2:
# ref_vol = get_im(args[2])
# else:
ref_vol = None
options.user_func = user_functions.factory[options.function]
options.CTF = False
options.snr = 1.0
options.an = -1.0
from multi_shc import multi_shc
out_params, out_vol, out_peaks = multi_shc(all_projs,
subset,
runs_count,
options,
mpi_comm=MPI_COMM_WORLD,
log=log,
ref_vol=ref_vol)
mpi_finalize()
def main(options):
"""
Projection subtraction program entry point.
:param options: Command-line arguments parsed by ArgumentParser.parse_args()
:return: Exit status
"""
rchop = lambda x, y: x if not x.endswith(y) or len(y) == 0 else x[:-len(y)]
options.output = rchop(options.output, ".star")
options.suffix = rchop(options.suffix, ".mrc")
options.suffix = rchop(options.suffix, ".mrcs")
star = StarFile(options.input)
npart = len(star['rlnImageName'])
sub_dens = EMData(options.submap)
if options.wholemap is not None:
dens = EMData(options.wholemap)
else:
print "Reference map is required."
return 1
# Write star header for output.star.
top_header = "\ndata_\n\nloop_\n"
headings = star.keys()
output_star = open("{0}.star".format(options.output), 'w')
output_star.write(top_header)
for i, heading in enumerate(headings):
output_star.write("_{0} #{1}\n".format(heading, i + 1))
if options.recenter: # Compute difference vector between new and old mass centers.
if options.wholemap is None:
print "Reference map required for recentering."
return 1
new_dens = dens - sub_dens
# Note the sign of the shift in coordinate frame is opposite the shift in the CoM.
recenter = Vec3f(*dens.phase_cog()[:3]) - Vec3f(
*new_dens.phase_cog()[:3])
else:
recenter = None
pool = None
if options.nproc > 1: # Compute subtraction in parallel.
pool = Pool(processes=options.nproc)
results = pool.imap(
lambda x: subtract(x,
dens,
sub_dens,
recenter=recenter,
no_frc=options.no_frc,
low_cutoff=options.low_cutoff,
high_cutoff=options.high_cutoff),
particles(star),
chunksize=min(npart / options.nproc, options.maxchunk))
else: # Use serial generator.
results = (subtract(x,
dens,
sub_dens,
recenter=recenter,
no_frc=options.no_frc,
low_cutoff=options.low_cutoff,
high_cutoff=options.high_cutoff)
for x in particles(star))
# Write subtraction results to .mrcs and .star files.
i = 0
nfile = 1
starpath = None
mrcs = None
mrcs_orig = None
for r in results:
if i % options.maxpart == 0:
mrcsuffix = options.suffix + "_%d" % nfile
nfile += 1
starpath = "{0}.mrcs".format(
os.path.sep.join(
os.path.relpath(mrcsuffix,
options.output).split(os.path.sep)[1:]))
mrcs = "{0}.mrcs".format(mrcsuffix)
mrcs_orig = "{0}_original.mrcs".format(mrcsuffix)
if os.path.exists(mrcs):
os.remove(mrcs)
if os.path.exists(mrcs_orig):
os.remove(mrcs_orig)
r.ptcl_norm_sub.append_image(mrcs)
if options.original:
r.ptcl.append_image(mrcs_orig)
if logger.getEffectiveLevel(
) == logging.DEBUG: # Write additional debug output.
ptcl_sub_img = r.ptcl.process("math.sub.optimal", {
"ref": r.ctfproj,
"actual": r.ctfproj_sub,
"return_subim": True
})
ptcl_lowpass = r.ptcl.process("filter.lowpass.gauss", {
"apix": 1.22,
"cutoff_freq": 0.05
})
ptcl_sub_lowpass = r.ptcl_norm_sub.process("filter.lowpass.gauss",
{
"apix": 1.22,
"cutoff_freq": 0.05
})
ptcl_sub_img.write_image("poreclass_subimg.mrcs", -1)
ptcl_lowpass.write_image("poreclass_lowpass.mrcs", -1)
ptcl_sub_lowpass.write_image("poreclass_sublowpass.mrcs", -1)
r.ctfproj.write_image("poreclass_ctfproj.mrcs", -1)
r.ctfproj_sub.write_image("poreclass_ctfprojsub.mrcs", -1)
assert r.meta.i == i # Assert particle order is preserved.
star['rlnImageName'][i] = "{0:06d}@{1}".format(
i % options.maxpart + 1, starpath) # Set new image name.
r.meta.update(star) # Update StarFile with altered fields.
line = ' '.join(str(star[key][i]) for key in headings)
output_star.write("{0}\n".format(line))
i += 1
output_star.close()
if pool is not None:
pool.close()
pool.join()
return 0
def execute(self):
'''
The main function - executes the job of performing all v all boot strapped probe generation
'''
if self.logger: E2progress(self.logger,0.0)
# all_v_all_cmd = self.get_all_v_all_cmd()
# all_v_all_output = self.get_all_v_all_output()
#
# # NOTE: calling the allvall program is probably not strictly necessary, seeing
# # as there is a generic framework for generating and executing alignment jobs
# # implemented below that would be easily adaptable to this - however I left it
# # because doing it this way is absolutely equivalent and has the same cost.
# all_v_all_cmd += " --output="+all_v_all_output
# print "executing",all_v_all_cmd
# if self.logger: E2progress(self.logger,0.01)
# if ( launch_childprocess(all_v_all_cmd) != 0 ):
# print "Failed to execute %s" %all_v_all_cmd
# sys.exit(1)
# if self.logger: E2progress(self.logger,0.02)
#
# images = []
# images.append(EMData(all_v_all_output,0))
# images.append(EMData(all_v_all_output,1))
# images.append(EMData(all_v_all_output,2))
# images.append(EMData(all_v_all_output,3))
# images.append(EMData(all_v_all_output,4))
# images.append(EMData(all_v_all_output,5))
# images.append(EMData(all_v_all_output,6))
#
start_n = len(self.files) # the number of averages produced
images = []
e = EMData(start_n,start_n)
e.to_zero()
images.append(e)
for j in range(6): images.append(e.copy())
# keep tracks of the names of the new files
big_n = images[0].get_xsize()*(images[0].get_xsize()-1)/2.0
iter = 1
current_files = self.files
alignment_jobs = []# a list of comparisons to be performed
for i in range(len(current_files)):
for j in range(i+1,len(current_files)):
alignment_jobs.append([i,j])
self.register_current_images(images)
self.register_current_files(self.files)
alignments_manager = EMTomoAlignments(self.options)
alignments_manager.execute(alignment_jobs, self.files,self)
self.write_current_images(self.files)
# this loop
while True:
couples = self.get_couples(images[0])
taken = range(images[0].get_xsize())
done = False
if len(couples) == 1 and len(taken) == 2: done = True
#print len(couples),len(taken)
new_files = []
# write the averages of the couples to disk, store the new names
for i,j in couples:
image_1 = EMData(current_files[j],0)
image_2 = EMData(current_files[i],0)
image_1_weight = 1
if image_1.has_attr("total_inc"): image_1_weight = image_1["total_inc"]
image_2_weight = 1
if image_2.has_attr("total_inc"): image_2_weight = image_2["total_inc"]
total_weight = image_1_weight+image_2_weight
image_1.mult(float(image_1_weight)/total_weight)
image_2.mult(float(image_2_weight)/total_weight)
d = {}
d["type"] = "eman"
d["tx"] = images[1].get(i,j)
d["ty"] = images[2].get(i,j)
d["tz"] = images[3].get(i,j)
d["az"] = images[4].get(i,j)
d["alt"] = images[5].get(i,j)
d["phi"] = images[6].get(i,j)
t = Transform(d)
image_1.process_inplace("xform",{"transform":t})
image_2 += image_1
image_2.set_attr("src_image",current_files[j]) # so we can recollect how it was created
image_2.set_attr("added_src_image",current_files[i]) # so we can recollect how it was created
image_2.set_attr("added_src_transform",t) # so we can recollect how it was created
image_2.set_attr("added_src_cmp",images[0](i,j)) # so we can recollect how it was created
image_2.set_attr("total_inc",total_weight) # so we can recollect how it was created
output_name = numbered_bdb("bdb:"+self.options.path+"#tomo_ave_0"+str(iter-1))
image_2.write_image(output_name,0)
if self.options.dbls: self.save_to_workflow_db(output_name)
new_files.append(output_name)
taken.remove(i)
taken.remove(j)
if done: break
num_new = len(new_files) # the number of averages produced
new_n = len(new_files) + len(taken)
new_images = []
e = EMData(new_n,new_n)
e.to_zero()
new_images.append(e)
for j in range(6): new_images.append(e.copy())
for i,idxi in enumerate(taken):
new_files.append(current_files[idxi])
for j,idxj in enumerate(taken):
if i == j: continue
else:
new_images[0].set(num_new+i,num_new+j,images[0].get(idxi,idxj))
new_images[1].set(num_new+i,num_new+j,images[1].get(idxi,idxj))
new_images[2].set(num_new+i,num_new+j,images[2].get(idxi,idxj))
new_images[3].set(num_new+i,num_new+j,images[3].get(idxi,idxj))
new_images[4].set(num_new+i,num_new+j,images[4].get(idxi,idxj))
new_images[5].set(num_new+i,num_new+j,images[5].get(idxi,idxj))
new_images[6].set(num_new+i,num_new+j,images[6].get(idxi,idxj))
alignment_jobs = []# a list of comparisons to be performed
for i in range(num_new):
for j in range(i+1,len(new_files)):
alignment_jobs.append([i,j])
if self.logger:
E2progress(self.logger,1.0-len(alignment_jobs)/big_n)
self.register_current_images(new_images)
self.register_current_files(new_files)
alignments_manager = EMTomoAlignments(self.options)
alignments_manager.execute(alignment_jobs, new_files,self)
self.write_current_images(new_files)
current_files = new_files
images = new_images
iter += 1
print couples,taken
if self.logger: E2progress(self.logger,1.0)
if __name__ == '__main__':
FLAGS = gflags.FLAGS
gflags.DEFINE_string('infilename', '', 'input file')
gflags.DEFINE_string('outfilename', '', 'output file or suffix if multiple')
gflags.DEFINE_string('imgsuffix', '',
'image suffix to replace if multi-input')
gflags.DEFINE_boolean('bootstrap', True, 'run bootstrap error analysis')
argv = FLAGS(sys.argv)
# Support multiple input files
infile_list = glob.glob(FLAGS.infilename)
for infile in infile_list:
print 'Processing %s' % infile
if len(infile_list) == 1:
outfile = FLAGS.outfilename
else:
# kludge
outfile = infile.replace(FLAGS.imgsuffix, FLAGS.outfilename)
# read images
input_imgs = EMData.read_images(infile)
if FLAGS.bootstrap:
(yvals, err_lo, err_hi, xdata) = runFFT_err(input_imgs)
numpy.savetxt(outfile,
numpy.array([xdata, yvals, err_lo, err_hi]))
else:
(yvals, dx) = runFFT(input_imgs)
numpy.savetxt(outfile,
numpy.array([numpy.arange(0, len(yvals) * dx, dx),
yvals]))
def runFFT_err(images, apix=3.7, nresamples=1000, conf_int=95):
"""Run radially averaged FFT with bootstrapped error estimates.
Args:
images: images to operate on
apix: angstroms per pixel
nresmaples: number of bootstrap resamples
conf_int: confidence interval to calculate
Rets:
curve: average y-vals
conf_lo: low confidence bound
conf_hi: high confidence bound
sf_dx: x-vals
"""
bootidx = numpy.random.randint(len(images) - 1,
size=(nresamples, len(images)))
nx = images[0]["nx"]
ny = images[0]["ny"]
# compute FFT on everything
# for now also computing sample average
df = []
sampleavg = EMData(nx + 2, ny, 1)
sampleavg.to_zero()
sampleavg.set_complex(1)
for d in images:
d.process_inplace("mask.ringmean")
d.process_inplace("normalize")
curimg = d.do_fft()
curimg.mult(d.get_ysize())
sampleavg.add_incoherent(curimg)
df.append(curimg)
sampleavg.mult(1.0 / len(images))
# calculate average for each bootstrap resample
fftavg = EMData(nx + 2, ny, 1)
curve = []
for r in range(nresamples):
fftavg.to_zero()
fftavg.set_complex(1)
for idx in bootidx[r]:
fftavg.add_incoherent(df[idx])
fftavg.mult(1.0 / len(bootidx[r]))
curve.append(fftavg.calc_radial_dist(ny, 0, 0.5, 1))
cdata = numpy.vstack(curve)
xdata = numpy.array(range(cdata.shape[-1])) * 1.0 / (apix * 2.0 * ny)
return (sampleavg.calc_radial_dist(ny, 0, 0.5, 1),
numpy.percentile(cdata, 50 - conf_int/2, axis=0),
numpy.percentile(cdata, 50 + conf_int/2, axis=0),
xdata)
def mref2polar(refimgs,
firstring=1,
outerradius=-1,
ringstep=1,
mode="F",
normbysquare=0,
log=None,
verbose=False):
"""
Generates polar representations of a series of images to be used as alignment references.
Arguments:
refimgs : Input reference image stack (filename or EMData object)
firstring : Inner alignment radius
outerradius : Outer alignment radius
ringstep : Alignment radius step size
mode : Mode, full circle ("F") vs. half circle ("H)
normbysquare : If other than 0, normalization by setting the norm to 1
log : Logger object
verbose : (boolean) Whether to write additional information to screen
Returns:
alignringlist : List of alignment-ring data
polarreflist : List of polar representation of refernences
"""
# Read reference stack
if isinstance(refimgs, str):
referencelist = EMData.read_images(refimgs)
else:
referencelist = [refimgs] # For single image
numrefs = len(referencelist)
polarreflist = []
# Get image dimensions (assuming square, and that images and references have the same dimension)
#print('referencelist', type(refimgs), type(referencelist), type(referencelist[0]))
#exit()
idim = referencelist[0]['nx']
# Calculate image center
halfdim = idim / 2 + 1
if outerradius <= 0:
outerradius = halfdim - 3
#print('outerradius1', outerradius)
# Prepare alignment rings
alignringlist = Numrinit(firstring, outerradius, ringstep, mode)
# Calculate ring weights
ringweightlist = ringwe(alignringlist, mode)
print_log_msg(
'Converting %s references to polar coordinates from radius %s to %s with step %s and mode "%s"'
% (numrefs, firstring, outerradius, ringstep, mode), log, verbose)
# Loop through reference images (adapted from sxisac2)
for refindex in range(numrefs):
# Convert to polar
cimage = Util.Polar2Dm(referencelist[refindex], halfdim, halfdim,
alignringlist, mode)
# Fourier transform of rings
Util.Frngs(cimage, alignringlist)
# Apply weights to rings
Util.Applyws(cimage, alignringlist, ringweightlist)
# Normalize
Util.Normalize_ring(cimage, alignringlist, normbysquare)
#normbysquare: if other than 0, normalizes by setting the norm to 1
# Copy to reference stack
polarreflist.append(cimage.copy())
return alignringlist, polarreflist
def __new__(cls,filename,application,force_plot=False,force_2d=False,old=None):
file_type = Util.get_filename_ext(filename)
em_file_type = EMUtil.get_image_ext_type(file_type)
if not file_exists(filename): return None
if force_plot and force_2d:
# ok this sucks but it suffices for the time being
print "Error, the force_plot and force_2d options are mutually exclusive"
return None
if force_plot:
from emplot2d import EMPlot2DWidget
if isinstance(old,EMPlot2DWidget): widget = old
else: widget = EMPlot2DWidget(application=application)
widget.set_data_from_file(filename)
return widget
if em_file_type != IMAGE_UNKNOWN or filename[:4] == "bdb:":
n = EMUtil.get_image_count(filename)
nx,ny,nz = gimme_image_dimensions3D(filename)
if n > 1 and nz == 1:
if force_2d:
a = EMData()
data=a.read_images(filename)
else:
data = None # This is like a flag - the ImageMXWidget only needs the file name
elif nz == 1:
data = [EMData(filename,0)]
else:
data = EMData()
data.read_image(filename,0,not force_2d) # This should be 3-D. We read the header-only here
data = [data]
if data != None and len(data) == 1: data = data[0]
if force_2d or isinstance(data,EMData) and data.get_zsize()==1:
if isinstance(data,list) or data.get_ysize() != 1:
from emimage2d import EMImage2DWidget
if isinstance(old,EMImage2DWidget): widget = old
else: widget= EMImage2DWidget(application=application)
else:
from emplot2d import EMPlot2DWidget
if isinstance(old,EMPlot2DWidget): widget = old
else: widget = EMPlot2DWidget(application=application)
widget.set_data_from_file(filename)
return widget
elif isinstance(data,EMData):
if isinstance(old,EMScene3D): widget = old
else: widget = EMScene3D()
# print n,data
for ii in xrange(n):
data=EMData(filename,ii)
datai = EMDataItem3D(data, transform=Transform())
widget.insertNewNode(os.path.basename(filename), datai, parentnode=widget)
isosurface = EMIsosurface(datai, transform=Transform())
widget.insertNewNode("Iso", isosurface, parentnode=datai)
return widget
elif data == None or isinstance(data,list):
from emimagemx import EMImageMXWidget
if isinstance(old,EMImageMXWidget): widget = old
else: widget = EMImageMXWidget(application=application)
data = filename
else:
print filename
raise # weirdness, this should never happen
widget.set_data(data,filename)
return widget
else:
from emplot2d import EMPlot2DWidget
if isinstance(old,EMPlot2DWidget): widget = old
else: widget = EMPlot2DWidget(application=application)
widget.set_data_from_file(filename)
return widget
def getmxinfo(fsp, fsp2, clsnum):
"""reads particle references associated with a particular class.
fsp2 is the matrix file and fsp is the raw particle file"""
mx = EMData(fsp2, 0)
imgs = [(fsp, i) for i in range(mx.get_ysize()) if mx.get(0, i) == clsnum]
return imgs
def __new__(cls,filename,application,force_plot=False,force_2d=False,old=None):
file_type = Util.get_filename_ext(filename)
em_file_type = EMUtil.get_image_ext_type(file_type)
if not file_exists(filename): return None
if force_plot and force_2d:
# ok this sucks but it suffices for the time being
print("Error, the force_plot and force_2d options are mutually exclusive")
return None
if force_plot:
from .emplot2d import EMPlot2DWidget
if isinstance(old,EMPlot2DWidget): widget = old
else: widget = EMPlot2DWidget(application=application)
widget.set_data_from_file(filename)
return widget
if em_file_type != IMAGE_UNKNOWN or filename[:4] == "bdb:":
n = EMUtil.get_image_count(filename)
nx,ny,nz = gimme_image_dimensions3D(filename)
if n > 1 and nz == 1:
if force_2d:
a = EMData()
data=a.read_images(filename)
else:
data = None # This is like a flag - the ImageMXWidget only needs the file name
elif nz == 1:
data = [EMData(filename,0)]
else:
data = EMData()
data.read_image(filename,0,not force_2d) # This should be 3-D. We read the header-only here
data = [data]
if data != None and len(data) == 1: data = data[0]
if force_2d or isinstance(data,EMData) and data.get_zsize()==1:
if isinstance(data,list) or data.get_ysize() != 1:
from .emimage2d import EMImage2DWidget
if isinstance(old,EMImage2DWidget): widget = old
else: widget= EMImage2DWidget(application=application)
else:
from .emplot2d import EMPlot2DWidget
if isinstance(old,EMPlot2DWidget): widget = old
else: widget = EMPlot2DWidget(application=application)
widget.set_data_from_file(filename)
return widget
elif isinstance(data,EMData):
if isinstance(old,EMScene3D): widget = old
else: widget = EMScene3D()
# print n,data
for ii in range(n):
data=EMData(filename,ii)
datai = EMDataItem3D(data, transform=Transform())
widget.insertNewNode(os.path.basename(filename), datai, parentnode=widget)
isosurface = EMIsosurface(datai, transform=Transform())
widget.insertNewNode("Iso", isosurface, parentnode=datai)
return widget
elif data == None or isinstance(data,list):
from .emimagemx import EMImageMXWidget
if isinstance(old,EMImageMXWidget): widget = old
else: widget = EMImageMXWidget(application=application)
data = filename
else:
print(filename)
raise # weirdness, this should never happen
widget.set_data(data,filename)
return widget
else:
from .emplot2d import EMPlot2DWidget
if isinstance(old,EMPlot2DWidget): widget = old
else: widget = EMPlot2DWidget(application=application)
widget.set_data_from_file(filename)
return widget
def execute(self, alignment_jobs, files, caller):
'''
The main function
@param alignment_jobs a list of alignment pair indices like this [[0,1],[2,1],[2,3],[0,5],...] etc the indices pair represent images to be aligned and correspond to the order of the files argument
@param files a list of filenames - used to read image based on the indices present in alignment_jobs
@param caller - the calling object - it needs to have a function called process_output that takes a dictionary as the argument
'''
options = self.options
align_data = EMAN2.parsemodopt(options.align)
align_cmp_data = EMAN2.parsemodopt(options.aligncmp)
cmp_data = EMAN2.parsemodopt(options.cmp)
ralign_data = None
if options.ralign != None:
ralign_data = EMAN2.parsemodopt(options.ralign)
ralign_cmp_data = EMAN2.parsemodopt(options.raligncmp)
data = {}
data["align"] = align_data
data["aligncmp"] = align_cmp_data
data["cmp"] = cmp_data
if ralign_data:
data["ralign"] = ralign_data
data["raligncmp"] = ralign_cmp_data
data["using_cuda"] = self.using_cuda
data["nsoln"] = self.nsoln
if self.options.parallel:
task_customers = []
tids = []
if options.shrink:
scratch_name_1 = numbered_bdb(
"bdb:tomo_scratch#scratch_shrink")
scratch_name_2 = numbered_bdb(
"bdb:tomo_scratch##scratch_shrink")
else:
print("no shrink")
for i, j in alignment_jobs:
if options.shrink or options.filter:
a = EMData(files[i], 0)
if options.filter:
filter_params = EMAN2.parsemodopt(options.filter)
a.process_inplace(filter_params[0], filter_params[1])
if options.shrink:
a.process_inplace("math.meanshrink",
{"n": options.shrink})
a.set_attr("src_image", files[i])
a.write_image(scratch_name_1, 0)
a = EMData(files[j], 0)
if options.filter:
filter_params = EMAN2.parsemodopt(options.filter)
a.process_inplace(filter_params[0], filter_params[1])
if options.shrink:
a.process_inplace("math.meanshrink",
{"n": options.shrink})
a.set_attr("src_image", files[j])
a.write_image(scratch_name_2, 0)
data["probe"] = ("cache", scratch_name_1, 0)
data["target"] = ("cache", scratch_name_2, 0)
else:
data["probe"] = ("cache", files[i], 0)
data["target"] = ("cache", files[j], 0)
data["target_idx"] = j
data["probe_idx"] = i
task = EMTomoAlignTaskDC(data=data)
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(self.options.parallel)
#print "Est %d CPUs"%etc.cpu_est()
tid = etc.send_task(task)
#print "Task submitted tid=",tid
task_customers.append(etc)
tids.append(tid)
self.dc_monitor(task_customers, tids, caller)
else:
n = len(alignment_jobs)
p = 0.0
for i, j in alignment_jobs:
probe = EMData(files[i], 0)
target = EMData(files[j], 0)
if options.filter:
print("filtered")
filter_params = EMAN2.parsemodopt(options.filter)
probe.process_inplace(filter_params[0], filter_params[1])
target.process_inplace(filter_params[0], filter_params[1])
if options.shrink:
probe.process_inplace("math.meanshrink",
{"n": options.shrink})
target.process_inplace("math.meanshrink",
{"n": options.shrink})
else:
print("no shrink")
data["target"] = target
data["probe"] = probe
data["target_idx"] = j
data["probe_idx"] = i
task = EMTomoAlignTask(data=data)
rslts = task.execute(self.progress_callback)
if options.shrink:
self.correction_translation(rslts, options.shrink)
caller.process_output(rslts)
p += 1.0
def execute(self):
'''
The main function - executes the job of performing all v all boot strapped probe generation
'''
if self.logger: E2progress(self.logger, 0.0)
# all_v_all_cmd = self.get_all_v_all_cmd()
# all_v_all_output = self.get_all_v_all_output()
#
# # NOTE: calling the allvall program is probably not strictly necessary, seeing
# # as there is a generic framework for generating and executing alignment jobs
# # implemented below that would be easily adaptable to this - however I left it
# # because doing it this way is absolutely equivalent and has the same cost.
# all_v_all_cmd += " --output="+all_v_all_output
# print "executing",all_v_all_cmd
# if self.logger: E2progress(self.logger,0.01)
# if ( launch_childprocess(all_v_all_cmd) != 0 ):
# print "Failed to execute %s" %all_v_all_cmd
# sys.exit(1)
# if self.logger: E2progress(self.logger,0.02)
#
# images = []
# images.append(EMData(all_v_all_output,0))
# images.append(EMData(all_v_all_output,1))
# images.append(EMData(all_v_all_output,2))
# images.append(EMData(all_v_all_output,3))
# images.append(EMData(all_v_all_output,4))
# images.append(EMData(all_v_all_output,5))
# images.append(EMData(all_v_all_output,6))
#
start_n = len(self.files) # the number of averages produced
images = []
e = EMData(start_n, start_n)
e.to_zero()
images.append(e)
for j in range(6):
images.append(e.copy())
# keep tracks of the names of the new files
big_n = images[0].get_xsize() * (images[0].get_xsize() - 1) / 2.0
iter = 1
current_files = self.files
alignment_jobs = [] # a list of comparisons to be performed
for i in range(len(current_files)):
for j in range(i + 1, len(current_files)):
alignment_jobs.append([i, j])
self.register_current_images(images)
self.register_current_files(self.files)
alignments_manager = EMTomoAlignments(self.options)
alignments_manager.execute(alignment_jobs, self.files, self)
self.write_current_images(self.files)
# this loop
while True:
couples = self.get_couples(images[0])
taken = list(range(images[0].get_xsize()))
done = False
if len(couples) == 1 and len(taken) == 2: done = True
#print len(couples),len(taken)
new_files = []
# write the averages of the couples to disk, store the new names
for i, j in couples:
image_1 = EMData(current_files[j], 0)
image_2 = EMData(current_files[i], 0)
image_1_weight = 1
if image_1.has_attr("total_inc"):
image_1_weight = image_1["total_inc"]
image_2_weight = 1
if image_2.has_attr("total_inc"):
image_2_weight = image_2["total_inc"]
total_weight = image_1_weight + image_2_weight
image_1.mult(old_div(float(image_1_weight), total_weight))
image_2.mult(old_div(float(image_2_weight), total_weight))
d = {}
d["type"] = "eman"
d["tx"] = images[1].get(i, j)
d["ty"] = images[2].get(i, j)
d["tz"] = images[3].get(i, j)
d["az"] = images[4].get(i, j)
d["alt"] = images[5].get(i, j)
d["phi"] = images[6].get(i, j)
t = Transform(d)
image_1.process_inplace("xform", {"transform": t})
image_2 += image_1
image_2.set_attr(
"src_image",
current_files[j]) # so we can recollect how it was created
image_2.set_attr(
"added_src_image",
current_files[i]) # so we can recollect how it was created
image_2.set_attr("added_src_transform",
t) # so we can recollect how it was created
image_2.set_attr("added_src_cmp", images[0](
i, j)) # so we can recollect how it was created
image_2.set_attr(
"total_inc",
total_weight) # so we can recollect how it was created
output_name = numbered_bdb("bdb:" + self.options.path +
"#tomo_ave_0" + str(iter - 1))
image_2.write_image(output_name, 0)
if self.options.dbls: self.save_to_workflow_db(output_name)
new_files.append(output_name)
taken.remove(i)
taken.remove(j)
if done: break
num_new = len(new_files) # the number of averages produced
new_n = len(new_files) + len(taken)
new_images = []
e = EMData(new_n, new_n)
e.to_zero()
new_images.append(e)
for j in range(6):
new_images.append(e.copy())
for i, idxi in enumerate(taken):
new_files.append(current_files[idxi])
for j, idxj in enumerate(taken):
if i == j: continue
else:
new_images[0].set(num_new + i, num_new + j,
images[0].get(idxi, idxj))
new_images[1].set(num_new + i, num_new + j,
images[1].get(idxi, idxj))
new_images[2].set(num_new + i, num_new + j,
images[2].get(idxi, idxj))
new_images[3].set(num_new + i, num_new + j,
images[3].get(idxi, idxj))
new_images[4].set(num_new + i, num_new + j,
images[4].get(idxi, idxj))
new_images[5].set(num_new + i, num_new + j,
images[5].get(idxi, idxj))
new_images[6].set(num_new + i, num_new + j,
images[6].get(idxi, idxj))
alignment_jobs = [] # a list of comparisons to be performed
for i in range(num_new):
for j in range(i + 1, len(new_files)):
alignment_jobs.append([i, j])
if self.logger:
E2progress(self.logger,
1.0 - old_div(len(alignment_jobs), big_n))
self.register_current_images(new_images)
self.register_current_files(new_files)
alignments_manager = EMTomoAlignments(self.options)
alignments_manager.execute(alignment_jobs, new_files, self)
self.write_current_images(new_files)
current_files = new_files
images = new_images
iter += 1
print(couples, taken)
if self.logger: E2progress(self.logger, 1.0)
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options] <image file> ...
This program can be used to visualize most files used in EMAN2. Running it without arguments
will open a browser window with more flexible functionality than the command-line.
"""
global app, win, options
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--classmx",
type=str,
help=
"<classmx>,<#> Show particles in one class from a classification matrix. Pass raw particle file as first argument to command."
)
parser.add_argument(
"--classes",
type=str,
help="<rawptcl>,<classmx> Show particles associated class-averages")
parser.add_argument("--pdb",
type=str,
help="<pdb file> Show PDB structure.")
parser.add_argument("--singleimage",
action="store_true",
default=False,
help="Display a stack in a single image view")
parser.add_argument(
"--plot",
action="store_true",
default=False,
help="Data file(s) should be plotted rather than displayed in 2-D")
parser.add_argument(
"--hist",
action="store_true",
default=False,
help=
"Data file(s) should be plotted as a histogram rather than displayed in 2-D."
)
parser.add_argument(
"--plot3d",
action="store_true",
default=False,
help="Data file(s) should be plotted rather than displayed in 3-D")
parser.add_argument(
"--fullrange",
action="store_true",
default=False,
help=
"A specialized flag that disables auto contrast for the display of particles stacks and 2D images only."
)
parser.add_argument("--newwidget",
action="store_true",
default=False,
help="Use the new 3D widgetD. Highly recommended!!!!")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-2)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
(options, args) = parser.parse_args()
# logid=E2init(sys.argv)
app = EMApp()
#gapp = app
#QtWidgets.QApplication(sys.argv)
win = []
if options.fullrange:
print(
"""The --fullrange option has been removed, and replaced with an option in user preferences.
To set your preferences for full-range 2-D display, please run:
e2procjson.py --setoption display2d.autocontrast:true
""")
sys.exit(0)
if len(args) < 1:
global dialog
file_list = []
dialog = embrowser.EMBrowserWidget(withmodal=False, multiselect=False)
dialog.show()
try:
dialog.raise_()
# QtCore.QObject.connect(dialog,QtCore.SIGNAL("ok"),on_browser_done)
# QtCore.QObject.connect(dialog,QtCore.SIGNAL("cancel"),on_browser_cancel)
except:
pass
elif options.pdb:
load_pdb(args, app)
elif options.plot:
plot(args, app)
elif options.hist:
hist(args, app)
elif options.plot3d:
plot_3d(args, app)
elif options.classes:
options.classes = options.classes.split(",")
imgs = EMData.read_images(args[0])
display(imgs, app, args[0])
win[0].child.mousedown.connect(lambda a, b: selectclass(
options.classes[0], options.classes[1], a, b))
try:
out = open("selected.lst", "w")
out.write("#LST\n")
out.close()
except:
pass
elif options.classmx:
options.classmx = options.classmx.split(",")
clsnum = int(options.classmx[1])
imgs = getmxim(args[0], options.classmx[0], clsnum)
display(imgs, app, args[0])
else:
for i in args:
if not file_exists(i):
print("%s doesn't exist" % i)
sys.exit(1)
display_file(i,
app,
options.singleimage,
usescenegraph=options.newwidget)
app.exec_()
def getmxim(fsp, fsp2, clsnum):
"""reads the raw particles associated with a particular class.
fsp2 is the matrix file and fsp is the raw particle file"""
mx = EMData(fsp2, 0)
dx = EMData(fsp2, 2)
dy = EMData(fsp2, 3)
da = EMData(fsp2, 4)
imgs = [(EMData(fsp, i), dx.get(0, i), dy.get(0, i), da.get(0, i))
for i in range(mx.get_ysize()) if mx.get(0, i) == clsnum]
for i in imgs:
print(i)
i[0].rotate_translate(i[3], 0, 0, i[1], i[2], 0)
imgs = [i[0] for i in imgs]
return imgs
def main(args):
from utilities import if_error_then_all_processes_exit_program, write_text_row, drop_image, model_gauss_noise, get_im, set_params_proj, wrap_mpi_bcast, model_circle
from logger import Logger, BaseLogger_Files
from mpi import mpi_init, mpi_finalize, MPI_COMM_WORLD, mpi_comm_rank, mpi_comm_size, mpi_barrier
import user_functions
import sys
import os
from applications import MPI_start_end
from optparse import OptionParser, SUPPRESS_HELP
from global_def import SPARXVERSION
from EMAN2 import EMData
from multi_shc import multi_shc
progname = os.path.basename(sys.argv[0])
usage = progname + " stack [output_directory] --ir=inner_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translational_search_step --delta=angular_step --center=center_type --maxit1=max_iter1 --maxit2=max_iter2 --L2threshold=0.1 --ref_a=S --sym=c1"
usage += """
stack 2D images in a stack file: (default required string)
directory output directory name: into which the results will be written (if it does not exist, it will be created, if it does exist, the results will be written possibly overwriting previous results) (default required string)
"""
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--radius", type="int", help="radius of the particle: has to be less than < int(nx/2)-1 (default required int)")
parser.add_option("--xr", type="string", default='0', help="range for translation search in x direction: search is +/xr in pixels (default '0')")
parser.add_option("--yr", type="string", default='0', help="range for translation search in y direction: if omitted will be set to xr, search is +/yr in pixels (default '0')")
parser.add_option("--mask3D", type="string", default=None, help="3D mask file: (default sphere)")
parser.add_option("--moon_elimination", type="string", default='', help="elimination of disconnected pieces: two arguments: mass in KDa and pixel size in px/A separated by comma, no space (default none)")
parser.add_option("--ir", type="int", default=1, help="inner radius for rotational search: > 0 (default 1)")
# 'radius' and 'ou' are the same as per Pawel's request; 'ou' is hidden from the user
# the 'ou' variable is not changed to 'radius' in the 'sparx' program. This change is at interface level only for sxviper.
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--ou", type="int", default=-1, help=SUPPRESS_HELP)
parser.add_option("--rs", type="int", default=1, help="step between rings in rotational search: >0 (default 1)")
parser.add_option("--ts", type="string", default='1.0', help="step size of the translation search in x-y directions: search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional (default '1.0')")
parser.add_option("--delta", type="string", default='2.0', help="angular step of reference projections: (default '2.0')")
parser.add_option("--center", type="float", default=-1.0, help="centering of 3D template: average shift method; 0: no centering; 1: center of gravity (default -1.0)")
parser.add_option("--maxit1", type="int", default=400, help="maximum number of iterations performed for the GA part: (default 400)")
parser.add_option("--maxit2", type="int", default=50, help="maximum number of iterations performed for the finishing up part: (default 50)")
parser.add_option("--L2threshold", type="float", default=0.03, help="stopping criterion of GA: given as a maximum relative dispersion of volumes' L2 norms: (default 0.03)")
parser.add_option("--ref_a", type="string", default='S', help="method for generating the quasi-uniformly distributed projection directions: (default S)")
parser.add_option("--sym", type="string", default='c1', help="point-group symmetry of the structure: (default c1)")
# parser.add_option("--function", type="string", default="ref_ali3d", help="name of the reference preparation function (ref_ali3d by default)")
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--function", type="string", default="ref_ali3d", help= SUPPRESS_HELP)
parser.add_option("--nruns", type="int", default=6, help="GA population: aka number of quasi-independent volumes (default 6)")
parser.add_option("--doga", type="float", default=0.1, help="do GA when fraction of orientation changes less than 1.0 degrees is at least doga: (default 0.1)")
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--npad", type="int", default= 2, help="padding size for 3D reconstruction (default=2)")
parser.add_option("--fl", type="float", default=0.25, help="cut-off frequency applied to the template volume: using a hyperbolic tangent low-pass filter (default 0.25)")
parser.add_option("--aa", type="float", default=0.1, help="fall-off of hyperbolic tangent low-pass filter: (default 0.1)")
parser.add_option("--pwreference", type="string", default='', help="text file with a reference power spectrum: (default none)")
parser.add_option("--debug", action="store_true", default=False, help="debug info printout: (default False)")
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help = SUPPRESS_HELP)
#parser.add_option("--an", type="string", default= "-1", help="NOT USED angular neighborhood for local searches (phi and theta)")
#parser.add_option("--CTF", action="store_true", default=False, help="NOT USED Consider CTF correction during the alignment ")
#parser.add_option("--snr", type="float", default= 1.0, help="NOT USED Signal-to-Noise Ratio of the data (default 1.0)")
# (options, args) = parser.parse_args(sys.argv[1:])
required_option_list = ['radius']
(options, args) = parser.parse_args(args)
# option_dict = vars(options)
# print parser
if options.return_options:
return parser
if options.moon_elimination == "":
options.moon_elimination = []
else:
options.moon_elimination = map(float, options.moon_elimination.split(","))
# Making sure all required options appeared.
for required_option in required_option_list:
if not options.__dict__[required_option]:
print "\n ==%s== mandatory option is missing.\n"%required_option
print "Please run '" + progname + " -h' for detailed options"
return 1
if len(args) < 2 or len(args) > 3:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
return 1
mpi_init(0, [])
log = Logger(BaseLogger_Files())
# 'radius' and 'ou' are the same as per Pawel's request; 'ou' is hidden from the user
# the 'ou' variable is not changed to 'radius' in the 'sparx' program. This change is at interface level only for sxviper.
options.ou = options.radius
runs_count = options.nruns
mpi_rank = mpi_comm_rank(MPI_COMM_WORLD)
mpi_size = mpi_comm_size(MPI_COMM_WORLD) # Total number of processes, passed by --np option.
if mpi_rank == 0:
all_projs = EMData.read_images(args[0])
subset = range(len(all_projs))
# if mpi_size > len(all_projs):
# ERROR('Number of processes supplied by --np needs to be less than or equal to %d (total number of images) ' % len(all_projs), 'sxviper', 1)
# mpi_finalize()
# return
else:
all_projs = None
subset = None
outdir = args[1]
if mpi_rank == 0:
if mpi_size % options.nruns != 0:
ERROR('Number of processes needs to be a multiple of total number of runs. Total runs by default are 3, you can change it by specifying --nruns option.', 'sxviper', 1)
mpi_finalize()
return
if os.path.exists(outdir):
ERROR('Output directory exists, please change the name and restart the program', "sxviper", 1)
mpi_finalize()
return
os.mkdir(outdir)
import global_def
global_def.LOGFILE = os.path.join(outdir, global_def.LOGFILE)
mpi_barrier(MPI_COMM_WORLD)
if outdir[-1] != "/":
outdir += "/"
log.prefix = outdir
# if len(args) > 2:
# ref_vol = get_im(args[2])
# else:
ref_vol = None
options.user_func = user_functions.factory[options.function]
options.CTF = False
options.snr = 1.0
options.an = -1.0
from multi_shc import multi_shc
out_params, out_vol, out_peaks = multi_shc(all_projs, subset, runs_count, options, mpi_comm=MPI_COMM_WORLD, log=log, ref_vol=ref_vol)
mpi_finalize()
def get_attr_dict(self):
a = EMData()
a.read_image(self.file_name, self.idx, True)
return a.get_attr_dict()
def internal_test_image(self, nx, ny=1, nz=1):
from EMAN2 import EMData
e = EMData()
e.set_size(nx, ny, nz)
e.process_inplace("testimage.tomo.objects")
return e
def write_image(self,*args): a = EMData() a.read_image(self.file_name,self.idx) a.write_image(*args)
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + """ [options] <inputfile> <outputfile>
Forms chains of 2D images based on their similarities.
Functionality:
Order a 2-D stack of image based on pair-wise similarity (computed as a cross-correlation coefficent).
Options 1-3 require image stack to be aligned. The program will apply orientation parameters if present in headers.
The ways to use the program:
1. Use option initial to specify which image will be used as an initial seed to form the chain.
sp_chains.py input_stack.hdf output_stack.hdf --initial=23 --radius=25
2. If options initial is omitted, the program will determine which image best serves as initial seed to form the chain
sp_chains.py input_stack.hdf output_stack.hdf --radius=25
3. Use option circular to form a circular chain.
sp_chains.py input_stack.hdf output_stack.hdf --circular--radius=25
4. New circular code based on pairwise alignments
sp_chains.py aclf.hdf chain.hdf circle.hdf --align --radius=25 --xr=2 --pairwiseccc=lcc.txt
5. Circular ordering based on pairwise alignments
sp_chains.py vols.hdf chain.hdf mask.hdf --dd --radius=25
"""
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option(
"--dd",
action="store_true",
help="Circular ordering without adjustment of orientations",
default=False)
parser.add_option(
"--circular",
action="store_true",
help=
"Select circular ordering (first image has to be similar to the last)",
default=False)
parser.add_option(
"--align",
action="store_true",
help=
"Compute all pairwise alignments and from the table of image similarities find the best chain",
default=False)
parser.add_option(
"--initial",
type="int",
default=-1,
help=
"Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)"
)
parser.add_option(
"--radius",
type="int",
default=-1,
help="Radius of a circular mask for similarity based ordering")
# import params for 2D alignment
parser.add_option(
"--ou",
type="int",
default=-1,
help=
"outer radius for 2D alignment < nx/2-1 (set to the radius of the particle)"
)
parser.add_option(
"--xr",
type="int",
default=0,
help="range for translation search in x direction, search is +/xr (0)")
parser.add_option(
"--yr",
type="int",
default=0,
help="range for translation search in y direction, search is +/yr (0)")
# parser.add_option("--nomirror", action="store_true", default=False, help="Disable checking mirror orientations of images (default False)")
parser.add_option("--pairwiseccc",
type="string",
default=" ",
help="Input/output pairwise ccc file")
(options, args) = parser.parse_args()
sp_global_def.BATCH = True
if options.dd:
nargs = len(args)
if nargs != 3:
ERROR("Must provide name of input and two output files!")
return
stack = args[0]
new_stack = args[1]
from sp_utilities import model_circle
from sp_statistics import ccc
from sp_statistics import mono
lend = EMUtil.get_image_count(stack)
lccc = [None] * (old_div(lend * (lend - 1), 2))
for i in range(lend - 1):
v1 = get_im(stack, i)
if (i == 0 and nargs == 2):
nx = v1.get_xsize()
ny = v1.get_ysize()
nz = v1.get_ysize()
if options.ou < 1:
radius = old_div(nx, 2) - 2
else:
radius = options.ou
mask = model_circle(radius, nx, ny, nz)
else:
mask = get_im(args[2])
for j in range(i + 1, lend):
lccc[mono(i, j)] = [ccc(v1, get_im(stack, j), mask), 0]
order = tsp(lccc)
if (len(order) != lend):
ERROR("Problem with data length")
return
sxprint("Total sum of cccs :", TotalDistance(order, lccc))
sxprint("ordering :", order)
for i in range(lend):
get_im(stack, order[i]).write_image(new_stack, i)
elif options.align:
nargs = len(args)
if nargs != 3:
ERROR("Must provide name of input and two output files!")
return
from sp_utilities import get_params2D, model_circle
from sp_fundamentals import rot_shift2D
from sp_statistics import ccc
from time import time
from sp_alignment import align2d, align2d_scf
stack = args[0]
new_stack = args[1]
d = EMData.read_images(stack)
if (len(d) < 6):
ERROR(
"Chains requires at least six images in the input stack to be executed"
)
return
"""
# will align anyway
try:
ttt = d[0].get_attr('xform.params2d')
for i in xrange(len(d)):
alpha, sx, sy, mirror, scale = get_params2D(d[i])
d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror)
except:
pass
"""
nx = d[0].get_xsize()
ny = d[0].get_ysize()
if options.ou < 1:
radius = old_div(nx, 2) - 2
else:
radius = options.ou
mask = model_circle(radius, nx, ny)
if (options.xr < 0):
xrng = 0
else:
xrng = options.xr
if (options.yr < 0):
yrng = xrng
else:
yrng = options.yr
initial = max(options.initial, 0)
from sp_statistics import mono
lend = len(d)
lccc = [None] * (old_div(lend * (lend - 1), 2))
from sp_utilities import read_text_row
if options.pairwiseccc == " " or not os.path.exists(
options.pairwiseccc):
st = time()
for i in range(lend - 1):
for j in range(i + 1, lend):
# j>i meaning mono entry (i,j) or (j,i) indicates T i->j (from smaller index to larger)
# alpha, sx, sy, mir, peak = align2d(d[i],d[j], xrng, yrng, step=options.ts, first_ring=options.ir, last_ring=radius, mode = "F")
alpha, sx, sy, mir, peak = align2d_scf(d[i],
d[j],
xrng,
yrng,
ou=radius)
lccc[mono(i, j)] = [
ccc(d[j], rot_shift2D(d[i], alpha, sx, sy, mir, 1.0),
mask), alpha, sx, sy, mir
]
# print " %4d %10.1f"%(i,time()-st)
if ((not os.path.exists(options.pairwiseccc))
and (options.pairwiseccc != " ")):
write_text_row([[initial, 0, 0, 0, 0]] + lccc,
options.pairwiseccc)
elif (os.path.exists(options.pairwiseccc)):
lccc = read_text_row(options.pairwiseccc)
initial = int(lccc[0][0] + 0.1)
del lccc[0]
for i in range(len(lccc)):
T = Transform({
"type": "2D",
"alpha": lccc[i][1],
"tx": lccc[i][2],
"ty": lccc[i][3],
"mirror": int(lccc[i][4] + 0.1)
})
lccc[i] = [lccc[i][0], T]
tdummy = Transform({"type": "2D"})
maxsum = -1.023
for m in range(0, lend): # initial, initial+1):
indc = list(range(lend))
lsnake = [[m, tdummy, 0.0]]
del indc[m]
lsum = 0.0
while len(indc) > 1:
maxcit = -111.
for i in range(len(indc)):
cuc = lccc[mono(indc[i], lsnake[-1][0])][0]
if cuc > maxcit:
maxcit = cuc
qi = indc[i]
# Here we need transformation from the current to the previous,
# meaning indc[i] -> lsnake[-1][0]
T = lccc[mono(indc[i], lsnake[-1][0])][1]
# If direction is from larger to smaller index, the transformation has to be inverted
if (indc[i] > lsnake[-1][0]): T = T.inverse()
lsnake.append([qi, T, maxcit])
lsum += maxcit
del indc[indc.index(qi)]
T = lccc[mono(indc[-1], lsnake[-1][0])][1]
if (indc[-1] > lsnake[-1][0]): T = T.inverse()
lsnake.append(
[indc[-1], T, lccc[mono(indc[-1], lsnake[-1][0])][0]])
sxprint(" initial image and lsum ", m, lsum)
# print lsnake
if (lsum > maxsum):
maxsum = lsum
init = m
snake = [lsnake[i] for i in range(lend)]
sxprint(" Initial image selected : ", init, maxsum, " ",
TotalDistance([snake[m][0] for m in range(lend)], lccc))
# for q in snake: print q
from copy import deepcopy
trans = deepcopy([snake[i][1] for i in range(len(snake))])
sxprint([snake[i][0] for i in range(len(snake))])
"""
for m in xrange(lend):
prms = trans[m].get_params("2D")
print " %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2])
"""
for k in range(lend - 2, 0, -1):
T = snake[k][1]
for i in range(k + 1, lend):
trans[i] = T * trans[i]
# To add - apply all transformations and do the overall centering.
for m in range(lend):
prms = trans[m].get_params("2D")
# print(" %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2]) )
# rot_shift2D(d[snake[m][0]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image(new_stack, m)
rot_shift2D(d[snake[m][0]], prms["alpha"], 0.0, 0.0,
prms["mirror"]).write_image(new_stack, m)
order = tsp(lccc)
if (len(order) != lend):
ERROR("Problem with data length")
return
sxprint(TotalDistance(order, lccc))
sxprint(order)
ibeg = order.index(init)
order = [order[(i + ibeg) % lend] for i in range(lend)]
sxprint(TotalDistance(order, lccc))
sxprint(order)
snake = [tdummy]
for i in range(1, lend):
# Here we need transformation from the current to the previous,
# meaning order[i] -> order[i-1]]
T = lccc[mono(order[i], order[i - 1])][1]
# If direction is from larger to smaller index, the transformation has to be inverted
if (order[i] > order[i - 1]): T = T.inverse()
snake.append(T)
assert (len(snake) == lend)
from copy import deepcopy
trans = deepcopy(snake)
for k in range(lend - 2, 0, -1):
T = snake[k]
for i in range(k + 1, lend):
trans[i] = T * trans[i]
# Try to smooth the angles - complicated, I am afraid one would have to use angles forward and backwards
# and find their average??
# In addition, one would have to recenter them
"""
trms = []
for m in xrange(lend):
prms = trans[m].get_params("2D")
trms.append([prms["alpha"], prms["mirror"]])
for i in xrange(3):
for m in xrange(lend):
mb = (m-1)%lend
me = (m+1)%lend
# angles order mb,m,me
# calculate predicted angles mb->m
"""
best_params = []
for m in range(lend):
prms = trans[m].get_params("2D")
# rot_shift2D(d[order[m]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image("metro.hdf", m)
rot_shift2D(d[order[m]], prms["alpha"], 0.0, 0.0,
prms["mirror"]).write_image(args[2], m)
best_params.append(
[m, order[m], prms["alpha"], 0.0, 0.0, prms["mirror"]])
# Write alignment parameters
outdir = os.path.dirname(args[2])
aligndoc = os.path.join(outdir, "chains_params.txt")
write_text_row(best_params, aligndoc)
"""
# This was an effort to get number of loops, inconclusive, to say the least
from numpy import outer, zeros, float32, sqrt
lend = len(d)
cor = zeros(lend,float32)
cor = outer(cor, cor)
for i in xrange(lend): cor[i][i] = 1.0
for i in xrange(lend-1):
for j in xrange(i+1, lend):
cor[i,j] = lccc[mono(i,j)][0]
cor[j,i] = cor[i,j]
lmbd, eigvec = pca(cor)
from sp_utilities import write_text_file
nvec=20
print [lmbd[j] for j in xrange(nvec)]
print " G"
mm = [-1]*lend
for i in xrange(lend): # row
mi = -1.0e23
for j in xrange(nvec):
qt = eigvec[j][i]
if(abs(qt)>mi):
mi = abs(qt)
mm[i] = j
for j in xrange(nvec):
qt = eigvec[j][i]
print round(qt,3), # eigenvector
print mm[i]
print
for j in xrange(nvec):
qt = []
for i in xrange(lend):
if(mm[i] == j): qt.append(i)
if(len(qt)>0): write_text_file(qt,"loop%02d.txt"%j)
"""
"""
print [lmbd[j] for j in xrange(nvec)]
print " B"
mm = [-1]*lend
for i in xrange(lend): # row
mi = -1.0e23
for j in xrange(nvec):
qt = eigvec[j][i]/sqrt(lmbd[j])
if(abs(qt)>mi):
mi = abs(qt)
mm[i] = j
for j in xrange(nvec):
qt = eigvec[j][i]/sqrt(lmbd[j])
print round(qt,3), # eigenvector
print mm[i]
print
"""
"""
lend=3
cor = zeros(lend,float32)
cor = outer(cor, cor)
cor[0][0] =136.77
cor[0][1] = 79.15
cor[0][2] = 37.13
cor[1][0] = 79.15
cor[2][0] = 37.13
cor[1][1] = 50.04
cor[1][2] = 21.65
cor[2][1] = 21.65
cor[2][2] = 13.26
lmbd, eigvec = pca(cor)
print lmbd
print eigvec
for i in xrange(lend): # row
for j in xrange(lend): print eigvec[j][i], # eigenvector
print
print " B"
for i in xrange(lend): # row
for j in xrange(lend): print eigvec[j][i]/sqrt(lmbd[j]), # eigenvector
print
print " G"
for i in xrange(lend): # row
for j in xrange(lend): print eigvec[j][i]*sqrt(lmbd[j]), # eigenvector
print
"""
else:
nargs = len(args)
if nargs != 2:
ERROR("Must provide name of input and output file!")
return
from sp_utilities import get_params2D, model_circle
from sp_fundamentals import rot_shift2D
from sp_statistics import ccc
from time import time
from sp_alignment import align2d
stack = args[0]
new_stack = args[1]
d = EMData.read_images(stack)
try:
sxprint("Using 2D alignment parameters from header.")
ttt = d[0].get_attr('xform.params2d')
for i in range(len(d)):
alpha, sx, sy, mirror, scale = get_params2D(d[i])
d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror)
except:
pass
nx = d[0].get_xsize()
ny = d[0].get_ysize()
if options.radius < 1:
radius = old_div(nx, 2) - 2
else:
radius = options.radius
mask = model_circle(radius, nx, ny)
init = options.initial
if init > -1:
sxprint(" initial image: %d" % init)
temp = d[init].copy()
temp.write_image(new_stack, 0)
del d[init]
k = 1
lsum = 0.0
while len(d) > 1:
maxcit = -111.
for i in range(len(d)):
cuc = ccc(d[i], temp, mask)
if cuc > maxcit:
maxcit = cuc
qi = i
# sxprint k, maxcit
lsum += maxcit
temp = d[qi].copy()
del d[qi]
temp.write_image(new_stack, k)
k += 1
sxprint(lsum)
d[0].write_image(new_stack, k)
else:
if options.circular:
sxprint("Using options.circular, no alignment")
# figure the "best circular" starting image
maxsum = -1.023
for m in range(len(d)):
indc = list(range(len(d)))
lsnake = [-1] * (len(d) + 1)
lsnake[0] = m
lsnake[-1] = m
del indc[m]
temp = d[m].copy()
lsum = 0.0
direction = +1
k = 1
while len(indc) > 1:
maxcit = -111.
for i in range(len(indc)):
cuc = ccc(d[indc[i]], temp, mask)
if cuc > maxcit:
maxcit = cuc
qi = indc[i]
lsnake[k] = qi
lsum += maxcit
del indc[indc.index(qi)]
direction = -direction
for i in range(1, len(d)):
if (direction > 0):
if (lsnake[i] == -1):
temp = d[lsnake[i - 1]].copy()
# print " forw ",lsnake[i-1]
k = i
break
else:
if (lsnake[len(d) - i] == -1):
temp = d[lsnake[len(d) - i + 1]].copy()
# print " back ",lsnake[len(d) - i +1]
k = len(d) - i
break
lsnake[lsnake.index(-1)] = indc[-1]
# print " initial image and lsum ",m,lsum
# print lsnake
if (lsum > maxsum):
maxsum = lsum
init = m
snake = [lsnake[i] for i in range(len(d))]
sxprint(" Initial image selected : ", init, maxsum)
sxprint(lsnake)
for m in range(len(d)):
d[snake[m]].write_image(new_stack, m)
else:
# figure the "best" starting image
sxprint("Straight chain, no alignment")
maxsum = -1.023
for m in range(len(d)):
indc = list(range(len(d)))
lsnake = [m]
del indc[m]
temp = d[m].copy()
lsum = 0.0
while len(indc) > 1:
maxcit = -111.
for i in range(len(indc)):
cuc = ccc(d[indc[i]], temp, mask)
if cuc > maxcit:
maxcit = cuc
qi = indc[i]
lsnake.append(qi)
lsum += maxcit
temp = d[qi].copy()
del indc[indc.index(qi)]
lsnake.append(indc[-1])
# sxprint " initial image and lsum ",m,lsum
# sxprint lsnake
if (lsum > maxsum):
maxsum = lsum
init = m
snake = [lsnake[i] for i in range(len(d))]
sxprint(" Initial image selected : ", init, maxsum)
sxprint(lsnake)
for m in range(len(d)):
d[snake[m]].write_image(new_stack, m)
def get_attr_dict(self): a = EMData() a.read_image(self.file_name,self.idx,True) return a.get_attr_dict()
def apsh(refimgs,
imgs2align,
outangles=None,
refanglesdoc=None,
outaligndoc=None,
outerradius=-1,
maxshift=0,
ringstep=1,
mode="F",
log=None,
verbose=False):
"""
Generates polar representations of a series of images to be used as alignment references.
Arguments:
refimgs : Input reference image stack (filename or EMData object)
imgs2align : Image stack to be aligned (filename or EMData object)
outangles : Output Euler angles doc file
refanglesdoc : Input Euler angles for reference projections
outaligndoc : Output 2D alignment doc file
outerradius : Outer alignment radius
maxshift : Maximum shift allowed
ringstep : Alignment radius step size
mode : Mode, full circle ("F") vs. half circle ("H")
log : Logger object
verbose : (boolean) Whether to write additional information to screen
"""
# Generate polar representation(s) of reference(s)
alignrings, polarrefs = mref2polar(refimgs,
outerradius=outerradius,
ringstep=ringstep,
log=log,
verbose=verbose)
# Read image stack (as a filename or already an EMDataobject)
if isinstance(imgs2align, str):
imagestacklist = EMData.read_images(imgs2align)
else:
imagestacklist = [imgs2align]
# Get number of images
numimg = len(imagestacklist)
# Get image dimensions (assuming square, and that images and references have the same dimension)
idim = imagestacklist[0]['nx']
# Calculate image center
halfdim = idim / 2 + 1
# Set constants
currshift = 0
scale = 1
# Initialize output angles
outangleslist = []
outalignlist = []
if outerradius <= 0:
outerradius = halfdim - 3
# Set search range
txrng = tyrng = search_range(idim, outerradius, currshift, maxshift)
print_log_msg(
'Running multireference alignment allowing a maximum shift of %s\n' %
maxshift, log, verbose)
# Loop through images
for imgindex in range(numimg):
currimg = imagestacklist[imgindex]
# Perform multi-reference alignment (adapted from alignment.mref_ali2d)
best2dparamslist = [
angt, sxst, syst, mirrorfloat, bestreffloat, peakt
] = Util.multiref_polar_ali_2d(currimg, polarrefs, txrng, tyrng,
ringstep, mode, alignrings, halfdim,
halfdim)
bestref = int(bestreffloat)
mirrorflag = int(mirrorfloat)
# Store parameters
params2dlist = [angt, sxst, syst, mirrorflag, scale]
outalignlist.append(params2dlist)
if refanglesdoc:
refangleslist = read_text_row(refanglesdoc)
besteulers = refangleslist[bestref]
else:
besteulers = [0] * 5
# Check for mirroring
if mirrorflag == 1:
tempeulers = list(
compose_transform3(besteulers[0], besteulers[1], besteulers[2],
besteulers[3], besteulers[4], 0, 1, 0, 180,
0, 0, 0, 0, 1))
combinedparams = list(
compose_transform3(tempeulers[0], tempeulers[1], tempeulers[2],
tempeulers[3], tempeulers[4], 0, 1, 0, 0,
-angt, 0, 0, 0, 1))
else:
combinedparams = list(
compose_transform3(besteulers[0], besteulers[1], besteulers[2],
besteulers[3], besteulers[4], 0, 1, 0, 0,
-angt, 0, 0, 0, 1))
# compose_transform3: returns phi,theta,psi, tx,ty,tz, scale
outangleslist.append(combinedparams)
# Set transformations as image attribute
set_params2D(currimg, params2dlist, xform="xform.align2d"
) # sometimes I get a vector error with sxheader
set_params_proj(currimg, besteulers,
xform="xform.projection") # use shifts
if outangles or outaligndoc:
msg = ''
if outangles:
write_text_row(outangleslist, outangles)
msg += 'Wrote alignment angles to %s\n' % outangles
print_log_msg(msg, log, verbose)
if outaligndoc:
write_text_row(outalignlist, outaligndoc)
msg += 'Wrote 2D alignment parameters to %s\n' % outaligndoc
print_log_msg(msg, log, verbose)
return outalignlist
def getmxinfo(fsp,fsp2,clsnum): """reads particle references associated with a particular class. fsp2 is the matrix file and fsp is the raw particle file""" mx=EMData(fsp2,0) imgs=[(fsp,i) for i in range(mx.get_ysize()) if mx.get(0,i)==clsnum] return imgs
def execute(self,alignment_jobs,files,caller):
'''
The main function
@param alignment_jobs a list of alignment pair indices like this [[0,1],[2,1],[2,3],[0,5],...] etc the indices pair represent images to be aligned and correspond to the order of the files argument
@param files a list of filenames - used to read image based on the indices present in alignment_jobs
@param caller - the calling object - it needs to have a function called process_output that takes a dictionary as the argument
'''
options = self.options
align_data = EMAN2.parsemodopt(options.align)
align_cmp_data = EMAN2.parsemodopt(options.aligncmp)
cmp_data = EMAN2.parsemodopt(options.cmp)
ralign_data = None
if options.ralign != None:
ralign_data = EMAN2.parsemodopt(options.ralign)
ralign_cmp_data = EMAN2.parsemodopt(options.raligncmp)
data = {}
data["align"] = align_data
data["aligncmp"] = align_cmp_data
data["cmp"] = cmp_data
if ralign_data:
data["ralign"] = ralign_data
data["raligncmp"] = ralign_cmp_data
data["using_cuda"] = self.using_cuda
data["nsoln"] = self.nsoln
if self.options.parallel :
task_customers = []
tids = []
if options.shrink:
scratch_name_1 = numbered_bdb("bdb:tomo_scratch#scratch_shrink")
scratch_name_2 = numbered_bdb("bdb:tomo_scratch##scratch_shrink")
else: print "no shrink"
for i,j in alignment_jobs:
if options.shrink or options.filter:
a = EMData(files[i],0)
if options.filter:
filter_params = EMAN2.parsemodopt(options.filter)
a.process_inplace(filter_params[0],filter_params[1])
if options.shrink:
a.process_inplace("math.meanshrink",{"n":options.shrink})
a.set_attr("src_image",files[i])
a.write_image(scratch_name_1,0)
a = EMData(files[j],0)
if options.filter:
filter_params = EMAN2.parsemodopt(options.filter)
a.process_inplace(filter_params[0],filter_params[1])
if options.shrink:
a.process_inplace("math.meanshrink",{"n":options.shrink})
a.set_attr("src_image",files[j])
a.write_image(scratch_name_2,0)
data["probe"] = ("cache",scratch_name_1,0)
data["target"] = ("cache",scratch_name_2,0)
else:
data["probe"] = ("cache",files[i],0)
data["target"] = ("cache",files[j],0)
data["target_idx"] = j
data["probe_idx"] = i
task = EMTomoAlignTaskDC(data=data)
from EMAN2PAR import EMTaskCustomer
etc=EMTaskCustomer(self.options.parallel)
#print "Est %d CPUs"%etc.cpu_est()
tid=etc.send_task(task)
#print "Task submitted tid=",tid
task_customers.append(etc)
tids.append(tid)
self.dc_monitor(task_customers,tids,caller)
else:
n = len(alignment_jobs)
p = 0.0
for i,j in alignment_jobs:
probe = EMData(files[i],0)
target = EMData(files[j],0)
if options.filter:
print "filtered"
filter_params = EMAN2.parsemodopt(options.filter)
probe.process_inplace(filter_params[0],filter_params[1])
target.process_inplace(filter_params[0],filter_params[1])
if options.shrink:
probe.process_inplace("math.meanshrink",{"n":options.shrink})
target.process_inplace("math.meanshrink",{"n":options.shrink})
else:
print "no shrink"
data["target"] = target
data["probe"] = probe
data["target_idx"] = j
data["probe_idx"] = i
task = EMTomoAlignTask(data=data)
rslts = task.execute(self.progress_callback)
if options.shrink:
self.correction_translation(rslts,options.shrink)
caller.process_output(rslts)
p += 1.0
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options] <image file> ...
This program can be used to visualize most files used in EMAN2. Running it without arguments
will open a browser window with more flexible functionality than the command-line.
"""
global app,win,options
parser = EMArgumentParser(usage=usage,version=EMANVERSION)
parser.add_argument("--classmx",type=str,help="<classmx>,<#> Show particles in one class from a classification matrix. Pass raw particle file as first argument to command.")
parser.add_argument("--classes",type=str,help="<rawptcl>,<classmx> Show particles associated class-averages")
parser.add_argument("--pdb",type=str,help="<pdb file> Show PDB structure.")
parser.add_argument("--singleimage",action="store_true",default=False,help="Display a stack in a single image view")
parser.add_argument("--plot",action="store_true",default=False,help="Data file(s) should be plotted rather than displayed in 2-D")
parser.add_argument("--plot3",action="store_true",default=False,help="Data file(s) should be plotted rather than displayed in 3-D")
parser.add_argument("--fullrange",action="store_true",default=False,help="A specialized flag that disables auto contrast for the display of particles stacks and 2D images only.")
parser.add_argument("--newwidget",action="store_true",default=False,help="Use the new 3D widgetD. Highly recommended!!!!")
parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-2)
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higner number means higher level of verboseness")
(options, args) = parser.parse_args()
# logid=E2init(sys.argv)
app = EMApp()
#gapp = app
#QtGui.QApplication(sys.argv)
win=[]
if options.fullrange:
fullrangeparms = set_full_range()
if len(args) < 1:
global dialog
file_list = []
dialog = embrowser.EMBrowserWidget(withmodal=False,multiselect=False)
dialog.show()
try: dialog.raise_()
# QtCore.QObject.connect(dialog,QtCore.SIGNAL("ok"),on_browser_done)
# QtCore.QObject.connect(dialog,QtCore.SIGNAL("cancel"),on_browser_cancel)
except: pass
elif options.pdb:
load_pdb(args,app)
elif options.plot:
plot(args,app)
elif options.plot3:
plot_3d(args,app)
elif options.classes:
options.classes=options.classes.split(",")
imgs=EMData.read_images(args[0])
display(imgs,app,args[0])
QtCore.QObject.connect(win[0].child,QtCore.SIGNAL("mousedown"),lambda a,b:selectclass(options.classes[0],options.classes[1],a,b))
try:
out=file("selected.lst","w")
out.write("#LST\n")
out.close()
except: pass
elif options.classmx:
options.classmx=options.classmx.split(",")
clsnum=int(options.classmx[1])
imgs=getmxim(args[0],options.classmx[0],clsnum)
display(imgs,app,args[0])
else:
for i in args:
if not file_exists(i):
print "%s doesn't exist" %i
sys.exit(1)
display_file(i,app,options.singleimage,usescenegraph=options.newwidget)
if options.fullrange:
revert_full_range(fullrangeparms)
app.exec_()
def write_image(self, *args):
a = EMData()
a.read_image(self.file_name, self.idx)
a.write_image(*args)
def main():
from optparse import OptionParser
from global_def import SPARXVERSION
from EMAN2 import EMData
from logger import Logger, BaseLogger_Files
import sys, os, time
global Tracker, Blockdata
from global_def import ERROR
progname = os.path.basename(sys.argv[0])
usage = progname + " --output_dir=output_dir --isac_dir=output_dir_of_isac "
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--pw_adjustment", type ="string", default ='analytical_model', \
help="adjust power spectrum of 2-D averages to an analytic model. Other opions: no_adjustment; bfactor; a text file of 1D rotationally averaged PW")
#### Four options for --pw_adjustment:
# 1> analytical_model(default);
# 2> no_adjustment;
# 3> bfactor;
# 4> adjust_to_given_pw2(user has to provide a text file that contains 1D rotationally averaged PW)
# options in common
parser.add_option(
"--isac_dir",
type="string",
default='',
help="ISAC run output directory, input directory for this command")
parser.add_option(
"--output_dir",
type="string",
default='',
help="output directory where computed averages are saved")
parser.add_option(
"--pixel_size",
type="float",
default=-1.0,
help=
"pixel_size of raw images. one can put 1.0 in case of negative stain data"
)
parser.add_option(
"--fl",
type="float",
default=-1.0,
help=
"low pass filter, = -1.0, not applied; =0.0, using FH1 (initial resolution), = 1.0 using FH2 (resolution after local alignment), or user provided value in absolute freqency [0.0:0.5]"
)
parser.add_option("--stack",
type="string",
default="",
help="data stack used in ISAC")
parser.add_option("--radius", type="int", default=-1, help="radius")
parser.add_option("--xr",
type="float",
default=-1.0,
help="local alignment search range")
#parser.add_option("--ts", type ="float", default =1.0, help= "local alignment search step")
parser.add_option("--fh",
type="float",
default=-1.0,
help="local alignment high frequencies limit")
#parser.add_option("--maxit", type ="int", default =5, help= "local alignment iterations")
parser.add_option("--navg",
type="int",
default=1000000,
help="number of aveages")
parser.add_option("--local_alignment",
action="store_true",
default=False,
help="do local alignment")
parser.add_option(
"--noctf",
action="store_true",
default=False,
help=
"no ctf correction, useful for negative stained data. always ctf for cryo data"
)
parser.add_option(
"--B_start",
type="float",
default=45.0,
help=
"start frequency (Angstrom) of power spectrum for B_factor estimation")
parser.add_option(
"--Bfactor",
type="float",
default=-1.0,
help=
"User defined bactors (e.g. 25.0[A^2]). By default, the program automatically estimates B-factor. "
)
(options, args) = parser.parse_args(sys.argv[1:])
adjust_to_analytic_model = False
adjust_to_given_pw2 = False
B_enhance = False
no_adjustment = False
if options.pw_adjustment == 'analytical_model':
adjust_to_analytic_model = True
elif options.pw_adjustment == 'no_adjustment':
no_adjustment = True
elif options.pw_adjustment == 'bfactor':
B_enhance = True
else:
adjust_to_given_pw2 = True
from utilities import get_im, bcast_number_to_all, write_text_file, read_text_file, wrap_mpi_bcast, write_text_row
from utilities import cmdexecute
from filter import filt_tanl
from logger import Logger, BaseLogger_Files
import user_functions
import string
from string import split, atoi, atof
import json
mpi_init(0, [])
nproc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
Blockdata = {}
# MPI stuff
Blockdata["nproc"] = nproc
Blockdata["myid"] = myid
Blockdata["main_node"] = 0
Blockdata["shared_comm"] = mpi_comm_split_type(MPI_COMM_WORLD,
MPI_COMM_TYPE_SHARED, 0,
MPI_INFO_NULL)
Blockdata["myid_on_node"] = mpi_comm_rank(Blockdata["shared_comm"])
Blockdata["no_of_processes_per_group"] = mpi_comm_size(
Blockdata["shared_comm"])
masters_from_groups_vs_everything_else_comm = mpi_comm_split(
MPI_COMM_WORLD, Blockdata["main_node"] == Blockdata["myid_on_node"],
Blockdata["myid_on_node"])
Blockdata["color"], Blockdata["no_of_groups"], balanced_processor_load_on_nodes = get_colors_and_subsets(Blockdata["main_node"], MPI_COMM_WORLD, Blockdata["myid"], \
Blockdata["shared_comm"], Blockdata["myid_on_node"], masters_from_groups_vs_everything_else_comm)
# We need two nodes for processing of volumes
Blockdata["node_volume"] = [
Blockdata["no_of_groups"] - 3, Blockdata["no_of_groups"] - 2,
Blockdata["no_of_groups"] - 1
] # For 3D stuff take three last nodes
# We need two CPUs for processing of volumes, they are taken to be main CPUs on each volume
# We have to send the two myids to all nodes so we can identify main nodes on two selected groups.
Blockdata["nodes"] = [Blockdata["node_volume"][0]*Blockdata["no_of_processes_per_group"],Blockdata["node_volume"][1]*Blockdata["no_of_processes_per_group"], \
Blockdata["node_volume"][2]*Blockdata["no_of_processes_per_group"]]
# End of Blockdata: sorting requires at least three nodes, and the used number of nodes be integer times of three
global_def.BATCH = True
global_def.MPI = True
if adjust_to_given_pw2:
checking_flag = 0
if (Blockdata["myid"] == Blockdata["main_node"]):
if not os.path.exists(options.pw_adjustment): checking_flag = 1
checking_flag = bcast_number_to_all(checking_flag,
Blockdata["main_node"],
MPI_COMM_WORLD)
if checking_flag == 1:
ERROR("User provided power spectrum does not exist",
"sxcompute_isac_avg.py", 1, Blockdata["myid"])
Tracker = {}
Constants = {}
Constants["isac_dir"] = options.isac_dir
Constants["masterdir"] = options.output_dir
Constants["pixel_size"] = options.pixel_size
Constants["orgstack"] = options.stack
Constants["radius"] = options.radius
Constants["xrange"] = options.xr
Constants["FH"] = options.fh
Constants["low_pass_filter"] = options.fl
#Constants["maxit"] = options.maxit
Constants["navg"] = options.navg
Constants["B_start"] = options.B_start
Constants["Bfactor"] = options.Bfactor
if adjust_to_given_pw2: Constants["modelpw"] = options.pw_adjustment
Tracker["constants"] = Constants
# -------------------------------------------------------------
#
# Create and initialize Tracker dictionary with input options # State Variables
#<<<---------------------->>>imported functions<<<---------------------------------------------
#x_range = max(Tracker["constants"]["xrange"], int(1./Tracker["ini_shrink"])+1)
#y_range = x_range
####-----------------------------------------------------------
# Create Master directory and associated subdirectories
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
if Tracker["constants"]["masterdir"] == Tracker["constants"]["isac_dir"]:
masterdir = os.path.join(Tracker["constants"]["isac_dir"], "sharpen")
else:
masterdir = Tracker["constants"]["masterdir"]
if (Blockdata["myid"] == Blockdata["main_node"]):
msg = "Postprocessing ISAC 2D averages starts"
print(line, "Postprocessing ISAC 2D averages starts")
if not masterdir:
timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime())
masterdir = "sharpen_" + Tracker["constants"]["isac_dir"]
os.mkdir(masterdir)
else:
if os.path.exists(masterdir):
print("%s already exists" % masterdir)
else:
os.mkdir(masterdir)
subdir_path = os.path.join(masterdir, "ali2d_local_params_avg")
if not os.path.exists(subdir_path): os.mkdir(subdir_path)
subdir_path = os.path.join(masterdir, "params_avg")
if not os.path.exists(subdir_path): os.mkdir(subdir_path)
li = len(masterdir)
else:
li = 0
li = mpi_bcast(li, 1, MPI_INT, Blockdata["main_node"], MPI_COMM_WORLD)[0]
masterdir = mpi_bcast(masterdir, li, MPI_CHAR, Blockdata["main_node"],
MPI_COMM_WORLD)
masterdir = string.join(masterdir, "")
Tracker["constants"]["masterdir"] = masterdir
log_main = Logger(BaseLogger_Files())
log_main.prefix = Tracker["constants"]["masterdir"] + "/"
while not os.path.exists(Tracker["constants"]["masterdir"]):
print("Node ", Blockdata["myid"], " waiting...",
Tracker["constants"]["masterdir"])
sleep(1)
mpi_barrier(MPI_COMM_WORLD)
if (Blockdata["myid"] == Blockdata["main_node"]):
init_dict = {}
print(Tracker["constants"]["isac_dir"])
Tracker["directory"] = os.path.join(Tracker["constants"]["isac_dir"],
"2dalignment")
core = read_text_row(
os.path.join(Tracker["directory"], "initial2Dparams.txt"))
for im in range(len(core)):
init_dict[im] = core[im]
del core
else:
init_dict = 0
init_dict = wrap_mpi_bcast(init_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
###
do_ctf = True
if options.noctf: do_ctf = False
if (Blockdata["myid"] == Blockdata["main_node"]):
if do_ctf: print("CTF correction is on")
else: print("CTF correction is off")
if options.local_alignment: print("local refinement is on")
else: print("local refinement is off")
if B_enhance: print("Bfactor is to be applied on averages")
elif adjust_to_given_pw2:
print("PW of averages is adjusted to a given 1D PW curve")
elif adjust_to_analytic_model:
print("PW of averages is adjusted to analytical model")
else:
print("PW of averages is not adjusted")
#Tracker["constants"]["orgstack"] = "bdb:"+ os.path.join(Tracker["constants"]["isac_dir"],"../","sparx_stack")
image = get_im(Tracker["constants"]["orgstack"], 0)
Tracker["constants"]["nnxo"] = image.get_xsize()
if Tracker["constants"]["pixel_size"] == -1.0:
print(
"Pixel size value is not provided by user. extracting it from ctf header entry of the original stack."
)
try:
ctf_params = image.get_attr("ctf")
Tracker["constants"]["pixel_size"] = ctf_params.apix
except:
ERROR(
"Pixel size could not be extracted from the original stack.",
"sxcompute_isac_avg.py", 1,
Blockdata["myid"]) # action=1 - fatal error, exit
## Now fill in low-pass filter
isac_shrink_path = os.path.join(Tracker["constants"]["isac_dir"],
"README_shrink_ratio.txt")
if not os.path.exists(isac_shrink_path):
ERROR(
"%s does not exist in the specified ISAC run output directory"
% (isac_shrink_path), "sxcompute_isac_avg.py", 1,
Blockdata["myid"]) # action=1 - fatal error, exit
isac_shrink_file = open(isac_shrink_path, "r")
isac_shrink_lines = isac_shrink_file.readlines()
isac_shrink_ratio = float(
isac_shrink_lines[5]
) # 6th line: shrink ratio (= [target particle radius]/[particle radius]) used in the ISAC run
isac_radius = float(
isac_shrink_lines[6]
) # 7th line: particle radius at original pixel size used in the ISAC run
isac_shrink_file.close()
print("Extracted parameter values")
print("ISAC shrink ratio : {0}".format(isac_shrink_ratio))
print("ISAC particle radius : {0}".format(isac_radius))
Tracker["ini_shrink"] = isac_shrink_ratio
else:
Tracker["ini_shrink"] = 0.0
Tracker = wrap_mpi_bcast(Tracker,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
#print(Tracker["constants"]["pixel_size"], "pixel_size")
x_range = max(Tracker["constants"]["xrange"],
int(1. / Tracker["ini_shrink"] + 0.99999))
a_range = y_range = x_range
if (Blockdata["myid"] == Blockdata["main_node"]):
parameters = read_text_row(
os.path.join(Tracker["constants"]["isac_dir"],
"all_parameters.txt"))
else:
parameters = 0
parameters = wrap_mpi_bcast(parameters,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
params_dict = {}
list_dict = {}
#parepare params_dict
#navg = min(Tracker["constants"]["navg"]*Blockdata["nproc"], EMUtil.get_image_count(os.path.join(Tracker["constants"]["isac_dir"], "class_averages.hdf")))
navg = min(
Tracker["constants"]["navg"],
EMUtil.get_image_count(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf")))
global_dict = {}
ptl_list = []
memlist = []
if (Blockdata["myid"] == Blockdata["main_node"]):
print("Number of averages computed in this run is %d" % navg)
for iavg in range(navg):
params_of_this_average = []
image = get_im(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf"), iavg)
members = sorted(image.get_attr("members"))
memlist.append(members)
for im in range(len(members)):
abs_id = members[im]
global_dict[abs_id] = [iavg, im]
P = combine_params2( init_dict[abs_id][0], init_dict[abs_id][1], init_dict[abs_id][2], init_dict[abs_id][3], \
parameters[abs_id][0], parameters[abs_id][1]/Tracker["ini_shrink"], parameters[abs_id][2]/Tracker["ini_shrink"], parameters[abs_id][3])
if parameters[abs_id][3] == -1:
print(
"WARNING: Image #{0} is an unaccounted particle with invalid 2D alignment parameters and should not be the member of any classes. Please check the consitency of input dataset."
.format(abs_id)
) # How to check what is wrong about mirror = -1 (Toshio 2018/01/11)
params_of_this_average.append([P[0], P[1], P[2], P[3], 1.0])
ptl_list.append(abs_id)
params_dict[iavg] = params_of_this_average
list_dict[iavg] = members
write_text_row(
params_of_this_average,
os.path.join(Tracker["constants"]["masterdir"], "params_avg",
"params_avg_%03d.txt" % iavg))
ptl_list.sort()
init_params = [None for im in range(len(ptl_list))]
for im in range(len(ptl_list)):
init_params[im] = [ptl_list[im]] + params_dict[global_dict[
ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
write_text_row(
init_params,
os.path.join(Tracker["constants"]["masterdir"],
"init_isac_params.txt"))
else:
params_dict = 0
list_dict = 0
memlist = 0
params_dict = wrap_mpi_bcast(params_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
list_dict = wrap_mpi_bcast(list_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
memlist = wrap_mpi_bcast(memlist,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
# Now computing!
del init_dict
tag_sharpen_avg = 1000
## always apply low pass filter to B_enhanced images to suppress noise in high frequencies
enforced_to_H1 = False
if B_enhance:
if Tracker["constants"]["low_pass_filter"] == -1.0:
enforced_to_H1 = True
if navg < Blockdata["nproc"]: # Each CPU do one average
ERROR("number of nproc is larger than number of averages",
"sxcompute_isac_avg.py", 1, Blockdata["myid"])
else:
FH_list = [[0, 0.0, 0.0] for im in range(navg)]
image_start, image_end = MPI_start_end(navg, Blockdata["nproc"],
Blockdata["myid"])
if Blockdata["myid"] == Blockdata["main_node"]:
cpu_dict = {}
for iproc in range(Blockdata["nproc"]):
local_image_start, local_image_end = MPI_start_end(
navg, Blockdata["nproc"], iproc)
for im in range(local_image_start, local_image_end):
cpu_dict[im] = iproc
else:
cpu_dict = 0
cpu_dict = wrap_mpi_bcast(cpu_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
slist = [None for im in range(navg)]
ini_list = [None for im in range(navg)]
avg1_list = [None for im in range(navg)]
avg2_list = [None for im in range(navg)]
plist_dict = {}
data_list = [None for im in range(navg)]
if Blockdata["myid"] == Blockdata["main_node"]:
if B_enhance:
print(
"Avg ID B-factor FH1(Res before ali) FH2(Res after ali)"
)
else:
print("Avg ID FH1(Res before ali) FH2(Res after ali)")
for iavg in range(image_start, image_end):
mlist = EMData.read_images(Tracker["constants"]["orgstack"],
list_dict[iavg])
for im in range(len(mlist)):
#mlist[im]= get_im(Tracker["constants"]["orgstack"], list_dict[iavg][im])
set_params2D(mlist[im],
params_dict[iavg][im],
xform="xform.align2d")
if options.local_alignment:
"""
new_average1 = within_group_refinement([mlist[kik] for kik in range(0,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou=Tracker["constants"]["radius"], rs=1.0, xrng=[x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH=max(Tracker["constants"]["FH"], FH1), FF=0.02, method="")
new_average2 = within_group_refinement([mlist[kik] for kik in range(1,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou= Tracker["constants"]["radius"], rs=1.0, xrng=[ x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH = max(Tracker["constants"]["FH"], FH1), FF=0.02, method="")
new_avg, frc, plist = compute_average(mlist, Tracker["constants"]["radius"], do_ctf)
"""
new_avg, plist, FH2 = refinement_2d_local(
mlist,
Tracker["constants"]["radius"],
a_range,
x_range,
y_range,
CTF=do_ctf,
SNR=1.0e10)
plist_dict[iavg] = plist
FH1 = -1.0
else:
new_avg, frc, plist = compute_average(
mlist, Tracker["constants"]["radius"], do_ctf)
FH1 = get_optimistic_res(frc)
FH2 = -1.0
#write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d.txt"%iavg))
FH_list[iavg] = [iavg, FH1, FH2]
if B_enhance:
new_avg, gb = apply_enhancement(
new_avg, Tracker["constants"]["B_start"],
Tracker["constants"]["pixel_size"],
Tracker["constants"]["Bfactor"])
print(" %6d %6.3f %4.3f %4.3f" % (iavg, gb, FH1, FH2))
elif adjust_to_given_pw2:
roo = read_text_file(Tracker["constants"]["modelpw"], -1)
roo = roo[0] # always on the first column
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], roo)
print(" %6d %4.3f %4.3f " % (iavg, FH1, FH2))
elif adjust_to_analytic_model:
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], None)
print(" %6d %4.3f %4.3f " % (iavg, FH1, FH2))
elif no_adjustment:
pass
if Tracker["constants"]["low_pass_filter"] != -1.0:
if Tracker["constants"]["low_pass_filter"] == 0.0:
low_pass_filter = FH1
elif Tracker["constants"]["low_pass_filter"] == 1.0:
low_pass_filter = FH2
if not options.local_alignment: low_pass_filter = FH1
else:
low_pass_filter = Tracker["constants"]["low_pass_filter"]
if low_pass_filter >= 0.45: low_pass_filter = 0.45
new_avg = filt_tanl(new_avg, low_pass_filter, 0.02)
else: # No low pass filter but if enforced
if enforced_to_H1: new_avg = filt_tanl(new_avg, FH1, 0.02)
if B_enhance: new_avg = fft(new_avg)
new_avg.set_attr("members", list_dict[iavg])
new_avg.set_attr("n_objects", len(list_dict[iavg]))
slist[iavg] = new_avg
print(
strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>",
"Refined average %7d" % iavg)
## send to main node to write
mpi_barrier(MPI_COMM_WORLD)
for im in range(navg):
# avg
if cpu_dict[im] == Blockdata[
"myid"] and Blockdata["myid"] != Blockdata["main_node"]:
send_EMData(slist[im], Blockdata["main_node"], tag_sharpen_avg)
elif cpu_dict[im] == Blockdata["myid"] and Blockdata[
"myid"] == Blockdata["main_node"]:
slist[im].set_attr("members", memlist[im])
slist[im].set_attr("n_objects", len(memlist[im]))
slist[im].write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
elif cpu_dict[im] != Blockdata["myid"] and Blockdata[
"myid"] == Blockdata["main_node"]:
new_avg_other_cpu = recv_EMData(cpu_dict[im], tag_sharpen_avg)
new_avg_other_cpu.set_attr("members", memlist[im])
new_avg_other_cpu.set_attr("n_objects", len(memlist[im]))
new_avg_other_cpu.write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
if options.local_alignment:
if cpu_dict[im] == Blockdata["myid"]:
write_text_row(
plist_dict[im],
os.path.join(Tracker["constants"]["masterdir"],
"ali2d_local_params_avg",
"ali2d_local_params_avg_%03d.txt" % im))
if cpu_dict[im] == Blockdata[
"myid"] and cpu_dict[im] != Blockdata["main_node"]:
wrap_mpi_send(plist_dict[im], Blockdata["main_node"],
MPI_COMM_WORLD)
wrap_mpi_send(FH_list, Blockdata["main_node"],
MPI_COMM_WORLD)
elif cpu_dict[im] != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
plist_dict[im] = dummy
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
FH_list[im] = dummy[im]
else:
if cpu_dict[im] == Blockdata[
"myid"] and cpu_dict[im] != Blockdata["main_node"]:
wrap_mpi_send(FH_list, Blockdata["main_node"],
MPI_COMM_WORLD)
elif cpu_dict[im] != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
FH_list[im] = dummy[im]
mpi_barrier(MPI_COMM_WORLD)
mpi_barrier(MPI_COMM_WORLD)
if options.local_alignment:
if Blockdata["myid"] == Blockdata["main_node"]:
ali3d_local_params = [None for im in range(len(ptl_list))]
for im in range(len(ptl_list)):
ali3d_local_params[im] = [ptl_list[im]] + plist_dict[
global_dict[ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
write_text_row(
ali3d_local_params,
os.path.join(Tracker["constants"]["masterdir"],
"ali2d_local_params.txt"))
write_text_row(
FH_list,
os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))
else:
if Blockdata["myid"] == Blockdata["main_node"]:
write_text_row(
FH_list,
os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))
mpi_barrier(MPI_COMM_WORLD)
target_xr = 3
target_yr = 3
if (Blockdata["myid"] == 0):
cmd = "{} {} {} {} {} {} {} {} {} {}".format("sxchains.py", os.path.join(Tracker["constants"]["masterdir"],"class_averages.hdf"),\
os.path.join(Tracker["constants"]["masterdir"],"junk.hdf"),os.path.join(Tracker["constants"]["masterdir"],"ordered_class_averages.hdf"),\
"--circular","--radius=%d"%Tracker["constants"]["radius"] , "--xr=%d"%(target_xr+1),"--yr=%d"%(target_yr+1),"--align", ">/dev/null")
junk = cmdexecute(cmd)
cmd = "{} {}".format(
"rm -rf",
os.path.join(Tracker["constants"]["masterdir"], "junk.hdf"))
junk = cmdexecute(cmd)
from mpi import mpi_finalize
mpi_finalize()
exit()
