def build_object_layer_z_level(self, width, height, background_layer, z_level):
# get_image(x pixels, y pixels, width pixels, height pixels)
background_layer.image.fill(utilities.colors.key)
objects_to_draw = []
for each in self.entity_group["Flora"]:
if each.height >= z_level:
objects_to_draw.append(each)
for each in self.entity_group["Terrain"]:
if each.height >= z_level:
objects_to_draw.append(each)
for each in self.entity_group["Structure"]:
if each.height >= z_level:
objects_to_draw.append(each)
rows_to_draw = []
for y_level in range(int(self.height / 20)):
this_row = []
for entity in objects_to_draw:
if entity.tile_y == y_level:
this_row.append(entity)
rows_to_draw.append(this_row)
for row in rows_to_draw:
for entity in row:
image_height = entity.height * 20
image_slice = utilities.get_image(entity.sprite.image, 0, image_height - (z_level * 20), entity.width * 20, 20)
background_layer.image.blit(image_slice, [entity.tile_x * 20 + int((entity.footprint[0] * 20 - entity.width * 20) / 2),
(entity.tile_y - z_level + 1) * 20])
background_layer.image.set_colorkey(utilities.colors.key)
background_layer.image = background_layer.image.convert_alpha()
def image_decimate(img, decimation=2, fit_to_fft = True, frequency_low=0, frequency_high=0):
"""
Window 2D image to FFT-friendly size, apply Butterworth low pass filter,
and decimate image by integer factor
"""
from filter import filt_btwl
from fundamentals import smallprime, window2d
from utilities import get_image
if type(img) == str: img=get_image(img)
nz = img.get_zsize()
if( nz > 1): ERROR("This command works only for 2-D images", "image_decimate", 1)
if decimation <= 1 : ERROR("Improper decimation ratio", "image_decimate", 1)
if(decimation == 1.0): return img.copy()
if frequency_low <= 0 :
frequency_low = 0.5/decimation-0.02
if frequency_low <= 0 : ERROR("Butterworth pass-band frequency is too low","image_decimation",1)
frequency_high = min(0.5/decimation + 0.02, 0.499)
if fit_to_fft:
nx = img.get_xsize()
ny = img.get_ysize()
nx_fft_m = smallprime(nx)
ny_fft_m = smallprime(ny)
e = Util.window(img, nx_fft_m, ny_fft_m, 1, 0,0,0)
e = filt_btwl(e, frequency_low, frequency_high)
else:
e = filt_btwl(img, frequency_low, frequency_high)
return Util.decimate(e, int(decimation), int(decimation), 1)
def image_decimate(img, decimation=2, fit_to_fft = True, frequency_low=0, frequency_high=0):
"""
Window 2D image to FFT-friendly size, apply Butterworth low pass filter,
and decimate image by integer factor
"""
from filter import filt_btwl
from fundamentals import smallprime, window2d
from utilities import get_image
if type(img) == str: img=get_image(img)
nz = img.get_zsize()
if( nz > 1): ERROR("This command works only for 2-D images", "image_decimate", 1)
if decimation <= 1 : ERROR("Improper decimation ratio", "image_decimate", 1)
if(decimation == 1.0): return img.copy()
if frequency_low <= 0 :
frequency_low = 0.5/decimation-0.02
if frequency_low <= 0 : ERROR("Butterworth passband frequency is too low","image_decimation",1)
frequency_high = min(0.5/decimation + 0.02, 0.499)
if fit_to_fft:
nx = img.get_xsize()
ny = img.get_ysize()
nx_fft_m = smallprime(nx)
ny_fft_m = smallprime(ny)
e = Util.window(img, nx_fft_m, ny_fft_m, 1, 0,0,0)
e = filt_btwl(e, frequency_low, frequency_high)
else:
e = filt_btwl(img, frequency_low, frequency_high)
return Util.decimate(e, int(decimation), int(decimation), 1)
def resample(img, sub_rate=0.5):
"""
resample image based on the value of sub_rate.
the input image can be either 2D image or 3D volume.
sub_rate < 1.0, subsampling the image.
sub_rate > 1.0, upsampling the image using new gridding interpolation.
fit_to_fft will change the ouput image size to an fft_friendly size
"""
from fundamentals import subsample
from utilities import get_pixel_size, set_pixel_size
if type(img) == str:
from utilities import get_image
img = get_image(img)
nx = img.get_xsize()
ny = img.get_ysize()
nz = img.get_zsize()
if( ny == 1): ERROR("Only 2D or 3D images allowed","resample",1)
if sub_rate == 1.0: return img.copy()
elif sub_rate < 1.0:
e = subsample(img, sub_rate)
else: # sub_rate>1
new_nx = int(nx*sub_rate+0.5)
new_ny = int(ny*sub_rate+0.5)
if nz==1:
new_nz = 1
else:
new_nz = int(ny*sub_rate+0.5)
if ( nx!=ny and nz==1 ):
nn = max(new_nx, new_ny)
e = Util.pad(img, nn, nn, 1, 0, 0, 0, "circumference")
e, kb = prepi(e)
e = Util.window( e.rot_scale_conv_new(0.0, 0.0, 0.0, kb, sub_rate), new_nx, new_ny, 1, 0,0,0)
elif ((nx!=ny or nx!=nz or ny!=nz) and nz>1):
nn = max(new_nx, new_ny,new_nz)
e = Util.pad(img, nn, nn, nn, 0, 0, 0, "circumference")
e, kb = prepi3D(e)
e = Util.window( e.rot_scale_conv_new_3D(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, kb, sub_rate), new_nx, new_ny, new_nz, 0,0,0)
else:
if nz==1:
e, kb = prepi(Util.pad(img, new_nx, new_ny, 1, 0, 0, 0, "circumference"))
e = e.rot_scale_conv_new(0.0, 0.0, 0.0, kb, sub_rate)
else:
e, kb = prepi3D(Util.pad(img, new_nx, new_ny, new_nz, 0, 0, 0, "circumference"))
e = e.rot_scale_conv_new_3D(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, kb, sub_rate)
# Automatically adjust pixel size for ctf parameters
from utilities import get_pixel_size, set_pixel_size
apix = get_pixel_size(e)
apix /= sub_rate
set_pixel_size(e, apix)
return e
def draw_actors(self, screen, screen_width, screen_height, z_level):
actors_to_draw = []
for each in self.entity_group["Avatar"]:
if each.height >= z_level:
actors_to_draw.append(each)
for each in self.entity_group["Npc"]:
if each.height >= z_level:
actors_to_draw.append(each)
for each in self.entity_group["Creature"]:
if each.height >= z_level:
actors_to_draw.append(each)
for each in self.entity_group["Projectile"]:
actors_to_draw.append(each)
rows_to_draw = []
for y_level in range(int(self.height / 20)):
this_row = []
for entity in actors_to_draw:
if entity.tile_y == y_level:
this_row.append(entity)
rows_to_draw.append(this_row)
for row in rows_to_draw:
for entity in row:
if utilities.any_tile_visible(screen_width, screen_height, self.x_shift, self.y_shift, entity):
# screen.blit(entity.sprite.image, [(entity.sprite.rect.x + self.x_shift), (entity.sprite.rect.y + self.y_shift)])
image_height = entity.height * 20
image_slice = utilities.get_image(entity.sprite.image, 0, image_height - (z_level * 20), entity.width * 20, 20)
screen.blit(image_slice, [entity.sprite.rect.x + self.x_shift, entity.sprite.rect.y - (z_level - 2) * 20 + self.y_shift])
if entity.my_type == "Avatar" or entity.my_type == "Npc" or entity.my_type == "Creature":
if entity.equipped["Helmet"]:
screen.blit(entity.equipped["Helmet"].sprite.image,
[entity.sprite.rect.x + self.x_shift, entity.sprite.rect.y + self.y_shift])
if entity.equipped["Gloves"]:
screen.blit(entity.equipped["Gloves"].sprite.image, [entity.sprite.rect.x + self.x_shift,
entity.sprite.rect.y + self.y_shift])
if entity.equipped["Body Armor"]:
screen.blit(entity.equipped["Body Armor"].sprite.image,
[entity.sprite.rect.x + self.x_shift, entity.sprite.rect.y + self.y_shift])
if entity.equipped["Boots"]:
screen.blit(entity.equipped["Boots"].sprite.image,
[entity.sprite.rect.x + self.x_shift, entity.sprite.rect.y + self.y_shift])
if entity.equipped["Weapon"]:
screen.blit(entity.equipped["Weapon"].sprite.image,
[(entity.sprite.rect.x + self.x_shift - 1),
(entity.sprite.rect.y + self.y_shift - 42)])
def main():
def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
if mirror:
m = 1
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
540.0 - psi, 0, 0, 1.0)
else:
m = 0
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
360.0 - psi, 0, 0, 1.0)
return alpha, sx, sy, m
progname = os.path.basename(sys.argv[0])
usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl=15. --aa=0.01 --sym=symmetry --CTF"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--output_dir",
type="string",
default="./",
help="output directory")
parser.add_option("--ave2D",
type="string",
default=False,
help="write to the disk a stack of 2D averages")
parser.add_option("--var2D",
type="string",
default=False,
help="write to the disk a stack of 2D variances")
parser.add_option("--ave3D",
type="string",
default=False,
help="write to the disk reconstructed 3D average")
parser.add_option("--var3D",
type="string",
default=False,
help="compute 3D variability (time consuming!)")
parser.add_option("--img_per_grp",
type="int",
default=10,
help="number of neighbouring projections")
parser.add_option("--no_norm",
action="store_true",
default=False,
help="do not use normalization")
#parser.add_option("--radius", type="int" , default=-1 , help="radius for 3D variability" )
parser.add_option("--npad",
type="int",
default=2,
help="number of time to pad the original images")
parser.add_option("--sym", type="string", default="c1", help="symmetry")
parser.add_option(
"--fl",
type="float",
default=0.0,
help=
"cutoff freqency in absolute frequency (0.0-0.5). (Default - no filtration)"
)
parser.add_option(
"--aa",
type="float",
default=0.0,
help=
"fall off of the filter. Put 0.01 if user has no clue about falloff (Default - no filtration)"
)
parser.add_option("--CTF",
action="store_true",
default=False,
help="use CFT correction")
parser.add_option("--VERBOSE",
action="store_true",
default=False,
help="Long output for debugging")
#parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
#parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" )
#parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" )
#parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
#parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
parser.add_option(
"--VAR",
action="store_true",
default=False,
help="stack on input consists of 2D variances (Default False)")
parser.add_option(
"--decimate",
type="float",
default=1.0,
help=
"image decimate rate, a number larger (expand image) or less (shrink image) than 1. default is 1"
)
parser.add_option(
"--window",
type="int",
default=0,
help=
"reduce images to a small image size without changing pixel_size. Default value is zero."
)
#parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)")
parser.add_option(
"--nvec",
type="int",
default=0,
help="number of eigenvectors, default = 0 meaning no PCA calculated")
parser.add_option(
"--symmetrize",
action="store_true",
default=False,
help="Prepare input stack for handling symmetry (Default False)")
(options, args) = parser.parse_args()
#####
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD
from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
from applications import MPI_start_end
from reconstruction import recons3d_em, recons3d_em_MPI
from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
from utilities import print_begin_msg, print_end_msg, print_msg
from utilities import read_text_row, get_image, get_im
from utilities import bcast_EMData_to_all, bcast_number_to_all
from utilities import get_symt
# This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015
from EMAN2db import db_open_dict
# Set up global variables related to bdb cache
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
# Set up global variables related to ERROR function
global_def.BATCH = True
# detect if program is running under MPI
RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ
if RUNNING_UNDER_MPI:
global_def.MPI = True
if options.symmetrize:
if RUNNING_UNDER_MPI:
try:
sys.argv = mpi_init(len(sys.argv), sys.argv)
try:
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
if (number_of_proc > 1):
ERROR(
"Cannot use more than one CPU for symmetry prepration",
"sx3dvariability", 1)
except:
pass
except:
pass
if options.output_dir != "./" and not os.path.exists(
options.output_dir):
os.mkdir(options.output_dir)
# Input
#instack = "Clean_NORM_CTF_start_wparams.hdf"
#instack = "bdb:data"
from logger import Logger, BaseLogger_Files
if os.path.exists(os.path.join(options.output_dir, "log.txt")):
os.remove(os.path.join(options.output_dir, "log.txt"))
log_main = Logger(BaseLogger_Files())
log_main.prefix = os.path.join(options.output_dir, "./")
instack = args[0]
sym = options.sym.lower()
if (sym == "c1"):
ERROR("There is no need to symmetrize stack for C1 symmetry",
"sx3dvariability", 1)
line = ""
for a in sys.argv:
line += " " + a
log_main.add(line)
if (instack[:4] != "bdb:"):
if output_dir == "./": stack = "bdb:data"
else: stack = "bdb:" + options.output_dir + "/data"
delete_bdb(stack)
junk = cmdexecute("sxcpy.py " + instack + " " + stack)
else:
stack = instack
qt = EMUtil.get_all_attributes(stack, 'xform.projection')
na = len(qt)
ts = get_symt(sym)
ks = len(ts)
angsa = [None] * na
for k in xrange(ks):
#Qfile = "Q%1d"%k
if options.output_dir != "./":
Qfile = os.path.join(options.output_dir, "Q%1d" % k)
else:
Qfile = os.path.join(options.output_dir, "Q%1d" % k)
#delete_bdb("bdb:Q%1d"%k)
delete_bdb("bdb:" + Qfile)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
junk = cmdexecute("e2bdb.py " + stack + " --makevstack=bdb:" +
Qfile)
#DB = db_open_dict("bdb:Q%1d"%k)
DB = db_open_dict("bdb:" + Qfile)
for i in xrange(na):
ut = qt[i] * ts[k]
DB.set_attr(i, "xform.projection", ut)
#bt = ut.get_params("spider")
#angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
#write_text_row(angsa, 'ptsma%1d.txt'%k)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
#junk = cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k))
DB.close()
if options.output_dir == "./": delete_bdb("bdb:sdata")
else: delete_bdb("bdb:" + options.output_dir + "/" + "sdata")
#junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
sdata = "bdb:" + options.output_dir + "/" + "sdata"
print(sdata)
junk = cmdexecute("e2bdb.py " + options.output_dir +
" --makevstack=" + sdata + " --filt=Q")
#junk = cmdexecute("ls EMAN2DB/sdata*")
#a = get_im("bdb:sdata")
a = get_im(sdata)
a.set_attr("variabilitysymmetry", sym)
#a.write_image("bdb:sdata")
a.write_image(sdata)
else:
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
if len(args) == 1:
stack = args[0]
else:
print(("usage: " + usage))
print(("Please run '" + progname + " -h' for detailed options"))
return 1
t0 = time()
# obsolete flags
options.MPI = True
options.nvec = 0
options.radiuspca = -1
options.iter = 40
options.abs = 0.0
options.squ = 0.0
if options.fl > 0.0 and options.aa == 0.0:
ERROR("Fall off has to be given for the low-pass filter",
"sx3dvariability", 1, myid)
if options.VAR and options.SND:
ERROR("Only one of var and SND can be set!", "sx3dvariability",
myid)
exit()
if options.VAR and (options.ave2D or options.ave3D or options.var2D):
ERROR(
"When VAR is set, the program cannot output ave2D, ave3D or var2D",
"sx3dvariability", 1, myid)
exit()
#if options.SND and (options.ave2D or options.ave3D):
# ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid)
# exit()
if options.nvec > 0:
ERROR("PCA option not implemented", "sx3dvariability", 1, myid)
exit()
if options.nvec > 0 and options.ave3D == None:
ERROR("When doing PCA analysis, one must set ave3D",
"sx3dvariability",
myid=myid)
exit()
import string
options.sym = options.sym.lower()
# if global_def.CACHE_DISABLE:
# from utilities import disable_bdb_cache
# disable_bdb_cache()
# global_def.BATCH = True
if myid == main_node:
if options.output_dir != "./" and not os.path.exists(
options.output_dir):
os.mkdir(options.output_dir)
img_per_grp = options.img_per_grp
nvec = options.nvec
radiuspca = options.radiuspca
from logger import Logger, BaseLogger_Files
#if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt"))
log_main = Logger(BaseLogger_Files())
log_main.prefix = os.path.join(options.output_dir, "./")
if myid == main_node:
line = ""
for a in sys.argv:
line += " " + a
log_main.add(line)
log_main.add("-------->>>Settings given by all options<<<-------")
log_main.add("instack :" + stack)
log_main.add("output_dir :" + options.output_dir)
log_main.add("var3d :" + options.var3D)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
#print_begin_msg("sx3dvariability")
msg = "sx3dvariability"
log_main.add(msg)
print(line, msg)
msg = ("%-70s: %s\n" % ("Input stack", stack))
log_main.add(msg)
print(line, msg)
symbaselen = 0
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
if options.sym != "c1":
imgdata = get_im(stack)
try:
i = imgdata.get_attr("variabilitysymmetry").lower()
if (i != options.sym):
ERROR(
"The symmetry provided does not agree with the symmetry of the input stack",
"sx3dvariability",
myid=myid)
except:
ERROR(
"Input stack is not prepared for symmetry, please follow instructions",
"sx3dvariability",
myid=myid)
from utilities import get_symt
i = len(get_symt(options.sym))
if ((nima / i) * i != nima):
ERROR(
"The length of the input stack is incorrect for symmetry processing",
"sx3dvariability",
myid=myid)
symbaselen = nima / i
else:
symbaselen = nima
else:
nima = 0
nx = 0
ny = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
Tracker = {}
Tracker["total_stack"] = nima
if options.decimate == 1.:
if options.window != 0:
nx = options.window
ny = options.window
else:
if options.window == 0:
nx = int(nx * options.decimate)
ny = int(ny * options.decimate)
else:
nx = int(options.window * options.decimate)
ny = nx
Tracker["nx"] = nx
Tracker["ny"] = ny
Tracker["nz"] = nx
symbaselen = bcast_number_to_all(symbaselen)
if radiuspca == -1: radiuspca = nx / 2 - 2
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = "%-70s: %d\n" % ("Number of projection", nima)
log_main.add(msg)
print(line, msg)
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
"""
if options.SND:
from projection import prep_vol, prgs
from statistics import im_diff
from utilities import get_im, model_circle, get_params_proj, set_params_proj
from utilities import get_ctf, generate_ctf
from filter import filt_ctf
imgdata = EMData.read_images(stack, range(img_begin, img_end))
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
bcast_EMData_to_all(vol, myid)
volft, kb = prep_vol(vol)
mask = model_circle(nx/2-2, nx, ny)
varList = []
for i in xrange(img_begin, img_end):
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
if options.CTF:
ctf_params = get_ctf(imgdata[i-img_begin])
ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
diff2 = diff*diff
set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
varList.append(diff2)
mpi_barrier(MPI_COMM_WORLD)
"""
if options.VAR:
#varList = EMData.read_images(stack, range(img_begin, img_end))
varList = []
this_image = EMData()
for index_of_particle in xrange(img_begin, img_end):
this_image.read_image(stack, index_of_particle)
varList.append(
image_decimate_window_xform_ctf(this_image,
options.decimate,
options.window,
options.CTF))
else:
from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
from utilities import model_blank, nearest_proj, model_circle
from applications import pca
from statistics import avgvar, avgvar_ctf, ccc
from filter import filt_tanl
from morphology import threshold, square_root
from projection import project, prep_vol, prgs
from sets import Set
if myid == main_node:
t1 = time()
proj_angles = []
aveList = []
tab = EMUtil.get_all_attributes(stack, 'xform.projection')
for i in xrange(nima):
t = tab[i].get_params('spider')
phi = t['phi']
theta = t['theta']
psi = t['psi']
x = theta
if x > 90.0: x = 180.0 - x
x = x * 10000 + psi
proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
t2 = time()
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = "%-70s: %d\n" % ("Number of neighboring projections",
img_per_grp)
log_main.add(msg)
print(line, msg)
msg = "...... Finding neighboring projections\n"
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
msg = "Number of images per group: %d" % img_per_grp
log_main.add(msg)
print(line, msg)
msg = "Now grouping projections"
log_main.add(msg)
print(line, msg)
proj_angles.sort()
proj_angles_list = [0.0] * (nima * 4)
if myid == main_node:
for i in xrange(nima):
proj_angles_list[i * 4] = proj_angles[i][1]
proj_angles_list[i * 4 + 1] = proj_angles[i][2]
proj_angles_list[i * 4 + 2] = proj_angles[i][3]
proj_angles_list[i * 4 + 3] = proj_angles[i][4]
proj_angles_list = bcast_list_to_all(proj_angles_list, myid,
main_node)
proj_angles = []
for i in xrange(nima):
proj_angles.append([
proj_angles_list[i * 4], proj_angles_list[i * 4 + 1],
proj_angles_list[i * 4 + 2],
int(proj_angles_list[i * 4 + 3])
])
del proj_angles_list
proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp,
range(img_begin, img_end))
all_proj = Set()
for im in proj_list:
for jm in im:
all_proj.add(proj_angles[jm][3])
all_proj = list(all_proj)
if options.VERBOSE:
print("On node %2d, number of images needed to be read = %5d" %
(myid, len(all_proj)))
index = {}
for i in xrange(len(all_proj)):
index[all_proj[i]] = i
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("%-70s: %.2f\n" %
("Finding neighboring projections lasted [s]",
time() - t2))
log_main.add(msg)
print(msg)
msg = ("%-70s: %d\n" %
("Number of groups processed on the main node",
len(proj_list)))
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
print("Grouping projections took: ", (time() - t2) / 60,
"[min]")
print("Number of groups on main node: ", len(proj_list))
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("...... calculating the stack of 2D variances \n")
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
print("Now calculating the stack of 2D variances")
proj_params = [0.0] * (nima * 5)
aveList = []
varList = []
if nvec > 0:
eigList = [[] for i in xrange(nvec)]
if options.VERBOSE:
print("Begin to read images on processor %d" % (myid))
ttt = time()
#imgdata = EMData.read_images(stack, all_proj)
imgdata = []
for index_of_proj in xrange(len(all_proj)):
#img = EMData()
#img.read_image(stack, all_proj[index_of_proj])
dmg = image_decimate_window_xform_ctf(
get_im(stack, all_proj[index_of_proj]), options.decimate,
options.window, options.CTF)
#print dmg.get_xsize(), "init"
imgdata.append(dmg)
if options.VERBOSE:
print("Reading images on processor %d done, time = %.2f" %
(myid, time() - ttt))
print("On processor %d, we got %d images" %
(myid, len(imgdata)))
mpi_barrier(MPI_COMM_WORLD)
'''
imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
if options.fl > 0.0:
for k in xrange(len(imgdata2)):
imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
if myid == main_node:
vol.write_image("vol_ctf.hdf")
print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf"))
del vol, imgdata2
mpi_barrier(MPI_COMM_WORLD)
'''
from applications import prepare_2d_forPCA
from utilities import model_blank
for i in xrange(len(proj_list)):
ki = proj_angles[proj_list[i][0]][3]
if ki >= symbaselen: continue
mi = index[ki]
phiM, thetaM, psiM, s2xM, s2yM = get_params_proj(imgdata[mi])
grp_imgdata = []
for j in xrange(img_per_grp):
mj = index[proj_angles[proj_list[i][j]][3]]
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[mj])
alpha, sx, sy, mirror = params_3D_2D_NEW(
phi, theta, psi, s2x, s2y, mirror_list[i][j])
if thetaM <= 90:
if mirror == 0:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, phiM - phi, 0.0, 0.0, 1.0)
else:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, 180 - (phiM - phi), 0.0,
0.0, 1.0)
else:
if mirror == 0:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, -(phiM - phi), 0.0, 0.0,
1.0)
else:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, -(180 - (phiM - phi)), 0.0,
0.0, 1.0)
set_params2D(imgdata[mj], [alpha, sx, sy, mirror, 1.0])
grp_imgdata.append(imgdata[mj])
#print grp_imgdata[j].get_xsize(), imgdata[mj].get_xsize()
if not options.no_norm:
#print grp_imgdata[j].get_xsize()
mask = model_circle(nx / 2 - 2, nx, nx)
for k in xrange(img_per_grp):
ave, std, minn, maxx = Util.infomask(
grp_imgdata[k], mask, False)
grp_imgdata[k] -= ave
grp_imgdata[k] /= std
del mask
if options.fl > 0.0:
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2 * nx
ny2 = 2 * ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d(
fft(
filt_tanl(
filt_ctf(
fft(
pad(grp_imgdata[k], nx2, ny2,
1, 0.0)),
grp_imgdata[k].get_attr("ctf"),
binary=1), options.fl,
options.aa)), nx, ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
for k in xrange(img_per_grp):
grp_imgdata[k] = filt_tanl(grp_imgdata[k],
options.fl, options.aa)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
from utilities import pad, read_text_file
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2 * nx
ny2 = 2 * ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d(
fft(
filt_ctf(fft(
pad(grp_imgdata[k], nx2, ny2, 1, 0.0)),
grp_imgdata[k].get_attr("ctf"),
binary=1)), nx, ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
'''
if i < 10 and myid == main_node:
for k in xrange(10):
grp_imgdata[k].write_image("grp%03d.hdf"%i, k)
'''
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("pp.hdf", pp)
"""
ave, grp_imgdata = prepare_2d_forPCA(grp_imgdata)
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("qq.hdf", pp)
"""
var = model_blank(nx, ny)
for q in grp_imgdata:
Util.add_img2(var, q)
Util.mul_scalar(var, 1.0 / (len(grp_imgdata) - 1))
# Switch to std dev
var = square_root(threshold(var))
#if options.CTF: ave, var = avgvar_ctf(grp_imgdata, mode="a")
#else: ave, var = avgvar(grp_imgdata, mode="a")
"""
if myid == main_node:
ave.write_image("avgv.hdf",i)
var.write_image("varv.hdf",i)
"""
set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])
aveList.append(ave)
varList.append(var)
if options.VERBOSE:
print("%5.2f%% done on processor %d" %
(i * 100.0 / len(proj_list), myid))
if nvec > 0:
eig = pca(input_stacks=grp_imgdata,
subavg="",
mask_radius=radiuspca,
nvec=nvec,
incore=True,
shuffle=False,
genbuf=True)
for k in xrange(nvec):
set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
eigList[k].append(eig[k])
"""
if myid == 0 and i == 0:
for k in xrange(nvec):
eig[k].write_image("eig.hdf", k)
"""
del imgdata
# To this point, all averages, variances, and eigenvectors are computed
if options.ave2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node:
for im in xrange(len(aveList)):
aveList[im].write_image(
os.path.join(options.output_dir,
options.ave2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i,
SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im + i + 70000)
"""
nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('pix_err', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
"""
tmpvol = fpol(ave, Tracker["nx"],
Tracker["nx"], 1)
tmpvol.write_image(
os.path.join(options.output_dir,
options.ave2D), km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in xrange(len(aveList)):
send_EMData(aveList[im], main_node, im + myid + 70000)
"""
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
members = aveList[im].get_attr('pix_err')
mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
except:
mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
"""
if options.ave3D:
from fundamentals import fpol
if options.VERBOSE:
print("Reconstructing 3D average volume")
ave3D = recons3d_4nn_MPI(myid,
aveList,
symmetry=options.sym,
npad=options.npad)
bcast_EMData_to_all(ave3D, myid)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
ave3D = fpol(ave3D, Tracker["nx"], Tracker["nx"],
Tracker["nx"])
ave3D.write_image(
os.path.join(options.output_dir, options.ave3D))
msg = ("%-70s: %s\n" % (
"Writing to the disk volume reconstructed from averages as",
options.ave3D))
log_main.add(msg)
print(line, msg)
del ave, var, proj_list, stack, phi, theta, psi, s2x, s2y, alpha, sx, sy, mirror, aveList
if nvec > 0:
for k in xrange(nvec):
if options.VERBOSE:
print("Reconstruction eigenvolumes", k)
cont = True
ITER = 0
mask2d = model_circle(radiuspca, nx, nx)
while cont:
#print "On node %d, iteration %d"%(myid, ITER)
eig3D = recons3d_4nn_MPI(myid,
eigList[k],
symmetry=options.sym,
npad=options.npad)
bcast_EMData_to_all(eig3D, myid, main_node)
if options.fl > 0.0:
eig3D = filt_tanl(eig3D, options.fl, options.aa)
if myid == main_node:
eig3D.write_image(
os.path.join(options.outpout_dir,
"eig3d_%03d.hdf" % (k, ITER)))
Util.mul_img(eig3D,
model_circle(radiuspca, nx, nx, nx))
eig3Df, kb = prep_vol(eig3D)
del eig3D
cont = False
icont = 0
for l in xrange(len(eigList[k])):
phi, theta, psi, s2x, s2y = get_params_proj(
eigList[k][l])
proj = prgs(eig3Df, kb,
[phi, theta, psi, s2x, s2y])
cl = ccc(proj, eigList[k][l], mask2d)
if cl < 0.0:
icont += 1
cont = True
eigList[k][l] *= -1.0
u = int(cont)
u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node,
MPI_COMM_WORLD)
icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM,
main_node, MPI_COMM_WORLD)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S",
localtime()) + " =>"
u = int(u[0])
msg = (" Eigenvector: ", k, " number changed ",
int(icont[0]))
log_main.add(msg)
print(line, msg)
else:
u = 0
u = bcast_number_to_all(u, main_node)
cont = bool(u)
ITER += 1
del eig3Df, kb
mpi_barrier(MPI_COMM_WORLD)
del eigList, mask2d
if options.ave3D: del ave3D
if options.var2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node:
for im in xrange(len(varList)):
tmpvol = fpol(varList[im], Tracker["nx"],
Tracker["nx"], 1)
tmpvol.write_image(
os.path.join(options.output_dir,
options.var2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i,
SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im + i + 70000)
tmpvol = fpol(ave, Tracker["nx"],
Tracker["nx"], 1)
tmpvol.write_image(
os.path.join(options.output_dir,
options.var2D, km))
km += 1
else:
mpi_send(len(varList), 1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in xrange(len(varList)):
send_EMData(varList[im], main_node, im + myid +
70000) # What with the attributes??
mpi_barrier(MPI_COMM_WORLD)
if options.var3D:
if myid == main_node and options.VERBOSE:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("Reconstructing 3D variability volume")
log_main.add(msg)
print(line, msg)
t6 = time()
# radiusvar = options.radius
# if( radiusvar < 0 ): radiusvar = nx//2 -3
res = recons3d_4nn_MPI(myid,
varList,
symmetry=options.sym,
npad=options.npad)
#res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
if myid == main_node:
from fundamentals import fpol
res = fpol(res, Tracker["nx"], Tracker["nx"], Tracker["nx"])
res.write_image(os.path.join(options.output_dir,
options.var3D))
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("%-70s: %.2f\n" %
("Reconstructing 3D variability took [s]", time() - t6))
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
print("Reconstruction took: %.2f [min]" %
((time() - t6) / 60))
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("%-70s: %.2f\n" %
("Total time for these computations [s]", time() - t0))
print(line, msg)
log_main.add(msg)
if options.VERBOSE:
print("Total time for these computations: %.2f [min]" %
((time() - t0) / 60))
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("sx3dvariability")
print(line, msg)
log_main.add(msg)
from mpi import mpi_finalize
mpi_finalize()
if RUNNING_UNDER_MPI:
global_def.MPI = False
global_def.BATCH = False
def main():
arglist = []
for arg in sys.argv:
arglist.append(arg)
progname = os.path.basename(arglist[0])
usage = (
progname
+ " prj_stack volume fsc_curve <mask> --CTF --snr=signal_noise_ratio --list=file --group=ID --sym=symmetry -verbose=(0|1) --MPI"
)
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--CTF", action="store_true", default=False, help="perform ctf correction")
parser.add_option("--snr", type="float", default=1.0, help="Signal-to-Noise Ratio in the data")
parser.add_option("--sym", type="string", default="c1", help="symmetry")
parser.add_option("--list", type="string", help="file with list of images to be used in the first column")
parser.add_option(
"--group",
type="int",
default=-1,
help="perform reconstruction using images for a given group number (group is attribute in the header)",
)
parser.add_option("--MPI", action="store_true", default=False, help="use MPI version ")
parser.add_option("--npad", type="int", default=2, help="number of times padding (default 2)")
parser.add_option("--verbose", type="int", default=0, help="verbose level: 0 no, 1 yes")
(options, args) = parser.parse_args(arglist[1:])
if len(args) != 3 and len(args) != 4:
print usage
sys.exit(-1)
prj_stack = args[0]
vol_stack = args[1]
fsc_curve = args[2]
if len(args) == 3:
mask = None
else:
mask = get_image(args[3])
if options.MPI:
from mpi import mpi_init
sys.argv = mpi_init(len(sys.argv), sys.argv)
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
if options.list and options.group > -1:
ERROR("options group and list cannot be used together", "recon3d_n", 1)
sys.exit()
from applications import recons3d_f
global_def.BATCH = True
recons3d_f(
prj_stack,
vol_stack,
fsc_curve,
mask,
options.CTF,
options.snr,
options.sym,
options.list,
options.group,
options.npad,
options.verbose,
options.MPI,
)
global_def.BATCH = False
if options.MPI:
from mpi import mpi_finalize
mpi_finalize()
def main():
def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
# the final ali2d parameters already combine shifts operation first and rotation operation second for parameters converted from 3D
if mirror:
m = 1
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 540.0-psi, 0, 0, 1.0)
else:
m = 0
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 360.0-psi, 0, 0, 1.0)
return alpha, sx, sy, m
progname = os.path.basename(sys.argv[0])
usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl= --aa= --sym=symmetry --CTF"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--output_dir", type="string" , default="./", help="Output directory")
parser.add_option("--ave2D", type="string" , default=False, help="Write to the disk a stack of 2D averages")
parser.add_option("--var2D", type="string" , default=False, help="Write to the disk a stack of 2D variances")
parser.add_option("--ave3D", type="string" , default=False, help="Write to the disk reconstructed 3D average")
parser.add_option("--var3D", type="string" , default=False, help="Compute 3D variability (time consuming!)")
parser.add_option("--img_per_grp", type="int" , default=100, help="Number of neighbouring projections.(Default is 100)")
parser.add_option("--no_norm", action="store_true", default=False, help="Do not use normalization.(Default is to apply normalization)")
#parser.add_option("--radius", type="int" , default=-1 , help="radius for 3D variability" )
parser.add_option("--npad", type="int" , default=2 , help="Number of time to pad the original images.(Default is 2 times padding)")
parser.add_option("--sym" , type="string" , default="c1", help="Symmetry. (Default is no symmetry)")
parser.add_option("--fl", type="float" , default=0.0, help="Low pass filter cutoff in absolute frequency (0.0 - 0.5) and is applied to decimated images. (Default - no filtration)")
parser.add_option("--aa", type="float" , default=0.02 , help="Fall off of the filter. Use default value if user has no clue about falloff (Default value is 0.02)")
parser.add_option("--CTF", action="store_true", default=False, help="Use CFT correction.(Default is no CTF correction)")
#parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
#parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" )
#parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" )
#parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
#parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
parser.add_option("--VAR" , action="store_true", default=False, help="Stack of input consists of 2D variances (Default False)")
parser.add_option("--decimate", type ="float", default=0.25, help="Image decimate rate, a number less than 1. (Default is 0.25)")
parser.add_option("--window", type ="int", default=0, help="Target image size relative to original image size. (Default value is zero.)")
#parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)")
#parser.add_option("--nvec", type="int" , default=0 , help="Number of eigenvectors, (Default = 0 meaning no PCA calculated)")
parser.add_option("--symmetrize", action="store_true", default=False, help="Prepare input stack for handling symmetry (Default False)")
parser.add_option("--overhead", type ="float", default=0.5, help="python overhead per CPU.")
(options,args) = parser.parse_args()
#####
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD
from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
#from mpi import *
from applications import MPI_start_end
from reconstruction import recons3d_em, recons3d_em_MPI
from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
from utilities import print_begin_msg, print_end_msg, print_msg
from utilities import read_text_row, get_image, get_im, wrap_mpi_send, wrap_mpi_recv
from utilities import bcast_EMData_to_all, bcast_number_to_all
from utilities import get_symt
# This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015
from EMAN2db import db_open_dict
# Set up global variables related to bdb cache
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
# Set up global variables related to ERROR function
global_def.BATCH = True
# detect if program is running under MPI
RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ
if RUNNING_UNDER_MPI: global_def.MPI = True
if options.output_dir =="./": current_output_dir = os.path.abspath(options.output_dir)
else: current_output_dir = options.output_dir
if options.symmetrize :
if RUNNING_UNDER_MPI:
try:
sys.argv = mpi_init(len(sys.argv), sys.argv)
try:
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
if( number_of_proc > 1 ):
ERROR("Cannot use more than one CPU for symmetry preparation","sx3dvariability",1)
except:
pass
except:
pass
if not os.path.exists(current_output_dir): os.mkdir(current_output_dir)
# Input
#instack = "Clean_NORM_CTF_start_wparams.hdf"
#instack = "bdb:data"
from logger import Logger,BaseLogger_Files
if os.path.exists(os.path.join(current_output_dir, "log.txt")): os.remove(os.path.join(current_output_dir, "log.txt"))
log_main=Logger(BaseLogger_Files())
log_main.prefix = os.path.join(current_output_dir, "./")
instack = args[0]
sym = options.sym.lower()
if( sym == "c1" ):
ERROR("There is no need to symmetrize stack for C1 symmetry","sx3dvariability",1)
line =""
for a in sys.argv:
line +=" "+a
log_main.add(line)
if(instack[:4] !="bdb:"):
#if output_dir =="./": stack = "bdb:data"
stack = "bdb:"+current_output_dir+"/data"
delete_bdb(stack)
junk = cmdexecute("sxcpy.py "+instack+" "+stack)
else: stack = instack
qt = EMUtil.get_all_attributes(stack,'xform.projection')
na = len(qt)
ts = get_symt(sym)
ks = len(ts)
angsa = [None]*na
for k in range(ks):
#Qfile = "Q%1d"%k
#if options.output_dir!="./": Qfile = os.path.join(options.output_dir,"Q%1d"%k)
Qfile = os.path.join(current_output_dir, "Q%1d"%k)
#delete_bdb("bdb:Q%1d"%k)
delete_bdb("bdb:"+Qfile)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:"+Qfile)
#DB = db_open_dict("bdb:Q%1d"%k)
DB = db_open_dict("bdb:"+Qfile)
for i in range(na):
ut = qt[i]*ts[k]
DB.set_attr(i, "xform.projection", ut)
#bt = ut.get_params("spider")
#angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
#write_text_row(angsa, 'ptsma%1d.txt'%k)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
#junk = cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k))
DB.close()
#if options.output_dir =="./": delete_bdb("bdb:sdata")
delete_bdb("bdb:" + current_output_dir + "/"+"sdata")
#junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
sdata = "bdb:"+current_output_dir+"/"+"sdata"
print(sdata)
junk = cmdexecute("e2bdb.py " + current_output_dir +" --makevstack="+sdata +" --filt=Q")
#junk = cmdexecute("ls EMAN2DB/sdata*")
#a = get_im("bdb:sdata")
a = get_im(sdata)
a.set_attr("variabilitysymmetry",sym)
#a.write_image("bdb:sdata")
a.write_image(sdata)
else:
from fundamentals import window2d
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
shared_comm = mpi_comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL)
myid_on_node = mpi_comm_rank(shared_comm)
no_of_processes_per_group = mpi_comm_size(shared_comm)
masters_from_groups_vs_everything_else_comm = mpi_comm_split(MPI_COMM_WORLD, main_node == myid_on_node, myid_on_node)
color, no_of_groups, balanced_processor_load_on_nodes = get_colors_and_subsets(main_node, MPI_COMM_WORLD, myid, \
shared_comm, myid_on_node, masters_from_groups_vs_everything_else_comm)
overhead_loading = options.overhead*number_of_proc
#memory_per_node = options.memory_per_node
#if memory_per_node == -1.: memory_per_node = 2.*no_of_processes_per_group
keepgoing = 1
current_window = options.window
current_decimate = options.decimate
if len(args) == 1: stack = args[0]
else:
print(( "usage: " + usage))
print(( "Please run '" + progname + " -h' for detailed options"))
return 1
t0 = time()
# obsolete flags
options.MPI = True
#options.nvec = 0
options.radiuspca = -1
options.iter = 40
options.abs = 0.0
options.squ = 0.0
if options.fl > 0.0 and options.aa == 0.0:
ERROR("Fall off has to be given for the low-pass filter", "sx3dvariability", 1, myid)
#if options.VAR and options.SND:
# ERROR("Only one of var and SND can be set!", "sx3dvariability", myid)
if options.VAR and (options.ave2D or options.ave3D or options.var2D):
ERROR("When VAR is set, the program cannot output ave2D, ave3D or var2D", "sx3dvariability", 1, myid)
#if options.SND and (options.ave2D or options.ave3D):
# ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid)
#if options.nvec > 0 :
# ERROR("PCA option not implemented", "sx3dvariability", 1, myid)
#if options.nvec > 0 and options.ave3D == None:
# ERROR("When doing PCA analysis, one must set ave3D", "sx3dvariability", 1, myid)
if current_decimate>1.0 or current_decimate<0.0:
ERROR("Decimate rate should be a value between 0.0 and 1.0", "sx3dvariability", 1, myid)
if current_window < 0.0:
ERROR("Target window size should be always larger than zero", "sx3dvariability", 1, myid)
if myid == main_node:
img = get_image(stack, 0)
nx = img.get_xsize()
ny = img.get_ysize()
if(min(nx, ny) < current_window): keepgoing = 0
keepgoing = bcast_number_to_all(keepgoing, main_node, MPI_COMM_WORLD)
if keepgoing == 0: ERROR("The target window size cannot be larger than the size of decimated image", "sx3dvariability", 1, myid)
import string
options.sym = options.sym.lower()
# if global_def.CACHE_DISABLE:
# from utilities import disable_bdb_cache
# disable_bdb_cache()
# global_def.BATCH = True
if myid == main_node:
if not os.path.exists(current_output_dir): os.mkdir(current_output_dir)# Never delete output_dir in the program!
img_per_grp = options.img_per_grp
#nvec = options.nvec
radiuspca = options.radiuspca
from logger import Logger,BaseLogger_Files
#if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt"))
log_main=Logger(BaseLogger_Files())
log_main.prefix = os.path.join(current_output_dir, "./")
if myid == main_node:
line = ""
for a in sys.argv: line +=" "+a
log_main.add(line)
log_main.add("-------->>>Settings given by all options<<<-------")
log_main.add("Symmetry : %s"%options.sym)
log_main.add("Input stack : %s"%stack)
log_main.add("Output_dir : %s"%current_output_dir)
if options.ave3D: log_main.add("Ave3d : %s"%options.ave3D)
if options.var3D: log_main.add("Var3d : %s"%options.var3D)
if options.ave2D: log_main.add("Ave2D : %s"%options.ave2D)
if options.var2D: log_main.add("Var2D : %s"%options.var2D)
if options.VAR: log_main.add("VAR : True")
else: log_main.add("VAR : False")
if options.CTF: log_main.add("CTF correction : True ")
else: log_main.add("CTF correction : False ")
log_main.add("Image per group : %5d"%options.img_per_grp)
log_main.add("Image decimate rate : %4.3f"%current_decimate)
log_main.add("Low pass filter : %4.3f"%options.fl)
current_fl = options.fl
if current_fl == 0.0: current_fl = 0.5
log_main.add("Current low pass filter is equivalent to cutoff frequency %4.3f for original image size"%round((current_fl*current_decimate),3))
log_main.add("Window size : %5d "%current_window)
log_main.add("sx3dvariability begins")
symbaselen = 0
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
nnxo = nx
nnyo = ny
if options.sym != "c1" :
imgdata = get_im(stack)
try:
i = imgdata.get_attr("variabilitysymmetry").lower()
if(i != options.sym):
ERROR("The symmetry provided does not agree with the symmetry of the input stack", "sx3dvariability", 1, myid)
except:
ERROR("Input stack is not prepared for symmetry, please follow instructions", "sx3dvariability", 1, myid)
from utilities import get_symt
i = len(get_symt(options.sym))
if((nima/i)*i != nima):
ERROR("The length of the input stack is incorrect for symmetry processing", "sx3dvariability", 1, myid)
symbaselen = nima/i
else: symbaselen = nima
else:
nima = 0
nx = 0
ny = 0
nnxo = 0
nnyo = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
nnxo = bcast_number_to_all(nnxo)
nnyo = bcast_number_to_all(nnyo)
if current_window > max(nx, ny):
ERROR("Window size is larger than the original image size", "sx3dvariability", 1)
if current_decimate == 1.:
if current_window !=0:
nx = current_window
ny = current_window
else:
if current_window == 0:
nx = int(nx*current_decimate+0.5)
ny = int(ny*current_decimate+0.5)
else:
nx = int(current_window*current_decimate+0.5)
ny = nx
symbaselen = bcast_number_to_all(symbaselen)
# check FFT prime number
from fundamentals import smallprime
is_fft_friendly = (nx == smallprime(nx))
if not is_fft_friendly:
if myid == main_node:
log_main.add("The target image size is not a product of small prime numbers")
log_main.add("Program adjusts the input settings!")
### two cases
if current_decimate == 1.:
nx = smallprime(nx)
ny = nx
current_window = nx # update
if myid == main_node:
log_main.add("The window size is updated to %d."%current_window)
else:
if current_window == 0:
nx = smallprime(int(nx*current_decimate+0.5))
current_decimate = float(nx)/nnxo
ny = nx
if (myid == main_node):
log_main.add("The decimate rate is updated to %f."%current_decimate)
else:
nx = smallprime(int(current_window*current_decimate+0.5))
ny = nx
current_window = int(nx/current_decimate+0.5)
if (myid == main_node):
log_main.add("The window size is updated to %d."%current_window)
if myid == main_node:
log_main.add("The target image size is %d"%nx)
if radiuspca == -1: radiuspca = nx/2-2
if myid == main_node: log_main.add("%-70s: %d\n"%("Number of projection", nima))
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
"""
if options.SND:
from projection import prep_vol, prgs
from statistics import im_diff
from utilities import get_im, model_circle, get_params_proj, set_params_proj
from utilities import get_ctf, generate_ctf
from filter import filt_ctf
imgdata = EMData.read_images(stack, range(img_begin, img_end))
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
bcast_EMData_to_all(vol, myid)
volft, kb = prep_vol(vol)
mask = model_circle(nx/2-2, nx, ny)
varList = []
for i in xrange(img_begin, img_end):
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
if options.CTF:
ctf_params = get_ctf(imgdata[i-img_begin])
ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
diff2 = diff*diff
set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
varList.append(diff2)
mpi_barrier(MPI_COMM_WORLD)
"""
if options.VAR: # 2D variance images have no shifts
#varList = EMData.read_images(stack, range(img_begin, img_end))
from EMAN2 import Region
for index_of_particle in range(img_begin,img_end):
image = get_im(stack, index_of_proj)
if current_window > 0: varList.append(fdecimate(window2d(image,current_window,current_window), nx,ny))
else: varList.append(fdecimate(image, nx,ny))
else:
from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
from utilities import model_blank, nearest_proj, model_circle, write_text_row, wrap_mpi_gatherv
from applications import pca
from statistics import avgvar, avgvar_ctf, ccc
from filter import filt_tanl
from morphology import threshold, square_root
from projection import project, prep_vol, prgs
from sets import Set
from utilities import wrap_mpi_recv, wrap_mpi_bcast, wrap_mpi_send
import numpy as np
if myid == main_node:
t1 = time()
proj_angles = []
aveList = []
tab = EMUtil.get_all_attributes(stack, 'xform.projection')
for i in range(nima):
t = tab[i].get_params('spider')
phi = t['phi']
theta = t['theta']
psi = t['psi']
x = theta
if x > 90.0: x = 180.0 - x
x = x*10000+psi
proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
t2 = time()
log_main.add( "%-70s: %d\n"%("Number of neighboring projections", img_per_grp))
log_main.add("...... Finding neighboring projections\n")
log_main.add( "Number of images per group: %d"%img_per_grp)
log_main.add( "Now grouping projections")
proj_angles.sort()
proj_angles_list = np.full((nima, 4), 0.0, dtype=np.float32)
for i in range(nima):
proj_angles_list[i][0] = proj_angles[i][1]
proj_angles_list[i][1] = proj_angles[i][2]
proj_angles_list[i][2] = proj_angles[i][3]
proj_angles_list[i][3] = proj_angles[i][4]
else: proj_angles_list = 0
proj_angles_list = wrap_mpi_bcast(proj_angles_list, main_node, MPI_COMM_WORLD)
proj_angles = []
for i in range(nima):
proj_angles.append([proj_angles_list[i][0], proj_angles_list[i][1], proj_angles_list[i][2], int(proj_angles_list[i][3])])
del proj_angles_list
proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp, range(img_begin, img_end))
all_proj = Set()
for im in proj_list:
for jm in im:
all_proj.add(proj_angles[jm][3])
all_proj = list(all_proj)
index = {}
for i in range(len(all_proj)): index[all_proj[i]] = i
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
log_main.add("%-70s: %.2f\n"%("Finding neighboring projections lasted [s]", time()-t2))
log_main.add("%-70s: %d\n"%("Number of groups processed on the main node", len(proj_list)))
log_main.add("Grouping projections took: %12.1f [m]"%((time()-t2)/60.))
log_main.add("Number of groups on main node: ", len(proj_list))
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
log_main.add("...... Calculating the stack of 2D variances \n")
# Memory estimation. There are two memory consumption peaks
# peak 1. Compute ave, var;
# peak 2. Var volume reconstruction;
# proj_params = [0.0]*(nima*5)
aveList = []
varList = []
#if nvec > 0: eigList = [[] for i in range(nvec)]
dnumber = len(all_proj)# all neighborhood set for assigned to myid
pnumber = len(proj_list)*2. + img_per_grp # aveList and varList
tnumber = dnumber+pnumber
vol_size2 = nx**3*4.*8/1.e9
vol_size1 = 2.*nnxo**3*4.*8/1.e9
proj_size = nnxo*nnyo*len(proj_list)*4.*2./1.e9 # both aveList and varList
orig_data_size = nnxo*nnyo*4.*tnumber/1.e9
reduced_data_size = nx*nx*4.*tnumber/1.e9
full_data = np.full((number_of_proc, 2), -1., dtype=np.float16)
full_data[myid] = orig_data_size, reduced_data_size
if myid != main_node: wrap_mpi_send(full_data, main_node, MPI_COMM_WORLD)
if myid == main_node:
for iproc in range(number_of_proc):
if iproc != main_node:
dummy = wrap_mpi_recv(iproc, MPI_COMM_WORLD)
full_data[np.where(dummy>-1)] = dummy[np.where(dummy>-1)]
del dummy
mpi_barrier(MPI_COMM_WORLD)
full_data = wrap_mpi_bcast(full_data, main_node, MPI_COMM_WORLD)
# find the CPU with heaviest load
minindx = np.argsort(full_data, 0)
heavy_load_myid = minindx[-1][1]
total_mem = sum(full_data)
if myid == main_node:
if current_window == 0:
log_main.add("Nx: current image size = %d. Decimated by %f from %d"%(nx, current_decimate, nnxo))
else:
log_main.add("Nx: current image size = %d. Windowed to %d, and decimated by %f from %d"%(nx, current_window, current_decimate, nnxo))
log_main.add("Nproj: number of particle images.")
log_main.add("Navg: number of 2D average images.")
log_main.add("Nvar: number of 2D variance images.")
log_main.add("Img_per_grp: user defined image per group for averaging = %d"%img_per_grp)
log_main.add("Overhead: total python overhead memory consumption = %f"%overhead_loading)
log_main.add("Total memory) = 4.0*nx^2*(nproj + navg +nvar+ img_per_grp)/1.0e9 + overhead: %12.3f [GB]"%\
(total_mem[1] + overhead_loading))
del full_data
mpi_barrier(MPI_COMM_WORLD)
if myid == heavy_load_myid:
log_main.add("Begin reading and preprocessing images on processor. Wait... ")
ttt = time()
#imgdata = EMData.read_images(stack, all_proj)
imgdata = [ None for im in range(len(all_proj))]
for index_of_proj in range(len(all_proj)):
#image = get_im(stack, all_proj[index_of_proj])
if( current_window > 0): imgdata[index_of_proj] = fdecimate(window2d(get_im(stack, all_proj[index_of_proj]),current_window,current_window), nx, ny)
else: imgdata[index_of_proj] = fdecimate(get_im(stack, all_proj[index_of_proj]), nx, ny)
if (current_decimate> 0.0 and options.CTF):
ctf = imgdata[index_of_proj].get_attr("ctf")
ctf.apix = ctf.apix/current_decimate
imgdata[index_of_proj].set_attr("ctf", ctf)
if myid == heavy_load_myid and index_of_proj%100 == 0:
log_main.add(" ...... %6.2f%% "%(index_of_proj/float(len(all_proj))*100.))
mpi_barrier(MPI_COMM_WORLD)
if myid == heavy_load_myid:
log_main.add("All_proj preprocessing cost %7.2f m"%((time()-ttt)/60.))
log_main.add("Wait untill reading on all CPUs done...")
'''
imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
if options.fl > 0.0:
for k in xrange(len(imgdata2)):
imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
if myid == main_node:
vol.write_image("vol_ctf.hdf")
print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf"))
del vol, imgdata2
mpi_barrier(MPI_COMM_WORLD)
'''
from applications import prepare_2d_forPCA
from utilities import model_blank
from EMAN2 import Transform
if not options.no_norm:
mask = model_circle(nx/2-2, nx, nx)
if options.CTF:
from utilities import pad
from filter import filt_ctf
from filter import filt_tanl
if myid == heavy_load_myid:
log_main.add("Start computing 2D aveList and varList. Wait...")
ttt = time()
inner=nx//2-4
outer=inner+2
xform_proj_for_2D = [ None for i in range(len(proj_list))]
for i in range(len(proj_list)):
ki = proj_angles[proj_list[i][0]][3]
if ki >= symbaselen: continue
mi = index[ki]
dpar = Util.get_transform_params(imgdata[mi], "xform.projection", "spider")
phiM, thetaM, psiM, s2xM, s2yM = dpar["phi"],dpar["theta"],dpar["psi"],-dpar["tx"]*current_decimate,-dpar["ty"]*current_decimate
grp_imgdata = []
for j in range(img_per_grp):
mj = index[proj_angles[proj_list[i][j]][3]]
cpar = Util.get_transform_params(imgdata[mj], "xform.projection", "spider")
alpha, sx, sy, mirror = params_3D_2D_NEW(cpar["phi"], cpar["theta"],cpar["psi"], -cpar["tx"]*current_decimate, -cpar["ty"]*current_decimate, mirror_list[i][j])
if thetaM <= 90:
if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, phiM - cpar["phi"], 0.0, 0.0, 1.0)
else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, 180-(phiM - cpar["phi"]), 0.0, 0.0, 1.0)
else:
if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(phiM- cpar["phi"]), 0.0, 0.0, 1.0)
else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(180-(phiM - cpar["phi"])), 0.0, 0.0, 1.0)
imgdata[mj].set_attr("xform.align2d", Transform({"type":"2D","alpha":alpha,"tx":sx,"ty":sy,"mirror":mirror,"scale":1.0}))
grp_imgdata.append(imgdata[mj])
if not options.no_norm:
for k in range(img_per_grp):
ave, std, minn, maxx = Util.infomask(grp_imgdata[k], mask, False)
grp_imgdata[k] -= ave
grp_imgdata[k] /= std
if options.fl > 0.0:
for k in range(img_per_grp):
grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
# Because of background issues, only linear option works.
if options.CTF: ave, var = aves_wiener(grp_imgdata, SNR = 1.0e5, interpolation_method = "linear")
else: ave, var = ave_var(grp_imgdata)
# Switch to std dev
# threshold is not really needed,it is just in case due to numerical accuracy something turns out negative.
var = square_root(threshold(var))
set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])
aveList.append(ave)
varList.append(var)
xform_proj_for_2D[i] = [phiM, thetaM, 0.0, 0.0, 0.0]
'''
if nvec > 0:
eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True)
for k in range(nvec):
set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
eigList[k].append(eig[k])
"""
if myid == 0 and i == 0:
for k in xrange(nvec):
eig[k].write_image("eig.hdf", k)
"""
'''
if (myid == heavy_load_myid) and (i%100 == 0):
log_main.add(" ......%6.2f%% "%(i/float(len(proj_list))*100.))
del imgdata, grp_imgdata, cpar, dpar, all_proj, proj_angles, index
if not options.no_norm: del mask
if myid == main_node: del tab
# At this point, all averages and variances are computed
mpi_barrier(MPI_COMM_WORLD)
if (myid == heavy_load_myid):
log_main.add("Computing aveList and varList took %12.1f [m]"%((time()-ttt)/60.))
xform_proj_for_2D = wrap_mpi_gatherv(xform_proj_for_2D, main_node, MPI_COMM_WORLD)
if (myid == main_node):
write_text_row(xform_proj_for_2D, os.path.join(current_output_dir, "params.txt"))
del xform_proj_for_2D
mpi_barrier(MPI_COMM_WORLD)
if options.ave2D:
from fundamentals import fpol
from applications import header
if myid == main_node:
log_main.add("Compute ave2D ... ")
km = 0
for i in range(number_of_proc):
if i == main_node :
for im in range(len(aveList)):
aveList[im].write_image(os.path.join(current_output_dir, options.ave2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
nl = int(nl[0])
for im in range(nl):
ave = recv_EMData(i, im+i+70000)
"""
nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('pix_err', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
"""
tmpvol=fpol(ave, nx, nx,1)
tmpvol.write_image(os.path.join(current_output_dir, options.ave2D), km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in range(len(aveList)):
send_EMData(aveList[im], main_node,im+myid+70000)
"""
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
members = aveList[im].get_attr('pix_err')
mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
except:
mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
"""
if myid == main_node:
header(os.path.join(current_output_dir, options.ave2D), params='xform.projection', fimport = os.path.join(current_output_dir, "params.txt"))
mpi_barrier(MPI_COMM_WORLD)
if options.ave3D:
from fundamentals import fpol
t5 = time()
if myid == main_node: log_main.add("Reconstruct ave3D ... ")
ave3D = recons3d_4nn_MPI(myid, aveList, symmetry=options.sym, npad=options.npad)
bcast_EMData_to_all(ave3D, myid)
if myid == main_node:
if current_decimate != 1.0: ave3D = resample(ave3D, 1./current_decimate)
ave3D = fpol(ave3D, nnxo, nnxo, nnxo) # always to the orignal image size
set_pixel_size(ave3D, 1.0)
ave3D.write_image(os.path.join(current_output_dir, options.ave3D))
log_main.add("Ave3D reconstruction took %12.1f [m]"%((time()-t5)/60.0))
log_main.add("%-70s: %s\n"%("The reconstructed ave3D is saved as ", options.ave3D))
mpi_barrier(MPI_COMM_WORLD)
del ave, var, proj_list, stack, alpha, sx, sy, mirror, aveList
'''
if nvec > 0:
for k in range(nvec):
if myid == main_node:log_main.add("Reconstruction eigenvolumes", k)
cont = True
ITER = 0
mask2d = model_circle(radiuspca, nx, nx)
while cont:
#print "On node %d, iteration %d"%(myid, ITER)
eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad)
bcast_EMData_to_all(eig3D, myid, main_node)
if options.fl > 0.0:
eig3D = filt_tanl(eig3D, options.fl, options.aa)
if myid == main_node:
eig3D.write_image(os.path.join(options.outpout_dir, "eig3d_%03d.hdf"%(k, ITER)))
Util.mul_img( eig3D, model_circle(radiuspca, nx, nx, nx) )
eig3Df, kb = prep_vol(eig3D)
del eig3D
cont = False
icont = 0
for l in range(len(eigList[k])):
phi, theta, psi, s2x, s2y = get_params_proj(eigList[k][l])
proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y])
cl = ccc(proj, eigList[k][l], mask2d)
if cl < 0.0:
icont += 1
cont = True
eigList[k][l] *= -1.0
u = int(cont)
u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD)
icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
if myid == main_node:
u = int(u[0])
log_main.add(" Eigenvector: ",k," number changed ",int(icont[0]))
else: u = 0
u = bcast_number_to_all(u, main_node)
cont = bool(u)
ITER += 1
del eig3Df, kb
mpi_barrier(MPI_COMM_WORLD)
del eigList, mask2d
'''
if options.ave3D: del ave3D
if options.var2D:
from fundamentals import fpol
from applications import header
if myid == main_node:
log_main.add("Compute var2D...")
km = 0
for i in range(number_of_proc):
if i == main_node :
for im in range(len(varList)):
tmpvol=fpol(varList[im], nx, nx,1)
tmpvol.write_image(os.path.join(current_output_dir, options.var2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
nl = int(nl[0])
for im in range(nl):
ave = recv_EMData(i, im+i+70000)
tmpvol=fpol(ave, nx, nx,1)
tmpvol.write_image(os.path.join(current_output_dir, options.var2D), km)
km += 1
else:
mpi_send(len(varList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in range(len(varList)):
send_EMData(varList[im], main_node, im+myid+70000)# What with the attributes??
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
from applications import header
header(os.path.join(current_output_dir, options.var2D), params = 'xform.projection',fimport = os.path.join(current_output_dir, "params.txt"))
mpi_barrier(MPI_COMM_WORLD)
if options.var3D:
if myid == main_node: log_main.add("Reconstruct var3D ...")
t6 = time()
# radiusvar = options.radius
# if( radiusvar < 0 ): radiusvar = nx//2 -3
res = recons3d_4nn_MPI(myid, varList, symmetry = options.sym, npad=options.npad)
#res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
if myid == main_node:
from fundamentals import fpol
if current_decimate != 1.0: res = resample(res, 1./current_decimate)
res = fpol(res, nnxo, nnxo, nnxo)
set_pixel_size(res, 1.0)
res.write_image(os.path.join(current_output_dir, options.var3D))
log_main.add("%-70s: %s\n"%("The reconstructed var3D is saved as ", options.var3D))
log_main.add("Var3D reconstruction took %f12.1 [m]"%((time()-t6)/60.0))
log_main.add("Total computation time %f12.1 [m]"%((time()-t0)/60.0))
log_main.add("sx3dvariability finishes")
from mpi import mpi_finalize
mpi_finalize()
if RUNNING_UNDER_MPI: global_def.MPI = False
global_def.BATCH = False
def fdownsample(img, sub_rate=0.5, RetReal = True):
"""
resample image based on the value of sub_rate.
the input image can be either 2D image or 3D volume.
sub_rate < 1.0, subsampling the image.
sub_rate > 1.0, upsampling the image using new gridding interpolation.
fit_to_fft will change the ouput image size to an fft_friendly size
"""
from fundamentals import fdecimate
from utilities import get_pixel_size, set_pixel_size
if type(img) == str:
from utilities import get_image
img = get_image(img)
nx = img.get_xsize()
if img.is_complex():
nx -= (2-nx%2)
ny = img.get_ysize()
nz = img.get_zsize()
if( ny == 1): ERROR("Only 2D or 3D images allowed","resample",1)
if sub_rate == 1.0: return img.copy()
elif sub_rate < 1.0:
nnx = int(nx*sub_rate+0.5)
nny = int(ny*sub_rate+0.5)
nnz = int(nz*sub_rate+0.5)
e = fdecimate(img, nnx, nny, nnz, RetReal = RetReal)
else: # sub_rate>1
ERROR("fdownsample","upscaling not implemented",1)
"""
new_nx = int(nx*sub_rate+0.5)
new_ny = int(ny*sub_rate+0.5)
if nz==1:
new_nz = 1
else:
new_nz = int(ny*sub_rate+0.5)
if ( nx!=ny and nz==1 ):
nn = max(new_nx, new_ny)
e = Util.pad(img, nn, nn, 1, 0, 0, 0, "circumference")
e, kb = prepi(e)
e = Util.window( e.rot_scale_conv_new(0.0, 0.0, 0.0, kb, sub_rate), new_nx, new_ny, 1, 0,0,0)
elif ((nx!=ny or nx!=nz or ny!=nz) and nz>1):
nn = max(new_nx, new_ny,new_nz)
e = Util.pad(img, nn, nn, nn, 0, 0, 0, "circumference")
e, kb = prepi3D(e)
e = Util.window( e.rot_scale_conv_new_3D(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, kb, sub_rate), new_nx, new_ny, new_nz, 0,0,0)
else:
if nz==1:
e, kb = prepi(Util.pad(img, new_nx, new_ny, 1, 0, 0, 0, "circumference"))
e = e.rot_scale_conv_new(0.0, 0.0, 0.0, kb, sub_rate)
else:
e, kb = prepi3D(Util.pad(img, new_nx, new_ny, new_nz, 0, 0, 0, "circumference"))
e = e.rot_scale_conv_new_3D(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, kb, sub_rate)
"""
# Automatically adjust pixel size for ctf parameters
from utilities import get_pixel_size, set_pixel_size
apix = get_pixel_size(e)
apix /= sub_rate
set_pixel_size(e, apix)
cc = e.get_attr_default("xform.projection", None)
if cc:
cp = cc.get_params("spider")
cp["tx"] *= sub_rate
cp["ty"] *= sub_rate
from utilities import set_params_proj
set_params_proj(e, [cp["phi"], cp["theta"], cp["psi"], -cp["tx"], -cp["ty"]]) # have to invert as set inverts them again
cc = e.get_attr_default("xform.align2d", None)
if cc:
cp = cc.get_params("2D")
cp["tx"] *= sub_rate
cp["ty"] *= sub_rate
from utilities import set_params2D
set_params2D(e, [cp["alpha"], cp["tx"], cp["ty"], cp["mirror"], cp["scale"]])
return e
def rec3D_MPI(data, snr, symmetry, mask3D, fsc_curve, myid, main_node = 0, rstep = 1.0, odd_start=0, eve_start=1, finfo=None, index=-1, npad = 4, hparams=None):
'''
This function is to be called within an MPI program to do a reconstruction on a dataset kept
in the memory, computes reconstruction and through odd-even, in order to get the resolution
'''
import os
from statistics import fsc_mask
from utilities import model_blank, reduce_EMData_to_root, get_image, send_EMData, recv_EMData
from random import randint
from mpi import mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD
nproc = mpi_comm_size(MPI_COMM_WORLD)
if nproc==1:
assert main_node==0
main_node_odd = main_node
main_node_eve = main_node
main_node_all = main_node
elif nproc==2:
main_node_odd = main_node
main_node_eve = (main_node+1)%2
main_node_all = main_node
tag_voleve = 1000
tag_fftvol_eve = 1001
tag_weight_eve = 1002
else:
#spread CPUs between different nodes to save memory
main_node_odd = main_node
main_node_eve = (int(main_node)+nproc-1)%int(nproc)
main_node_all = (int(main_node)+nproc//2)%int(nproc)
tag_voleve = 1000
tag_fftvol_eve = 1001
tag_weight_eve = 1002
tag_fftvol_odd = 1003
tag_weight_odd = 1004
tag_volall = 1005
if index !=-1 :
grpdata = []
for i in xrange( len(data) ):
if data[i].get_attr( 'group' ) == index:
grpdata.append( data[i] )
imgdata = grpdata
else:
imgdata = data
nx = get_image_size( imgdata, myid )
if nx==0:
ERROR("Warning: no images were given for reconstruction, this usually means there is an empty group, returning empty volume","rec3D",0)
return model_blank( 2, 2, 2 ), None, model_blank(nx,nx,nx), model_blank(nx,nx,nx)
fftvol_odd_file,weight_odd_file = prepare_recons_ctf(nx, imgdata, snr, symmetry, myid, main_node_odd, odd_start, 2, finfo, npad)
fftvol_eve_file,weight_eve_file = prepare_recons_ctf(nx, imgdata, snr, symmetry, myid, main_node_eve, eve_start, 2, finfo, npad)
del imgdata
if nproc == 1:
fftvol = get_image(fftvol_odd_file)
weight = get_image(weight_odd_file)
volodd = recons_ctf_from_fftvol(nx, fftvol, weight, snr, symmetry, npad)
fftvol = get_image(fftvol_eve_file)
weight = get_image(weight_eve_file)
voleve = recons_ctf_from_fftvol(nx, fftvol, weight, snr, symmetry, npad)
fscdat = fsc_mask( volodd, voleve, mask3D, rstep, fsc_curve)
fftvol = get_image( fftvol_odd_file )
fftvol_tmp = get_image(fftvol_eve_file)
fftvol += fftvol_tmp
fftvol_tmp = None
weight = get_image( weight_odd_file )
weight_tmp = get_image(weight_eve_file)
weight += weight_tmp
weight_tmp = None
volall = recons_ctf_from_fftvol(nx, fftvol, weight, snr, symmetry, npad)
# if helical, find & apply symmetry to volume
if hparams is not None:
volodd,voleve,volall = hsymVols(volodd,voleve,volall,hparams)
fscdat = fsc_mask( volodd, voleve, mask3D, rstep, fsc_curve)
os.system( "rm -f " + fftvol_odd_file + " " + weight_odd_file )
os.system( "rm -f " + fftvol_eve_file + " " + weight_eve_file )
return volall,fscdat,volodd,voleve
if nproc == 2:
if myid == main_node_odd:
fftvol = get_image( fftvol_odd_file )
weight = get_image( weight_odd_file )
volodd = recons_ctf_from_fftvol(nx, fftvol, weight, snr, symmetry, npad)
voleve = recv_EMData(main_node_eve, tag_voleve)
fscdat = fsc_mask( volodd, voleve, mask3D, rstep, fsc_curve)
else:
assert myid == main_node_eve
fftvol = get_image( fftvol_eve_file )
weight = get_image( weight_eve_file )
voleve = recons_ctf_from_fftvol(nx, fftvol, weight, snr, symmetry, npad)
send_EMData(voleve, main_node_odd, tag_voleve)
if myid == main_node_odd:
fftvol = get_image( fftvol_odd_file )
fftvol_tmp = recv_EMData( main_node_eve, tag_fftvol_eve )
fftvol += fftvol_tmp
fftvol_tmp = None
weight = get_image( weight_odd_file )
weight_tmp = recv_EMData( main_node_eve, tag_weight_eve )
weight += weight_tmp
weight_tmp = None
volall = recons_ctf_from_fftvol(nx, fftvol, weight, snr, symmetry, npad)
# if helical, find & apply symmetry to volume
if hparams is not None:
volodd,voleve,volall = hsymVols(volodd,voleve,volall,hparams)
fscdat = fsc_mask( volodd, voleve, mask3D, rstep, fsc_curve)
os.system( "rm -f " + fftvol_odd_file + " " + weight_odd_file )
os.system( "rm -f " + fftvol_eve_file + " " + weight_eve_file )
return volall,fscdat,volodd,voleve
else:
assert myid == main_node_eve
fftvol = get_image( fftvol_eve_file )
send_EMData(fftvol, main_node_odd, tag_fftvol_eve )
weight = get_image( weight_eve_file )
send_EMData(weight, main_node_odd, tag_weight_eve )
os.system( "rm -f " + fftvol_eve_file + " " + weight_eve_file )
return model_blank(nx,nx,nx), None, model_blank(nx,nx,nx), model_blank(nx,nx,nx)
# cases from all other number of processors situations
if myid == main_node_odd:
fftvol = get_image( fftvol_odd_file )
send_EMData(fftvol, main_node_eve, tag_fftvol_odd )
if not(finfo is None):
finfo.write("fftvol odd sent\n")
finfo.flush()
weight = get_image( weight_odd_file )
send_EMData(weight, main_node_all, tag_weight_odd )
if not(finfo is None):
finfo.write("weight odd sent\n")
finfo.flush()
volodd = recons_ctf_from_fftvol(nx, fftvol, weight, snr, symmetry, npad)
del fftvol, weight
voleve = recv_EMData(main_node_eve, tag_voleve)
fscdat = fsc_mask(volodd, voleve, mask3D, rstep, fsc_curve)
volall = recv_EMData(main_node_all, tag_volall)
# if helical, find & apply symmetry to volume
if hparams is not None:
volodd,voleve,volall = hsymVols(volodd,voleve,volall,hparams)
fscdat = fsc_mask( volodd, voleve, mask3D, rstep, fsc_curve)
os.system( "rm -f " + fftvol_odd_file + " " + weight_odd_file );
return volall,fscdat,volodd,voleve
if myid == main_node_eve:
ftmp = recv_EMData(main_node_odd, tag_fftvol_odd)
fftvol = get_image( fftvol_eve_file )
Util.add_img( ftmp, fftvol )
send_EMData(ftmp, main_node_all, tag_fftvol_eve )
del ftmp
weight = get_image( weight_eve_file )
send_EMData(weight, main_node_all, tag_weight_eve )
voleve = recons_ctf_from_fftvol(nx, fftvol, weight, snr, symmetry, npad)
send_EMData(voleve, main_node_odd, tag_voleve)
os.system( "rm -f " + fftvol_eve_file + " " + weight_eve_file );
return model_blank(nx,nx,nx), None, model_blank(nx,nx,nx), model_blank(nx,nx,nx)
if myid == main_node_all:
fftvol = recv_EMData(main_node_eve, tag_fftvol_eve)
if not(finfo is None):
finfo.write( "fftvol odd received\n" )
finfo.flush()
weight = recv_EMData(main_node_odd, tag_weight_odd)
weight_tmp = recv_EMData(main_node_eve, tag_weight_eve)
Util.add_img( weight, weight_tmp )
weight_tmp = None
volall = recons_ctf_from_fftvol(nx, fftvol, weight, snr, symmetry, npad)
send_EMData(volall, main_node_odd, tag_volall)
return model_blank(nx,nx,nx),None, model_blank(nx,nx,nx), model_blank(nx,nx,nx)
return model_blank(nx,nx,nx),None, model_blank(nx,nx,nx), model_blank(nx,nx,nx)
def main():
arglist = []
for arg in sys.argv:
arglist.append(arg)
progname = os.path.basename(arglist[0])
usage = progname + " prj_stack volume fsc_curve <mask> --CTF --snr=signal_noise_ratio --list=file --group=ID --sym=symmetry -verbose=(0|1) --MPI"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--CTF",
action="store_true",
default=False,
help="perform ctf correction")
parser.add_option("--snr",
type="float",
default=1.0,
help="Signal-to-Noise Ratio in the data")
parser.add_option("--sym", type="string", default="c1", help="symmetry")
parser.add_option(
"--list",
type="string",
help="file with list of images to be used in the first column")
parser.add_option(
"--group",
type="int",
default=-1,
help=
"perform reconstruction using images for a given group number (group is attribute in the header)"
)
parser.add_option("--MPI",
action="store_true",
default=False,
help="use MPI version ")
parser.add_option("--npad",
type="int",
default=2,
help="number of times padding (default 2)")
parser.add_option("--verbose",
type="int",
default=0,
help="verbose level: 0 no, 1 yes")
(options, args) = parser.parse_args(arglist[1:])
if len(args) != 3 and len(args) != 4:
print(usage)
sys.exit(-1)
prj_stack = args[0]
vol_stack = args[1]
fsc_curve = args[2]
if len(args) == 3: mask = None
else: mask = get_image(args[3])
if options.MPI:
from mpi import mpi_init
sys.argv = mpi_init(len(sys.argv), sys.argv)
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
if (options.list and options.group > -1):
ERROR("options group and list cannot be used together", "recon3d_n", 1)
sys.exit()
from applications import recons3d_f
global_def.BATCH = True
recons3d_f(prj_stack, vol_stack, fsc_curve, mask, options.CTF, options.snr,
options.sym, options.list, options.group, options.npad,
options.verbose, options.MPI)
global_def.BATCH = False
if options.MPI:
from mpi import mpi_finalize
mpi_finalize()
def main():
import global_def
from optparse import OptionParser
from EMAN2 import EMUtil
import os
import sys
from time import time
progname = os.path.basename(sys.argv[0])
usage = progname + " proj_stack output_averages --MPI"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--img_per_group",type="int" , default=100 , help="number of images per group" )
parser.add_option("--radius", type="int" , default=-1 , help="radius for alignment" )
parser.add_option("--xr", type="string" , default="2 1", help="range for translation search in x direction, search is +/xr")
parser.add_option("--yr", type="string" , default="-1", help="range for translation search in y direction, search is +/yr (default = same as xr)")
parser.add_option("--ts", type="string" , default="1 0.5", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional")
parser.add_option("--iter", type="int" , default=30, help="number of iterations within alignment (default = 30)" )
parser.add_option("--num_ali", type="int" , default=5, help="number of alignments performed for stability (default = 5)" )
parser.add_option("--thld_err", type="float" , default=1.0, help="threshold of pixel error (default = 1.732)" )
parser.add_option("--grouping" , type="string" , default="GRP", help="do grouping of projections: PPR - per projection, GRP - different size groups, exclusive (default), GEV - grouping equal size")
parser.add_option("--delta", type="float" , default=-1.0, help="angular step for reference projections (required for GEV method)")
parser.add_option("--fl", type="float" , default=0.3, help="cut-off frequency of hyperbolic tangent low-pass Fourier filter")
parser.add_option("--aa", type="float" , default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter")
parser.add_option("--CTF", action="store_true", default=False, help="Consider CTF correction during the alignment ")
parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
(options,args) = parser.parse_args()
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, MPI_COMM_WORLD, MPI_TAG_UB
from mpi import mpi_barrier, mpi_send, mpi_recv, mpi_bcast, MPI_INT, mpi_finalize, MPI_FLOAT
from applications import MPI_start_end, within_group_refinement, ali2d_ras
from pixel_error import multi_align_stability
from utilities import send_EMData, recv_EMData
from utilities import get_image, bcast_number_to_all, set_params2D, get_params2D
from utilities import group_proj_by_phitheta, model_circle, get_input_from_string
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
if len(args) == 2:
stack = args[0]
outdir = args[1]
else:
ERROR("incomplete list of arguments", "sxproj_stability", 1, myid=myid)
exit()
if not options.MPI:
ERROR("Non-MPI not supported!", "sxproj_stability", myid=myid)
exit()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
global_def.BATCH = True
#if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "sxproj_stability", 1, myid)
#mpi_barrier(MPI_COMM_WORLD)
img_per_grp = options.img_per_group
radius = options.radius
ite = options.iter
num_ali = options.num_ali
thld_err = options.thld_err
xrng = get_input_from_string(options.xr)
if options.yr == "-1": yrng = xrng
else : yrng = get_input_from_string(options.yr)
step = get_input_from_string(options.ts)
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
else:
nima = 0
nx = 0
ny = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
if radius == -1: radius = nx/2-2
mask = model_circle(radius, nx, nx)
st = time()
if options.grouping == "GRP":
if myid == main_node:
print " A ",myid," ",time()-st
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
proj_params = []
for i in xrange(nima):
dp = proj_attr[i].get_params("spider")
phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp["psi"], -dp["tx"], -dp["ty"]
proj_params.append([phi, theta, psi, s2x, s2y])
# Here is where the grouping is done, I didn't put enough annotation in the group_proj_by_phitheta,
# So I will briefly explain it here
# proj_list : Returns a list of list of particle numbers, each list contains img_per_grp particle numbers
# except for the last one. Depending on the number of particles left, they will either form a
# group or append themselves to the last group
# angle_list : Also returns a list of list, each list contains three numbers (phi, theta, delta), (phi,
# theta) is the projection angle of the center of the group, delta is the range of this group
# mirror_list: Also returns a list of list, each list contains img_per_grp True or False, which indicates
# whether it should take mirror position.
# In this program angle_list and mirror list are not of interest.
proj_list_all, angle_list, mirror_list = group_proj_by_phitheta(proj_params, img_per_grp=img_per_grp)
del proj_params
print " B number of groups ",myid," ",len(proj_list_all),time()-st
mpi_barrier(MPI_COMM_WORLD)
# Number of groups, actually there could be one or two more groups, since the size of the remaining group varies
# we will simply assign them to main node.
n_grp = nima/img_per_grp-1
# Divide proj_list_all equally to all nodes, and becomes proj_list
proj_list = []
for i in xrange(n_grp):
proc_to_stay = i%number_of_proc
if proc_to_stay == main_node:
if myid == main_node: proj_list.append(proj_list_all[i])
elif myid == main_node:
mpi_send(len(proj_list_all[i]), 1, MPI_INT, proc_to_stay, MPI_TAG_UB, MPI_COMM_WORLD)
mpi_send(proj_list_all[i], len(proj_list_all[i]), MPI_INT, proc_to_stay, MPI_TAG_UB, MPI_COMM_WORLD)
elif myid == proc_to_stay:
img_per_grp = mpi_recv(1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
img_per_grp = int(img_per_grp[0])
temp = mpi_recv(img_per_grp, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
proj_list.append(map(int, temp))
del temp
mpi_barrier(MPI_COMM_WORLD)
print " C ",myid," ",time()-st
if myid == main_node:
# Assign the remaining groups to main_node
for i in xrange(n_grp, len(proj_list_all)):
proj_list.append(proj_list_all[i])
del proj_list_all, angle_list, mirror_list
# Compute stability per projection projection direction, equal number assigned, thus overlaps
elif options.grouping == "GEV":
if options.delta == -1.0: ERROR("Angular step for reference projections is required for GEV method","sxproj_stability",1)
from utilities import even_angles, nearestk_to_refdir, getvec
refproj = even_angles(options.delta)
img_begin, img_end = MPI_start_end(len(refproj), number_of_proc, myid)
# Now each processor keeps its own share of reference projections
refprojdir = refproj[img_begin: img_end]
del refproj
ref_ang = [0.0]*(len(refprojdir)*2)
for i in xrange(len(refprojdir)):
ref_ang[i*2] = refprojdir[0][0]
ref_ang[i*2+1] = refprojdir[0][1]+i*0.1
print " A ",myid," ",time()-st
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
# the solution below is very slow, do not use it unless there is a problem with the i/O
"""
for i in xrange(number_of_proc):
if myid == i:
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
mpi_barrier(MPI_COMM_WORLD)
"""
print " B ",myid," ",time()-st
proj_ang = [0.0]*(nima*2)
for i in xrange(nima):
dp = proj_attr[i].get_params("spider")
proj_ang[i*2] = dp["phi"]
proj_ang[i*2+1] = dp["theta"]
print " C ",myid," ",time()-st
asi = Util.nearestk_to_refdir(proj_ang, ref_ang, img_per_grp)
del proj_ang, ref_ang
proj_list = []
for i in xrange(len(refprojdir)):
proj_list.append(asi[i*img_per_grp:(i+1)*img_per_grp])
del asi
print " D ",myid," ",time()-st
#from sys import exit
#exit()
# Compute stability per projection
elif options.grouping == "PPR":
print " A ",myid," ",time()-st
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
print " B ",myid," ",time()-st
proj_params = []
for i in xrange(nima):
dp = proj_attr[i].get_params("spider")
phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp["psi"], -dp["tx"], -dp["ty"]
proj_params.append([phi, theta, psi, s2x, s2y])
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
print " C ",myid," ",time()-st
from utilities import nearest_proj
proj_list, mirror_list = nearest_proj(proj_params, img_per_grp, range(img_begin, img_begin+1))#range(img_begin, img_end))
refprojdir = proj_params[img_begin: img_end]
del proj_params, mirror_list
print " D ",myid," ",time()-st
else: ERROR("Incorrect projection grouping option","sxproj_stability",1)
"""
from utilities import write_text_file
for i in xrange(len(proj_list)):
write_text_file(proj_list[i],"projlist%06d_%04d"%(i,myid))
"""
###########################################################################################################
# Begin stability test
from utilities import get_params_proj, read_text_file
#if myid == 0:
# from utilities import read_text_file
# proj_list[0] = map(int, read_text_file("lggrpp0.txt"))
from utilities import model_blank
aveList = [model_blank(nx,ny)]*len(proj_list)
if options.grouping == "GRP": refprojdir = [[0.0,0.0,-1.0]]*len(proj_list)
for i in xrange(len(proj_list)):
print " E ",myid," ",time()-st
class_data = EMData.read_images(stack, proj_list[i])
#print " R ",myid," ",time()-st
if options.CTF :
from filter import filt_ctf
for im in xrange(len(class_data)): # MEM LEAK!!
atemp = class_data[im].copy()
btemp = filt_ctf(atemp, atemp.get_attr("ctf"), binary=1)
class_data[im] = btemp
#class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1)
for im in class_data:
try:
t = im.get_attr("xform.align2d") # if they are there, no need to set them!
except:
try:
t = im.get_attr("xform.projection")
d = t.get_params("spider")
set_params2D(im, [0.0,-d["tx"],-d["ty"],0,1.0])
except:
set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0])
#print " F ",myid," ",time()-st
# Here, we perform realignment num_ali times
all_ali_params = []
for j in xrange(num_ali):
if( xrng[0] == 0.0 and yrng[0] == 0.0 ):
avet = ali2d_ras(class_data, randomize = True, ir = 1, ou = radius, rs = 1, step = 1.0, dst = 90.0, maxit = ite, check_mirror = True, FH=options.fl, FF=options.aa)
else:
avet = within_group_refinement(class_data, mask, True, 1, radius, 1, xrng, yrng, step, 90.0, ite, options.fl, options.aa)
ali_params = []
for im in xrange(len(class_data)):
alpha, sx, sy, mirror, scale = get_params2D(class_data[im])
ali_params.extend( [alpha, sx, sy, mirror] )
all_ali_params.append(ali_params)
#aveList[i] = avet
#print " G ",myid," ",time()-st
del ali_params
# We determine the stability of this group here.
# stable_set contains all particles deemed stable, it is a list of list
# each list has two elements, the first is the pixel error, the second is the image number
# stable_set is sorted based on pixel error
#from utilities import write_text_file
#write_text_file(all_ali_params, "all_ali_params%03d.txt"%myid)
stable_set, mir_stab_rate, average_pix_err = multi_align_stability(all_ali_params, 0.0, 10000.0, thld_err, False, 2*radius+1)
#print " H ",myid," ",time()-st
if(len(stable_set) > 5):
stable_set_id = []
members = []
pix_err = []
# First put the stable members into attr 'members' and 'pix_err'
for s in stable_set:
# s[1] - number in this subset
stable_set_id.append(s[1])
# the original image number
members.append(proj_list[i][s[1]])
pix_err.append(s[0])
# Then put the unstable members into attr 'members' and 'pix_err'
from fundamentals import rot_shift2D
avet.to_zero()
if options.grouping == "GRP":
aphi = 0.0
atht = 0.0
vphi = 0.0
vtht = 0.0
l = -1
for j in xrange(len(proj_list[i])):
# Here it will only work if stable_set_id is sorted in the increasing number, see how l progresses
if j in stable_set_id:
l += 1
avet += rot_shift2D(class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3] )
if options.grouping == "GRP":
phi, theta, psi, sxs, sys = get_params_proj(class_data[j])
if( theta > 90.0):
phi = (phi+540.0)%360.0
theta = 180.0 - theta
aphi += phi
atht += theta
vphi += phi*phi
vtht += theta*theta
else:
members.append(proj_list[i][j])
pix_err.append(99999.99)
aveList[i] = avet.copy()
if l>1 :
l += 1
aveList[i] /= l
if options.grouping == "GRP":
aphi /= l
atht /= l
vphi = (vphi - l*aphi*aphi)/l
vtht = (vtht - l*atht*atht)/l
from math import sqrt
refprojdir[i] = [aphi, atht, (sqrt(max(vphi,0.0))+sqrt(max(vtht,0.0)))/2.0]
# Here more information has to be stored, PARTICULARLY WHAT IS THE REFERENCE DIRECTION
aveList[i].set_attr('members', members)
aveList[i].set_attr('refprojdir',refprojdir[i])
aveList[i].set_attr('pixerr', pix_err)
else:
print " empty group ",i, refprojdir[i]
aveList[i].set_attr('members',[-1])
aveList[i].set_attr('refprojdir',refprojdir[i])
aveList[i].set_attr('pixerr', [99999.])
del class_data
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node :
for im in xrange(len(aveList)):
aveList[im].write_image(args[1], km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im+i+70000)
nm = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('pixerr', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
ave.write_image(args[1], km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
for im in xrange(len(aveList)):
send_EMData(aveList[im], main_node,im+myid+70000)
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
members = aveList[im].get_attr('pixerr')
mpi_send(members, len(members), MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
except:
mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
global_def.BATCH = False
mpi_barrier(MPI_COMM_WORLD)
from mpi import mpi_finalize
mpi_finalize()
def launches(result="",
timeOut=1,
units="miles",
task="none",
parameter="none",
tz="none"):
"""
:type timeOut: Optional[int]
Returns details about the Launches
Parameters
----------
timeOut : time out in seconds
Returns
-------
a string in speech format containing details of the information requested
"""
# Get instance of the number to words engine
p = inflect.engine()
# Previous Launch
if (task == "last-long"):
SPEECH = "The last launch was for the " + result[
'mission_name'] + " mission, on " + convert_date_to_speech(
UTC_to_local(result['launch_date_utc'], tz), "long", tz) + "."
droneshipslist = ships(1, "", "getDroneShipsList", "")
SPEECH = SPEECH + " It was launched by a " + rocket(
result) + " rocket,"
core_speech = " and had "
cores_used = core_reuse(result).split(":")
core_speech_r = ""
core_speech_n = ""
if (int(cores_used[0]) > 0):
core_speech_r = cores_used[0] + " reused " + p.plural(
"core", int(cores_used[0]))
core_speech = core_speech + core_speech_r
if (int(cores_used[1]) > 0):
core_speech = core_speech + " and "
if (int(cores_used[1]) > 0):
core_speech_n = cores_used[1] + " new " + p.plural(
"core", int(cores_used[1]))
core_speech = core_speech + core_speech_n
if (result['rocket']['first_stage']['cores']):
core_speech = core_speech + ". "
cores_down = cores_landed(result)
core_count = 0
for core_down in cores_down:
core_count = core_count + 1
coreinfo = core_down.split(":")
core_status = coreinfo[0]
landing_vehicle = coreinfo[1]
if (core_status == "YES"):
core_speech = core_speech + "Core " + str(
core_count
) + " landed successfully " + coreinfo[1] + "."
else:
core_speech = core_speech + "Core " + str(
core_count
) + " did not land on its target of " + coreinfo[1] + "."
RET = SPEECH + core_speech
if (task == "next-long"):
SPEECH = "The next launch is for the " + result[
'mission_name'] + " mission, on " + convert_date_to_speech(
UTC_to_local(result['launch_date_utc'], tz), "long")
details = result['details']
if details is not None:
listofpads = json.loads(
json.dumps(landingpads(1, "", "getLandingPadsList", "")))
for lv in listofpads:
s_name = ''
if (lv["landing_type"] == "ASDS"):
s_name = " the drone ship "
else:
s_name = " the landing zone "
s_name = s_name + lv["full_name"]
details = details.replace(lv["id"], "," + s_name)
SPEECH = SPEECH + ". " + details + ". "
SPEECH = SPEECH + " It'll be launched by a " + rocket(
result) + " rocket,"
core_speech = " and will have "
cores_used = core_reuse(result).split(":")
core_speech_r = ""
core_speech_n = ""
if (int(cores_used[0]) > 0):
core_speech_r = cores_used[0] + " reused " + p.plural(
"core", int(cores_used[0]))
core_speech = core_speech + core_speech_r
if (int(cores_used[1]) > 0):
core_speech = core_speech + " and "
if (int(cores_used[1]) > 0):
core_speech = cores_used[1] + " new " + p.plural(
"core", int(cores_used[1]))
RET = SPEECH + core_speech
# Next launch date (short form)
if (task == "next-date-short"):
if (parameter == "speech"):
RET = convert_date_to_speech(result['launch_date_utc'], "short")
if (parameter == "text"):
longlaunchdateutc = str(UTC_to_local(result['launch_date_utc'],
tz))
launchyear = longlaunchdateutc[0:4]
launchmonth = num_to_month(int(longlaunchdateutc[6:7]))
launchday = p.ordinal(int(longlaunchdateutc[8:10]))
launchhour = longlaunchdateutc[11:13]
launchmin = longlaunchdateutc[14:16]
RET = launchday + " of " + launchmonth + " " + launchyear + " at " + launchhour + ":" + launchmin
if (tz == "UTC"):
RET = RET + " UTC"
else:
RET = RET + " local time"
# retrieve launch images
if (task == "image"):
RET = get_image(result, parameter)
return RET
def ali3d_MPI(stack,
ref_vol,
outdir,
maskfile=None,
ir=1,
ou=-1,
rs=1,
xr="4 2 2 1",
yr="-1",
ts="1 1 0.5 0.25",
delta="10 6 4 4",
an="-1",
center=0,
maxit=5,
term=95,
CTF=False,
fourvar=False,
snr=1.0,
ref_a="S",
sym="c1",
sort=True,
cutoff=999.99,
pix_cutoff="0",
two_tail=False,
model_jump="1 1 1 1 1",
restart=False,
save_half=False,
protos=None,
oplane=None,
lmask=-1,
ilmask=-1,
findseam=False,
vertstep=None,
hpars="-1",
hsearch="0.0 50.0",
full_output=False,
compare_repro=False,
compare_ref_free="-1",
ref_free_cutoff="-1 -1 -1 -1",
wcmask=None,
debug=False,
recon_pad=4,
olmask=75):
from alignment import Numrinit, prepare_refrings
from utilities import model_circle, get_image, drop_image, get_input_from_string
from utilities import bcast_list_to_all, bcast_number_to_all, reduce_EMData_to_root, bcast_EMData_to_all
from utilities import send_attr_dict
from utilities import get_params_proj, file_type
from fundamentals import rot_avg_image
import os
import types
from utilities import print_begin_msg, print_end_msg, print_msg
from mpi import mpi_bcast, mpi_comm_size, mpi_comm_rank, MPI_FLOAT, MPI_COMM_WORLD, mpi_barrier, mpi_reduce
from mpi import mpi_reduce, MPI_INT, MPI_SUM, mpi_finalize
from filter import filt_ctf
from projection import prep_vol, prgs
from statistics import hist_list, varf3d_MPI, fsc_mask
from numpy import array, bincount, array2string, ones
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
if myid == main_node:
if os.path.exists(outdir):
ERROR(
'Output directory exists, please change the name and restart the program',
"ali3d_MPI", 1)
os.mkdir(outdir)
mpi_barrier(MPI_COMM_WORLD)
if debug:
from time import sleep
while not os.path.exists(outdir):
print "Node ", myid, " waiting..."
sleep(5)
info_file = os.path.join(outdir, "progress%04d" % myid)
finfo = open(info_file, 'w')
else:
finfo = None
mjump = get_input_from_string(model_jump)
xrng = get_input_from_string(xr)
if yr == "-1": yrng = xrng
else: yrng = get_input_from_string(yr)
step = get_input_from_string(ts)
delta = get_input_from_string(delta)
ref_free_cutoff = get_input_from_string(ref_free_cutoff)
pix_cutoff = get_input_from_string(pix_cutoff)
lstp = min(len(xrng), len(yrng), len(step), len(delta))
if an == "-1":
an = [-1] * lstp
else:
an = get_input_from_string(an)
# make sure pix_cutoff is set for all iterations
if len(pix_cutoff) < lstp:
for i in xrange(len(pix_cutoff), lstp):
pix_cutoff.append(pix_cutoff[-1])
# don't waste time on sub-pixel alignment for low-resolution ang incr
for i in range(len(step)):
if (delta[i] > 4 or delta[i] == -1) and step[i] < 1:
step[i] = 1
first_ring = int(ir)
rstep = int(rs)
last_ring = int(ou)
max_iter = int(maxit)
center = int(center)
nrefs = EMUtil.get_image_count(ref_vol)
nmasks = 0
if maskfile:
# read number of masks within each maskfile (mc)
nmasks = EMUtil.get_image_count(maskfile)
# open masks within maskfile (mc)
maskF = EMData.read_images(maskfile, xrange(nmasks))
vol = EMData.read_images(ref_vol, xrange(nrefs))
nx = vol[0].get_xsize()
## make sure box sizes are the same
if myid == main_node:
im = EMData.read_images(stack, [0])
bx = im[0].get_xsize()
if bx != nx:
print_msg(
"Error: Stack box size (%i) differs from initial model (%i)\n"
% (bx, nx))
sys.exit()
del im, bx
# for helical processing:
helicalrecon = False
if protos is not None or hpars != "-1" or findseam is True:
helicalrecon = True
# if no out-of-plane param set, use 5 degrees
if oplane is None:
oplane = 5.0
if protos is not None:
proto = get_input_from_string(protos)
if len(proto) != nrefs:
print_msg("Error: insufficient protofilament numbers supplied")
sys.exit()
if hpars != "-1":
hpars = get_input_from_string(hpars)
if len(hpars) != 2 * nrefs:
print_msg("Error: insufficient helical parameters supplied")
sys.exit()
## create helical parameter file for helical reconstruction
if helicalrecon is True and myid == main_node:
from hfunctions import createHpar
# create initial helical parameter files
dp = [0] * nrefs
dphi = [0] * nrefs
vdp = [0] * nrefs
vdphi = [0] * nrefs
for iref in xrange(nrefs):
hpar = os.path.join(outdir, "hpar%02d.spi" % (iref))
params = False
if hpars != "-1":
# if helical parameters explicitly given, set twist & rise
params = [float(hpars[iref * 2]), float(hpars[(iref * 2) + 1])]
dp[iref], dphi[iref], vdp[iref], vdphi[iref] = createHpar(
hpar, proto[iref], params, vertstep)
# get values for helical search parameters
hsearch = get_input_from_string(hsearch)
if len(hsearch) != 2:
print_msg("Error: specify outer and inner radii for helical search")
sys.exit()
if last_ring < 0 or last_ring > int(nx / 2) - 2:
last_ring = int(nx / 2) - 2
if myid == main_node:
# import user_functions
# user_func = user_functions.factory[user_func_name]
print_begin_msg("ali3d_MPI")
print_msg("Input stack : %s\n" % (stack))
print_msg("Reference volume : %s\n" % (ref_vol))
print_msg("Output directory : %s\n" % (outdir))
if nmasks > 0:
print_msg("Maskfile (number of masks) : %s (%i)\n" %
(maskfile, nmasks))
print_msg("Inner radius : %i\n" % (first_ring))
print_msg("Outer radius : %i\n" % (last_ring))
print_msg("Ring step : %i\n" % (rstep))
print_msg("X search range : %s\n" % (xrng))
print_msg("Y search range : %s\n" % (yrng))
print_msg("Translational step : %s\n" % (step))
print_msg("Angular step : %s\n" % (delta))
print_msg("Angular search range : %s\n" % (an))
print_msg("Maximum iteration : %i\n" % (max_iter))
print_msg("Center type : %i\n" % (center))
print_msg("CTF correction : %s\n" % (CTF))
print_msg("Signal-to-Noise Ratio : %f\n" % (snr))
print_msg("Reference projection method : %s\n" % (ref_a))
print_msg("Symmetry group : %s\n" % (sym))
print_msg("Fourier padding for 3D : %i\n" % (recon_pad))
print_msg("Number of reference models : %i\n" % (nrefs))
print_msg("Sort images between models : %s\n" % (sort))
print_msg("Allow images to jump : %s\n" % (mjump))
print_msg("CC cutoff standard dev : %f\n" % (cutoff))
print_msg("Two tail cutoff : %s\n" % (two_tail))
print_msg("Termination pix error : %f\n" % (term))
print_msg("Pixel error cutoff : %s\n" % (pix_cutoff))
print_msg("Restart : %s\n" % (restart))
print_msg("Full output : %s\n" % (full_output))
print_msg("Compare reprojections : %s\n" % (compare_repro))
print_msg("Compare ref free class avgs : %s\n" % (compare_ref_free))
print_msg("Use cutoff from ref free : %s\n" % (ref_free_cutoff))
if protos:
print_msg("Protofilament numbers : %s\n" % (proto))
print_msg("Using helical search range : %s\n" % hsearch)
if findseam is True:
print_msg("Using seam-based reconstruction\n")
if hpars != "-1":
print_msg("Using hpars : %s\n" % hpars)
if vertstep != None:
print_msg("Using vertical step : %.2f\n" % vertstep)
if save_half is True:
print_msg("Saving even/odd halves\n")
for i in xrange(100):
print_msg("*")
print_msg("\n\n")
if maskfile:
if type(maskfile) is types.StringType: mask3D = get_image(maskfile)
else: mask3D = maskfile
else: mask3D = model_circle(last_ring, nx, nx, nx)
numr = Numrinit(first_ring, last_ring, rstep, "F")
mask2D = model_circle(last_ring, nx, nx) - model_circle(first_ring, nx, nx)
fscmask = model_circle(last_ring, nx, nx, nx)
if CTF:
from filter import filt_ctf
from reconstruction_rjh import rec3D_MPI_noCTF
if myid == main_node:
active = EMUtil.get_all_attributes(stack, 'active')
list_of_particles = []
for im in xrange(len(active)):
if active[im]: list_of_particles.append(im)
del active
nima = len(list_of_particles)
else:
nima = 0
total_nima = bcast_number_to_all(nima, source_node=main_node)
if myid != main_node:
list_of_particles = [-1] * total_nima
list_of_particles = bcast_list_to_all(list_of_particles,
source_node=main_node)
image_start, image_end = MPI_start_end(total_nima, number_of_proc, myid)
# create a list of images for each node
list_of_particles = list_of_particles[image_start:image_end]
nima = len(list_of_particles)
if debug:
finfo.write("image_start, image_end: %d %d\n" %
(image_start, image_end))
finfo.flush()
data = EMData.read_images(stack, list_of_particles)
t_zero = Transform({
"type": "spider",
"phi": 0,
"theta": 0,
"psi": 0,
"tx": 0,
"ty": 0
})
transmulti = [[t_zero for i in xrange(nrefs)] for j in xrange(nima)]
for iref, im in ((iref, im) for iref in xrange(nrefs)
for im in xrange(nima)):
if nrefs == 1:
transmulti[im][iref] = data[im].get_attr("xform.projection")
else:
# if multi models, keep track of eulers for all models
try:
transmulti[im][iref] = data[im].get_attr("eulers_txty.%i" %
iref)
except:
data[im].set_attr("eulers_txty.%i" % iref, t_zero)
scoremulti = [[0.0 for i in xrange(nrefs)] for j in xrange(nima)]
pixelmulti = [[0.0 for i in xrange(nrefs)] for j in xrange(nima)]
ref_res = [0.0 for x in xrange(nrefs)]
apix = data[0].get_attr('apix_x')
# for oplane parameter, create cylindrical mask
if oplane is not None and myid == main_node:
from hfunctions import createCylMask
cmaskf = os.path.join(outdir, "mask3D_cyl.mrc")
mask3D = createCylMask(data, olmask, lmask, ilmask, cmaskf)
# if finding seam of helix, create wedge masks
if findseam is True:
wedgemask = []
for pf in xrange(nrefs):
wedgemask.append(EMData())
# wedgemask option
if wcmask is not None:
wcmask = get_input_from_string(wcmask)
if len(wcmask) != 3:
print_msg(
"Error: wcmask option requires 3 values: x y radius")
sys.exit()
# determine if particles have helix info:
try:
data[0].get_attr('h_angle')
original_data = []
boxmask = True
from hfunctions import createBoxMask
except:
boxmask = False
# prepare particles
for im in xrange(nima):
data[im].set_attr('ID', list_of_particles[im])
data[im].set_attr('pix_score', int(0))
if CTF:
# only phaseflip particles, not full CTF correction
ctf_params = data[im].get_attr("ctf")
st = Util.infomask(data[im], mask2D, False)
data[im] -= st[0]
data[im] = filt_ctf(data[im], ctf_params, sign=-1, binary=1)
data[im].set_attr('ctf_applied', 1)
# for window mask:
if boxmask is True:
h_angle = data[im].get_attr("h_angle")
original_data.append(data[im].copy())
bmask = createBoxMask(nx, apix, ou, lmask, h_angle)
data[im] *= bmask
del bmask
if debug:
finfo.write('%d loaded \n' % nima)
finfo.flush()
if myid == main_node:
# initialize data for the reference preparation function
ref_data = [mask3D, max(center, 0), None, None, None, None]
# for method -1, switch off centering in user function
from time import time
# this is needed for gathering of pixel errors
disps = []
recvcount = []
disps_score = []
recvcount_score = []
for im in xrange(number_of_proc):
if (im == main_node):
disps.append(0)
disps_score.append(0)
else:
disps.append(disps[im - 1] + recvcount[im - 1])
disps_score.append(disps_score[im - 1] + recvcount_score[im - 1])
ib, ie = MPI_start_end(total_nima, number_of_proc, im)
recvcount.append(ie - ib)
recvcount_score.append((ie - ib) * nrefs)
pixer = [0.0] * nima
cs = [0.0] * 3
total_iter = 0
volodd = EMData.read_images(ref_vol, xrange(nrefs))
voleve = EMData.read_images(ref_vol, xrange(nrefs))
if restart:
# recreate initial volumes from alignments stored in header
itout = "000_00"
for iref in xrange(nrefs):
if (nrefs == 1):
modout = ""
else:
modout = "_model_%02d" % (iref)
if (sort):
group = iref
for im in xrange(nima):
imgroup = data[im].get_attr('group')
if imgroup == iref:
data[im].set_attr('xform.projection',
transmulti[im][iref])
else:
group = int(999)
for im in xrange(nima):
data[im].set_attr('xform.projection', transmulti[im][iref])
fscfile = os.path.join(outdir, "fsc_%s%s" % (itout, modout))
vol[iref], fscc, volodd[iref], voleve[iref] = rec3D_MPI_noCTF(
data,
sym,
fscmask,
fscfile,
myid,
main_node,
index=group,
npad=recon_pad)
if myid == main_node:
if helicalrecon:
from hfunctions import processHelicalVol
vstep = None
if vertstep is not None:
vstep = (vdp[iref], vdphi[iref])
print_msg(
"Old rise and twist for model %i : %8.3f, %8.3f\n"
% (iref, dp[iref], dphi[iref]))
hvals = processHelicalVol(vol[iref], voleve[iref],
volodd[iref], iref, outdir,
itout, dp[iref], dphi[iref],
apix, hsearch, findseam, vstep,
wcmask)
(vol[iref], voleve[iref], volodd[iref], dp[iref],
dphi[iref], vdp[iref], vdphi[iref]) = hvals
print_msg(
"New rise and twist for model %i : %8.3f, %8.3f\n"
% (iref, dp[iref], dphi[iref]))
# get new FSC from symmetrized half volumes
fscc = fsc_mask(volodd[iref], voleve[iref], mask3D, rstep,
fscfile)
else:
vol[iref].write_image(
os.path.join(outdir, "vol_%s.hdf" % itout), -1)
if save_half is True:
volodd[iref].write_image(
os.path.join(outdir, "volodd_%s.hdf" % itout), -1)
voleve[iref].write_image(
os.path.join(outdir, "voleve_%s.hdf" % itout), -1)
if nmasks > 1:
# Read mask for multiplying
ref_data[0] = maskF[iref]
ref_data[2] = vol[iref]
ref_data[3] = fscc
# call user-supplied function to prepare reference image, i.e., center and filter it
vol[iref], cs, fl = ref_ali3d(ref_data)
vol[iref].write_image(
os.path.join(outdir, "volf_%s.hdf" % (itout)), -1)
if (apix == 1):
res_msg = "Models filtered at spatial frequency of:\t"
res = fl
else:
res_msg = "Models filtered at resolution of: \t"
res = apix / fl
ares = array2string(array(res), precision=2)
print_msg("%s%s\n\n" % (res_msg, ares))
bcast_EMData_to_all(vol[iref], myid, main_node)
# write out headers, under MPI writing has to be done sequentially
mpi_barrier(MPI_COMM_WORLD)
# projection matching
for N_step in xrange(lstp):
terminate = 0
Iter = -1
while (Iter < max_iter - 1 and terminate == 0):
Iter += 1
total_iter += 1
itout = "%03g_%02d" % (delta[N_step], Iter)
if myid == main_node:
print_msg(
"ITERATION #%3d, inner iteration #%3d\nDelta = %4.1f, an = %5.2f, xrange = %5.2f, yrange = %5.2f, step = %5.2f\n\n"
% (N_step, Iter, delta[N_step], an[N_step], xrng[N_step],
yrng[N_step], step[N_step]))
for iref in xrange(nrefs):
if myid == main_node: start_time = time()
volft, kb = prep_vol(vol[iref])
## constrain projections to out of plane parameter
theta1 = None
theta2 = None
if oplane is not None:
theta1 = 90 - oplane
theta2 = 90 + oplane
refrings = prepare_refrings(volft,
kb,
nx,
delta[N_step],
ref_a,
sym,
numr,
MPI=True,
phiEqpsi="Minus",
initial_theta=theta1,
delta_theta=theta2)
del volft, kb
if myid == main_node:
print_msg(
"Time to prepare projections for model %i: %s\n" %
(iref, legibleTime(time() - start_time)))
start_time = time()
for im in xrange(nima):
data[im].set_attr("xform.projection", transmulti[im][iref])
if an[N_step] == -1:
t1, peak, pixer[im] = proj_ali_incore(
data[im], refrings, numr, xrng[N_step],
yrng[N_step], step[N_step], finfo)
else:
t1, peak, pixer[im] = proj_ali_incore_local(
data[im], refrings, numr, xrng[N_step],
yrng[N_step], step[N_step], an[N_step], finfo)
#data[im].set_attr("xform.projection"%iref, t1)
if nrefs > 1:
data[im].set_attr("eulers_txty.%i" % iref, t1)
scoremulti[im][iref] = peak
from pixel_error import max_3D_pixel_error
# t1 is the current param, t2 is old
t2 = transmulti[im][iref]
pixelmulti[im][iref] = max_3D_pixel_error(t1, t2, numr[-3])
transmulti[im][iref] = t1
if myid == main_node:
print_msg("Time of alignment for model %i: %s\n" %
(iref, legibleTime(time() - start_time)))
start_time = time()
# gather scoring data from all processors
from mpi import mpi_gatherv
scoremultisend = sum(scoremulti, [])
pixelmultisend = sum(pixelmulti, [])
tmp = mpi_gatherv(scoremultisend, len(scoremultisend), MPI_FLOAT,
recvcount_score, disps_score, MPI_FLOAT,
main_node, MPI_COMM_WORLD)
tmp1 = mpi_gatherv(pixelmultisend, len(pixelmultisend), MPI_FLOAT,
recvcount_score, disps_score, MPI_FLOAT,
main_node, MPI_COMM_WORLD)
tmp = mpi_bcast(tmp, (total_nima * nrefs), MPI_FLOAT, 0,
MPI_COMM_WORLD)
tmp1 = mpi_bcast(tmp1, (total_nima * nrefs), MPI_FLOAT, 0,
MPI_COMM_WORLD)
tmp = map(float, tmp)
tmp1 = map(float, tmp1)
score = array(tmp).reshape(-1, nrefs)
pixelerror = array(tmp1).reshape(-1, nrefs)
score_local = array(scoremulti)
mean_score = score.mean(axis=0)
std_score = score.std(axis=0)
cut = mean_score - (cutoff * std_score)
cut2 = mean_score + (cutoff * std_score)
res_max = score_local.argmax(axis=1)
minus_cc = [0.0 for x in xrange(nrefs)]
minus_pix = [0.0 for x in xrange(nrefs)]
minus_ref = [0.0 for x in xrange(nrefs)]
#output pixel errors
if (myid == main_node):
from statistics import hist_list
lhist = 20
pixmin = pixelerror.min(axis=1)
region, histo = hist_list(pixmin, lhist)
if (region[0] < 0.0): region[0] = 0.0
print_msg(
"Histogram of pixel errors\n ERROR number of particles\n"
)
for lhx in xrange(lhist):
print_msg(" %10.3f %7d\n" % (region[lhx], histo[lhx]))
# Terminate if 95% within 1 pixel error
im = 0
for lhx in xrange(lhist):
if (region[lhx] > 1.0): break
im += histo[lhx]
print_msg("Percent of particles with pixel error < 1: %f\n\n" %
(im / float(total_nima) * 100))
term_cond = float(term) / 100
if (im / float(total_nima) > term_cond):
terminate = 1
print_msg("Terminating internal loop\n")
del region, histo
terminate = mpi_bcast(terminate, 1, MPI_INT, 0, MPI_COMM_WORLD)
terminate = int(terminate[0])
for im in xrange(nima):
if (sort == False):
data[im].set_attr('group', 999)
elif (mjump[N_step] == 1):
data[im].set_attr('group', int(res_max[im]))
pix_run = data[im].get_attr('pix_score')
if (pix_cutoff[N_step] == 1
and (terminate == 1 or Iter == max_iter - 1)):
if (pixelmulti[im][int(res_max[im])] > 1):
data[im].set_attr('pix_score', int(777))
if (score_local[im][int(res_max[im])] < cut[int(
res_max[im])]) or (two_tail and score_local[im][int(
res_max[im])] > cut2[int(res_max[im])]):
data[im].set_attr('group', int(888))
minus_cc[int(res_max[im])] = minus_cc[int(res_max[im])] + 1
if (pix_run == 777):
data[im].set_attr('group', int(777))
minus_pix[int(
res_max[im])] = minus_pix[int(res_max[im])] + 1
if (compare_ref_free != "-1") and (ref_free_cutoff[N_step] !=
-1) and (total_iter > 1):
id = data[im].get_attr('ID')
if id in rejects:
data[im].set_attr('group', int(666))
minus_ref[int(
res_max[im])] = minus_ref[int(res_max[im])] + 1
minus_cc_tot = mpi_reduce(minus_cc, nrefs, MPI_FLOAT, MPI_SUM, 0,
MPI_COMM_WORLD)
minus_pix_tot = mpi_reduce(minus_pix, nrefs, MPI_FLOAT, MPI_SUM, 0,
MPI_COMM_WORLD)
minus_ref_tot = mpi_reduce(minus_ref, nrefs, MPI_FLOAT, MPI_SUM, 0,
MPI_COMM_WORLD)
if (myid == main_node):
if (sort):
tot_max = score.argmax(axis=1)
res = bincount(tot_max)
else:
res = ones(nrefs) * total_nima
print_msg("Particle distribution: \t\t%s\n" % (res * 1.0))
afcut1 = res - minus_cc_tot
afcut2 = afcut1 - minus_pix_tot
afcut3 = afcut2 - minus_ref_tot
print_msg("Particle distribution after cc cutoff:\t\t%s\n" %
(afcut1))
print_msg("Particle distribution after pix cutoff:\t\t%s\n" %
(afcut2))
print_msg("Particle distribution after ref cutoff:\t\t%s\n\n" %
(afcut3))
res = [0.0 for i in xrange(nrefs)]
for iref in xrange(nrefs):
if (center == -1):
from utilities import estimate_3D_center_MPI, rotate_3D_shift
dummy = EMData()
cs[0], cs[1], cs[2], dummy, dummy = estimate_3D_center_MPI(
data, total_nima, myid, number_of_proc, main_node)
cs = mpi_bcast(cs, 3, MPI_FLOAT, main_node, MPI_COMM_WORLD)
cs = [-float(cs[0]), -float(cs[1]), -float(cs[2])]
rotate_3D_shift(data, cs)
if (sort):
group = iref
for im in xrange(nima):
imgroup = data[im].get_attr('group')
if imgroup == iref:
data[im].set_attr('xform.projection',
transmulti[im][iref])
else:
group = int(999)
for im in xrange(nima):
data[im].set_attr('xform.projection',
transmulti[im][iref])
if (nrefs == 1):
modout = ""
else:
modout = "_model_%02d" % (iref)
fscfile = os.path.join(outdir, "fsc_%s%s" % (itout, modout))
vol[iref], fscc, volodd[iref], voleve[iref] = rec3D_MPI_noCTF(
data,
sym,
fscmask,
fscfile,
myid,
main_node,
index=group,
npad=recon_pad)
if myid == main_node:
print_msg("3D reconstruction time for model %i: %s\n" %
(iref, legibleTime(time() - start_time)))
start_time = time()
# Compute Fourier variance
if fourvar:
outvar = os.path.join(outdir, "volVar_%s.hdf" % (itout))
ssnr_file = os.path.join(outdir, "ssnr_%s" % (itout))
varf = varf3d_MPI(data,
ssnr_text_file=ssnr_file,
mask2D=None,
reference_structure=vol[iref],
ou=last_ring,
rw=1.0,
npad=1,
CTF=None,
sign=1,
sym=sym,
myid=myid)
if myid == main_node:
print_msg(
"Time to calculate 3D Fourier variance for model %i: %s\n"
% (iref, legibleTime(time() - start_time)))
start_time = time()
varf = 1.0 / varf
varf.write_image(outvar, -1)
else:
varf = None
if myid == main_node:
if helicalrecon:
from hfunctions import processHelicalVol
vstep = None
if vertstep is not None:
vstep = (vdp[iref], vdphi[iref])
print_msg(
"Old rise and twist for model %i : %8.3f, %8.3f\n"
% (iref, dp[iref], dphi[iref]))
hvals = processHelicalVol(vol[iref], voleve[iref],
volodd[iref], iref, outdir,
itout, dp[iref], dphi[iref],
apix, hsearch, findseam,
vstep, wcmask)
(vol[iref], voleve[iref], volodd[iref], dp[iref],
dphi[iref], vdp[iref], vdphi[iref]) = hvals
print_msg(
"New rise and twist for model %i : %8.3f, %8.3f\n"
% (iref, dp[iref], dphi[iref]))
# get new FSC from symmetrized half volumes
fscc = fsc_mask(volodd[iref], voleve[iref], mask3D,
rstep, fscfile)
print_msg(
"Time to search and apply helical symmetry for model %i: %s\n\n"
% (iref, legibleTime(time() - start_time)))
start_time = time()
else:
vol[iref].write_image(
os.path.join(outdir, "vol_%s.hdf" % (itout)), -1)
if save_half is True:
volodd[iref].write_image(
os.path.join(outdir, "volodd_%s.hdf" % (itout)),
-1)
voleve[iref].write_image(
os.path.join(outdir, "voleve_%s.hdf" % (itout)),
-1)
if nmasks > 1:
# Read mask for multiplying
ref_data[0] = maskF[iref]
ref_data[2] = vol[iref]
ref_data[3] = fscc
ref_data[4] = varf
# call user-supplied function to prepare reference image, i.e., center and filter it
vol[iref], cs, fl = ref_ali3d(ref_data)
vol[iref].write_image(
os.path.join(outdir, "volf_%s.hdf" % (itout)), -1)
if (apix == 1):
res_msg = "Models filtered at spatial frequency of:\t"
res[iref] = fl
else:
res_msg = "Models filtered at resolution of: \t"
res[iref] = apix / fl
del varf
bcast_EMData_to_all(vol[iref], myid, main_node)
if compare_ref_free != "-1": compare_repro = True
if compare_repro:
outfile_repro = comp_rep(refrings, data, itout, modout,
vol[iref], group, nima, nx, myid,
main_node, outdir)
mpi_barrier(MPI_COMM_WORLD)
if compare_ref_free != "-1":
ref_free_output = os.path.join(
outdir, "ref_free_%s%s" % (itout, modout))
rejects = compare(compare_ref_free, outfile_repro,
ref_free_output, yrng[N_step],
xrng[N_step], rstep, nx, apix,
ref_free_cutoff[N_step],
number_of_proc, myid, main_node)
# retrieve alignment params from all processors
par_str = ['xform.projection', 'ID', 'group']
if nrefs > 1:
for iref in xrange(nrefs):
par_str.append('eulers_txty.%i' % iref)
if myid == main_node:
from utilities import recv_attr_dict
recv_attr_dict(main_node, stack, data, par_str, image_start,
image_end, number_of_proc)
else:
send_attr_dict(main_node, data, par_str, image_start,
image_end)
if myid == main_node:
ares = array2string(array(res), precision=2)
print_msg("%s%s\n\n" % (res_msg, ares))
dummy = EMData()
if full_output:
nimat = EMUtil.get_image_count(stack)
output_file = os.path.join(outdir, "paramout_%s" % itout)
foutput = open(output_file, 'w')
for im in xrange(nimat):
# save the parameters for each of the models
outstring = ""
dummy.read_image(stack, im, True)
param3d = dummy.get_attr('xform.projection')
g = dummy.get_attr("group")
# retrieve alignments in EMAN-format
pE = param3d.get_params('eman')
outstring += "%f\t%f\t%f\t%f\t%f\t%i\n" % (
pE["az"], pE["alt"], pE["phi"], pE["tx"], pE["ty"],
g)
foutput.write(outstring)
foutput.close()
del dummy
mpi_barrier(MPI_COMM_WORLD)
# mpi_finalize()
if myid == main_node: print_end_msg("ali3d_MPI")
def defocus_get_fast(indir, writetodoc="w", Pixel_size=1, volt=120, Cs=2, wgh=.1, round_off=100, dz_max0=50000, f_l0=30, f_h0=5, nr_1=5, nr_2=5, prefix="roo", docf="a",skip="#", micdir="no", print_screen="p"):
"""
Estimate defocus using user supplied 1D power spectrum area
writetodoc="a" return the estimated defoci in a list, and write them down also in a text file
writetodoc="l" output estimated defocus in a list
writetodoc="w" output estimated defocus in a text file
"""
import os
import types
from utilities import set_arb_params, get_image
if writetodoc[0] != "a" and writetodoc[0] != "l" and writetodoc[0] != "a": writetodoc= "a"
if print_screen[0] != "p" and print_screen[0] != "n" : print_screen = "n"
if os.path.exists(indir) == False : ERROR("roodir doesn't exist", "defocus_get_fast",1)
ctf_dicts = ["defocus", "Pixel_size", "voltage", "Cs", "amp_contrast", "B_factor", "sign"]
flist = os.listdir(indir)
res = []
f_l = f_l0
f_h = f_h0
if(f_l <= 1 and f_l> 0) :
f_l = 1./f_l
f_h = 1./f_h
if(f_h > f_l or f_l <= 0 or f_h <= 0):
f_h = 8
f_l = 30
if nr_1 <= 1 : nr_1 = 5.
if nr_2 <= 1 : nr_2 = 5.
if round_off <= 0 : round_off = 100.
if dz_max0 <= 1 : dz_max0 = 100000.
dz_max=dz_max0
if writetodoc[0] == "w" or writetodoc == "a":
fdefo_nam = "defocus.txt"
out = open(fdefo_nam, "w")
out.write("#defocus: %s\n")
defocus = 0
ncount = 0
for i, v in enumerate(flist):
(fnam, fext) = os.path.splitext(v)
if(fnam[0:len(prefix)] == prefix):
ncount += 1
fnam_root = fnam[len(prefix):]
nr1 = int(nr_1)
nr2 = int(nr_2)
istart = int(f_l)
istop = int(f_h)
fnam_roo = os.path.join(indir, v)
defocus = defocus_get(fnam_roo, volt, Pixel_size, Cs, wgh, istart, istop, docf, skip, round_off, nr1, nr2)
if(defocus > dz_max):
while(nr1 <= 7 or nr2 <= 7):
nr1 += 1
nr2 += 1
defocus = defocus_get(fnam_roo, volt, Pixel_size, Cs, wgh,istart, istop, docf,skip, round_off, nr1, nr2)
#if(print_screen[0] == "p" or print_screen[0] == "P" ): print "defocus",defocus,"Euclidean distance",dis,"starting feq",istart,"stop freq",istop,"P R E", nr1,"P R B", nr2
if(defocus<dz_max): break
if(defocus > dz_max):
while(nr1 >= 2 and nr2 >= 2):
nr1 -= 1
nr2 -= 1
defocus = defocus_get(fnam_roo, volt,Pixel_size, Cs, wgh, istart, istop, docf, skip, round_off, nr1, nr2)
#if(print_sreen[0] == "p" or print_screen=="P"): print "defocus",defocus,"Euclidean distance",dis,"starting feq",istart,"stop freq",istop,"P R E", nr1,"P R B", nr2
if(defocus < dz_max): break
if(defocus > dz_max):
while(istart > istop):
nr1 = 5
nr2 = 5
istart -=.5
defocus = defocus_get(fnam_roo, volt, Pixel_size, Cs, wgh, istart, istop, docf,skip, round_off, nr1, nr2)
#if(print_screen[0] == "p" or print_screen == "P"): print "defocus",defocus,"Euclidean distance",dis,"starting feq",istart,"stop freq",istop,"P R E", nr1,"P R B", nr2
if(defocus < dz_max): break
if(defocus > dz_max):
while(istart > istop):
nr1 = 5
nr2 = 5
istop += 0.5
defocus = defocus_get(fnam_roo, volt, Pixel_size, Cs, wgh, istart, istop, docf, skip, round_off, nr1, nr2)
if(print_screen == "p" or print_screen == "P"): print "defocus",defocus,"Euclidean distance", dis, "starting feq", istart, "stop freq", istop,"P R E", nr1,"P R B", nr2
if(defocus < dz_max): break
if(defocus >= dz_max): ERROR("defocus_get_fast fails at estimating defocus", fnam, action = 0)
print "micrograph", flist[i], '%5d'%(defocus) # screen output, give the user a general impression about estimated defoci
if(writetodoc[0] == "w" or writetodoc[0] != "l"): out.write("%d\t%5d\t%s\n" % (ncount,defocus,flist[i]))
if(writetodoc[0] == "l"): res.append(defocus)
if type(micdir) is types.StringType :
ctf_param = [defocus, Pixel_size, volt, Cs, wgh, 0, 1]
mic_name = os.path.join(micdir,"micrograph"+ fnam_root+ ".hdf")
if os.path.exists(mic_name) :
e = get_image (mic_name)
U______set_arb_params(e, ctf_param, ctf_dicts) # THIS IS INCORRECT< PLEASE CHANGE
e.write_image(mic_name,0, EMUtil.ImageType.IMAGE_HDF, True)
print "ctf parameters is written back into headers of ", mic_name
#else : print mic_name, " Not found"
if(len(res) == 0 and writetodoc == "l" ): ERROR("No input file is found, check the input directory of file prefix", indir, 1)
else:
if(writetodoc[0] == "a"):
out.close()
return res
if(writetodoc[0] == "l"): return res
if(writetodoc[0] == "w"): out.close()
def rec3D_MPI_noCTF(data, symmetry, mask3D, fsc_curve, myid, main_node = 0, rstep = 1.0, odd_start=0, eve_start=1, finfo=None, index = -1, npad = 4, hparams=None):
'''
This function is to be called within an MPI program to do a reconstruction on a dataset kept in the memory
Computes reconstruction and through odd-even, in order to get the resolution
if index > -1, projections should have attribute group set and only those whose group matches index will be used in the reconstruction
this is for multireference alignment
'''
import os
from statistics import fsc_mask
from utilities import model_blank, reduce_EMData_to_root, get_image,send_EMData, recv_EMData
from random import randint
from mpi import mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD
nproc = mpi_comm_size(MPI_COMM_WORLD)
if nproc==1:
assert main_node==0
main_node_odd = main_node
main_node_eve = main_node
main_node_all = main_node
elif nproc==2:
main_node_odd = main_node
main_node_eve = (main_node+1)%2
main_node_all = main_node
tag_voleve = 1000
tag_fftvol_eve = 1001
tag_weight_eve = 1002
else:
#spread CPUs between different nodes to save memory
main_node_odd = main_node
main_node_eve = (int(main_node)+nproc-1)%int(nproc)
main_node_all = (int(main_node)+nproc//2)%int(nproc)
tag_voleve = 1000
tag_fftvol_eve = 1001
tag_weight_eve = 1002
tag_fftvol_odd = 1003
tag_weight_odd = 1004
tag_volall = 1005
nx = data[0].get_xsize()
fftvol_odd_file,weight_odd_file = prepare_recons(data, symmetry, myid, main_node_odd, odd_start, 2, index, finfo, npad)
fftvol_eve_file,weight_eve_file = prepare_recons(data, symmetry, myid, main_node_eve, eve_start, 2, index, finfo, npad)
if nproc == 1:
fftvol = get_image( fftvol_odd_file )
weight = get_image( weight_odd_file )
volodd = recons_from_fftvol(nx, fftvol, weight, symmetry, npad)
fftvol = get_image( fftvol_eve_file )
weight = get_image( weight_eve_file )
voleve = recons_from_fftvol(nx, fftvol, weight, symmetry, npad)
fftvol = get_image( fftvol_odd_file )
Util.add_img( fftvol, get_image(fftvol_eve_file) )
weight = get_image( weight_odd_file )
Util.add_img( weight, get_image(weight_eve_file) )
volall = recons_from_fftvol(nx, fftvol, weight, symmetry, npad)
# if helical, find & apply symmetry to volume
if hparams is not None:
volodd,voleve,volall = hsymVols(volodd,voleve,volall,hparams)
fscdat = fsc_mask( volodd, voleve, mask3D, rstep, fsc_curve)
os.system( "rm -f " + fftvol_odd_file + " " + weight_odd_file );
os.system( "rm -f " + fftvol_eve_file + " " + weight_eve_file );
return volall,fscdat,volodd,voleve
if nproc == 2:
if myid == main_node_odd:
fftvol = get_image( fftvol_odd_file )
weight = get_image( weight_odd_file )
volodd = recons_from_fftvol(nx, fftvol, weight, symmetry, npad)
voleve = recv_EMData(main_node_eve, tag_voleve)
else:
assert myid == main_node_eve
fftvol = get_image( fftvol_eve_file )
weight = get_image( weight_eve_file )
voleve = recons_from_fftvol(nx, fftvol, weight, symmetry, npad)
send_EMData(voleve, main_node_odd, tag_voleve)
if myid == main_node_odd:
fftvol = get_image( fftvol_odd_file )
fftvol_tmp = recv_EMData( main_node_eve, tag_fftvol_eve )
Util.add_img( fftvol, fftvol_tmp )
fftvol_tmp = None
weight = get_image( weight_odd_file )
weight_tmp = recv_EMData( main_node_eve, tag_weight_eve )
Util.add_img( weight, weight_tmp )
weight_tmp = None
volall = recons_from_fftvol(nx, fftvol, weight, symmetry, npad)
# if helical, find & apply symmetry to volume
if hparams is not None:
volodd,voleve,volall = hsymVols(volodd,voleve,volall,hparams)
fscdat = fsc_mask( volodd, voleve, mask3D, rstep, fsc_curve)
os.system( "rm -f " + fftvol_odd_file + " " + weight_odd_file );
os.system( "rm -f " + fftvol_eve_file + " " + weight_eve_file );
return volall,fscdat,volodd,voleve
else:
assert myid == main_node_eve
fftvol = get_image( fftvol_eve_file )
send_EMData(fftvol, main_node_odd, tag_fftvol_eve )
weight = get_image( weight_eve_file )
send_EMData(weight, main_node_odd, tag_weight_eve )
import os
os.system( "rm -f " + fftvol_eve_file + " " + weight_eve_file );
return model_blank(nx,nx,nx), None, model_blank(nx,nx,nx), model_blank(nx,nx,nx)
# cases from all other number of processors situations
if myid == main_node_odd:
fftvol = get_image( fftvol_odd_file )
send_EMData(fftvol, main_node_eve, tag_fftvol_odd )
if not(finfo is None):
finfo.write("fftvol odd sent\n")
finfo.flush()
weight = get_image( weight_odd_file )
send_EMData(weight, main_node_all, tag_weight_odd )
if not(finfo is None):
finfo.write("weight odd sent\n")
finfo.flush()
volodd = recons_from_fftvol(nx, fftvol, weight, symmetry, npad)
del fftvol, weight
voleve = recv_EMData(main_node_eve, tag_voleve)
volall = recv_EMData(main_node_all, tag_volall)
# if helical, find & apply symmetry to volume
if hparams is not None:
volodd,voleve,volall = hsymVols(volodd,voleve,volall,hparams)
fscdat = fsc_mask( volodd, voleve, mask3D, rstep, fsc_curve)
os.system( "rm -f " + fftvol_odd_file + " " + weight_odd_file );
return volall,fscdat,volodd,voleve
if myid == main_node_eve:
ftmp = recv_EMData(main_node_odd, tag_fftvol_odd)
fftvol = get_image( fftvol_eve_file )
Util.add_img( ftmp, fftvol )
send_EMData(ftmp, main_node_all, tag_fftvol_eve )
del ftmp
weight = get_image( weight_eve_file )
send_EMData(weight, main_node_all, tag_weight_eve )
voleve = recons_from_fftvol(nx, fftvol, weight, symmetry, npad)
send_EMData(voleve, main_node_odd, tag_voleve)
os.system( "rm -f " + fftvol_eve_file + " " + weight_eve_file );
return model_blank(nx,nx,nx), None, model_blank(nx,nx,nx), model_blank(nx,nx,nx)
if myid == main_node_all:
fftvol = recv_EMData(main_node_eve, tag_fftvol_eve)
if not(finfo is None):
finfo.write( "fftvol odd received\n" )
finfo.flush()
weight = recv_EMData(main_node_odd, tag_weight_odd)
weight_tmp = recv_EMData(main_node_eve, tag_weight_eve)
Util.add_img( weight, weight_tmp )
weight_tmp = None
volall = recons_from_fftvol(nx, fftvol, weight, symmetry, npad)
send_EMData(volall, main_node_odd, tag_volall)
return model_blank(nx,nx,nx),None, model_blank(nx,nx,nx), model_blank(nx,nx,nx)
return model_blank(nx,nx,nx), None, model_blank(nx,nx,nx), model_blank(nx,nx,nx)
def main():
def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
if mirror:
m = 1
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 540.0-psi, 0, 0, 1.0)
else:
m = 0
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 360.0-psi, 0, 0, 1.0)
return alpha, sx, sy, m
progname = os.path.basename(sys.argv[0])
usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl=0.2 --aa=0.1 --sym=symmetry --CTF"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--ave2D", type="string" , default=False, help="write to the disk a stack of 2D averages")
parser.add_option("--var2D", type="string" , default=False, help="write to the disk a stack of 2D variances")
parser.add_option("--ave3D", type="string" , default=False, help="write to the disk reconstructed 3D average")
parser.add_option("--var3D", type="string" , default=False, help="compute 3D variability (time consuming!)")
parser.add_option("--img_per_grp", type="int" , default=10 , help="number of neighbouring projections")
parser.add_option("--no_norm", action="store_true", default=False, help="do not use normalization")
parser.add_option("--radiusvar", type="int" , default=-1 , help="radius for 3D var" )
parser.add_option("--npad", type="int" , default=2 , help="number of time to pad the original images")
parser.add_option("--sym" , type="string" , default="c1" , help="symmetry")
parser.add_option("--fl", type="float" , default=0.0 , help="stop-band frequency (Default - no filtration)")
parser.add_option("--aa", type="float" , default=0.0 , help="fall off of the filter (Default - no filtration)")
parser.add_option("--CTF", action="store_true", default=False, help="use CFT correction")
parser.add_option("--VERBOSE", action="store_true", default=False, help="Long output for debugging")
#parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
#parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" )
#parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" )
#parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
#parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
parser.add_option("--VAR" , action="store_true", default=False, help="stack on input consists of 2D variances (Default False)")
parser.add_option("--decimate", type="float", default=1.0, help="image decimate rate, a number large than 1. default is 1")
parser.add_option("--window", type="int", default=0, help="reduce images to a small image size without changing pixel_size. Default value is zero.")
#parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)")
parser.add_option("--nvec", type="int" , default=0 , help="number of eigenvectors, default = 0 meaning no PCA calculated")
parser.add_option("--symmetrize", action="store_true", default=False, help="Prepare input stack for handling symmetry (Default False)")
(options,args) = parser.parse_args()
#####
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD, MPI_TAG_UB
from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
from applications import MPI_start_end
from reconstruction import recons3d_em, recons3d_em_MPI
from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
from utilities import print_begin_msg, print_end_msg, print_msg
from utilities import read_text_row, get_image, get_im
from utilities import bcast_EMData_to_all, bcast_number_to_all
from utilities import get_symt
# This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015
from EMAN2db import db_open_dict
if options.symmetrize :
try:
sys.argv = mpi_init(len(sys.argv), sys.argv)
try:
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
if( number_of_proc > 1 ):
ERROR("Cannot use more than one CPU for symmetry prepration","sx3dvariability",1)
except:
pass
except:
pass
# Input
#instack = "Clean_NORM_CTF_start_wparams.hdf"
#instack = "bdb:data"
instack = args[0]
sym = options.sym
if( sym == "c1" ):
ERROR("Thre is no need to symmetrize stack for C1 symmetry","sx3dvariability",1)
if(instack[:4] !="bdb:"):
stack = "bdb:data"
delete_bdb(stack)
cmdexecute("sxcpy.py "+instack+" "+stack)
else:
stack = instack
qt = EMUtil.get_all_attributes(stack,'xform.projection')
na = len(qt)
ts = get_symt(sym)
ks = len(ts)
angsa = [None]*na
for k in xrange(ks):
delete_bdb("bdb:Q%1d"%k)
cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
DB = db_open_dict("bdb:Q%1d"%k)
for i in xrange(na):
ut = qt[i]*ts[k]
DB.set_attr(i, "xform.projection", ut)
#bt = ut.get_params("spider")
#angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
#write_text_row(angsa, 'ptsma%1d.txt'%k)
#cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
#cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k))
DB.close()
delete_bdb("bdb:sdata")
cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
#cmdexecute("ls EMAN2DB/sdata*")
a = get_im("bdb:sdata")
a.set_attr("variabilitysymmetry",sym)
a.write_image("bdb:sdata")
else:
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
if len(args) == 1:
stack = args[0]
else:
print( "usage: " + usage)
print( "Please run '" + progname + " -h' for detailed options")
return 1
t0 = time()
# obsolete flags
options.MPI = True
options.nvec = 0
options.radiuspca = -1
options.iter = 40
options.abs = 0.0
options.squ = 0.0
if options.fl > 0.0 and options.aa == 0.0:
ERROR("Fall off has to be given for the low-pass filter", "sx3dvariability", 1, myid)
if options.VAR and options.SND:
ERROR("Only one of var and SND can be set!", "sx3dvariability", myid)
exit()
if options.VAR and (options.ave2D or options.ave3D or options.var2D):
ERROR("When VAR is set, the program cannot output ave2D, ave3D or var2D", "sx3dvariability", 1, myid)
exit()
#if options.SND and (options.ave2D or options.ave3D):
# ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid)
# exit()
if options.nvec > 0 :
ERROR("PCA option not implemented", "sx3dvariability", 1, myid)
exit()
if options.nvec > 0 and options.ave3D == None:
ERROR("When doing PCA analysis, one must set ave3D", "sx3dvariability", myid=myid)
exit()
import string
options.sym = options.sym.lower()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
global_def.BATCH = True
if myid == main_node:
print_begin_msg("sx3dvariability")
print_msg("%-70s: %s\n"%("Input stack", stack))
img_per_grp = options.img_per_grp
nvec = options.nvec
radiuspca = options.radiuspca
symbaselen = 0
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
if options.sym != "c1" :
imgdata = get_im(stack)
try:
i = imgdata.get_attr("variabilitysymmetry")
if(i != options.sym):
ERROR("The symmetry provided does not agree with the symmetry of the input stack", "sx3dvariability", myid=myid)
except:
ERROR("Input stack is not prepared for symmetry, please follow instructions", "sx3dvariability", myid=myid)
from utilities import get_symt
i = len(get_symt(options.sym))
if((nima/i)*i != nima):
ERROR("The length of the input stack is incorrect for symmetry processing", "sx3dvariability", myid=myid)
symbaselen = nima/i
else: symbaselen = nima
else:
nima = 0
nx = 0
ny = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
Tracker ={}
Tracker["nx"] =nx
Tracker["ny"] =ny
Tracker["total_stack"]=nima
if options.decimate==1.:
if options.window !=0:
nx = options.window
ny = options.window
else:
if options.window ==0:
nx = int(nx/options.decimate)
ny = int(ny/options.decimate)
else:
nx = int(options.window/options.decimate)
ny = nx
symbaselen = bcast_number_to_all(symbaselen)
if radiuspca == -1: radiuspca = nx/2-2
if myid == main_node:
print_msg("%-70s: %d\n"%("Number of projection", nima))
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
"""
if options.SND:
from projection import prep_vol, prgs
from statistics import im_diff
from utilities import get_im, model_circle, get_params_proj, set_params_proj
from utilities import get_ctf, generate_ctf
from filter import filt_ctf
imgdata = EMData.read_images(stack, range(img_begin, img_end))
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
bcast_EMData_to_all(vol, myid)
volft, kb = prep_vol(vol)
mask = model_circle(nx/2-2, nx, ny)
varList = []
for i in xrange(img_begin, img_end):
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
if options.CTF:
ctf_params = get_ctf(imgdata[i-img_begin])
ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
diff2 = diff*diff
set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
varList.append(diff2)
mpi_barrier(MPI_COMM_WORLD)
"""
if options.VAR:
#varList = EMData.read_images(stack, range(img_begin, img_end))
varList = []
this_image = EMData()
for index_of_particle in xrange(img_begin,img_end):
this_image.read_image(stack,index_of_particle)
varList.append(image_decimate_window_xform_ctf(img,options.decimate,options.window,options.CTF))
else:
from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
from utilities import model_blank, nearest_proj, model_circle
from applications import pca
from statistics import avgvar, avgvar_ctf, ccc
from filter import filt_tanl
from morphology import threshold, square_root
from projection import project, prep_vol, prgs
from sets import Set
if myid == main_node:
t1 = time()
proj_angles = []
aveList = []
tab = EMUtil.get_all_attributes(stack, 'xform.projection')
for i in xrange(nima):
t = tab[i].get_params('spider')
phi = t['phi']
theta = t['theta']
psi = t['psi']
x = theta
if x > 90.0: x = 180.0 - x
x = x*10000+psi
proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
t2 = time()
print_msg("%-70s: %d\n"%("Number of neighboring projections", img_per_grp))
print_msg("...... Finding neighboring projections\n")
if options.VERBOSE:
print "Number of images per group: ", img_per_grp
print "Now grouping projections"
proj_angles.sort()
proj_angles_list = [0.0]*(nima*4)
if myid == main_node:
for i in xrange(nima):
proj_angles_list[i*4] = proj_angles[i][1]
proj_angles_list[i*4+1] = proj_angles[i][2]
proj_angles_list[i*4+2] = proj_angles[i][3]
proj_angles_list[i*4+3] = proj_angles[i][4]
proj_angles_list = bcast_list_to_all(proj_angles_list, myid, main_node)
proj_angles = []
for i in xrange(nima):
proj_angles.append([proj_angles_list[i*4], proj_angles_list[i*4+1], proj_angles_list[i*4+2], int(proj_angles_list[i*4+3])])
del proj_angles_list
proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp, range(img_begin, img_end))
all_proj = Set()
for im in proj_list:
for jm in im:
all_proj.add(proj_angles[jm][3])
all_proj = list(all_proj)
if options.VERBOSE:
print "On node %2d, number of images needed to be read = %5d"%(myid, len(all_proj))
index = {}
for i in xrange(len(all_proj)): index[all_proj[i]] = i
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
print_msg("%-70s: %.2f\n"%("Finding neighboring projections lasted [s]", time()-t2))
print_msg("%-70s: %d\n"%("Number of groups processed on the main node", len(proj_list)))
if options.VERBOSE:
print "Grouping projections took: ", (time()-t2)/60 , "[min]"
print "Number of groups on main node: ", len(proj_list)
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
print_msg("...... calculating the stack of 2D variances \n")
if options.VERBOSE:
print "Now calculating the stack of 2D variances"
proj_params = [0.0]*(nima*5)
aveList = []
varList = []
if nvec > 0:
eigList = [[] for i in xrange(nvec)]
if options.VERBOSE: print "Begin to read images on processor %d"%(myid)
ttt = time()
#imgdata = EMData.read_images(stack, all_proj)
img = EMData()
imgdata = []
for index_of_proj in xrange(len(all_proj)):
img.read_image(stack, all_proj[index_of_proj])
dmg = image_decimate_window_xform_ctf(img,options.decimate,options.window,options.CTF)
#print dmg.get_xsize(), "init"
imgdata.append(dmg)
if options.VERBOSE:
print "Reading images on processor %d done, time = %.2f"%(myid, time()-ttt)
print "On processor %d, we got %d images"%(myid, len(imgdata))
mpi_barrier(MPI_COMM_WORLD)
'''
imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
if options.fl > 0.0:
for k in xrange(len(imgdata2)):
imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
if myid == main_node:
vol.write_image("vol_ctf.hdf")
print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf"))
del vol, imgdata2
mpi_barrier(MPI_COMM_WORLD)
'''
from applications import prepare_2d_forPCA
from utilities import model_blank
for i in xrange(len(proj_list)):
ki = proj_angles[proj_list[i][0]][3]
if ki >= symbaselen: continue
mi = index[ki]
phiM, thetaM, psiM, s2xM, s2yM = get_params_proj(imgdata[mi])
grp_imgdata = []
for j in xrange(img_per_grp):
mj = index[proj_angles[proj_list[i][j]][3]]
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[mj])
alpha, sx, sy, mirror = params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror_list[i][j])
if thetaM <= 90:
if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, phiM-phi, 0.0, 0.0, 1.0)
else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, 180-(phiM-phi), 0.0, 0.0, 1.0)
else:
if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(phiM-phi), 0.0, 0.0, 1.0)
else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(180-(phiM-phi)), 0.0, 0.0, 1.0)
set_params2D(imgdata[mj], [alpha, sx, sy, mirror, 1.0])
grp_imgdata.append(imgdata[mj])
#print grp_imgdata[j].get_xsize(), imgdata[mj].get_xsize()
if not options.no_norm:
#print grp_imgdata[j].get_xsize()
mask = model_circle(nx/2-2, nx, nx)
for k in xrange(img_per_grp):
ave, std, minn, maxx = Util.infomask(grp_imgdata[k], mask, False)
grp_imgdata[k] -= ave
grp_imgdata[k] /= std
del mask
if options.fl > 0.0:
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2*nx
ny2 = 2*ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d(fft( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa) ),nx,ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
for k in xrange(img_per_grp):
grp_imgdata[k] = filt_tanl( grp_imgdata[k], options.fl, options.aa)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
from utilities import pad, read_text_file
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2*nx
ny2 = 2*ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d( fft( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1) ) , nx,ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
'''
if i < 10 and myid == main_node:
for k in xrange(10):
grp_imgdata[k].write_image("grp%03d.hdf"%i, k)
'''
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("pp.hdf", pp)
"""
ave, grp_imgdata = prepare_2d_forPCA(grp_imgdata)
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("qq.hdf", pp)
"""
var = model_blank(nx,ny)
for q in grp_imgdata: Util.add_img2( var, q )
Util.mul_scalar( var, 1.0/(len(grp_imgdata)-1))
# Switch to std dev
var = square_root(threshold(var))
#if options.CTF: ave, var = avgvar_ctf(grp_imgdata, mode="a")
#else: ave, var = avgvar(grp_imgdata, mode="a")
"""
if myid == main_node:
ave.write_image("avgv.hdf",i)
var.write_image("varv.hdf",i)
"""
set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])
aveList.append(ave)
varList.append(var)
if options.VERBOSE:
print "%5.2f%% done on processor %d"%(i*100.0/len(proj_list), myid)
if nvec > 0:
eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True)
for k in xrange(nvec):
set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
eigList[k].append(eig[k])
"""
if myid == 0 and i == 0:
for k in xrange(nvec):
eig[k].write_image("eig.hdf", k)
"""
del imgdata
# To this point, all averages, variances, and eigenvectors are computed
if options.ave2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node :
for im in xrange(len(aveList)):
aveList[im].write_image(options.ave2D, km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im+i+70000)
"""
nm = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('pix_err', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
"""
tmpvol=fpol(ave, Tracker["nx"],Tracker["nx"],Tracker["nx"])
tmpvol.write_image(options.ave2D, km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
for im in xrange(len(aveList)):
send_EMData(aveList[im], main_node,im+myid+70000)
"""
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
members = aveList[im].get_attr('pix_err')
mpi_send(members, len(members), MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
except:
mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
"""
if options.ave3D:
from fundamentals import fpol
if options.VERBOSE:
print "Reconstructing 3D average volume"
ave3D = recons3d_4nn_MPI(myid, aveList, symmetry=options.sym, npad=options.npad)
bcast_EMData_to_all(ave3D, myid)
if myid == main_node:
ave3D=fpol(ave3D,Tracker["nx"],Tracker["nx"],Tracker["nx"])
ave3D.write_image(options.ave3D)
print_msg("%-70s: %s\n"%("Writing to the disk volume reconstructed from averages as", options.ave3D))
del ave, var, proj_list, stack, phi, theta, psi, s2x, s2y, alpha, sx, sy, mirror, aveList
if nvec > 0:
for k in xrange(nvec):
if options.VERBOSE:
print "Reconstruction eigenvolumes", k
cont = True
ITER = 0
mask2d = model_circle(radiuspca, nx, nx)
while cont:
#print "On node %d, iteration %d"%(myid, ITER)
eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad)
bcast_EMData_to_all(eig3D, myid, main_node)
if options.fl > 0.0:
eig3D = filt_tanl(eig3D, options.fl, options.aa)
if myid == main_node:
eig3D.write_image("eig3d_%03d.hdf"%k, ITER)
Util.mul_img( eig3D, model_circle(radiuspca, nx, nx, nx) )
eig3Df, kb = prep_vol(eig3D)
del eig3D
cont = False
icont = 0
for l in xrange(len(eigList[k])):
phi, theta, psi, s2x, s2y = get_params_proj(eigList[k][l])
proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y])
cl = ccc(proj, eigList[k][l], mask2d)
if cl < 0.0:
icont += 1
cont = True
eigList[k][l] *= -1.0
u = int(cont)
u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD)
icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
if myid == main_node:
u = int(u[0])
print " Eigenvector: ",k," number changed ",int(icont[0])
else: u = 0
u = bcast_number_to_all(u, main_node)
cont = bool(u)
ITER += 1
del eig3Df, kb
mpi_barrier(MPI_COMM_WORLD)
del eigList, mask2d
if options.ave3D: del ave3D
if options.var2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node :
for im in xrange(len(varList)):
tmpvol=fpol(varList[im], Tracker["nx"], Tracker["nx"],1)
tmpvol.write_image(options.var2D, km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im+i+70000)
tmpvol=fpol(ave, Tracker["nx"], Tracker["nx"],1)
tmpvol.write_image(options.var2D, km)
km += 1
else:
mpi_send(len(varList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
for im in xrange(len(varList)):
send_EMData(varList[im], main_node, im+myid+70000)# What with the attributes??
mpi_barrier(MPI_COMM_WORLD)
if options.var3D:
if myid == main_node and options.VERBOSE:
print "Reconstructing 3D variability volume"
t6 = time()
radiusvar = options.radiusvar
if( radiusvar < 0 ): radiusvar = nx//2 -3
res = recons3d_4nn_MPI(myid, varList, symmetry=options.sym, npad=options.npad)
#res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
if myid == main_node:
from fundamentals import fpol
res =fpol(res, Tracker["nx"], Tracker["nx"], Tracker["nx"])
res.write_image(options.var3D)
if myid == main_node:
print_msg("%-70s: %.2f\n"%("Reconstructing 3D variability took [s]", time()-t6))
if options.VERBOSE:
print "Reconstruction took: %.2f [min]"%((time()-t6)/60)
if myid == main_node:
print_msg("%-70s: %.2f\n"%("Total time for these computations [s]", time()-t0))
if options.VERBOSE:
print "Total time for these computations: %.2f [min]"%((time()-t0)/60)
print_end_msg("sx3dvariability")
global_def.BATCH = False
from mpi import mpi_finalize
mpi_finalize()
def ali3d_MPI(stack, ref_vol, outdir, maskfile = None, ir = 1, ou = -1, rs = 1,
xr = "4 2 2 1", yr = "-1", ts = "1 1 0.5 0.25", delta = "10 6 4 4", an = "-1",
center = 0, maxit = 5, term = 95, CTF = False, fourvar = False, snr = 1.0, ref_a = "S", sym = "c1",
sort=True, cutoff=999.99, pix_cutoff="0", two_tail=False, model_jump="1 1 1 1 1", restart=False, save_half=False,
protos=None, oplane=None, lmask=-1, ilmask=-1, findseam=False, vertstep=None, hpars="-1", hsearch="73.0 170.0",
full_output = False, compare_repro = False, compare_ref_free = "-1", ref_free_cutoff= "-1 -1 -1 -1",
wcmask = None, debug = False, recon_pad = 4):
from alignment import Numrinit, prepare_refrings
from utilities import model_circle, get_image, drop_image, get_input_from_string
from utilities import bcast_list_to_all, bcast_number_to_all, reduce_EMData_to_root, bcast_EMData_to_all
from utilities import send_attr_dict
from utilities import get_params_proj, file_type
from fundamentals import rot_avg_image
import os
import types
from utilities import print_begin_msg, print_end_msg, print_msg
from mpi import mpi_bcast, mpi_comm_size, mpi_comm_rank, MPI_FLOAT, MPI_COMM_WORLD, mpi_barrier, mpi_reduce
from mpi import mpi_reduce, MPI_INT, MPI_SUM, mpi_finalize
from filter import filt_ctf
from projection import prep_vol, prgs
from statistics import hist_list, varf3d_MPI, fsc_mask
from numpy import array, bincount, array2string, ones
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
if myid == main_node:
if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "ali3d_MPI", 1)
os.mkdir(outdir)
mpi_barrier(MPI_COMM_WORLD)
if debug:
from time import sleep
while not os.path.exists(outdir):
print "Node ",myid," waiting..."
sleep(5)
info_file = os.path.join(outdir, "progress%04d"%myid)
finfo = open(info_file, 'w')
else:
finfo = None
mjump = get_input_from_string(model_jump)
xrng = get_input_from_string(xr)
if yr == "-1": yrng = xrng
else : yrng = get_input_from_string(yr)
step = get_input_from_string(ts)
delta = get_input_from_string(delta)
ref_free_cutoff = get_input_from_string(ref_free_cutoff)
pix_cutoff = get_input_from_string(pix_cutoff)
lstp = min(len(xrng), len(yrng), len(step), len(delta))
if an == "-1":
an = [-1] * lstp
else:
an = get_input_from_string(an)
# make sure pix_cutoff is set for all iterations
if len(pix_cutoff)<lstp:
for i in xrange(len(pix_cutoff),lstp):
pix_cutoff.append(pix_cutoff[-1])
# don't waste time on sub-pixel alignment for low-resolution ang incr
for i in range(len(step)):
if (delta[i] > 4 or delta[i] == -1) and step[i] < 1:
step[i] = 1
first_ring = int(ir)
rstep = int(rs)
last_ring = int(ou)
max_iter = int(maxit)
center = int(center)
nrefs = EMUtil.get_image_count( ref_vol )
nmasks = 0
if maskfile:
# read number of masks within each maskfile (mc)
nmasks = EMUtil.get_image_count( maskfile )
# open masks within maskfile (mc)
maskF = EMData.read_images(maskfile, xrange(nmasks))
vol = EMData.read_images(ref_vol, xrange(nrefs))
nx = vol[0].get_xsize()
## make sure box sizes are the same
if myid == main_node:
im=EMData.read_images(stack,[0])
bx = im[0].get_xsize()
if bx!=nx:
print_msg("Error: Stack box size (%i) differs from initial model (%i)\n"%(bx,nx))
sys.exit()
del im,bx
# for helical processing:
helicalrecon = False
if protos is not None or hpars != "-1" or findseam is True:
helicalrecon = True
# if no out-of-plane param set, use 5 degrees
if oplane is None:
oplane=5.0
if protos is not None:
proto = get_input_from_string(protos)
if len(proto) != nrefs:
print_msg("Error: insufficient protofilament numbers supplied")
sys.exit()
if hpars != "-1":
hpars = get_input_from_string(hpars)
if len(hpars) != 2*nrefs:
print_msg("Error: insufficient helical parameters supplied")
sys.exit()
## create helical parameter file for helical reconstruction
if helicalrecon is True and myid == main_node:
from hfunctions import createHpar
# create initial helical parameter files
dp=[0]*nrefs
dphi=[0]*nrefs
vdp=[0]*nrefs
vdphi=[0]*nrefs
for iref in xrange(nrefs):
hpar = os.path.join(outdir,"hpar%02d.spi"%(iref))
params = False
if hpars != "-1":
# if helical parameters explicitly given, set twist & rise
params = [float(hpars[iref*2]),float(hpars[(iref*2)+1])]
dp[iref],dphi[iref],vdp[iref],vdphi[iref] = createHpar(hpar,proto[iref],params,vertstep)
# get values for helical search parameters
hsearch = get_input_from_string(hsearch)
if len(hsearch) != 2:
print_msg("Error: specify outer and inner radii for helical search")
sys.exit()
if last_ring < 0 or last_ring > int(nx/2)-2 : last_ring = int(nx/2) - 2
if myid == main_node:
# import user_functions
# user_func = user_functions.factory[user_func_name]
print_begin_msg("ali3d_MPI")
print_msg("Input stack : %s\n"%(stack))
print_msg("Reference volume : %s\n"%(ref_vol))
print_msg("Output directory : %s\n"%(outdir))
if nmasks > 0:
print_msg("Maskfile (number of masks) : %s (%i)\n"%(maskfile,nmasks))
print_msg("Inner radius : %i\n"%(first_ring))
print_msg("Outer radius : %i\n"%(last_ring))
print_msg("Ring step : %i\n"%(rstep))
print_msg("X search range : %s\n"%(xrng))
print_msg("Y search range : %s\n"%(yrng))
print_msg("Translational step : %s\n"%(step))
print_msg("Angular step : %s\n"%(delta))
print_msg("Angular search range : %s\n"%(an))
print_msg("Maximum iteration : %i\n"%(max_iter))
print_msg("Center type : %i\n"%(center))
print_msg("CTF correction : %s\n"%(CTF))
print_msg("Signal-to-Noise Ratio : %f\n"%(snr))
print_msg("Reference projection method : %s\n"%(ref_a))
print_msg("Symmetry group : %s\n"%(sym))
print_msg("Fourier padding for 3D : %i\n"%(recon_pad))
print_msg("Number of reference models : %i\n"%(nrefs))
print_msg("Sort images between models : %s\n"%(sort))
print_msg("Allow images to jump : %s\n"%(mjump))
print_msg("CC cutoff standard dev : %f\n"%(cutoff))
print_msg("Two tail cutoff : %s\n"%(two_tail))
print_msg("Termination pix error : %f\n"%(term))
print_msg("Pixel error cutoff : %s\n"%(pix_cutoff))
print_msg("Restart : %s\n"%(restart))
print_msg("Full output : %s\n"%(full_output))
print_msg("Compare reprojections : %s\n"%(compare_repro))
print_msg("Compare ref free class avgs : %s\n"%(compare_ref_free))
print_msg("Use cutoff from ref free : %s\n"%(ref_free_cutoff))
if protos:
print_msg("Protofilament numbers : %s\n"%(proto))
print_msg("Using helical search range : %s\n"%hsearch)
if findseam is True:
print_msg("Using seam-based reconstruction\n")
if hpars != "-1":
print_msg("Using hpars : %s\n"%hpars)
if vertstep != None:
print_msg("Using vertical step : %.2f\n"%vertstep)
if save_half is True:
print_msg("Saving even/odd halves\n")
for i in xrange(100) : print_msg("*")
print_msg("\n\n")
if maskfile:
if type(maskfile) is types.StringType: mask3D = get_image(maskfile)
else: mask3D = maskfile
else: mask3D = model_circle(last_ring, nx, nx, nx)
numr = Numrinit(first_ring, last_ring, rstep, "F")
mask2D = model_circle(last_ring,nx,nx) - model_circle(first_ring,nx,nx)
fscmask = model_circle(last_ring,nx,nx,nx)
if CTF:
from filter import filt_ctf
from reconstruction_rjh import rec3D_MPI_noCTF
if myid == main_node:
active = EMUtil.get_all_attributes(stack, 'active')
list_of_particles = []
for im in xrange(len(active)):
if active[im]: list_of_particles.append(im)
del active
nima = len(list_of_particles)
else:
nima = 0
total_nima = bcast_number_to_all(nima, source_node = main_node)
if myid != main_node:
list_of_particles = [-1]*total_nima
list_of_particles = bcast_list_to_all(list_of_particles, source_node = main_node)
image_start, image_end = MPI_start_end(total_nima, number_of_proc, myid)
# create a list of images for each node
list_of_particles = list_of_particles[image_start: image_end]
nima = len(list_of_particles)
if debug:
finfo.write("image_start, image_end: %d %d\n" %(image_start, image_end))
finfo.flush()
data = EMData.read_images(stack, list_of_particles)
t_zero = Transform({"type":"spider","phi":0,"theta":0,"psi":0,"tx":0,"ty":0})
transmulti = [[t_zero for i in xrange(nrefs)] for j in xrange(nima)]
for iref,im in ((iref,im) for iref in xrange(nrefs) for im in xrange(nima)):
if nrefs == 1:
transmulti[im][iref] = data[im].get_attr("xform.projection")
else:
# if multi models, keep track of eulers for all models
try:
transmulti[im][iref] = data[im].get_attr("eulers_txty.%i"%iref)
except:
data[im].set_attr("eulers_txty.%i"%iref,t_zero)
scoremulti = [[0.0 for i in xrange(nrefs)] for j in xrange(nima)]
pixelmulti = [[0.0 for i in xrange(nrefs)] for j in xrange(nima)]
ref_res = [0.0 for x in xrange(nrefs)]
apix = data[0].get_attr('apix_x')
# for oplane parameter, create cylindrical mask
if oplane is not None and myid == main_node:
from hfunctions import createCylMask
cmaskf=os.path.join(outdir, "mask3D_cyl.mrc")
mask3D = createCylMask(data,ou,lmask,ilmask,cmaskf)
# if finding seam of helix, create wedge masks
if findseam is True:
wedgemask=[]
for pf in xrange(nrefs):
wedgemask.append(EMData())
# wedgemask option
if wcmask is not None:
wcmask = get_input_from_string(wcmask)
if len(wcmask) != 3:
print_msg("Error: wcmask option requires 3 values: x y radius")
sys.exit()
# determine if particles have helix info:
try:
data[0].get_attr('h_angle')
original_data = []
boxmask = True
from hfunctions import createBoxMask
except:
boxmask = False
# prepare particles
for im in xrange(nima):
data[im].set_attr('ID', list_of_particles[im])
data[im].set_attr('pix_score', int(0))
if CTF:
# only phaseflip particles, not full CTF correction
ctf_params = data[im].get_attr("ctf")
st = Util.infomask(data[im], mask2D, False)
data[im] -= st[0]
data[im] = filt_ctf(data[im], ctf_params, sign = -1, binary=1)
data[im].set_attr('ctf_applied', 1)
# for window mask:
if boxmask is True:
h_angle = data[im].get_attr("h_angle")
original_data.append(data[im].copy())
bmask = createBoxMask(nx,apix,ou,lmask,h_angle)
data[im]*=bmask
del bmask
if debug:
finfo.write( '%d loaded \n' % nima )
finfo.flush()
if myid == main_node:
# initialize data for the reference preparation function
ref_data = [ mask3D, max(center,0), None, None, None, None ]
# for method -1, switch off centering in user function
from time import time
# this is needed for gathering of pixel errors
disps = []
recvcount = []
disps_score = []
recvcount_score = []
for im in xrange(number_of_proc):
if( im == main_node ):
disps.append(0)
disps_score.append(0)
else:
disps.append(disps[im-1] + recvcount[im-1])
disps_score.append(disps_score[im-1] + recvcount_score[im-1])
ib, ie = MPI_start_end(total_nima, number_of_proc, im)
recvcount.append( ie - ib )
recvcount_score.append((ie-ib)*nrefs)
pixer = [0.0]*nima
cs = [0.0]*3
total_iter = 0
volodd = EMData.read_images(ref_vol, xrange(nrefs))
voleve = EMData.read_images(ref_vol, xrange(nrefs))
if restart:
# recreate initial volumes from alignments stored in header
itout = "000_00"
for iref in xrange(nrefs):
if(nrefs == 1):
modout = ""
else:
modout = "_model_%02d"%(iref)
if(sort):
group = iref
for im in xrange(nima):
imgroup = data[im].get_attr('group')
if imgroup == iref:
data[im].set_attr('xform.projection',transmulti[im][iref])
else:
group = int(999)
for im in xrange(nima):
data[im].set_attr('xform.projection',transmulti[im][iref])
fscfile = os.path.join(outdir, "fsc_%s%s"%(itout,modout))
vol[iref], fscc, volodd[iref], voleve[iref] = rec3D_MPI_noCTF(data, sym, fscmask, fscfile, myid, main_node, index = group, npad = recon_pad)
if myid == main_node:
if helicalrecon:
from hfunctions import processHelicalVol
vstep=None
if vertstep is not None:
vstep=(vdp[iref],vdphi[iref])
print_msg("Old rise and twist for model %i : %8.3f, %8.3f\n"%(iref,dp[iref],dphi[iref]))
hvals=processHelicalVol(vol[iref],voleve[iref],volodd[iref],iref,outdir,itout,
dp[iref],dphi[iref],apix,hsearch,findseam,vstep,wcmask)
(vol[iref],voleve[iref],volodd[iref],dp[iref],dphi[iref],vdp[iref],vdphi[iref])=hvals
print_msg("New rise and twist for model %i : %8.3f, %8.3f\n"%(iref,dp[iref],dphi[iref]))
# get new FSC from symmetrized half volumes
fscc = fsc_mask( volodd[iref], voleve[iref], mask3D, rstep, fscfile)
else:
vol[iref].write_image(os.path.join(outdir, "vol_%s.hdf"%itout),-1)
if save_half is True:
volodd[iref].write_image(os.path.join(outdir, "volodd_%s.hdf"%itout),-1)
voleve[iref].write_image(os.path.join(outdir, "voleve_%s.hdf"%itout),-1)
if nmasks > 1:
# Read mask for multiplying
ref_data[0] = maskF[iref]
ref_data[2] = vol[iref]
ref_data[3] = fscc
# call user-supplied function to prepare reference image, i.e., center and filter it
vol[iref], cs,fl = ref_ali3d(ref_data)
vol[iref].write_image(os.path.join(outdir, "volf_%s.hdf"%(itout)),-1)
if (apix == 1):
res_msg = "Models filtered at spatial frequency of:\t"
res = fl
else:
res_msg = "Models filtered at resolution of: \t"
res = apix / fl
ares = array2string(array(res), precision = 2)
print_msg("%s%s\n\n"%(res_msg,ares))
bcast_EMData_to_all(vol[iref], myid, main_node)
# write out headers, under MPI writing has to be done sequentially
mpi_barrier(MPI_COMM_WORLD)
# projection matching
for N_step in xrange(lstp):
terminate = 0
Iter = -1
while(Iter < max_iter-1 and terminate == 0):
Iter += 1
total_iter += 1
itout = "%03g_%02d" %(delta[N_step], Iter)
if myid == main_node:
print_msg("ITERATION #%3d, inner iteration #%3d\nDelta = %4.1f, an = %5.2f, xrange = %5.2f, yrange = %5.2f, step = %5.2f\n\n"%(N_step, Iter, delta[N_step], an[N_step], xrng[N_step],yrng[N_step],step[N_step]))
for iref in xrange(nrefs):
if myid == main_node: start_time = time()
volft,kb = prep_vol( vol[iref] )
## constrain projections to out of plane parameter
theta1 = None
theta2 = None
if oplane is not None:
theta1 = 90-oplane
theta2 = 90+oplane
refrings = prepare_refrings( volft, kb, nx, delta[N_step], ref_a, sym, numr, MPI=True, phiEqpsi = "Minus", initial_theta=theta1, delta_theta=theta2)
del volft,kb
if myid== main_node:
print_msg( "Time to prepare projections for model %i: %s\n" % (iref, legibleTime(time()-start_time)) )
start_time = time()
for im in xrange( nima ):
data[im].set_attr("xform.projection", transmulti[im][iref])
if an[N_step] == -1:
t1, peak, pixer[im] = proj_ali_incore(data[im],refrings,numr,xrng[N_step],yrng[N_step],step[N_step],finfo)
else:
t1, peak, pixer[im] = proj_ali_incore_local(data[im],refrings,numr,xrng[N_step],yrng[N_step],step[N_step],an[N_step],finfo)
#data[im].set_attr("xform.projection"%iref, t1)
if nrefs > 1: data[im].set_attr("eulers_txty.%i"%iref,t1)
scoremulti[im][iref] = peak
from pixel_error import max_3D_pixel_error
# t1 is the current param, t2 is old
t2 = transmulti[im][iref]
pixelmulti[im][iref] = max_3D_pixel_error(t1,t2,numr[-3])
transmulti[im][iref] = t1
if myid == main_node:
print_msg("Time of alignment for model %i: %s\n"%(iref, legibleTime(time()-start_time)))
start_time = time()
# gather scoring data from all processors
from mpi import mpi_gatherv
scoremultisend = sum(scoremulti,[])
pixelmultisend = sum(pixelmulti,[])
tmp = mpi_gatherv(scoremultisend,len(scoremultisend),MPI_FLOAT, recvcount_score, disps_score, MPI_FLOAT, main_node,MPI_COMM_WORLD)
tmp1 = mpi_gatherv(pixelmultisend,len(pixelmultisend),MPI_FLOAT, recvcount_score, disps_score, MPI_FLOAT, main_node,MPI_COMM_WORLD)
tmp = mpi_bcast(tmp,(total_nima * nrefs), MPI_FLOAT,0, MPI_COMM_WORLD)
tmp1 = mpi_bcast(tmp1,(total_nima * nrefs), MPI_FLOAT,0, MPI_COMM_WORLD)
tmp = map(float,tmp)
tmp1 = map(float,tmp1)
score = array(tmp).reshape(-1,nrefs)
pixelerror = array(tmp1).reshape(-1,nrefs)
score_local = array(scoremulti)
mean_score = score.mean(axis=0)
std_score = score.std(axis=0)
cut = mean_score - (cutoff * std_score)
cut2 = mean_score + (cutoff * std_score)
res_max = score_local.argmax(axis=1)
minus_cc = [0.0 for x in xrange(nrefs)]
minus_pix = [0.0 for x in xrange(nrefs)]
minus_ref = [0.0 for x in xrange(nrefs)]
#output pixel errors
if(myid == main_node):
from statistics import hist_list
lhist = 20
pixmin = pixelerror.min(axis=1)
region, histo = hist_list(pixmin, lhist)
if(region[0] < 0.0): region[0] = 0.0
print_msg("Histogram of pixel errors\n ERROR number of particles\n")
for lhx in xrange(lhist):
print_msg(" %10.3f %7d\n"%(region[lhx], histo[lhx]))
# Terminate if 95% within 1 pixel error
im = 0
for lhx in xrange(lhist):
if(region[lhx] > 1.0): break
im += histo[lhx]
print_msg( "Percent of particles with pixel error < 1: %f\n\n"% (im/float(total_nima)*100))
term_cond = float(term)/100
if(im/float(total_nima) > term_cond):
terminate = 1
print_msg("Terminating internal loop\n")
del region, histo
terminate = mpi_bcast(terminate, 1, MPI_INT, 0, MPI_COMM_WORLD)
terminate = int(terminate[0])
for im in xrange(nima):
if(sort==False):
data[im].set_attr('group',999)
elif (mjump[N_step]==1):
data[im].set_attr('group',int(res_max[im]))
pix_run = data[im].get_attr('pix_score')
if (pix_cutoff[N_step]==1 and (terminate==1 or Iter == max_iter-1)):
if (pixelmulti[im][int(res_max[im])] > 1):
data[im].set_attr('pix_score',int(777))
if (score_local[im][int(res_max[im])]<cut[int(res_max[im])]) or (two_tail and score_local[im][int(res_max[im])]>cut2[int(res_max[im])]):
data[im].set_attr('group',int(888))
minus_cc[int(res_max[im])] = minus_cc[int(res_max[im])] + 1
if(pix_run == 777):
data[im].set_attr('group',int(777))
minus_pix[int(res_max[im])] = minus_pix[int(res_max[im])] + 1
if (compare_ref_free != "-1") and (ref_free_cutoff[N_step] != -1) and (total_iter > 1):
id = data[im].get_attr('ID')
if id in rejects:
data[im].set_attr('group',int(666))
minus_ref[int(res_max[im])] = minus_ref[int(res_max[im])] + 1
minus_cc_tot = mpi_reduce(minus_cc,nrefs,MPI_FLOAT,MPI_SUM,0,MPI_COMM_WORLD)
minus_pix_tot = mpi_reduce(minus_pix,nrefs,MPI_FLOAT,MPI_SUM,0,MPI_COMM_WORLD)
minus_ref_tot = mpi_reduce(minus_ref,nrefs,MPI_FLOAT,MPI_SUM,0,MPI_COMM_WORLD)
if (myid == main_node):
if(sort):
tot_max = score.argmax(axis=1)
res = bincount(tot_max)
else:
res = ones(nrefs) * total_nima
print_msg("Particle distribution: \t\t%s\n"%(res*1.0))
afcut1 = res - minus_cc_tot
afcut2 = afcut1 - minus_pix_tot
afcut3 = afcut2 - minus_ref_tot
print_msg("Particle distribution after cc cutoff:\t\t%s\n"%(afcut1))
print_msg("Particle distribution after pix cutoff:\t\t%s\n"%(afcut2))
print_msg("Particle distribution after ref cutoff:\t\t%s\n\n"%(afcut3))
res = [0.0 for i in xrange(nrefs)]
for iref in xrange(nrefs):
if(center == -1):
from utilities import estimate_3D_center_MPI, rotate_3D_shift
dummy=EMData()
cs[0], cs[1], cs[2], dummy, dummy = estimate_3D_center_MPI(data, total_nima, myid, number_of_proc, main_node)
cs = mpi_bcast(cs, 3, MPI_FLOAT, main_node, MPI_COMM_WORLD)
cs = [-float(cs[0]), -float(cs[1]), -float(cs[2])]
rotate_3D_shift(data, cs)
if(sort):
group = iref
for im in xrange(nima):
imgroup = data[im].get_attr('group')
if imgroup == iref:
data[im].set_attr('xform.projection',transmulti[im][iref])
else:
group = int(999)
for im in xrange(nima):
data[im].set_attr('xform.projection',transmulti[im][iref])
if(nrefs == 1):
modout = ""
else:
modout = "_model_%02d"%(iref)
fscfile = os.path.join(outdir, "fsc_%s%s"%(itout,modout))
vol[iref], fscc, volodd[iref], voleve[iref] = rec3D_MPI_noCTF(data, sym, fscmask, fscfile, myid, main_node, index=group, npad=recon_pad)
if myid == main_node:
print_msg("3D reconstruction time for model %i: %s\n"%(iref, legibleTime(time()-start_time)))
start_time = time()
# Compute Fourier variance
if fourvar:
outvar = os.path.join(outdir, "volVar_%s.hdf"%(itout))
ssnr_file = os.path.join(outdir, "ssnr_%s"%(itout))
varf = varf3d_MPI(data, ssnr_text_file=ssnr_file, mask2D=None, reference_structure=vol[iref], ou=last_ring, rw=1.0, npad=1, CTF=None, sign=1, sym=sym, myid=myid)
if myid == main_node:
print_msg("Time to calculate 3D Fourier variance for model %i: %s\n"%(iref, legibleTime(time()-start_time)))
start_time = time()
varf = 1.0/varf
varf.write_image(outvar,-1)
else: varf = None
if myid == main_node:
if helicalrecon:
from hfunctions import processHelicalVol
vstep=None
if vertstep is not None:
vstep=(vdp[iref],vdphi[iref])
print_msg("Old rise and twist for model %i : %8.3f, %8.3f\n"%(iref,dp[iref],dphi[iref]))
hvals=processHelicalVol(vol[iref],voleve[iref],volodd[iref],iref,outdir,itout,
dp[iref],dphi[iref],apix,hsearch,findseam,vstep,wcmask)
(vol[iref],voleve[iref],volodd[iref],dp[iref],dphi[iref],vdp[iref],vdphi[iref])=hvals
print_msg("New rise and twist for model %i : %8.3f, %8.3f\n"%(iref,dp[iref],dphi[iref]))
# get new FSC from symmetrized half volumes
fscc = fsc_mask( volodd[iref], voleve[iref], mask3D, rstep, fscfile)
print_msg("Time to search and apply helical symmetry for model %i: %s\n\n"%(iref, legibleTime(time()-start_time)))
start_time = time()
else:
vol[iref].write_image(os.path.join(outdir, "vol_%s.hdf"%(itout)),-1)
if save_half is True:
volodd[iref].write_image(os.path.join(outdir, "volodd_%s.hdf"%(itout)),-1)
voleve[iref].write_image(os.path.join(outdir, "voleve_%s.hdf"%(itout)),-1)
if nmasks > 1:
# Read mask for multiplying
ref_data[0] = maskF[iref]
ref_data[2] = vol[iref]
ref_data[3] = fscc
ref_data[4] = varf
# call user-supplied function to prepare reference image, i.e., center and filter it
vol[iref], cs,fl = ref_ali3d(ref_data)
vol[iref].write_image(os.path.join(outdir, "volf_%s.hdf"%(itout)),-1)
if (apix == 1):
res_msg = "Models filtered at spatial frequency of:\t"
res[iref] = fl
else:
res_msg = "Models filtered at resolution of: \t"
res[iref] = apix / fl
del varf
bcast_EMData_to_all(vol[iref], myid, main_node)
if compare_ref_free != "-1": compare_repro = True
if compare_repro:
outfile_repro = comp_rep(refrings, data, itout, modout, vol[iref], group, nima, nx, myid, main_node, outdir)
mpi_barrier(MPI_COMM_WORLD)
if compare_ref_free != "-1":
ref_free_output = os.path.join(outdir,"ref_free_%s%s"%(itout,modout))
rejects = compare(compare_ref_free, outfile_repro,ref_free_output,yrng[N_step], xrng[N_step], rstep,nx,apix,ref_free_cutoff[N_step], number_of_proc, myid, main_node)
# retrieve alignment params from all processors
par_str = ['xform.projection','ID','group']
if nrefs > 1:
for iref in xrange(nrefs):
par_str.append('eulers_txty.%i'%iref)
if myid == main_node:
from utilities import recv_attr_dict
recv_attr_dict(main_node, stack, data, par_str, image_start, image_end, number_of_proc)
else: send_attr_dict(main_node, data, par_str, image_start, image_end)
if myid == main_node:
ares = array2string(array(res), precision = 2)
print_msg("%s%s\n\n"%(res_msg,ares))
dummy = EMData()
if full_output:
nimat = EMUtil.get_image_count(stack)
output_file = os.path.join(outdir, "paramout_%s"%itout)
foutput = open(output_file, 'w')
for im in xrange(nimat):
# save the parameters for each of the models
outstring = ""
dummy.read_image(stack,im,True)
param3d = dummy.get_attr('xform.projection')
g = dummy.get_attr("group")
# retrieve alignments in EMAN-format
pE = param3d.get_params('eman')
outstring += "%f\t%f\t%f\t%f\t%f\t%i\n" %(pE["az"], pE["alt"], pE["phi"], pE["tx"], pE["ty"],g)
foutput.write(outstring)
foutput.close()
del dummy
mpi_barrier(MPI_COMM_WORLD)
# mpi_finalize()
if myid == main_node: print_end_msg("ali3d_MPI")
def rec3D_MPI_noCTF(data,
symmetry,
mask3D,
fsc_curve,
myid,
main_node=0,
rstep=1.0,
odd_start=0,
eve_start=1,
finfo=None,
index=-1,
npad=4):
'''
This function is to be called within an MPI program to do a reconstruction on a dataset kept in the memory
Computes reconstruction and through odd-even, in order to get the resolution
if index > -1, projections should have attribute group set and only those whose group matches index will be used in the reconstruction
this is for multireference alignment
'''
import os
from statistics import fsc_mask
from utilities import model_blank, reduce_EMData_to_root, get_image, send_EMData, recv_EMData
from random import randint
from mpi import mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD
from reconstruction import recons_from_fftvol, prepare_recons
nproc = mpi_comm_size(MPI_COMM_WORLD)
if nproc == 1:
assert main_node == 0
main_node_odd = main_node
main_node_eve = main_node
main_node_all = main_node
elif nproc == 2:
main_node_odd = main_node
main_node_eve = (main_node + 1) % 2
main_node_all = main_node
tag_voleve = 1000
tag_fftvol_eve = 1001
tag_weight_eve = 1002
else:
#spread CPUs between different nodes to save memory
main_node_odd = main_node
main_node_eve = (int(main_node) + nproc - 1) % int(nproc)
main_node_all = (int(main_node) + nproc // 2) % int(nproc)
tag_voleve = 1000
tag_fftvol_eve = 1001
tag_weight_eve = 1002
tag_fftvol_odd = 1003
tag_weight_odd = 1004
tag_volall = 1005
nx = data[0].get_xsize()
fftvol_odd_file, weight_odd_file = prepare_recons(data, symmetry, myid,
main_node_odd, odd_start,
2, index, finfo, npad)
fftvol_eve_file, weight_eve_file = prepare_recons(data, symmetry, myid,
main_node_eve, eve_start,
2, index, finfo, npad)
if nproc == 1:
fftvol = get_image(fftvol_odd_file)
weight = get_image(weight_odd_file)
volodd = recons_from_fftvol(nx, fftvol, weight, symmetry, npad)
fftvol = get_image(fftvol_eve_file)
weight = get_image(weight_eve_file)
voleve = recons_from_fftvol(nx, fftvol, weight, symmetry, npad)
fscdat = fsc_mask(volodd, voleve, mask3D, rstep, fsc_curve)
fftvol = get_image(fftvol_odd_file)
Util.add_img(fftvol, get_image(fftvol_eve_file))
weight = get_image(weight_odd_file)
Util.add_img(weight, get_image(weight_eve_file))
volall = recons_from_fftvol(nx, fftvol, weight, symmetry, npad)
os.system("rm -f " + fftvol_odd_file + " " + weight_odd_file)
os.system("rm -f " + fftvol_eve_file + " " + weight_eve_file)
return volall, fscdat, volodd, voleve
if nproc == 2:
if myid == main_node_odd:
fftvol = get_image(fftvol_odd_file)
weight = get_image(weight_odd_file)
volodd = recons_from_fftvol(nx, fftvol, weight, symmetry, npad)
voleve = recv_EMData(main_node_eve, tag_voleve)
fscdat = fsc_mask(volodd, voleve, mask3D, rstep, fsc_curve)
else:
assert myid == main_node_eve
fftvol = get_image(fftvol_eve_file)
weight = get_image(weight_eve_file)
voleve = recons_from_fftvol(nx, fftvol, weight, symmetry, npad)
send_EMData(voleve, main_node_odd, tag_voleve)
if myid == main_node_odd:
fftvol = get_image(fftvol_odd_file)
fftvol_tmp = recv_EMData(main_node_eve, tag_fftvol_eve)
Util.add_img(fftvol, fftvol_tmp)
fftvol_tmp = None
weight = get_image(weight_odd_file)
weight_tmp = recv_EMData(main_node_eve, tag_weight_eve)
Util.add_img(weight, weight_tmp)
weight_tmp = None
volall = recons_from_fftvol(nx, fftvol, weight, symmetry, npad)
os.system("rm -f " + fftvol_odd_file + " " + weight_odd_file)
return volall, fscdat, volodd, voleve
else:
assert myid == main_node_eve
fftvol = get_image(fftvol_eve_file)
send_EMData(fftvol, main_node_odd, tag_fftvol_eve)
weight = get_image(weight_eve_file)
send_EMData(weight, main_node_odd, tag_weight_eve)
os.system("rm -f " + fftvol_eve_file + " " + weight_eve_file)
return model_blank(nx, nx, nx), None, model_blank(nx, nx,
nx), model_blank(
nx, nx, nx)
# cases from all other number of processors situations
if myid == main_node_odd:
fftvol = get_image(fftvol_odd_file)
send_EMData(fftvol, main_node_eve, tag_fftvol_odd)
if not (finfo is None):
finfo.write("fftvol odd sent\n")
finfo.flush()
weight = get_image(weight_odd_file)
send_EMData(weight, main_node_all, tag_weight_odd)
if not (finfo is None):
finfo.write("weight odd sent\n")
finfo.flush()
volodd = recons_from_fftvol(nx, fftvol, weight, symmetry, npad)
del fftvol, weight
voleve = recv_EMData(main_node_eve, tag_voleve)
fscdat = fsc_mask(volodd, voleve, mask3D, rstep, fsc_curve)
volall = recv_EMData(main_node_all, tag_volall)
os.system("rm -f " + fftvol_odd_file + " " + weight_odd_file)
return volall, fscdat, volodd, voleve
if myid == main_node_eve:
ftmp = recv_EMData(main_node_odd, tag_fftvol_odd)
fftvol = get_image(fftvol_eve_file)
Util.add_img(ftmp, fftvol)
send_EMData(ftmp, main_node_all, tag_fftvol_eve)
del ftmp
weight = get_image(weight_eve_file)
send_EMData(weight, main_node_all, tag_weight_eve)
voleve = recons_from_fftvol(nx, fftvol, weight, symmetry, npad)
send_EMData(voleve, main_node_odd, tag_voleve)
os.system("rm -f " + fftvol_eve_file + " " + weight_eve_file)
return model_blank(nx, nx,
nx), None, model_blank(nx, nx,
nx), model_blank(nx, nx, nx)
if myid == main_node_all:
fftvol = recv_EMData(main_node_eve, tag_fftvol_eve)
if not (finfo is None):
finfo.write("fftvol odd received\n")
finfo.flush()
weight = recv_EMData(main_node_odd, tag_weight_odd)
weight_tmp = recv_EMData(main_node_eve, tag_weight_eve)
Util.add_img(weight, weight_tmp)
weight_tmp = None
volall = recons_from_fftvol(nx, fftvol, weight, symmetry, npad)
send_EMData(volall, main_node_odd, tag_volall)
return model_blank(nx, nx,
nx), None, model_blank(nx, nx,
nx), model_blank(nx, nx, nx)
return model_blank(nx, nx,
nx), None, model_blank(nx, nx,
nx), model_blank(nx, nx, nx)
def main():
import global_def
from optparse import OptionParser
from EMAN2 import EMUtil
import os
import sys
from time import time
progname = os.path.basename(sys.argv[0])
usage = progname + " proj_stack output_averages --MPI"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--img_per_group",
type="int",
default=100,
help="number of images per group")
parser.add_option("--radius",
type="int",
default=-1,
help="radius for alignment")
parser.add_option(
"--xr",
type="string",
default="2 1",
help="range for translation search in x direction, search is +/xr")
parser.add_option(
"--yr",
type="string",
default="-1",
help=
"range for translation search in y direction, search is +/yr (default = same as xr)"
)
parser.add_option(
"--ts",
type="string",
default="1 0.5",
help=
"step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional"
)
parser.add_option(
"--iter",
type="int",
default=30,
help="number of iterations within alignment (default = 30)")
parser.add_option(
"--num_ali",
type="int",
default=5,
help="number of alignments performed for stability (default = 5)")
parser.add_option("--thld_err",
type="float",
default=1.0,
help="threshold of pixel error (default = 1.732)")
parser.add_option(
"--grouping",
type="string",
default="GRP",
help=
"do grouping of projections: PPR - per projection, GRP - different size groups, exclusive (default), GEV - grouping equal size"
)
parser.add_option(
"--delta",
type="float",
default=-1.0,
help="angular step for reference projections (required for GEV method)"
)
parser.add_option(
"--fl",
type="float",
default=0.3,
help="cut-off frequency of hyperbolic tangent low-pass Fourier filter")
parser.add_option(
"--aa",
type="float",
default=0.2,
help="fall-off of hyperbolic tangent low-pass Fourier filter")
parser.add_option("--CTF",
action="store_true",
default=False,
help="Consider CTF correction during the alignment ")
parser.add_option("--MPI",
action="store_true",
default=False,
help="use MPI version")
(options, args) = parser.parse_args()
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, MPI_COMM_WORLD
from mpi import mpi_barrier, mpi_send, mpi_recv, mpi_bcast, MPI_INT, mpi_finalize, MPI_FLOAT
from applications import MPI_start_end, within_group_refinement, ali2d_ras
from pixel_error import multi_align_stability
from utilities import send_EMData, recv_EMData
from utilities import get_image, bcast_number_to_all, set_params2D, get_params2D
from utilities import group_proj_by_phitheta, model_circle, get_input_from_string
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
if len(args) == 2:
stack = args[0]
outdir = args[1]
else:
ERROR("incomplete list of arguments", "sxproj_stability", 1, myid=myid)
exit()
if not options.MPI:
ERROR("Non-MPI not supported!", "sxproj_stability", myid=myid)
exit()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
global_def.BATCH = True
#if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "sxproj_stability", 1, myid)
#mpi_barrier(MPI_COMM_WORLD)
img_per_grp = options.img_per_group
radius = options.radius
ite = options.iter
num_ali = options.num_ali
thld_err = options.thld_err
xrng = get_input_from_string(options.xr)
if options.yr == "-1": yrng = xrng
else: yrng = get_input_from_string(options.yr)
step = get_input_from_string(options.ts)
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
else:
nima = 0
nx = 0
ny = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
if radius == -1: radius = nx / 2 - 2
mask = model_circle(radius, nx, nx)
st = time()
if options.grouping == "GRP":
if myid == main_node:
print " A ", myid, " ", time() - st
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
proj_params = []
for i in xrange(nima):
dp = proj_attr[i].get_params("spider")
phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[
"psi"], -dp["tx"], -dp["ty"]
proj_params.append([phi, theta, psi, s2x, s2y])
# Here is where the grouping is done, I didn't put enough annotation in the group_proj_by_phitheta,
# So I will briefly explain it here
# proj_list : Returns a list of list of particle numbers, each list contains img_per_grp particle numbers
# except for the last one. Depending on the number of particles left, they will either form a
# group or append themselves to the last group
# angle_list : Also returns a list of list, each list contains three numbers (phi, theta, delta), (phi,
# theta) is the projection angle of the center of the group, delta is the range of this group
# mirror_list: Also returns a list of list, each list contains img_per_grp True or False, which indicates
# whether it should take mirror position.
# In this program angle_list and mirror list are not of interest.
proj_list_all, angle_list, mirror_list = group_proj_by_phitheta(
proj_params, img_per_grp=img_per_grp)
del proj_params
print " B number of groups ", myid, " ", len(
proj_list_all), time() - st
mpi_barrier(MPI_COMM_WORLD)
# Number of groups, actually there could be one or two more groups, since the size of the remaining group varies
# we will simply assign them to main node.
n_grp = nima / img_per_grp - 1
# Divide proj_list_all equally to all nodes, and becomes proj_list
proj_list = []
for i in xrange(n_grp):
proc_to_stay = i % number_of_proc
if proc_to_stay == main_node:
if myid == main_node: proj_list.append(proj_list_all[i])
elif myid == main_node:
mpi_send(len(proj_list_all[i]), 1, MPI_INT, proc_to_stay,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(proj_list_all[i], len(proj_list_all[i]), MPI_INT,
proc_to_stay, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
elif myid == proc_to_stay:
img_per_grp = mpi_recv(1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
img_per_grp = int(img_per_grp[0])
temp = mpi_recv(img_per_grp, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
proj_list.append(map(int, temp))
del temp
mpi_barrier(MPI_COMM_WORLD)
print " C ", myid, " ", time() - st
if myid == main_node:
# Assign the remaining groups to main_node
for i in xrange(n_grp, len(proj_list_all)):
proj_list.append(proj_list_all[i])
del proj_list_all, angle_list, mirror_list
# Compute stability per projection projection direction, equal number assigned, thus overlaps
elif options.grouping == "GEV":
if options.delta == -1.0:
ERROR(
"Angular step for reference projections is required for GEV method",
"sxproj_stability", 1)
from utilities import even_angles, nearestk_to_refdir, getvec
refproj = even_angles(options.delta)
img_begin, img_end = MPI_start_end(len(refproj), number_of_proc, myid)
# Now each processor keeps its own share of reference projections
refprojdir = refproj[img_begin:img_end]
del refproj
ref_ang = [0.0] * (len(refprojdir) * 2)
for i in xrange(len(refprojdir)):
ref_ang[i * 2] = refprojdir[0][0]
ref_ang[i * 2 + 1] = refprojdir[0][1] + i * 0.1
print " A ", myid, " ", time() - st
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
# the solution below is very slow, do not use it unless there is a problem with the i/O
"""
for i in xrange(number_of_proc):
if myid == i:
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
mpi_barrier(MPI_COMM_WORLD)
"""
print " B ", myid, " ", time() - st
proj_ang = [0.0] * (nima * 2)
for i in xrange(nima):
dp = proj_attr[i].get_params("spider")
proj_ang[i * 2] = dp["phi"]
proj_ang[i * 2 + 1] = dp["theta"]
print " C ", myid, " ", time() - st
asi = Util.nearestk_to_refdir(proj_ang, ref_ang, img_per_grp)
del proj_ang, ref_ang
proj_list = []
for i in xrange(len(refprojdir)):
proj_list.append(asi[i * img_per_grp:(i + 1) * img_per_grp])
del asi
print " D ", myid, " ", time() - st
#from sys import exit
#exit()
# Compute stability per projection
elif options.grouping == "PPR":
print " A ", myid, " ", time() - st
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
print " B ", myid, " ", time() - st
proj_params = []
for i in xrange(nima):
dp = proj_attr[i].get_params("spider")
phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[
"psi"], -dp["tx"], -dp["ty"]
proj_params.append([phi, theta, psi, s2x, s2y])
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
print " C ", myid, " ", time() - st
from utilities import nearest_proj
proj_list, mirror_list = nearest_proj(
proj_params, img_per_grp,
range(img_begin, img_begin + 1)) #range(img_begin, img_end))
refprojdir = proj_params[img_begin:img_end]
del proj_params, mirror_list
print " D ", myid, " ", time() - st
else:
ERROR("Incorrect projection grouping option", "sxproj_stability", 1)
"""
from utilities import write_text_file
for i in xrange(len(proj_list)):
write_text_file(proj_list[i],"projlist%06d_%04d"%(i,myid))
"""
###########################################################################################################
# Begin stability test
from utilities import get_params_proj, read_text_file
#if myid == 0:
# from utilities import read_text_file
# proj_list[0] = map(int, read_text_file("lggrpp0.txt"))
from utilities import model_blank
aveList = [model_blank(nx, ny)] * len(proj_list)
if options.grouping == "GRP":
refprojdir = [[0.0, 0.0, -1.0]] * len(proj_list)
for i in xrange(len(proj_list)):
print " E ", myid, " ", time() - st
class_data = EMData.read_images(stack, proj_list[i])
#print " R ",myid," ",time()-st
if options.CTF:
from filter import filt_ctf
for im in xrange(len(class_data)): # MEM LEAK!!
atemp = class_data[im].copy()
btemp = filt_ctf(atemp, atemp.get_attr("ctf"), binary=1)
class_data[im] = btemp
#class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1)
for im in class_data:
try:
t = im.get_attr(
"xform.align2d") # if they are there, no need to set them!
except:
try:
t = im.get_attr("xform.projection")
d = t.get_params("spider")
set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0])
except:
set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0])
#print " F ",myid," ",time()-st
# Here, we perform realignment num_ali times
all_ali_params = []
for j in xrange(num_ali):
if (xrng[0] == 0.0 and yrng[0] == 0.0):
avet = ali2d_ras(class_data,
randomize=True,
ir=1,
ou=radius,
rs=1,
step=1.0,
dst=90.0,
maxit=ite,
check_mirror=True,
FH=options.fl,
FF=options.aa)
else:
avet = within_group_refinement(class_data, mask, True, 1,
radius, 1, xrng, yrng, step,
90.0, ite, options.fl,
options.aa)
ali_params = []
for im in xrange(len(class_data)):
alpha, sx, sy, mirror, scale = get_params2D(class_data[im])
ali_params.extend([alpha, sx, sy, mirror])
all_ali_params.append(ali_params)
#aveList[i] = avet
#print " G ",myid," ",time()-st
del ali_params
# We determine the stability of this group here.
# stable_set contains all particles deemed stable, it is a list of list
# each list has two elements, the first is the pixel error, the second is the image number
# stable_set is sorted based on pixel error
#from utilities import write_text_file
#write_text_file(all_ali_params, "all_ali_params%03d.txt"%myid)
stable_set, mir_stab_rate, average_pix_err = multi_align_stability(
all_ali_params, 0.0, 10000.0, thld_err, False, 2 * radius + 1)
#print " H ",myid," ",time()-st
if (len(stable_set) > 5):
stable_set_id = []
members = []
pix_err = []
# First put the stable members into attr 'members' and 'pix_err'
for s in stable_set:
# s[1] - number in this subset
stable_set_id.append(s[1])
# the original image number
members.append(proj_list[i][s[1]])
pix_err.append(s[0])
# Then put the unstable members into attr 'members' and 'pix_err'
from fundamentals import rot_shift2D
avet.to_zero()
if options.grouping == "GRP":
aphi = 0.0
atht = 0.0
vphi = 0.0
vtht = 0.0
l = -1
for j in xrange(len(proj_list[i])):
# Here it will only work if stable_set_id is sorted in the increasing number, see how l progresses
if j in stable_set_id:
l += 1
avet += rot_shift2D(class_data[j], stable_set[l][2][0],
stable_set[l][2][1],
stable_set[l][2][2],
stable_set[l][2][3])
if options.grouping == "GRP":
phi, theta, psi, sxs, sys = get_params_proj(
class_data[j])
if (theta > 90.0):
phi = (phi + 540.0) % 360.0
theta = 180.0 - theta
aphi += phi
atht += theta
vphi += phi * phi
vtht += theta * theta
else:
members.append(proj_list[i][j])
pix_err.append(99999.99)
aveList[i] = avet.copy()
if l > 1:
l += 1
aveList[i] /= l
if options.grouping == "GRP":
aphi /= l
atht /= l
vphi = (vphi - l * aphi * aphi) / l
vtht = (vtht - l * atht * atht) / l
from math import sqrt
refprojdir[i] = [
aphi, atht,
(sqrt(max(vphi, 0.0)) + sqrt(max(vtht, 0.0))) / 2.0
]
# Here more information has to be stored, PARTICULARLY WHAT IS THE REFERENCE DIRECTION
aveList[i].set_attr('members', members)
aveList[i].set_attr('refprojdir', refprojdir[i])
aveList[i].set_attr('pixerr', pix_err)
else:
print " empty group ", i, refprojdir[i]
aveList[i].set_attr('members', [-1])
aveList[i].set_attr('refprojdir', refprojdir[i])
aveList[i].set_attr('pixerr', [99999.])
del class_data
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node:
for im in xrange(len(aveList)):
aveList[im].write_image(args[1], km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im + i + 70000)
nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('pixerr', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
ave.write_image(args[1], km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
for im in xrange(len(aveList)):
send_EMData(aveList[im], main_node, im + myid + 70000)
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
members = aveList[im].get_attr('pixerr')
mpi_send(members, len(members), MPI_FLOAT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
except:
mpi_send([-999.0, -999.0, -999.0], 3, MPI_FLOAT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
global_def.BATCH = False
mpi_barrier(MPI_COMM_WORLD)
from mpi import mpi_finalize
mpi_finalize()
def rec3D_MPI(data,
snr,
symmetry,
mask3D,
fsc_curve,
myid,
main_node=0,
rstep=1.0,
odd_start=0,
eve_start=1,
finfo=None,
index=-1,
npad=4,
hparams=None):
'''
This function is to be called within an MPI program to do a reconstruction on a dataset kept
in the memory, computes reconstruction and through odd-even, in order to get the resolution
'''
import os
from statistics import fsc_mask
from utilities import model_blank, reduce_EMData_to_root, get_image, send_EMData, recv_EMData
from random import randint
from mpi import mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD
nproc = mpi_comm_size(MPI_COMM_WORLD)
if nproc == 1:
assert main_node == 0
main_node_odd = main_node
main_node_eve = main_node
main_node_all = main_node
elif nproc == 2:
main_node_odd = main_node
main_node_eve = (main_node + 1) % 2
main_node_all = main_node
tag_voleve = 1000
tag_fftvol_eve = 1001
tag_weight_eve = 1002
else:
#spread CPUs between different nodes to save memory
main_node_odd = main_node
main_node_eve = (int(main_node) + nproc - 1) % int(nproc)
main_node_all = (int(main_node) + nproc // 2) % int(nproc)
tag_voleve = 1000
tag_fftvol_eve = 1001
tag_weight_eve = 1002
tag_fftvol_odd = 1003
tag_weight_odd = 1004
tag_volall = 1005
if index != -1:
grpdata = []
for i in xrange(len(data)):
if data[i].get_attr('group') == index:
grpdata.append(data[i])
imgdata = grpdata
else:
imgdata = data
nx = get_image_size(imgdata, myid)
if nx == 0:
ERROR(
"Warning: no images were given for reconstruction, this usually means there is an empty group, returning empty volume",
"rec3D", 0)
return model_blank(2, 2,
2), None, model_blank(nx, nx,
nx), model_blank(nx, nx, nx)
fftvol_odd_file, weight_odd_file = prepare_recons_ctf(
nx, imgdata, snr, symmetry, myid, main_node_odd, odd_start, 2, finfo,
npad)
fftvol_eve_file, weight_eve_file = prepare_recons_ctf(
nx, imgdata, snr, symmetry, myid, main_node_eve, eve_start, 2, finfo,
npad)
del imgdata
if nproc == 1:
fftvol = get_image(fftvol_odd_file)
weight = get_image(weight_odd_file)
volodd = recons_ctf_from_fftvol(nx, fftvol, weight, snr, symmetry,
npad)
fftvol = get_image(fftvol_eve_file)
weight = get_image(weight_eve_file)
voleve = recons_ctf_from_fftvol(nx, fftvol, weight, snr, symmetry,
npad)
fscdat = fsc_mask(volodd, voleve, mask3D, rstep, fsc_curve)
fftvol = get_image(fftvol_odd_file)
fftvol_tmp = get_image(fftvol_eve_file)
fftvol += fftvol_tmp
fftvol_tmp = None
weight = get_image(weight_odd_file)
weight_tmp = get_image(weight_eve_file)
weight += weight_tmp
weight_tmp = None
volall = recons_ctf_from_fftvol(nx, fftvol, weight, snr, symmetry,
npad)
# if helical, find & apply symmetry to volume
if hparams is not None:
volodd, voleve, volall = hsymVols(volodd, voleve, volall, hparams)
fscdat = fsc_mask(volodd, voleve, mask3D, rstep, fsc_curve)
os.system("rm -f " + fftvol_odd_file + " " + weight_odd_file)
os.system("rm -f " + fftvol_eve_file + " " + weight_eve_file)
return volall, fscdat, volodd, voleve
if nproc == 2:
if myid == main_node_odd:
fftvol = get_image(fftvol_odd_file)
weight = get_image(weight_odd_file)
volodd = recons_ctf_from_fftvol(nx, fftvol, weight, snr, symmetry,
npad)
voleve = recv_EMData(main_node_eve, tag_voleve)
fscdat = fsc_mask(volodd, voleve, mask3D, rstep, fsc_curve)
else:
assert myid == main_node_eve
fftvol = get_image(fftvol_eve_file)
weight = get_image(weight_eve_file)
voleve = recons_ctf_from_fftvol(nx, fftvol, weight, snr, symmetry,
npad)
send_EMData(voleve, main_node_odd, tag_voleve)
if myid == main_node_odd:
fftvol = get_image(fftvol_odd_file)
fftvol_tmp = recv_EMData(main_node_eve, tag_fftvol_eve)
fftvol += fftvol_tmp
fftvol_tmp = None
weight = get_image(weight_odd_file)
weight_tmp = recv_EMData(main_node_eve, tag_weight_eve)
weight += weight_tmp
weight_tmp = None
volall = recons_ctf_from_fftvol(nx, fftvol, weight, snr, symmetry,
npad)
# if helical, find & apply symmetry to volume
if hparams is not None:
volodd, voleve, volall = hsymVols(volodd, voleve, volall,
hparams)
fscdat = fsc_mask(volodd, voleve, mask3D, rstep, fsc_curve)
os.system("rm -f " + fftvol_odd_file + " " + weight_odd_file)
os.system("rm -f " + fftvol_eve_file + " " + weight_eve_file)
return volall, fscdat, volodd, voleve
else:
assert myid == main_node_eve
fftvol = get_image(fftvol_eve_file)
send_EMData(fftvol, main_node_odd, tag_fftvol_eve)
weight = get_image(weight_eve_file)
send_EMData(weight, main_node_odd, tag_weight_eve)
os.system("rm -f " + fftvol_eve_file + " " + weight_eve_file)
return model_blank(nx, nx, nx), None, model_blank(nx, nx,
nx), model_blank(
nx, nx, nx)
# cases from all other number of processors situations
if myid == main_node_odd:
fftvol = get_image(fftvol_odd_file)
send_EMData(fftvol, main_node_eve, tag_fftvol_odd)
if not (finfo is None):
finfo.write("fftvol odd sent\n")
finfo.flush()
weight = get_image(weight_odd_file)
send_EMData(weight, main_node_all, tag_weight_odd)
if not (finfo is None):
finfo.write("weight odd sent\n")
finfo.flush()
volodd = recons_ctf_from_fftvol(nx, fftvol, weight, snr, symmetry,
npad)
del fftvol, weight
voleve = recv_EMData(main_node_eve, tag_voleve)
fscdat = fsc_mask(volodd, voleve, mask3D, rstep, fsc_curve)
volall = recv_EMData(main_node_all, tag_volall)
# if helical, find & apply symmetry to volume
if hparams is not None:
volodd, voleve, volall = hsymVols(volodd, voleve, volall, hparams)
fscdat = fsc_mask(volodd, voleve, mask3D, rstep, fsc_curve)
os.system("rm -f " + fftvol_odd_file + " " + weight_odd_file)
return volall, fscdat, volodd, voleve
if myid == main_node_eve:
ftmp = recv_EMData(main_node_odd, tag_fftvol_odd)
fftvol = get_image(fftvol_eve_file)
Util.add_img(ftmp, fftvol)
send_EMData(ftmp, main_node_all, tag_fftvol_eve)
del ftmp
weight = get_image(weight_eve_file)
send_EMData(weight, main_node_all, tag_weight_eve)
voleve = recons_ctf_from_fftvol(nx, fftvol, weight, snr, symmetry,
npad)
send_EMData(voleve, main_node_odd, tag_voleve)
os.system("rm -f " + fftvol_eve_file + " " + weight_eve_file)
return model_blank(nx, nx,
nx), None, model_blank(nx, nx,
nx), model_blank(nx, nx, nx)
if myid == main_node_all:
fftvol = recv_EMData(main_node_eve, tag_fftvol_eve)
if not (finfo is None):
finfo.write("fftvol odd received\n")
finfo.flush()
weight = recv_EMData(main_node_odd, tag_weight_odd)
weight_tmp = recv_EMData(main_node_eve, tag_weight_eve)
Util.add_img(weight, weight_tmp)
weight_tmp = None
volall = recons_ctf_from_fftvol(nx, fftvol, weight, snr, symmetry,
npad)
send_EMData(volall, main_node_odd, tag_volall)
return model_blank(nx, nx,
nx), None, model_blank(nx, nx,
nx), model_blank(nx, nx, nx)
return model_blank(nx, nx,
nx), None, model_blank(nx, nx,
nx), model_blank(nx, nx, nx)
