def extract_classav_vstack(inpath,classes,outpath,verbose=0):
"""Extracts particles from class-averages into a virtual stack. inpath is the path to a file with class-averages,
outpath is the output BDB, and classes is a list/tuple of class average numbers. Returns the number of images copied"""
vstack=db_open_dict(outpath) # open output db
outn=0
for avn in classes:
av=EMData(inpath,avn)
try:
imgsrc=av["class_ptcl_src"]
imgns=av["class_ptcl_idxs"]
except:
raise Exception("Particle doesn't have source image info (%s,d)"%(inpath,avn))
# If we are writing to a virtual stack
try:
src=db_open_dict(imgsrc) # this is the source database file
except:
raise Exception("Cannot open source images as BDB (%s)"%imgsrc)
for n in imgns:
# here we make the new entry in the vstack
d=src.get(n,nodata=1).get_attr_dict() # get the metadata for the image
d["data_path"]=src.get_data_path(n) # this is how we avoid copying the image data
vstack[outn]=d
outn+=1
if verbose>0 : print("Class %d: %d particles"%(avn,len(imgns)))
if verbose>0 : print("%d total particles written to %s"%(outn,outpath))
return outn
def __init__(self, mediator):
QtGui.QWidget.__init__(self)
self.mediator = mediator
self.db = db_open_dict("bdb:emboxerrctgui")
self.qualitydb = db_open_dict("bdb:e2boxercache#quality")
self.setWindowIcon(QtGui.QIcon(get_image_directory() +"green_boxes.png"))
self.setWindowTitle("e2RCTboxer")
# Here is where additional tools can be added
self.manual_tool = ManualPicker(self.mediator, self.db)
self.pair_picker_tool = PairPickerTool(self.mediator, self.db)
vbox = QtGui.QVBoxLayout(self)
self.tab_widget = QtGui.QTabWidget()
self.tab_widget.addTab(self.get_main_tab(),"Main")
self.tab_widget.addTab(self.get_processor_tab(),"Processor")
self.tab_widget.addTab(self.get_filter_tab(),"Filter")
vbox.addWidget(self.tab_widget)
self.add_controls(vbox) # add done button, among other things
self.setLayout(vbox)
self.add_picker_tools()
self.updateGeometry()
# Initialize tools
self.tools_stacked_widget.setCurrentIndex(self.db.get("toolidx",dfl=0))
self.current_tool_combobox.setCurrentIndex(self.db.get("toolidx",dfl=0))
def extract_classav_vstack(inpath,classes,outpath,verbose=0): """Extracts particles from class-averages into a virtual stack. inpath is the path to a file with class-averages, outpath is the output BDB, and classes is a list/tuple of class average numbers. Returns the number of images copied""" vstack=db_open_dict(outpath) # open output db outn=0 for avn in classes: av=EMData(inpath,avn) try: imgsrc=av["class_ptcl_src"] imgns=av["class_ptcl_idxs"] except: raise Exception,"Particle doesn't have source image info (%s,d)"%(inpath,avn) # If we are writing to a virtual stack try: src=db_open_dict(imgsrc) # this is the source database file except: raise Exception,"Cannot open source images as BDB (%s)"%imgsrc for n in imgns: # here we make the new entry in the vstack d=src.get(n,nodata=1).get_attr_dict() # get the metadata for the image d["data_path"]=src.get_data_path(n) # this is how we avoid copying the image data vstack[outn]=d outn+=1 if verbose>0 : print "Class %d: %d particles"%(avn,len(imgns)) if verbose>0 : print "%d total particles written to %s"(outn,outpath) return outn
def pspec_and_ctf_fit(options, debug=False):
"Power spectrum and CTF fitting"
global logid
img_sets = []
db_parms = db_open_dict("bdb:e2ctf.parms")
for i, filename in enumerate(options.filenames):
name = get_file_tag(filename)
# compute the power spectra
if debug: print "Processing ", filename
apix = options.apix
if apix <= 0: apix = EMData(filename, 0, 1)["apix_x"]
im_1d, bg_1d, im_2d, bg_2d = powspec_with_bg(
filename,
radius=options.bgmask,
edgenorm=not options.nonorm,
oversamp=options.oversamp)
ds = 1.0 / (apix * im_2d.get_ysize())
if not options.nosmooth: bg_1d = smooth_bg(bg_1d, ds)
Util.save_data(0, ds, bg_1d, "ctf.bgb4.txt")
# Fit the CTF parameters
if debug: print "Fit CTF"
ctf = ctf_fit(im_1d,
bg_1d,
im_2d,
bg_2d,
options.voltage,
options.cs,
options.ac,
apix,
bgadj=not options.nosmooth,
autohp=options.autohp)
db_parms[name] = [ctf.to_string(), im_1d, bg_1d, im_2d, bg_2d]
if debug:
Util.save_data(0, ds, im_1d, "ctf.fg.txt")
Util.save_data(0, ds, bg_1d, "ctf.bg.txt")
Util.save_data(0, ds, ctf.snr, "ctf.snr.txt")
img_sets.append((filename, ctf, im_1d, bg_1d, im_2d, bg_2d))
if logid: E2progress(logid, float(i + 1) / len(options.filenames))
project_db = db_open_dict("bdb:project")
project_db["global.microscope_voltage"] = options.voltage
project_db["global.microscope_cs"] = options.cs
project_db["global.apix"] = apix
#db_close_dict("bdb:project")
#db_close_dict("bdb:e2ctf.parms")
return img_sets
def write_e2ctf_output(options):
"write wiener filtered and/or phase flipped particle data to the local database"
global logid
if options.phaseflip or options.wiener:
db_parms=db_open_dict("bdb:e2ctf.parms")
for i,filename in enumerate(options.filenames):
name=base_name(filename)
if debug: print("Processing ",filename)
if options.phaseflip: phaseout="bdb:particles#"+name+"_ctf_flip"
else: phaseout=None
if options.wiener:
if options.autohp: wienerout="bdb:particles#"+name+"_ctf_wiener_hp"
else: wienerout="bdb:particles#"+name+"_ctf_wiener"
else : wienerout=None
#if options.phaseflip: phaseout=name+"_ctf_flip.hed"
#else: phaseout=None
#if options.wiener: wienerout=name+"_ctf_wiener.hed"
#else : wienerout=None
if phaseout : print("Phase image out: ",phaseout,"\t", end=' ')
if wienerout : print("Wiener image out: ",wienerout, end=' ')
print("")
ctf=EMAN2Ctf()
ctf.from_string(db_parms[name][0])
process_stack(filename,phaseout,wienerout,not options.nonorm,options.oversamp,ctf,invert=options.invert)
if logid : E2progress(logid,float(i+1)/len(options.filenames))
def get_gui_arg_img_sets(filenames):
'''
returns the img_sets list required to intialized the GUI correctly
'''
img_sets = []
if db_check_dict("bdb:e2ctf.parms"):
db_parms = db_open_dict("bdb:e2ctf.parms", ro=True)
else:
return img_sets
for file in filenames:
name = get_file_tag(file)
if not db_parms.has_key(name):
print "error, you must first run auto fit before running the gui - there are no parameters for", name
return []
img_set = db_parms[name]
ctf = EMAN2Ctf()
ctf.from_string(
img_set[0]) # convert to ctf object seeing as it's a string
img_set[0] = ctf
actual = [file]
actual.extend(img_set)
img_sets.append(actual)
return img_sets
def get_params(self):
params = []
params = []
self.table_tool = EMTomoPtclReportTool(self.project_dict,
self.window_title)
table = self.table_tool.get_particle_table_with_ptcls(self.ptcls_list)
params.append(
ParamDef(name="blurb",
vartype="text",
desc_short="",
desc_long="",
property=None,
defaultunits=self.__doc__,
choices=None))
params.append(table)
self.add_filt_params(params)
db = db_open_dict(self.form_db_name)
pname = ParamDef(
"name",
vartype="string",
desc_short="Filtered set name",
desc_long="The processed sets will be referred to by this name",
property=None,
defaultunits=db.get("name", dfl="filt"),
choices=[])
params.append(pname)
return params
def write_e2ctf_output(options):
"write wiener filtered and/or phase flipped particle data to the local database"
global logid
if options.phaseflip or options.wiener:
db_parms=db_open_dict("bdb:e2ctf.parms")
for i,filename in enumerate(options.filenames):
name=get_file_tag(filename)
if debug: print "Processing ",filename
if options.phaseflip: phaseout="bdb:particles#"+name+"_ctf_flip"
else: phaseout=None
if options.wiener:
if options.autohp: wienerout="bdb:particles#"+name+"_ctf_wiener_hp"
else: wienerout="bdb:particles#"+name+"_ctf_wiener"
else : wienerout=None
#if options.phaseflip: phaseout=name+"_ctf_flip.hed"
#else: phaseout=None
#if options.wiener: wienerout=name+"_ctf_wiener.hed"
#else : wienerout=None
if phaseout : print "Phase image out: ",phaseout,"\t",
if wienerout : print "Wiener image out: ",wienerout,
print ""
ctf=EMAN2Ctf()
ctf.from_string(db_parms[name][0])
process_stack(filename,phaseout,wienerout,not options.nonorm,options.oversamp,ctf,invert=options.invert)
if logid : E2progress(logid,float(i+1)/len(options.filenames))
def execute(self, progress_callback):
'''
Called to perform class averaging
May boot strap the original average, iteratively refines averages, aligns final average to ref
'''
from EMAN2db import db_open_dict
progress_callback(0)
cache_name = self.data["probe"][1]
cache = db_open_dict(cache_name)
probe = cache[self.data["probe"][2]]
cache_name = self.data["target"][1]
cache = db_open_dict(cache_name)
target = cache[self.data["target"][2]]
return self.align_task.align(probe, target, progress_callback)
def execute(self,progress_callback): ''' Called to perform class averaging May boot strap the original average, iteratively refines averages, aligns final average to ref ''' from EMAN2db import db_open_dict progress_callback(0) cache_name=self.data["probe"][1] cache=db_open_dict(cache_name) probe = cache[self.data["probe"][2]] cache_name=self.data["target"][1] cache=db_open_dict(cache_name) target = cache[self.data["target"][2]] return self.align_task.align(probe, target, progress_callback)
def send_bdb(stdout, path):
"Sends a BDB to stdout as a set of compressed pickled key/value pairs, terminated by a None key"
db = db_open_dict(path)
keys = list(db.keys())
for k in keys:
write_obj(stdout, k)
write_obj(stdout, db[k])
write_obj(stdout, None)
db.close()
def send_bdb(stdout,path): "Sends a BDB to stdout as a set of compressed pickled key/value pairs, terminated by a None key" db=db_open_dict(path) keys=db.keys() for k in keys: write_obj(stdout,k) write_obj(stdout,db[k]) write_obj(stdout,None) db.close()
def pspec_and_ctf_fit(options,debug=False):
"Power spectrum and CTF fitting"
global logid
img_sets=[]
db_parms=db_open_dict("bdb:e2ctf.parms")
for i,filename in enumerate(options.filenames):
name=get_file_tag(filename)
# compute the power spectra
if debug : print "Processing ",filename
apix=options.apix
if apix<=0 : apix=EMData(filename,0,1)["apix_x"]
im_1d,bg_1d,im_2d,bg_2d=powspec_with_bg(filename,radius=options.bgmask,edgenorm=not options.nonorm,oversamp=options.oversamp)
ds=1.0/(apix*im_2d.get_ysize())
if not options.nosmooth : bg_1d=smooth_bg(bg_1d,ds)
Util.save_data(0,ds,bg_1d,"ctf.bgb4.txt")
# Fit the CTF parameters
if debug : print "Fit CTF"
ctf=ctf_fit(im_1d,bg_1d,im_2d,bg_2d,options.voltage,options.cs,options.ac,apix,bgadj=not options.nosmooth,autohp=options.autohp)
db_parms[name]=[ctf.to_string(),im_1d,bg_1d,im_2d,bg_2d]
if debug:
Util.save_data(0,ds,im_1d,"ctf.fg.txt")
Util.save_data(0,ds,bg_1d,"ctf.bg.txt")
Util.save_data(0,ds,ctf.snr,"ctf.snr.txt")
img_sets.append((filename,ctf,im_1d,bg_1d,im_2d,bg_2d))
if logid : E2progress(logid,float(i+1)/len(options.filenames))
project_db = db_open_dict("bdb:project")
project_db["global.microscope_voltage"] = options.voltage
project_db["global.microscope_cs"] = options.cs
project_db["global.apix"] = apix
#db_close_dict("bdb:project")
#db_close_dict("bdb:e2ctf.parms")
return img_sets
def add_filt_params(self,params):
'''
'''
db = db_open_dict(self.form_db_name)
az = ParamDef(name="az",vartype="float",desc_short="Az rotation",desc_long="Rotate your model about the z axis",property=None,defaultunits=db.get("az",0.0),choices=None)
alt = ParamDef(name="alt",vartype="float",desc_short="Alt rotation",desc_long="Rotate your model about the x axis",property=None,defaultunits=db.get("alt",0.0),choices=None)
phi = ParamDef(name="phi",vartype="float",desc_short="Phi rotation",desc_long="Rotate your model about the z' axis",property=None,defaultunits=db.get("phi",0.0),choices=None)
ppostproc = ParamDef("filter",vartype="string",desc_short="Filter",desc_long="A post processor applied to the reconstructed model",property=None,defaultunits=db.get("filter",dfl="None"),choices=self.get_postprocess_filt_options())
ppostprocargs = ParamDef(name="filterargs",vartype="string",desc_short="params",desc_long="Parameters for the post processor see \"e2help.py processors\"",property=None,defaultunits=db.get("filterargs",dfl=""),choices=[])
params.append([ppostproc,ppostprocargs])
params.append([az,alt,phi])
def add_filt_params(self,params):
'''
'''
db = db_open_dict(self.form_db_name)
az = ParamDef(name="az",vartype="float",desc_short="Az rotation",desc_long="Rotate your model about the z axis",property=None,defaultunits=db.get("az",0.0),choices=None)
alt = ParamDef(name="alt",vartype="float",desc_short="Alt rotation",desc_long="Rotate your model about the x axis",property=None,defaultunits=db.get("alt",0.0),choices=None)
phi = ParamDef(name="phi",vartype="float",desc_short="Phi rotation",desc_long="Rotate your model about the z' axis",property=None,defaultunits=db.get("phi",0.0),choices=None)
ppostproc = ParamDef("filter",vartype="string",desc_short="Filter",desc_long="A post processor applied to the reconstructed model",property=None,defaultunits=db.get("filter",dfl="None"),choices=self.get_postprocess_filt_options())
ppostprocargs = ParamDef(name="filterargs",vartype="string",desc_short="params",desc_long="Parameters for the post processor see \"e2help.py processors\"",property=None,defaultunits=db.get("filterargs",dfl=""),choices=[])
params.append([ppostproc,ppostprocargs])
params.append([az,alt,phi])
def save_to_workflow_db(self, output_name):
'''
Used by the workflow to automatically add the names of averages to a list
that is displayed in a form
'''
if self.options.dbls:
pdb = db_open_dict("bdb:project")
tmp_data = pdb.get(self.options.dbls, dfl={})
s = {}
s["Original Data"] = output_name
tmp_data[output_name] = s
# global.spr_ref_free_class_aves
pdb[self.options.dbls] = tmp_data
def recv_bdb(stdin,path):
"Receives a BDB from stdin as a set of None terminated compressed pickled key/value pairs"
try :os.makedirs(path[4:].split("#")[0])
except: pass
# sys.stderr.write("open %s\n"%path)
db=db_open_dict(path)
db.realopen()
db.bdb.truncate() # empty the existing database
while (1):
k=read_obj(stdin)
if k==None or (isinstance(k,str) and len(k)==0) : break
db[k]=read_obj(stdin)
db.close()
def recv_bdb(stdin,path):
"Receives a BDB from stdin as a set of None terminated compressed pickled key/value pairs"
try :os.makedirs(path[4:].split("#")[0])
except: pass
# sys.stderr.write("open %s\n"%path)
db=db_open_dict(path)
db.realopen()
db.bdb.truncate() # empty the existing database
while (1):
k=read_obj(stdin)
if k==None or (isinstance(k,str) and len(k)==0) : break
db[k]=read_obj(stdin)
db.close()
def save_to_workflow_db(self,output_name):
'''
Used by the workflow to automatically add the names of averages to a list
that is displayed in a form
'''
if self.options.dbls:
pdb = db_open_dict("bdb:project")
tmp_data = pdb.get(self.options.dbls, dfl={})
s = {}
s["Original Data"] = output_name
tmp_data[output_name]= s
# global.spr_ref_free_class_aves
pdb[self.options.dbls] = tmp_data
def db_compute_fsc(a, b, apix, pathname, dbname): fsc = a.calc_fourier_shell_correlation(b) third = len(fsc)/3 xaxis = fsc[0:third] plot = fsc[third:2*third] error = fsc[2*third:] convergence_db_name = "bdb:"+pathname+"#convergence.results" db = db_open_dict(convergence_db_name) tmpaxis = [x/apix for x in xaxis] db[dbname] = [tmpaxis[1:],plot[1:]] #db["error_"+s+"_fsc"] = [xaxis,error] #we're not plotting the errors db_close_dict(convergence_db_name)
def get_params(self):
params = []
db = db_open_dict(self.form_db_name)
p,n = self.get_tomo_boxer_basic_table() # note n is unused, it's a refactoring residual
params.append(ParamDef(name="blurb",vartype="text",desc_short="Interactive use of e2spt_boxer",desc_long="",property=None,defaultunits=self.__doc__,choices=None))
params.append(p)
pylong = ParamDef(name="yshort",vartype="boolean",desc_short="yshort",desc_long="Use Z axis as normal",property=None,defaultunits=db.get("yshort",dfl=True),choices=None)
pinmem = ParamDef(name="inmemory",vartype="boolean",desc_short="inmemory",desc_long="Load the tomo into memory",property=None,defaultunits=db.get("inmemory",dfl=True),choices=None)
papix = ParamDef(name="apix",vartype="float",desc_short=u"\u212B per pixel", desc_long="Angstroms per pixel",property=None,defaultunits=db.get("apix",dfl=1.0),choices=None )
params.append([pylong, pinmem, papix])
# db = db_open_dict(self.form_db_name)
# params.append(ParamDef(name="interface_boxsize",vartype="int",desc_short="Box size",desc_long="An integer value",property=None,defaultunits=db.get("interface_boxsize",dfl=128),choices=[]))
# #db_close_dict(self.form_db_name)
return params
def get_params(self):
params = []
params = []
self.table_tool = EMTomoPtclReportTool(self.project_dict,self.window_title)
table = self.table_tool.get_particle_table_with_ptcls(self.ptcls_list)
params.append(ParamDef(name="blurb",vartype="text",desc_short="",desc_long="",property=None,defaultunits=self.__doc__,choices=None))
params.append(table)
self.add_filt_params(params)
db = db_open_dict(self.form_db_name)
pname = ParamDef("name",vartype="string",desc_short="Filtered set name",desc_long="The processed sets will be referred to by this name",property=None,defaultunits=db.get("name",dfl="filt"),choices=[])
params.append(pname)
return params
def get_params(self):
params = []
db = db_open_dict(self.form_db_name)
p,n = self.get_tomo_boxer_basic_table() # note n is unused, it's a refactoring residual
params.append(ParamDef(name="blurb",vartype="text",desc_short="Interactive use of e2spt_boxer",desc_long="",property=None,defaultunits=self.__doc__,choices=None))
params.append(p)
pylong = ParamDef(name="yshort",vartype="boolean",desc_short="yshort",desc_long="Use Z axis as normal",property=None,defaultunits=db.get("yshort",dfl=True),choices=None)
pinmem = ParamDef(name="inmemory",vartype="boolean",desc_short="inmemory",desc_long="Load the tomo into memory",property=None,defaultunits=db.get("inmemory",dfl=True),choices=None)
papix = ParamDef(name="apix",vartype="float",desc_short=u"\u212B per pixel", desc_long="Angstroms per pixel",property=None,defaultunits=db.get("apix",dfl=1.0),choices=None )
params.append([pylong, pinmem, papix])
# db = db_open_dict(self.form_db_name)
# params.append(ParamDef(name="interface_boxsize",vartype="int",desc_short="Box size",desc_long="An integer value",property=None,defaultunits=db.get("interface_boxsize",dfl=128),choices=[]))
# #db_close_dict(self.form_db_name)
return params
def get_params(self):
ptcl_opts = EMPartSetOptions(tpr_ptcls_dict)
self.particles_map, self.particles_name_map, choices, self.name_map = ptcl_opts.get_particle_options(
)
#if as_string:
#params.append(ParamDef(name="particle_set_choice",vartype="string",desc_long="Choose the particle data set you wish to use to generate a starting data for e2refine2d",desc_short=title,property=None,defaultunits=db.get("particle_set_choice",dfl=""),choices=choices))
db = db_open_dict(self.form_db_name)
params = []
params.append(
ParamDef(name="blurb",
vartype="text",
desc_short="",
desc_long="",
property=None,
defaultunits=self.__doc__,
choices=None))
if len(choices) > 0:
if len(choices) == 1:
vartype = "choice"
else:
vartype = "string"
params.append(
ParamDef(
name="tomo_filt_choice",
vartype=vartype,
desc_long="Choose from the filtered tomogram particles",
desc_short="Choose data",
property=None,
defaultunits=db.get("tomo_filt_choice", dfl=""),
choices=choices))
else:
params.append(
ParamDef(
name="blurb2",
vartype="text",
desc_short="",
desc_long="",
property=None,
defaultunits=
"There are no particles in the project. Go back to earlier stages and box/import particles",
choices=None))
return params
def get_params(self):
ptcl_opts = EMPartSetOptions(tpr_ptcls_dict)
self.particles_map, self.particles_name_map, choices, self.name_map = ptcl_opts.get_particle_options()
#if as_string:
#params.append(ParamDef(name="particle_set_choice",vartype="string",desc_long="Choose the particle data set you wish to use to generate a starting data for e2refine2d",desc_short=title,property=None,defaultunits=db.get("particle_set_choice",dfl=""),choices=choices))
db = db_open_dict(self.form_db_name)
params = []
params.append(ParamDef(name="blurb",vartype="text",desc_short="",desc_long="",property=None,defaultunits=self.__doc__,choices=None))
if len(choices) > 0:
if len(choices) == 1:
vartype = "choice"
else:
vartype = "string"
params.append(ParamDef(name="tomo_filt_choice",vartype=vartype,desc_long="Choose from the filtered tomogram particles",desc_short="Choose data",property=None,defaultunits=db.get("tomo_filt_choice",dfl=""),choices=choices))
else:
params.append(ParamDef(name="blurb2",vartype="text",desc_short="",desc_long="",property=None,defaultunits="There are no particles in the project. Go back to earlier stages and box/import particles",choices=None))
return params
def on_recall_params(self):
if len(self.setlist.selectedItems()) == 0: return
val = self.curset
name = str(self.setlist.item(val).text())
data = [name]
name = get_file_tag(name)
db_parms=db_open_dict("bdb:e2ctf.parms")
if not db_parms.has_key(name):
print "error, ctf parameters do not exist for:",name
#
data.extend(db_parms[name])
ctf=EMAN2Ctf()
ctf.from_string(data[1])
data[1] = ctf
self.data[val] = data
self.newSet(self.curset)
def on_recall_params(self):
if len(self.setlist.selectedItems()) == 0: return
val = self.curset
name = str(self.setlist.item(val).text())
data = [name]
name = get_file_tag(name)
db_parms = db_open_dict("bdb:e2ctf.parms")
if not db_parms.has_key(name):
print "error, ctf parameters do not exist for:", name
#
data.extend(db_parms[name])
ctf = EMAN2Ctf()
ctf.from_string(data[1])
data[1] = ctf
self.data[val] = data
self.newSet(self.curset)
def on_save_params(self):
if len(self.setlist.selectedItems()) == 0: return
val = self.curset
name = str(self.setlist.item(val).text())
name = get_file_tag(name)
# if not db_check_dict(name):
# print "error, the db doesn't exist:",name
#
db_parms=db_open_dict("bdb:e2ctf.parms")
ctf = self.data[val][1].to_string()
output = []
for i,val in enumerate(self.data[val]):
# ignore i == 0 it's just the filename
if i > 1:
output.append(val)
elif i == 1:
output.append(ctf)
db_parms[name] = output
def on_save_params(self):
if len(self.setlist.selectedItems()) == 0: return
val = self.curset
name = str(self.setlist.item(val).text())
name = get_file_tag(name)
# if not db_check_dict(name):
# print "error, the db doesn't exist:",name
#
db_parms = db_open_dict("bdb:e2ctf.parms")
ctf = self.data[val][1].to_string()
output = []
for i, val in enumerate(self.data[val]):
# ignore i == 0 it's just the filename
if i > 1:
output.append(val)
elif i == 1:
output.append(ctf)
db_parms[name] = output
def get_gui_arg_img_sets(filenames):
'''
returns the img_sets list required to intialized the GUI correctly
'''
img_sets = []
if db_check_dict("bdb:e2ctf.parms"):
db_parms=db_open_dict("bdb:e2ctf.parms",ro=True)
else: return img_sets
for file in filenames:
name = get_file_tag(file)
if not db_parms.has_key(name):
print "error, you must first run auto fit before running the gui - there are no parameters for",name
return []
img_set = db_parms[name]
ctf=EMAN2Ctf()
ctf.from_string(img_set[0]) # convert to ctf object seeing as it's a string
img_set[0] = ctf
actual = [file]
actual.extend(img_set)
img_sets.append(actual)
return img_sets
def get_params(self):
table_params = []
params = []
db = db_open_dict(self.form_db_name)
p,n = self.get_tomo_hunter_basic_table() # note n is unused, it's a refactoring residual
params.append(ParamDef(name="blurb",vartype="text",desc_short="Interactive use of tomohunter",desc_long="",property=None,defaultunits=self.__doc__,choices=None))
params.append(p)
psavesteps = ParamDef(name="savesteps",vartype="boolean",desc_short="Savesteps",desc_long="Save the steps",property=None,defaultunits=db.get("savesteps",dfl=True),choices=None)
psaveali = ParamDef(name="saveali",vartype="boolean",desc_short="Saveali",desc_long="Save the alignments",property=None,defaultunits=db.get("saveali",dfl=True),choices=None)
params.append([psavesteps, psaveali])
piter = ParamDef(name="number of iterations",vartype="int",desc_short="Number of iterations", desc_long="Number of iterations",property=None,defaultunits=db.get("number of iterations",dfl=5),choices=None )
pncoarse = ParamDef(name="coarse number",vartype="int",desc_short="Coarse Number", desc_long="Coarse number",property=None,defaultunits=db.get("coarse number",dfl=6),choices=None )
params.append([piter, pncoarse])
pshrink = ParamDef(name="Percentage to shrink",vartype="int",desc_short="Shrink", desc_long="Percentage to shrink",property=None,defaultunits=db.get("Percentage to shrink",dfl=2),choices=None )
pshrinkrefine = ParamDef(name="Percentage to shrink, refinement",vartype="int",desc_short="Shrink refine", desc_long="Percentage to shrink for refinement",property=None,defaultunits=db.get("Percentage to shrink, refinement",dfl=2),choices=None )
params.append([pshrink, pshrinkrefine])
proc_data = dump_processors_list()
masks = {}
for key in list(proc_data.keys()):
if len(key) >= 5 and key[:5] == "mask.":
masks[key] = proc_data[key]
masks["None"] = ["Choose this to stop masking from occurring"]
pmask = ParamDef("mask",vartype="string",desc_short="Mask",desc_long="The mask to apply to the subtomos",property=None,defaultunits=db.get("mask",dfl="None"),choices=masks)
pmaskparams = ParamDef("maskparams",vartype="string",desc_short="Params",desc_long="Parameters for the mask",property=None,defaultunits=db.get("maskparams",dfl=""))
params.append([pmask, pmaskparams])
filters = {}
for key in list(proc_data.keys()):
if len(key) >= 7 and key[:7] == "filter.":
filters[key] = proc_data[key]
filters["None"] = ["Choose this to stop filtering from occurring"]
pfilter = ParamDef("filter",vartype="string",desc_short="Filter",desc_long="The Filter to apply to the subtomos",property=None,defaultunits=db.get("filter",dfl="None"),choices=filters)
pfilterparams = ParamDef("filterparams",vartype="string",desc_short="Params",desc_long="Parameters for the filter",property=None,defaultunits=db.get("filterparams",dfl=""))
params.append([pfilter, pfilterparams])
ali_data = dump_aligners_list()
caligners = {}
for key in list(ali_data.keys()):
if len(key) >= 19 and key[:19] == "rotate_translate_3d":
caligners[key] = ali_data[key]
pali = ParamDef("aligner3D",vartype="string",desc_short="Aligner3D",desc_long="The 3D course aligner",property=None,defaultunits=db.get("aligner3D",dfl="rotate_translate_3d"),choices=caligners)
paliparams = ParamDef("ali3dparams",vartype="string",desc_short="Params",desc_long="Parameters for the 3D aligner",property=None,defaultunits=db.get("ali3dparams",dfl="search=10:delta=15:dphi=15:verbose=1"))
params.append([pali, paliparams])
craligners = {}
for key in list(ali_data.keys()):
if len(key) >= 9 and key[:9] == "refine_3d":
craligners[key] = ali_data[key]
prali = ParamDef("raligner3D",vartype="string",desc_short="RAligner3D",desc_long="The 3D refine aligner",property=None,defaultunits=db.get("raligner3D",dfl="refine_3d_grid"),choices=craligners)
praliparams = ParamDef("rali3dparams",vartype="string",desc_short="Params",desc_long="Parameters for the 3D refine aligner",property=None,defaultunits=db.get("rali3dparams",dfl="verbose=1"))
params.append([prali, praliparams])
ppostfilter = ParamDef("postfilter",vartype="string",desc_short="PostFilter",desc_long="The Filter to apply to the average",property=None,defaultunits=db.get("postfilter",dfl="None"),choices=filters)
ppostfilterparams = ParamDef("postfilterparams",vartype="string",desc_short="Params",desc_long="Parameters for the postfilter",property=None,defaultunits=db.get("postfilterparams",dfl=""))
params.append([ppostfilter, ppostfilterparams])
pparallel = ParamDef("parallel",vartype="string",desc_short="Parallel",desc_long="Parallelization parameters",property=None,defaultunits=db.get("parallel",dfl=""))
params.append(pparallel)
#pylong = ParamDef(name="yshort",vartype="boolean",desc_short="yshort",desc_long="Use Z axis as normal",property=None,defaultunits=1,choices=None)
#pinmem = ParamDef(name="inmemory",vartype="boolean",desc_short="inmemory",desc_long="Load the tomo into memory",property=None,defaultunits=1,choices=None)
#papix = ParamDef(name="apix",vartype="float",desc_short=u"\u212B per pixel", desc_long="Angstroms per pixel",property=None,defaultunits=1.0,choices=None )
#params.append([pylong, pinmem, papix])
#params.append(papix)
table_params.append(["Main",params])
advanced_params = []
r,rn = self.get_tomo_hunter_ref_table() # note rn is unused, it's a refactoring residual
advanced_params.append(ParamDef(name="blurb",vartype="text",desc_short="Interactive use tomo refs",desc_long="",property=None,defaultunits="Use this for selecting references. If no reference is chosen, then reference free alignment will be executed",choices=None))
advanced_params.append(r)
table_params.append(["References",advanced_params])
# db = db_open_dict(self.form_db_name)
# params.append(ParamDef(name="interface_boxsize",vartype="int",desc_short="Box size",desc_long="An integer value",property=None,defaultunits=db.get("interface_boxsize",dfl=128),choices=[]))
# #db_close_dict(self.form_db_name)
return table_params
from math import *
from os import *
import EMAN2fsc
from EMAN2db import db_open_dict
from EMAN2 import *
if len(sys.argv) > 1: dirs = sys.argv[1:]
else: dirs = [i for i in os.listdir(".") if "refine_" == i[:7] and len(i) == 9]
# iterates over all refine directories
for d in dirs:
print("Updating ", d)
db2 = db_open_dict("bdb:%s#register" % d, True)
try:
initm = db2["cmd_dict"]["model"]
print(" Initial model ", initm)
except:
print("No initial model recorded")
initm = None
db = db_open_dict("bdb:%s#convergence.results" % d)
if d[-5:] == "_even":
# For e2refine_evenodd results
db2 = db_open_dict("bdb:%s#convergence.results" % d[:-5])
tdflist = [
i for i in db_list_dicts("bdb:%s" % d) if "threed_filt" in i
]
def shiftali_MPI(stack, maskfile=None, maxit=100, CTF=False, snr=1.0, Fourvar=False, search_rng=-1, oneDx=False, search_rng_y=-1):
from applications import MPI_start_end
from utilities import model_circle, model_blank, get_image, peak_search, get_im
from utilities import reduce_EMData_to_root, bcast_EMData_to_all, send_attr_dict, file_type, bcast_number_to_all, bcast_list_to_all
from statistics import varf2d_MPI
from fundamentals import fft, ccf, rot_shift3D, rot_shift2D
from utilities import get_params2D, set_params2D
from utilities import print_msg, print_begin_msg, print_end_msg
import os
import sys
from mpi import mpi_init, mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD
from mpi import mpi_reduce, mpi_bcast, mpi_barrier, mpi_gatherv
from mpi import MPI_SUM, MPI_FLOAT, MPI_INT
from EMAN2 import Processor
from time import time
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
ftp = file_type(stack)
if myid == main_node:
print_begin_msg("shiftali_MPI")
max_iter=int(maxit)
if myid == main_node:
if ftp == "bdb":
from EMAN2db import db_open_dict
dummy = db_open_dict(stack, True)
nima = EMUtil.get_image_count(stack)
else:
nima = 0
nima = bcast_number_to_all(nima, source_node = main_node)
list_of_particles = range(nima)
image_start, image_end = MPI_start_end(nima, number_of_proc, myid)
list_of_particles = list_of_particles[image_start: image_end]
# read nx and ctf_app (if CTF) and broadcast to all nodes
if myid == main_node:
ima = EMData()
ima.read_image(stack, list_of_particles[0], True)
nx = ima.get_xsize()
ny = ima.get_ysize()
if CTF: ctf_app = ima.get_attr_default('ctf_applied', 2)
del ima
else:
nx = 0
ny = 0
if CTF: ctf_app = 0
nx = bcast_number_to_all(nx, source_node = main_node)
ny = bcast_number_to_all(ny, source_node = main_node)
if CTF:
ctf_app = bcast_number_to_all(ctf_app, source_node = main_node)
if ctf_app > 0: ERROR("data cannot be ctf-applied", "shiftali_MPI", 1, myid)
if maskfile == None:
mrad = min(nx, ny)
mask = model_circle(mrad//2-2, nx, ny)
else:
mask = get_im(maskfile)
if CTF:
from filter import filt_ctf
from morphology import ctf_img
ctf_abs_sum = EMData(nx, ny, 1, False)
ctf_2_sum = EMData(nx, ny, 1, False)
else:
ctf_2_sum = None
from global_def import CACHE_DISABLE
if CACHE_DISABLE:
data = EMData.read_images(stack, list_of_particles)
else:
for i in xrange(number_of_proc):
if myid == i:
data = EMData.read_images(stack, list_of_particles)
if ftp == "bdb": mpi_barrier(MPI_COMM_WORLD)
for im in xrange(len(data)):
data[im].set_attr('ID', list_of_particles[im])
st = Util.infomask(data[im], mask, False)
data[im] -= st[0]
if CTF:
ctf_params = data[im].get_attr("ctf")
ctfimg = ctf_img(nx, ctf_params, ny=ny)
Util.add_img2(ctf_2_sum, ctfimg)
Util.add_img_abs(ctf_abs_sum, ctfimg)
if CTF:
reduce_EMData_to_root(ctf_2_sum, myid, main_node)
reduce_EMData_to_root(ctf_abs_sum, myid, main_node)
else: ctf_2_sum = None
if CTF:
if myid != main_node:
del ctf_2_sum
del ctf_abs_sum
else:
temp = EMData(nx, ny, 1, False)
for i in xrange(0,nx,2):
for j in xrange(ny):
temp.set_value_at(i,j,snr)
Util.add_img(ctf_2_sum, temp)
del temp
total_iter = 0
# apply initial xform.align2d parameters stored in header
init_params = []
for im in xrange(len(data)):
t = data[im].get_attr('xform.align2d')
init_params.append(t)
p = t.get_params("2d")
data[im] = rot_shift2D(data[im], p['alpha'], sx=p['tx'], sy=p['ty'], mirror=p['mirror'], scale=p['scale'])
# fourier transform all images, and apply ctf if CTF
for im in xrange(len(data)):
if CTF:
ctf_params = data[im].get_attr("ctf")
data[im] = filt_ctf(fft(data[im]), ctf_params)
else:
data[im] = fft(data[im])
sx_sum=0
sy_sum=0
sx_sum_total=0
sy_sum_total=0
shift_x = [0.0]*len(data)
shift_y = [0.0]*len(data)
ishift_x = [0.0]*len(data)
ishift_y = [0.0]*len(data)
for Iter in xrange(max_iter):
if myid == main_node:
start_time = time()
print_msg("Iteration #%4d\n"%(total_iter))
total_iter += 1
avg = EMData(nx, ny, 1, False)
for im in data: Util.add_img(avg, im)
reduce_EMData_to_root(avg, myid, main_node)
if myid == main_node:
if CTF:
tavg = Util.divn_filter(avg, ctf_2_sum)
else: tavg = Util.mult_scalar(avg, 1.0/float(nima))
else:
tavg = EMData(nx, ny, 1, False)
if Fourvar:
bcast_EMData_to_all(tavg, myid, main_node)
vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF)
if myid == main_node:
if Fourvar:
tavg = fft(Util.divn_img(fft(tavg), vav))
vav_r = Util.pack_complex_to_real(vav)
# normalize and mask tavg in real space
tavg = fft(tavg)
stat = Util.infomask( tavg, mask, False )
tavg -= stat[0]
Util.mul_img(tavg, mask)
# For testing purposes: shift tavg to some random place and see if the centering is still correct
#tavg = rot_shift3D(tavg,sx=3,sy=-4)
tavg = fft(tavg)
if Fourvar: del vav
bcast_EMData_to_all(tavg, myid, main_node)
sx_sum=0
sy_sum=0
if search_rng > 0: nwx = 2*search_rng+1
else: nwx = nx
if search_rng_y > 0: nwy = 2*search_rng_y+1
else: nwy = ny
not_zero = 0
for im in xrange(len(data)):
if oneDx:
ctx = Util.window(ccf(data[im],tavg),nwx,1)
p1 = peak_search(ctx)
p1_x = -int(p1[0][3])
ishift_x[im] = p1_x
sx_sum += p1_x
else:
p1 = peak_search(Util.window(ccf(data[im],tavg), nwx,nwy))
p1_x = -int(p1[0][4])
p1_y = -int(p1[0][5])
ishift_x[im] = p1_x
ishift_y[im] = p1_y
sx_sum += p1_x
sy_sum += p1_y
if not_zero == 0:
if (not(ishift_x[im] == 0.0)) or (not(ishift_y[im] == 0.0)):
not_zero = 1
sx_sum = mpi_reduce(sx_sum, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
if not oneDx:
sy_sum = mpi_reduce(sy_sum, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
if myid == main_node:
sx_sum_total = int(sx_sum[0])
if not oneDx:
sy_sum_total = int(sy_sum[0])
else:
sx_sum_total = 0
sy_sum_total = 0
sx_sum_total = bcast_number_to_all(sx_sum_total, source_node = main_node)
if not oneDx:
sy_sum_total = bcast_number_to_all(sy_sum_total, source_node = main_node)
sx_ave = round(float(sx_sum_total)/nima)
sy_ave = round(float(sy_sum_total)/nima)
for im in xrange(len(data)):
p1_x = ishift_x[im] - sx_ave
p1_y = ishift_y[im] - sy_ave
params2 = {"filter_type" : Processor.fourier_filter_types.SHIFT, "x_shift" : p1_x, "y_shift" : p1_y, "z_shift" : 0.0}
data[im] = Processor.EMFourierFilter(data[im], params2)
shift_x[im] += p1_x
shift_y[im] += p1_y
# stop if all shifts are zero
not_zero = mpi_reduce(not_zero, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
if myid == main_node:
not_zero_all = int(not_zero[0])
else:
not_zero_all = 0
not_zero_all = bcast_number_to_all(not_zero_all, source_node = main_node)
if myid == main_node:
print_msg("Time of iteration = %12.2f\n"%(time()-start_time))
start_time = time()
if not_zero_all == 0: break
#for im in xrange(len(data)): data[im] = fft(data[im]) This should not be required as only header information is used
# combine shifts found with the original parameters
for im in xrange(len(data)):
t0 = init_params[im]
t1 = Transform()
t1.set_params({"type":"2D","alpha":0,"scale":t0.get_scale(),"mirror":0,"tx":shift_x[im],"ty":shift_y[im]})
# combine t0 and t1
tt = t1*t0
data[im].set_attr("xform.align2d", tt)
# write out headers and STOP, under MPI writing has to be done sequentially
mpi_barrier(MPI_COMM_WORLD)
par_str = ["xform.align2d", "ID"]
if myid == main_node:
from utilities import file_type
if(file_type(stack) == "bdb"):
from utilities import recv_attr_dict_bdb
recv_attr_dict_bdb(main_node, stack, data, par_str, image_start, image_end, number_of_proc)
else:
from utilities import recv_attr_dict
recv_attr_dict(main_node, stack, data, par_str, image_start, image_end, number_of_proc)
else: send_attr_dict(main_node, data, par_str, image_start, image_end)
if myid == main_node: print_end_msg("shiftali_MPI")
def main():
def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
if mirror:
m = 1
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 540.0-psi, 0, 0, 1.0)
else:
m = 0
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 360.0-psi, 0, 0, 1.0)
return alpha, sx, sy, m
progname = os.path.basename(sys.argv[0])
usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl=0.2 --aa=0.1 --sym=symmetry --CTF"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--ave2D", type="string" , default=False, help="write to the disk a stack of 2D averages")
parser.add_option("--var2D", type="string" , default=False, help="write to the disk a stack of 2D variances")
parser.add_option("--ave3D", type="string" , default=False, help="write to the disk reconstructed 3D average")
parser.add_option("--var3D", type="string" , default=False, help="compute 3D variability (time consuming!)")
parser.add_option("--img_per_grp", type="int" , default=10 , help="number of neighbouring projections")
parser.add_option("--no_norm", action="store_true", default=False, help="do not use normalization")
parser.add_option("--radiusvar", type="int" , default=-1 , help="radius for 3D var" )
parser.add_option("--npad", type="int" , default=2 , help="number of time to pad the original images")
parser.add_option("--sym" , type="string" , default="c1" , help="symmetry")
parser.add_option("--fl", type="float" , default=0.0 , help="stop-band frequency (Default - no filtration)")
parser.add_option("--aa", type="float" , default=0.0 , help="fall off of the filter (Default - no filtration)")
parser.add_option("--CTF", action="store_true", default=False, help="use CFT correction")
parser.add_option("--VERBOSE", action="store_true", default=False, help="Long output for debugging")
#parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
#parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" )
#parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" )
#parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
#parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
parser.add_option("--VAR" , action="store_true", default=False, help="stack on input consists of 2D variances (Default False)")
parser.add_option("--decimate", type="float", default=1.0, help="image decimate rate, a number large than 1. default is 1")
parser.add_option("--window", type="int", default=0, help="reduce images to a small image size without changing pixel_size. Default value is zero.")
#parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)")
parser.add_option("--nvec", type="int" , default=0 , help="number of eigenvectors, default = 0 meaning no PCA calculated")
parser.add_option("--symmetrize", action="store_true", default=False, help="Prepare input stack for handling symmetry (Default False)")
(options,args) = parser.parse_args()
#####
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD, MPI_TAG_UB
from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
from applications import MPI_start_end
from reconstruction import recons3d_em, recons3d_em_MPI
from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
from utilities import print_begin_msg, print_end_msg, print_msg
from utilities import read_text_row, get_image, get_im
from utilities import bcast_EMData_to_all, bcast_number_to_all
from utilities import get_symt
# This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015
from EMAN2db import db_open_dict
if options.symmetrize :
try:
sys.argv = mpi_init(len(sys.argv), sys.argv)
try:
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
if( number_of_proc > 1 ):
ERROR("Cannot use more than one CPU for symmetry prepration","sx3dvariability",1)
except:
pass
except:
pass
# Input
#instack = "Clean_NORM_CTF_start_wparams.hdf"
#instack = "bdb:data"
instack = args[0]
sym = options.sym
if( sym == "c1" ):
ERROR("Thre is no need to symmetrize stack for C1 symmetry","sx3dvariability",1)
if(instack[:4] !="bdb:"):
stack = "bdb:data"
delete_bdb(stack)
cmdexecute("sxcpy.py "+instack+" "+stack)
else:
stack = instack
qt = EMUtil.get_all_attributes(stack,'xform.projection')
na = len(qt)
ts = get_symt(sym)
ks = len(ts)
angsa = [None]*na
for k in xrange(ks):
delete_bdb("bdb:Q%1d"%k)
cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
DB = db_open_dict("bdb:Q%1d"%k)
for i in xrange(na):
ut = qt[i]*ts[k]
DB.set_attr(i, "xform.projection", ut)
#bt = ut.get_params("spider")
#angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
#write_text_row(angsa, 'ptsma%1d.txt'%k)
#cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
#cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k))
DB.close()
delete_bdb("bdb:sdata")
cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
#cmdexecute("ls EMAN2DB/sdata*")
a = get_im("bdb:sdata")
a.set_attr("variabilitysymmetry",sym)
a.write_image("bdb:sdata")
else:
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
if len(args) == 1:
stack = args[0]
else:
print( "usage: " + usage)
print( "Please run '" + progname + " -h' for detailed options")
return 1
t0 = time()
# obsolete flags
options.MPI = True
options.nvec = 0
options.radiuspca = -1
options.iter = 40
options.abs = 0.0
options.squ = 0.0
if options.fl > 0.0 and options.aa == 0.0:
ERROR("Fall off has to be given for the low-pass filter", "sx3dvariability", 1, myid)
if options.VAR and options.SND:
ERROR("Only one of var and SND can be set!", "sx3dvariability", myid)
exit()
if options.VAR and (options.ave2D or options.ave3D or options.var2D):
ERROR("When VAR is set, the program cannot output ave2D, ave3D or var2D", "sx3dvariability", 1, myid)
exit()
#if options.SND and (options.ave2D or options.ave3D):
# ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid)
# exit()
if options.nvec > 0 :
ERROR("PCA option not implemented", "sx3dvariability", 1, myid)
exit()
if options.nvec > 0 and options.ave3D == None:
ERROR("When doing PCA analysis, one must set ave3D", "sx3dvariability", myid=myid)
exit()
import string
options.sym = options.sym.lower()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
global_def.BATCH = True
if myid == main_node:
print_begin_msg("sx3dvariability")
print_msg("%-70s: %s\n"%("Input stack", stack))
img_per_grp = options.img_per_grp
nvec = options.nvec
radiuspca = options.radiuspca
symbaselen = 0
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
if options.sym != "c1" :
imgdata = get_im(stack)
try:
i = imgdata.get_attr("variabilitysymmetry")
if(i != options.sym):
ERROR("The symmetry provided does not agree with the symmetry of the input stack", "sx3dvariability", myid=myid)
except:
ERROR("Input stack is not prepared for symmetry, please follow instructions", "sx3dvariability", myid=myid)
from utilities import get_symt
i = len(get_symt(options.sym))
if((nima/i)*i != nima):
ERROR("The length of the input stack is incorrect for symmetry processing", "sx3dvariability", myid=myid)
symbaselen = nima/i
else: symbaselen = nima
else:
nima = 0
nx = 0
ny = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
Tracker ={}
Tracker["nx"] =nx
Tracker["ny"] =ny
Tracker["total_stack"]=nima
if options.decimate==1.:
if options.window !=0:
nx = options.window
ny = options.window
else:
if options.window ==0:
nx = int(nx/options.decimate)
ny = int(ny/options.decimate)
else:
nx = int(options.window/options.decimate)
ny = nx
symbaselen = bcast_number_to_all(symbaselen)
if radiuspca == -1: radiuspca = nx/2-2
if myid == main_node:
print_msg("%-70s: %d\n"%("Number of projection", nima))
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
"""
if options.SND:
from projection import prep_vol, prgs
from statistics import im_diff
from utilities import get_im, model_circle, get_params_proj, set_params_proj
from utilities import get_ctf, generate_ctf
from filter import filt_ctf
imgdata = EMData.read_images(stack, range(img_begin, img_end))
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
bcast_EMData_to_all(vol, myid)
volft, kb = prep_vol(vol)
mask = model_circle(nx/2-2, nx, ny)
varList = []
for i in xrange(img_begin, img_end):
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
if options.CTF:
ctf_params = get_ctf(imgdata[i-img_begin])
ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
diff2 = diff*diff
set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
varList.append(diff2)
mpi_barrier(MPI_COMM_WORLD)
"""
if options.VAR:
#varList = EMData.read_images(stack, range(img_begin, img_end))
varList = []
this_image = EMData()
for index_of_particle in xrange(img_begin,img_end):
this_image.read_image(stack,index_of_particle)
varList.append(image_decimate_window_xform_ctf(img,options.decimate,options.window,options.CTF))
else:
from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
from utilities import model_blank, nearest_proj, model_circle
from applications import pca
from statistics import avgvar, avgvar_ctf, ccc
from filter import filt_tanl
from morphology import threshold, square_root
from projection import project, prep_vol, prgs
from sets import Set
if myid == main_node:
t1 = time()
proj_angles = []
aveList = []
tab = EMUtil.get_all_attributes(stack, 'xform.projection')
for i in xrange(nima):
t = tab[i].get_params('spider')
phi = t['phi']
theta = t['theta']
psi = t['psi']
x = theta
if x > 90.0: x = 180.0 - x
x = x*10000+psi
proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
t2 = time()
print_msg("%-70s: %d\n"%("Number of neighboring projections", img_per_grp))
print_msg("...... Finding neighboring projections\n")
if options.VERBOSE:
print "Number of images per group: ", img_per_grp
print "Now grouping projections"
proj_angles.sort()
proj_angles_list = [0.0]*(nima*4)
if myid == main_node:
for i in xrange(nima):
proj_angles_list[i*4] = proj_angles[i][1]
proj_angles_list[i*4+1] = proj_angles[i][2]
proj_angles_list[i*4+2] = proj_angles[i][3]
proj_angles_list[i*4+3] = proj_angles[i][4]
proj_angles_list = bcast_list_to_all(proj_angles_list, myid, main_node)
proj_angles = []
for i in xrange(nima):
proj_angles.append([proj_angles_list[i*4], proj_angles_list[i*4+1], proj_angles_list[i*4+2], int(proj_angles_list[i*4+3])])
del proj_angles_list
proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp, range(img_begin, img_end))
all_proj = Set()
for im in proj_list:
for jm in im:
all_proj.add(proj_angles[jm][3])
all_proj = list(all_proj)
if options.VERBOSE:
print "On node %2d, number of images needed to be read = %5d"%(myid, len(all_proj))
index = {}
for i in xrange(len(all_proj)): index[all_proj[i]] = i
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
print_msg("%-70s: %.2f\n"%("Finding neighboring projections lasted [s]", time()-t2))
print_msg("%-70s: %d\n"%("Number of groups processed on the main node", len(proj_list)))
if options.VERBOSE:
print "Grouping projections took: ", (time()-t2)/60 , "[min]"
print "Number of groups on main node: ", len(proj_list)
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
print_msg("...... calculating the stack of 2D variances \n")
if options.VERBOSE:
print "Now calculating the stack of 2D variances"
proj_params = [0.0]*(nima*5)
aveList = []
varList = []
if nvec > 0:
eigList = [[] for i in xrange(nvec)]
if options.VERBOSE: print "Begin to read images on processor %d"%(myid)
ttt = time()
#imgdata = EMData.read_images(stack, all_proj)
img = EMData()
imgdata = []
for index_of_proj in xrange(len(all_proj)):
img.read_image(stack, all_proj[index_of_proj])
dmg = image_decimate_window_xform_ctf(img,options.decimate,options.window,options.CTF)
#print dmg.get_xsize(), "init"
imgdata.append(dmg)
if options.VERBOSE:
print "Reading images on processor %d done, time = %.2f"%(myid, time()-ttt)
print "On processor %d, we got %d images"%(myid, len(imgdata))
mpi_barrier(MPI_COMM_WORLD)
'''
imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
if options.fl > 0.0:
for k in xrange(len(imgdata2)):
imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
if myid == main_node:
vol.write_image("vol_ctf.hdf")
print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf"))
del vol, imgdata2
mpi_barrier(MPI_COMM_WORLD)
'''
from applications import prepare_2d_forPCA
from utilities import model_blank
for i in xrange(len(proj_list)):
ki = proj_angles[proj_list[i][0]][3]
if ki >= symbaselen: continue
mi = index[ki]
phiM, thetaM, psiM, s2xM, s2yM = get_params_proj(imgdata[mi])
grp_imgdata = []
for j in xrange(img_per_grp):
mj = index[proj_angles[proj_list[i][j]][3]]
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[mj])
alpha, sx, sy, mirror = params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror_list[i][j])
if thetaM <= 90:
if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, phiM-phi, 0.0, 0.0, 1.0)
else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, 180-(phiM-phi), 0.0, 0.0, 1.0)
else:
if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(phiM-phi), 0.0, 0.0, 1.0)
else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(180-(phiM-phi)), 0.0, 0.0, 1.0)
set_params2D(imgdata[mj], [alpha, sx, sy, mirror, 1.0])
grp_imgdata.append(imgdata[mj])
#print grp_imgdata[j].get_xsize(), imgdata[mj].get_xsize()
if not options.no_norm:
#print grp_imgdata[j].get_xsize()
mask = model_circle(nx/2-2, nx, nx)
for k in xrange(img_per_grp):
ave, std, minn, maxx = Util.infomask(grp_imgdata[k], mask, False)
grp_imgdata[k] -= ave
grp_imgdata[k] /= std
del mask
if options.fl > 0.0:
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2*nx
ny2 = 2*ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d(fft( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa) ),nx,ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
for k in xrange(img_per_grp):
grp_imgdata[k] = filt_tanl( grp_imgdata[k], options.fl, options.aa)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
from utilities import pad, read_text_file
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2*nx
ny2 = 2*ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d( fft( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1) ) , nx,ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
'''
if i < 10 and myid == main_node:
for k in xrange(10):
grp_imgdata[k].write_image("grp%03d.hdf"%i, k)
'''
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("pp.hdf", pp)
"""
ave, grp_imgdata = prepare_2d_forPCA(grp_imgdata)
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("qq.hdf", pp)
"""
var = model_blank(nx,ny)
for q in grp_imgdata: Util.add_img2( var, q )
Util.mul_scalar( var, 1.0/(len(grp_imgdata)-1))
# Switch to std dev
var = square_root(threshold(var))
#if options.CTF: ave, var = avgvar_ctf(grp_imgdata, mode="a")
#else: ave, var = avgvar(grp_imgdata, mode="a")
"""
if myid == main_node:
ave.write_image("avgv.hdf",i)
var.write_image("varv.hdf",i)
"""
set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])
aveList.append(ave)
varList.append(var)
if options.VERBOSE:
print "%5.2f%% done on processor %d"%(i*100.0/len(proj_list), myid)
if nvec > 0:
eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True)
for k in xrange(nvec):
set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
eigList[k].append(eig[k])
"""
if myid == 0 and i == 0:
for k in xrange(nvec):
eig[k].write_image("eig.hdf", k)
"""
del imgdata
# To this point, all averages, variances, and eigenvectors are computed
if options.ave2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node :
for im in xrange(len(aveList)):
aveList[im].write_image(options.ave2D, km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im+i+70000)
"""
nm = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('pix_err', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
"""
tmpvol=fpol(ave, Tracker["nx"],Tracker["nx"],Tracker["nx"])
tmpvol.write_image(options.ave2D, km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
for im in xrange(len(aveList)):
send_EMData(aveList[im], main_node,im+myid+70000)
"""
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
members = aveList[im].get_attr('pix_err')
mpi_send(members, len(members), MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
except:
mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
"""
if options.ave3D:
from fundamentals import fpol
if options.VERBOSE:
print "Reconstructing 3D average volume"
ave3D = recons3d_4nn_MPI(myid, aveList, symmetry=options.sym, npad=options.npad)
bcast_EMData_to_all(ave3D, myid)
if myid == main_node:
ave3D=fpol(ave3D,Tracker["nx"],Tracker["nx"],Tracker["nx"])
ave3D.write_image(options.ave3D)
print_msg("%-70s: %s\n"%("Writing to the disk volume reconstructed from averages as", options.ave3D))
del ave, var, proj_list, stack, phi, theta, psi, s2x, s2y, alpha, sx, sy, mirror, aveList
if nvec > 0:
for k in xrange(nvec):
if options.VERBOSE:
print "Reconstruction eigenvolumes", k
cont = True
ITER = 0
mask2d = model_circle(radiuspca, nx, nx)
while cont:
#print "On node %d, iteration %d"%(myid, ITER)
eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad)
bcast_EMData_to_all(eig3D, myid, main_node)
if options.fl > 0.0:
eig3D = filt_tanl(eig3D, options.fl, options.aa)
if myid == main_node:
eig3D.write_image("eig3d_%03d.hdf"%k, ITER)
Util.mul_img( eig3D, model_circle(radiuspca, nx, nx, nx) )
eig3Df, kb = prep_vol(eig3D)
del eig3D
cont = False
icont = 0
for l in xrange(len(eigList[k])):
phi, theta, psi, s2x, s2y = get_params_proj(eigList[k][l])
proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y])
cl = ccc(proj, eigList[k][l], mask2d)
if cl < 0.0:
icont += 1
cont = True
eigList[k][l] *= -1.0
u = int(cont)
u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD)
icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
if myid == main_node:
u = int(u[0])
print " Eigenvector: ",k," number changed ",int(icont[0])
else: u = 0
u = bcast_number_to_all(u, main_node)
cont = bool(u)
ITER += 1
del eig3Df, kb
mpi_barrier(MPI_COMM_WORLD)
del eigList, mask2d
if options.ave3D: del ave3D
if options.var2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node :
for im in xrange(len(varList)):
tmpvol=fpol(varList[im], Tracker["nx"], Tracker["nx"],1)
tmpvol.write_image(options.var2D, km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im+i+70000)
tmpvol=fpol(ave, Tracker["nx"], Tracker["nx"],1)
tmpvol.write_image(options.var2D, km)
km += 1
else:
mpi_send(len(varList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
for im in xrange(len(varList)):
send_EMData(varList[im], main_node, im+myid+70000)# What with the attributes??
mpi_barrier(MPI_COMM_WORLD)
if options.var3D:
if myid == main_node and options.VERBOSE:
print "Reconstructing 3D variability volume"
t6 = time()
radiusvar = options.radiusvar
if( radiusvar < 0 ): radiusvar = nx//2 -3
res = recons3d_4nn_MPI(myid, varList, symmetry=options.sym, npad=options.npad)
#res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
if myid == main_node:
from fundamentals import fpol
res =fpol(res, Tracker["nx"], Tracker["nx"], Tracker["nx"])
res.write_image(options.var3D)
if myid == main_node:
print_msg("%-70s: %.2f\n"%("Reconstructing 3D variability took [s]", time()-t6))
if options.VERBOSE:
print "Reconstruction took: %.2f [min]"%((time()-t6)/60)
if myid == main_node:
print_msg("%-70s: %.2f\n"%("Total time for these computations [s]", time()-t0))
if options.VERBOSE:
print "Total time for these computations: %.2f [min]"%((time()-t0)/60)
print_end_msg("sx3dvariability")
global_def.BATCH = False
from mpi import mpi_finalize
mpi_finalize()
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options] imagefile ...
This program can be used to extract various metadata/header information from images of any file format,
including BDB databases (though e2bdb.py has additional database-specific functionality).
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument("-H",
"--header",
action="store_true",
help="Show all header information",
default=False)
parser.add_argument("-N",
"--number",
type=int,
help="Image number for single image info",
default=-1)
parser.add_argument(
"-Q",
"--quality",
type=int,
help="Include only images with a single quality value (integer 0-9)",
default=-1)
parser.add_argument(
"--dfmin",
type=float,
help="Include only images with defocus >= the specified value",
default=None)
parser.add_argument(
"--dfmax",
type=float,
help="Include only images with defocus <= the specified value",
default=None)
parser.add_argument(
"--nameonly",
action="store_true",
help="Only display the matching filenames. No other info.",
default=False)
parser.add_argument(
"-s",
"--stat",
action="store_true",
help="Show statistical information about the image(s).",
default=False)
parser.add_argument("-E",
"--euler",
action="store_true",
help="Show Euler angles from header",
default=False)
parser.add_argument("-a",
"--all",
action="store_true",
help="Show info for all images in file",
default=False)
parser.add_argument(
"-C",
"--check",
action="store_true",
help=
"Checks to make sure all image numbers are populated with images, and that all images have valid CTF parameters",
default=False)
parser.add_argument(
"-c",
"--count",
action="store_true",
help="Just show a count of the number of particles in each file",
default=False)
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-2)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higner number means higher level of verboseness")
(options, args) = parser.parse_args()
if len(args) < 1:
print(usage)
parser.error("Specify image file")
if options.header and options.stat:
print("Only one of --header and --stat may be specified")
sys.exit(1)
nimgs = 0
for imagefile in args:
if options.quality != -1 or options.dfmin != None or options.dfmax != None:
try:
db = js_open_dict(info_name(imagefile))
if options.quality != -1:
q = db["quality"]
db.close()
if q != options.quality: continue
except:
sys.stderr.write(
"No quality for {}. Including\n".format(imagefile))
if options.nameonly:
print(imagefile)
continue
if imagefile.lower()[:4] != "bdb:":
try:
nimg = EMUtil.get_image_count(imagefile)
except:
print(imagefile, " is not a recognized image format")
continue
nimgs += nimg
imgtype = EMUtil.get_image_type(imagefile)
imgtypename = EMUtil.get_imagetype_name(imgtype)
if imgtype == EMUtil.ImageType.IMAGE_SPIDER and not options.stat:
image_index = -1
d = EMData()
try:
d.read_image(imagefile, max(options.number, 0), True)
except:
print("Image read error (%s)" % imagefile)
continue
if options.count: print("%d\t" % (nimg), end=' ')
if d["nz"] == 1:
print("%s\t %d images in %s format\t%d x %d" %
(imagefile, nimg, imgtypename, d["nx"], d["ny"]))
else:
print(
"%s\t %d images in %s format\t%d x %d x %d" %
(imagefile, nimg, imgtypename, d["nx"], d["ny"], d["nz"]))
else:
dct = db_open_dict(imagefile, ro=True)
d = dct.get_header(0)
nimg = len(dct)
nimgs += nimg
d = EMData(imagefile, max(options.number, 0), True)
if d["nz"] == 1:
print("%s\t %d images in BDB format\t%d x %d" %
(imagefile, len(dct), d["nx"], d["ny"]))
else:
print("%s\t %d images in BDB format\t%d x %d x %d" %
(imagefile, len(dct), d["nx"], d["ny"], d["nz"]))
if options.all or options.check:
imgn = list(range(nimg))
elif options.number >= 0:
imgn = [options.number]
elif options.header or options.stat or options.euler:
# If we request header information, without specifying an image #, assume the first image.
imgn = [0]
else:
imgn = []
nptcl = 0
d = EMData()
for i in imgn:
if options.stat: d.read_image(imagefile, i)
else:
try:
d.read_image(imagefile, i, True) #Jesus
except:
print("Error reading image ", imagefile, i)
if options.check:
try:
ctf = d["ctf"]
snr = ctf.snr[0]
except:
print("Error with CTF on ", imagefile, i)
else:
print("%d. " % i, end='')
try:
print("%d ptcl\t" % d["ptcl_repr"], end=' ')
nptcl += d["ptcl_repr"]
except:
print("\t", end='')
if options.stat:
print(
"apix=%-5.2f min=%-10.4g max=%-10.4g mean=%-10.4g sigma=%-9.4g skewness=%-9.4g kurtosis=%-9.4g"
% (d["apix_x"], d["minimum"], d["maximum"], d["mean"],
d["sigma"], d["skewness"], d["kurtosis"]),
end=' ')
try:
c = d["ctf"]
print(" defocus=%-6.2f B=%-1.0f" %
(c.defocus, c.bfactor))
except:
print(" ")
if options.euler:
# d=EMData(imagefile,i,True) #Jesus
try:
print("%s" % (str(d["xform.projection"])))
except:
print("No transform information", end=' ')
if options.header:
# d=EMData(imagefile,i, True) #Jesus
print("")
keys = list(d.get_attr_dict().keys())
keys.sort()
for k in keys:
print("\t%s: %s" % (k, str(d[k])))
print("======================")
if nimgs > 1: print("%d total images" % nimgs)
try:
print("representing %d particles" % nptcl)
except:
pass
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options] <volume> <mask> <output>
WARNING: Do not use this program. The whole concept has been replaced by "gold standard" refinement.
Note that this method has not yet been published, and is not yet a reliable
scheme for resolution determination. DO NOT EXPECT RELIABLE ANSWERS AT THIS POINT !
This will compute an FSC resolution curve from a single structure by
comparing the power spectrum of the noise region to the power spectrum
of the reconstructed particle. The 'mask' input should mask out the particle,
and should be a 'soft' mask. Sharp edges on the mask may ruin the results.
proc3d (EMAN1) with the automask2 option can produce appropriate masks. The
3rd parameter (in automask2) should be about 10 percent of the box size. This will not
work with particles that have been tightly masked already. If doing an EMAN1
reconstruction with the amask= option, you must also use the refmaskali option
(in the refine command). Note that the resultant curve is guaranteed to go
to zero at near Nyquist because it assumes that the SNR is zero near Nyquist.
If your data is undersampled, the resulting curve may also be inaccurate.
Models should also not be heavily low-pass filtered
"""
parser = EMArgumentParser(usage=usage,version=EMANVERSION)
parser.add_pos_argument(name="volume",help="Volume to compute resolution on.", default="", guitype='filebox', row=0, col=0,rowspan=1, colspan=2)
parser.add_pos_argument(name="mask",help="Mask for the volume. The volume is masked by this file (should be ssoft mask).", default="", guitype='filebox', row=1, col=0,rowspan=1, colspan=2)
parser.add_pos_argument(name="output",help="Name of the output file. These data will contain FSC stats.", default="", guitype='strbox', row=2, col=0,rowspan=1, colspan=2)
parser.add_header(name="eotestheader", help='Options below this label are specific to e2resolution', title="### e2resolution options ###", row=3, col=0, rowspan=1, colspan=2)
parser.add_argument("--apix", "-A", type=float, help="A/voxel", default=-1.0, guitype='floatbox', row=4, col=0, rowspan=1, colspan=1)
parser.add_argument("--path", type=str,help="The name the e2refine directory that contains the reconstruction data. If specified will place curves generated in bdb:path#convergence.results", guitype='dirbox', default=None, dirbasename='refine', row=4, col=1,rowspan=1, colspan=1)
parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higner number means higher level of verboseness")
(options, args) = parser.parse_args()
if len(args)<3 : parser.error("Input and output files required")
E2n=E2init(sys.argv,options.ppid)
#options.align=parsemodopt(options.align)
print "WARNING: e2resolution is an experimental program. It does not (yet) produce reliable resolution curves in most cases."
print "read models"
data=EMData(args[0],0)
mask=EMData(args[1],0)
# mask.to_one()
# mask.process_inplace("mask.gaussian",{"outer_radius":mask.get_xsize()/6,"inner_radius":mask.get_xsize()/6})
mask.write_image("msk.mrc",0)
# inverted mask
maski=mask.copy()
maski*=-1.0
maski+=1.0
maski.process_inplace("mask.gaussian",{"outer_radius":mask.get_xsize()/6,"inner_radius":mask.get_xsize()/3})
maski.write_image("msk2.mrc",0)
noise=data.copy()
noise*=maski
data*=mask
print "compute FFT"
dataf=data.do_fft()
noisef=noise.do_fft()
print "compute power 1"
datapow=dataf.calc_radial_dist(dataf.get_ysize()/2-1,1,1,1)
print "compute power 2"
noisepow=noisef.calc_radial_dist(noisef.get_ysize()/2-1,1,1,1)
x=range(1,len(datapow)+1)
if options.apix>0:
x=[i/(len(datapow)*options.apix*2.0) for i in x]
else:
x=[i/(len(datapow)*data["apix_x"]*2.0) for i in x]
# normalize noise near Nyquist
s=0
sn=0
for i in range(int(len(noisepow)*.9),len(noisepow)-1):
if datapow[i]<datapow[i+1] or noisepow[i]<noisepow[i+1] : continue
s+=datapow[i]/noisepow[i]
sn+=1.0
if sn==0 :
print "Warning, strange normalization"
s=datapow[int(len(noisepow)*.9)]/noisepow[int(len(noisepow)*.9)]
else: s/=sn
noisepow=[i*s for i in noisepow]
#
# # normalize based on volume
# datapow=[v/mask["mean"] for v in datapow]
# noisepow=[v/maski["mean"] for v in noisepow]
# compute signal to noise ratio
snr=[]
for i in range(len(datapow)):
try: snr.append((datapow[i]-noisepow[i])/noisepow[i])
except: snr.append(0)
# convert to FSC
fsc=[i/(2.0+i) for i in snr]
out=file(args[2],"w")
for i in range(len(fsc)): out.write("%f\t%f\n"%(x[i],fsc[i]))
out.close()
out=file(args[2]+".dat","w")
for i in range(len(fsc)): out.write("%f\t%f\n"%(x[i],datapow[i]))
out.close()
out=file(args[2]+".noi","w")
for i in range(len(noisepow)): out.write("%f\t%f\n"%(x[i],noisepow[i]))
out.close()
if options.path != None:
s = base_name(args[0])
db = db_open_dict("bdb:"+options.path+"#convergence.results")
db[s+"res_fsc"] = [x,fsc] # warning, changing this naming convention will disrupt forms in the workflow (make them fail)
db[s+"res_datapow"] = [x,datapow]
db[s+"res_noisepow"] = [x,noisepow]
db_close_dict("bdb:"+options.path+"#convergence.results")
E2end(E2n)
def get_params(self):
table_params = []
params = []
db = db_open_dict(self.form_db_name)
p,n = self.get_tomo_hunter_basic_table() # note n is unused, it's a refactoring residual
params.append(ParamDef(name="blurb",vartype="text",desc_short="Interactive use of tomohunter",desc_long="",property=None,defaultunits=self.__doc__,choices=None))
params.append(p)
psavesteps = ParamDef(name="savesteps",vartype="boolean",desc_short="Savesteps",desc_long="Save the steps",property=None,defaultunits=db.get("savesteps",dfl=True),choices=None)
psaveali = ParamDef(name="saveali",vartype="boolean",desc_short="Saveali",desc_long="Save the alignments",property=None,defaultunits=db.get("saveali",dfl=True),choices=None)
params.append([psavesteps, psaveali])
piter = ParamDef(name="number of iterations",vartype="int",desc_short="Number of iterations", desc_long="Number of iterations",property=None,defaultunits=db.get("number of iterations",dfl=5),choices=None )
pncoarse = ParamDef(name="coarse number",vartype="int",desc_short="Coarse Number", desc_long="Coarse number",property=None,defaultunits=db.get("coarse number",dfl=6),choices=None )
params.append([piter, pncoarse])
pshrink = ParamDef(name="Percentage to shrink",vartype="int",desc_short="Shrink", desc_long="Percentage to shrink",property=None,defaultunits=db.get("Percentage to shrink",dfl=2),choices=None )
pshrinkrefine = ParamDef(name="Percentage to shrink, refinement",vartype="int",desc_short="Shrink refine", desc_long="Percentage to shrink for refienment",property=None,defaultunits=db.get("Percentage to shrink, refinement",dfl=2),choices=None )
params.append([pshrink, pshrinkrefine])
proc_data = dump_processors_list()
masks = {}
for key in proc_data.keys():
if len(key) >= 5 and key[:5] == "mask.":
masks[key] = proc_data[key]
masks["None"] = ["Choose this to stop masking from occuring"]
pmask = ParamDef("mask",vartype="string",desc_short="Mask",desc_long="The mask to apply to the subtomos",property=None,defaultunits=db.get("mask",dfl="None"),choices=masks)
pmaskparams = ParamDef("maskparams",vartype="string",desc_short="Params",desc_long="Parameters for the mask",property=None,defaultunits=db.get("maskparams",dfl=""))
params.append([pmask, pmaskparams])
filters = {}
for key in proc_data.keys():
if len(key) >= 7 and key[:7] == "filter.":
filters[key] = proc_data[key]
filters["None"] = ["Choose this to stop filtering from occuring"]
pfilter = ParamDef("filter",vartype="string",desc_short="Filter",desc_long="The Filter to apply to the subtomos",property=None,defaultunits=db.get("filter",dfl="None"),choices=filters)
pfilterparams = ParamDef("filterparams",vartype="string",desc_short="Params",desc_long="Parameters for the filter",property=None,defaultunits=db.get("filterparams",dfl=""))
params.append([pfilter, pfilterparams])
ali_data = dump_aligners_list()
caligners = {}
for key in ali_data.keys():
if len(key) >= 19 and key[:19] == "rotate_translate_3d":
caligners[key] = ali_data[key]
pali = ParamDef("aligner3D",vartype="string",desc_short="Aligner3D",desc_long="The 3D course aligner",property=None,defaultunits=db.get("aligner3D",dfl="rotate_translate_3d"),choices=caligners)
paliparams = ParamDef("ali3dparams",vartype="string",desc_short="Params",desc_long="Parameters for the 3D aligner",property=None,defaultunits=db.get("ali3dparams",dfl="search=10:delta=15:dphi=15:verbose=1"))
params.append([pali, paliparams])
craligners = {}
for key in ali_data.keys():
if len(key) >= 9 and key[:9] == "refine_3d":
craligners[key] = ali_data[key]
prali = ParamDef("raligner3D",vartype="string",desc_short="RAligner3D",desc_long="The 3D refine aligner",property=None,defaultunits=db.get("raligner3D",dfl="refine_3d_grid"),choices=craligners)
praliparams = ParamDef("rali3dparams",vartype="string",desc_short="Params",desc_long="Parameters for the 3D refine aligner",property=None,defaultunits=db.get("rali3dparams",dfl="verbose=1"))
params.append([prali, praliparams])
ppostfilter = ParamDef("postfilter",vartype="string",desc_short="PostFilter",desc_long="The Filter to apply to the average",property=None,defaultunits=db.get("postfilter",dfl="None"),choices=filters)
ppostfilterparams = ParamDef("postfilterparams",vartype="string",desc_short="Params",desc_long="Parameters for the postfilter",property=None,defaultunits=db.get("postfilterparams",dfl=""))
params.append([ppostfilter, ppostfilterparams])
pparallel = ParamDef("parallel",vartype="string",desc_short="Parallel",desc_long="Parallalization parameters",property=None,defaultunits=db.get("parallel",dfl=""))
params.append(pparallel)
#pylong = ParamDef(name="yshort",vartype="boolean",desc_short="yshort",desc_long="Use Z axis as normal",property=None,defaultunits=1,choices=None)
#pinmem = ParamDef(name="inmemory",vartype="boolean",desc_short="inmemory",desc_long="Load the tomo into memory",property=None,defaultunits=1,choices=None)
#papix = ParamDef(name="apix",vartype="float",desc_short=u"\u212B per pixel", desc_long="Angstroms per pixel",property=None,defaultunits=1.0,choices=None )
#params.append([pylong, pinmem, papix])
#params.append(papix)
table_params.append(["Main",params])
advanced_params = []
r,rn = self.get_tomo_hunter_ref_table() # note rn is unused, it's a refactoring residual
advanced_params.append(ParamDef(name="blurb",vartype="text",desc_short="Interactive use tomo refs",desc_long="",property=None,defaultunits="Use this for selecting references. If no reference is chosen, then reference free alignment will be executed",choices=None))
advanced_params.append(r)
table_params.append(["References",advanced_params])
# db = db_open_dict(self.form_db_name)
# params.append(ParamDef(name="interface_boxsize",vartype="int",desc_short="Box size",desc_long="An integer value",property=None,defaultunits=db.get("interface_boxsize",dfl=128),choices=[]))
# #db_close_dict(self.form_db_name)
return table_params
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options] imagefile ...
This program can be used to extract various metadata/header information from images of any file format,
including BDB databases (though e2bdb.py has additional database-specific functionality).
"""
parser = EMArgumentParser(usage=usage,version=EMANVERSION)
parser.add_argument("-H", "--header", action="store_true",help="Show all header information",default=False)
parser.add_argument("-N", "--number", type=int, help="Image number for single image info",default=-1)
parser.add_argument("-s", "--stat", action="store_true",help="Show statistical information about the image(s).",default=False)
parser.add_argument("-E", "--euler", action="store_true",help="Show Euler angles from header",default=False)
parser.add_argument("-a", "--all", action="store_true",help="Show info for all images in file",default=False)
parser.add_argument("-c", "--count", action="store_true",help="Just show a count of the number of particles in each file",default=False)
parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-2)
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higner number means higher level of verboseness")
(options, args) = parser.parse_args()
if len(args)<1:
print usage
parser.error("Specify image file")
if options.header and options.stat :
print "Only one of --header and --stat may be specified"
sys.exit(1)
nimgs=0
for imagefile in args:
if imagefile.lower()[:4]!="bdb:" :
nimg = EMUtil.get_image_count(imagefile)
nimgs+=nimg
imgtype = EMUtil.get_image_type(imagefile)
imgtypename = EMUtil.get_imagetype_name(imgtype)
if imgtype == EMUtil.ImageType.IMAGE_SPIDER and not options.stat:
image_index = -1
d=EMData()
try: d.read_image(imagefile, max(options.number,0), True)
except :
print "Image read error (%s)"%imagefile
continue
if options.count : print "%d\t"%(nimg),
if d["nz"]==1 : print "%s\t %d images in %s format\t%d x %d"%(imagefile,nimg,imgtypename,d["nx"],d["ny"])
else : print "%s\t %d images in %s format\t%d x %d x %d"%(imagefile,nimg,imgtypename,d["nx"],d["ny"],d["nz"])
else :
dct=db_open_dict(imagefile,ro=True)
d=dct.get_header(0)
nimg=len(dct)
nimgs+=nimg
d=EMData(imagefile, max(options.number,0), True)
if d["nz"]==1 : print "%s\t %d images in BDB format\t%d x %d"%(imagefile,len(dct),d["nx"],d["ny"])
else : print "%s\t %d images in BDB format\t%d x %d x %d"%(imagefile,len(dct),d["nx"],d["ny"],d["nz"])
if options.all:
imgn = xrange(nimg)
elif options.number >= 0:
imgn = [options.number]
elif options.header or options.stat or options.euler:
# If we request header information, without specifying an image #, assume the first image.
imgn = [0]
else:
imgn = []
nptcl=0
d=EMData()
for i in imgn:
if options.stat : d.read_image(imagefile,i)
else : d.read_image(imagefile,i,True) #Jesus
print "%d. "%i,
try:
print "%d ptcl\t"%d["ptcl_repr"],
nptcl+=d["ptcl_repr"]
except:
print "\t"
if options.stat :
print "min=%1.4g\tmax=%1.4g\tmean=%1.4g\tsigma=%1.4g\tskewness=%1.4g \tkurtosis=%1.4g"%(d["minimum"],d["maximum"],d["mean"],d["sigma"],d["skewness"],d["kurtosis"]),
try:
c=d["ctf"]
print "\tdefocus=%1.2f\tB=%1.0f"%(c.defocus,c.bfactor),
except:
print " "
if options.euler:
# d=EMData(imagefile,i,True) #Jesus
try: print "%s"%(str(d["xform.projection"])),
except : print "No transform information",
if options.header :
# d=EMData(imagefile,i, True) #Jesus
print ""
keys=d.get_attr_dict().keys()
keys.sort()
for k in keys:
print "\t%s: %s"%(k,str(d[k]))
print "======================"
print ""
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options] imagefile ...
This program can be used to extract various metadata/header information from images of any file format,
including BDB databases (though e2bdb.py has additional database-specific functionality).
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument("-H",
"--header",
action="store_true",
help="Show all header information",
default=False)
parser.add_argument("-N",
"--number",
type=int,
help="Image number for single image info",
default=-1)
parser.add_argument(
"-s",
"--stat",
action="store_true",
help="Show statistical information about the image(s).",
default=False)
parser.add_argument("-E",
"--euler",
action="store_true",
help="Show Euler angles from header",
default=False)
parser.add_argument("-a",
"--all",
action="store_true",
help="Show info for all images in file",
default=False)
parser.add_argument(
"-c",
"--count",
action="store_true",
help="Just show a count of the number of particles in each file",
default=False)
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-2)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higner number means higher level of verboseness")
(options, args) = parser.parse_args()
if len(args) < 1:
print usage
parser.error("Specify image file")
if options.header and options.stat:
print "Only one of --header and --stat may be specified"
sys.exit(1)
nimgs = 0
for imagefile in args:
if imagefile.lower()[:4] != "bdb:":
nimg = EMUtil.get_image_count(imagefile)
nimgs += nimg
imgtype = EMUtil.get_image_type(imagefile)
imgtypename = EMUtil.get_imagetype_name(imgtype)
if imgtype == EMUtil.ImageType.IMAGE_SPIDER and not options.stat:
image_index = -1
d = EMData()
try:
d.read_image(imagefile, max(options.number, 0), True)
except:
print "Image read error (%s)" % imagefile
continue
if options.count: print "%d\t" % (nimg),
if d["nz"] == 1:
print "%s\t %d images in %s format\t%d x %d" % (
imagefile, nimg, imgtypename, d["nx"], d["ny"])
else:
print "%s\t %d images in %s format\t%d x %d x %d" % (
imagefile, nimg, imgtypename, d["nx"], d["ny"], d["nz"])
else:
dct = db_open_dict(imagefile, ro=True)
d = dct.get_header(0)
nimg = len(dct)
nimgs += nimg
d = EMData(imagefile, max(options.number, 0), True)
if d["nz"] == 1:
print "%s\t %d images in BDB format\t%d x %d" % (
imagefile, len(dct), d["nx"], d["ny"])
else:
print "%s\t %d images in BDB format\t%d x %d x %d" % (
imagefile, len(dct), d["nx"], d["ny"], d["nz"])
if options.all:
imgn = xrange(nimg)
elif options.number >= 0:
imgn = [options.number]
elif options.header or options.stat or options.euler:
# If we request header information, without specifying an image #, assume the first image.
imgn = [0]
else:
imgn = []
nptcl = 0
d = EMData()
for i in imgn:
if options.stat: d.read_image(imagefile, i)
else: d.read_image(imagefile, i, True) #Jesus
print "%d. " % i,
try:
print "%d ptcl\t" % d["ptcl_repr"],
nptcl += d["ptcl_repr"]
except:
print "\t"
if options.stat:
print "min=%1.4g\tmax=%1.4g\tmean=%1.4g\tsigma=%1.4g\tskewness=%1.4g \tkurtosis=%1.4g" % (
d["minimum"], d["maximum"], d["mean"], d["sigma"],
d["skewness"], d["kurtosis"]),
try:
c = d["ctf"]
print "\tdefocus=%1.2f\tB=%1.0f" % (c.defocus, c.bfactor),
except:
print " "
if options.euler:
# d=EMData(imagefile,i,True) #Jesus
try:
print "%s" % (str(d["xform.projection"])),
except:
print "No transform information",
if options.header:
# d=EMData(imagefile,i, True) #Jesus
print ""
keys = d.get_attr_dict().keys()
keys.sort()
for k in keys:
print "\t%s: %s" % (k, str(d[k]))
print "======================"
print ""
if nimgs > 1: print "%d total images" % nimgs
try:
print "representing %d particles" % nptcl
except:
pass
def main():
def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
if mirror:
m = 1
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
540.0 - psi, 0, 0, 1.0)
else:
m = 0
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
360.0 - psi, 0, 0, 1.0)
return alpha, sx, sy, m
progname = os.path.basename(sys.argv[0])
usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl=15. --aa=0.01 --sym=symmetry --CTF"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--output_dir",
type="string",
default="./",
help="output directory")
parser.add_option("--ave2D",
type="string",
default=False,
help="write to the disk a stack of 2D averages")
parser.add_option("--var2D",
type="string",
default=False,
help="write to the disk a stack of 2D variances")
parser.add_option("--ave3D",
type="string",
default=False,
help="write to the disk reconstructed 3D average")
parser.add_option("--var3D",
type="string",
default=False,
help="compute 3D variability (time consuming!)")
parser.add_option("--img_per_grp",
type="int",
default=10,
help="number of neighbouring projections")
parser.add_option("--no_norm",
action="store_true",
default=False,
help="do not use normalization")
#parser.add_option("--radius", type="int" , default=-1 , help="radius for 3D variability" )
parser.add_option("--npad",
type="int",
default=2,
help="number of time to pad the original images")
parser.add_option("--sym", type="string", default="c1", help="symmetry")
parser.add_option(
"--fl",
type="float",
default=0.0,
help=
"cutoff freqency in absolute frequency (0.0-0.5). (Default - no filtration)"
)
parser.add_option(
"--aa",
type="float",
default=0.0,
help=
"fall off of the filter. Put 0.01 if user has no clue about falloff (Default - no filtration)"
)
parser.add_option("--CTF",
action="store_true",
default=False,
help="use CFT correction")
parser.add_option("--VERBOSE",
action="store_true",
default=False,
help="Long output for debugging")
#parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
#parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" )
#parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" )
#parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
#parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
parser.add_option(
"--VAR",
action="store_true",
default=False,
help="stack on input consists of 2D variances (Default False)")
parser.add_option(
"--decimate",
type="float",
default=1.0,
help=
"image decimate rate, a number larger (expand image) or less (shrink image) than 1. default is 1"
)
parser.add_option(
"--window",
type="int",
default=0,
help=
"reduce images to a small image size without changing pixel_size. Default value is zero."
)
#parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)")
parser.add_option(
"--nvec",
type="int",
default=0,
help="number of eigenvectors, default = 0 meaning no PCA calculated")
parser.add_option(
"--symmetrize",
action="store_true",
default=False,
help="Prepare input stack for handling symmetry (Default False)")
(options, args) = parser.parse_args()
#####
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD
from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
from applications import MPI_start_end
from reconstruction import recons3d_em, recons3d_em_MPI
from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
from utilities import print_begin_msg, print_end_msg, print_msg
from utilities import read_text_row, get_image, get_im
from utilities import bcast_EMData_to_all, bcast_number_to_all
from utilities import get_symt
# This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015
from EMAN2db import db_open_dict
# Set up global variables related to bdb cache
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
# Set up global variables related to ERROR function
global_def.BATCH = True
# detect if program is running under MPI
RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ
if RUNNING_UNDER_MPI:
global_def.MPI = True
if options.symmetrize:
if RUNNING_UNDER_MPI:
try:
sys.argv = mpi_init(len(sys.argv), sys.argv)
try:
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
if (number_of_proc > 1):
ERROR(
"Cannot use more than one CPU for symmetry prepration",
"sx3dvariability", 1)
except:
pass
except:
pass
if options.output_dir != "./" and not os.path.exists(
options.output_dir):
os.mkdir(options.output_dir)
# Input
#instack = "Clean_NORM_CTF_start_wparams.hdf"
#instack = "bdb:data"
from logger import Logger, BaseLogger_Files
if os.path.exists(os.path.join(options.output_dir, "log.txt")):
os.remove(os.path.join(options.output_dir, "log.txt"))
log_main = Logger(BaseLogger_Files())
log_main.prefix = os.path.join(options.output_dir, "./")
instack = args[0]
sym = options.sym.lower()
if (sym == "c1"):
ERROR("There is no need to symmetrize stack for C1 symmetry",
"sx3dvariability", 1)
line = ""
for a in sys.argv:
line += " " + a
log_main.add(line)
if (instack[:4] != "bdb:"):
if output_dir == "./": stack = "bdb:data"
else: stack = "bdb:" + options.output_dir + "/data"
delete_bdb(stack)
junk = cmdexecute("sxcpy.py " + instack + " " + stack)
else:
stack = instack
qt = EMUtil.get_all_attributes(stack, 'xform.projection')
na = len(qt)
ts = get_symt(sym)
ks = len(ts)
angsa = [None] * na
for k in xrange(ks):
#Qfile = "Q%1d"%k
if options.output_dir != "./":
Qfile = os.path.join(options.output_dir, "Q%1d" % k)
else:
Qfile = os.path.join(options.output_dir, "Q%1d" % k)
#delete_bdb("bdb:Q%1d"%k)
delete_bdb("bdb:" + Qfile)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
junk = cmdexecute("e2bdb.py " + stack + " --makevstack=bdb:" +
Qfile)
#DB = db_open_dict("bdb:Q%1d"%k)
DB = db_open_dict("bdb:" + Qfile)
for i in xrange(na):
ut = qt[i] * ts[k]
DB.set_attr(i, "xform.projection", ut)
#bt = ut.get_params("spider")
#angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
#write_text_row(angsa, 'ptsma%1d.txt'%k)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
#junk = cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k))
DB.close()
if options.output_dir == "./": delete_bdb("bdb:sdata")
else: delete_bdb("bdb:" + options.output_dir + "/" + "sdata")
#junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
sdata = "bdb:" + options.output_dir + "/" + "sdata"
print(sdata)
junk = cmdexecute("e2bdb.py " + options.output_dir +
" --makevstack=" + sdata + " --filt=Q")
#junk = cmdexecute("ls EMAN2DB/sdata*")
#a = get_im("bdb:sdata")
a = get_im(sdata)
a.set_attr("variabilitysymmetry", sym)
#a.write_image("bdb:sdata")
a.write_image(sdata)
else:
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
if len(args) == 1:
stack = args[0]
else:
print(("usage: " + usage))
print(("Please run '" + progname + " -h' for detailed options"))
return 1
t0 = time()
# obsolete flags
options.MPI = True
options.nvec = 0
options.radiuspca = -1
options.iter = 40
options.abs = 0.0
options.squ = 0.0
if options.fl > 0.0 and options.aa == 0.0:
ERROR("Fall off has to be given for the low-pass filter",
"sx3dvariability", 1, myid)
if options.VAR and options.SND:
ERROR("Only one of var and SND can be set!", "sx3dvariability",
myid)
exit()
if options.VAR and (options.ave2D or options.ave3D or options.var2D):
ERROR(
"When VAR is set, the program cannot output ave2D, ave3D or var2D",
"sx3dvariability", 1, myid)
exit()
#if options.SND and (options.ave2D or options.ave3D):
# ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid)
# exit()
if options.nvec > 0:
ERROR("PCA option not implemented", "sx3dvariability", 1, myid)
exit()
if options.nvec > 0 and options.ave3D == None:
ERROR("When doing PCA analysis, one must set ave3D",
"sx3dvariability",
myid=myid)
exit()
import string
options.sym = options.sym.lower()
# if global_def.CACHE_DISABLE:
# from utilities import disable_bdb_cache
# disable_bdb_cache()
# global_def.BATCH = True
if myid == main_node:
if options.output_dir != "./" and not os.path.exists(
options.output_dir):
os.mkdir(options.output_dir)
img_per_grp = options.img_per_grp
nvec = options.nvec
radiuspca = options.radiuspca
from logger import Logger, BaseLogger_Files
#if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt"))
log_main = Logger(BaseLogger_Files())
log_main.prefix = os.path.join(options.output_dir, "./")
if myid == main_node:
line = ""
for a in sys.argv:
line += " " + a
log_main.add(line)
log_main.add("-------->>>Settings given by all options<<<-------")
log_main.add("instack :" + stack)
log_main.add("output_dir :" + options.output_dir)
log_main.add("var3d :" + options.var3D)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
#print_begin_msg("sx3dvariability")
msg = "sx3dvariability"
log_main.add(msg)
print(line, msg)
msg = ("%-70s: %s\n" % ("Input stack", stack))
log_main.add(msg)
print(line, msg)
symbaselen = 0
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
if options.sym != "c1":
imgdata = get_im(stack)
try:
i = imgdata.get_attr("variabilitysymmetry").lower()
if (i != options.sym):
ERROR(
"The symmetry provided does not agree with the symmetry of the input stack",
"sx3dvariability",
myid=myid)
except:
ERROR(
"Input stack is not prepared for symmetry, please follow instructions",
"sx3dvariability",
myid=myid)
from utilities import get_symt
i = len(get_symt(options.sym))
if ((nima / i) * i != nima):
ERROR(
"The length of the input stack is incorrect for symmetry processing",
"sx3dvariability",
myid=myid)
symbaselen = nima / i
else:
symbaselen = nima
else:
nima = 0
nx = 0
ny = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
Tracker = {}
Tracker["total_stack"] = nima
if options.decimate == 1.:
if options.window != 0:
nx = options.window
ny = options.window
else:
if options.window == 0:
nx = int(nx * options.decimate)
ny = int(ny * options.decimate)
else:
nx = int(options.window * options.decimate)
ny = nx
Tracker["nx"] = nx
Tracker["ny"] = ny
Tracker["nz"] = nx
symbaselen = bcast_number_to_all(symbaselen)
if radiuspca == -1: radiuspca = nx / 2 - 2
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = "%-70s: %d\n" % ("Number of projection", nima)
log_main.add(msg)
print(line, msg)
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
"""
if options.SND:
from projection import prep_vol, prgs
from statistics import im_diff
from utilities import get_im, model_circle, get_params_proj, set_params_proj
from utilities import get_ctf, generate_ctf
from filter import filt_ctf
imgdata = EMData.read_images(stack, range(img_begin, img_end))
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
bcast_EMData_to_all(vol, myid)
volft, kb = prep_vol(vol)
mask = model_circle(nx/2-2, nx, ny)
varList = []
for i in xrange(img_begin, img_end):
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
if options.CTF:
ctf_params = get_ctf(imgdata[i-img_begin])
ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
diff2 = diff*diff
set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
varList.append(diff2)
mpi_barrier(MPI_COMM_WORLD)
"""
if options.VAR:
#varList = EMData.read_images(stack, range(img_begin, img_end))
varList = []
this_image = EMData()
for index_of_particle in xrange(img_begin, img_end):
this_image.read_image(stack, index_of_particle)
varList.append(
image_decimate_window_xform_ctf(this_image,
options.decimate,
options.window,
options.CTF))
else:
from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
from utilities import model_blank, nearest_proj, model_circle
from applications import pca
from statistics import avgvar, avgvar_ctf, ccc
from filter import filt_tanl
from morphology import threshold, square_root
from projection import project, prep_vol, prgs
from sets import Set
if myid == main_node:
t1 = time()
proj_angles = []
aveList = []
tab = EMUtil.get_all_attributes(stack, 'xform.projection')
for i in xrange(nima):
t = tab[i].get_params('spider')
phi = t['phi']
theta = t['theta']
psi = t['psi']
x = theta
if x > 90.0: x = 180.0 - x
x = x * 10000 + psi
proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
t2 = time()
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = "%-70s: %d\n" % ("Number of neighboring projections",
img_per_grp)
log_main.add(msg)
print(line, msg)
msg = "...... Finding neighboring projections\n"
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
msg = "Number of images per group: %d" % img_per_grp
log_main.add(msg)
print(line, msg)
msg = "Now grouping projections"
log_main.add(msg)
print(line, msg)
proj_angles.sort()
proj_angles_list = [0.0] * (nima * 4)
if myid == main_node:
for i in xrange(nima):
proj_angles_list[i * 4] = proj_angles[i][1]
proj_angles_list[i * 4 + 1] = proj_angles[i][2]
proj_angles_list[i * 4 + 2] = proj_angles[i][3]
proj_angles_list[i * 4 + 3] = proj_angles[i][4]
proj_angles_list = bcast_list_to_all(proj_angles_list, myid,
main_node)
proj_angles = []
for i in xrange(nima):
proj_angles.append([
proj_angles_list[i * 4], proj_angles_list[i * 4 + 1],
proj_angles_list[i * 4 + 2],
int(proj_angles_list[i * 4 + 3])
])
del proj_angles_list
proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp,
range(img_begin, img_end))
all_proj = Set()
for im in proj_list:
for jm in im:
all_proj.add(proj_angles[jm][3])
all_proj = list(all_proj)
if options.VERBOSE:
print("On node %2d, number of images needed to be read = %5d" %
(myid, len(all_proj)))
index = {}
for i in xrange(len(all_proj)):
index[all_proj[i]] = i
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("%-70s: %.2f\n" %
("Finding neighboring projections lasted [s]",
time() - t2))
log_main.add(msg)
print(msg)
msg = ("%-70s: %d\n" %
("Number of groups processed on the main node",
len(proj_list)))
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
print("Grouping projections took: ", (time() - t2) / 60,
"[min]")
print("Number of groups on main node: ", len(proj_list))
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("...... calculating the stack of 2D variances \n")
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
print("Now calculating the stack of 2D variances")
proj_params = [0.0] * (nima * 5)
aveList = []
varList = []
if nvec > 0:
eigList = [[] for i in xrange(nvec)]
if options.VERBOSE:
print("Begin to read images on processor %d" % (myid))
ttt = time()
#imgdata = EMData.read_images(stack, all_proj)
imgdata = []
for index_of_proj in xrange(len(all_proj)):
#img = EMData()
#img.read_image(stack, all_proj[index_of_proj])
dmg = image_decimate_window_xform_ctf(
get_im(stack, all_proj[index_of_proj]), options.decimate,
options.window, options.CTF)
#print dmg.get_xsize(), "init"
imgdata.append(dmg)
if options.VERBOSE:
print("Reading images on processor %d done, time = %.2f" %
(myid, time() - ttt))
print("On processor %d, we got %d images" %
(myid, len(imgdata)))
mpi_barrier(MPI_COMM_WORLD)
'''
imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
if options.fl > 0.0:
for k in xrange(len(imgdata2)):
imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
if myid == main_node:
vol.write_image("vol_ctf.hdf")
print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf"))
del vol, imgdata2
mpi_barrier(MPI_COMM_WORLD)
'''
from applications import prepare_2d_forPCA
from utilities import model_blank
for i in xrange(len(proj_list)):
ki = proj_angles[proj_list[i][0]][3]
if ki >= symbaselen: continue
mi = index[ki]
phiM, thetaM, psiM, s2xM, s2yM = get_params_proj(imgdata[mi])
grp_imgdata = []
for j in xrange(img_per_grp):
mj = index[proj_angles[proj_list[i][j]][3]]
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[mj])
alpha, sx, sy, mirror = params_3D_2D_NEW(
phi, theta, psi, s2x, s2y, mirror_list[i][j])
if thetaM <= 90:
if mirror == 0:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, phiM - phi, 0.0, 0.0, 1.0)
else:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, 180 - (phiM - phi), 0.0,
0.0, 1.0)
else:
if mirror == 0:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, -(phiM - phi), 0.0, 0.0,
1.0)
else:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, -(180 - (phiM - phi)), 0.0,
0.0, 1.0)
set_params2D(imgdata[mj], [alpha, sx, sy, mirror, 1.0])
grp_imgdata.append(imgdata[mj])
#print grp_imgdata[j].get_xsize(), imgdata[mj].get_xsize()
if not options.no_norm:
#print grp_imgdata[j].get_xsize()
mask = model_circle(nx / 2 - 2, nx, nx)
for k in xrange(img_per_grp):
ave, std, minn, maxx = Util.infomask(
grp_imgdata[k], mask, False)
grp_imgdata[k] -= ave
grp_imgdata[k] /= std
del mask
if options.fl > 0.0:
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2 * nx
ny2 = 2 * ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d(
fft(
filt_tanl(
filt_ctf(
fft(
pad(grp_imgdata[k], nx2, ny2,
1, 0.0)),
grp_imgdata[k].get_attr("ctf"),
binary=1), options.fl,
options.aa)), nx, ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
for k in xrange(img_per_grp):
grp_imgdata[k] = filt_tanl(grp_imgdata[k],
options.fl, options.aa)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
from utilities import pad, read_text_file
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2 * nx
ny2 = 2 * ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d(
fft(
filt_ctf(fft(
pad(grp_imgdata[k], nx2, ny2, 1, 0.0)),
grp_imgdata[k].get_attr("ctf"),
binary=1)), nx, ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
'''
if i < 10 and myid == main_node:
for k in xrange(10):
grp_imgdata[k].write_image("grp%03d.hdf"%i, k)
'''
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("pp.hdf", pp)
"""
ave, grp_imgdata = prepare_2d_forPCA(grp_imgdata)
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("qq.hdf", pp)
"""
var = model_blank(nx, ny)
for q in grp_imgdata:
Util.add_img2(var, q)
Util.mul_scalar(var, 1.0 / (len(grp_imgdata) - 1))
# Switch to std dev
var = square_root(threshold(var))
#if options.CTF: ave, var = avgvar_ctf(grp_imgdata, mode="a")
#else: ave, var = avgvar(grp_imgdata, mode="a")
"""
if myid == main_node:
ave.write_image("avgv.hdf",i)
var.write_image("varv.hdf",i)
"""
set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])
aveList.append(ave)
varList.append(var)
if options.VERBOSE:
print("%5.2f%% done on processor %d" %
(i * 100.0 / len(proj_list), myid))
if nvec > 0:
eig = pca(input_stacks=grp_imgdata,
subavg="",
mask_radius=radiuspca,
nvec=nvec,
incore=True,
shuffle=False,
genbuf=True)
for k in xrange(nvec):
set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
eigList[k].append(eig[k])
"""
if myid == 0 and i == 0:
for k in xrange(nvec):
eig[k].write_image("eig.hdf", k)
"""
del imgdata
# To this point, all averages, variances, and eigenvectors are computed
if options.ave2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node:
for im in xrange(len(aveList)):
aveList[im].write_image(
os.path.join(options.output_dir,
options.ave2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i,
SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im + i + 70000)
"""
nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('pix_err', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
"""
tmpvol = fpol(ave, Tracker["nx"],
Tracker["nx"], 1)
tmpvol.write_image(
os.path.join(options.output_dir,
options.ave2D), km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in xrange(len(aveList)):
send_EMData(aveList[im], main_node, im + myid + 70000)
"""
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
members = aveList[im].get_attr('pix_err')
mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
except:
mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
"""
if options.ave3D:
from fundamentals import fpol
if options.VERBOSE:
print("Reconstructing 3D average volume")
ave3D = recons3d_4nn_MPI(myid,
aveList,
symmetry=options.sym,
npad=options.npad)
bcast_EMData_to_all(ave3D, myid)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
ave3D = fpol(ave3D, Tracker["nx"], Tracker["nx"],
Tracker["nx"])
ave3D.write_image(
os.path.join(options.output_dir, options.ave3D))
msg = ("%-70s: %s\n" % (
"Writing to the disk volume reconstructed from averages as",
options.ave3D))
log_main.add(msg)
print(line, msg)
del ave, var, proj_list, stack, phi, theta, psi, s2x, s2y, alpha, sx, sy, mirror, aveList
if nvec > 0:
for k in xrange(nvec):
if options.VERBOSE:
print("Reconstruction eigenvolumes", k)
cont = True
ITER = 0
mask2d = model_circle(radiuspca, nx, nx)
while cont:
#print "On node %d, iteration %d"%(myid, ITER)
eig3D = recons3d_4nn_MPI(myid,
eigList[k],
symmetry=options.sym,
npad=options.npad)
bcast_EMData_to_all(eig3D, myid, main_node)
if options.fl > 0.0:
eig3D = filt_tanl(eig3D, options.fl, options.aa)
if myid == main_node:
eig3D.write_image(
os.path.join(options.outpout_dir,
"eig3d_%03d.hdf" % (k, ITER)))
Util.mul_img(eig3D,
model_circle(radiuspca, nx, nx, nx))
eig3Df, kb = prep_vol(eig3D)
del eig3D
cont = False
icont = 0
for l in xrange(len(eigList[k])):
phi, theta, psi, s2x, s2y = get_params_proj(
eigList[k][l])
proj = prgs(eig3Df, kb,
[phi, theta, psi, s2x, s2y])
cl = ccc(proj, eigList[k][l], mask2d)
if cl < 0.0:
icont += 1
cont = True
eigList[k][l] *= -1.0
u = int(cont)
u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node,
MPI_COMM_WORLD)
icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM,
main_node, MPI_COMM_WORLD)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S",
localtime()) + " =>"
u = int(u[0])
msg = (" Eigenvector: ", k, " number changed ",
int(icont[0]))
log_main.add(msg)
print(line, msg)
else:
u = 0
u = bcast_number_to_all(u, main_node)
cont = bool(u)
ITER += 1
del eig3Df, kb
mpi_barrier(MPI_COMM_WORLD)
del eigList, mask2d
if options.ave3D: del ave3D
if options.var2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node:
for im in xrange(len(varList)):
tmpvol = fpol(varList[im], Tracker["nx"],
Tracker["nx"], 1)
tmpvol.write_image(
os.path.join(options.output_dir,
options.var2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i,
SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im + i + 70000)
tmpvol = fpol(ave, Tracker["nx"],
Tracker["nx"], 1)
tmpvol.write_image(
os.path.join(options.output_dir,
options.var2D, km))
km += 1
else:
mpi_send(len(varList), 1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in xrange(len(varList)):
send_EMData(varList[im], main_node, im + myid +
70000) # What with the attributes??
mpi_barrier(MPI_COMM_WORLD)
if options.var3D:
if myid == main_node and options.VERBOSE:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("Reconstructing 3D variability volume")
log_main.add(msg)
print(line, msg)
t6 = time()
# radiusvar = options.radius
# if( radiusvar < 0 ): radiusvar = nx//2 -3
res = recons3d_4nn_MPI(myid,
varList,
symmetry=options.sym,
npad=options.npad)
#res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
if myid == main_node:
from fundamentals import fpol
res = fpol(res, Tracker["nx"], Tracker["nx"], Tracker["nx"])
res.write_image(os.path.join(options.output_dir,
options.var3D))
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("%-70s: %.2f\n" %
("Reconstructing 3D variability took [s]", time() - t6))
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
print("Reconstruction took: %.2f [min]" %
((time() - t6) / 60))
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("%-70s: %.2f\n" %
("Total time for these computations [s]", time() - t0))
print(line, msg)
log_main.add(msg)
if options.VERBOSE:
print("Total time for these computations: %.2f [min]" %
((time() - t0) / 60))
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("sx3dvariability")
print(line, msg)
log_main.add(msg)
from mpi import mpi_finalize
mpi_finalize()
if RUNNING_UNDER_MPI:
global_def.MPI = False
global_def.BATCH = False
def main():
global debug
progname = os.path.basename(sys.argv[0])
usage = """prog [options] <path or db> ...
Various utilities related to BDB databases.
examples :
e2bdb.py -c Is perhaps the most critical function, as it cleans up the database cache. See the Wiki for more.
e2bdb.py <path> -s will list the contents of the database in a directory in bdb: notation
e2bdb.py <path> -l Will give useful summary info about stacks in a directory
e2bdb.py <database> --dump Gives a mechanism to dump all of the metadata in a database, even if the database contains no images
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--cleanup",
"-c",
action="store_true",
default=False,
help=
"This option will clean up the database cache so files can safely be moved or accessed on another computer via NFS."
)
parser.add_argument(
"--force",
"-F",
action="store_true",
default=False,
help=
"This will force an action that would normally fail due to failed checks."
)
parser.add_argument(
"--delete",
action="store_true",
default=False,
help="This will delete (or at least empty) the named database(s)")
parser.add_argument("--all",
"-a",
action="store_true",
help="List per-particle info",
default=False)
parser.add_argument("--long",
"-l",
action="store_true",
help="Long listing",
default=False)
parser.add_argument("--short",
"-s",
action="store_true",
help="Dense listing of names only",
default=False)
parser.add_argument(
"--filt",
type=str,
help="Only include dictionary names containing the specified string",
default=None)
parser.add_argument(
"--filtexclude",
type=str,
help="Exclude dictionary names containing the specified string",
default=None)
parser.add_argument(
"--match",
type=str,
help=
"Only include dictionaries matching the provided Python regular expression",
default=None)
parser.add_argument(
"--exclude",
type=str,
help="The name of a database containing a list of exclusion keys",
default=None)
parser.add_argument(
"--dump",
"-D",
action="store_true",
help=
"List contents of an entire database, eg 'e2bdb.py -D refine_01#register",
default=False)
parser.add_argument(
"--smalldump",
action="store_true",
help=
"Lists contents of an entire database, but only list 2 items per dictionary to better see headers",
default=False)
parser.add_argument(
"--extractplots",
action="store_true",
help=
"If a database contains sets of plots, such as bdb:refine_xx#convergence.results, this will extract the plots as text files."
)
parser.add_argument(
"--check",
action="store_true",
help=
"Check for self-consistency and errors in the structure of specified databases",
default=False)
parser.add_argument("--nocache",
action="store_true",
help="Don't use the database cache for this operation",
default=False)
parser.add_argument(
"--merge",
action="store_true",
help=
"This will merge the contents of BDB 2-N into BDB 1 (including BDB 1's contents)",
default=False)
parser.add_argument(
"--makevstack",
type=str,
help=
"Creates a 'virtual' BDB stack with its own metadata, but the binary data taken from the (filtered) list of stacks",
default=None)
parser.add_argument(
"--appendvstack",
type=str,
help=
"Appends to/creates a 'virtual' BDB stack with its own metadata, but the binary data taken from the (filtered) list of stacks",
default=None)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument(
"--list",
type=str,
help=
"Specify the name of a file with a list of images to use in creation of virtual stacks. Please see source for details.",
default=None)
parser.add_argument(
"--exlist",
type=str,
help=
"Specify the name of a file with a list of images to exclude in creation of virtual stacks. Please see source for details.",
default=None)
parser.add_argument(
"--restore",
type=str,
help=
"Write changes in the derived virtual stack back to the original stack",
default=None)
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
parser.add_argument(
"--checkctf",
action="store_true",
help=
"Verfies that all images in the file contain CTF information, and gives some basic statistics",
default=False)
parser.add_argument(
"--step",
type=str,
default="0,1",
help=
"Specify <init>,<step>[,<max>]. Processes only a subset of the input data. For example, 0,2 would process only the even numbered particles"
)
(options, args) = parser.parse_args()
if options.nocache: EMAN2db.BDB_CACHE_DISABLE = True
if options.cleanup:
db_cleanup(options.force)
sys.exit(0)
try:
options.step = int(options.step.split(",")[0]), int(
options.step.split(",")[1]), int(
options.step.split(",")[2]) # convert strings to tuple
except:
try:
options.step = int(options.step.split(",")[0]), int(
options.step.split(",")[1])
except:
print("Invalid --step specification")
sys.exit(1)
if options.all: options.long = 1
if len(args) == 0: args.append("bdb:.")
logid = 0
if options.makevstack:
logid = E2init(sys.argv)
vstack = db_open_dict(options.makevstack)
vstackn = 0
elif options.appendvstack:
logid = E2init(sys.argv)
vstack = db_open_dict(options.appendvstack)
vstackn = len(vstack)
else:
vstack = None
if options.merge:
print("WARNING: Merge mode\nCombining contents of: ",
", ".join(args[1:]))
print("into ", args[0])
if input("Proceed (y/n) :").lower() != "y":
print("Aborting")
sys.exit(1)
for i, path in enumerate(args):
if path.lower()[:4] == "bdb:" and not "#" in path:
path = "bdb:.#" + path[4:]
if path.lower()[:4] != "bdb:": path = "bdb:" + path
if i == 0:
outdb = db_open_dict(path)
continue
indb = db_open_dict(path, True)
for k in list(indb.keys()):
outdb[k] = indb[k]
print("Merging complete")
sys.exit(0)
for path in args:
if path.lower()[:4] == "bdb:" and not "#" in path:
uu = os.path.split(path)
if (uu[0] == ''): path = "bdb:.#" + path[4:]
else: path = uu[0] + "#" + uu[1]
if path.lower()[:4] != "bdb:": path = "bdb:" + path
if '#' in path:
if len(args) > 1: print("\n", path, ":")
path, dbs = path.rsplit("#", 1)
path += "#"
dbs = [dbs]
else:
if not '#' in path and path[-1] != '/': path += '#'
if len(args) > 1: print("\n", path[:-1], ":")
dbs = db_list_dicts(path)
dbs.sort()
if options.filt:
dbs = [db for db in dbs if options.filt in db]
if options.filtexclude:
dbs = [db for db in dbs if options.filtexclude not in db]
if options.match != None:
dbs = [db for db in dbs if re.match(options.match, db)]
if options.list:
if options.makevstack == None and options.appendvstack == None:
print(
"ERROR, this option is used for virtual stack creation, please add makevstack or appendvstack options, and restart"
)
sys.exit(1)
vdata = open(options.list, 'r').readlines()
n = len(vdata[0].split())
slist = []
for line in vdata:
line = line.split()
for i in range(n):
val = int(line[i])
slist.append(val)
del n, val, vdata
if options.exlist:
if options.makevstack == None:
print(
"ERROR, this option is used for virtual stack creation, please add makevstack or appendvstack options, and restart"
)
sys.exit(1)
vdata = open(options.exlist, 'r').readlines()
n = len(vdata[0].split())
slist = []
for line in vdata:
line = line.split()
for i in range(n):
val = int(line[i])
slist.append(val)
n = EMUtil.get_image_count(args[0])
good = set(range(n)) - set(slist)
slist = [i for i in good]
slist.sort()
del n, val, vdata, good
if options.makevstack != None or options.appendvstack != None:
vspath = os.path.realpath(vstack.path) + "/"
if options.verbose > 2: print("vspath: ", vspath)
for db in dbs:
dct, keys = db_open_dict(path + db, ro=True, with_keys=True)
if dct == vstack: continue
if len(options.step) == 2:
if keys == None:
vals = list(
range(options.step[0], len(dct), options.step[1]))
else:
vals = keys[options.step[0]::options.step[
1]] # we apply --step even if we have a list of keys
else:
if keys == None:
vals = list(
range(options.step[0], options.step[2],
options.step[1]))
else:
vals = keys[options.step[0]:options.step[2]:options.step[
1]] # we apply --step even if we have a list of keys
if options.list != None or options.exlist != None: vals = slist
for n in vals:
try:
d = dct.get(n, nodata=1).get_attr_dict()
except:
traceback.print_exc()
print("---\nerror reading ", db, n)
continue
# This block converts an absolute path to the actual data to a relative path
try:
dpath = os.path.realpath(dct.get_data_path(n))
if options.verbose > 2: print("dpath: ", dpath)
if os.name == 'nt':
vspath = vspath.replace("\\", '/')
dpath = dpath.replace('\\', '/')
rpath = makerelpath(vspath, dpath)
if options.verbose > 2: print("rpath: ", rpath)
except:
print("error with data_path ", db, n)
continue
d["data_path"] = rpath
d["data_n"] = n
d["data_source"] = path + db
if d["data_path"] == None:
print("error with data_path ", db, n)
continue
vstack[vstackn] = d
vstackn += 1
if vstackn % 100 == 0:
try:
print("\r ", vstackn, " ", end=' ')
sys.stdout.flush()
except:
pass
print("\r ", vstackn, " ")
dct.close()
try:
maxname = max([len(s) for s in dbs])
except:
print("Error reading ", path)
if options.restore:
nima = EMUtil.get_image_count(options.restore)
IB = db_open_dict(options.restore)
source_old = None
if len(options.step) == 3: nima = min(options.step[2], nima)
for i in range(options.step[0], nima, options.step[1]):
source = IB.get_header(i)
source_path = source["source_path"]
ID = source["source_n"]
if (source_old != source_path):
if (source_old != None): DB.close()
DB = db_open_dict(source_path, ro=True)
source_old = source_path
target = DB.get_header(ID)
try:
source["data_path"] = target["data_path"]
source["data_n"] = target["data_n"]
source["source_path"] = target["source_path"]
source["source_n"] = target["source_n"]
except:
# top level does not have data_path
del source['data_path']
del source['data_n']
source["source_path"] = target["source_path"]
source["source_n"] = target["source_n"]
DB.set_header(ID, source)
DB.close()
if options.extractplots:
for db in dbs:
print("#### Extracting plots from ", db)
dct = db_open_dict(path + db, ro=True)
#### Dump
keys = list(dct.keys())
keys.sort()
for k in keys:
v = dct[k]
try:
ns = [len(i) for i in v]
fsp = db + "-" + k + ".txt"
print("%s (%d columns)" % (fsp, len(ns)))
out = open(fsp, "w")
for i in range(ns[0]):
for j in range(len(ns)):
out.write(str(v[j][i]))
if j < len(ns) - 1: out.write("\t")
out.write("\n")
out.close()
except:
continue
dct.close()
if options.smalldump:
for db in dbs:
print("##### ", db)
dct = db_open_dict(path + db, ro=True)
#### Dump
keys = list(dct.keys())
keys.sort()
if len(options.step) == 3: keys = keys[:options.step[2]]
for k in keys[options.step[0]::options.step[1]]:
v = dct[k]
print("%s : " % k, end=' ')
if isinstance(v, list) or isinstance(v, tuple):
for i in v:
print("\n\t%s" % str(i), end=' ')
print("")
elif isinstance(v, dict):
ks2 = list(v.keys())
ks2.sort()
kc = 0
for i in ks2:
if kc >= 2:
print("...")
break
print("\n\t%s : %s" % (i, v[i]), end=' ')
kc += 1
print("")
else:
print(str(v))
dct.close()
if options.checkctf:
for db in dbs:
print("##### CTF -> ", db)
dct = db_open_dict(path + db, ro=True)
keys = list(dct.keys())
if len(options.step) == 3: keys = keys[:options.step[2]]
defocus = set()
for k in keys[options.step[0]::options.step[1]]:
v = dct.get_header(k)
try:
ctf = v["ctf"]
except:
if k != "maxrec": print("CTF missing on image %s" % k)
continue
defocus.add(ctf.defocus)
defocus = list(defocus)
print("Defocuses found: ", end=' ')
for i in defocus:
print("%1.3f, " % i, end=' ')
print("\n\nRange: %1.3f - %1.3f (%d unique values)" %
(min(defocus), max(defocus), len(defocus)))
if options.dump:
for db in dbs:
print("##### ", db)
dct = db_open_dict(path + db, ro=True)
#### Dump
keys = list(dct.keys())
if len(options.step) == 3: keys = keys[:options.step[2]]
keys.sort()
for k in keys[options.step[0]::options.step[1]]:
v = dct[k]
print("%s : " % k, end=' ')
if isinstance(v, list) or isinstance(v, tuple):
for i in v:
print("\n\t%s" % str(i), end=' ')
print("")
elif isinstance(v, dict):
ks2 = list(v.keys())
ks2.sort()
for i in ks2:
print("\n\t%s : %s" % (i, v[i]), end=' ')
print("")
else:
print(str(v))
dct.close()
# long listing, one db per line
elif options.long:
width = maxname + 3
fmt = "%%-%ds %%-07d %%14s %%s" % width
fmt2 = "%%-%ds (not an image stack)" % width
total = [0, 0]
for db in dbs:
dct = db_open_dict(path + db, True)
### Info on all particles
if options.all:
mx = len(dct)
if len(options.step) == 3: mx = min(mx, options.step[2])
for i in range(options.step[0], mx, options.step[1]):
try:
im = dct[i]
if im == None: raise Exception
except:
continue
print(
"%d. %d x %d x %d\tA/pix=%1.2f\tM=%1.4f\tS=%1.4f\tSk=%1.4f"
% (i, im["nx"], im["ny"], im["nz"], im["apix_x"],
im["mean"], im["sigma"], im["skewness"]),
end=' ')
try:
print("\t%s" % str(im["model_id"]))
except:
pass
try:
print("\tdf=%1.3f\tB=%1.1f" %
(im["ctf"].defocus, im["ctf"].bfactor))
except:
print(" ")
first = EMData()
try:
first.read_image(path + db, 0, True)
size = first.get_xsize() * first.get_ysize(
) * first.get_zsize() * len(dct) * 4
total[0] += len(dct)
total[1] += size
print(fmt % (db, len(dct), "%dx%dx%d apix: %1.2f" %
(first.get_xsize(), first.get_ysize(),
first.get_zsize(), first["apix_x"]),
human_size(size)),
end=' ')
except:
print(fmt2 % db)
try:
print("\tdf: %1.3f\tB: %1.0f" %
(first["ctf"].defocus, first["ctf"].bfactor))
except:
print("")
dct.close()
print(fmt % ("TOTAL", total[0], "", human_size(total[1])))
elif options.check:
from pickle import loads
for db in dbs:
dct = db_open_dict(path + db, ro=True)
dct.realopen()
keys = list(dct.bdb.keys())
allkvp = {}
for k in keys:
s1, s2 = k.split("\x80", 1) # start of a pickled string.
s2 = loads("\x80" + s2) # the pickled part
if len(
s1
) > 0: # If anything unpickled, then it is an axbxc prefix identifying the location of a binary
st = allkvp.setdefault(
s1, set()) # set of all positions seen so far
v = loads(dct.bdb.get(k)) # position in binary file
if v in st:
print(
"Error: value %d seen multiple times in %s (%s,%s)"
% (v, db, s1, s2))
st.add(v)
print("%s : " % db, end=' ')
for i in list(allkvp.keys()):
if options.verbose > 0:
print("%s %d/%d\t" %
(i, len(allkvp[i]), int(max(allkvp[i])) + 1),
end=' ')
if len(allkvp[i]) != int(max(allkvp[i]) + 1):
print(
"\nMismatch found in %s. Could be normal if file has been rewritten multiple times, but is unusual"
% db)
if options.verbose > 0: print("")
else: print(" done")
dct.close()
elif options.short:
for db in dbs:
print(path + db, end=' ')
print(" ")
elif not options.makevstack and not options.appendvstack:
# Nicely formatted 'ls' style display
cols = int(floor(old_div(80.0, (maxname + 3))))
width = old_div(80, cols)
rows = int(ceil(old_div(float(len(dbs)), cols)))
fmt = "%%-%ds" % width
for r in range(rows):
for c in range(cols):
try:
print(fmt % dbs[r + c * rows], end=' ')
except:
pass
print(" ")
if options.delete:
if not options.force:
print("You are requesting to delete the following databases:")
for db in dbs:
print(db, " ", end=' ')
if input("\nAre you sure (y/n) ? ")[0].lower() != 'y':
print("Aborted")
sys.exit(1)
for db in dbs:
db_remove_dict(path + db)
if logid: E2end(logid)
from math import *
from os import *
import EMAN2fsc
from EMAN2db import db_open_dict
from EMAN2 import *
if len(sys.argv)>1 : dirs=sys.argv[1:]
else : dirs=[i for i in os.listdir(".") if "refine_"==i[:7] and len(i)==9]
# iterates over all refine directories
for d in dirs:
print "Updating ",d
db2=db_open_dict("bdb:%s#register"%d,True)
try :
initm=db2["cmd_dict"]["model"]
print " Initial model ",initm
except:
print "No initial model recorded"
initm=None
db=db_open_dict("bdb:%s#convergence.results"%d)
if d[-5:]=="_even" :
# For e2refine_evenodd results
db2=db_open_dict("bdb:%s#convergence.results"%d[:-5])
tdflist = [i for i in db_list_dicts("bdb:%s"%d) if "threed_filt" in i]
for i in tdflist:
dictname = re.sub('^threed_filt_','conv_even_odd_', i)
def main():
def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
# the final ali2d parameters already combine shifts operation first and rotation operation second for parameters converted from 3D
if mirror:
m = 1
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 540.0-psi, 0, 0, 1.0)
else:
m = 0
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 360.0-psi, 0, 0, 1.0)
return alpha, sx, sy, m
progname = os.path.basename(sys.argv[0])
usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl= --aa= --sym=symmetry --CTF"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--output_dir", type="string" , default="./", help="Output directory")
parser.add_option("--ave2D", type="string" , default=False, help="Write to the disk a stack of 2D averages")
parser.add_option("--var2D", type="string" , default=False, help="Write to the disk a stack of 2D variances")
parser.add_option("--ave3D", type="string" , default=False, help="Write to the disk reconstructed 3D average")
parser.add_option("--var3D", type="string" , default=False, help="Compute 3D variability (time consuming!)")
parser.add_option("--img_per_grp", type="int" , default=100, help="Number of neighbouring projections.(Default is 100)")
parser.add_option("--no_norm", action="store_true", default=False, help="Do not use normalization.(Default is to apply normalization)")
#parser.add_option("--radius", type="int" , default=-1 , help="radius for 3D variability" )
parser.add_option("--npad", type="int" , default=2 , help="Number of time to pad the original images.(Default is 2 times padding)")
parser.add_option("--sym" , type="string" , default="c1", help="Symmetry. (Default is no symmetry)")
parser.add_option("--fl", type="float" , default=0.0, help="Low pass filter cutoff in absolute frequency (0.0 - 0.5) and is applied to decimated images. (Default - no filtration)")
parser.add_option("--aa", type="float" , default=0.02 , help="Fall off of the filter. Use default value if user has no clue about falloff (Default value is 0.02)")
parser.add_option("--CTF", action="store_true", default=False, help="Use CFT correction.(Default is no CTF correction)")
#parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
#parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" )
#parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" )
#parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
#parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
parser.add_option("--VAR" , action="store_true", default=False, help="Stack of input consists of 2D variances (Default False)")
parser.add_option("--decimate", type ="float", default=0.25, help="Image decimate rate, a number less than 1. (Default is 0.25)")
parser.add_option("--window", type ="int", default=0, help="Target image size relative to original image size. (Default value is zero.)")
#parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)")
#parser.add_option("--nvec", type="int" , default=0 , help="Number of eigenvectors, (Default = 0 meaning no PCA calculated)")
parser.add_option("--symmetrize", action="store_true", default=False, help="Prepare input stack for handling symmetry (Default False)")
parser.add_option("--overhead", type ="float", default=0.5, help="python overhead per CPU.")
(options,args) = parser.parse_args()
#####
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD
from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
#from mpi import *
from applications import MPI_start_end
from reconstruction import recons3d_em, recons3d_em_MPI
from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
from utilities import print_begin_msg, print_end_msg, print_msg
from utilities import read_text_row, get_image, get_im, wrap_mpi_send, wrap_mpi_recv
from utilities import bcast_EMData_to_all, bcast_number_to_all
from utilities import get_symt
# This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015
from EMAN2db import db_open_dict
# Set up global variables related to bdb cache
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
# Set up global variables related to ERROR function
global_def.BATCH = True
# detect if program is running under MPI
RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ
if RUNNING_UNDER_MPI: global_def.MPI = True
if options.output_dir =="./": current_output_dir = os.path.abspath(options.output_dir)
else: current_output_dir = options.output_dir
if options.symmetrize :
if RUNNING_UNDER_MPI:
try:
sys.argv = mpi_init(len(sys.argv), sys.argv)
try:
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
if( number_of_proc > 1 ):
ERROR("Cannot use more than one CPU for symmetry preparation","sx3dvariability",1)
except:
pass
except:
pass
if not os.path.exists(current_output_dir): os.mkdir(current_output_dir)
# Input
#instack = "Clean_NORM_CTF_start_wparams.hdf"
#instack = "bdb:data"
from logger import Logger,BaseLogger_Files
if os.path.exists(os.path.join(current_output_dir, "log.txt")): os.remove(os.path.join(current_output_dir, "log.txt"))
log_main=Logger(BaseLogger_Files())
log_main.prefix = os.path.join(current_output_dir, "./")
instack = args[0]
sym = options.sym.lower()
if( sym == "c1" ):
ERROR("There is no need to symmetrize stack for C1 symmetry","sx3dvariability",1)
line =""
for a in sys.argv:
line +=" "+a
log_main.add(line)
if(instack[:4] !="bdb:"):
#if output_dir =="./": stack = "bdb:data"
stack = "bdb:"+current_output_dir+"/data"
delete_bdb(stack)
junk = cmdexecute("sxcpy.py "+instack+" "+stack)
else: stack = instack
qt = EMUtil.get_all_attributes(stack,'xform.projection')
na = len(qt)
ts = get_symt(sym)
ks = len(ts)
angsa = [None]*na
for k in range(ks):
#Qfile = "Q%1d"%k
#if options.output_dir!="./": Qfile = os.path.join(options.output_dir,"Q%1d"%k)
Qfile = os.path.join(current_output_dir, "Q%1d"%k)
#delete_bdb("bdb:Q%1d"%k)
delete_bdb("bdb:"+Qfile)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:"+Qfile)
#DB = db_open_dict("bdb:Q%1d"%k)
DB = db_open_dict("bdb:"+Qfile)
for i in range(na):
ut = qt[i]*ts[k]
DB.set_attr(i, "xform.projection", ut)
#bt = ut.get_params("spider")
#angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
#write_text_row(angsa, 'ptsma%1d.txt'%k)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
#junk = cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k))
DB.close()
#if options.output_dir =="./": delete_bdb("bdb:sdata")
delete_bdb("bdb:" + current_output_dir + "/"+"sdata")
#junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
sdata = "bdb:"+current_output_dir+"/"+"sdata"
print(sdata)
junk = cmdexecute("e2bdb.py " + current_output_dir +" --makevstack="+sdata +" --filt=Q")
#junk = cmdexecute("ls EMAN2DB/sdata*")
#a = get_im("bdb:sdata")
a = get_im(sdata)
a.set_attr("variabilitysymmetry",sym)
#a.write_image("bdb:sdata")
a.write_image(sdata)
else:
from fundamentals import window2d
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
shared_comm = mpi_comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL)
myid_on_node = mpi_comm_rank(shared_comm)
no_of_processes_per_group = mpi_comm_size(shared_comm)
masters_from_groups_vs_everything_else_comm = mpi_comm_split(MPI_COMM_WORLD, main_node == myid_on_node, myid_on_node)
color, no_of_groups, balanced_processor_load_on_nodes = get_colors_and_subsets(main_node, MPI_COMM_WORLD, myid, \
shared_comm, myid_on_node, masters_from_groups_vs_everything_else_comm)
overhead_loading = options.overhead*number_of_proc
#memory_per_node = options.memory_per_node
#if memory_per_node == -1.: memory_per_node = 2.*no_of_processes_per_group
keepgoing = 1
current_window = options.window
current_decimate = options.decimate
if len(args) == 1: stack = args[0]
else:
print(( "usage: " + usage))
print(( "Please run '" + progname + " -h' for detailed options"))
return 1
t0 = time()
# obsolete flags
options.MPI = True
#options.nvec = 0
options.radiuspca = -1
options.iter = 40
options.abs = 0.0
options.squ = 0.0
if options.fl > 0.0 and options.aa == 0.0:
ERROR("Fall off has to be given for the low-pass filter", "sx3dvariability", 1, myid)
#if options.VAR and options.SND:
# ERROR("Only one of var and SND can be set!", "sx3dvariability", myid)
if options.VAR and (options.ave2D or options.ave3D or options.var2D):
ERROR("When VAR is set, the program cannot output ave2D, ave3D or var2D", "sx3dvariability", 1, myid)
#if options.SND and (options.ave2D or options.ave3D):
# ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid)
#if options.nvec > 0 :
# ERROR("PCA option not implemented", "sx3dvariability", 1, myid)
#if options.nvec > 0 and options.ave3D == None:
# ERROR("When doing PCA analysis, one must set ave3D", "sx3dvariability", 1, myid)
if current_decimate>1.0 or current_decimate<0.0:
ERROR("Decimate rate should be a value between 0.0 and 1.0", "sx3dvariability", 1, myid)
if current_window < 0.0:
ERROR("Target window size should be always larger than zero", "sx3dvariability", 1, myid)
if myid == main_node:
img = get_image(stack, 0)
nx = img.get_xsize()
ny = img.get_ysize()
if(min(nx, ny) < current_window): keepgoing = 0
keepgoing = bcast_number_to_all(keepgoing, main_node, MPI_COMM_WORLD)
if keepgoing == 0: ERROR("The target window size cannot be larger than the size of decimated image", "sx3dvariability", 1, myid)
import string
options.sym = options.sym.lower()
# if global_def.CACHE_DISABLE:
# from utilities import disable_bdb_cache
# disable_bdb_cache()
# global_def.BATCH = True
if myid == main_node:
if not os.path.exists(current_output_dir): os.mkdir(current_output_dir)# Never delete output_dir in the program!
img_per_grp = options.img_per_grp
#nvec = options.nvec
radiuspca = options.radiuspca
from logger import Logger,BaseLogger_Files
#if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt"))
log_main=Logger(BaseLogger_Files())
log_main.prefix = os.path.join(current_output_dir, "./")
if myid == main_node:
line = ""
for a in sys.argv: line +=" "+a
log_main.add(line)
log_main.add("-------->>>Settings given by all options<<<-------")
log_main.add("Symmetry : %s"%options.sym)
log_main.add("Input stack : %s"%stack)
log_main.add("Output_dir : %s"%current_output_dir)
if options.ave3D: log_main.add("Ave3d : %s"%options.ave3D)
if options.var3D: log_main.add("Var3d : %s"%options.var3D)
if options.ave2D: log_main.add("Ave2D : %s"%options.ave2D)
if options.var2D: log_main.add("Var2D : %s"%options.var2D)
if options.VAR: log_main.add("VAR : True")
else: log_main.add("VAR : False")
if options.CTF: log_main.add("CTF correction : True ")
else: log_main.add("CTF correction : False ")
log_main.add("Image per group : %5d"%options.img_per_grp)
log_main.add("Image decimate rate : %4.3f"%current_decimate)
log_main.add("Low pass filter : %4.3f"%options.fl)
current_fl = options.fl
if current_fl == 0.0: current_fl = 0.5
log_main.add("Current low pass filter is equivalent to cutoff frequency %4.3f for original image size"%round((current_fl*current_decimate),3))
log_main.add("Window size : %5d "%current_window)
log_main.add("sx3dvariability begins")
symbaselen = 0
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
nnxo = nx
nnyo = ny
if options.sym != "c1" :
imgdata = get_im(stack)
try:
i = imgdata.get_attr("variabilitysymmetry").lower()
if(i != options.sym):
ERROR("The symmetry provided does not agree with the symmetry of the input stack", "sx3dvariability", 1, myid)
except:
ERROR("Input stack is not prepared for symmetry, please follow instructions", "sx3dvariability", 1, myid)
from utilities import get_symt
i = len(get_symt(options.sym))
if((nima/i)*i != nima):
ERROR("The length of the input stack is incorrect for symmetry processing", "sx3dvariability", 1, myid)
symbaselen = nima/i
else: symbaselen = nima
else:
nima = 0
nx = 0
ny = 0
nnxo = 0
nnyo = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
nnxo = bcast_number_to_all(nnxo)
nnyo = bcast_number_to_all(nnyo)
if current_window > max(nx, ny):
ERROR("Window size is larger than the original image size", "sx3dvariability", 1)
if current_decimate == 1.:
if current_window !=0:
nx = current_window
ny = current_window
else:
if current_window == 0:
nx = int(nx*current_decimate+0.5)
ny = int(ny*current_decimate+0.5)
else:
nx = int(current_window*current_decimate+0.5)
ny = nx
symbaselen = bcast_number_to_all(symbaselen)
# check FFT prime number
from fundamentals import smallprime
is_fft_friendly = (nx == smallprime(nx))
if not is_fft_friendly:
if myid == main_node:
log_main.add("The target image size is not a product of small prime numbers")
log_main.add("Program adjusts the input settings!")
### two cases
if current_decimate == 1.:
nx = smallprime(nx)
ny = nx
current_window = nx # update
if myid == main_node:
log_main.add("The window size is updated to %d."%current_window)
else:
if current_window == 0:
nx = smallprime(int(nx*current_decimate+0.5))
current_decimate = float(nx)/nnxo
ny = nx
if (myid == main_node):
log_main.add("The decimate rate is updated to %f."%current_decimate)
else:
nx = smallprime(int(current_window*current_decimate+0.5))
ny = nx
current_window = int(nx/current_decimate+0.5)
if (myid == main_node):
log_main.add("The window size is updated to %d."%current_window)
if myid == main_node:
log_main.add("The target image size is %d"%nx)
if radiuspca == -1: radiuspca = nx/2-2
if myid == main_node: log_main.add("%-70s: %d\n"%("Number of projection", nima))
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
"""
if options.SND:
from projection import prep_vol, prgs
from statistics import im_diff
from utilities import get_im, model_circle, get_params_proj, set_params_proj
from utilities import get_ctf, generate_ctf
from filter import filt_ctf
imgdata = EMData.read_images(stack, range(img_begin, img_end))
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
bcast_EMData_to_all(vol, myid)
volft, kb = prep_vol(vol)
mask = model_circle(nx/2-2, nx, ny)
varList = []
for i in xrange(img_begin, img_end):
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
if options.CTF:
ctf_params = get_ctf(imgdata[i-img_begin])
ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
diff2 = diff*diff
set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
varList.append(diff2)
mpi_barrier(MPI_COMM_WORLD)
"""
if options.VAR: # 2D variance images have no shifts
#varList = EMData.read_images(stack, range(img_begin, img_end))
from EMAN2 import Region
for index_of_particle in range(img_begin,img_end):
image = get_im(stack, index_of_proj)
if current_window > 0: varList.append(fdecimate(window2d(image,current_window,current_window), nx,ny))
else: varList.append(fdecimate(image, nx,ny))
else:
from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
from utilities import model_blank, nearest_proj, model_circle, write_text_row, wrap_mpi_gatherv
from applications import pca
from statistics import avgvar, avgvar_ctf, ccc
from filter import filt_tanl
from morphology import threshold, square_root
from projection import project, prep_vol, prgs
from sets import Set
from utilities import wrap_mpi_recv, wrap_mpi_bcast, wrap_mpi_send
import numpy as np
if myid == main_node:
t1 = time()
proj_angles = []
aveList = []
tab = EMUtil.get_all_attributes(stack, 'xform.projection')
for i in range(nima):
t = tab[i].get_params('spider')
phi = t['phi']
theta = t['theta']
psi = t['psi']
x = theta
if x > 90.0: x = 180.0 - x
x = x*10000+psi
proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
t2 = time()
log_main.add( "%-70s: %d\n"%("Number of neighboring projections", img_per_grp))
log_main.add("...... Finding neighboring projections\n")
log_main.add( "Number of images per group: %d"%img_per_grp)
log_main.add( "Now grouping projections")
proj_angles.sort()
proj_angles_list = np.full((nima, 4), 0.0, dtype=np.float32)
for i in range(nima):
proj_angles_list[i][0] = proj_angles[i][1]
proj_angles_list[i][1] = proj_angles[i][2]
proj_angles_list[i][2] = proj_angles[i][3]
proj_angles_list[i][3] = proj_angles[i][4]
else: proj_angles_list = 0
proj_angles_list = wrap_mpi_bcast(proj_angles_list, main_node, MPI_COMM_WORLD)
proj_angles = []
for i in range(nima):
proj_angles.append([proj_angles_list[i][0], proj_angles_list[i][1], proj_angles_list[i][2], int(proj_angles_list[i][3])])
del proj_angles_list
proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp, range(img_begin, img_end))
all_proj = Set()
for im in proj_list:
for jm in im:
all_proj.add(proj_angles[jm][3])
all_proj = list(all_proj)
index = {}
for i in range(len(all_proj)): index[all_proj[i]] = i
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
log_main.add("%-70s: %.2f\n"%("Finding neighboring projections lasted [s]", time()-t2))
log_main.add("%-70s: %d\n"%("Number of groups processed on the main node", len(proj_list)))
log_main.add("Grouping projections took: %12.1f [m]"%((time()-t2)/60.))
log_main.add("Number of groups on main node: ", len(proj_list))
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
log_main.add("...... Calculating the stack of 2D variances \n")
# Memory estimation. There are two memory consumption peaks
# peak 1. Compute ave, var;
# peak 2. Var volume reconstruction;
# proj_params = [0.0]*(nima*5)
aveList = []
varList = []
#if nvec > 0: eigList = [[] for i in range(nvec)]
dnumber = len(all_proj)# all neighborhood set for assigned to myid
pnumber = len(proj_list)*2. + img_per_grp # aveList and varList
tnumber = dnumber+pnumber
vol_size2 = nx**3*4.*8/1.e9
vol_size1 = 2.*nnxo**3*4.*8/1.e9
proj_size = nnxo*nnyo*len(proj_list)*4.*2./1.e9 # both aveList and varList
orig_data_size = nnxo*nnyo*4.*tnumber/1.e9
reduced_data_size = nx*nx*4.*tnumber/1.e9
full_data = np.full((number_of_proc, 2), -1., dtype=np.float16)
full_data[myid] = orig_data_size, reduced_data_size
if myid != main_node: wrap_mpi_send(full_data, main_node, MPI_COMM_WORLD)
if myid == main_node:
for iproc in range(number_of_proc):
if iproc != main_node:
dummy = wrap_mpi_recv(iproc, MPI_COMM_WORLD)
full_data[np.where(dummy>-1)] = dummy[np.where(dummy>-1)]
del dummy
mpi_barrier(MPI_COMM_WORLD)
full_data = wrap_mpi_bcast(full_data, main_node, MPI_COMM_WORLD)
# find the CPU with heaviest load
minindx = np.argsort(full_data, 0)
heavy_load_myid = minindx[-1][1]
total_mem = sum(full_data)
if myid == main_node:
if current_window == 0:
log_main.add("Nx: current image size = %d. Decimated by %f from %d"%(nx, current_decimate, nnxo))
else:
log_main.add("Nx: current image size = %d. Windowed to %d, and decimated by %f from %d"%(nx, current_window, current_decimate, nnxo))
log_main.add("Nproj: number of particle images.")
log_main.add("Navg: number of 2D average images.")
log_main.add("Nvar: number of 2D variance images.")
log_main.add("Img_per_grp: user defined image per group for averaging = %d"%img_per_grp)
log_main.add("Overhead: total python overhead memory consumption = %f"%overhead_loading)
log_main.add("Total memory) = 4.0*nx^2*(nproj + navg +nvar+ img_per_grp)/1.0e9 + overhead: %12.3f [GB]"%\
(total_mem[1] + overhead_loading))
del full_data
mpi_barrier(MPI_COMM_WORLD)
if myid == heavy_load_myid:
log_main.add("Begin reading and preprocessing images on processor. Wait... ")
ttt = time()
#imgdata = EMData.read_images(stack, all_proj)
imgdata = [ None for im in range(len(all_proj))]
for index_of_proj in range(len(all_proj)):
#image = get_im(stack, all_proj[index_of_proj])
if( current_window > 0): imgdata[index_of_proj] = fdecimate(window2d(get_im(stack, all_proj[index_of_proj]),current_window,current_window), nx, ny)
else: imgdata[index_of_proj] = fdecimate(get_im(stack, all_proj[index_of_proj]), nx, ny)
if (current_decimate> 0.0 and options.CTF):
ctf = imgdata[index_of_proj].get_attr("ctf")
ctf.apix = ctf.apix/current_decimate
imgdata[index_of_proj].set_attr("ctf", ctf)
if myid == heavy_load_myid and index_of_proj%100 == 0:
log_main.add(" ...... %6.2f%% "%(index_of_proj/float(len(all_proj))*100.))
mpi_barrier(MPI_COMM_WORLD)
if myid == heavy_load_myid:
log_main.add("All_proj preprocessing cost %7.2f m"%((time()-ttt)/60.))
log_main.add("Wait untill reading on all CPUs done...")
'''
imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
if options.fl > 0.0:
for k in xrange(len(imgdata2)):
imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
if myid == main_node:
vol.write_image("vol_ctf.hdf")
print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf"))
del vol, imgdata2
mpi_barrier(MPI_COMM_WORLD)
'''
from applications import prepare_2d_forPCA
from utilities import model_blank
from EMAN2 import Transform
if not options.no_norm:
mask = model_circle(nx/2-2, nx, nx)
if options.CTF:
from utilities import pad
from filter import filt_ctf
from filter import filt_tanl
if myid == heavy_load_myid:
log_main.add("Start computing 2D aveList and varList. Wait...")
ttt = time()
inner=nx//2-4
outer=inner+2
xform_proj_for_2D = [ None for i in range(len(proj_list))]
for i in range(len(proj_list)):
ki = proj_angles[proj_list[i][0]][3]
if ki >= symbaselen: continue
mi = index[ki]
dpar = Util.get_transform_params(imgdata[mi], "xform.projection", "spider")
phiM, thetaM, psiM, s2xM, s2yM = dpar["phi"],dpar["theta"],dpar["psi"],-dpar["tx"]*current_decimate,-dpar["ty"]*current_decimate
grp_imgdata = []
for j in range(img_per_grp):
mj = index[proj_angles[proj_list[i][j]][3]]
cpar = Util.get_transform_params(imgdata[mj], "xform.projection", "spider")
alpha, sx, sy, mirror = params_3D_2D_NEW(cpar["phi"], cpar["theta"],cpar["psi"], -cpar["tx"]*current_decimate, -cpar["ty"]*current_decimate, mirror_list[i][j])
if thetaM <= 90:
if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, phiM - cpar["phi"], 0.0, 0.0, 1.0)
else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, 180-(phiM - cpar["phi"]), 0.0, 0.0, 1.0)
else:
if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(phiM- cpar["phi"]), 0.0, 0.0, 1.0)
else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(180-(phiM - cpar["phi"])), 0.0, 0.0, 1.0)
imgdata[mj].set_attr("xform.align2d", Transform({"type":"2D","alpha":alpha,"tx":sx,"ty":sy,"mirror":mirror,"scale":1.0}))
grp_imgdata.append(imgdata[mj])
if not options.no_norm:
for k in range(img_per_grp):
ave, std, minn, maxx = Util.infomask(grp_imgdata[k], mask, False)
grp_imgdata[k] -= ave
grp_imgdata[k] /= std
if options.fl > 0.0:
for k in range(img_per_grp):
grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
# Because of background issues, only linear option works.
if options.CTF: ave, var = aves_wiener(grp_imgdata, SNR = 1.0e5, interpolation_method = "linear")
else: ave, var = ave_var(grp_imgdata)
# Switch to std dev
# threshold is not really needed,it is just in case due to numerical accuracy something turns out negative.
var = square_root(threshold(var))
set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])
aveList.append(ave)
varList.append(var)
xform_proj_for_2D[i] = [phiM, thetaM, 0.0, 0.0, 0.0]
'''
if nvec > 0:
eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True)
for k in range(nvec):
set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
eigList[k].append(eig[k])
"""
if myid == 0 and i == 0:
for k in xrange(nvec):
eig[k].write_image("eig.hdf", k)
"""
'''
if (myid == heavy_load_myid) and (i%100 == 0):
log_main.add(" ......%6.2f%% "%(i/float(len(proj_list))*100.))
del imgdata, grp_imgdata, cpar, dpar, all_proj, proj_angles, index
if not options.no_norm: del mask
if myid == main_node: del tab
# At this point, all averages and variances are computed
mpi_barrier(MPI_COMM_WORLD)
if (myid == heavy_load_myid):
log_main.add("Computing aveList and varList took %12.1f [m]"%((time()-ttt)/60.))
xform_proj_for_2D = wrap_mpi_gatherv(xform_proj_for_2D, main_node, MPI_COMM_WORLD)
if (myid == main_node):
write_text_row(xform_proj_for_2D, os.path.join(current_output_dir, "params.txt"))
del xform_proj_for_2D
mpi_barrier(MPI_COMM_WORLD)
if options.ave2D:
from fundamentals import fpol
from applications import header
if myid == main_node:
log_main.add("Compute ave2D ... ")
km = 0
for i in range(number_of_proc):
if i == main_node :
for im in range(len(aveList)):
aveList[im].write_image(os.path.join(current_output_dir, options.ave2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
nl = int(nl[0])
for im in range(nl):
ave = recv_EMData(i, im+i+70000)
"""
nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('pix_err', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
"""
tmpvol=fpol(ave, nx, nx,1)
tmpvol.write_image(os.path.join(current_output_dir, options.ave2D), km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in range(len(aveList)):
send_EMData(aveList[im], main_node,im+myid+70000)
"""
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
members = aveList[im].get_attr('pix_err')
mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
except:
mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
"""
if myid == main_node:
header(os.path.join(current_output_dir, options.ave2D), params='xform.projection', fimport = os.path.join(current_output_dir, "params.txt"))
mpi_barrier(MPI_COMM_WORLD)
if options.ave3D:
from fundamentals import fpol
t5 = time()
if myid == main_node: log_main.add("Reconstruct ave3D ... ")
ave3D = recons3d_4nn_MPI(myid, aveList, symmetry=options.sym, npad=options.npad)
bcast_EMData_to_all(ave3D, myid)
if myid == main_node:
if current_decimate != 1.0: ave3D = resample(ave3D, 1./current_decimate)
ave3D = fpol(ave3D, nnxo, nnxo, nnxo) # always to the orignal image size
set_pixel_size(ave3D, 1.0)
ave3D.write_image(os.path.join(current_output_dir, options.ave3D))
log_main.add("Ave3D reconstruction took %12.1f [m]"%((time()-t5)/60.0))
log_main.add("%-70s: %s\n"%("The reconstructed ave3D is saved as ", options.ave3D))
mpi_barrier(MPI_COMM_WORLD)
del ave, var, proj_list, stack, alpha, sx, sy, mirror, aveList
'''
if nvec > 0:
for k in range(nvec):
if myid == main_node:log_main.add("Reconstruction eigenvolumes", k)
cont = True
ITER = 0
mask2d = model_circle(radiuspca, nx, nx)
while cont:
#print "On node %d, iteration %d"%(myid, ITER)
eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad)
bcast_EMData_to_all(eig3D, myid, main_node)
if options.fl > 0.0:
eig3D = filt_tanl(eig3D, options.fl, options.aa)
if myid == main_node:
eig3D.write_image(os.path.join(options.outpout_dir, "eig3d_%03d.hdf"%(k, ITER)))
Util.mul_img( eig3D, model_circle(radiuspca, nx, nx, nx) )
eig3Df, kb = prep_vol(eig3D)
del eig3D
cont = False
icont = 0
for l in range(len(eigList[k])):
phi, theta, psi, s2x, s2y = get_params_proj(eigList[k][l])
proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y])
cl = ccc(proj, eigList[k][l], mask2d)
if cl < 0.0:
icont += 1
cont = True
eigList[k][l] *= -1.0
u = int(cont)
u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD)
icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
if myid == main_node:
u = int(u[0])
log_main.add(" Eigenvector: ",k," number changed ",int(icont[0]))
else: u = 0
u = bcast_number_to_all(u, main_node)
cont = bool(u)
ITER += 1
del eig3Df, kb
mpi_barrier(MPI_COMM_WORLD)
del eigList, mask2d
'''
if options.ave3D: del ave3D
if options.var2D:
from fundamentals import fpol
from applications import header
if myid == main_node:
log_main.add("Compute var2D...")
km = 0
for i in range(number_of_proc):
if i == main_node :
for im in range(len(varList)):
tmpvol=fpol(varList[im], nx, nx,1)
tmpvol.write_image(os.path.join(current_output_dir, options.var2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
nl = int(nl[0])
for im in range(nl):
ave = recv_EMData(i, im+i+70000)
tmpvol=fpol(ave, nx, nx,1)
tmpvol.write_image(os.path.join(current_output_dir, options.var2D), km)
km += 1
else:
mpi_send(len(varList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in range(len(varList)):
send_EMData(varList[im], main_node, im+myid+70000)# What with the attributes??
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
from applications import header
header(os.path.join(current_output_dir, options.var2D), params = 'xform.projection',fimport = os.path.join(current_output_dir, "params.txt"))
mpi_barrier(MPI_COMM_WORLD)
if options.var3D:
if myid == main_node: log_main.add("Reconstruct var3D ...")
t6 = time()
# radiusvar = options.radius
# if( radiusvar < 0 ): radiusvar = nx//2 -3
res = recons3d_4nn_MPI(myid, varList, symmetry = options.sym, npad=options.npad)
#res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
if myid == main_node:
from fundamentals import fpol
if current_decimate != 1.0: res = resample(res, 1./current_decimate)
res = fpol(res, nnxo, nnxo, nnxo)
set_pixel_size(res, 1.0)
res.write_image(os.path.join(current_output_dir, options.var3D))
log_main.add("%-70s: %s\n"%("The reconstructed var3D is saved as ", options.var3D))
log_main.add("Var3D reconstruction took %f12.1 [m]"%((time()-t6)/60.0))
log_main.add("Total computation time %f12.1 [m]"%((time()-t0)/60.0))
log_main.add("sx3dvariability finishes")
from mpi import mpi_finalize
mpi_finalize()
if RUNNING_UNDER_MPI: global_def.MPI = False
global_def.BATCH = False
def main():
global debug,logid
progname = os.path.basename(sys.argv[0])
usage = """%prog [options] <input stack/image> ...
Various CTF-related operations on images, including automatic fitting. Note that automatic fitting is limited to 5 microns
underfocus at most. Input particles should be unmasked and unfiltered. A minimum of ~20% padding around the
particles is required for background extraction, even if this brings the edge of another particle into the box in some cases.
Particles should be reasonably well centered. Can also optionally phase flip and Wiener filter particles. Wiener filtration comes
after phase-flipping, so if phase flipping is performed Wiener filtered particles will also be phase-flipped. Note that both
operations are performed on oversampled images if specified (though final real-space images are clipped back to their original
size. Increasing padding during the particle picking process will improve the accuracy of phase-flipping, particularly for
images far from focus."""
parser = OptionParser(usage=usage,version=EMANVERSION)
parser.add_option("--gui",action="store_true",help="Start the GUI for interactive fitting",default=False)
parser.add_option("--auto_fit",action="store_true",help="Runs automated CTF fitting on the input images",default=False)
parser.add_option("--bgmask",type="int",help="Compute the background power spectrum from the edge of the image, specify a mask radius in pixels which would largely mask out the particles. Default is boxsize/2.",default=0)
parser.add_option("--apix",type="float",help="Angstroms per pixel for all images",default=0)
parser.add_option("--voltage",type="float",help="Microscope voltage in KV",default=0)
parser.add_option("--cs",type="float",help="Microscope Cs (spherical aberation)",default=0)
parser.add_option("--ac",type="float",help="Amplitude contrast (percentage, default=10)",default=10)
parser.add_option("--autohp",action="store_true",help="Automatic high pass filter of the SNR only to remove initial sharp peak, phase-flipped data is not directly affected (default false)",default=False)
#parser.add_option("--invert",action="store_true",help="Invert the contrast of the particles in output files (default false)",default=False)
parser.add_option("--nonorm",action="store_true",help="Suppress per image real-space normalization",default=False)
parser.add_option("--nosmooth",action="store_true",help="Disable smoothing of the background (running-average of the log with adjustment at the zeroes of the CTF)",default=False)
#parser.add_option("--phaseflip",action="store_true",help="Perform phase flipping after CTF determination and writes to specified file.",default=False)
#parser.add_option("--wiener",action="store_true",help="Wiener filter (optionally phaseflipped) particles.",default=False)
parser.add_option("--oversamp",type="int",help="Oversampling factor",default=1)
parser.add_option("--sf",type="string",help="The name of a file containing a structure factor curve. Can improve B-factor determination.",default=None)
parser.add_option("--debug",action="store_true",default=False)
(options, args) = parser.parse_args()
if len(args)<1 : parser.error("Input image required")
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
if options.auto_fit:
if options.voltage==0 : parser.error("Please specify voltage")
if options.cs==0 : parser.error("Please specify Cs")
if options.apix==0 : print "Using A/pix from header"
debug=options.debug
global sfcurve
if options.sf :
sfcurve=XYData()
sfcurve.read_file(options.sf)
logid=E2init(sys.argv)
# if options.oversamp>1 : options.apix/=float(options.oversamp)
db_project=db_open_dict("bdb:project")
db_parms=db_open_dict("bdb:e2ctf.parms")
db_misc=db_open_dict("bdb:e2ctf.misc")
options.filenames = args
### Power spectrum and CTF fitting
if options.auto_fit:
img_sets=pspec_and_ctf_fit(options,debug) # converted to a function so to work with the workflow
### This computes the intensity of the background subtracted power spectrum at each CTF maximum for all sets
global envelopes # needs to be a global for the Simplex minimizer
# envelopes is essentially a cache of information that could be useful at later stages of refinement
# as according to Steven Ludtke
for i in img_sets:
envelopes.append(ctf_env_points(i[2],i[3],i[1]))
# we use a simplex minimizer to try to rescale the individual sets to match as best they can
scales=[1.0]*len(img_sets)
if (len(img_sets)>3) :
incr=[0.2]*len(img_sets)
simp=Simplex(env_cmp,scales,incr)
scales=simp.minimize(maxiters=1000)[0]
# print scales
print " "
# apply the final rescaling
envelope=[]
for i in range(len(scales)):
cur=envelopes[i]
for j in range(len(cur)):
envelope.append((cur[j][0],cur[j][1]*scales[i]))
envelope.sort()
envelope=[i for i in envelope if i[1]>0] # filter out all negative peak values
db_misc=db_open_dict("bdb:e2ctf.misc")
db_misc["envelope"]=envelope
#db_close_dict("bdb:e2ctf.misc")
#out=file("envelope.txt","w")
#for i in envelope: out.write("%f\t%f\n"%(i[0],i[1]))
#out.close()
### GUI - user can update CTF parameters interactively
if options.gui :
img_sets = get_gui_arg_img_sets(options.filenames)
if len(img_sets) == 0:
E2end(logid)
sys.exit(1)
app=EMApp()
gui=GUIctf(app,img_sets)
gui.show_guis()
app.exec_()
print "done execution"
### Process input files
#if debug : print "Phase flipping / Wiener filtration"
# write wiener filtered and/or phase flipped particle data to the local database
#if options.phaseflip or options.wiener: # only put this if statement here to make the program flow obvious
# write_e2ctf_output(options) # converted to a function so to work with the workflow
E2end(logid)
def filter_shrink(input_bdb_name,
output_stack_path,
output_bdb_name,
alignYN=False,
filtrad=None,
shrink=None,
verbose=False):
"""
Filters and shrinks image stack.
Arguments:
input_bdb_name : Input BDB stack (in the form bdb:DIRECTORY#STACK)
output_stack_path : Name for output image stack (MRCS/HDF/etc.)
output_bdb_name : Name for output BDB stack (in the form bdb:DIRECTORY#STACK)
alignYN: (boolean) Whether to apply alignments
filtrad : Filter radius, reciprocal pixels
shrink : Downsampling factor
combined_params_file : Output combined alignment parameters
verbose : (boolean) Whether to write to screen
"""
input_stack = EMData.read_images(input_bdb_name)
num_imgs = EMUtil.get_image_count(input_bdb_name)
box_size = input_stack[0].get_attr('nx')
if options.shrink:
sub_rate = float(1) / options.shrink
box_size = int(float(box_size) / options.shrink + 0.5)
if verbose:
print('sub_rate', sub_rate, type(sub_rate), box_size,
options.shrink)
# Initialize stack & BDB
aligned_stack_obj = EMData(box_size, box_size, num_imgs)
new_bdb_dict = db_open_dict(output_bdb_name)
# Loop through images
for img_num in range(len(input_stack)):
img_orig = input_stack[img_num]
try:
alpha, sx, sy, mirror, scale = get_params2D(img_orig)
except RuntimeError:
print('\nERROR! Exiting with RuntimeError')
img_prev = input_stack[img_num - 1]
print('\nPrevious particle: %s %s' %
(img_num - 1, img_prev.get_attr_dict()))
print('\nCurrent particle: %s %s' %
(img_num, img_orig.get_attr_dict()))
exit()
# Optionally apply alignment parameters
if alignYN:
img_ali = rot_shift2D(img_orig, alpha, sx, sy, mirror, scale,
"quadratic")
else:
img_ali = img_orig
if verbose and img_num == 0:
print('\nimg_orig.get_attr_dict0', img_orig.get_attr_dict())
img_ali_dict = img_ali.get_attr_dict()
print('\nimg_ali.get_attr_dict1\n', img_ali.get_attr_dict())
if filtrad:
img_ali = filt_gaussl(img_ali, filtrad)
if shrink:
img_ali = resample(img_ali, sub_rate)
#### (Maybe update resample_ratio)
img_ali_dict = img_ali.get_attr_dict()
img_ali_dict["data_path"] = os.path.join(
'..', STACKFILEDIR, os.path.basename(output_stack_path))
img_ali_dict["ptcl_source_coord_id"] = img_num
new_bdb_dict[img_num] = img_ali_dict
aligned_stack_obj.insert_clip(img_ali, (0, 0, img_num))
# End image-loop
aligned_stack_obj.write_image(output_stack_path)
db_close_dict(output_bdb_name)
return num_imgs
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 2111-1307 USA
#
#
from builtins import range
import sys
from EMAN2 import *
from EMAN2db import db_open_dict
if len(sys.argv)==1 :
print("Please provide the name of a refine_xx directory to extract FSC curves from")
exit(1)
db=db_open_dict("bdb:%s#convergence.results"%sys.argv[1],ro=True)
for k in list(db.keys()):
curve=db[k]
out=open("fsc_%s_%s.txt"%(sys.argv[1].rsplit("_",1)[-1],k),"w")
for i in range(len(curve[0])): out.write("%1.5f\t%1.4f\n"%(curve[0][i],curve[1][i]))
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options] <volume> <mask> <output>
WARNING: Do not use this program. The whole concept has been replaced by "gold standard" refinement.
Note that this method has not yet been published, and is not yet a reliable
scheme for resolution determination. DO NOT EXPECT RELIABLE ANSWERS AT THIS POINT !
This will compute an FSC resolution curve from a single structure by
comparing the power spectrum of the noise region to the power spectrum
of the reconstructed particle. The 'mask' input should mask out the particle,
and should be a 'soft' mask. Sharp edges on the mask may ruin the results.
proc3d (EMAN1) with the automask2 option can produce appropriate masks. The
3rd parameter (in automask2) should be about 10 percent of the box size. This will not
work with particles that have been tightly masked already. If doing an EMAN1
reconstruction with the amask= option, you must also use the refmaskali option
(in the refine command). Note that the resultant curve is guaranteed to go
to zero at near Nyquist because it assumes that the SNR is zero near Nyquist.
If your data is undersampled, the resulting curve may also be inaccurate.
Models should also not be heavily low-pass filtered
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_pos_argument(name="volume",
help="Volume to compute resolution on.",
default="",
guitype='filebox',
row=0,
col=0,
rowspan=1,
colspan=2)
parser.add_pos_argument(
name="mask",
help=
"Mask for the volume. The volume is masked by this file (should be ssoft mask).",
default="",
guitype='filebox',
row=1,
col=0,
rowspan=1,
colspan=2)
parser.add_pos_argument(
name="output",
help="Name of the output file. These data will contain FSC stats.",
default="",
guitype='strbox',
row=2,
col=0,
rowspan=1,
colspan=2)
parser.add_header(
name="eotestheader",
help='Options below this label are specific to e2resolution',
title="### e2resolution options ###",
row=3,
col=0,
rowspan=1,
colspan=2)
parser.add_argument("--apix",
"-A",
type=float,
help="A/voxel",
default=-1.0,
guitype='floatbox',
row=4,
col=0,
rowspan=1,
colspan=1)
parser.add_argument(
"--path",
type=str,
help=
"The name the e2refine directory that contains the reconstruction data. If specified will place curves generated in bdb:path#convergence.results",
guitype='dirbox',
default=None,
dirbasename='refine',
row=4,
col=1,
rowspan=1,
colspan=1)
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higner number means higher level of verboseness")
(options, args) = parser.parse_args()
if len(args) < 3: parser.error("Input and output files required")
E2n = E2init(sys.argv, options.ppid)
#options.align=parsemodopt(options.align)
print(
"WARNING: e2resolution is an experimental program. It does not (yet) produce reliable resolution curves in most cases."
)
print("read models")
data = EMData(args[0], 0)
mask = EMData(args[1], 0)
# mask.to_one()
# mask.process_inplace("mask.gaussian",{"outer_radius":mask.get_xsize()/6,"inner_radius":mask.get_xsize()/6})
mask.write_image("msk.mrc", 0)
# inverted mask
maski = mask.copy()
maski *= -1.0
maski += 1.0
maski.process_inplace(
"mask.gaussian", {
"outer_radius": old_div(mask.get_xsize(), 6),
"inner_radius": old_div(mask.get_xsize(), 3)
})
maski.write_image("msk2.mrc", 0)
noise = data.copy()
noise *= maski
data *= mask
print("compute FFT")
dataf = data.do_fft()
noisef = noise.do_fft()
print("compute power 1")
datapow = dataf.calc_radial_dist(
old_div(dataf.get_ysize(), 2) - 1, 1, 1, 1)
print("compute power 2")
noisepow = noisef.calc_radial_dist(
old_div(noisef.get_ysize(), 2) - 1, 1, 1, 1)
x = list(range(1, len(datapow) + 1))
if options.apix > 0:
x = [old_div(i, (len(datapow) * options.apix * 2.0)) for i in x]
else:
x = [old_div(i, (len(datapow) * data["apix_x"] * 2.0)) for i in x]
# normalize noise near Nyquist
s = 0
sn = 0
for i in range(int(len(noisepow) * .9), len(noisepow) - 1):
if datapow[i] < datapow[i + 1] or noisepow[i] < noisepow[i + 1]:
continue
s += old_div(datapow[i], noisepow[i])
sn += 1.0
if sn == 0:
print("Warning, strange normalization")
s = old_div(datapow[int(len(noisepow) * .9)],
noisepow[int(len(noisepow) * .9)])
else:
s /= sn
noisepow = [i * s for i in noisepow]
#
# # normalize based on volume
# datapow=[v/mask["mean"] for v in datapow]
# noisepow=[v/maski["mean"] for v in noisepow]
# compute signal to noise ratio
snr = []
for i in range(len(datapow)):
try:
snr.append(old_div((datapow[i] - noisepow[i]), noisepow[i]))
except:
snr.append(0)
# convert to FSC
fsc = [old_div(i, (2.0 + i)) for i in snr]
out = open(args[2], "w")
for i in range(len(fsc)):
out.write("%f\t%f\n" % (x[i], fsc[i]))
out.close()
out = open(args[2] + ".dat", "w")
for i in range(len(fsc)):
out.write("%f\t%f\n" % (x[i], datapow[i]))
out.close()
out = open(args[2] + ".noi", "w")
for i in range(len(noisepow)):
out.write("%f\t%f\n" % (x[i], noisepow[i]))
out.close()
if options.path != None:
s = base_name(args[0])
db = db_open_dict("bdb:" + options.path + "#convergence.results")
db[s + "res_fsc"] = [
x, fsc
] # warning, changing this naming convention will disrupt forms in the workflow (make them fail)
db[s + "res_datapow"] = [x, datapow]
db[s + "res_noisepow"] = [x, noisepow]
db_close_dict("bdb:" + options.path + "#convergence.results")
E2end(E2n)
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 2111-1307 USA
#
#
import sys
from EMAN2 import *
from EMAN2db import db_open_dict
if len(sys.argv)==1 :
print "Please provide the name of a refine_xx directory to extract FSC curves from"
exit(1)
db=db_open_dict("bdb:%s#convergence.results"%sys.argv[1],ro=True)
for k in db.keys():
curve=db[k]
out=file("fsc_%s_%s.txt"%(sys.argv[1].rsplit("_",1)[-1],k),"w")
for i in xrange(len(curve[0])): out.write("%1.5f\t%1.4f\n"%(curve[0][i],curve[1][i]))
def main(args):
progname = os.path.basename(sys.argv[0])
usage = ( progname + " stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter" +
" --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp" +
" --generation=generation --rand_seed=rand_seed>" )
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--radius", type="int", default=-1, help="<Particle radius>, it has to be provided.")
parser.add_option("--img_per_grp", type="int", default=100, help="<number of images per group> in the ideal case (essentially maximum size of class) (100)")
parser.add_option("--CTF", action="store_true", default=False, help="<CTF flag>, if set the data will be phase-flipped")
parser.add_option("--ir", type="int", default=1, help="<inner ring> of the resampling to polar coordinates (1)")
parser.add_option("--rs", type="int", default=1, help="<ring step> of the resampling to polar coordinates (1)")
parser.add_option("--xr", type="int", default=-1, help="<x range> of translational search (By default set by the program) (advanced)")
parser.add_option("--yr", type="int", default=-1, help="<y range> of translational search (same as xr) (advanced)")
parser.add_option("--ts", type="float", default=1.0, help="<search step> of translational search (1.0)")
parser.add_option("--maxit", type="int", default=30, help="number of iterations for reference-free alignment (30)")
#parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)")
parser.add_option("--center_method", type="int", default=7, help="<Method for centering> of global 2D average during initial prealignment of data (default : 7; 0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7)")
parser.add_option("--dst", type="float", default=90.0, help="discrete angle used in within group alignment ")
parser.add_option("--FL", type="float", default=0.2, help="<lowest stopband> frequency used in the tangent filter (0.2)")
parser.add_option("--FH", type="float", default=0.3, help="<highest stopband> frequency used in the tangent filter (0.3)")
parser.add_option("--FF", type="float", default=0.2, help="<fall-off of the tangent> filter (0.2)")
parser.add_option("--init_iter", type="int", default=3, help="<init_iter> number of iterations of ISAC program in initialization (3)")
parser.add_option("--main_iter", type="int", default=3, help="<main_iter> number of iterations of ISAC program in main part (3)")
parser.add_option("--iter_reali", type="int", default=1, help="<iter_reali> number of iterations in ISAC before checking stability (1)")
parser.add_option("--match_first", type="int", default=1, help="number of iterations to run 2-way matching in the first phase (1)")
parser.add_option("--max_round", type="int", default=20, help="maximum rounds of generating candidate averages in the first phase (20)")
parser.add_option("--match_second", type="int", default=5, help="number of iterations to run 2-way (or 3-way) matching in the second phase (5)")
parser.add_option("--stab_ali", type="int", default=5, help="number of alignments when checking stability (5)")
parser.add_option("--thld_err", type="float", default=0.7, help="the threshold of pixel error when checking stability (0.7)")
parser.add_option("--indep_run", type="int", default=4, help="number of independent runs for reproducibility (default=4, only values 2, 3 and 4 are supported (4)")
parser.add_option("--thld_grp", type="int", default=10, help="minimum size of class (10)")
parser.add_option("--n_generations", type="int", default=100, help="<n_generations> program stops when reaching this total number of generations (advanced)")
#parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)")
parser.add_option("--rand_seed", type="int", default=None, help="random seed set before calculations, useful for testing purposes (default None - total randomness)")
parser.add_option("--new", action="store_true", default=False, help="use new code (default = False)")
parser.add_option("--debug", action="store_true", default=False, help="debug info printout (default = False)")
# must be switched off in production
parser.add_option("--use_latest_master_directory", action="store_true", dest="use_latest_master_directory", default=False)
parser.add_option("--restart_section", type="string", default="", help="<restart section name> (no spaces) followed immediately by comma, followed immediately by generation to restart, example: \n--restart_section=candidate_class_averages,1 (Sections: restart, candidate_class_averages, reproducible_class_averages)")
parser.add_option("--stop_after_candidates", action="store_true", default=False, help="<stop_after_candidates> stops after the 'candidate_class_averages' section")
parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help = SUPPRESS_HELP)
(options, args) = parser.parse_args(args)
if options.return_options:
return parser
if len(args) > 2:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
sys.exit()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
from isac import iter_isac
global_def.BATCH = True
global_def.BATCH = True
command_line_provided_stack_filename = args[0]
global_def.BATCH = True
main_node = 0
mpi_init(0, [])
myid = mpi_comm_rank(MPI_COMM_WORLD)
nproc = mpi_comm_size(MPI_COMM_WORLD)
radi = options.radius
center_method = options.center_method
if(radi < 1): ERROR("Particle radius has to be provided!","sxisac",1,myid)
use_latest_master_directory = options.use_latest_master_directory
stop_after_candidates = options.stop_after_candidates
program_state_stack.restart_location_title_from_command_line = options.restart_section
from utilities import qw
program_state_stack.PROGRAM_STATE_VARIABLES = set(qw("""
isac_generation
"""))
# create or reuse master directory
masterdir = ""
stack_processed_by_ali2d_base__filename = ""
stack_processed_by_ali2d_base__filename__without_master_dir = ""
error_status = 0
if len(args) == 2:
masterdir = args[1]
elif len(args) == 1:
if use_latest_master_directory:
all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)]
import re; r = re.compile("^master.*$")
all_dirs = filter(r.match, all_dirs)
if len(all_dirs)>0:
# all_dirs = max(all_dirs, key=os.path.getctime)
masterdir = max(all_dirs, key=os.path.getmtime)
#Create folder for all results or check if there is one created already
if(myid == main_node):
if( masterdir == ""):
timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime())
masterdir = "master"+timestring
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
elif not os.path.exists(masterdir):
# os.path.exists(masterdir) does not exist
masterdir = args[1]
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
if(args[0][:4] == "bdb:"): filename = args[0][4:]
else: filename = args[0][:-4]
filename = os.path.basename(filename)
stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join(masterdir, filename )
stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename
if_error_all_processes_quit_program(error_status)
# send masterdir to all processes
masterdir = send_string_to_all(masterdir)
if myid == 0:
if options.restart_section != "":
if os.path.exists(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)):
stored_stack, stored_state = restore_program_stack_and_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE))
import re
if "," in options.restart_section:
parsed_restart_section_option = options.restart_section.split(",")
stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%parsed_restart_section_option[0], stored_state[-1]["location_in_program"])
generation_str_format = parsed_restart_section_option[1]
if generation_str_format != "":
isac_generation_from_command_line = int(generation_str_format)
stored_state[-1]["isac_generation"] = isac_generation_from_command_line
else:
isac_generation_from_command_line = 1
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = -1
stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%options.restart_section, stored_state[-1]["location_in_program"])
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
store_program_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE), stored_state, stored_stack)
else:
print "Please remove the restart_section option from the command line. The program must be started from the beginning."
mpi_finalize()
sys.exit()
else:
isac_generation_from_command_line = -1
program_state_stack(locals(), getframeinfo(currentframe()), os.path.join(masterdir,NAME_OF_JSON_STATE_FILE))
stack_processed_by_ali2d_base__filename = send_string_to_all(stack_processed_by_ali2d_base__filename)
stack_processed_by_ali2d_base__filename__without_master_dir = \
send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir)
# PARAMETERS OF THE PROCEDURE
if( options.xr == -1 ):
# Default values
target_nx = 76
target_radius = 29
target_xr = 1
else: # nx//2
# Check below!
target_xr = options.xr
target_nx = 76 + target_xr - 1 # subtract one, which is default
target_radius = 29
mpi_barrier(MPI_COMM_WORLD)
# Initialization of stacks
if(myid == main_node):
number_of_images_in_stack = EMUtil.get_image_count(command_line_provided_stack_filename)
else:
number_of_images_in_stack = 0
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node)
nxrsteps = 4
init2dir = os.path.join(masterdir,"2dalignment")
if(myid == 0):
import subprocess
from logger import Logger, BaseLogger_Files
# Create output directory
log2d = Logger(BaseLogger_Files())
log2d.prefix = os.path.join(init2dir)
cmd = "mkdir -p "+log2d.prefix
outcome = subprocess.call(cmd, shell=True)
log2d.prefix += "/"
# outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True)
else:
outcome = 0
log2d = None
if(myid == main_node):
a = get_im(command_line_provided_stack_filename)
nnxo = a.get_xsize()
else:
nnxo = 0
nnxo = bcast_number_to_all(nnxo, source_node = main_node)
txrm = (nnxo - 2*(radi+1))//2
if(txrm < 0): ERROR( "ERROR!! Radius of the structure larger than the window data size permits %d"%(radi), "sxisac",1, myid)
if(txrm/nxrsteps>0):
tss = ""
txr = ""
while(txrm/nxrsteps>0):
tts=txrm/nxrsteps
tss += " %d"%tts
txr += " %d"%(tts*nxrsteps)
txrm =txrm//2
else:
tss = "1"
txr = "%d"%txrm
# section ali2d_base
# centering method is set to #7
params2d, aligned_images = ali2d_base(command_line_provided_stack_filename, init2dir, None, 1, radi, 1, txr, txr, tss, \
False, 90.0, center_method, 14, options.CTF, 1.0, False, \
"ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False)
if( myid == main_node ):
write_text_row(params2d,os.path.join(init2dir, "initial2Dparams.txt"))
del params2d
mpi_barrier(MPI_COMM_WORLD)
# We assume the target image size will be target_nx, radius will be 29, and xr = 1.
# Note images can be also padded, in which case shrink_ratio > 1.
shrink_ratio = float(target_radius)/float(radi)
nx = aligned_images[0].get_xsize()
nima = len(aligned_images)
newx = int(nx*shrink_ratio + 0.5)
from fundamentals import rot_shift2D, resample
from utilities import pad, combine_params2
if(shrink_ratio < 1.0):
if newx > target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx :
msk = model_circle(newx//2-2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
elif(shrink_ratio == 1.0):
if newx > target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx :
msk = model_circle(nx//2-2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
#aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
elif(shrink_ratio > 1.0):
target_radius = radi
msk = model_circle(target_radius, nx, nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
del msk
gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images, myid)
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node)
if( myid == main_node ):
for i in range(number_of_images_in_stack): aligned_images[i].write_image(stack_processed_by_ali2d_base__filename,i)
# It has to be explicitly closed
from EMAN2db import db_open_dict
DB = db_open_dict(stack_processed_by_ali2d_base__filename)
DB.close()
fp = open("README_shrink_ratio.txt", "w")
output_text = """
Since, for processing purposes, isac changes the image dimensions,
adjustment of pixel size needs to be made in subsequent steps, (e.g.
running sxviper.py). The shrink ratio for this particular isac run is
--------
%.5f
--------
To get the pixel size for the isac output the user needs to divide
the original pixel size by the above value. This info is saved in
the following file: README_shrink_ratio.txt
"""%shrink_ratio
fp.write(output_text); fp.flush() ;fp.close()
print output_text
mpi_barrier(MPI_COMM_WORLD)
global_def.BATCH = True
os.chdir(masterdir)
if program_state_stack(locals(), getframeinfo(currentframe())):
# if 1:
pass
if (myid == main_node):
cmdexecute("sxheader.py --consecutive --params=originalid %s"%stack_processed_by_ali2d_base__filename__without_master_dir)
cmdexecute("e2bdb.py %s --makevstack=%s_000"%(stack_processed_by_ali2d_base__filename__without_master_dir, stack_processed_by_ali2d_base__filename__without_master_dir))
if (myid == main_node):
main_dir_no = get_latest_directory_increment_value("./", NAME_OF_MAIN_DIR, myformat="%04d")
print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no
if isac_generation_from_command_line < 0:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = -1
if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no:
for i in xrange(isac_generation_from_command_line+1, main_dir_no + 1):
if i == isac_generation_from_command_line+1:
backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%i + " 000_backup" + "%05d"%backup_dir_no)
cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%i)
# it includes both command line and json file
my_restart_section = stored_state[-1]["location_in_program"].split("___")[-1]
if "restart" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no)
cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%isac_generation_from_command_line )
elif "candidate_class_averages" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no)
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line)
# cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line)
elif "reproducible_class_averages" in my_restart_section:
cmdexecute("rm -rf " + NAME_OF_MAIN_DIR + "%04d/ali_params_generation_*"%isac_generation_from_command_line)
cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_generation*"%isac_generation_from_command_line)
else:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 0
isac_generation_from_command_line = mpi_bcast(isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]
isac_generation = isac_generation_from_command_line - 1
if (myid == main_node):
if isac_generation == 0:
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation)
write_text_file([1], os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_accounted.txt"%isac_generation))
write_text_file(range(number_of_images_in_stack), os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_unaccounted.txt"%isac_generation))
# Stopping criterion should be inside the program.
while True:
isac_generation += 1
if isac_generation > options.n_generations:
break
data64_stack_current = "bdb:../"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d"%isac_generation
if(myid == main_node):
accounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1),"generation_%d_accounted.txt"%(isac_generation - 1)))
number_of_accounted_images = len(accounted_images)
# unaccounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1),"generation_%d_unaccounted.txt"%(isac_generation - 1)))
# number_of_unaccounted_images = len(unaccounted_images)
else:
number_of_accounted_images = 0
number_of_accounted_images = int(mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
if number_of_accounted_images == 0:
os.chdir("..")
break
program_state_stack.restart_location_title = "restart"
if program_state_stack(locals(), getframeinfo(currentframe())):
if (myid == main_node):
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation)
# reference the original stack
list_file = os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1), "generation_%d_unaccounted.txt"%(isac_generation - 1))
cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\
stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file))
mpi_barrier(MPI_COMM_WORLD)
os.chdir(NAME_OF_MAIN_DIR + "%04d"%isac_generation)
program_state_stack.restart_location_title = "candidate_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new)
# program_state_stack.restart_location_title = "stopped_program1"
# program_state_stack(locals(), getframeinfo(currentframe()))
program_state_stack.restart_location_title = "stop_after_candidates"
program_state_stack(locals(), getframeinfo(currentframe()))
if stop_after_candidates:
mpi_finalize()
sys.exit()
exit_program = 0
if(myid == main_node):
if not os.path.exists("class_averages_candidate_generation_%d.hdf"%isac_generation):
print "This generation (%d) no class averages were generated!"%isac_generation
exit_program = 1
exit_program = int(mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
if exit_program:
os.chdir("..")
break
program_state_stack.restart_location_title = "reproducible_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new)
pass
os.chdir("..")
if (myid == main_node):
cmdexecute("rm -f class_averages.hdf")
cpy(["generation_%04d/class_averages_generation_%d.hdf"%(i,i) for i in xrange(1, isac_generation)], "class_averages.hdf")
# program_state_stack.restart_location_title = "stopped_program2"
# program_state_stack(locals(), getframeinfo(currentframe()))
program_state_stack(locals(), getframeinfo(currentframe()), last_call="__LastCall")
mpi_finalize()
def main():
global debug, logid
progname = os.path.basename(sys.argv[0])
usage = """%prog [options] <input stack/image> ...
Various CTF-related operations on images, including automatic fitting. Note that automatic fitting is limited to 5 microns
underfocus at most. Input particles should be unmasked and unfiltered. A minimum of ~20% padding around the
particles is required for background extraction, even if this brings the edge of another particle into the box in some cases.
Particles should be reasonably well centered. Can also optionally phase flip and Wiener filter particles. Wiener filtration comes
after phase-flipping, so if phase flipping is performed Wiener filtered particles will also be phase-flipped. Note that both
operations are performed on oversampled images if specified (though final real-space images are clipped back to their original
size. Increasing padding during the particle picking process will improve the accuracy of phase-flipping, particularly for
images far from focus."""
parser = OptionParser(usage=usage, version=EMANVERSION)
parser.add_option("--gui",
action="store_true",
help="Start the GUI for interactive fitting",
default=False)
parser.add_option("--auto_fit",
action="store_true",
help="Runs automated CTF fitting on the input images",
default=False)
parser.add_option(
"--bgmask",
type="int",
help=
"Compute the background power spectrum from the edge of the image, specify a mask radius in pixels which would largely mask out the particles. Default is boxsize/2.",
default=0)
parser.add_option("--apix",
type="float",
help="Angstroms per pixel for all images",
default=0)
parser.add_option("--voltage",
type="float",
help="Microscope voltage in KV",
default=0)
parser.add_option("--cs",
type="float",
help="Microscope Cs (spherical aberation)",
default=0)
parser.add_option("--ac",
type="float",
help="Amplitude contrast (percentage, default=10)",
default=10)
parser.add_option(
"--autohp",
action="store_true",
help=
"Automatic high pass filter of the SNR only to remove initial sharp peak, phase-flipped data is not directly affected (default false)",
default=False)
#parser.add_option("--invert",action="store_true",help="Invert the contrast of the particles in output files (default false)",default=False)
parser.add_option("--nonorm",
action="store_true",
help="Suppress per image real-space normalization",
default=False)
parser.add_option(
"--nosmooth",
action="store_true",
help=
"Disable smoothing of the background (running-average of the log with adjustment at the zeroes of the CTF)",
default=False)
#parser.add_option("--phaseflip",action="store_true",help="Perform phase flipping after CTF determination and writes to specified file.",default=False)
#parser.add_option("--wiener",action="store_true",help="Wiener filter (optionally phaseflipped) particles.",default=False)
parser.add_option("--oversamp",
type="int",
help="Oversampling factor",
default=1)
parser.add_option(
"--sf",
type="string",
help=
"The name of a file containing a structure factor curve. Can improve B-factor determination.",
default=None)
parser.add_option("--debug", action="store_true", default=False)
(options, args) = parser.parse_args()
if len(args) < 1: parser.error("Input image required")
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
if options.auto_fit:
if options.voltage == 0: parser.error("Please specify voltage")
if options.cs == 0: parser.error("Please specify Cs")
if options.apix == 0: print "Using A/pix from header"
debug = options.debug
global sfcurve
if options.sf:
sfcurve = XYData()
sfcurve.read_file(options.sf)
logid = E2init(sys.argv)
# if options.oversamp>1 : options.apix/=float(options.oversamp)
db_project = db_open_dict("bdb:project")
db_parms = db_open_dict("bdb:e2ctf.parms")
db_misc = db_open_dict("bdb:e2ctf.misc")
options.filenames = args
### Power spectrum and CTF fitting
if options.auto_fit:
img_sets = pspec_and_ctf_fit(
options,
debug) # converted to a function so to work with the workflow
### This computes the intensity of the background subtracted power spectrum at each CTF maximum for all sets
global envelopes # needs to be a global for the Simplex minimizer
# envelopes is essentially a cache of information that could be useful at later stages of refinement
# as according to Steven Ludtke
for i in img_sets:
envelopes.append(ctf_env_points(i[2], i[3], i[1]))
# we use a simplex minimizer to try to rescale the individual sets to match as best they can
scales = [1.0] * len(img_sets)
if (len(img_sets) > 3):
incr = [0.2] * len(img_sets)
simp = Simplex(env_cmp, scales, incr)
scales = simp.minimize(maxiters=1000)[0]
# print scales
print " "
# apply the final rescaling
envelope = []
for i in range(len(scales)):
cur = envelopes[i]
for j in range(len(cur)):
envelope.append((cur[j][0], cur[j][1] * scales[i]))
envelope.sort()
envelope = [i for i in envelope
if i[1] > 0] # filter out all negative peak values
db_misc = db_open_dict("bdb:e2ctf.misc")
db_misc["envelope"] = envelope
#db_close_dict("bdb:e2ctf.misc")
#out=file("envelope.txt","w")
#for i in envelope: out.write("%f\t%f\n"%(i[0],i[1]))
#out.close()
### GUI - user can update CTF parameters interactively
if options.gui:
img_sets = get_gui_arg_img_sets(options.filenames)
if len(img_sets) == 0:
E2end(logid)
sys.exit(1)
app = EMApp()
gui = GUIctf(app, img_sets)
gui.show_guis()
app.exec_()
print "done execution"
### Process input files
#if debug : print "Phase flipping / Wiener filtration"
# write wiener filtered and/or phase flipped particle data to the local database
#if options.phaseflip or options.wiener: # only put this if statement here to make the program flow obvious
# write_e2ctf_output(options) # converted to a function so to work with the workflow
E2end(logid)
def main(args):
progname = os.path.basename(sys.argv[0])
usage = (
progname +
" stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter"
+
" --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp"
+ " --generation=generation --rand_seed=rand_seed>")
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option(
"--radius",
type="int",
help=
"particle radius: there is no default, a sensible number has to be provided, units - pixels (default required int)"
)
parser.add_option(
"--target_radius",
type="int",
default=29,
help=
"target particle radius: actual particle radius on which isac will process data. Images will be shrinked/enlarged to achieve this radius (default 29)"
)
parser.add_option(
"--target_nx",
type="int",
default=76,
help=
"target particle image size: actual image size on which isac will process data. Images will be shrinked/enlarged according to target particle radius and then cut/padded to achieve target_nx size. When xr > 0, the final image size for isac processing is 'target_nx + xr - 1' (default 76)"
)
parser.add_option(
"--img_per_grp",
type="int",
default=100,
help=
"number of images per class: in the ideal case (essentially maximum size of class) (default 100)"
)
parser.add_option(
"--CTF",
action="store_true",
default=False,
help=
"apply phase-flip for CTF correction: if set the data will be phase-flipped using CTF information included in image headers (default False)"
)
parser.add_option(
"--ir",
type="int",
default=1,
help=
"inner ring: of the resampling to polar coordinates. units - pixels (default 1)"
)
parser.add_option(
"--rs",
type="int",
default=1,
help=
"ring step: of the resampling to polar coordinates. units - pixels (default 1)"
)
parser.add_option(
"--xr",
type="int",
default=1,
help=
"x range: of translational search. By default, set by the program. (default 1)"
)
parser.add_option(
"--yr",
type="int",
default=-1,
help=
"y range: of translational search. By default, same as xr. (default -1)"
)
parser.add_option(
"--ts",
type="float",
default=1.0,
help=
"search step: of translational search: units - pixels (default 1.0)")
parser.add_option(
"--maxit",
type="int",
default=30,
help="number of iterations for reference-free alignment: (default 30)")
#parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)")
parser.add_option(
"--center_method",
type="int",
default=-1,
help=
"method for centering: of global 2D average during initial prealignment of data (0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7) (default -1)"
)
parser.add_option(
"--dst",
type="float",
default=90.0,
help="discrete angle used in within group alignment: (default 90.0)")
parser.add_option(
"--FL",
type="float",
default=0.2,
help=
"lowest stopband: frequency used in the tangent filter (default 0.2)")
parser.add_option(
"--FH",
type="float",
default=0.3,
help=
"highest stopband: frequency used in the tangent filter (default 0.3)")
parser.add_option("--FF",
type="float",
default=0.2,
help="fall-off of the tangent filter: (default 0.2)")
parser.add_option(
"--init_iter",
type="int",
default=3,
help=
"SAC initialization iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC initialization (default 3)"
)
parser.add_option(
"--main_iter",
type="int",
default=3,
help=
"SAC main iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC (default 3)"
)
parser.add_option(
"--iter_reali",
type="int",
default=1,
help=
"SAC stability check interval: every iter_reali iterations of SAC stability checking is performed (default 1)"
)
parser.add_option(
"--match_first",
type="int",
default=1,
help=
"number of iterations to run 2-way matching in the first phase: (default 1)"
)
parser.add_option(
"--max_round",
type="int",
default=20,
help=
"maximum rounds: of generating candidate class averages in the first phase (default 20)"
)
parser.add_option(
"--match_second",
type="int",
default=5,
help=
"number of iterations to run 2-way (or 3-way) matching in the second phase: (default 5)"
)
parser.add_option(
"--stab_ali",
type="int",
default=5,
help="number of alignments when checking stability: (default 5)")
parser.add_option(
"--thld_err",
type="float",
default=0.7,
help=
"threshold of pixel error when checking stability: equals root mean square of distances between corresponding pixels from set of found transformations and theirs average transformation, depends linearly on square of radius (parameter ou). units - pixels. (default 0.7)"
)
parser.add_option(
"--indep_run",
type="int",
default=4,
help=
"level of m-way matching for reproducibility tests: By default, perform full ISAC to 4-way matching. Value indep_run=2 will restrict ISAC to 2-way matching and 3 to 3-way matching. Note the number of used MPI processes requested in mpirun must be a multiplicity of indep_run. (default 4)"
)
parser.add_option("--thld_grp",
type="int",
default=10,
help="minimum size of reproducible class (default 10)")
parser.add_option(
"--n_generations",
type="int",
default=10,
help=
"maximum number of generations: program stops when reaching this total number of generations: (default 10)"
)
#parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)")
parser.add_option(
"--rand_seed",
type="int",
help=
"random seed set before calculations: useful for testing purposes. By default, total randomness (type int)"
)
parser.add_option("--new",
action="store_true",
default=False,
help="use new code: (default False)")
parser.add_option("--debug",
action="store_true",
default=False,
help="debug info printout: (default False)")
# must be switched off in production
parser.add_option(
"--use_latest_master_directory",
action="store_true",
default=False,
help=
"use latest master directory: when active, the program looks for the latest directory that starts with the word 'master', so the user does not need to provide a directory name. (default False)"
)
parser.add_option(
"--restart_section",
type="string",
default=' ',
help=
"restart section: each generation (iteration) contains three sections: 'restart', 'candidate_class_averages', and 'reproducible_class_averages'. To restart from a particular step, for example, generation 4 and section 'candidate_class_averages' the following option is needed: '--restart_section=candidate_class_averages,4'. The option requires no white space before or after the comma. The default behavior is to restart execution from where it stopped intentionally or unintentionally. For default restart, it is assumed that the name of the directory is provided as argument. Alternatively, the '--use_latest_master_directory' option can be used. (default ' ')"
)
parser.add_option(
"--stop_after_candidates",
action="store_true",
default=False,
help=
"stop after candidates: stops after the 'candidate_class_averages' section. (default False)"
)
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--return_options",
action="store_true",
dest="return_options",
default=False,
help=SUPPRESS_HELP)
parser.add_option(
"--skip_prealignment",
action="store_true",
default=False,
help=
"skip pre-alignment step: to be used if images are already centered. 2dalignment directory will still be generated but the parameters will be zero. (default False)"
)
required_option_list = ['radius']
(options, args) = parser.parse_args(args)
if options.return_options:
return parser
if len(args) > 2:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
sys.exit()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
global_def.BATCH = True
from isac import iter_isac
from fundamentals import rot_shift2D, resample
from utilities import pad, combine_params2
command_line_provided_stack_filename = args[0]
main_node = 0
mpi_init(0, [])
myid = mpi_comm_rank(MPI_COMM_WORLD)
nproc = mpi_comm_size(MPI_COMM_WORLD)
mpi_barrier(MPI_COMM_WORLD)
if (myid == main_node):
print "****************************************************************"
Util.version()
print "****************************************************************"
sys.stdout.flush()
mpi_barrier(MPI_COMM_WORLD)
# Making sure all required options appeared.
for required_option in required_option_list:
if not options.__dict__[required_option]:
print "\n ==%s== mandatory option is missing.\n" % required_option
print "Please run '" + progname + " -h' for detailed options"
return 1
radi = options.radius
target_radius = options.target_radius
target_nx = options.target_nx
center_method = options.center_method
if (radi < 1):
ERROR("Particle radius has to be provided!", "sxisac", 1, myid)
use_latest_master_directory = options.use_latest_master_directory
stop_after_candidates = options.stop_after_candidates
# program_state_stack.restart_location_title_from_command_line = options.restart_section
from utilities import qw
program_state_stack.PROGRAM_STATE_VARIABLES = set(
qw("""
isac_generation
"""))
# create or reuse master directory
masterdir = ""
stack_processed_by_ali2d_base__filename = ""
stack_processed_by_ali2d_base__filename__without_master_dir = ""
error_status = 0
if len(args) == 2:
masterdir = args[1]
elif len(args) == 1:
if use_latest_master_directory:
all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)]
import re
r = re.compile("^master.*$")
all_dirs = filter(r.match, all_dirs)
if len(all_dirs) > 0:
# all_dirs = max(all_dirs, key=os.path.getctime)
masterdir = max(all_dirs, key=os.path.getmtime)
#Create folder for all results or check if there is one created already
if (myid == main_node):
if (masterdir == ""):
timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM,
localtime())
masterdir = "master" + timestring
cmd = "{} {}".format("mkdir", masterdir)
junk = cmdexecute(cmd)
elif not os.path.exists(masterdir):
# os.path.exists(masterdir) does not exist
masterdir = args[1]
cmd = "{} {}".format("mkdir", masterdir)
junk = cmdexecute(cmd)
if (args[0][:4] == "bdb:"): filename = args[0][4:]
else: filename = args[0][:-4]
filename = os.path.basename(filename)
stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join(
masterdir, filename)
stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename
if_error_then_all_processes_exit_program(error_status)
# send masterdir to all processes
masterdir = send_string_to_all(masterdir)
if myid == 0:
if options.restart_section != " ":
if os.path.exists(os.path.join(masterdir,
NAME_OF_JSON_STATE_FILE)):
stored_stack, stored_state = restore_program_stack_and_state(
os.path.join(masterdir, NAME_OF_JSON_STATE_FILE))
import re
if "," in options.restart_section:
parsed_restart_section_option = options.restart_section.split(
",")
stored_state[-1]["location_in_program"] = re.sub(
r"___.*$", "___%s" % parsed_restart_section_option[0],
stored_state[-1]["location_in_program"])
generation_str_format = parsed_restart_section_option[1]
if generation_str_format != "":
isac_generation_from_command_line = int(
generation_str_format)
stored_state[-1][
"isac_generation"] = isac_generation_from_command_line
else:
isac_generation_from_command_line = 1
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = -1
stored_state[-1]["location_in_program"] = re.sub(
r"___.*$", "___%s" % options.restart_section,
stored_state[-1]["location_in_program"])
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
store_program_state(
os.path.join(masterdir, NAME_OF_JSON_STATE_FILE),
stored_state, stored_stack)
else:
print "Please remove the restart_section option from the command line. The program must be started from the beginning."
mpi_finalize()
sys.exit()
else:
isac_generation_from_command_line = -1
program_state_stack(locals(), getframeinfo(currentframe()),
os.path.join(masterdir, NAME_OF_JSON_STATE_FILE))
stack_processed_by_ali2d_base__filename = send_string_to_all(
stack_processed_by_ali2d_base__filename)
stack_processed_by_ali2d_base__filename__without_master_dir = \
send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir)
# previous code 2016-05-05--20-14-12-153
# # PARAMETERS OF THE PROCEDURE
# if( options.xr == -1 ):
# # Default values
# # target_nx = 76
# # target_radius = 29
# target_xr = 1
# else: # nx//2
# # Check below!
# target_xr = options.xr
# # target_nx = 76 + target_xr - 1 # subtract one, which is default
# target_nx += target_xr - 1 # subtract one, which is default
# # target_radius = 29
target_xr = options.xr
target_nx += target_xr - 1 # subtract one, which is default
if (options.yr == -1):
yr = options.xr
else:
yr = options.yr
mpi_barrier(MPI_COMM_WORLD)
# Initialization of stacks
if (myid == main_node):
print "command_line_provided_stack_filename", command_line_provided_stack_filename
number_of_images_in_stack = EMUtil.get_image_count(
command_line_provided_stack_filename)
else:
number_of_images_in_stack = 0
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack,
source_node=main_node)
nxrsteps = 4
init2dir = os.path.join(masterdir, "2dalignment")
# from mpi import mpi_finalize
# mpi_finalize()
# sys.stdout.flush()
# sys.exit()
if not os.path.exists(
os.path.join(init2dir, "Finished_initial_2d_alignment.txt")):
if (myid == 0):
import subprocess
from logger import Logger, BaseLogger_Files
# Create output directory
log2d = Logger(BaseLogger_Files())
log2d.prefix = os.path.join(init2dir)
cmd = "mkdir -p " + log2d.prefix
outcome = subprocess.call(cmd, shell=True)
log2d.prefix += "/"
# outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True)
else:
outcome = 0
log2d = None
if (myid == main_node):
a = get_im(command_line_provided_stack_filename)
nnxo = a.get_xsize()
else:
nnxo = 0
nnxo = bcast_number_to_all(nnxo, source_node=main_node)
image_start, image_end = MPI_start_end(number_of_images_in_stack,
nproc, myid)
if options.skip_prealignment:
params2d = [[0.0, 0.0, 0.0, 0]
for i in xrange(image_start, image_end)]
else:
original_images = EMData.read_images(
command_line_provided_stack_filename,
range(image_start, image_end))
# We assume the target radius will be 29, and xr = 1.
shrink_ratio = float(target_radius) / float(radi)
for im in xrange(len(original_images)):
if (shrink_ratio != 1.0):
original_images[im] = resample(original_images[im],
shrink_ratio)
nx = original_images[0].get_xsize()
# nx = int(nx*shrink_ratio + 0.5)
txrm = (nx - 2 * (target_radius + 1)) // 2
if (txrm < 0):
ERROR(
"ERROR!! Radius of the structure larger than the window data size permits %d"
% (radi), "sxisac", 1, myid)
if (txrm / nxrsteps > 0):
tss = ""
txr = ""
while (txrm / nxrsteps > 0):
tts = txrm / nxrsteps
tss += " %d" % tts
txr += " %d" % (tts * nxrsteps)
txrm = txrm // 2
else:
tss = "1"
txr = "%d" % txrm
# print "nx, txr, txrm, tss", nx, txr, txrm, tss
# from mpi import mpi_finalize
# mpi_finalize()
# sys.stdout.flush()
# sys.exit()
# section ali2d_base
params2d = ali2d_base(original_images, init2dir, None, 1, target_radius, 1, txr, txr, tss, \
False, 90.0, center_method, 14, options.CTF, 1.0, False, \
"ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False)
del original_images
for i in xrange(len(params2d)):
alpha, sx, sy, mirror = combine_params2(
0, params2d[i][1], params2d[i][2], 0, -params2d[i][0], 0,
0, 0)
sx /= shrink_ratio
sy /= shrink_ratio
params2d[i][0] = 0.0
params2d[i][1] = sx
params2d[i][2] = sy
params2d[i][3] = 0
#set_params2D(aligned_images[i],[0.0, sx,sy,0.,1.0])
mpi_barrier(MPI_COMM_WORLD)
tmp = params2d[:]
tmp = wrap_mpi_gatherv(tmp, main_node, MPI_COMM_WORLD)
if (myid == main_node):
if options.skip_prealignment:
print "========================================="
print "Even though there is no alignment step, '%s' params are set to zero for later use." % os.path.join(
init2dir, "initial2Dparams.txt")
print "========================================="
write_text_row(tmp, os.path.join(init2dir, "initial2Dparams.txt"))
del tmp
mpi_barrier(MPI_COMM_WORLD)
# We assume the target image size will be target_nx, radius will be 29, and xr = 1.
# Note images can be also padded, in which case shrink_ratio > 1.
shrink_ratio = float(target_radius) / float(radi)
aligned_images = EMData.read_images(
command_line_provided_stack_filename,
range(image_start, image_end))
nx = aligned_images[0].get_xsize()
nima = len(aligned_images)
newx = int(nx * shrink_ratio + 0.5)
while not os.path.exists(os.path.join(init2dir,
"initial2Dparams.txt")):
import time
time.sleep(1)
mpi_barrier(MPI_COMM_WORLD)
params = read_text_row(os.path.join(init2dir, "initial2Dparams.txt"))
params = params[image_start:image_end]
msk = model_circle(radi, nx, nx)
for im in xrange(nima):
st = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= st[0]
if options.CTF:
aligned_images[im] = filt_ctf(
aligned_images[im],
aligned_images[im].get_attr("ctf"),
binary=True)
if (shrink_ratio < 1.0):
if newx > target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
#aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = resample(aligned_images[im],
shrink_ratio)
aligned_images[im] = Util.window(aligned_images[im],
target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = resample(aligned_images[im],
shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx:
msk = model_circle(newx // 2 - 2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = resample(aligned_images[im],
shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx,
target_nx, 1, 0.0)
elif (shrink_ratio == 1.0):
if newx > target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = Util.window(aligned_images[im],
target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx:
msk = model_circle(newx // 2 - 2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
#aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx,
target_nx, 1, 0.0)
elif (shrink_ratio > 1.0):
if newx > target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = resample(aligned_images[im],
shrink_ratio)
aligned_images[im] = Util.window(aligned_images[im],
target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = resample(aligned_images[im],
shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx:
msk = model_circle(newx // 2 - 2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = resample(aligned_images[im],
shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx,
target_nx, 1, 0.0)
del msk
gather_compacted_EMData_to_root(number_of_images_in_stack,
aligned_images, myid)
number_of_images_in_stack = bcast_number_to_all(
number_of_images_in_stack, source_node=main_node)
if (myid == main_node):
for i in range(number_of_images_in_stack):
aligned_images[i].write_image(
stack_processed_by_ali2d_base__filename, i)
# It has to be explicitly closed
from EMAN2db import db_open_dict
DB = db_open_dict(stack_processed_by_ali2d_base__filename)
DB.close()
fp = open(os.path.join(masterdir, "README_shrink_ratio.txt"), "w")
output_text = """
Since, for processing purposes, isac changes the image dimensions,
adjustment of pixel size needs to be made in subsequent steps, (e.g.
running sxviper.py). The shrink ratio for this particular isac run is
--------
%.5f
%.5f
--------
To get the pixel size for the isac output the user needs to divide
the original pixel size by the above value. This info is saved in
the following file: README_shrink_ratio.txt
""" % (shrink_ratio, radi)
fp.write(output_text)
fp.flush()
fp.close()
print output_text
fp = open(
os.path.join(init2dir, "Finished_initial_2d_alignment.txt"),
"w")
fp.flush()
fp.close()
else:
if (myid == main_node):
print "Skipping 2d alignment since it was already done!"
mpi_barrier(MPI_COMM_WORLD)
# from mpi import mpi_finalize
# mpi_finalize()
# sys.stdout.flush()
# sys.exit()
os.chdir(masterdir)
if program_state_stack(locals(), getframeinfo(currentframe())):
# if 1:
pass
if (myid == main_node):
junk = cmdexecute(
"sxheader.py --consecutive --params=originalid %s" %
stack_processed_by_ali2d_base__filename__without_master_dir)
junk = cmdexecute(
"e2bdb.py %s --makevstack=%s_000" %
(stack_processed_by_ali2d_base__filename__without_master_dir,
stack_processed_by_ali2d_base__filename__without_master_dir))
if (myid == main_node):
main_dir_no = get_latest_directory_increment_value("./",
NAME_OF_MAIN_DIR,
myformat="%04d")
print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no
if isac_generation_from_command_line < 0:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(
NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1][
"isac_generation"]
else:
isac_generation_from_command_line = -1
if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no:
for i in xrange(isac_generation_from_command_line + 1,
main_dir_no + 1):
if i == isac_generation_from_command_line + 1:
backup_dir_no = get_nonexistent_directory_increment_value(
"./", "000_backup", myformat="%05d", start_value=1)
junk = cmdexecute("mkdir -p " + "000_backup" +
"%05d" % backup_dir_no)
junk = cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % i +
" 000_backup" + "%05d" % backup_dir_no)
junk = cmdexecute(
"rm " + "EMAN2DB/" +
stack_processed_by_ali2d_base__filename__without_master_dir[
4:] + "_%03d.bdb" % i)
# it includes both command line and json file
my_restart_section = stored_state[-1]["location_in_program"].split(
"___")[-1]
if "restart" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value(
"./", "000_backup", myformat="%05d", start_value=1)
junk = cmdexecute("mkdir -p " + "000_backup" +
"%05d" % backup_dir_no)
junk = cmdexecute("mv " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation_from_command_line +
" 000_backup" + "%05d" % backup_dir_no)
junk = cmdexecute(
"rm " + "EMAN2DB/" +
stack_processed_by_ali2d_base__filename__without_master_dir[
4:] + "_%03d.bdb" % isac_generation_from_command_line)
elif "candidate_class_averages" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value(
"./", "000_backup", myformat="%05d", start_value=1)
junk = cmdexecute("mkdir -p " + "000_backup" +
"%05d" % backup_dir_no)
junk = cmdexecute("mv " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation_from_command_line +
" 000_backup" + "%05d" % backup_dir_no)
junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation_from_command_line)
# junk = cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line)
elif "reproducible_class_averages" in my_restart_section:
junk = cmdexecute("rm -rf " + NAME_OF_MAIN_DIR +
"%04d/ali_params_generation_*" %
isac_generation_from_command_line)
junk = cmdexecute("rm -f " + NAME_OF_MAIN_DIR +
"%04d/class_averages_generation*" %
isac_generation_from_command_line)
else:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(
NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1][
"isac_generation"]
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 0
isac_generation_from_command_line = mpi_bcast(
isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]
isac_generation = isac_generation_from_command_line - 1
if (myid == main_node):
if isac_generation == 0:
junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation)
write_text_file(
[1],
os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation,
"generation_%d_accounted.txt" % isac_generation))
write_text_file(
range(number_of_images_in_stack),
os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation,
"generation_%d_unaccounted.txt" %
isac_generation))
# Stopping criterion should be inside the program.
while True:
isac_generation += 1
if isac_generation > options.n_generations:
break
data64_stack_current = "bdb:../" + stack_processed_by_ali2d_base__filename__without_master_dir[
4:] + "_%03d" % isac_generation
program_state_stack.restart_location_title = "restart"
if program_state_stack(locals(), getframeinfo(currentframe())):
if (myid == main_node):
junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation)
# reference the original stack
list_file = os.path.join(
NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1),
"generation_%d_unaccounted.txt" % (isac_generation - 1))
junk = cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\
stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file))
mpi_barrier(MPI_COMM_WORLD)
os.chdir(NAME_OF_MAIN_DIR + "%04d" % isac_generation)
program_state_stack.restart_location_title = "candidate_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new)
# program_state_stack.restart_location_title = "stopped_program1"
# program_state_stack(locals(), getframeinfo(currentframe()))
program_state_stack.restart_location_title = "stop_after_candidates"
program_state_stack(locals(), getframeinfo(currentframe()))
if stop_after_candidates:
mpi_finalize()
sys.exit()
exit_program = 0
if (myid == main_node):
if not os.path.exists(
"class_averages_candidate_generation_%d.hdf" %
isac_generation):
print "This generation (%d) no class average candidates were generated! Finishing." % isac_generation
exit_program = 1
exit_program = int(
mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
if exit_program:
os.chdir("..")
break
program_state_stack.restart_location_title = "reproducible_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new)
pass
os.chdir("..")
if (myid == main_node):
accounted_images = read_text_file(
os.path.join(NAME_OF_MAIN_DIR + "%04d" % (isac_generation),
"generation_%d_accounted.txt" %
(isac_generation)))
number_of_accounted_images = len(accounted_images)
un_accounted_images = read_text_file(
os.path.join(
NAME_OF_MAIN_DIR + "%04d" % (isac_generation),
"generation_%d_unaccounted.txt" % (isac_generation)))
number_of_un_accounted_images = len(un_accounted_images)
else:
number_of_accounted_images = 0
number_of_un_accounted_images = 0
number_of_accounted_images = int(
mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0,
MPI_COMM_WORLD)[0])
number_of_un_accounted_images = int(
mpi_bcast(number_of_un_accounted_images, 1, MPI_INT, 0,
MPI_COMM_WORLD)[0])
if number_of_accounted_images == 0:
if (myid == main_node):
print "This generation (%d) there are no accounted images! Finishing." % isac_generation
break
while (myid == main_node):
def files_are_missing(isac_generation):
for i in xrange(1, isac_generation + 1):
if not os.path.exists(
"generation_%04d/class_averages_generation_%d.hdf"
% (i, i)):
print "Error: generation_%04d/class_averages_generation_%d.hdf is missing! Exiting." % (
i, i)
return 1
return 0
if files_are_missing(isac_generation):
break
junk = cmdexecute("rm -f class_averages.hdf")
cpy([
"generation_%04d/class_averages_generation_%d.hdf" % (i, i)
for i in xrange(1, isac_generation + 1)
], "class_averages.hdf")
break
if number_of_un_accounted_images == 0:
if (myid == main_node):
print "This generation (%d) there are no un accounted images! Finishing." % isac_generation
break
program_state_stack(locals(),
getframeinfo(currentframe()),
last_call="__LastCall")
mpi_barrier(MPI_COMM_WORLD)
mpi_finalize()
def main(args):
progname = os.path.basename(sys.argv[0])
usage = ( progname + " stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter" +
" --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp" +
" --generation=generation --rand_seed=rand_seed>" )
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--radius", type="int", help="particle radius: there is no default, a sensible number has to be provided, units - pixels (default required int)")
parser.add_option("--target_radius", type="int", default=29, help="target particle radius: actual particle radius on which isac will process data. Images will be shrinked/enlarged to achieve this radius (default 29)")
parser.add_option("--target_nx", type="int", default=76, help="target particle image size: actual image size on which isac will process data. Images will be shrinked/enlarged according to target particle radius and then cut/padded to achieve target_nx size. When xr > 0, the final image size for isac processing is 'target_nx + xr - 1' (default 76)")
parser.add_option("--img_per_grp", type="int", default=100, help="number of images per class: in the ideal case (essentially maximum size of class) (default 100)")
parser.add_option("--CTF", action="store_true", default=False, help="apply phase-flip for CTF correction: if set the data will be phase-flipped using CTF information included in image headers (default False)")
parser.add_option("--ir", type="int", default=1, help="inner ring: of the resampling to polar coordinates. units - pixels (default 1)")
parser.add_option("--rs", type="int", default=1, help="ring step: of the resampling to polar coordinates. units - pixels (default 1)")
parser.add_option("--xr", type="int", default=1, help="x range: of translational search. By default, set by the program. (default 1)")
parser.add_option("--yr", type="int", default=-1, help="y range: of translational search. By default, same as xr. (default -1)")
parser.add_option("--ts", type="float", default=1.0, help="search step: of translational search: units - pixels (default 1.0)")
parser.add_option("--maxit", type="int", default=30, help="number of iterations for reference-free alignment: (default 30)")
#parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)")
parser.add_option("--center_method", type="int", default=-1, help="method for centering: of global 2D average during initial prealignment of data (0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7) (default -1)")
parser.add_option("--dst", type="float", default=90.0, help="discrete angle used in within group alignment: (default 90.0)")
parser.add_option("--FL", type="float", default=0.2, help="lowest stopband: frequency used in the tangent filter (default 0.2)")
parser.add_option("--FH", type="float", default=0.3, help="highest stopband: frequency used in the tangent filter (default 0.3)")
parser.add_option("--FF", type="float", default=0.2, help="fall-off of the tangent filter: (default 0.2)")
parser.add_option("--init_iter", type="int", default=3, help="SAC initialization iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC initialization (default 3)")
parser.add_option("--main_iter", type="int", default=3, help="SAC main iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC (default 3)")
parser.add_option("--iter_reali", type="int", default=1, help="SAC stability check interval: every iter_reali iterations of SAC stability checking is performed (default 1)")
parser.add_option("--match_first", type="int", default=1, help="number of iterations to run 2-way matching in the first phase: (default 1)")
parser.add_option("--max_round", type="int", default=20, help="maximum rounds: of generating candidate class averages in the first phase (default 20)")
parser.add_option("--match_second", type="int", default=5, help="number of iterations to run 2-way (or 3-way) matching in the second phase: (default 5)")
parser.add_option("--stab_ali", type="int", default=5, help="number of alignments when checking stability: (default 5)")
parser.add_option("--thld_err", type="float", default=0.7, help="threshold of pixel error when checking stability: equals root mean square of distances between corresponding pixels from set of found transformations and theirs average transformation, depends linearly on square of radius (parameter ou). units - pixels. (default 0.7)")
parser.add_option("--indep_run", type="int", default=4, help="level of m-way matching for reproducibility tests: By default, perform full ISAC to 4-way matching. Value indep_run=2 will restrict ISAC to 2-way matching and 3 to 3-way matching. Note the number of used MPI processes requested in mpirun must be a multiplicity of indep_run. (default 4)")
parser.add_option("--thld_grp", type="int", default=10, help="minimum size of reproducible class (default 10)")
parser.add_option("--n_generations", type="int", default=100, help="maximum number of generations: program stops when reaching this total number of generations: (default 100)")
#parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)")
parser.add_option("--rand_seed", type="int", help="random seed set before calculations: useful for testing purposes. By default, total randomness (type int)")
parser.add_option("--new", action="store_true", default=False, help="use new code: (default False)")
parser.add_option("--debug", action="store_true", default=False, help="debug info printout: (default False)")
# must be switched off in production
parser.add_option("--use_latest_master_directory",action="store_true", default=False, help="use latest master directory: when active, the program looks for the latest directory that starts with the word 'master', so the user does not need to provide a directory name. (default False)")
parser.add_option("--restart_section", type="string", default=' ', help="restart section: each generation (iteration) contains three sections: 'restart', 'candidate_class_averages', and 'reproducible_class_averages'. To restart from a particular step, for example, generation 4 and section 'candidate_class_averages' the following option is needed: '--restart_section=candidate_class_averages,4'. The option requires no white space before or after the comma. The default behavior is to restart execution from where it stopped intentionally or unintentionally. For default restart, it is assumed that the name of the directory is provided as argument. Alternatively, the '--use_latest_master_directory' option can be used. (default ' ')")
parser.add_option("--stop_after_candidates", action="store_true", default=False, help="stop after candidates: stops after the 'candidate_class_averages' section. (default False)")
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help = SUPPRESS_HELP)
parser.add_option("--skip_prealignment", action="store_true", default=False, help="skip pre-alignment step: to be used if images are already centered. 2dalignment directory will still be generated but the parameters will be zero. (default False)")
required_option_list = ['radius']
(options, args) = parser.parse_args(args)
if options.return_options:
return parser
if len(args) > 2:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
sys.exit()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
global_def.BATCH = True
from isac import iter_isac
from fundamentals import rot_shift2D, resample
from utilities import pad, combine_params2
command_line_provided_stack_filename = args[0]
main_node = 0
mpi_init(0, [])
myid = mpi_comm_rank(MPI_COMM_WORLD)
nproc = mpi_comm_size(MPI_COMM_WORLD)
mpi_barrier(MPI_COMM_WORLD)
if(myid == main_node):
print "****************************************************************"
Util.version()
print "****************************************************************"
sys.stdout.flush()
mpi_barrier(MPI_COMM_WORLD)
# Making sure all required options appeared.
for required_option in required_option_list:
if not options.__dict__[required_option]:
print "\n ==%s== mandatory option is missing.\n"%required_option
print "Please run '" + progname + " -h' for detailed options"
return 1
radi = options.radius
target_radius = options.target_radius
target_nx = options.target_nx
center_method = options.center_method
if(radi < 1): ERROR("Particle radius has to be provided!","sxisac",1,myid)
use_latest_master_directory = options.use_latest_master_directory
stop_after_candidates = options.stop_after_candidates
# program_state_stack.restart_location_title_from_command_line = options.restart_section
from utilities import qw
program_state_stack.PROGRAM_STATE_VARIABLES = set(qw("""
isac_generation
"""))
# create or reuse master directory
masterdir = ""
stack_processed_by_ali2d_base__filename = ""
stack_processed_by_ali2d_base__filename__without_master_dir = ""
error_status = 0
if len(args) == 2:
masterdir = args[1]
elif len(args) == 1:
if use_latest_master_directory:
all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)]
import re; r = re.compile("^master.*$")
all_dirs = filter(r.match, all_dirs)
if len(all_dirs)>0:
# all_dirs = max(all_dirs, key=os.path.getctime)
masterdir = max(all_dirs, key=os.path.getmtime)
#Create folder for all results or check if there is one created already
if(myid == main_node):
if( masterdir == ""):
timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime())
masterdir = "master"+timestring
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
elif not os.path.exists(masterdir):
# os.path.exists(masterdir) does not exist
masterdir = args[1]
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
if(args[0][:4] == "bdb:"): filename = args[0][4:]
else: filename = args[0][:-4]
filename = os.path.basename(filename)
stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join(masterdir, filename )
stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename
if_error_then_all_processes_exit_program(error_status)
# send masterdir to all processes
masterdir = send_string_to_all(masterdir)
if myid == 0:
if options.restart_section != " ":
if os.path.exists(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)):
stored_stack, stored_state = restore_program_stack_and_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE))
import re
if "," in options.restart_section:
parsed_restart_section_option = options.restart_section.split(",")
stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%parsed_restart_section_option[0], stored_state[-1]["location_in_program"])
generation_str_format = parsed_restart_section_option[1]
if generation_str_format != "":
isac_generation_from_command_line = int(generation_str_format)
stored_state[-1]["isac_generation"] = isac_generation_from_command_line
else:
isac_generation_from_command_line = 1
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = -1
stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%options.restart_section, stored_state[-1]["location_in_program"])
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
store_program_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE), stored_state, stored_stack)
else:
print "Please remove the restart_section option from the command line. The program must be started from the beginning."
mpi_finalize()
sys.exit()
else:
isac_generation_from_command_line = -1
program_state_stack(locals(), getframeinfo(currentframe()), os.path.join(masterdir,NAME_OF_JSON_STATE_FILE))
stack_processed_by_ali2d_base__filename = send_string_to_all(stack_processed_by_ali2d_base__filename)
stack_processed_by_ali2d_base__filename__without_master_dir = \
send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir)
# previous code 2016-05-05--20-14-12-153
# # PARAMETERS OF THE PROCEDURE
# if( options.xr == -1 ):
# # Default values
# # target_nx = 76
# # target_radius = 29
# target_xr = 1
# else: # nx//2
# # Check below!
# target_xr = options.xr
# # target_nx = 76 + target_xr - 1 # subtract one, which is default
# target_nx += target_xr - 1 # subtract one, which is default
# # target_radius = 29
target_xr = options.xr
target_nx += target_xr - 1 # subtract one, which is default
if (options.yr == -1):
yr = options.xr
else:
yr = options.yr
mpi_barrier(MPI_COMM_WORLD)
# Initialization of stacks
if(myid == main_node):
print "command_line_provided_stack_filename", command_line_provided_stack_filename
number_of_images_in_stack = EMUtil.get_image_count(command_line_provided_stack_filename)
else:
number_of_images_in_stack = 0
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node)
nxrsteps = 4
init2dir = os.path.join(masterdir,"2dalignment")
# from mpi import mpi_finalize
# mpi_finalize()
# sys.stdout.flush()
# sys.exit()
if not os.path.exists(os.path.join(init2dir, "Finished_initial_2d_alignment.txt")):
if(myid == 0):
import subprocess
from logger import Logger, BaseLogger_Files
# Create output directory
log2d = Logger(BaseLogger_Files())
log2d.prefix = os.path.join(init2dir)
cmd = "mkdir -p "+log2d.prefix
outcome = subprocess.call(cmd, shell=True)
log2d.prefix += "/"
# outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True)
else:
outcome = 0
log2d = None
if(myid == main_node):
a = get_im(command_line_provided_stack_filename)
nnxo = a.get_xsize()
else:
nnxo = 0
nnxo = bcast_number_to_all(nnxo, source_node = main_node)
image_start, image_end = MPI_start_end(number_of_images_in_stack, nproc, myid)
if options.skip_prealignment:
params2d = [[0.0,0.0,0.0,0] for i in xrange(image_start, image_end)]
else:
original_images = EMData.read_images(command_line_provided_stack_filename, range(image_start,image_end))
# We assume the target radius will be 29, and xr = 1.
shrink_ratio = float(target_radius)/float(radi)
for im in xrange(len(original_images)):
if(shrink_ratio != 1.0):
original_images[im] = resample(original_images[im], shrink_ratio)
nx = original_images[0].get_xsize()
# nx = int(nx*shrink_ratio + 0.5)
txrm = (nx - 2*(target_radius+1))//2
if(txrm < 0): ERROR( "ERROR!! Radius of the structure larger than the window data size permits %d"%(radi), "sxisac",1, myid)
if(txrm/nxrsteps>0):
tss = ""
txr = ""
while(txrm/nxrsteps>0):
tts=txrm/nxrsteps
tss += " %d"%tts
txr += " %d"%(tts*nxrsteps)
txrm =txrm//2
else:
tss = "1"
txr = "%d"%txrm
# print "nx, txr, txrm, tss", nx, txr, txrm, tss
# from mpi import mpi_finalize
# mpi_finalize()
# sys.stdout.flush()
# sys.exit()
# section ali2d_base
params2d = ali2d_base(original_images, init2dir, None, 1, target_radius, 1, txr, txr, tss, \
False, 90.0, center_method, 14, options.CTF, 1.0, False, \
"ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False)
del original_images
for i in xrange(len(params2d)):
alpha, sx, sy, mirror = combine_params2(0, params2d[i][1],params2d[i][2], 0, -params2d[i][0], 0, 0, 0)
sx /= shrink_ratio
sy /= shrink_ratio
params2d[i][0] = 0.0
params2d[i][1] = sx
params2d[i][2] = sy
params2d[i][3] = 0
#set_params2D(aligned_images[i],[0.0, sx,sy,0.,1.0])
mpi_barrier(MPI_COMM_WORLD)
tmp = params2d[:]
tmp = wrap_mpi_gatherv(tmp, main_node, MPI_COMM_WORLD)
if( myid == main_node ):
if options.skip_prealignment:
print "========================================="
print "Even though there is no alignment step, '%s' params are set to zero for later use."%os.path.join(init2dir, "initial2Dparams.txt")
print "========================================="
write_text_row(tmp,os.path.join(init2dir, "initial2Dparams.txt"))
del tmp
mpi_barrier(MPI_COMM_WORLD)
# We assume the target image size will be target_nx, radius will be 29, and xr = 1.
# Note images can be also padded, in which case shrink_ratio > 1.
shrink_ratio = float(target_radius)/float(radi)
aligned_images = EMData.read_images(command_line_provided_stack_filename, range(image_start,image_end))
nx = aligned_images[0].get_xsize()
nima = len(aligned_images)
newx = int(nx*shrink_ratio + 0.5)
while not os.path.exists(os.path.join(init2dir, "initial2Dparams.txt")):
import time
time.sleep(1)
mpi_barrier(MPI_COMM_WORLD)
params = read_text_row(os.path.join(init2dir, "initial2Dparams.txt"))
params = params[image_start:image_end]
msk = model_circle(radi, nx, nx)
for im in xrange(nima):
st = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= st[0]
if options.CTF:
aligned_images[im] = filt_ctf(aligned_images[im], aligned_images[im].get_attr("ctf"), binary = True)
if(shrink_ratio < 1.0):
if newx > target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
#aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx :
msk = model_circle(newx//2-2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
elif(shrink_ratio == 1.0):
if newx > target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx :
msk = model_circle(newx//2-2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
#aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
elif(shrink_ratio > 1.0):
if newx > target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx :
msk = model_circle(newx//2-2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
del msk
gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images, myid)
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node)
if( myid == main_node ):
for i in range(number_of_images_in_stack): aligned_images[i].write_image(stack_processed_by_ali2d_base__filename,i)
# It has to be explicitly closed
from EMAN2db import db_open_dict
DB = db_open_dict(stack_processed_by_ali2d_base__filename)
DB.close()
fp = open(os.path.join(masterdir,"README_shrink_ratio.txt"), "w")
output_text = """
Since, for processing purposes, isac changes the image dimensions,
adjustment of pixel size needs to be made in subsequent steps, (e.g.
running sxviper.py). The shrink ratio for this particular isac run is
--------
%.5f
%.5f
--------
To get the pixel size for the isac output the user needs to divide
the original pixel size by the above value. This info is saved in
the following file: README_shrink_ratio.txt
"""%(shrink_ratio, radi)
fp.write(output_text); fp.flush() ;fp.close()
print output_text
fp = open(os.path.join(init2dir, "Finished_initial_2d_alignment.txt"), "w"); fp.flush() ;fp.close()
else:
if( myid == main_node ):
print "Skipping 2d alignment since it was already done!"
mpi_barrier(MPI_COMM_WORLD)
# from mpi import mpi_finalize
# mpi_finalize()
# sys.stdout.flush()
# sys.exit()
os.chdir(masterdir)
if program_state_stack(locals(), getframeinfo(currentframe())):
# if 1:
pass
if (myid == main_node):
cmdexecute("sxheader.py --consecutive --params=originalid %s"%stack_processed_by_ali2d_base__filename__without_master_dir)
cmdexecute("e2bdb.py %s --makevstack=%s_000"%(stack_processed_by_ali2d_base__filename__without_master_dir, stack_processed_by_ali2d_base__filename__without_master_dir))
if (myid == main_node):
main_dir_no = get_latest_directory_increment_value("./", NAME_OF_MAIN_DIR, myformat="%04d")
print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no
if isac_generation_from_command_line < 0:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = -1
if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no:
for i in xrange(isac_generation_from_command_line+1, main_dir_no + 1):
if i == isac_generation_from_command_line+1:
backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%i + " 000_backup" + "%05d"%backup_dir_no)
cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%i)
# it includes both command line and json file
my_restart_section = stored_state[-1]["location_in_program"].split("___")[-1]
if "restart" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no)
cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%isac_generation_from_command_line )
elif "candidate_class_averages" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no)
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line)
# cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line)
elif "reproducible_class_averages" in my_restart_section:
cmdexecute("rm -rf " + NAME_OF_MAIN_DIR + "%04d/ali_params_generation_*"%isac_generation_from_command_line)
cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_generation*"%isac_generation_from_command_line)
else:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 0
isac_generation_from_command_line = mpi_bcast(isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]
isac_generation = isac_generation_from_command_line - 1
if (myid == main_node):
if isac_generation == 0:
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation)
write_text_file([1], os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_accounted.txt"%isac_generation))
write_text_file(range(number_of_images_in_stack), os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_unaccounted.txt"%isac_generation))
# Stopping criterion should be inside the program.
while True:
isac_generation += 1
if isac_generation > options.n_generations:
break
data64_stack_current = "bdb:../"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d"%isac_generation
program_state_stack.restart_location_title = "restart"
if program_state_stack(locals(), getframeinfo(currentframe())):
if (myid == main_node):
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation)
# reference the original stack
list_file = os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1), "generation_%d_unaccounted.txt"%(isac_generation - 1))
cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\
stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file))
mpi_barrier(MPI_COMM_WORLD)
os.chdir(NAME_OF_MAIN_DIR + "%04d"%isac_generation)
program_state_stack.restart_location_title = "candidate_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new)
# program_state_stack.restart_location_title = "stopped_program1"
# program_state_stack(locals(), getframeinfo(currentframe()))
program_state_stack.restart_location_title = "stop_after_candidates"
program_state_stack(locals(), getframeinfo(currentframe()))
if stop_after_candidates:
mpi_finalize()
sys.exit()
exit_program = 0
if(myid == main_node):
if not os.path.exists("class_averages_candidate_generation_%d.hdf"%isac_generation):
print "This generation (%d) no class average candidates were generated! Finishing."%isac_generation
exit_program = 1
exit_program = int(mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
if exit_program:
os.chdir("..")
break
program_state_stack.restart_location_title = "reproducible_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new)
pass
os.chdir("..")
if(myid == main_node):
accounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation),"generation_%d_accounted.txt"%(isac_generation)))
number_of_accounted_images = len(accounted_images)
un_accounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation),"generation_%d_unaccounted.txt"%(isac_generation)))
number_of_un_accounted_images = len(un_accounted_images)
else:
number_of_accounted_images = 0
number_of_un_accounted_images = 0
number_of_accounted_images = int(mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
number_of_un_accounted_images = int(mpi_bcast(number_of_un_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
if number_of_accounted_images == 0:
if(myid == main_node):
print "This generation (%d) there are no accounted images! Finishing."%isac_generation
break
while (myid == main_node):
def files_are_missing(isac_generation):
for i in xrange(1, isac_generation + 1):
if not os.path.exists("generation_%04d/class_averages_generation_%d.hdf"%(i,i)):
print "Error: generation_%04d/class_averages_generation_%d.hdf is missing! Exiting."%(i,i)
return 1
return 0
if files_are_missing(isac_generation):
break
cmdexecute("rm -f class_averages.hdf")
cpy(["generation_%04d/class_averages_generation_%d.hdf"%(i,i) for i in xrange(1, isac_generation + 1)], "class_averages.hdf")
break
if number_of_un_accounted_images == 0:
if(myid == main_node):
print "This generation (%d) there are no un accounted images! Finishing."%isac_generation
break
program_state_stack(locals(), getframeinfo(currentframe()), last_call="__LastCall")
mpi_barrier(MPI_COMM_WORLD)
mpi_finalize()
def isac_substack(args):
from utilities import get_im, write_text_file
from EMAN2db import db_open_dict
from e2bdb import makerelpath
# To make the execution exit upon fatal error by ERROR in global_def.py
global_def.BATCH = True
# Check error conditions
subcommand_name = 'isac_substack'
if not os.path.exists(args.input_isac_class_avgs_path):
ERROR('Input ISAC class average stack file does not exist. Please check the file path and restart the program.', subcommand_name) # action=1 - fatal error, exit
if os.path.exists(args.output_directory):
ERROR('Output directory exists. Please change the name and restart the program.', subcommand_name) # action=1 - fatal error, exit
assert(os.path.exists(args.input_isac_class_avgs_path))
assert(not os.path.exists(args.output_directory))
# Create output directory
os.mkdir(args.output_directory)
# Retrieve original particle IDs of member particles listed in ISAC class average stack
n_img_processed = EMUtil.get_image_count(args.input_isac_class_avgs_path)
isac_substack_particle_id_list = []
for i_img in xrange(n_img_processed):
isac_substack_particle_id_list += get_im(args.input_isac_class_avgs_path, i_img).get_attr('members')
isac_substack_particle_id_list.sort()
# Save the substack particle id list
isac_substack_particle_id_list_file_path = os.path.join(args.output_directory, 'isac_substack_particle_id_list.txt')
write_text_file(isac_substack_particle_id_list, isac_substack_particle_id_list_file_path)
# Open the output BDB dictionary
assert(args.output_directory != '')
output_virtual_bdb_stack_real_path = 'bdb:%s#isac_substack' % args.output_directory
output_virtual_bdb_stack = db_open_dict(output_virtual_bdb_stack_real_path)
# Convert an absolute path to the actual output data to a relative path by eliminating any symbolic links
output_virtual_bdb_stack_real_path=os.path.realpath(output_virtual_bdb_stack.path)+"/"
# Open the input BDB dictionary
input_bdb_stack = db_open_dict(args.input_bdb_stack_path, ro=True) # Read only
# Copy the header from input to output BDB dictionary
n_img_detected = len(isac_substack_particle_id_list)
print_progress("Detected %d ISAC validated particles in %s"%(n_img_detected, args.input_isac_class_avgs_path))
print(" ")
# Loop through all ISAC validated particles
n_img_processed = 0
for i_img_detected, isac_substack_particle_id in enumerate(isac_substack_particle_id_list):
# Print progress
if i_img_detected % 1000 == 0:
try:
print_progress("Progress %5.2f%%: Processing %6dth entry (Particle ID %6d)."%(float(i_img_detected)/n_img_detected*100.0, i_img_detected, isac_substack_particle_id))
sys.stdout.flush()
except:
pass
# Read a particle image header from input bdb stack
try:
img_header = input_bdb_stack.get(isac_substack_particle_id, nodata=1).get_attr_dict() # Need only header information
except:
ERROR('Failed to read image header of particle #%d from %s. Skipping this image...' % (isac_substack_particle_id, args.input_bdb_stack_path), subcommand_name, action = 0) # action = 0 - non-fatal, print a warning;
continue
# Convert an absolute path to the actual input data to a relative path by eliminating any symbolic links
try:
input_bdb_stack_real_path = os.path.realpath(input_bdb_stack.get_data_path(isac_substack_particle_id))
# Conver the path to OS specific format
if os.name == 'nt':
output_virtual_bdb_stack_real_path = output_virtual_bdb_stack_real_path.replace("\\", '/')
input_bdb_stack_real_path = input_bdb_stack_real_path.replace('\\', '/')
# Takes a pair of paths /a/b/c/d and /a/b/e/f/g and returns a relative path to b from a, ../../e/f/g
common_relative_path = makerelpath(output_virtual_bdb_stack_real_path, input_bdb_stack_real_path)
except:
ERROR('Failure to find common relative data path for particle image #%d. Skipping this image...' % (isac_substack_particle_id), subcommand_name, action = 0) # action = 0 - non-fatal, print a warning;
continue
assert(img_header["data_path"] != None)
# Update the image header for output
img_header["data_path"] = common_relative_path
img_header["data_n"] = isac_substack_particle_id
img_header["data_source"] = args.input_bdb_stack_path
# Register the image header to output virtual bdb stack
output_virtual_bdb_stack[n_img_processed] = img_header
# Increment process image counts
n_img_processed += 1
# Close input and output bdb stacks
output_virtual_bdb_stack.close()
input_bdb_stack.close()
# Print summary of processing
print(" ")
print_progress("Summary of processing...")
print_progress("Detected : %6d"%(n_img_detected))
print_progress("Processed : %6d"%(n_img_processed))
print(" ")
def main(args):
progname = os.path.basename(sys.argv[0])
usage = (
progname +
" stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter"
+
" --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp"
+ " --generation=generation --rand_seed=rand_seed>")
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--radius",
type="int",
default=-1,
help="<Particle radius>, it has to be provided.")
parser.add_option(
"--img_per_grp",
type="int",
default=100,
help=
"<number of images per group> in the ideal case (essentially maximum size of class) (100)"
)
parser.add_option("--CTF",
action="store_true",
default=False,
help="<CTF flag>, if set the data will be phase-flipped")
parser.add_option(
"--ir",
type="int",
default=1,
help="<inner ring> of the resampling to polar coordinates (1)")
parser.add_option(
"--rs",
type="int",
default=1,
help="<ring step> of the resampling to polar coordinates (1)")
parser.add_option(
"--xr",
type="int",
default=-1,
help=
"<x range> of translational search (By default set by the program) (advanced)"
)
parser.add_option(
"--yr",
type="int",
default=-1,
help="<y range> of translational search (same as xr) (advanced)")
parser.add_option("--ts",
type="float",
default=1.0,
help="<search step> of translational search (1.0)")
parser.add_option(
"--maxit",
type="int",
default=30,
help="number of iterations for reference-free alignment (30)")
#parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)")
parser.add_option(
"--center_method",
type="int",
default=7,
help=
"<Method for centering> of global 2D average during initial prealignment of data (default : 7; 0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7)"
)
parser.add_option("--dst",
type="float",
default=90.0,
help="discrete angle used in within group alignment ")
parser.add_option(
"--FL",
type="float",
default=0.2,
help="<lowest stopband> frequency used in the tangent filter (0.2)")
parser.add_option(
"--FH",
type="float",
default=0.3,
help="<highest stopband> frequency used in the tangent filter (0.3)")
parser.add_option("--FF",
type="float",
default=0.2,
help="<fall-off of the tangent> filter (0.2)")
parser.add_option(
"--init_iter",
type="int",
default=3,
help=
"<init_iter> number of iterations of ISAC program in initialization (3)"
)
parser.add_option(
"--main_iter",
type="int",
default=3,
help="<main_iter> number of iterations of ISAC program in main part (3)"
)
parser.add_option(
"--iter_reali",
type="int",
default=1,
help=
"<iter_reali> number of iterations in ISAC before checking stability (1)"
)
parser.add_option(
"--match_first",
type="int",
default=1,
help="number of iterations to run 2-way matching in the first phase (1)"
)
parser.add_option(
"--max_round",
type="int",
default=20,
help=
"maximum rounds of generating candidate averages in the first phase (20)"
)
parser.add_option(
"--match_second",
type="int",
default=5,
help=
"number of iterations to run 2-way (or 3-way) matching in the second phase (5)"
)
parser.add_option("--stab_ali",
type="int",
default=5,
help="number of alignments when checking stability (5)")
parser.add_option(
"--thld_err",
type="float",
default=0.7,
help="the threshold of pixel error when checking stability (0.7)")
parser.add_option(
"--indep_run",
type="int",
default=4,
help=
"number of independent runs for reproducibility (default=4, only values 2, 3 and 4 are supported (4)"
)
parser.add_option("--thld_grp",
type="int",
default=10,
help="minimum size of class (10)")
parser.add_option(
"--n_generations",
type="int",
default=100,
help=
"<n_generations> program stops when reaching this total number of generations (advanced)"
)
#parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)")
parser.add_option(
"--rand_seed",
type="int",
default=None,
help=
"random seed set before calculations, useful for testing purposes (default None - total randomness)"
)
parser.add_option("--new",
action="store_true",
default=False,
help="use new code (default = False)")
parser.add_option("--debug",
action="store_true",
default=False,
help="debug info printout (default = False)")
# must be switched off in production
parser.add_option("--use_latest_master_directory",
action="store_true",
dest="use_latest_master_directory",
default=False)
parser.add_option(
"--restart_section",
type="string",
default="",
help=
"<restart section name> (no spaces) followed immediately by comma, followed immediately by generation to restart, example: \n--restart_section=candidate_class_averages,1 (Sections: restart, candidate_class_averages, reproducible_class_averages)"
)
parser.add_option(
"--stop_after_candidates",
action="store_true",
default=False,
help=
"<stop_after_candidates> stops after the 'candidate_class_averages' section"
)
parser.add_option("--return_options",
action="store_true",
dest="return_options",
default=False,
help=SUPPRESS_HELP)
(options, args) = parser.parse_args(args)
if options.return_options:
return parser
if len(args) > 2:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
sys.exit()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
from isac import iter_isac
global_def.BATCH = True
global_def.BATCH = True
command_line_provided_stack_filename = args[0]
global_def.BATCH = True
main_node = 0
mpi_init(0, [])
myid = mpi_comm_rank(MPI_COMM_WORLD)
nproc = mpi_comm_size(MPI_COMM_WORLD)
radi = options.radius
center_method = options.center_method
if (radi < 1):
ERROR("Particle radius has to be provided!", "sxisac", 1, myid)
use_latest_master_directory = options.use_latest_master_directory
stop_after_candidates = options.stop_after_candidates
program_state_stack.restart_location_title_from_command_line = options.restart_section
from utilities import qw
program_state_stack.PROGRAM_STATE_VARIABLES = set(
qw("""
isac_generation
"""))
# create or reuse master directory
masterdir = ""
stack_processed_by_ali2d_base__filename = ""
stack_processed_by_ali2d_base__filename__without_master_dir = ""
error_status = 0
if len(args) == 2:
masterdir = args[1]
elif len(args) == 1:
if use_latest_master_directory:
all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)]
import re
r = re.compile("^master.*$")
all_dirs = filter(r.match, all_dirs)
if len(all_dirs) > 0:
# all_dirs = max(all_dirs, key=os.path.getctime)
masterdir = max(all_dirs, key=os.path.getmtime)
#Create folder for all results or check if there is one created already
if (myid == main_node):
if (masterdir == ""):
timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM,
localtime())
masterdir = "master" + timestring
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
elif not os.path.exists(masterdir):
# os.path.exists(masterdir) does not exist
masterdir = args[1]
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
if (args[0][:4] == "bdb:"): filename = args[0][4:]
else: filename = args[0][:-4]
filename = os.path.basename(filename)
stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join(
masterdir, filename)
stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename
if_error_all_processes_quit_program(error_status)
# send masterdir to all processes
masterdir = send_string_to_all(masterdir)
if myid == 0:
if options.restart_section != "":
if os.path.exists(os.path.join(masterdir,
NAME_OF_JSON_STATE_FILE)):
stored_stack, stored_state = restore_program_stack_and_state(
os.path.join(masterdir, NAME_OF_JSON_STATE_FILE))
import re
if "," in options.restart_section:
parsed_restart_section_option = options.restart_section.split(
",")
stored_state[-1]["location_in_program"] = re.sub(
r"___.*$", "___%s" % parsed_restart_section_option[0],
stored_state[-1]["location_in_program"])
generation_str_format = parsed_restart_section_option[1]
if generation_str_format != "":
isac_generation_from_command_line = int(
generation_str_format)
stored_state[-1][
"isac_generation"] = isac_generation_from_command_line
else:
isac_generation_from_command_line = 1
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = -1
stored_state[-1]["location_in_program"] = re.sub(
r"___.*$", "___%s" % options.restart_section,
stored_state[-1]["location_in_program"])
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
store_program_state(
os.path.join(masterdir, NAME_OF_JSON_STATE_FILE),
stored_state, stored_stack)
else:
print "Please remove the restart_section option from the command line. The program must be started from the beginning."
mpi_finalize()
sys.exit()
else:
isac_generation_from_command_line = -1
program_state_stack(locals(), getframeinfo(currentframe()),
os.path.join(masterdir, NAME_OF_JSON_STATE_FILE))
stack_processed_by_ali2d_base__filename = send_string_to_all(
stack_processed_by_ali2d_base__filename)
stack_processed_by_ali2d_base__filename__without_master_dir = \
send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir)
# PARAMETERS OF THE PROCEDURE
if (options.xr == -1):
# Default values
target_nx = 76
target_radius = 29
target_xr = 1
else: # nx//2
# Check below!
target_xr = options.xr
target_nx = 76 + target_xr - 1 # subtract one, which is default
target_radius = 29
mpi_barrier(MPI_COMM_WORLD)
# Initialization of stacks
if (myid == main_node):
number_of_images_in_stack = EMUtil.get_image_count(
command_line_provided_stack_filename)
else:
number_of_images_in_stack = 0
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack,
source_node=main_node)
nxrsteps = 4
init2dir = os.path.join(masterdir, "2dalignment")
if (myid == 0):
import subprocess
from logger import Logger, BaseLogger_Files
# Create output directory
log2d = Logger(BaseLogger_Files())
log2d.prefix = os.path.join(init2dir)
cmd = "mkdir -p " + log2d.prefix
outcome = subprocess.call(cmd, shell=True)
log2d.prefix += "/"
# outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True)
else:
outcome = 0
log2d = None
if (myid == main_node):
a = get_im(command_line_provided_stack_filename)
nnxo = a.get_xsize()
else:
nnxo = 0
nnxo = bcast_number_to_all(nnxo, source_node=main_node)
txrm = (nnxo - 2 * (radi + 1)) // 2
if (txrm < 0):
ERROR(
"ERROR!! Radius of the structure larger than the window data size permits %d"
% (radi), "sxisac", 1, myid)
if (txrm / nxrsteps > 0):
tss = ""
txr = ""
while (txrm / nxrsteps > 0):
tts = txrm / nxrsteps
tss += " %d" % tts
txr += " %d" % (tts * nxrsteps)
txrm = txrm // 2
else:
tss = "1"
txr = "%d" % txrm
# section ali2d_base
# centering method is set to #7
params2d, aligned_images = ali2d_base(command_line_provided_stack_filename, init2dir, None, 1, radi, 1, txr, txr, tss, \
False, 90.0, center_method, 14, options.CTF, 1.0, False, \
"ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False)
if (myid == main_node):
write_text_row(params2d, os.path.join(init2dir, "initial2Dparams.txt"))
del params2d
mpi_barrier(MPI_COMM_WORLD)
# We assume the target image size will be target_nx, radius will be 29, and xr = 1.
# Note images can be also padded, in which case shrink_ratio > 1.
shrink_ratio = float(target_radius) / float(radi)
nx = aligned_images[0].get_xsize()
nima = len(aligned_images)
newx = int(nx * shrink_ratio + 0.5)
from fundamentals import rot_shift2D, resample
from utilities import pad, combine_params2
if (shrink_ratio < 1.0):
if newx > target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
aligned_images[im] = Util.window(aligned_images[im], target_nx,
target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx:
msk = model_circle(nx // 2 - 2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx,
target_nx, 1, 0.0)
elif (shrink_ratio == 1.0):
if newx > target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
aligned_images[im] = Util.window(aligned_images[im], target_nx,
target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx:
msk = model_circle(nx // 2 - 2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx,
target_nx, 1, 0.0)
elif (shrink_ratio > 1.0):
if newx > target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
aligned_images[im] = Util.window(aligned_images[im], target_nx,
target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx:
msk = model_circle(target_radius, nx, nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx,
target_nx, 1, 0.0)
del msk
gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images,
myid)
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack,
source_node=main_node)
if (myid == main_node):
for i in range(number_of_images_in_stack):
aligned_images[i].write_image(
stack_processed_by_ali2d_base__filename, i)
# It has to be explicitly closed
from EMAN2db import db_open_dict
DB = db_open_dict(stack_processed_by_ali2d_base__filename)
DB.close()
mpi_barrier(MPI_COMM_WORLD)
global_def.BATCH = True
os.chdir(masterdir)
if program_state_stack(locals(), getframeinfo(currentframe())):
# if 1:
pass
if (myid == main_node):
cmdexecute(
"sxheader.py --consecutive --params=originalid %s" %
stack_processed_by_ali2d_base__filename__without_master_dir)
cmdexecute(
"e2bdb.py %s --makevstack=%s_000" %
(stack_processed_by_ali2d_base__filename__without_master_dir,
stack_processed_by_ali2d_base__filename__without_master_dir))
if (myid == main_node):
main_dir_no = get_latest_directory_increment_value("./",
NAME_OF_MAIN_DIR,
myformat="%04d")
print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no
if isac_generation_from_command_line < 0:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(
NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1][
"isac_generation"]
else:
isac_generation_from_command_line = -1
if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no:
for i in xrange(isac_generation_from_command_line + 1,
main_dir_no + 1):
if i == isac_generation_from_command_line + 1:
backup_dir_no = get_nonexistent_directory_increment_value(
"./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" +
"%05d" % backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % i +
" 000_backup" + "%05d" % backup_dir_no)
cmdexecute(
"rm " + "EMAN2DB/" +
stack_processed_by_ali2d_base__filename__without_master_dir[
4:] + "_%03d.bdb" % i)
# it includes both command line and json file
my_restart_section = stored_state[-1]["location_in_program"].split(
"___")[-1]
if "restart" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value(
"./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" +
"%05d" % backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation_from_command_line +
" 000_backup" + "%05d" % backup_dir_no)
cmdexecute(
"rm " + "EMAN2DB/" +
stack_processed_by_ali2d_base__filename__without_master_dir[
4:] + "_%03d.bdb" % isac_generation_from_command_line)
elif "candidate_class_averages" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value(
"./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" +
"%05d" % backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation_from_command_line +
" 000_backup" + "%05d" % backup_dir_no)
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation_from_command_line)
# cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line)
elif "reproducible_class_averages" in my_restart_section:
cmdexecute("rm -rf " + NAME_OF_MAIN_DIR +
"%04d/ali_params_generation_*" %
isac_generation_from_command_line)
cmdexecute("rm -f " + NAME_OF_MAIN_DIR +
"%04d/class_averages_generation*" %
isac_generation_from_command_line)
else:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(
NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1][
"isac_generation"]
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 0
isac_generation_from_command_line = mpi_bcast(
isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]
isac_generation = isac_generation_from_command_line - 1
if (myid == main_node):
if isac_generation == 0:
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation)
write_text_file(
[1],
os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation,
"generation_%d_accounted.txt" % isac_generation))
write_text_file(
range(number_of_images_in_stack),
os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation,
"generation_%d_unaccounted.txt" %
isac_generation))
# Stopping criterion should be inside the program.
while True:
isac_generation += 1
if isac_generation > options.n_generations:
break
data64_stack_current = "bdb:../" + stack_processed_by_ali2d_base__filename__without_master_dir[
4:] + "_%03d" % isac_generation
if (myid == main_node):
accounted_images = read_text_file(
os.path.join(
NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1),
"generation_%d_accounted.txt" % (isac_generation - 1)))
number_of_accounted_images = len(accounted_images)
# unaccounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1),"generation_%d_unaccounted.txt"%(isac_generation - 1)))
# number_of_unaccounted_images = len(unaccounted_images)
else:
number_of_accounted_images = 0
number_of_accounted_images = int(
mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0,
MPI_COMM_WORLD)[0])
if number_of_accounted_images == 0:
os.chdir("..")
break
program_state_stack.restart_location_title = "restart"
if program_state_stack(locals(), getframeinfo(currentframe())):
if (myid == main_node):
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation)
# reference the original stack
list_file = os.path.join(
NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1),
"generation_%d_unaccounted.txt" % (isac_generation - 1))
cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\
stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file))
mpi_barrier(MPI_COMM_WORLD)
os.chdir(NAME_OF_MAIN_DIR + "%04d" % isac_generation)
program_state_stack.restart_location_title = "candidate_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new)
# program_state_stack.restart_location_title = "stopped_program1"
# program_state_stack(locals(), getframeinfo(currentframe()))
program_state_stack.restart_location_title = "stop_after_candidates"
program_state_stack(locals(), getframeinfo(currentframe()))
if stop_after_candidates:
mpi_finalize()
sys.exit()
exit_program = 0
if (myid == main_node):
if not os.path.exists(
"class_averages_candidate_generation_%d.hdf" %
isac_generation):
print "This generation (%d) no class averages were generated!" % isac_generation
exit_program = 1
exit_program = int(
mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
if exit_program:
os.chdir("..")
break
program_state_stack.restart_location_title = "reproducible_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new)
pass
os.chdir("..")
if (myid == main_node):
cmdexecute("rm -f class_averages.hdf")
cpy([
"generation_%04d/class_averages_generation_%d.hdf" % (i, i)
for i in xrange(1, isac_generation)
], "class_averages.hdf")
# program_state_stack.restart_location_title = "stopped_program2"
# program_state_stack(locals(), getframeinfo(currentframe()))
program_state_stack(locals(),
getframeinfo(currentframe()),
last_call="__LastCall")
mpi_finalize()
def shiftali_MPI(stack,
maskfile=None,
maxit=100,
CTF=False,
snr=1.0,
Fourvar=False,
search_rng=-1,
oneDx=False,
search_rng_y=-1):
number_of_proc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
main_node = 0
ftp = file_type(stack)
if myid == main_node:
print_begin_msg("shiftali_MPI")
max_iter = int(maxit)
if myid == main_node:
if ftp == "bdb":
from EMAN2db import db_open_dict
dummy = db_open_dict(stack, True)
nima = EMUtil.get_image_count(stack)
else:
nima = 0
nima = bcast_number_to_all(nima, source_node=main_node)
list_of_particles = list(range(nima))
image_start, image_end = MPI_start_end(nima, number_of_proc, myid)
list_of_particles = list_of_particles[image_start:image_end]
# read nx and ctf_app (if CTF) and broadcast to all nodes
if myid == main_node:
ima = EMData()
ima.read_image(stack, list_of_particles[0], True)
nx = ima.get_xsize()
ny = ima.get_ysize()
if CTF: ctf_app = ima.get_attr_default('ctf_applied', 2)
del ima
else:
nx = 0
ny = 0
if CTF: ctf_app = 0
nx = bcast_number_to_all(nx, source_node=main_node)
ny = bcast_number_to_all(ny, source_node=main_node)
if CTF:
ctf_app = bcast_number_to_all(ctf_app, source_node=main_node)
if ctf_app > 0:
ERROR("data cannot be ctf-applied", myid=myid)
if maskfile == None:
mrad = min(nx, ny)
mask = model_circle(mrad // 2 - 2, nx, ny)
else:
mask = get_im(maskfile)
if CTF:
from sp_filter import filt_ctf
from sp_morphology import ctf_img
ctf_abs_sum = EMData(nx, ny, 1, False)
ctf_2_sum = EMData(nx, ny, 1, False)
else:
ctf_2_sum = None
from sp_global_def import CACHE_DISABLE
if CACHE_DISABLE:
data = EMData.read_images(stack, list_of_particles)
else:
for i in range(number_of_proc):
if myid == i:
data = EMData.read_images(stack, list_of_particles)
if ftp == "bdb": mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
for im in range(len(data)):
data[im].set_attr('ID', list_of_particles[im])
st = Util.infomask(data[im], mask, False)
data[im] -= st[0]
if CTF:
ctf_params = data[im].get_attr("ctf")
ctfimg = ctf_img(nx, ctf_params, ny=ny)
Util.add_img2(ctf_2_sum, ctfimg)
Util.add_img_abs(ctf_abs_sum, ctfimg)
if CTF:
reduce_EMData_to_root(ctf_2_sum, myid, main_node)
reduce_EMData_to_root(ctf_abs_sum, myid, main_node)
else:
ctf_2_sum = None
if CTF:
if myid != main_node:
del ctf_2_sum
del ctf_abs_sum
else:
temp = EMData(nx, ny, 1, False)
for i in range(0, nx, 2):
for j in range(ny):
temp.set_value_at(i, j, snr)
Util.add_img(ctf_2_sum, temp)
del temp
total_iter = 0
# apply initial xform.align2d parameters stored in header
init_params = []
for im in range(len(data)):
t = data[im].get_attr('xform.align2d')
init_params.append(t)
p = t.get_params("2d")
data[im] = rot_shift2D(data[im],
p['alpha'],
sx=p['tx'],
sy=p['ty'],
mirror=p['mirror'],
scale=p['scale'])
# fourier transform all images, and apply ctf if CTF
for im in range(len(data)):
if CTF:
ctf_params = data[im].get_attr("ctf")
data[im] = filt_ctf(fft(data[im]), ctf_params)
else:
data[im] = fft(data[im])
sx_sum = 0
sy_sum = 0
sx_sum_total = 0
sy_sum_total = 0
shift_x = [0.0] * len(data)
shift_y = [0.0] * len(data)
ishift_x = [0.0] * len(data)
ishift_y = [0.0] * len(data)
for Iter in range(max_iter):
if myid == main_node:
start_time = time()
print_msg("Iteration #%4d\n" % (total_iter))
total_iter += 1
avg = EMData(nx, ny, 1, False)
for im in data:
Util.add_img(avg, im)
reduce_EMData_to_root(avg, myid, main_node)
if myid == main_node:
if CTF:
tavg = Util.divn_filter(avg, ctf_2_sum)
else:
tavg = Util.mult_scalar(avg, 1.0 / float(nima))
else:
tavg = EMData(nx, ny, 1, False)
if Fourvar:
bcast_EMData_to_all(tavg, myid, main_node)
vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF)
if myid == main_node:
if Fourvar:
tavg = fft(Util.divn_img(fft(tavg), vav))
vav_r = Util.pack_complex_to_real(vav)
# normalize and mask tavg in real space
tavg = fft(tavg)
stat = Util.infomask(tavg, mask, False)
tavg -= stat[0]
Util.mul_img(tavg, mask)
# For testing purposes: shift tavg to some random place and see if the centering is still correct
#tavg = rot_shift3D(tavg,sx=3,sy=-4)
tavg = fft(tavg)
if Fourvar: del vav
bcast_EMData_to_all(tavg, myid, main_node)
sx_sum = 0
sy_sum = 0
if search_rng > 0: nwx = 2 * search_rng + 1
else: nwx = nx
if search_rng_y > 0: nwy = 2 * search_rng_y + 1
else: nwy = ny
not_zero = 0
for im in range(len(data)):
if oneDx:
ctx = Util.window(ccf(data[im], tavg), nwx, 1)
p1 = peak_search(ctx)
p1_x = -int(p1[0][3])
ishift_x[im] = p1_x
sx_sum += p1_x
else:
p1 = peak_search(Util.window(ccf(data[im], tavg), nwx, nwy))
p1_x = -int(p1[0][4])
p1_y = -int(p1[0][5])
ishift_x[im] = p1_x
ishift_y[im] = p1_y
sx_sum += p1_x
sy_sum += p1_y
if not_zero == 0:
if (not (ishift_x[im] == 0.0)) or (not (ishift_y[im] == 0.0)):
not_zero = 1
sx_sum = mpi.mpi_reduce(sx_sum, 1, mpi.MPI_INT, mpi.MPI_SUM, main_node,
mpi.MPI_COMM_WORLD)
if not oneDx:
sy_sum = mpi.mpi_reduce(sy_sum, 1, mpi.MPI_INT, mpi.MPI_SUM,
main_node, mpi.MPI_COMM_WORLD)
if myid == main_node:
sx_sum_total = int(sx_sum[0])
if not oneDx:
sy_sum_total = int(sy_sum[0])
else:
sx_sum_total = 0
sy_sum_total = 0
sx_sum_total = bcast_number_to_all(sx_sum_total, source_node=main_node)
if not oneDx:
sy_sum_total = bcast_number_to_all(sy_sum_total,
source_node=main_node)
sx_ave = round(float(sx_sum_total) / nima)
sy_ave = round(float(sy_sum_total) / nima)
for im in range(len(data)):
p1_x = ishift_x[im] - sx_ave
p1_y = ishift_y[im] - sy_ave
params2 = {
"filter_type": Processor.fourier_filter_types.SHIFT,
"x_shift": p1_x,
"y_shift": p1_y,
"z_shift": 0.0
}
data[im] = Processor.EMFourierFilter(data[im], params2)
shift_x[im] += p1_x
shift_y[im] += p1_y
# stop if all shifts are zero
not_zero = mpi.mpi_reduce(not_zero, 1, mpi.MPI_INT, mpi.MPI_SUM,
main_node, mpi.MPI_COMM_WORLD)
if myid == main_node:
not_zero_all = int(not_zero[0])
else:
not_zero_all = 0
not_zero_all = bcast_number_to_all(not_zero_all, source_node=main_node)
if myid == main_node:
print_msg("Time of iteration = %12.2f\n" % (time() - start_time))
start_time = time()
if not_zero_all == 0: break
#for im in xrange(len(data)): data[im] = fft(data[im]) This should not be required as only header information is used
# combine shifts found with the original parameters
for im in range(len(data)):
t0 = init_params[im]
t1 = Transform()
t1.set_params({
"type": "2D",
"alpha": 0,
"scale": t0.get_scale(),
"mirror": 0,
"tx": shift_x[im],
"ty": shift_y[im]
})
# combine t0 and t1
tt = t1 * t0
data[im].set_attr("xform.align2d", tt)
# write out headers and STOP, under MPI writing has to be done sequentially
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
par_str = ["xform.align2d", "ID"]
if myid == main_node:
from sp_utilities import file_type
if (file_type(stack) == "bdb"):
from sp_utilities import recv_attr_dict_bdb
recv_attr_dict_bdb(main_node, stack, data, par_str, image_start,
image_end, number_of_proc)
else:
from sp_utilities import recv_attr_dict
recv_attr_dict(main_node, stack, data, par_str, image_start,
image_end, number_of_proc)
else:
send_attr_dict(main_node, data, par_str, image_start, image_end)
if myid == main_node: print_end_msg("shiftali_MPI")
def main():
global debug
progname = os.path.basename(sys.argv[0])
usage = """prog [options] <path or db> ...
Various utilities related to BDB databases.
examples :
e2bdb.py -c Is perhaps the most critical function, as it cleans up the database cache. See the Wiki for more.
e2bdb.py <path> -s will list the contents of the database in a directory in bdb: notation
e2bdb.py <path> -l Will give useful summary info about stacks in a directory
e2bdb.py <database> --dump Gives a mechanism to dump all of the metadata in a database, even if the database contains no images
"""
parser = EMArgumentParser(usage=usage,version=EMANVERSION)
parser.add_argument("--cleanup","-c",action="store_true",default=False,help="This option will clean up the database cache so files can safely be moved or accessed on another computer via NFS.")
parser.add_argument("--force","-F",action="store_true",default=False,help="This will force an action that would normally fail due to failed checks.")
parser.add_argument("--delete",action="store_true",default=False,help="This will delete (or at least empty) the named database(s)")
parser.add_argument("--all","-a",action="store_true",help="List per-particle info",default=False)
parser.add_argument("--long","-l",action="store_true",help="Long listing",default=False)
parser.add_argument("--short","-s",action="store_true",help="Dense listing of names only",default=False)
parser.add_argument("--filt",type=str,help="Only include dictionary names containing the specified string",default=None)
parser.add_argument("--filtexclude",type=str,help="Exclude dictionary names containing the specified string",default=None)
parser.add_argument("--match",type=str,help="Only include dictionaries matching the provided Python regular expression",default=None)
parser.add_argument("--exclude",type=str,help="The name of a database containing a list of exclusion keys",default=None)
parser.add_argument("--dump","-D",action="store_true",help="List contents of an entire database, eg 'e2bdb.py -D refine_01#register",default=False)
parser.add_argument("--smalldump",action="store_true",help="Lists contents of an entire database, but only list 2 items per dictionary to better see headers",default=False)
parser.add_argument("--extractplots",action="store_true",help="If a database contains sets of plots, such as bdb:refine_xx#convergence.results, this will extract the plots as text files.")
parser.add_argument("--check",action="store_true",help="Check for self-consistency and errors in the structure of specified databases",default=False)
parser.add_argument("--nocache",action="store_true",help="Don't use the database cache for this operation",default=False)
parser.add_argument("--merge",action="store_true",help="This will merge the contents of BDB 2-N into BDB 1 (including BDB 1's contents)",default=False)
parser.add_argument("--makevstack",type=str,help="Creates a 'virtual' BDB stack with its own metadata, but the binary data taken from the (filtered) list of stacks",default=None)
parser.add_argument("--appendvstack",type=str,help="Appends to/creates a 'virtual' BDB stack with its own metadata, but the binary data taken from the (filtered) list of stacks",default=None)
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument("--list",type=str,help="Specify the name of a file with a list of images to use in creation of virtual stacks. Please see source for details.",default=None)
parser.add_argument("--exlist",type=str,help="Specify the name of a file with a list of images to exclude in creation of virtual stacks. Please see source for details.",default=None)
parser.add_argument("--restore",type=str,help="Write changes in the derived virtual stack back to the original stack",default=None)
parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
parser.add_argument("--checkctf",action="store_true",help="Verfies that all images in the file contain CTF information, and gives some basic statistics",default=False)
parser.add_argument("--step",type=str,default="0,1",help="Specify <init>,<step>[,<max>]. Processes only a subset of the input data. For example, 0,2 would process only the even numbered particles")
(options, args) = parser.parse_args()
if options.nocache : EMAN2db.BDB_CACHE_DISABLE=True
if options.cleanup :
db_cleanup(options.force)
sys.exit(0)
try : options.step=int(options.step.split(",")[0]),int(options.step.split(",")[1]),int(options.step.split(",")[2]) # convert strings to tuple
except:
try:
options.step=int(options.step.split(",")[0]),int(options.step.split(",")[1])
except:
print "Invalid --step specification"
sys.exit(1)
if options.all : options.long=1
if len(args)==0 : args.append("bdb:.")
logid=0
if options.makevstack :
logid=E2init(sys.argv)
vstack=db_open_dict(options.makevstack)
vstackn=0
elif options.appendvstack :
logid=E2init(sys.argv)
vstack=db_open_dict(options.appendvstack)
vstackn=len(vstack)
else : vstack=None
if options.merge :
print "WARNING: Merge mode\nCombining contents of: ",", ".join(args[1:])
print "into ",args[0]
if raw_input("Proceed (y/n) :").lower() != "y" :
print "Aborting"
sys.exit(1)
for i,path in enumerate(args):
if path.lower()[:4]=="bdb:" and not "#" in path : path="bdb:.#"+path[4:]
if path.lower()[:4]!="bdb:" : path="bdb:"+path
if i==0 :
outdb=db_open_dict(path)
continue
indb=db_open_dict(path,True)
for k in indb.keys():
outdb[k]=indb[k]
print "Merging complete"
sys.exit(0)
for path in args:
if path.lower()[:4]=="bdb:" and not "#" in path :
uu = os.path.split(path)
if(uu[0] == ''): path="bdb:.#"+path[4:]
else: path=uu[0]+"#"+uu[1]
if path.lower()[:4]!="bdb:" : path="bdb:"+path
if '#' in path :
if len(args)>1 : print "\n",path,":"
path,dbs=path.rsplit("#",1)
path+="#"
dbs=[dbs]
else:
if not '#' in path and path[-1]!='/' : path+='#'
if len(args)>1 : print "\n",path[:-1],":"
dbs=db_list_dicts(path)
dbs.sort()
if options.filt:
dbs=[db for db in dbs if options.filt in db]
if options.filtexclude:
dbs=[db for db in dbs if options.filtexclude not in db]
if options.match!=None:
dbs=[db for db in dbs if re.match(options.match,db)]
if options.list :
if options.makevstack==None and options.appendvstack==None :
print "ERROR, this option is used for virtual stack creation, please add makevstack or appendvstack options, and restart"
sys.exit(1)
vdata=open(options.list,'r').readlines()
n=len(vdata[0].split())
slist=[]
for line in vdata:
line=line.split()
for i in xrange(n):
val=int(line[i])
slist.append(val)
del n,val,vdata
if options.exlist :
if options.makevstack==None:
print "ERROR, this option is used for virtual stack creation, please add makevstack or appendvstack options, and restart"
sys.exit(1)
vdata=open(options.exlist,'r').readlines()
n=len(vdata[0].split())
slist=[]
for line in vdata:
line=line.split()
for i in xrange(n):
val=int(line[i])
slist.append(val)
n = EMUtil.get_image_count(args[0])
good = set(range(n)) - set(slist)
slist = [i for i in good]
slist.sort()
del n,val,vdata,good
if options.makevstack!=None or options.appendvstack!=None :
vspath=os.path.realpath(vstack.path)+"/"
if options.verbose>2 : print "vspath: ",vspath
for db in dbs:
dct,keys=db_open_dict(path+db,ro=True,with_keys=True)
if dct==vstack : continue
if len(options.step)==2 :
if keys == None: vals = xrange(options.step[0],len(dct),options.step[1])
else: vals = keys[options.step[0]::options.step[1]] # we apply --step even if we have a list of keys
else:
if keys == None: vals = xrange(options.step[0],options.step[2],options.step[1])
else: vals = keys[options.step[0]:options.step[2]:options.step[1]] # we apply --step even if we have a list of keys
if options.list !=None or options.exlist != None: vals=slist
for n in vals:
try: d=dct.get(n,nodata=1).get_attr_dict()
except:
traceback.print_exc()
print "---\nerror reading ",db,n
continue
# This block converts an absolute path to the actual data to a relative path
try:
dpath=os.path.realpath(dct.get_data_path(n))
if options.verbose>2 : print "dpath: ",dpath
if os.name == 'nt':
vspath=vspath.replace("\\", '/')
dpath=dpath.replace('\\', '/')
rpath=makerelpath(vspath,dpath)
if options.verbose>2 : print "rpath: ",rpath
except:
print "error with data_path ",db,n
continue
d["data_path"]=rpath
d["data_n"]=n
d["data_source"]= path+db
if d["data_path"]==None :
print "error with data_path ",db,n
continue
vstack[vstackn]=d
vstackn+=1
if vstackn%100==0:
try:
print "\r ",vstackn," ",
sys.stdout.flush()
except: pass
print "\r ",vstackn," "
dct.close()
try: maxname=max([len(s) for s in dbs])
except:
print "Error reading ",path
if options.restore :
nima = EMUtil.get_image_count(options.restore)
IB = db_open_dict(options.restore)
source_old = None
if len(options.step)==3 : nima=min(options.step[2],nima)
for i in xrange(options.step[0],nima,options.step[1]):
source = IB.get_header(i)
source_path = source["source_path"]
ID = source["source_n"]
if( source_old != source_path):
if( source_old != None): DB.close()
DB = db_open_dict(source_path,ro=True)
source_old = source_path
target = DB.get_header( ID )
try:
source["data_path"] = target["data_path"]
source["data_n"] = target["data_n"]
source["source_path"] = target["source_path"]
source["source_n"] = target["source_n"]
except:
# top level does not have data_path
del source['data_path']
del source['data_n']
source["source_path"] = target["source_path"]
source["source_n"] = target["source_n"]
DB.set_header(ID, source)
DB.close()
if options.extractplots :
for db in dbs:
print "#### Extracting plots from ",db
dct=db_open_dict(path+db,ro=True)
#### Dump
keys=dct.keys()
keys.sort()
for k in keys:
v=dct[k]
try :
ns=[len(i) for i in v]
fsp=db+"-"+k+".txt"
print "%s (%d columns)"%(fsp,len(ns))
out=file(fsp,"w")
for i in range(ns[0]):
for j in range(len(ns)):
out.write(str(v[j][i]))
if j<len(ns)-1 : out.write("\t")
out.write("\n")
out.close()
except: continue
dct.close()
if options.smalldump :
for db in dbs:
print "##### ",db
dct=db_open_dict(path+db,ro=True)
#### Dump
keys=dct.keys()
keys.sort()
if len(options.step)==3 : keys=keys[:options.step[2]]
for k in keys[options.step[0]::options.step[1]]:
v=dct[k]
print "%s : "%k,
if isinstance (v,list) or isinstance(v,tuple) :
for i in v: print "\n\t%s"%str(i),
print ""
elif isinstance(v,dict) :
ks2=v.keys()
ks2.sort()
kc=0
for i in ks2:
if kc>=2 :
print "..."
break
print "\n\t%s : %s"%(i,v[i]),
kc+=1
print ""
else : print str(v)
dct.close()
if options.checkctf:
for db in dbs:
print "##### CTF -> ",db
dct=db_open_dict(path+db,ro=True)
keys=dct.keys()
if len(options.step)==3 : keys=keys[:options.step[2]]
defocus=set()
for k in keys[options.step[0]::options.step[1]]:
v=dct.get_header(k)
try:
ctf=v["ctf"]
except:
if k!="maxrec" : print "CTF missing on image %s"%k
continue
defocus.add(ctf.defocus)
defocus=list(defocus)
print "Defocuses found: ",
for i in defocus: print "%1.3f, "%i,
print "\n\nRange: %1.3f - %1.3f (%d unique values)"%(min(defocus),max(defocus),len(defocus))
if options.dump :
for db in dbs:
print "##### ",db
dct=db_open_dict(path+db,ro=True)
#### Dump
keys=dct.keys()
if len(options.step)==3 : keys=keys[:options.step[2]]
keys.sort()
for k in keys[options.step[0]::options.step[1]]:
v=dct[k]
print "%s : "%k,
if isinstance (v,list) or isinstance(v,tuple) :
for i in v: print "\n\t%s"%str(i),
print ""
elif isinstance(v,dict) :
ks2=v.keys()
ks2.sort()
for i in ks2:
print "\n\t%s : %s"%(i,v[i]),
print ""
else : print str(v)
dct.close()
# long listing, one db per line
elif options.long :
width=maxname+3
fmt="%%-%ds %%-07d %%14s %%s"%width
fmt2="%%-%ds (not an image stack)"%width
total=[0,0]
for db in dbs:
dct=db_open_dict(path+db,True)
### Info on all particles
if options.all :
mx=len(dct)
if len(options.step)==3 : mx=min(mx,options.step[2])
for i in range(options.step[0],mx,options.step[1]):
try:
im=dct[i]
if im==None : raise Exception
except: continue
print "%d. %d x %d x %d\tA/pix=%1.2f\tM=%1.4f\tS=%1.4f\tSk=%1.4f"%(i,im["nx"],im["ny"],im["nz"],im["apix_x"],im["mean"],im["sigma"],im["skewness"]),
try: print "\t%s"%str(im["model_id"])
except: pass
try:
print "\tdf=%1.3f\tB=%1.1f"%(im["ctf"].defocus,im["ctf"].bfactor)
except: print " "
first=EMData()
try:
first.read_image(path+db,0,True)
size=first.get_xsize()*first.get_ysize()*first.get_zsize()*len(dct)*4;
total[0]+=len(dct)
total[1]+=size
print fmt%(db,len(dct),"%dx%dx%d apix: %1.2f"%(first.get_xsize(),first.get_ysize(),first.get_zsize(),first["apix_x"]),human_size(size)),
except:
print fmt2%db
try: print "\tdf: %1.3f\tB: %1.0f"%(first["ctf"].defocus,first["ctf"].bfactor)
except: print ""
dct.close()
print fmt%("TOTAL",total[0],"",human_size(total[1]))
elif options.check :
from cPickle import loads
for db in dbs:
dct=db_open_dict(path+db,ro=True)
dct.realopen()
keys=dct.bdb.keys()
allkvp={}
for k in keys:
s1,s2=k.split("\x80",1) # start of a pickled string.
s2=loads("\x80"+s2) # the pickled part
if len(s1)>0 : # If anything unpickled, then it is an axbxc prefix identifying the location of a binary
st=allkvp.setdefault(s1,set()) # set of all positions seen so far
v=loads(dct.bdb.get(k)) # position in binary file
if v in st : print "Error: value %d seen multiple times in %s (%s,%s)"%(v,db,s1,s2)
st.add(v)
print "%s : "%db,
for i in allkvp.keys():
if options.verbose>0 : print "%s %d/%d\t"%(i,len(allkvp[i]),int(max(allkvp[i]))+1),
if len(allkvp[i])!=int(max(allkvp[i])+1) : print "\nMismatch found in %s. Could be normal if file has been rewritten multiple times, but is unusual"%db
if options.verbose>0 : print ""
else : print " done"
dct.close()
elif options.short :
for db in dbs:
print path+db,
print " "