def sigma_analysis(self):
#assumes that self.total_signal() and self.background() have already been called.
self.addColumn('MeanIsigI',
0) #mean I over sig(I) for spots within the shell
self.MeanIsigI.format = "%.3f"
index = 0
for row in xrange(self.S_table_rows):
if self.PxlBkgrd[row] == None: continue
IsigI_values = flex.double()
for idx in xrange(self.Population[row]):
# Use International Tables Vol F (2001) equation 11.2.5.9 (p. 214)
I_s = flex.sum(self.persist_fstats.master[
self.persist_indices[index]].wts)
I_bg = flex.sum(self.persist_fstats.master[
self.persist_indices[index]].bkg)
m_sz = len(self.persist_fstats.master[
self.persist_indices[index]].wts)
n_sz = self.MnWndwSz[row]
gain = 1.00
spot_variance = gain * (I_s + I_bg + (m_sz * m_sz) *
(1. / n_sz) * self.PxlBkgrd[row])
IsigI_values.append(I_s / math.sqrt(spot_variance))
index += 1
if len(IsigI_values) > 0:
self.MeanIsigI[row] = flex.mean(IsigI_values)
def total_signal(self, fstats, indices):
#for index in indices:
# print index,fstats.master[index].intensity(),flex.sum(fstats.master[index].wts)
self.addColumn('Integrated',
0) #total integrated signal within the shell
self.Integrated.format = "%.0f"
self.addColumn('MeanI', 0) #mean integrated signal within the shell
self.MeanI.format = "%.0f"
self.addColumn('MeanBkg',
0) #mean integrated background within the shell
self.MeanBkg.format = "%.1f"
self.addColumn('MeanSz',
0) #mean pixel count for spots within the shell
self.MeanSz.format = "%.0f"
self.addColumn('MnEccen', 0) #mean eccentricity spots within the shell
self.MnEccen.format = "%.3f"
self.addColumn('MnSkew', 0) #mean eccentricity spots within the shell
self.MnSkew.format = "%.3f"
index = 0
for row in xrange(self.S_table_rows):
row_values = flex.double()
bkg_values = flex.double()
sz_values = flex.double()
eccen_values = flex.double()
skew_values = flex.double()
for idx in xrange(self.Population[row]):
spot_signal = flex.sum(fstats.master[indices[index]].wts)
bkg_signal = flex.sum(fstats.master[indices[index]].bkg)
spot_sz = len(fstats.master[indices[index]].wts)
spot_eccen = fstats.master[indices[index]].model_eccentricity()
spot_skew = fstats.master[indices[index]].skewness()
self.Integrated[row] += spot_signal
row_values.append(spot_signal)
bkg_values.append(bkg_signal)
sz_values.append(spot_sz)
eccen_values.append(spot_eccen)
skew_values.append(spot_skew)
index += 1
if len(row_values) > 0:
self.MeanI[row] = flex.mean(row_values)
self.MeanBkg[row] = flex.mean(bkg_values)
self.MeanSz[row] = flex.mean(sz_values)
self.MnEccen[row] = flex.mean(eccen_values)
self.MnSkew[row] = flex.mean(skew_values)
self.persist_fstats = fstats
self.persist_indices = indices
def total_signal(self,fstats,indices):
#for index in indices:
# print index,fstats.master[index].intensity(),flex.sum(fstats.master[index].wts)
self.addColumn('Integrated',0) #total integrated signal within the shell
self.Integrated.format = "%.0f"
self.addColumn('MeanI',0) #mean integrated signal within the shell
self.MeanI.format = "%.0f"
self.addColumn('MeanBkg',0) #mean integrated background within the shell
self.MeanBkg.format = "%.1f"
self.addColumn('MeanSz',0) #mean pixel count for spots within the shell
self.MeanSz.format = "%.0f"
self.addColumn('MnEccen',0) #mean eccentricity spots within the shell
self.MnEccen.format = "%.3f"
self.addColumn('MnSkew',0) #mean eccentricity spots within the shell
self.MnSkew.format = "%.3f"
index=0
for row in xrange(self.S_table_rows):
row_values = flex.double()
bkg_values = flex.double()
sz_values = flex.double()
eccen_values = flex.double()
skew_values = flex.double()
for idx in xrange(self.Population[row]):
spot_signal = flex.sum(fstats.master[indices[index]].wts)
bkg_signal = flex.sum(fstats.master[indices[index]].bkg)
spot_sz = len(fstats.master[indices[index]].wts)
spot_eccen = fstats.master[indices[index]].model_eccentricity()
spot_skew = fstats.master[indices[index]].skewness()
self.Integrated[row]+=spot_signal
row_values.append(spot_signal)
bkg_values.append(bkg_signal)
sz_values.append(spot_sz)
eccen_values.append(spot_eccen)
skew_values.append(spot_skew)
index+=1
if len(row_values)>0:
self.MeanI[row]=flex.mean(row_values)
self.MeanBkg[row]=flex.mean(bkg_values)
self.MeanSz[row]=flex.mean(sz_values)
self.MnEccen[row]=flex.mean(eccen_values)
self.MnSkew[row]=flex.mean(skew_values)
self.persist_fstats = fstats
self.persist_indices = indices
def sigma_analysis(self):
#assumes that self.total_signal() and self.background() have already been called.
self.addColumn('MeanIsigI',0) #mean I over sig(I) for spots within the shell
self.MeanIsigI.format = "%.3f"
index=0
for row in xrange(self.S_table_rows):
if self.PxlBkgrd[row] == None: continue
IsigI_values = flex.double()
for idx in xrange(self.Population[row]):
# Use International Tables Vol F (2001) equation 11.2.5.9 (p. 214)
I_s = flex.sum(self.persist_fstats.master[self.persist_indices[index]].wts)
I_bg = flex.sum(self.persist_fstats.master[self.persist_indices[index]].bkg)
m_sz = len(self.persist_fstats.master[self.persist_indices[index]].wts)
n_sz = self.MnWndwSz[row]
gain = 1.00
spot_variance = gain * (I_s + I_bg + (m_sz*m_sz) * (1./n_sz) * self.PxlBkgrd[row])
IsigI_values.append( I_s / math.sqrt(spot_variance))
index+=1
if len(IsigI_values)>0: self.MeanIsigI[row]=flex.mean(IsigI_values)
def run_distl(self, params):
""" Performs a quick DISTL spotfinding and returns Bragg spots information.
"""
from spotfinder.applications import signal_strength
# run DISTL spotfinder
with misc.Capturing() as distl_output:
Org = signal_strength.run_signal_strength(params)
# Extract relevant spotfinding info
for frame in Org.S.images.keys():
saturation = Org.Files.imageindex(frame).saturation
Bragg_spots = [flex.sum(spot.wts) for spot in Org.S.images[frame]['inlier_spots']]
return Bragg_spots
def run_distl(self, params):
""" Performs a quick DISTL spotfinding and returns Bragg spots information.
"""
from spotfinder.applications import signal_strength
# run DISTL spotfinder
with misc.Capturing() as distl_output:
Org = signal_strength.run_signal_strength(params)
# Extract relevant spotfinding info
for frame in Org.S.images.keys():
saturation = Org.Files.imageindex(frame).saturation
Bragg_spots = [
flex.sum(spot.wts)
for spot in Org.S.images[frame]['inlier_spots']
]
return Bragg_spots
def run_distl(self, params):
""" Performs a quick DISTL spotfinding and returns Bragg spots information.
"""
from spotfinder.applications import signal_strength
# run DISTL spotfinder
try:
with util.Capturing() as distl_output:
Org = signal_strength.run_signal_strength(params)
except NotImplementedError as e:
print("NOT IMPLEMENTED ERROR FOR {}".format(self.img))
# Extract relevant spotfinding info
for frame in Org.S.images.keys():
saturation = Org.Files.imageindex(frame).saturation
Bragg_spots = [
flex.sum(spot.wts)
for spot in Org.S.images[frame]['inlier_spots']
]
return Bragg_spots
def run_signal_strength_core(params,E):
verbose = params.distl.verbose
if params.distl.res.inner!=None:
params.distl_lowres_limit = params.distl.res.inner
if params.distl.res.outer!=None:
params.force_method2_resolution_limit = params.distl.res.outer
params.distl_highres_limit = params.distl.res.outer
params.distl_force_binning = False
params.distl_permit_binning = False
params.wedgelimit = len(E.argv)
params.spotfinder_header_tests = False
Org = DistlOrganizer(verbose = True, argument_module=E,
phil_params=params)
Org.printSpots()
#Image analysis requested by NE-CAT (Point of contact: Craig Ogata)
for key in Org.S.images.keys():
# List of spots between specified high- and low-resolution limits
if Org.S.images[key].has_key('lo_pass_resolution_spots'):
spots = Org.S.images[key]['lo_pass_resolution_spots']
elif Org.S.images[key].has_key('inlier_spots'):
spots = Org.S.images[key]['inlier_spots']
else:
spots = []
saturation = Org.Files.imageindex(key).saturation
#Total number of spots in this range
print
print "Number of focus spots on image #%d within the input resolution range: %d"%(
key,len(spots))
signals=flex.double()
saturations=flex.double()
#Each spot
for i,spot in enumerate(spots):
signals.append(flex.sum(spot.wts))
saturations.append(flex.max(spot.wts)/saturation)
if verbose:
#peak height given in ADC units above local background
#integrated signal strength given in pixel-ADC units above local background
print "%2d: Area in pixels=%d Peak=%.1f, Total integrated signal=%.1f (in pixel-ADC units above background)"%(
i, spot.area(), flex.max(spot.wts), flex.sum(spot.wts))
#peak signal-to-noise expressed in standard deviations above local background
print " Peak signal-to-noise=%.1f"%(spot.intensity())
#peak height expressed in ADC units, without background subtraction
image = Org.Files.imageindex(key)
print " Peak position x=%4d y=%4d (pixels); pixel value=%5d"%(
spot.max_pxl_x(), spot.max_pxl_y(),
image.linearintdata[(spot.max_pxl_x(),spot.max_pxl_y())])
#Gory detail, looping through each pixel on each spot
for j,pixel in enumerate(spot.bodypixels):
print " body pixel x=%4d y=%4d; pixel value=%5d; ADC height above background=%.1f"%(
pixel.x,pixel.y,image.linearintdata[(pixel.x,pixel.y)],spot.wts[j])
if signals.size()>0:
print "Total integrated signal, pixel-ADC units above local background (just the good Bragg candidates) %d"%(
flex.sum(flex.double([flex.sum(spot.wts) for spot in Org.S.images[key]['inlier_spots']]))
)
print "Signals range from %.1f to %.1f with mean integrated signal %.1f"%(
flex.min(signals), flex.max(signals), flex.mean(signals) )
print "Saturations range from %.1f%% to %.1f%% with mean saturation %.1f%%"%(
100.*flex.min(saturations), 100.*flex.max(saturations), 100.*flex.mean(saturations) )
if params.distl.pdf_output != None:
#later, put this in a separate module so reportlab is not imported unless requested
from labelit.publications.sublattice.sublattice_pdf import SublatticePDF,graphic
from labelit.publications.sublattice.sublattice_pdf import PointTransform
class genPDF(SublatticePDF):
def make_image_plots_detail(self):
params.pdf_output.window_fraction=1.0
params.pdf_output.window_offset_x=0.0
params.pdf_output.window_offset_y=0.0
params.pdf_output.markup_inliers=True
couple=(params.pdf_output.window_offset_x,
params.pdf_output.window_offset_y)
#instead of self.R.setTransform, which requires pickled spotfinder:
self.R.T = PointTransform()
self.R.S = self.R.spotfinder
self.R.T.setImage(spotfinder=self.R.S,subwindow_origin=couple,commands=params)
self.R.title(self.image_name)
#try:
pil_image = graphic(filein = self.image_name,
couple = couple,
commands = params)
self.R.image(pil_image)
#except:
# print "failure, file %s"%self.filename
if params.pdf_output.markup_inliers:
self.R.show_ellipse(
image_number=self.R.spotfinder.images.keys()[0],
tags = ['goodspots','spots_non-ice','hi_pass_resolution_spots',
'spots_unimodal'],
detail=True)
self.R.c.showPage()
return self
pdf = genPDF(params.distl.pdf_output)
pdf.filename = params.distl.pdf_output
pdf.image_name = params.distl.image
pdf.set_spotfinder(Org.S)
pdf.make_image_plots_detail()
if params.distl.image_viewer == True:
try:
from rstbx.viewer.spotfinder_wrap import spot_wrapper
spot_wrapper(params).display(path = params.distl.image,
organizer = Org)
except ImportError,e:
from libtbx.utils import Sorry
# must use phenix.wxpython for wx display
raise Sorry(str(e)+" Try setting env variable PHENIX_GUI_ENVIRONMENT=1")
def triage_image(self):
""" Performs a quick DISTL spotfinding without grid search.
"""
from spotfinder.command_line.signal_strength import master_params as sf_params
sf_params = sf_params.extract()
sf_params.distl.image = self.img
E = Empty()
E.argv = ['Empty']
E.argv.append(sf_params.distl.image)
log_info = ['{}\n'.format(self.img)]
img_filename = os.path.basename(self.img)
# Perform spotfinding
# ... using spotfinding grid search
if self.params.image_triage.type == 'grid_search':
log_info.append('\n CCTBX TRIAGE grid search:')
# Determine grid search extent
a_min = self.params.image_triage.grid_search.area_min
a_max = self.params.image_triage.grid_search.area_max
a_step = (a_max -
a_min) // self.params.image_triage.grid_search.step_size
h_min = self.params.image_triage.grid_search.height_min
h_max = self.params.image_triage.grid_search.height_max
h_step = (h_max -
h_min) // self.params.image_triage.grid_search.step_size
# Cycle through grid points
spotlist = []
for spa in range(a_min, a_max + 1, a_step):
for sph in range(h_min, h_max + 1, h_step):
sf_params.distl.minimum_spot_area = spa
sf_params.distl.minimum_spot_height = sph
sf_params.distl.minimum_signal_height = sph
# Perform spotfinding
Bragg_spots = self.run_distl(sf_params)
N_Bragg_spots = len(Bragg_spots)
if N_Bragg_spots > 0:
total_intensity = flex.sum(flex.double(Bragg_spots))
else:
total_intensity = 0
spotlist.append({
'bragg': N_Bragg_spots,
'ti': total_intensity,
'spa': spa,
'sph': sph
})
log_info.append('{:<{w}}: H = {:<2}, A = {:<2}, Bragg = {:<6.0f} '\
'total intensity = {:<12.4f}'.format(img_filename, sph, spa,
N_Bragg_spots, total_intensity, w = len(img_filename)))
# Pick best spotfinding result (highest total intensity seems to work for now
pick = sorted(spotlist, key=lambda j: j['ti'])[-1]
N_Bragg_spots = pick['bragg']
start_sph = pick['sph']
start_sih = pick['sph']
start_spa = pick['spa']
# ... using spotfinding without grid search
else:
# Set spotfinding params
sf_params.distl.minimum_spot_area = self.params.cctbx.grid_search.area_median
sf_params.distl.minimum_spot_height = self.params.cctbx.grid_search.height_median
sf_params.distl.minimum_signal_height = self.params.cctbx.grid_search.height_median
# Perform spotfinding
Bragg_spots = self.run_distl(sf_params)
# Extract spotfinding results
N_Bragg_spots = len(Bragg_spots)
start_sph = self.params.cctbx.grid_search.height_median
start_sih = self.params.cctbx.grid_search.height_median
start_spa = self.params.cctbx.grid_search.area_median
# Determine triage success
if N_Bragg_spots >= self.params.image_triage.min_Bragg_peaks:
log_info.append('ACCEPTED! Selected starting point:')
log_info.append('{:<{w}}: S = {:<2}, H = {:<2}, A = {:<2}, Bragg = {:<6.0f}'\
''.format(img_filename, start_sih, start_sph, start_spa,
N_Bragg_spots, w = len(img_filename)))
status = None
else:
log_info.append(
'REJECTED! ({} Bragg peaks found)'.format(N_Bragg_spots))
status = 'failed triage'
log_entry = "\n".join(log_info)
return status, log_entry, start_sph, start_spa
def run_signal_strength_core(params, E):
verbose = params.distl.verbose
if params.distl.res.inner != None:
params.distl_lowres_limit = params.distl.res.inner
if params.distl.res.outer != None:
params.force_method2_resolution_limit = params.distl.res.outer
params.distl_highres_limit = params.distl.res.outer
params.distl_force_binning = False
params.distl_permit_binning = False
params.wedgelimit = len(E.argv)
params.spotfinder_header_tests = False
Org = DistlOrganizer(verbose=True, argument_module=E, phil_params=params)
Org.printSpots()
#Image analysis requested by NE-CAT (Point of contact: Craig Ogata)
for key in Org.S.images.keys():
# List of spots between specified high- and low-resolution limits
if Org.S.images[key].has_key('lo_pass_resolution_spots'):
spots = Org.S.images[key]['lo_pass_resolution_spots']
elif Org.S.images[key].has_key('inlier_spots'):
spots = Org.S.images[key]['inlier_spots']
else:
spots = []
saturation = Org.Files.imageindex(key).saturation
#Total number of spots in this range
print
print "Number of focus spots on image #%d within the input resolution range: %d" % (
key, len(spots))
signals = flex.double()
saturations = flex.double()
#Each spot
for i, spot in enumerate(spots):
signals.append(flex.sum(spot.wts))
saturations.append(flex.max(spot.wts) / saturation)
if verbose:
#peak height given in ADC units above local background
#integrated signal strength given in pixel-ADC units above local background
print "%2d: Area in pixels=%d Peak=%.1f, Total integrated signal=%.1f (in pixel-ADC units above background)" % (
i, spot.area(), flex.max(spot.wts), flex.sum(spot.wts))
#peak signal-to-noise expressed in standard deviations above local background
print " Peak signal-to-noise=%.1f" % (spot.intensity())
#peak height expressed in ADC units, without background subtraction
image = Org.Files.imageindex(key)
print " Peak position x=%4d y=%4d (pixels); pixel value=%5d" % (
spot.max_pxl_x(), spot.max_pxl_y(),
image.linearintdata[(spot.max_pxl_x(), spot.max_pxl_y())])
#Gory detail, looping through each pixel on each spot
for j, pixel in enumerate(spot.bodypixels):
print " body pixel x=%4d y=%4d; pixel value=%5d; ADC height above background=%.1f" % (
pixel.x, pixel.y,
image.linearintdata[(pixel.x, pixel.y)], spot.wts[j])
if signals.size() > 0:
print "Total integrated signal, pixel-ADC units above local background (just the good Bragg candidates) %d" % (
flex.sum(
flex.double([
flex.sum(spot.wts)
for spot in Org.S.images[key]['inlier_spots']
])))
print "Signals range from %.1f to %.1f with mean integrated signal %.1f" % (
flex.min(signals), flex.max(signals), flex.mean(signals))
print "Saturations range from %.1f%% to %.1f%% with mean saturation %.1f%%" % (
100. * flex.min(saturations), 100. * flex.max(saturations),
100. * flex.mean(saturations))
if params.distl.pdf_output != None:
#later, put this in a separate module so reportlab is not imported unless requested
from labelit.publications.sublattice.sublattice_pdf import SublatticePDF, graphic
from labelit.publications.sublattice.sublattice_pdf import PointTransform
class genPDF(SublatticePDF):
def make_image_plots_detail(self):
params.pdf_output.window_fraction = 1.0
params.pdf_output.window_offset_x = 0.0
params.pdf_output.window_offset_y = 0.0
params.pdf_output.markup_inliers = True
couple = (params.pdf_output.window_offset_x,
params.pdf_output.window_offset_y)
#instead of self.R.setTransform, which requires pickled spotfinder:
self.R.T = PointTransform()
self.R.S = self.R.spotfinder
self.R.T.setImage(spotfinder=self.R.S,
subwindow_origin=couple,
commands=params)
self.R.title(self.image_name)
#try:
pil_image = graphic(filein=self.image_name,
couple=couple,
commands=params)
self.R.image(pil_image)
#except:
# print "failure, file %s"%self.filename
if params.pdf_output.markup_inliers:
self.R.show_ellipse(
image_number=self.R.spotfinder.images.keys()[0],
tags=[
'goodspots', 'spots_non-ice',
'hi_pass_resolution_spots', 'spots_unimodal'
],
detail=True)
self.R.c.showPage()
return self
pdf = genPDF(params.distl.pdf_output)
pdf.filename = params.distl.pdf_output
pdf.image_name = params.distl.image
pdf.set_spotfinder(Org.S)
pdf.make_image_plots_detail()
if params.distl.image_viewer == True:
try:
from rstbx.viewer.spotfinder_wrap import spot_wrapper
spot_wrapper(params).display(path=params.distl.image,
organizer=Org)
except ImportError, e:
from libtbx.utils import Sorry
# must use phenix.wxpython for wx display
raise Sorry(
str(e) + " Try setting env variable PHENIX_GUI_ENVIRONMENT=1")
def triage_image(self):
""" Performs a quick DISTL spotfinding without grid search.
"""
from spotfinder.command_line.signal_strength import master_params as sf_params
sf_params = sf_params.extract()
sf_params.distl.image = self.img
E = Empty()
E.argv=['Empty']
E.argv.append(sf_params.distl.image)
log_info = ['{}\n'.format(self.img)]
img_filename = os.path.basename(self.img)
# Perform spotfinding
# ... using spotfinding grid search
if self.params.image_triage.type == 'grid_search':
log_info.append('\n CCTBX TRIAGE grid search:')
# Determine grid search extent
a_min = self.params.image_triage.grid_search.area_min
a_max = self.params.image_triage.grid_search.area_max
a_step = (a_max - a_min) // self.params.image_triage.grid_search.step_size
h_min = self.params.image_triage.grid_search.height_min
h_max = self.params.image_triage.grid_search.height_max
h_step = (h_max - h_min) // self.params.image_triage.grid_search.step_size
# Cycle through grid points
spotlist = []
for spa in range(a_min, a_max + 1, a_step):
for sph in range(h_min, h_max + 1, h_step):
sf_params.distl.minimum_spot_area = spa
sf_params.distl.minimum_spot_height = sph
sf_params.distl.minimum_signal_height = sph
# Perform spotfinding
Bragg_spots = self.run_distl(sf_params)
N_Bragg_spots = len(Bragg_spots)
if N_Bragg_spots > 0:
total_intensity = flex.sum(flex.double(Bragg_spots))
else:
total_intensity = 0
spotlist.append({'bragg':N_Bragg_spots, 'ti':total_intensity,
'spa':spa, 'sph':sph})
log_info.append('{:<{w}}: H = {:<2}, A = {:<2}, Bragg = {:<6.0f} '\
'total intensity = {:<12.4f}'.format(img_filename, sph, spa,
N_Bragg_spots, total_intensity, w = len(img_filename)))
# Pick best spotfinding result (highest total intensity seems to work for now
pick = sorted(spotlist, key = lambda j: j['ti'])[-1]
N_Bragg_spots = pick['bragg']
start_sph = pick['sph']
start_sih = pick['sph']
start_spa = pick['spa']
# ... using spotfinding without grid search
else:
# Set spotfinding params
sf_params.distl.minimum_spot_area = self.params.cctbx.grid_search.area_median
sf_params.distl.minimum_spot_height = self.params.cctbx.grid_search.height_median
sf_params.distl.minimum_signal_height = self.params.cctbx.grid_search.height_median
# Perform spotfinding
Bragg_spots = self.run_distl(sf_params)
# Extract spotfinding results
N_Bragg_spots = len(Bragg_spots)
start_sph = self.params.cctbx.grid_search.height_median
start_sih = self.params.cctbx.grid_search.height_median
start_spa = self.params.cctbx.grid_search.area_median
# Determine triage success
if N_Bragg_spots >= self.params.image_triage.min_Bragg_peaks:
log_info.append('ACCEPTED! Selected starting point:')
log_info.append('{:<{w}}: S = {:<2}, H = {:<2}, A = {:<2}, Bragg = {:<6.0f}'\
''.format(img_filename, start_sih, start_sph, start_spa,
N_Bragg_spots, w = len(img_filename)))
status = None
else:
log_info.append('REJECTED!')
status = 'failed triage'
log_entry = "\n".join(log_info)
return status, log_entry, start_sph, start_spa
