def print_results(self):
""" Prints diagnostics from the final integration run. """
cons_s = Counter(self.s).most_common(1)[0][0]
cons_h = Counter(self.h).most_common(1)[0][0]
cons_a = Counter(self.a).most_common(1)[0][0]
final_table = []
final_table.append("\n\n{:-^80}\n".format('ANALYSIS OF RESULTS'))
#final_table.append("Total images: {}".format(len(
# self.final_objects)))
if self.params.advanced.integrate_with == 'cctbx':
final_table.append("Avg. signal height: {:<8.3f} std. dev: {:<6.2f}"\
" max: {:<3} min: {:<3} consensus: {:<3}"\
"".format(np.mean(self.s), np.std(self.s),
max(self.s), min(self.s), cons_s))
final_table.append("Avg. spot height: {:<8.3f} std. dev: {:<6.2f}"\
" max: {:<3} min: {:<3} consensus: {:<3}"\
"".format(np.mean(self.h), np.std(self.h),
max(self.h), min(self.h), cons_h))
final_table.append("Avg. spot areas: {:<8.3f} std. dev: {:<6.2f}"\
" max: {:<3} min: {:<3} consensus: {:<3}"\
"".format(np.mean(self.a), np.std(self.a),
max(self.a), min(self.a), cons_a))
final_table.append("Avg. resolution: {:<8.3f} std. dev: {:<6.2f}"\
" lowest: {:<6.3f} highest: {:<6.3f}"\
"".format(np.mean(self.hres), np.std(self.hres),
max(self.hres), min(self.hres)))
final_table.append("Avg. number of spots: {:<8.3f} std. dev: {:<6.2f}"\
"".format(np.mean(self.spots), np.std(self.spots)))
final_table.append("Avg. mosaicity: {:<8.3f} std. dev: {:<6.2f}"\
"".format(np.mean(self.mos), np.std(self.mos)))
# If more than one integrated image, plot various summary graphs
if len(self.final_objects) > 1 and self.params.analysis.summary_graphs:
plot = Plotter(self.params, self.final_objects, self.viz_dir)
if ( self.params.advanced.integrate_with == 'cctbx' and
self.params.cctbx.grid_search.type != None
):
plot.plot_spotfinding_heatmap(write_files=True)
plot.plot_res_histogram(write_files=True)
med_beamX, med_beamY = plot.plot_beam_xy(write_files=True,
return_values=True)
final_table.append("Median Beam Center: X = {:<4.2f}, Y = {:<4.2f}"\
"".format(med_beamX, med_beamY))
for item in final_table:
misc.main_log(self.logfile, item, (not self.gui_mode))
def run(self):
''' Run IOTA '''
# Import Images
self.stage = 'import'
self.run_import()
# Remove rejected images from image object list
acc_img_objects = [i.fail for i in self.img_objects if i.fail == None]
cmd.Command.end("Accepted {} of {} images -- DONE " \
"".format(len(acc_img_objects), len(self.img_objects)))
# Exit if none of the images have diffraction
if str(self.init.params.image_triage.type).lower() != 'none':
if len(acc_img_objects) == 0:
misc.main_log(self.init.logfile, 'No images have diffraction!', True)
misc.iota_exit()
else:
misc.main_log(self.init.logfile,
"{} out of {} images have diffraction "
"(at least {} Bragg peaks)"
"".format(len(acc_img_objects),
len(self.img_objects),
self.init.params.image_triage.min_Bragg_peaks))
# Check for -c option and exit if true
if self.init.params.image_conversion.convert_only:
misc.iota_exit()
# Process Images
self.stage = 'process'
self.run_process()
# Analysis of integration results
final_objects = [i for i in self.img_objects if i.fail == None]
if len(final_objects) > 0:
self.run_analysis()
else:
print 'No images successfully integrated!'
# Exit IOTA
misc.iota_exit()
def run(self):
self.args, self.phil_args = parse_command_args(self.iver,
self.help_message).parse_known_args()
# Check for type of input
if self.args.path == None: # No input
parse_command_args(self.iver, self.help_message).print_help()
if self.args.default: # Write out default params and exit
help_out, txt_out = inp.print_params()
print '\n{:-^70}\n'.format('IOTA Parameters')
print help_out
inp.write_defaults(os.path.abspath(os.path.curdir), txt_out)
misc.iota_exit()
else: # If input exists, check type
carg = os.path.abspath(self.args.path)
if os.path.exists(carg):
# If user provided a parameter file
if os.path.isfile(carg) and os.path.basename(carg).endswith('.param'):
msg = ''
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'file')
# If user provided a list of input files
elif os.path.isfile(carg) and os.path.basename(carg).endswith('.lst'):
msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg)
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'auto', self.now)
# If user provided a single filepath
elif os.path.isfile(carg) and not os.path.basename(carg).endswith('.lst'):
msg = "\nIOTA will run in SINGLE-FILE mode using {}:\n".format(carg)
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'auto', self.now)
# If user provided a data folder
elif os.path.isdir(carg):
msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg)
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'auto', self.now)
# If user provided gibberish
else:
print self.logo
print "ERROR: Invalid input! Need parameter filename or data folder."
misc.iota_exit()
# Identify indexing / integration program
if self.params.advanced.integrate_with == 'cctbx':
prg = " with CCTBX.XFEL\n"
elif self.params.advanced.integrate_with == 'dials':
prg = " with DIALS\n"
self.logo += prg
print self.logo
print '\n{}\n'.format(self.now)
if msg != '':
print msg
if self.args.analyze != None:
self.analyze_prior_results('{:003d}'.format(int(self.args.analyze)))
misc.iota_exit()
if self.params.mp_method == 'mpi':
rank, size = misc.get_mpi_rank_and_size()
self.master_process = rank == 0
else:
self.master_process = True
# Call function to read input folder structure (or input file) and
# generate list of image file paths
if self.params.cctbx.selection.select_only.flag_on:
self.gs_img_objects = self.make_int_object_list()
self.input_list = [i.conv_img for i in self.gs_img_objects]
else:
self.input_list = self.make_input_list()
# Check for -l option, output list of input files and exit
if self.args.list:
if list_file == None:
list_file = os.path.abspath("{}/input.lst".format(os.curdir))
print '\nINPUT LIST ONLY option selected'
print 'Input list in {} \n\n'.format(list_file)
with open(list_file, "w") as lf:
for i, input_file in enumerate(self.input_list, 1):
lf.write('{}\n'.format(input_file))
print "{}: {}".format(i, input_file)
lf.write('{}\n'.format(input_file))
print '\nExiting...\n\n'
misc.iota_exit()
# If fewer images than requested processors are supplied, set the number of
# processors to the number of images
if self.params.n_processors > len(self.input_list):
self.params.n_processors = len(self.input_list)
# Generate base folder paths
self.conv_base = misc.set_base_dir('converted_pickles', out_dir = self.params.output)
self.int_base = misc.set_base_dir('integration', out_dir = self.params.output)
self.obj_base = os.path.join(self.int_base, 'image_objects')
self.fin_base = os.path.join(self.int_base, 'final')
self.tmp_base = os.path.join(self.int_base, 'tmp')
self.viz_base = os.path.join(self.int_base, 'visualization')
# Generate base folders
os.makedirs(self.int_base)
os.makedirs(self.obj_base)
os.makedirs(self.fin_base)
os.makedirs(self.tmp_base)
# Determine input base
self.input_base = os.path.abspath(os.path.dirname(os.path.commonprefix(self.input_list)))
# Initialize main log
self.logfile = os.path.abspath(os.path.join(self.int_base, 'iota.log'))
# Log starting info
misc.main_log(self.logfile, '{:=^80} \n'.format(' IOTA MAIN LOG '))
misc.main_log(self.logfile, '{:-^80} \n'.format(' SETTINGS FOR THIS RUN '))
misc.main_log(self.logfile, self.txt_out)
if self.params.advanced.integrate_with == 'cctbx':
target_file = self.params.cctbx.target
elif self.params.advanced.integrate_with == 'dials':
target_file = self.params.dials.target
misc.main_log(self.logfile, '{:-^80} \n\n'
''.format(' TARGET FILE ({}) CONTENTS '
''.format(target_file)))
with open(target_file, 'r') as phil_file:
phil_file_contents = phil_file.read()
misc.main_log(self.logfile, phil_file_contents)
def process(self, single_image=False):
""" Image processing; selects method, runs requisite modules """
if self.status != 'bypass grid search':
self.status = 'processing'
#for CCTBX indexing / integration
if self.params.advanced.integrate_with == 'cctbx':
terminate = False
prev_status = self.status
prev_fail = 'first cycle'
prev_final = self.final
prev_epv = 9999
while not terminate:
if os.path.isfile(self.abort_file):
self.fail = 'aborted'
return self
# Run grid search if haven't already
if self.fail == None and 'grid search' not in self.status:
self.integrate_cctbx('grid search', single_image=single_image)
# Run selection if haven't already
if self.fail == None and self.status != 'selection':
self.select_cctbx()
# If smart grid search is active run multiple rounds until convergence
if self.params.cctbx.grid_search.type == 'smart':
if self.fail == None and self.final['epv'] < prev_epv:
prev_epv = self.final['epv']
prev_final = self.final
prev_status = self.status
prev_fail = self.fail
self.hmed = self.final['sph']
self.amed = self.final['spa']
self.generate_grid()
self.final['final'] = self.fin_file
if len(self.grid) == 0:
self.final = prev_final
self.status = prev_status
self.fail = prev_fail
terminate = True
continue
if self.verbose:
log_entry = '\nNew starting point: H = {}, A = {}\n'\
''.format(self.hmed, self.amed)
self.log_info.append(log_entry)
else:
if prev_fail != 'first cycle':
self.final = prev_final
self.status = prev_status
self.fail = prev_fail
if self.verbose:
log_entry = '\nFinal set of parameters: H = {}, A = {}'\
''.format(self.final['sph'], self.final['spa'])
self.log_info.append(log_entry)
terminate = True
# If brute force grid search is selected run one round
else:
terminate = True
# Run final integration if haven't already
if self.fail == None and self.status != 'final':
self.integrate_cctbx('integrate', single_image=single_image)
# If verbose output selected (default), write to main log
if self.verbose:
log_entry = "\n".join(self.log_info)
misc.main_log(self.main_log, log_entry)
misc.main_log(self.main_log, '\n\n')
# Make a temporary process log into a final process log
if os.path.isfile(self.int_log):
final_int_log = os.path.join(self.fin_path,
os.path.basename(self.int_log).split('.')[0] + '.log')
os.rename(self.int_log, final_int_log)
# For DIALS integration (WORK IN PROGRESS)
elif self.params.advanced.integrate_with == 'dials':
if os.path.isfile(self.abort_file):
self.fail = 'aborted'
return self
# Create DIALS integrator object
from iota.components.iota_dials import Integrator
integrator = Integrator(self.conv_img,
self.obj_base,
self.fin_base,
self.fin_file,
self.final,
self.int_log,
self.gain,
self.params)
# Run DIALS
self.fail, self.final, int_log = integrator.run()
if self.fail != None:
self.status = 'final'
self.log_info.append(int_log)
log_entry = "\n".join(self.log_info)
misc.main_log(self.main_log, log_entry)
misc.main_log(self.main_log, '\n{:-^100}\n'.format(''))
# Make a temporary process log into a final process log
final_int_log = self.int_log.split('.')[0] + ".log"
os.rename(self.int_log, final_int_log)
self.status = 'final'
ep.dump(self.obj_file, self)
def import_image(self):
""" Image conversion:
- Writes out data in pickle format (cctbx.xfel only)
- Moves beam center into center of image, crops / pads image (cctbx.xfel only)
- Adjusts beam center and distance (optional)
- Thresholds and masks beamstop shadow (optional)
"""
if os.path.isfile(self.abort_file):
self.fail = 'aborted'
return self
# Load image
img_data, img_type = self.load_image()
self.status = 'loaded'
if img_data is None:
self.log_info.append('\n{:-^100}\n'.format(self.raw_img))
self.log_info.append('FAILED TO IMPORT')
self.status = 'failed import'
self.fail = 'failed import'
return self
# if DIALS is selected, change image type to skip conversion step
if self.params.advanced.integrate_with == 'dials':
img_type = 'dials_input'
# Log initial image information
self.log_info.append('\n{:-^100}\n'.format(self.raw_img))
self.log_info.append('Imported image : {}'.format(self.raw_img))
self.log_info.append('Parameters : BEAM_X = {:<4.2f}, BEAM_Y = {:<4.2f}, '\
'PIXEL_SIZE = {:<8.6f}, IMG_SIZE = {:<4} X {:<4}, '\
'DIST = {}'.format(img_data['BEAM_CENTER_X'],
img_data['BEAM_CENTER_Y'],
img_data['PIXEL_SIZE'],
img_data['SIZE1'],
img_data['SIZE2'],
img_data['DISTANCE']))
# Deactivate image squaring if beam center is in the center of the image
# CCTBX.XFEL ONLY (Need a better displacement cutoff)
if ( self.params.advanced.integrate_with == 'dials' or
abs(img_data['BEAM_CENTER_X'] - img_data['BEAM_CENTER_Y']) < 0.1
):
self.params.image_conversion.square_mode = 'None'
# Check if conversion/modification is required and carry them out
if ( img_type == 'raw' or
self.params.image_conversion.square_mode != "None" or
self.params.image_conversion.beam_center.x != 0 or
self.params.image_conversion.beam_center.y != 0 or
self.params.image_conversion.beamstop != 0 or
self.params.image_conversion.distance != 0
):
# Check for and/or create a converted pickles folder
try:
if not os.path.isdir(self.conv_base):
os.makedirs(self.conv_base)
except OSError:
pass
# Generate converted image pickle filename
if self.params.image_conversion.rename_pickle_prefix != None:
if str(self.params.image_conversion.rename_pickle_prefix).lower() == "auto":
try:
prefix = os.getlogin()
except Exception:
prefix = 'converted'
else:
prefix = self.params.image_conversion.rename_pickle_prefix
number = int(os.path.basename(self.conv_base))
self.conv_img = os.path.abspath(os.path.join(self.conv_base,
"{}_{}_{:05d}.pickle".format(prefix, number, self.img_index)))
else: # This option preserves the input directory structure
img_path = misc.make_image_path(self.raw_img, self.input_base, self.conv_base)
self.conv_img = os.path.abspath(os.path.join(img_path,
os.path.basename(self.raw_img).split('.')[0] + ".pickle"))
try:
if not os.path.isdir(img_path):
os.makedirs(img_path)
except OSError:
pass
# Convert raw image to image pickle
beamstop = self.params.image_conversion.beamstop
distance = self.params.image_conversion.distance
beam_center = [self.params.image_conversion.beam_center.x,
self.params.image_conversion.beam_center.y]
square = self.params.image_conversion.square_mode
if beam_center != [0,0]:
pixel_size = img_data['PIXEL_SIZE']
img_data['BEAM_CENTER_X'] = int(round(beam_center[0] * pixel_size))
img_data['BEAM_CENTER_Y'] = int(round(beam_center[1] * pixel_size))
if distance != 0:
img_data['DISTANCE'] = distance
if square != "None":
img_data = self.square_pickle(img_data)
if beamstop != 0:
img_data = self.mask_image(img_data)
# Log converted image information
self.log_info.append('Converted image : {}'.format(self.conv_img))
self.log_info.append('Parameters : BEAM_X = {:<4.2f}, BEAM_Y = {:<4.2f}, '\
'PIXEL_SIZE = {:<8.6f}, IMG_SIZE = {:<4} X {:<4}, '\
'DIST = {}'.format(img_data['BEAM_CENTER_X'],
img_data['BEAM_CENTER_Y'],
img_data['PIXEL_SIZE'],
img_data['SIZE1'],
img_data['SIZE2'],
img_data['DISTANCE']))
self.input_base = self.conv_base
self.status = 'converted'
# Save converted image pickle
ep.dump(self.conv_img, img_data)
# Triage image (i.e. check for usable diffraction, using selected method)
if str(self.params.image_triage.type).lower() != 'none':
if self.params.advanced.integrate_with == 'cctbx':
from iota.components.iota_cctbx import Triage
triage = Triage(self.conv_img, self.gain, self.params)
self.fail, log_entry, self.hmed, self.amed = triage.triage_image()
elif self.params.advanced.integrate_with == 'dials':
from iota.components.iota_dials import Triage
triage = Triage(self.conv_img, self.gain, self.params)
self.fail, log_entry = triage.triage_image()
self.log_info.append(log_entry)
self.status = 'triaged'
else:
self.fail = None
# Generate integration result dictionary for cctbx.xfel or DIALS
if self.params.advanced.integrate_with == 'cctbx':
if self.params.cctbx.grid_search.type == 'smart':
self.hrange = 1
self.arange = 1
else:
self.hrange = self.params.cctbx.grid_search.height_range
self.arange = self.params.cctbx.grid_search.area_range
self.grid_points = []
self.generate_grid()
elif self.params.advanced.integrate_with == 'dials':
self.final = {'img':self.conv_img, 'a':0, 'b':0, 'c':0, 'alpha':0,
'beta':0, 'gamma':0, 'sg':'','strong':0, 'res':0,
'lres':0, 'mos':0, 'epv':0, 'info':'','final':None,
'program':'dials'}
# Generate names for output folders and files:
if not self.params.image_conversion.convert_only:
self.obj_path = misc.make_image_path(self.conv_img, self.input_base, self.obj_base)
self.obj_file = os.path.abspath(os.path.join(self.obj_path,
os.path.basename(self.conv_img).split('.')[0] + ".int"))
self.fin_path = misc.make_image_path(self.conv_img, self.input_base, self.fin_base)
self.fin_file = os.path.abspath(os.path.join(self.fin_path,
"int_{}.pickle".format(os.path.basename(self.conv_img).split('.')[0])))
self.final['final'] = self.fin_file
self.final['img'] = self.conv_img
self.int_log = os.path.join(self.fin_path,
os.path.basename(self.conv_img).split('.')[0] + '.tmp')
self.viz_path = misc.make_image_path(self.conv_img, self.input_base, self.viz_base)
self.viz_file = os.path.join(self.viz_path,
"int_{}.png".format(os.path.basename(self.conv_img).split('.')[0]))
# Create actual folders (if necessary)
try:
if not os.path.isdir(self.obj_path):
os.makedirs(self.obj_path)
if not os.path.isdir(self.fin_path):
os.makedirs(self.fin_path)
if not os.path.isdir(self.viz_path):
os.makedirs(self.viz_path)
except OSError:
pass
# Save image object to file
ep.dump(self.obj_file, self)
self.status = 'imported'
# If conversion only option is selected, write conversion info to log
if self.params.image_conversion.convert_only:
log_entry = "\n".join(self.log_info)
misc.main_log(self.main_log, log_entry)
return self
def print_results(self, final_table=None):
""" Prints diagnostics from the final integration run. """
cons_s = Counter(self.s).most_common(1)[0][0]
cons_h = Counter(self.h).most_common(1)[0][0]
cons_a = Counter(self.a).most_common(1)[0][0]
if final_table is None:
final_table = []
final_table.append("\n\n{:-^80}\n".format('ANALYSIS OF RESULTS'))
# In case no images were integrated
if self.final_objects is None:
final_table.append('NO IMAGES INTEGRATED!')
else:
if self.params.advanced.integrate_with == 'cctbx':
final_table.append(
"Avg. signal height: {:<8.3f} std. dev: "
"{:<6.2f} max: {:<3} min: {:<3} consensus: {:<3}"
"".format(np.mean(self.s), np.std(self.s), max(self.s),
min(self.s), cons_s))
final_table.append(
"Avg. spot height: {:<8.3f} std. dev: "
"{:<6.2f} max: {:<3} min: {:<3} consensus: {:<3}"
"".format(np.mean(self.h), np.std(self.h), max(self.h),
min(self.h), cons_h))
final_table.append(
"Avg. spot areas: {:<8.3f} std. dev: "
"{:<6.2f} max: {:<3} min: {:<3} consensus: {:<3}"
"".format(np.mean(self.a), np.std(self.a), max(self.a),
min(self.a), cons_a))
final_table.append(
"Avg. resolution: {:<8.3f} std. dev: "
"{:<6.2f} lowest: {:<6.3f} highest: {:<6.3f}"
"".format(np.mean(self.hres), np.std(self.hres),
max(self.hres), min(self.hres)))
final_table.append(
"Avg. number of spots: {:<8.3f} std. dev: {:<6.2f}"
"".format(np.mean(self.spots), np.std(self.spots)))
final_table.append(
"Avg. mosaicity: {:<8.3f} std. dev: {:<6.2f}"
"".format(np.mean(self.mos), np.std(self.mos)))
# If more than one integrated image, plot various summary graphs
if len(self.final_objects) > 1:
plot = Plotter(self.params, self.final_objects,
self.viz_dir)
if self.params.analysis.summary_graphs:
if (self.params.advanced.integrate_with == 'cctbx' and
self.params.cctbx.grid_search.type != None):
plot.plot_spotfinding_heatmap(write_files=True)
plot.plot_res_histogram(write_files=True)
med_beamX, med_beamY, pixel_size = plot.plot_beam_xy(
write_files=True, return_values=True)
else:
beamXY_info = plot.calculate_beam_xy()
beamX, beamY = beamXY_info[:2]
med_beamX = np.median(beamX)
med_beamY = np.median(beamY)
pixel_size = beamXY_info[-1]
final_table.append(
"Median Beam Center: X = {:<4.2f}, Y = {:<4.2f}"
"".format(med_beamX, med_beamY))
self.analysis_result.__setattr__('beamX_mm', med_beamX)
self.analysis_result.__setattr__('beamY_mm', med_beamY)
self.analysis_result.__setattr__('pixel_size', pixel_size)
self.analysis_result.__setattr__('final_table', final_table)
for item in final_table:
misc.main_log(self.logfile, item, (not self.gui_mode))
def run(self, gparams, target_phil=None, list_file=None):
''' Run initialization for IOTA GUI
gparams = IOTA parameters from the GUI elements in PHIL format
gtxt = text version of gparams
list_file = if "Write Input List" button pressed, specifies name
of list file
'''
self.params = gparams
self.target_phil = target_phil
# If input list for some reason isn't transmitted from main window, make it
if self.input_list is None:
self.input_list = self.make_input_list()
# Select range of images if turned on
if self.params.advanced.image_range.flag_on:
self.input_list = self.select_image_range(self.input_list)
# Select a random subset of images if turned on
if self.params.advanced.random_sample.flag_on and \
self.params.advanced.random_sample.number < len(self.input_list):
self.input_list = self.select_random_subset(self.input_list)
# Check for data not found
if len(self.input_list) == 0:
wx.MessageBox('ERROR: Data Not Found!', 'ERROR',
wx.OK | wx.ICON_ERROR)
return False
# If list-only option selected, output list only
if list_file != None:
with open(list_file, "w") as lf:
for i, input_file in enumerate(self.input_list, 1):
lf.write('{}\n'.format(input_file))
return True
# Run the sanity check procedure
if not self.sanity_check():
return False
# If fewer images than requested processors are supplied, set the number of
# processors to the number of images
if self.params.mp_method == 'multiprocessing':
if self.params.n_processors > len(self.input_list):
self.params.n_processors = len(self.input_list)
# Generate base folder paths
self.conv_base = misc.set_base_dir('converted_pickles',
out_dir=self.params.output)
self.int_base = misc.set_base_dir('integration',
out_dir=self.params.output)
self.obj_base = os.path.join(self.int_base, 'image_objects')
self.fin_base = os.path.join(self.int_base, 'final')
self.log_base = os.path.join(self.int_base, 'logs')
self.viz_base = os.path.join(self.int_base, 'visualization')
if str(self.params.advanced.temporary_output_folder).lower() in (
'none', ''):
self.tmp_base = os.path.join(self.int_base, 'tmp')
else:
self.tmp_base = os.path.join(
self.params.advanced.temporary_output_folder)
# Generate base folders
os.makedirs(self.int_base)
os.makedirs(self.obj_base)
os.makedirs(self.fin_base)
os.makedirs(self.log_base)
try:
if not os.path.isdir(self.tmp_base):
os.makedirs(self.tmp_base)
except OSError:
pass
# Determine input base
self.input_base = os.path.abspath(
os.path.dirname(os.path.commonprefix(self.input_list)))
# Initialize main log
self.logfile = os.path.abspath(os.path.join(self.int_base, 'iota.log'))
# Write target file and record its location in params
local_target_file = os.path.join(self.int_base, 'target.phil')
if type(self.target_phil) == list:
self.target_phil = '\n'.join(self.target_phil)
with open(local_target_file, 'w') as tf:
tf.write(self.target_phil)
if self.params.advanced.integrate_with == 'cctbx':
self.params.cctbx.target = local_target_file
elif self.params.advanced.integrate_with == 'dials':
self.params.dials.target = local_target_file
# Collect final params and convert to PHIL object
final_phil = inp.master_phil.format(python_object=self.params)
# Generate text of params
with misc.Capturing() as txt_output:
final_phil.show()
self.txt_out = ''
for one_output in txt_output:
self.txt_out += one_output + '\n'
# Log starting info
misc.main_log(self.logfile, '{:=^80} \n'.format(' IOTA MAIN LOG '))
misc.main_log(self.logfile,
'{:-^80} \n'.format(' SETTINGS FOR THIS RUN '))
misc.main_log(self.logfile, self.txt_out)
# Log cctbx.xfel / DIALS settings
misc.main_log(
self.logfile, '{:-^80} \n\n'
''.format(' TARGET FILE ({}) CONTENTS '
''.format(local_target_file)))
misc.main_log(self.logfile, self.target_phil)
return True
class Integrator(object):
""" Class for image integration (w/ grid search params) """
def __init__(self,
params,
source_image=None,
output_image=None,
viz=None,
log=None,
tag='grid search',
tmp_base=None,
gain=1,
single_image=False):
self.params = params
self.img = source_image
self.out_img = output_image
self.min_sigma = self.params.cctbx.selection.min_sigma
self.target = os.path.abspath(self.params.cctbx.target)
self.viz = viz
self.tag = tag
self.int_log = log
self.charts = self.params.analysis.charts
self.tmp_base = tmp_base
self.single_image = single_image
self.method = self.params.mp_method
self.queue = self.params.mp_queue
self.args = [
"target={}".format(self.target),
"indexing.data={}".format(self.img), "spots_pickle=None",
"subgroups_pickle=None", "refinements_pickle=None",
"rmsd_tolerance=5.0", "mosflm_rmsd_tolerance=5.0",
"integration.detector_gain={}".format(gain),
"indexing.verbose_cv=True"
]
# Add target unit cell if exists
if self.params.cctbx.target_unit_cell is not None:
t_uc = [
str(i)
for i in self.params.cctbx.target_unit_cell.parameters()
]
self.args.extend(['target_cell="{}"'.format(' '.join(t_uc))])
# Translate / add target lattice if exists
t_lat = self.params.cctbx.target_lattice_type
if t_lat is not None:
if t_lat == 'triclinic':
known_setting = 1
elif t_lat == 'monoclinic':
known_setting = 2
elif t_lat in ('orthorhombic', 'rhombohedral'):
known_setting = 5
elif t_lat == 'tetragonal':
known_setting = 9
elif t_lat == 'hexagonal':
known_setting = 12
elif t_lat == 'cubic':
known_setting = 22
self.args.extend(['known_setting={}'.format(known_setting)])
# Centering type if exists
t_ctype = self.params.cctbx.target_centering_type
if t_ctype is not None:
self.args.extend(['target_cell_centring_type={}'.format(t_ctype)])
# Resolution, if exists
hires = self.params.cctbx.resolution_limits.high
lowres = self.params.cctbx.resolution_limits.low
if hires is None:
hires = 1.5
if lowres is None:
lowres = 99.9
self.args.extend([
'force_method2_resolution_limit={}'.format(hires),
'distl_lowres_limit={}'.format(lowres),
'distl_highres_limit={}'.format(hires),
'distl.res.inner={}'.format(lowres),
'distl.res.outer={}'.format(hires)
])
# Add gain (necessary now)
self.args.extend(['integration.detector_gain={}'.format(gain)])
with open(os.path.join(self.params.output, 'test.txt'), 'w') as tf:
tf.write('\n'.join(self.args))
def integrate(self, grid_point):
""" Runs the integration module in cctbx.xfel; used by either grid-search or
final integration function. """
self.s = grid_point['sih']
self.h = grid_point['sph']
self.a = grid_point['spa']
args = self.args
# Generate advanced arguments (and PDF subfolder)
if self.charts and self.tag == 'grid search':
filename = os.path.basename(self.img).split('.')[0]
pdf_folder = os.path.join(self.viz, 'pdf_{}/s{}_h{}_a{}'\
''.format(filename, self.s, self.h, self.a))
if not os.path.exists(pdf_folder):
os.makedirs(pdf_folder)
self.args.extend([
"integration.enable_residual_map=True",
"integration.enable_residual_scatter=True",
"integration.mosaic.enable_AD14F7B=True",
"integration.graphics_backend=pdf",
"integration.pdf_output_dir={}".format(pdf_folder)
])
if self.tag == 'integrate':
args.append("indexing.completeness_pickle={}".format(self.out_img))
#Actually run integration using iota.bulletproof
error_message = ''
with misc.Capturing() as index_log:
arguments = [
"distl.minimum_signal_height={}".format(
str(self.s)), "distl.minimum_spot_height={}".format(
str(self.h)), "distl.minimum_spot_area={}".format(
str(self.a)), "indexing.open_wx_viewer=False"
] + list(args[1:])
tmppath = os.path.join(self.tmp_base,
str(uuid.uuid4()) + ".pickle")
assert not os.path.exists(tmppath)
# invoke the indexer in a way that will protect iota from any crashes
command = "iota.bulletproof {} {} {}" \
"".format(tmppath, self.target, " ".join(arguments))
try:
easy_run.fully_buffered(command,
join_stdout_stderr=True).show_stdout()
if not os.path.exists(tmppath):
print tmppath
print command
raise Exception("Indexing failed for an unknown reason")
# iota.bulletproof saves the needed results from indexing in a tmp file
result = easy_pickle.load(tmppath)
os.remove(tmppath)
if isinstance(result, str):
raise Exception(result)
else:
int_final = result
except Exception, e:
int_final = None
if hasattr(e, "classname"):
print e.classname, "for %s:" % self.img,
error_message = "{}: {}".format(
e.classname, e[0].replace('\n', ' ')[:50])
else:
print "Integration error for %s:" % self.img,
error_message = "{}".format(str(e).replace('\n', ' ')[:50])
print e
# Output results of integration (from the "info" object returned by
# run_one_index_core)
if int_final == None:
if error_message != '':
reason_for_failure = " - {}".format(error_message)
else:
reason_for_failure = ''
int_status = 'not integrated' + reason_for_failure
int_results = {'info': int_status}
elif int_final['observations'][0] == None:
int_status = 'no data recorded'
int_results = {'info': int_status}
else:
try:
obs = int_final['observations'][0]
cell = obs.unit_cell().parameters()
sg = int_final['pointgroup']
lres, hres = obs.d_max_min()
# Calculate number of spots w/ high I / sigmaI
Is = obs.data()
sigmas = obs.sigmas()
I_over_sigI = Is / sigmas
#spots = len(Is)
strong_spots = len(
[i for i in I_over_sigI if i >= self.min_sigma])
# Mosaicity parameters
mosaicity = round(
(int_final.get('ML_half_mosaicity_deg', [0])[0]), 6)
dom_size = int_final.get('ML_domain_size_ang', [0])[0]
ewald_proximal_volume = int_final.get('ewald_proximal_volume',
[0])[0]
# Assemble output for log file and/or integration result file
p_cell = "{:>6.2f}, {:>6.2f}, {:>6.2f}, {:>6.2f}, {:>6.2f}, {:>6.2f}"\
"".format(cell[0], cell[1], cell[2], cell[3], cell[4], cell[5])
int_status = 'RES: {:<4.2f} NSREF: {:<4} SG: {:<5} CELL: {}'\
''.format(hres, strong_spots, sg, p_cell)
int_results = {
'sg': sg,
'a': cell[0],
'b': cell[1],
'c': cell[2],
'alpha': cell[3],
'beta': cell[4],
'gamma': cell[5],
'wavelength': int_final['wavelength'],
'distance': int_final['distance'],
'beamX': int_final['xbeam'],
'beamY': int_final['ybeam'],
'strong': strong_spots,
'res': hres,
'lres': lres,
'mos': mosaicity,
'epv': ewald_proximal_volume,
'info': int_status,
'ok': True
}
except ValueError:
import traceback
print
print self.img
raise Exception("".join(
traceback.format_exception(*sys.exc_info())))
sys.exit()
# write integration logfile
if self.tag == 'integrate':
misc.main_log(self.int_log,
"{:-^100}\n{:-^100}\n{:-^100}\n"\
"".format("", " FINAL INTEGRATION: ", ""\
"S = {:>2}, H ={:>2}, A ={:>2} "\
"".format(self.s, self.h, self.a)))
else:
misc.main_log(self.int_log,
"{:-^100}\n".format(" INTEGRATION: "\
"S = {:>2}, H ={:>2}, A ={:>2} "\
"".format(self.s, self.h, self.a)))
for item in index_log:
misc.main_log(self.int_log, item)
misc.main_log(self.int_log, "\n[ {:^100} ]\n\n".format(int_status))
# In single-image mode, write a file with h, k, l, I, sigma
if self.single_image == True and self.tag == 'integrate':
hklI_filename = "{}.{}".format(
os.path.basename(self.out_img).split('.')[0], 'hkli')
hklI_file = os.path.join(os.path.dirname(self.out_img),
hklI_filename)
hklI = zip(obs.indices(), obs.data(), obs.sigmas())
for i in hklI:
with open(hklI_file, 'a') as f:
entry = '{},{},{},{},{}'.format(i[0][0], i[0][1], i[0][2],
i[1], i[2])
f.write('{}\n'.format(entry))
return int_results
def print_summary(self, write_files=True):
""" Prints summary and appends to general log file. Also outputs some of it
on stdout. Also writes out output list files.
"""
summary = []
misc.main_log(self.logfile,
"\n\n{:-^80}\n".format('SUMMARY'),
(not self.gui_mode))
summary.append('raw images read in: {}'\
''.format(len(self.all_objects)))
summary.append('raw images with no diffraction: {}'\
''.format(len(self.no_diff_objects)))
summary.append('raw images with diffraction: {}'\
''.format(len(self.diff_objects)))
if self.params.advanced.integrate_with == 'cctbx':
summary.append('failed indexing / integration: {}'\
''.format(len(self.not_int_objects)))
summary.append('failed prefilter: {}'\
''.format(len(self.filter_fail_objects)))
elif self.params.advanced.integrate_with == 'dials':
summary.append('failed spotfinding: {}'\
''.format(len(self.not_spf_objects)))
summary.append('failed indexing: {}'\
''.format(len(self.not_idx_objects)))
summary.append('failed integration: {}'\
''.format(len(self.not_int_objects)))
summary.append('final integrated pickles: {}'\
''.format(len(self.sorted_final_images)))
for item in summary:
misc.main_log(self.logfile, "{}".format(item), (not self.gui_mode))
misc.main_log(self.logfile, '\n\nIOTA version {0}'.format(self.ver))
misc.main_log(self.logfile, "{}\n".format(self.now))
# Write list files:
if write_files:
input_list_file = os.path.join(self.output_dir, 'input_images.lst')
blank_images_file = os.path.join(self.output_dir, 'blank_images.lst')
prefilter_fail_file = os.path.join(self.output_dir, 'failed_cctbx_prefilter.lst')
spotfinding_fail_file = os.path.join(self.output_dir, 'failed_dials_spotfinding.lst')
indexing_fail_file = os.path.join(self.output_dir, 'failed_dials_indexing.lst')
not_integrated_file = os.path.join(self.output_dir, 'not_integrated.lst')
integrated_file = os.path.join(self.output_dir, 'integrated.lst')
int_images_file = os.path.join(self.output_dir, 'int_image_pickles.lst')
if self.prime_data_path == None:
self.prime_data_path = integrated_file
if len(self.no_diff_objects) > 0:
with open(blank_images_file, 'w') as bif:
for obj in self.no_diff_objects:
bif.write('{}\n'.format(obj.conv_img))
if len(self.diff_objects) > 0:
with open(input_list_file, 'w') as ilf:
for obj in self.diff_objects:
ilf.write('{}\n'.format(obj.conv_img))
if len(self.not_int_objects) > 0:
with open(not_integrated_file, 'w') as nif:
for obj in self.not_int_objects:
nif.write('{}\n'.format(obj.conv_img))
if self.params.advanced.integrate_with == 'cctbx' and len(self.filter_fail_objects) > 0:
with open(prefilter_fail_file, 'w') as pff:
for obj in self.filter_fail_objects:
pff.write('{}\n'.format(obj.conv_img))
if self.params.advanced.integrate_with == 'dials':
if len(self.not_spf_objects) > 0:
with open(spotfinding_fail_file, 'w') as sff:
for obj in self.not_spf_objects:
sff.write('{}\n'.format(obj.conv_img))
if len(self.not_idx_objects) > 0:
with open(indexing_fail_file, 'w') as iff:
for obj in self.not_idx_objects:
iff.write('{}\n'.format(obj.conv_img))
if len(self.final_objects) > 0:
with open(integrated_file, 'w') as intf:
for obj in self.sorted_final_images:
intf.write('{}\n'.format(obj.final['final']))
with open(int_images_file, 'w') as ipf:
for obj in self.sorted_final_images:
ipf.write('{}\n'.format(obj.final['img']))
def unit_cell_analysis(self,
write_files=True):
""" Calls unit cell analysis module, which uses hierarchical clustering
(Zeldin, et al, Acta D, 2015) to split integration results according to
detected morphological groupings (if any). Most useful with preliminary
integration without target unit cell specified. """
# Will not run clustering if only one integration result found or if turned off
if len(self.final_objects) == 1:
unit_cell = (self.final_objects[0].final['a'],
self.final_objects[0].final['b'],
self.final_objects[0].final['c'],
self.final_objects[0].final['alpha'],
self.final_objects[0].final['beta'],
self.final_objects[0].final['gamma'])
point_group = self.final_objects[0].final['sg']
misc.main_log(self.logfile,
"\n\n{:-^80}\n".format(' UNIT CELL ANALYSIS '), True)
uc_line = "{:<6} {:^4}: {:<6.2f}, {:<6.2f}, {:<6.2f}, {:<6.2f}, "\
"{:<6.2f}, {:<6.2f}".format('(1)', point_group,
unit_cell[0], unit_cell[1], unit_cell[2],
unit_cell[3], unit_cell[4], unit_cell[5])
misc.main_log(self.logfile, uc_line, True)
self.cons_pg = point_group
self.cons_uc = unit_cell
else:
uc_table = []
uc_summary = []
# if self.params.analysis.run_clustering:
# # run hierarchical clustering analysis
# from xfel.clustering.cluster import Cluster
#
# counter = 0
# ucs = Cluster.from_files(self.pickles, use_b=True)
# clusters, _ = ucs.ab_cluster(self.params.analysis.cluster_threshold,
# log=False, write_file_lists=False,
# schnell=False, doplot=False)
# uc_table.append("\n\n{:-^80}\n"\
# "".format(' UNIT CELL ANALYSIS '))
#
# # extract clustering info and add to summary output list
# for cluster in clusters:
# sorted_pg_comp = sorted(cluster.pg_composition.items(),
# key=lambda x: -1 * x[1])
# pg_nums = [pg[1] for pg in sorted_pg_comp]
# cons_pg = sorted_pg_comp[np.argmax(pg_nums)]
#
# # write out lists of output pickles that comprise clusters with > 1 members
# if len(cluster.members) > 1:
# counter += 1
#
# # Sort clustered images by mosaicity, lowest to highest
# cluster_filenames = [j.path for j in cluster.members]
# clustered_objects = [i for i in self.final_objects if \
# i.final['final'] in cluster_filenames]
# sorted_cluster = sorted(clustered_objects,
# key=lambda i: i.final['mos'])
# # Write to file
# if write_files:
# output_file = os.path.join(self.output_dir, "uc_cluster_{}.lst".format(counter))
# for obj in sorted_cluster:
# with open(output_file, 'a') as scf:
# scf.write('{}\n'.format(obj.final['final']))
#
# mark_output = os.path.basename(output_file)
# else:
# mark_output = '*'
# output_file = None
# else:
# mark_output = ''
# output_file = None
#
# # format and record output
# uc_line = "{:<6} {:^4}: {:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), "\
# "{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), "\
# "{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}) "\
# "{}".format('({})'.format(len(cluster.members)), cons_pg[0],
# cluster.medians[0], cluster.stdevs[0],
# cluster.medians[1], cluster.stdevs[1],
# cluster.medians[2], cluster.stdevs[2],
# cluster.medians[3], cluster.stdevs[3],
# cluster.medians[4], cluster.stdevs[4],
# cluster.medians[5], cluster.stdevs[5],
# mark_output)
# uc_table.append(uc_line)
# uc_info = [len(cluster.members), cons_pg[0], cluster.medians,
# output_file, uc_line]
# uc_summary.append(uc_info)
# else:
# generate placeholder average unit cell - TEMPORARY, while we figure out how
# to eliminate scipy from cluster module
uc_table.append("\n\n{:-^80}\n" \
"".format(' UNIT CELL AVERAGING (no clustering) '))
uc_a = [i.final['a'] for i in self.final_objects]
uc_b = [i.final['b'] for i in self.final_objects]
uc_c = [i.final['c'] for i in self.final_objects]
uc_alpha = [i.final['alpha'] for i in self.final_objects]
uc_beta = [i.final['beta'] for i in self.final_objects]
uc_gamma = [i.final['gamma'] for i in self.final_objects]
uc_sg = [i.final['sg'] for i in self.final_objects]
cons_pg = Counter(uc_sg).most_common(1)[0][0]
uc_line = "{:<6} {:^4}: {:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), " \
"{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), " \
"{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}) " \
"{}".format('({})'.format(len(self.final_objects)), cons_pg,
np.median(uc_a), np.std(uc_a),
np.median(uc_b), np.std(uc_b),
np.median(uc_c), np.std(uc_c),
np.median(uc_alpha), np.std(uc_alpha),
np.median(uc_beta), np.std(uc_beta),
np.median(uc_gamma), np.std(uc_gamma), '')
unit_cell = (np.median(uc_a), np.median(uc_b), np.median(uc_c),
np.median(uc_alpha), np.median(uc_beta), np.median(uc_gamma))
uc_table.append(uc_line)
uc_info = [len(self.final_objects), cons_pg, unit_cell, None, uc_line]
uc_summary.append(uc_info)
uc_table.append('\nMost common unit cell:\n')
# select the most prevalent unit cell (most members in cluster)
uc_freqs = [i[0] for i in uc_summary]
uc_pick = uc_summary[np.argmax(uc_freqs)]
uc_table.append(uc_pick[4])
self.cons_pg = uc_pick[1]
self.cons_uc = uc_pick[2]
if uc_pick[3] != None:
self.prime_data_path = uc_pick[3]
for item in uc_table:
misc.main_log(self.logfile, item, (not self.gui_mode))
if self.gui_mode:
return self.cons_pg, self.cons_uc
def run(self):
self.args, self.phil_args = parse_command_args(self.iver,
self.help_message).parse_known_args()
# Check for type of input
if len(self.args.path) == 0 or self.args.path is None: # No input
parse_command_args(self.iver, self.help_message).print_help()
if self.args.default: # Write out default params and exit
help_out, txt_out = inp.print_params()
print '\n{:-^70}\n'.format('IOTA Parameters')
print help_out
inp.write_defaults(os.path.abspath(os.path.curdir), txt_out)
misc.iota_exit()
elif len(self.args.path) > 1: # If multiple paths / wildcards
file_list = ginp.make_input_list(self.args.path)
list_file = os.path.join(os.path.abspath(os.path.curdir), 'input.lst')
with open(list_file, 'w') as lf:
lf.write('\n'.join(file_list))
msg = "\nIOTA will run in AUTO mode using wildcard datapath:\n" \
"{} files found, compiled in {}\n".format(len(file_list), list_file)
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
list_file,
'auto',
self.now)
else: # If single path, check type
carg = os.path.abspath(self.args.path[0])
if os.path.exists(carg):
# If user provided a parameter file
if os.path.isfile(carg) and os.path.basename(carg).endswith('.param'):
msg = ''
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'file')
# If user provided a list of input files
elif os.path.isfile(carg) and os.path.basename(carg).endswith('.lst'):
msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg)
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'auto', self.now)
# If user provided a single filepath
elif os.path.isfile(carg) and not os.path.basename(carg).endswith('.lst'):
msg = "\nIOTA will run in SINGLE-FILE mode using {}:\n".format(carg)
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'auto', self.now)
# If user provided a data folder
elif os.path.isdir(carg):
msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg)
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'auto', self.now)
# If user provided gibberish
else:
print self.logo
print "ERROR: Invalid input! Need parameter filename or data folder."
misc.iota_exit()
# Identify indexing / integration program
if self.params.advanced.integrate_with == 'cctbx':
prg = " with CCTBX.XFEL\n"
elif self.params.advanced.integrate_with == 'dials':
prg = " with DIALS\n"
self.logo += prg
print self.logo
print '\n{}\n'.format(self.now)
if msg != '':
print msg
if self.args.analyze != None:
print 'ANALYSIS ONLY will be performed (analyzing run #{})'.format(
self.args.analyze)
self.analyze_prior_results('{:003d}'.format(int(self.args.analyze)))
misc.iota_exit()
if self.params.mp_method == 'mpi':
rank, size = misc.get_mpi_rank_and_size()
self.master_process = rank == 0
else:
self.master_process = True
# Call function to read input folder structure (or input file) and
# generate list of image file paths
if self.params.cctbx.selection.select_only.flag_on:
self.gs_img_objects = self.make_int_object_list()
self.input_list = [i.conv_img for i in self.gs_img_objects]
else:
self.input_list = self.make_input_list()
# Check for -l option, output list of input files and exit
if self.args.list:
list_file = os.path.abspath("{}/input.lst".format(os.curdir))
# Check if other files of this name exist under the current folder
list_folder = os.path.dirname(list_file)
list_files = [i for i in os.listdir(list_folder) if i.endswith(".lst")]
if len(list_files) > 0:
list_file = os.path.join(list_folder,
"input_{}.lst".format(len(list_files)))
print '\nINPUT LIST ONLY option selected'
print 'Input list in {} \n\n'.format(list_file)
with open(list_file, "w") as lf:
for i, input_file in enumerate(self.input_list, 1):
lf.write('{}\n'.format(input_file))
print "{}: {}".format(i, input_file)
lf.write('{}\n'.format(input_file))
print '\nExiting...\n\n'
misc.iota_exit()
# If fewer images than requested processors are supplied, set the number of
# processors to the number of images
if self.params.n_processors > len(self.input_list):
self.params.n_processors = len(self.input_list)
# Generate base folder paths
self.conv_base = misc.set_base_dir('converted_pickles', out_dir = self.params.output)
self.int_base = misc.set_base_dir('integration', out_dir = self.params.output)
self.obj_base = os.path.join(self.int_base, 'image_objects')
self.fin_base = os.path.join(self.int_base, 'final')
self.log_base = os.path.join(self.int_base, 'logs')
self.viz_base = os.path.join(self.int_base, 'visualization')
self.tmp_base = os.path.join('/tmp', '{}_{}'.format(os.getlogin(), time.time()))
# Generate base folders
os.makedirs(self.int_base)
os.makedirs(self.obj_base)
os.makedirs(self.fin_base)
os.makedirs(self.log_base)
os.makedirs(self.tmp_base)
# Determine input base
self.input_base = os.path.abspath(os.path.dirname(os.path.commonprefix(self.input_list)))
# Initialize main log
self.logfile = os.path.abspath(os.path.join(self.int_base, 'iota.log'))
# Log starting info
misc.main_log(self.logfile, '{:=^80} \n'.format(' IOTA MAIN LOG '))
misc.main_log(self.logfile, '{:-^80} \n'.format(' SETTINGS FOR THIS RUN '))
misc.main_log(self.logfile, self.txt_out)
if self.params.advanced.integrate_with == 'cctbx':
target_file = self.params.cctbx.target
elif self.params.advanced.integrate_with == 'dials':
target_file = self.params.dials.target
misc.main_log(self.logfile, '{:-^80} \n\n'
''.format(' TARGET FILE ({}) CONTENTS '
''.format(target_file)))
with open(target_file, 'r') as phil_file:
phil_file_contents = phil_file.read()
misc.main_log(self.logfile, phil_file_contents)
def unit_cell_analysis(self):
""" Calls unit cell analysis module, which uses hierarchical clustering
(Zeldin, et al, Acta D, 2015) to split integration results according to
detected morphological groupings (if any). Most useful with preliminary
integration without target unit cell specified. """
# Will not run clustering if only one integration result found or if turned off
if self.final_objects is None:
self.cons_uc = None
self.cons_pg = None
misc.main_log(self.logfile,
"\n\n{:-^80}\n".format(' UNIT CELL ANALYSIS '), True)
misc.main_log(self.logfile, '\n UNIT CELL CANNOT BE DETERMINED!',
True)
elif len(self.final_objects) == 1:
unit_cell = (self.final_objects[0].final['a'],
self.final_objects[0].final['b'],
self.final_objects[0].final['c'],
self.final_objects[0].final['alpha'],
self.final_objects[0].final['beta'],
self.final_objects[0].final['gamma'])
point_group = self.final_objects[0].final['sg']
misc.main_log(self.logfile,
"\n\n{:-^80}\n".format(' UNIT CELL ANALYSIS '), True)
uc_line = "{:<6} {:^4}: {:<6.2f}, {:<6.2f}, {:<6.2f}, {:<6.2f}, "\
"{:<6.2f}, {:<6.2f}".format('(1)', point_group,
unit_cell[0], unit_cell[1], unit_cell[2],
unit_cell[3], unit_cell[4], unit_cell[5])
misc.main_log(self.logfile, uc_line, True)
self.cons_pg = point_group
self.cons_uc = unit_cell
else:
uc_table = []
uc_summary = []
if self.params.analysis.run_clustering:
# run hierarchical clustering analysis
from xfel.clustering.cluster import Cluster
counter = 0
threshold = self.params.analysis.cluster_threshold
cluster_limit = self.params.analysis.cluster_limit
if self.params.analysis.cluster_n_images > 0:
n_images = self.params.analysis.cluster_n_images
else:
n_images = len(self.final_objects)
obj_list = []
if n_images < len(self.final_objects):
import random
for i in range(n_images):
random_number = random.randrange(
0, len(self.final_objects))
if self.final_objects[random_number] in obj_list:
while self.final_objects[
random_number] in obj_list:
random_number = random.randrange(
0, len(self.final_objects))
obj_list.append(self.final_objects[random_number])
else:
obj_list.append(self.final_objects[random_number])
if obj_list == []:
obj_list = self.final_objects
# Cluster from iterable (this doesn't keep filenames - bad!)
# with Capturing() as suppressed_output:
# uc_iterable = []
# for obj in obj_list:
# unit_cell = (float(obj.final['a']),
# float(obj.final['b']),
# float(obj.final['c']),
# float(obj.final['alpha']),
# float(obj.final['beta']),
# float(obj.final['gamma']),
# obj.final['sg'])
# uc_iterable.append(unit_cell)
# ucs = Cluster.from_iterable(iterable=uc_iterable)
# Cluster from files (slow, but will keep for now)
ucs = Cluster.from_files(pickle_list=self.pickles)
# Do clustering
clusters, _ = ucs.ab_cluster(threshold=threshold,
log=False,
write_file_lists=False,
schnell=False,
doplot=False)
uc_table.append("\n\n{:-^80}\n"\
"".format(' UNIT CELL ANALYSIS '))
# extract clustering info and add to summary output list
if cluster_limit is None:
if len(self.pickles) / 10 >= 10:
cluster_limit = 10
else:
cluster_limit = len(self.pickles) / 10
for cluster in clusters:
sorted_pg_comp = sorted(cluster.pg_composition.items(),
key=lambda x: -1 * x[1])
pg_nums = [pg[1] for pg in sorted_pg_comp]
cons_pg = sorted_pg_comp[np.argmax(pg_nums)]
if len(cluster.members) > cluster_limit:
counter += 1
# Sort clustered images by mosaicity, lowest to highest
cluster_filenames = [j.path for j in cluster.members]
clustered_objects = [i for i in self.final_objects if \
i.final['final'] in cluster_filenames]
sorted_cluster = sorted(clustered_objects,
key=lambda i: i.final['mos'])
# Write to file
if self.params.analysis.cluster_write_files:
output_file = os.path.join(
self.output_dir,
"uc_cluster_{}.lst".format(counter))
for obj in sorted_cluster:
with open(output_file, 'a') as scf:
scf.write('{}\n'.format(
obj.final['final']))
mark_output = os.path.basename(output_file)
else:
mark_output = '*'
output_file = None
# Populate clustering info for GUI display
uc_no_stdev = "{:<6.2f} {:<6.2f} {:<6.2f} " \
"{:<6.2f} {:<6.2f} {:<6.2f} " \
"".format(cluster.medians[0], cluster.medians[1],
cluster.medians[2], cluster.medians[3],
cluster.medians[4], cluster.medians[5])
cluster_info = {
'number': len(cluster.members),
'pg': cons_pg[0],
'uc': uc_no_stdev,
'filename': mark_output
}
self.clusters.append(cluster_info)
else:
mark_output = ''
output_file = None
# format and record output
uc_line = "{:<6} {:^4}: {:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), "\
"{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), "\
"{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}) "\
"{}".format('({})'.format(len(cluster.members)), cons_pg[0],
cluster.medians[0], cluster.stdevs[0],
cluster.medians[1], cluster.stdevs[1],
cluster.medians[2], cluster.stdevs[2],
cluster.medians[3], cluster.stdevs[3],
cluster.medians[4], cluster.stdevs[4],
cluster.medians[5], cluster.stdevs[5],
mark_output)
uc_table.append(uc_line)
lattices = ', '.join(
['{} ({})'.format(i[0], i[1]) for i in sorted_pg_comp])
uc_info = [
len(cluster.members), cons_pg[0], cluster.medians,
output_file, uc_line, lattices
]
uc_summary.append(uc_info)
else:
# generate average unit cell
uc_table.append("\n\n{:-^80}\n" \
"".format(' UNIT CELL AVERAGING (no clustering) '))
uc_a = [i.final['a'] for i in self.final_objects]
uc_b = [i.final['b'] for i in self.final_objects]
uc_c = [i.final['c'] for i in self.final_objects]
uc_alpha = [i.final['alpha'] for i in self.final_objects]
uc_beta = [i.final['beta'] for i in self.final_objects]
uc_gamma = [i.final['gamma'] for i in self.final_objects]
uc_sg = [i.final['sg'] for i in self.final_objects]
cons_pg = Counter(uc_sg).most_common(1)[0][0]
all_pgs = Counter(uc_sg).most_common()
uc_line = "{:<6} {:^4}: {:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), " \
"{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), " \
"{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}) " \
"{}".format('({})'.format(len(self.final_objects)), cons_pg,
np.median(uc_a), np.std(uc_a),
np.median(uc_b), np.std(uc_b),
np.median(uc_c), np.std(uc_c),
np.median(uc_alpha), np.std(uc_alpha),
np.median(uc_beta), np.std(uc_beta),
np.median(uc_gamma), np.std(uc_gamma), '')
unit_cell = (np.median(uc_a), np.median(uc_b), np.median(uc_c),
np.median(uc_alpha), np.median(uc_beta),
np.median(uc_gamma))
uc_table.append(uc_line)
lattices = ', '.join(
['{} ({})'.format(i[0], i[1]) for i in all_pgs])
uc_info = [
len(self.final_objects), cons_pg, unit_cell, None, uc_line,
lattices
]
uc_summary.append(uc_info)
uc_table.append('\nMost common unit cell:\n')
# select the most prevalent unit cell (most members in cluster)
uc_freqs = [i[0] for i in uc_summary]
uc_pick = uc_summary[np.argmax(uc_freqs)]
uc_table.append(uc_pick[4])
uc_table.append('\nBravais Lattices in Biggest Cluster: {}'
''.format(uc_pick[5]))
self.cons_pg = uc_pick[1]
self.cons_uc = uc_pick[2]
if uc_pick[3] != None:
self.prime_data_path = uc_pick[3]
for item in uc_table:
misc.main_log(self.logfile, item, (not self.gui_mode))
self.analysis_result.__setattr__('clusters', self.clusters)
self.analysis_result.__setattr__('cons_pg', self.cons_pg)
self.analysis_result.__setattr__('cons_uc', self.cons_uc)
if self.gui_mode:
return self.cons_pg, self.cons_uc, self.clusters