def import_int_file(self, init):
""" Replaces path settings in imported image object with new settings
NEED TO RE-DO LATER """
if os.path.isfile(self.abort_file):
self.fail = 'aborted'
return self
# Generate paths to output files
self.params = init.params
self.main_log = init.logfile
self.input_base = init.input_base
self.conv_base = init.conv_base
self.int_base = init.int_base
self.obj_base = init.obj_base
self.fin_base = init.fin_base
self.log_base = init.log_base
self.viz_base = init.viz_base
self.obj_path = misc.make_image_path(self.conv_img, self.input_base,
self.obj_base)
filename = misc.make_filename(self.conv_img)
self.obj_file = os.path.abspath(
os.path.join(self.obj_path, filename + ".int"))
self.fin_path = misc.make_image_path(self.conv_img, self.input_base,
self.fin_base)
self.log_path = misc.make_image_path(self.conv_img, self.input_base,
self.log_base)
self.fin_file = os.path.abspath(
os.path.join(self.fin_path, filename + "_int.pickle"))
self.final['final'] = self.fin_file
self.final['img'] = self.conv_img
self.viz_path = misc.make_image_path(self.conv_img, self.input_base,
self.viz_base)
self.viz_file = os.path.join(self.viz_path, filename + "_int.png")
# Create actual folders (if necessary)
try:
if not os.path.isdir(self.obj_path):
os.makedirs(self.obj_path)
if not os.path.isdir(self.fin_path):
os.makedirs(self.fin_path)
if not os.path.isdir(self.viz_path):
os.makedirs(self.viz_path)
except OSError:
pass
# Grid search / integration log file
self.int_log = os.path.join(
self.log_path,
os.path.basename(self.conv_img).split('.')[0] + '.tmp')
# Reset status to 'grid search' to pick up at selection (if no fail)
if self.fail == None:
self.status = 'bypass grid search'
return self
def import_image(self):
""" Image conversion:
- Writes out data in pickle format (cctbx.xfel only)
- Moves beam center into center of image, crops / pads image (cctbx.xfel only)
- Adjusts beam center and distance (optional)
- Thresholds and masks beamstop shadow (optional)
"""
if os.path.isfile(self.abort_file):
self.fail = 'aborted'
return self
# Load image
img_data, img_type = self.load_image()
self.status = 'loaded'
if img_data is None:
self.log_path = misc.make_image_path(self.conv_img,
self.input_base,
self.log_base)
self.int_log = os.path.join(
self.log_path,
misc.make_filename(self.conv_img) + '.tmp')
self.log_info.append('\n{:-^100}\n'.format(self.raw_img))
self.log_info.append('FAILED TO IMPORT')
self.status = 'failed import'
self.fail = 'failed import'
if not self.params.image_conversion.convert_only:
self.obj_path = misc.make_image_path(self.conv_img,
self.input_base,
self.obj_base)
rej_name = filter(None, misc.make_filename(
self.conv_img))[0] + '_reject.int'
self.obj_file = os.path.abspath(
os.path.join(self.obj_path, rej_name))
try:
if not os.path.isdir(self.obj_path):
os.makedirs(self.obj_path)
except OSError:
pass
# ep.dump(self.obj_file, self)
return self
# if DIALS is selected, change image type to skip conversion step
if self.params.advanced.integrate_with == 'dials':
img_type = 'dials_input'
if self.params.dials.auto_threshold:
try:
beam_x_px = int(img_data['BEAM_CENTER_X'] /
img_data['PIXEL_SIZE'])
beam_y_px = int(img_data['BEAM_CENTER_Y'] /
img_data['PIXEL_SIZE'])
data_array = img_data['DATA'].as_numpy_array().astype(
float)
self.center_int = np.nanmax(
data_array[beam_y_px - 20:beam_y_px + 20,
beam_x_px - 20:beam_x_px + 20])
except Exception, e:
print 'IMPORT ERROR: ', e
def import_image(self):
""" Image conversion:
- Writes out data in pickle format (cctbx.xfel only)
- Moves beam center into center of image, crops / pads image (cctbx.xfel only)
- Adjusts beam center and distance (optional)
- Thresholds and masks beamstop shadow (optional)
"""
if os.path.isfile(self.abort_file):
self.fail = 'aborted'
return self
# Load image
img_data, img_type = self.load_image()
self.status = 'loaded'
if img_data is None:
self.log_path = misc.make_image_path(self.conv_img, self.input_base,
self.log_base)
self.int_log = os.path.join(self.log_path,
misc.make_filename(self.conv_img) + '.tmp')
self.log_info.append('\n{:-^100}\n'.format(self.raw_img))
self.log_info.append('FAILED TO IMPORT')
self.status = 'failed import'
self.fail = 'failed import'
if not self.params.image_conversion.convert_only:
self.obj_path = misc.make_image_path(self.conv_img, self.input_base,
self.obj_base)
rej_name = filter(None, misc.make_filename(self.conv_img))[0] + '_reject.int'
self.obj_file = os.path.abspath(os.path.join(self.obj_path, rej_name))
try:
if not os.path.isdir(self.obj_path):
os.makedirs(self.obj_path)
except OSError:
pass
# ep.dump(self.obj_file, self)
return self
# if DIALS is selected, change image type to skip conversion step
if self.params.advanced.integrate_with == 'dials':
img_type = 'dials_input'
if self.params.dials.auto_threshold:
try:
beam_x_px = int(img_data['BEAM_CENTER_X'] / img_data['PIXEL_SIZE'])
beam_y_px = int(img_data['BEAM_CENTER_Y'] / img_data['PIXEL_SIZE'])
data_array = img_data['DATA'].as_numpy_array().astype(float)
self.center_int = np.nanmax(data_array[beam_y_px - 20:beam_y_px + 20,
beam_x_px - 20:beam_x_px + 20])
except Exception as e:
print 'IMPORT ERROR: ', e
# Log initial image information
self.log_info.append('\n{:-^100}\n'.format(self.raw_img))
self.log_info.append('Imported image : {}'.format(self.raw_img))
self.log_info.append('Parameters : BEAM_X = {:<4.2f}, BEAM_Y = {:<4.2f}, '\
'PIXEL_SIZE = {:<8.6f}, IMG_SIZE = {:<4} X {:<4}, '\
'DIST = {}'.format(img_data['BEAM_CENTER_X'],
img_data['BEAM_CENTER_Y'],
img_data['PIXEL_SIZE'],
img_data['SIZE1'],
img_data['SIZE2'],
img_data['DISTANCE']))
# Deactivate image squaring if beam center is in the center of the image
# CCTBX.XFEL ONLY (Need a better displacement cutoff)
if ( self.params.advanced.integrate_with == 'dials' or
abs(img_data['BEAM_CENTER_X'] - img_data['BEAM_CENTER_Y']) < 0.1
):
self.params.image_conversion.square_mode = 'no_modification'
# Check if conversion/modification is required and carry them out
if self.params.advanced.integrate_with != 'dials' and \
(
img_type == 'raw' or
self.params.image_conversion.square_mode != "no_modification" or
self.params.image_conversion.beam_center.x != 0 or
self.params.image_conversion.beam_center.y != 0 or
self.params.image_conversion.beamstop != 0 or
self.params.image_conversion.distance != 0
):
# Check for and/or create a converted pickles folder
try:
if not os.path.isdir(self.conv_base):
os.makedirs(self.conv_base)
except OSError:
pass
# Generate converted image pickle filename
rename_choice = str(self.params.image_conversion.rename_pickle).lower()
if rename_choice in ("keep_file_structure", "none"):
img_path = misc.make_image_path(self.raw_img, self.input_base,
self.conv_base)
img_filename = misc.make_filename(self.raw_img) + ".pickle"
self.conv_img = os.path.abspath(os.path.join(img_path, img_filename))
try:
if not os.path.isdir(img_path):
os.makedirs(img_path)
except OSError:
pass
else:
self.input_base = self.conv_base
if rename_choice == "auto_filename":
prefix = self.user_id
elif rename_choice == "custom_filename":
prefix = self.params.image_conversion.rename_pickle_prefix
else:
prefix = 'iota'
number = int(os.path.basename(self.conv_base))
self.conv_img = os.path.abspath(os.path.join(self.conv_base,
"{}_{}_{:05d}.pickle".format(prefix, number, self.img_index)))
# Convert raw image to image pickle
beamstop = self.params.image_conversion.beamstop
distance = self.params.image_conversion.distance
beam_center = [self.params.image_conversion.beam_center.x,
self.params.image_conversion.beam_center.y]
square = self.params.image_conversion.square_mode
mask_file = self.params.image_conversion.mask
if mask_file is not None:
img_data = self.apply_mask_from_file(img_data, mask_file)
if beam_center != [0,0]:
pixel_size = img_data['PIXEL_SIZE']
img_data['BEAM_CENTER_X'] = int(round(beam_center[0] * pixel_size))
img_data['BEAM_CENTER_Y'] = int(round(beam_center[1] * pixel_size))
if distance != 0:
img_data['DISTANCE'] = distance
if square in ('crop', 'pad'):
img_data = self.square_pickle(img_data)
if beamstop != 0:
img_data = self.mask_image(img_data)
# Log converted image information
self.log_info.append('Converted image : {}'.format(self.conv_img))
self.log_info.append('Parameters : BEAM_X = {:<4.2f}, BEAM_Y = {:<4.2f}, '\
'PIXEL_SIZE = {:<8.6f}, IMG_SIZE = {:<4} X {:<4}, '\
'DIST = {}'.format(img_data['BEAM_CENTER_X'],
img_data['BEAM_CENTER_Y'],
img_data['PIXEL_SIZE'],
img_data['SIZE1'],
img_data['SIZE2'],
img_data['DISTANCE']))
self.status = 'converted'
# Save converted image pickle
ep.dump(self.conv_img, img_data)
# Triage image (i.e. check for usable diffraction, using selected method)
if str(self.params.image_triage.type).lower() in ('simple', 'grid_search'):
if self.params.advanced.integrate_with == 'cctbx':
from iota.components.iota_cctbx import Triage
triage = Triage(self.conv_img, self.gain, self.params)
self.fail, log_entry, self.hmed, self.amed = triage.triage_image()
elif self.params.advanced.integrate_with == 'dials':
from iota.components.iota_dials import Triage
triage = Triage(img=self.conv_img,
gain=self.gain,
center_intensity=self.center_int,
params=self.params)
self.fail, log_entry = triage.triage_image()
self.log_info.append(log_entry)
self.status = 'triaged'
else:
self.fail = None
# Generate integration result dictionary for cctbx.xfel or DIALS
if self.params.advanced.integrate_with == 'cctbx':
gs_type = str(self.params.cctbx.grid_search.type).lower()
if gs_type == 'smart':
self.hrange = 1
self.arange = 1
elif gs_type in ('none', 'no_grid_search'):
self.hrange = 0
self.arange = 0
else:
self.hrange = self.params.cctbx.grid_search.height_range
self.arange = self.params.cctbx.grid_search.area_range
self.grid_points = []
self.generate_grid()
elif self.params.advanced.integrate_with == 'dials':
self.final = {'img':self.conv_img, 'a':0, 'b':0, 'c':0, 'alpha':0,
'beta':0, 'gamma':0, 'sg':'','strong':0, 'res':0,
'lres':0, 'mos':0, 'epv':0, 'info':'','final':None,
'wavelength': 0, 'distance':0, 'beamX': 0, 'beamY':0,
'program':'dials'}
# Generate names for output folders and files:
filename = misc.make_filename(self.conv_img)
self.test_filename = filename
if not self.params.image_conversion.convert_only:
self.obj_path = misc.make_image_path(self.conv_img, self.input_base, self.obj_base)
self.obj_file = os.path.abspath(os.path.join(self.obj_path,
filename + ".int"))
self.fin_path = misc.make_image_path(self.conv_img, self.input_base, self.fin_base)
self.log_path = misc.make_image_path(self.conv_img, self.input_base, self.log_base)
self.fin_file = os.path.abspath(os.path.join(self.fin_path,
"int_{}.pickle".format(filename)))
self.final['final'] = self.fin_file
self.final['img'] = self.conv_img
self.int_log = os.path.join(self.log_path, filename + '.tmp')
self.viz_path = misc.make_image_path(self.conv_img, self.input_base, self.viz_base)
self.viz_file = os.path.join(self.viz_path, "int_{}.png".format(filename))
# Create actual folders (if necessary)f
try:
if not os.path.isdir(self.obj_path):
os.makedirs(self.obj_path)
if not os.path.isdir(self.fin_path):
os.makedirs(self.fin_path)
if not os.path.isdir(self.log_path):
os.makedirs(self.log_path)
if not os.path.isdir(self.viz_path):
os.makedirs(self.viz_path)
except OSError:
pass
self.status = 'imported'
# Save image object to file
ep.dump(self.obj_file, self)
# If conversion only option is selected, write conversion info to log
else:
log_entry = "\n".join(self.log_info)
misc.main_log(self.main_log, log_entry)
return self
):
# Check for and/or create a converted pickles folder
try:
if not os.path.isdir(self.conv_base):
os.makedirs(self.conv_base)
except OSError:
pass
# Generate converted image pickle filename
rename_choice = str(
self.params.image_conversion.rename_pickle).lower()
if rename_choice in ("keep_file_structure", "none"):
img_path = misc.make_image_path(self.raw_img, self.input_base,
self.conv_base)
img_filename = misc.make_filename(self.raw_img) + ".pickle"
self.conv_img = os.path.abspath(
os.path.join(img_path, img_filename))
try:
if not os.path.isdir(img_path):
os.makedirs(img_path)
except OSError:
pass
else:
if rename_choice == "auto_filename":
prefix = self.user_id
elif rename_choice == "custom_filename":
prefix = self.params.image_conversion.rename_pickle_prefix
number = int(os.path.basename(self.conv_base))
self.conv_img = os.path.abspath(
os.path.join(
def __init__(self,
source_image,
object_folder,
int_folder,
final_filename,
final,
logfile,
gain=0.32,
center_intensity=0,
params=None):
'''Initialise the script.'''
self.params = params
self.int_base = int_folder
# Read settings from the DIALS target (.phil) file
# If none is provided, use default settings (and may God have mercy)
if self.params.dials.target != None:
with open(self.params.dials.target, 'r') as settings_file:
settings_file_contents = settings_file.read()
settings = parse(settings_file_contents)
current_phil = phil_scope.fetch(source=settings)
else:
current_phil = phil_scope
self.phil = current_phil.extract()
# Set general file-handling settings
file_basename = misc.make_filename(source_image)
self.phil.output.datablock_filename = "{}/{}.json".format(
object_folder, file_basename)
self.phil.output.indexed_filename = "{}/{}_indexed.pickle".format(
object_folder, file_basename)
self.phil.output.strong_filename = "{}/{}_strong.pickle".format(
object_folder, file_basename)
self.phil.output.refined_experiments_filename = "{}/{}_refined_experiments.json".format(
object_folder, file_basename)
self.phil.output.integrated_experiments_filename = "{}/{}_integrated_experiments.json".format(
object_folder, file_basename)
self.phil.output.integrated_filename = "{}/{}_integrated.pickle".format(
object_folder, file_basename)
self.phil.output.profile_filename = "{}/{}_profile.phil".format(
object_folder, file_basename)
self.phil.output.integration_pickle = final_filename
self.int_log = logfile
# Set customized parameters
beamX = self.params.image_conversion.beam_center.x
beamY = self.params.image_conversion.beam_center.y
if beamX != 0 or beamY != 0:
self.phil.geometry.detector.slow_fast_beam_centre = '{} {}'.format(
beamY, beamX)
if self.params.image_conversion.distance != 0:
self.phil.geometry.detector.distance = self.params.image_conversion.distance
if self.params.image_conversion.mask is not None:
self.phil.spotfinder.lookup.mask = self.params.image_conversion.mask
self.phil.integration.lookup.mask = self.params.image_conversion.mask
if self.params.dials.target_space_group is not None:
sg = self.params.dials.target_space_group
self.phil.indexing.known_symmetry.space_group = sg
if self.params.dials.target_unit_cell is not None:
uc = self.params.dials.target_unit_cell
self.phil.indexing.known_symmetry.unit_cell = uc
if self.params.dials.use_fft3d:
self.phil.indexing.stills.method_list = [
'fft1d', 'fft3d', 'real_space_grid_search'
]
if self.params.dials.significance_filter.flag_on:
if self.params.dials.significance_filter.sigma is not None:
sigma = self.params.dials.significance_filter.sigma
self.phil.significance_filter.enable = True
self.phil.significance_filter.isigi_cutoff = sigma
# # Write target file for this IOTA run
# with misc.Capturing() as output:
# mod_phil = current_phil.format(python_object=self.phil)
# mod_phil.show()
# txt_out = ''
# for one_output in output:
# txt_out += one_output + '\n'
# local_target_file = os.path.join(self.int_base, 'target.phil')
# with open(local_target_file, 'w') as tf:
# tf.write(txt_out)
self.img = [source_image]
self.obj_base = object_folder
self.fail = None
self.frame = None
self.final = final
self.final['final'] = final_filename
self.datablock = DataBlockFactory.from_filenames(self.img)[0]
self.obj_filename = "int_{}".format(os.path.basename(self.img[0]))
# Auto-set threshold and gain (not saved for target.phil)
if self.params.dials.auto_threshold:
# This is still experimental and I'm not sure if it does anything...
# rad_avg = RadAverageCalculator(datablock=self.datablock)
# means, res = rad_avg.make_radial_average(num_bins=20, lowres=90)
# threshold = int(np.min(means) * 5)
threshold = int(center_intensity)
self.phil.spotfinder.threshold.dispersion.global_threshold = threshold
if self.params.advanced.estimate_gain:
self.phil.spotfinder.threshold.dispersion.gain = gain