def import_int_file(self, init):
""" Replaces path settings in imported image object with new settings
NEED TO RE-DO LATER """
if os.path.isfile(self.abort_file):
self.fail = 'aborted'
return self
# Generate paths to output files
self.params = init.params
self.main_log = init.logfile
self.input_base = init.input_base
self.conv_base = init.conv_base
self.int_base = init.int_base
self.obj_base = init.obj_base
self.fin_base = init.fin_base
self.log_base = init.log_base
self.viz_base = init.viz_base
self.obj_path = misc.make_image_path(self.conv_img, self.input_base,
self.obj_base)
filename = misc.make_filename(self.conv_img)
self.obj_file = os.path.abspath(
os.path.join(self.obj_path, filename + ".int"))
self.fin_path = misc.make_image_path(self.conv_img, self.input_base,
self.fin_base)
self.log_path = misc.make_image_path(self.conv_img, self.input_base,
self.log_base)
self.fin_file = os.path.abspath(
os.path.join(self.fin_path, filename + "_int.pickle"))
self.final['final'] = self.fin_file
self.final['img'] = self.conv_img
self.viz_path = misc.make_image_path(self.conv_img, self.input_base,
self.viz_base)
self.viz_file = os.path.join(self.viz_path, filename + "_int.png")
# Create actual folders (if necessary)
try:
if not os.path.isdir(self.obj_path):
os.makedirs(self.obj_path)
if not os.path.isdir(self.fin_path):
os.makedirs(self.fin_path)
if not os.path.isdir(self.viz_path):
os.makedirs(self.viz_path)
except OSError:
pass
# Grid search / integration log file
self.int_log = os.path.join(
self.log_path,
os.path.basename(self.conv_img).split('.')[0] + '.tmp')
# Reset status to 'grid search' to pick up at selection (if no fail)
if self.fail == None:
self.status = 'bypass grid search'
return self
def import_int_file(self, init):
""" Replaces path settings in imported image object with new settings
NEED TO RE-DO LATER """
if os.path.isfile(self.abort_file):
self.fail = 'aborted'
return self
# Generate paths to output files
self.params = init.params
self.main_log = init.logfile
self.input_base = init.input_base
self.conv_base = init.conv_base
self.int_base = init.int_base
self.obj_base = init.obj_base
self.fin_base = init.fin_base
self.viz_base = init.viz_base
self.obj_path = misc.make_image_path(self.conv_img, self.input_base, self.obj_base)
self.obj_file = os.path.abspath(os.path.join(self.obj_path,
os.path.basename(self.conv_img).split('.')[0] + ".int"))
self.fin_path = misc.make_image_path(self.conv_img, self.input_base, self.fin_base)
self.fin_file = os.path.abspath(os.path.join(self.fin_path,
os.path.basename(self.conv_img).split('.')[0] + "_int.pickle"))
self.final['final'] = self.fin_file
self.final['img'] = self.conv_img
self.viz_path = misc.make_image_path(self.conv_img, self.input_base, self.viz_base)
self.viz_file = os.path.join(self.viz_path,
os.path.basename(self.conv_img).split('.')[0] + "_int.png")
# Create actual folders (if necessary)
try:
if not os.path.isdir(self.obj_path):
os.makedirs(self.obj_path)
if not os.path.isdir(self.fin_path):
os.makedirs(self.fin_path)
if not os.path.isdir(self.viz_path):
os.makedirs(self.viz_path)
except OSError:
pass
# Grid search / integration log file
self.int_log = os.path.join(self.fin_path,
os.path.basename(self.conv_img).split('.')[0] + '.tmp')
# Reset status to 'grid search' to pick up at selection (if no fail)
if self.fail == None:
self.status = 'bypass grid search'
return self
def import_image(self):
""" Image conversion:
- Writes out data in pickle format (cctbx.xfel only)
- Moves beam center into center of image, crops / pads image (cctbx.xfel only)
- Adjusts beam center and distance (optional)
- Thresholds and masks beamstop shadow (optional)
"""
if os.path.isfile(self.abort_file):
self.fail = 'aborted'
return self
# Load image
img_data, img_type = self.load_image()
self.status = 'loaded'
if img_data is None:
self.log_path = misc.make_image_path(self.conv_img,
self.input_base,
self.log_base)
self.int_log = os.path.join(
self.log_path,
os.path.basename(self.conv_img).split('.')[0] + '.tmp')
self.log_info.append('\n{:-^100}\n'.format(self.raw_img))
self.log_info.append('FAILED TO IMPORT')
self.status = 'failed import'
self.fail = 'failed import'
if not self.params.image_conversion.convert_only:
self.obj_path = misc.make_image_path(self.conv_img,
self.input_base,
self.obj_base)
rej_name = filter(
None,
os.path.basename(
self.conv_img).split('.'))[0] + '_reject.int'
self.obj_file = os.path.abspath(
os.path.join(self.obj_path, rej_name))
try:
if not os.path.isdir(self.obj_path):
os.makedirs(self.obj_path)
except OSError:
pass
ep.dump(self.obj_file, self)
return self
# if DIALS is selected, change image type to skip conversion step
if self.params.advanced.integrate_with == 'dials':
img_type = 'dials_input'
if self.params.dials.auto_threshold:
beam_x_px = img_data['BEAM_CENTER_X'] / img_data['PIXEL_SIZE']
beam_y_px = img_data['BEAM_CENTER_Y'] / img_data['PIXEL_SIZE']
data_array = img_data['DATA'].as_numpy_array().astype(float)
self.center_int = np.nanmax(
data_array[beam_y_px - 20:beam_y_px + 20,
beam_x_px - 20:beam_x_px + 20])
# Log initial image information
self.log_info.append('\n{:-^100}\n'.format(self.raw_img))
self.log_info.append('Imported image : {}'.format(self.raw_img))
self.log_info.append('Parameters : BEAM_X = {:<4.2f}, BEAM_Y = {:<4.2f}, '\
'PIXEL_SIZE = {:<8.6f}, IMG_SIZE = {:<4} X {:<4}, '\
'DIST = {}'.format(img_data['BEAM_CENTER_X'],
img_data['BEAM_CENTER_Y'],
img_data['PIXEL_SIZE'],
img_data['SIZE1'],
img_data['SIZE2'],
img_data['DISTANCE']))
# Deactivate image squaring if beam center is in the center of the image
# CCTBX.XFEL ONLY (Need a better displacement cutoff)
if (self.params.advanced.integrate_with == 'dials'
or abs(img_data['BEAM_CENTER_X'] - img_data['BEAM_CENTER_Y']) <
0.1):
self.params.image_conversion.square_mode = 'no_modification'
# Check if conversion/modification is required and carry them out
if self.params.advanced.integrate_with != 'dials' and \
(
img_type == 'raw' or
self.params.image_conversion.square_mode != "no_modification" or
self.params.image_conversion.beam_center.x != 0 or
self.params.image_conversion.beam_center.y != 0 or
self.params.image_conversion.beamstop != 0 or
self.params.image_conversion.distance != 0
):
# Check for and/or create a converted pickles folder
try:
if not os.path.isdir(self.conv_base):
os.makedirs(self.conv_base)
except OSError:
pass
# Generate converted image pickle filename
rename_choice = str(
self.params.image_conversion.rename_pickle).lower()
if rename_choice in ("keep_file_structure", "none"):
img_path = misc.make_image_path(self.raw_img, self.input_base,
self.conv_base)
img_filename = os.path.basename(
self.raw_img).split('.')[0] + ".pickle"
self.conv_img = os.path.abspath(
os.path.join(img_path, img_filename))
try:
if not os.path.isdir(img_path):
os.makedirs(img_path)
except OSError:
pass
else:
if rename_choice == "auto_filename":
prefix = self.user_id
elif rename_choice == "custom_filename":
prefix = self.params.image_conversion.rename_pickle_prefix
number = int(os.path.basename(self.conv_base))
self.conv_img = os.path.abspath(
os.path.join(
self.conv_base,
"{}_{}_{:05d}.pickle".format(prefix, number,
self.img_index)))
# Convert raw image to image pickle
beamstop = self.params.image_conversion.beamstop
distance = self.params.image_conversion.distance
beam_center = [
self.params.image_conversion.beam_center.x,
self.params.image_conversion.beam_center.y
]
square = self.params.image_conversion.square_mode
mask_file = self.params.image_conversion.mask
invert = self.params.image_conversion.invert_boolean_mask
if mask_file is not None:
img_data = self.apply_mask_from_file(img_data, mask_file,
invert)
if beam_center != [0, 0]:
pixel_size = img_data['PIXEL_SIZE']
img_data['BEAM_CENTER_X'] = int(
round(beam_center[0] * pixel_size))
img_data['BEAM_CENTER_Y'] = int(
round(beam_center[1] * pixel_size))
if distance != 0:
img_data['DISTANCE'] = distance
if square in ('crop', 'pad'):
img_data = self.square_pickle(img_data)
if beamstop != 0:
img_data = self.mask_image(img_data)
# Log converted image information
self.log_info.append('Converted image : {}'.format(self.conv_img))
self.log_info.append('Parameters : BEAM_X = {:<4.2f}, BEAM_Y = {:<4.2f}, '\
'PIXEL_SIZE = {:<8.6f}, IMG_SIZE = {:<4} X {:<4}, '\
'DIST = {}'.format(img_data['BEAM_CENTER_X'],
img_data['BEAM_CENTER_Y'],
img_data['PIXEL_SIZE'],
img_data['SIZE1'],
img_data['SIZE2'],
img_data['DISTANCE']))
self.input_base = self.conv_base
self.status = 'converted'
# Save converted image pickle
ep.dump(self.conv_img, img_data)
# Triage image (i.e. check for usable diffraction, using selected method)
if str(self.params.image_triage.type).lower() in ('simple',
'grid_search'):
if self.params.advanced.integrate_with == 'cctbx':
from iota.components.iota_cctbx import Triage
triage = Triage(self.conv_img, self.gain, self.params)
self.fail, log_entry, self.hmed, self.amed = triage.triage_image(
)
elif self.params.advanced.integrate_with == 'dials':
from iota.components.iota_dials import Triage
triage = Triage(img=self.conv_img,
gain=self.gain,
center_intensity=self.center_int,
params=self.params)
self.fail, log_entry = triage.triage_image()
self.log_info.append(log_entry)
self.status = 'triaged'
else:
self.fail = None
# Generate integration result dictionary for cctbx.xfel or DIALS
if self.params.advanced.integrate_with == 'cctbx':
gs_type = str(self.params.cctbx.grid_search.type).lower()
if gs_type == 'smart':
self.hrange = 1
self.arange = 1
elif gs_type in ('none', 'no_grid_search'):
self.hrange = 0
self.arange = 0
else:
self.hrange = self.params.cctbx.grid_search.height_range
self.arange = self.params.cctbx.grid_search.area_range
self.grid_points = []
self.generate_grid()
elif self.params.advanced.integrate_with == 'dials':
self.final = {
'img': self.conv_img,
'a': 0,
'b': 0,
'c': 0,
'alpha': 0,
'beta': 0,
'gamma': 0,
'sg': '',
'strong': 0,
'res': 0,
'lres': 0,
'mos': 0,
'epv': 0,
'info': '',
'final': None,
'program': 'dials'
}
# Generate names for output folders and files:
if not self.params.image_conversion.convert_only:
self.obj_path = misc.make_image_path(self.conv_img,
self.input_base,
self.obj_base)
self.obj_file = os.path.abspath(
os.path.join(
self.obj_path,
os.path.basename(self.conv_img).split('.')[0] + ".int"))
self.fin_path = misc.make_image_path(self.conv_img,
self.input_base,
self.fin_base)
self.log_path = misc.make_image_path(self.conv_img,
self.input_base,
self.log_base)
self.fin_file = os.path.abspath(
os.path.join(
self.fin_path, "int_{}.pickle".format(
os.path.basename(self.conv_img).split('.')[0])))
self.final['final'] = self.fin_file
self.final['img'] = self.conv_img
self.int_log = os.path.join(
self.log_path,
os.path.basename(self.conv_img).split('.')[0] + '.tmp')
self.viz_path = misc.make_image_path(self.conv_img,
self.input_base,
self.viz_base)
self.viz_file = os.path.join(
self.viz_path, "int_{}.png".format(
os.path.basename(self.conv_img).split('.')[0]))
# Create actual folders (if necessary)f
try:
if not os.path.isdir(self.obj_path):
os.makedirs(self.obj_path)
if not os.path.isdir(self.fin_path):
os.makedirs(self.fin_path)
if not os.path.isdir(self.log_path):
os.makedirs(self.log_path)
if not os.path.isdir(self.viz_path):
os.makedirs(self.viz_path)
except OSError:
pass
# Save image object to file
ep.dump(self.obj_file, self)
self.status = 'imported'
# If conversion only option is selected, write conversion info to log
if self.params.image_conversion.convert_only:
log_entry = "\n".join(self.log_info)
misc.main_log(self.main_log, log_entry)
return self
def import_image(self):
""" Image conversion:
- Writes out data in pickle format (cctbx.xfel only)
- Moves beam center into center of image, crops / pads image (cctbx.xfel only)
- Adjusts beam center and distance (optional)
- Thresholds and masks beamstop shadow (optional)
"""
if os.path.isfile(self.abort_file):
self.fail = 'aborted'
return self
# Load image
img_data, img_type = self.load_image()
self.status = 'loaded'
if img_data is None:
self.log_path = misc.make_image_path(self.conv_img,
self.input_base,
self.log_base)
self.int_log = os.path.join(
self.log_path,
os.path.basename(self.conv_img).split('.')[0] + '.tmp')
self.log_info.append('\n{:-^100}\n'.format(self.raw_img))
self.log_info.append('FAILED TO IMPORT')
self.status = 'failed import'
self.fail = 'failed import'
if not self.params.image_conversion.convert_only:
self.obj_path = misc.make_image_path(self.conv_img,
self.input_base,
self.obj_base)
rej_name = filter(
None,
os.path.basename(
self.conv_img).split('.'))[0] + '_reject.int'
self.obj_file = os.path.abspath(
os.path.join(self.obj_path, rej_name))
try:
if not os.path.isdir(self.obj_path):
os.makedirs(self.obj_path)
except OSError:
pass
# ep.dump(self.obj_file, self)
return self
# if DIALS is selected, change image type to skip conversion step
if self.params.advanced.integrate_with == 'dials':
img_type = 'dials_input'
if self.params.dials.auto_threshold:
try:
beam_x_px = int(img_data['BEAM_CENTER_X'] /
img_data['PIXEL_SIZE'])
beam_y_px = int(img_data['BEAM_CENTER_Y'] /
img_data['PIXEL_SIZE'])
data_array = img_data['DATA'].as_numpy_array().astype(
float)
self.center_int = np.nanmax(
data_array[beam_y_px - 20:beam_y_px + 20,
beam_x_px - 20:beam_x_px + 20])
except Exception, e:
print 'IMPORT ERROR: ', e
self.params.image_conversion.beamstop != 0 or
self.params.image_conversion.distance != 0
):
# Check for and/or create a converted pickles folder
try:
if not os.path.isdir(self.conv_base):
os.makedirs(self.conv_base)
except OSError:
pass
# Generate converted image pickle filename
rename_choice = str(
self.params.image_conversion.rename_pickle).lower()
if rename_choice in ("keep_file_structure", "none"):
img_path = misc.make_image_path(self.raw_img, self.input_base,
self.conv_base)
img_filename = os.path.basename(
self.raw_img).split('.')[0] + ".pickle"
self.conv_img = os.path.abspath(
os.path.join(img_path, img_filename))
try:
if not os.path.isdir(img_path):
os.makedirs(img_path)
except OSError:
pass
else:
if rename_choice == "auto_filename":
prefix = self.user_id
elif rename_choice == "custom_filename":
prefix = self.params.image_conversion.rename_pickle_prefix
number = int(os.path.basename(self.conv_base))
def import_image(self):
""" Image conversion:
- Writes out data in pickle format (cctbx.xfel only)
- Moves beam center into center of image, crops / pads image (cctbx.xfel only)
- Adjusts beam center and distance (optional)
- Thresholds and masks beamstop shadow (optional)
"""
if os.path.isfile(self.abort_file):
self.fail = 'aborted'
return self
# Load image
img_data, img_type = self.load_image()
self.status = 'loaded'
if img_data is None:
self.log_info.append('\n{:-^100}\n'.format(self.raw_img))
self.log_info.append('FAILED TO IMPORT')
self.status = 'failed import'
self.fail = 'failed import'
return self
# if DIALS is selected, change image type to skip conversion step
if self.params.advanced.integrate_with == 'dials':
img_type = 'dials_input'
# Log initial image information
self.log_info.append('\n{:-^100}\n'.format(self.raw_img))
self.log_info.append('Imported image : {}'.format(self.raw_img))
self.log_info.append('Parameters : BEAM_X = {:<4.2f}, BEAM_Y = {:<4.2f}, '\
'PIXEL_SIZE = {:<8.6f}, IMG_SIZE = {:<4} X {:<4}, '\
'DIST = {}'.format(img_data['BEAM_CENTER_X'],
img_data['BEAM_CENTER_Y'],
img_data['PIXEL_SIZE'],
img_data['SIZE1'],
img_data['SIZE2'],
img_data['DISTANCE']))
# Deactivate image squaring if beam center is in the center of the image
# CCTBX.XFEL ONLY (Need a better displacement cutoff)
if ( self.params.advanced.integrate_with == 'dials' or
abs(img_data['BEAM_CENTER_X'] - img_data['BEAM_CENTER_Y']) < 0.1
):
self.params.image_conversion.square_mode = 'None'
# Check if conversion/modification is required and carry them out
if ( img_type == 'raw' or
self.params.image_conversion.square_mode != "None" or
self.params.image_conversion.beam_center.x != 0 or
self.params.image_conversion.beam_center.y != 0 or
self.params.image_conversion.beamstop != 0 or
self.params.image_conversion.distance != 0
):
# Check for and/or create a converted pickles folder
try:
if not os.path.isdir(self.conv_base):
os.makedirs(self.conv_base)
except OSError:
pass
# Generate converted image pickle filename
if self.params.image_conversion.rename_pickle_prefix != None:
if str(self.params.image_conversion.rename_pickle_prefix).lower() == "auto":
try:
prefix = os.getlogin()
except Exception:
prefix = 'converted'
else:
prefix = self.params.image_conversion.rename_pickle_prefix
number = int(os.path.basename(self.conv_base))
self.conv_img = os.path.abspath(os.path.join(self.conv_base,
"{}_{}_{:05d}.pickle".format(prefix, number, self.img_index)))
else: # This option preserves the input directory structure
img_path = misc.make_image_path(self.raw_img, self.input_base, self.conv_base)
self.conv_img = os.path.abspath(os.path.join(img_path,
os.path.basename(self.raw_img).split('.')[0] + ".pickle"))
try:
if not os.path.isdir(img_path):
os.makedirs(img_path)
except OSError:
pass
# Convert raw image to image pickle
beamstop = self.params.image_conversion.beamstop
distance = self.params.image_conversion.distance
beam_center = [self.params.image_conversion.beam_center.x,
self.params.image_conversion.beam_center.y]
square = self.params.image_conversion.square_mode
if beam_center != [0,0]:
pixel_size = img_data['PIXEL_SIZE']
img_data['BEAM_CENTER_X'] = int(round(beam_center[0] * pixel_size))
img_data['BEAM_CENTER_Y'] = int(round(beam_center[1] * pixel_size))
if distance != 0:
img_data['DISTANCE'] = distance
if square != "None":
img_data = self.square_pickle(img_data)
if beamstop != 0:
img_data = self.mask_image(img_data)
# Log converted image information
self.log_info.append('Converted image : {}'.format(self.conv_img))
self.log_info.append('Parameters : BEAM_X = {:<4.2f}, BEAM_Y = {:<4.2f}, '\
'PIXEL_SIZE = {:<8.6f}, IMG_SIZE = {:<4} X {:<4}, '\
'DIST = {}'.format(img_data['BEAM_CENTER_X'],
img_data['BEAM_CENTER_Y'],
img_data['PIXEL_SIZE'],
img_data['SIZE1'],
img_data['SIZE2'],
img_data['DISTANCE']))
self.input_base = self.conv_base
self.status = 'converted'
# Save converted image pickle
ep.dump(self.conv_img, img_data)
# Triage image (i.e. check for usable diffraction, using selected method)
if str(self.params.image_triage.type).lower() != 'none':
if self.params.advanced.integrate_with == 'cctbx':
from iota.components.iota_cctbx import Triage
triage = Triage(self.conv_img, self.gain, self.params)
self.fail, log_entry, self.hmed, self.amed = triage.triage_image()
elif self.params.advanced.integrate_with == 'dials':
from iota.components.iota_dials import Triage
triage = Triage(self.conv_img, self.gain, self.params)
self.fail, log_entry = triage.triage_image()
self.log_info.append(log_entry)
self.status = 'triaged'
else:
self.fail = None
# Generate integration result dictionary for cctbx.xfel or DIALS
if self.params.advanced.integrate_with == 'cctbx':
if self.params.cctbx.grid_search.type == 'smart':
self.hrange = 1
self.arange = 1
else:
self.hrange = self.params.cctbx.grid_search.height_range
self.arange = self.params.cctbx.grid_search.area_range
self.grid_points = []
self.generate_grid()
elif self.params.advanced.integrate_with == 'dials':
self.final = {'img':self.conv_img, 'a':0, 'b':0, 'c':0, 'alpha':0,
'beta':0, 'gamma':0, 'sg':'','strong':0, 'res':0,
'lres':0, 'mos':0, 'epv':0, 'info':'','final':None,
'program':'dials'}
# Generate names for output folders and files:
if not self.params.image_conversion.convert_only:
self.obj_path = misc.make_image_path(self.conv_img, self.input_base, self.obj_base)
self.obj_file = os.path.abspath(os.path.join(self.obj_path,
os.path.basename(self.conv_img).split('.')[0] + ".int"))
self.fin_path = misc.make_image_path(self.conv_img, self.input_base, self.fin_base)
self.fin_file = os.path.abspath(os.path.join(self.fin_path,
"int_{}.pickle".format(os.path.basename(self.conv_img).split('.')[0])))
self.final['final'] = self.fin_file
self.final['img'] = self.conv_img
self.int_log = os.path.join(self.fin_path,
os.path.basename(self.conv_img).split('.')[0] + '.tmp')
self.viz_path = misc.make_image_path(self.conv_img, self.input_base, self.viz_base)
self.viz_file = os.path.join(self.viz_path,
"int_{}.png".format(os.path.basename(self.conv_img).split('.')[0]))
# Create actual folders (if necessary)
try:
if not os.path.isdir(self.obj_path):
os.makedirs(self.obj_path)
if not os.path.isdir(self.fin_path):
os.makedirs(self.fin_path)
if not os.path.isdir(self.viz_path):
os.makedirs(self.viz_path)
except OSError:
pass
# Save image object to file
ep.dump(self.obj_file, self)
self.status = 'imported'
# If conversion only option is selected, write conversion info to log
if self.params.image_conversion.convert_only:
log_entry = "\n".join(self.log_info)
misc.main_log(self.main_log, log_entry)
return self