def make_int_object_list(self):
""" Generates list of image objects from previous grid search """
from libtbx import easy_pickle as ep
if self.params.cctbx.selection.select_only.grid_search_path == None:
int_dir = misc.set_base_dir('integration', True)
else:
int_dir = self.params.cctbx.selection.select_only.grid_search_path
img_objects = []
cmd.Command.start("Importing saved grid search results")
for root, dirs, files in os.walk(int_dir):
for filename in files:
found_file = os.path.join(root, filename)
if found_file.endswith(('int')):
obj = ep.load(found_file)
img_objects.append(obj)
cmd.Command.end("Importing saved grid search results -- DONE")
# Pick a randomized subset of images
if self.params.advanced.random_sample.flag_on and \
self.params.advanced.random_sample.number < len(img_objects):
gs_img_objects = self.select_random_subset(img_objects)
else:
gs_img_objects = img_objects
return gs_img_objects
def update_settings(self):
idxs = self.inp_box.ctr.GetItemCount()
items = [self.inp_box.ctr.GetItemData(i) for i in range(idxs)]
inputs = []
reference = None
sequence = None
for i in items:
inp_type = i.type.type.GetString(i.type_selection)
if inp_type in ('processed pickle list', 'processed pickle folder',
'processed pickle'):
inputs.append(i.path)
elif inp_type == 'reference MTZ':
reference = i.path
elif inp_type == 'sequence':
sequence = i.path
self.pparams.data = inputs
if reference is not None:
self.pparams.hklisoin = reference
if self.opt_chk_useref.GetValue():
self.pparams.hklrefin = reference
self.out_dir = self.out_box.ctr.GetValue()
self.pparams.run_no = misc.set_base_dir(
out_dir=self.out_dir) # Need to change
self.pparams.title = self.project_title.ctr.GetValue()
self.pparams.n_residues = self.opt_spc_nres.ctr.GetValue()
self.pparams.n_processors = self.opt_spc_nproc.ctr.GetValue()
update_phil = master_phil.format(python_object=self.pparams)
self.regenerate_params(update_phil)
def make_int_object_list(self):
""" Generates list of image objects from previous grid search """
from libtbx import easy_pickle as ep
if self.params.cctbx.selection.select_only.grid_search_path == None:
int_dir = misc.set_base_dir('integration', True)
else:
int_dir = self.params.cctbx.selection.select_only.grid_search_path
img_objects = []
cmd.Command.start("Importing saved grid search results")
for root, dirs, files in os.walk(int_dir):
for filename in files:
found_file = os.path.join(root, filename)
if found_file.endswith(('int')):
obj = ep.load(found_file)
img_objects.append(obj)
cmd.Command.end("Importing saved grid search results -- DONE")
# Pick a randomized subset of images
if self.params.advanced.random_sample.flag_on and \
self.params.advanced.random_sample.number < len(img_objects):
gs_img_objects = self.select_random_subset(img_objects)
else:
gs_img_objects = img_objects
return gs_img_objects
def init_settings(self):
self.pparams = self.input_window.pparams
self.pparams.data = [self.input_window.inp_box.ctr.GetValue()]
self.out_dir = self.input_window.out_box.ctr.GetValue()
self.pparams.run_no = misc.set_base_dir(out_dir=self.out_dir) # Need to change
self.pparams.title = self.input_window.title_box.ctr.GetValue()
if str(self.input_window.ref_box.ctr.GetValue()).lower() != '':
self.pparams.hklisoin = self.input_window.ref_box.ctr.GetValue()
if self.input_window.opt_chk_useref.GetValue():
self.pparams.hklrefin = self.input_window.ref_box.ctr.GetValue()
self.pparams.n_residues = self.input_window.opt_spc_nres.GetValue()
self.pparams.n_processors = self.input_window.opt_spc_nproc.GetValue()
def run(self):
self.args, self.phil_args = parse_command_args(
self.iver, self.help_message).parse_known_args()
# Check for type of input
if len(self.args.path) == 0 or self.args.path is None: # No input
parse_command_args(self.iver, self.help_message).print_help()
if self.args.default: # Write out default params and exit
help_out, txt_out = inp.print_params()
print '\n{:-^70}\n'.format('IOTA Parameters')
print help_out
inp.write_defaults(os.path.abspath(os.path.curdir), txt_out)
misc.iota_exit()
elif len(self.args.path) > 1: # If multiple paths / wildcards
file_list = ginp.make_input_list(self.args.path)
list_file = os.path.join(os.path.abspath(os.path.curdir),
'input.lst')
with open(list_file, 'w') as lf:
lf.write('\n'.join(file_list))
msg = "\nIOTA will run in AUTO mode using wildcard datapath:\n" \
"{} files found, compiled in {}\n".format(len(file_list), list_file)
self.params, self.txt_out = inp.process_input(
self.args, self.phil_args, list_file, 'auto', self.now)
else: # If single path, check type
carg = os.path.abspath(self.args.path[0])
if os.path.exists(carg):
# If user provided a parameter file
if os.path.isfile(carg) and os.path.basename(carg).endswith(
'.param'):
msg = ''
self.params, self.txt_out = inp.process_input(
self.args, self.phil_args, carg, 'file')
# If user provided a list of input files
elif os.path.isfile(carg) and os.path.basename(carg).endswith(
'.lst'):
msg = "\nIOTA will run in AUTO mode using {}:\n".format(
carg)
self.params, self.txt_out = inp.process_input(
self.args, self.phil_args, carg, 'auto', self.now)
# If user provided a single filepath
elif os.path.isfile(
carg) and not os.path.basename(carg).endswith('.lst'):
msg = "\nIOTA will run in SINGLE-FILE mode using {}:\n".format(
carg)
self.params, self.txt_out = inp.process_input(
self.args, self.phil_args, carg, 'auto', self.now)
# If user provided a data folder
elif os.path.isdir(carg):
msg = "\nIOTA will run in AUTO mode using {}:\n".format(
carg)
self.params, self.txt_out = inp.process_input(
self.args, self.phil_args, carg, 'auto', self.now)
# If user provided gibberish
else:
print self.logo
print "ERROR: Invalid input! Need parameter filename or data folder."
misc.iota_exit()
# Identify indexing / integration program
if self.params.advanced.integrate_with == 'cctbx':
prg = " with CCTBX.XFEL\n"
elif self.params.advanced.integrate_with == 'dials':
prg = " with DIALS\n"
self.logo += prg
print self.logo
print '\n{}\n'.format(self.now)
if msg != '':
print msg
if self.args.analyze != None:
print 'ANALYSIS ONLY will be performed (analyzing run #{})'.format(
self.args.analyze)
self.analyze_prior_results('{:003d}'.format(int(
self.args.analyze)))
misc.iota_exit()
if self.params.mp_method == 'mpi':
rank, size = misc.get_mpi_rank_and_size()
self.master_process = rank == 0
else:
self.master_process = True
# Call function to read input folder structure (or input file) and
# generate list of image file paths
if self.params.cctbx.selection.select_only.flag_on:
self.gs_img_objects = self.make_int_object_list()
self.input_list = [i.conv_img for i in self.gs_img_objects]
else:
self.input_list = self.make_input_list()
# Check for -l option, output list of input files and exit
if self.args.list:
list_file = os.path.abspath("{}/input.lst".format(os.curdir))
# Check if other files of this name exist under the current folder
list_folder = os.path.dirname(list_file)
list_files = [
i for i in os.listdir(list_folder) if i.endswith(".lst")
]
if len(list_files) > 0:
list_file = os.path.join(
list_folder, "input_{}.lst".format(len(list_files)))
print '\nINPUT LIST ONLY option selected'
print 'Input list in {} \n\n'.format(list_file)
with open(list_file, "w") as lf:
for i, input_file in enumerate(self.input_list, 1):
lf.write('{}\n'.format(input_file))
print "{}: {}".format(i, input_file)
lf.write('{}\n'.format(input_file))
print '\nExiting...\n\n'
misc.iota_exit()
# If fewer images than requested processors are supplied, set the number of
# processors to the number of images
if self.params.n_processors > len(self.input_list):
self.params.n_processors = len(self.input_list)
# Generate base folder paths
self.conv_base = misc.set_base_dir('converted_pickles',
out_dir=self.params.output)
self.int_base = misc.set_base_dir('integration',
out_dir=self.params.output)
self.obj_base = os.path.join(self.int_base, 'image_objects')
self.fin_base = os.path.join(self.int_base, 'final')
self.log_base = os.path.join(self.int_base, 'logs')
self.viz_base = os.path.join(self.int_base, 'visualization')
self.tmp_base = os.path.join(
'/tmp', '{}_{}'.format(os.getlogin(), time.time()))
# Generate base folders
os.makedirs(self.int_base)
os.makedirs(self.obj_base)
os.makedirs(self.fin_base)
os.makedirs(self.log_base)
os.makedirs(self.tmp_base)
# Determine input base
self.input_base = os.path.abspath(
os.path.dirname(os.path.commonprefix(self.input_list)))
# Initialize main log
self.logfile = os.path.abspath(os.path.join(self.int_base, 'iota.log'))
# Log starting info
misc.main_log(self.logfile, '{:=^80} \n'.format(' IOTA MAIN LOG '))
misc.main_log(self.logfile,
'{:-^80} \n'.format(' SETTINGS FOR THIS RUN '))
misc.main_log(self.logfile, self.txt_out)
if self.params.advanced.integrate_with == 'cctbx':
target_file = self.params.cctbx.target
elif self.params.advanced.integrate_with == 'dials':
target_file = self.params.dials.target
misc.main_log(
self.logfile, '{:-^80} \n\n'
''.format(' TARGET FILE ({}) CONTENTS '
''.format(target_file)))
with open(target_file, 'r') as phil_file:
phil_file_contents = phil_file.read()
misc.main_log(self.logfile, phil_file_contents)
# Write target file and record its location in params
local_target_file = os.path.join(self.int_base, 'target.phil')
with open(local_target_file, 'w') as tf:
tf.write(phil_file_contents)
def run(self, gparams, target_phil=None, list_file=None):
''' Run initialization for IOTA GUI
gparams = IOTA parameters from the GUI elements in PHIL format
gtxt = text version of gparams
list_file = if "Write Input List" button pressed, specifies name
of list file
'''
self.params = gparams
self.target_phil = target_phil
# If input list for some reason isn't transmitted from main window, make it
if self.input_list is None:
self.input_list = self.make_input_list()
# Select range of images if turned on
if self.params.advanced.image_range.flag_on:
self.input_list = self.select_image_range(self.input_list)
# Select a random subset of images if turned on
if self.params.advanced.random_sample.flag_on and \
self.params.advanced.random_sample.number < len(self.input_list):
self.input_list = self.select_random_subset(self.input_list)
# Check for data not found
if len(self.input_list) == 0:
wx.MessageBox('ERROR: Data Not Found!', 'ERROR',
wx.OK | wx.ICON_ERROR)
return False
# If list-only option selected, output list only
if list_file != None:
with open(list_file, "w") as lf:
for i, input_file in enumerate(self.input_list, 1):
lf.write('{}\n'.format(input_file))
return True
# Run the sanity check procedure
if not self.sanity_check():
return False
# If fewer images than requested processors are supplied, set the number of
# processors to the number of images
if self.params.mp_method == 'multiprocessing':
if self.params.n_processors > len(self.input_list):
self.params.n_processors = len(self.input_list)
# Generate base folder paths
self.conv_base = misc.set_base_dir('converted_pickles',
out_dir=self.params.output)
self.int_base = misc.set_base_dir('integration',
out_dir=self.params.output)
self.obj_base = os.path.join(self.int_base, 'image_objects')
self.fin_base = os.path.join(self.int_base, 'final')
self.log_base = os.path.join(self.int_base, 'logs')
self.viz_base = os.path.join(self.int_base, 'visualization')
if str(self.params.advanced.temporary_output_folder).lower() in (
'none', ''):
self.tmp_base = os.path.join(self.int_base, 'tmp')
else:
self.tmp_base = os.path.join(
self.params.advanced.temporary_output_folder)
# Generate base folders
os.makedirs(self.int_base)
os.makedirs(self.obj_base)
os.makedirs(self.fin_base)
os.makedirs(self.log_base)
try:
if not os.path.isdir(self.tmp_base):
os.makedirs(self.tmp_base)
except OSError:
pass
# Determine input base
common_pfx = os.path.abspath(
os.path.dirname(os.path.commonprefix(self.input_list)))
if len(self.params.input) == 1:
self.input_base = os.path.commonprefix(
[self.params.input[0], common_pfx])
else:
self.input_base = common_pfx
# Initialize main log
self.logfile = os.path.abspath(os.path.join(self.int_base, 'iota.log'))
# Write target file and record its location in params
local_target_file = os.path.join(self.int_base, 'target.phil')
if type(self.target_phil) == list:
self.target_phil = '\n'.join(self.target_phil)
with open(local_target_file, 'w') as tf:
tf.write(self.target_phil)
if self.params.advanced.integrate_with == 'cctbx':
self.params.cctbx.target = local_target_file
elif self.params.advanced.integrate_with == 'dials':
self.params.dials.target = local_target_file
# Collect final params and convert to PHIL object
final_phil = inp.master_phil.format(python_object=self.params)
# Generate text of params
with misc.Capturing() as txt_output:
final_phil.show()
self.txt_out = ''
for one_output in txt_output:
self.txt_out += one_output + '\n'
# Log starting info
misc.main_log(self.logfile, '{:=^80} \n'.format(' IOTA MAIN LOG '))
misc.main_log(self.logfile,
'{:-^80} \n'.format(' SETTINGS FOR THIS RUN '))
misc.main_log(self.logfile, self.txt_out)
# Log cctbx.xfel / DIALS settings
misc.main_log(
self.logfile, '{:-^80} \n\n'
''.format(' TARGET FILE ({}) CONTENTS '
''.format(local_target_file)))
misc.main_log(self.logfile, self.target_phil)
return True
def run(self):
self.args, self.phil_args = parse_command_args(self.iver,
self.help_message).parse_known_args()
# Check for type of input
if self.args.path == None: # No input
parse_command_args(self.iver, self.help_message).print_help()
if self.args.default: # Write out default params and exit
help_out, txt_out = inp.print_params()
print '\n{:-^70}\n'.format('IOTA Parameters')
print help_out
inp.write_defaults(os.path.abspath(os.path.curdir), txt_out)
misc.iota_exit()
else: # If input exists, check type
carg = os.path.abspath(self.args.path)
if os.path.exists(carg):
# If user provided a parameter file
if os.path.isfile(carg) and os.path.basename(carg).endswith('.param'):
msg = ''
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'file')
# If user provided a list of input files
elif os.path.isfile(carg) and os.path.basename(carg).endswith('.lst'):
msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg)
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'auto', self.now)
# If user provided a single filepath
elif os.path.isfile(carg) and not os.path.basename(carg).endswith('.lst'):
msg = "\nIOTA will run in SINGLE-FILE mode using {}:\n".format(carg)
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'auto', self.now)
# If user provided a data folder
elif os.path.isdir(carg):
msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg)
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'auto', self.now)
# If user provided gibberish
else:
print self.logo
print "ERROR: Invalid input! Need parameter filename or data folder."
misc.iota_exit()
# Identify indexing / integration program
if self.params.advanced.integrate_with == 'cctbx':
prg = " with CCTBX.XFEL\n"
elif self.params.advanced.integrate_with == 'dials':
prg = " with DIALS\n"
self.logo += prg
print self.logo
print '\n{}\n'.format(self.now)
if msg != '':
print msg
if self.args.analyze != None:
self.analyze_prior_results('{:003d}'.format(int(self.args.analyze)))
misc.iota_exit()
if self.params.mp_method == 'mpi':
rank, size = misc.get_mpi_rank_and_size()
self.master_process = rank == 0
else:
self.master_process = True
# Call function to read input folder structure (or input file) and
# generate list of image file paths
if self.params.cctbx.selection.select_only.flag_on:
self.gs_img_objects = self.make_int_object_list()
self.input_list = [i.conv_img for i in self.gs_img_objects]
else:
self.input_list = self.make_input_list()
# Check for -l option, output list of input files and exit
if self.args.list:
if list_file == None:
list_file = os.path.abspath("{}/input.lst".format(os.curdir))
print '\nINPUT LIST ONLY option selected'
print 'Input list in {} \n\n'.format(list_file)
with open(list_file, "w") as lf:
for i, input_file in enumerate(self.input_list, 1):
lf.write('{}\n'.format(input_file))
print "{}: {}".format(i, input_file)
lf.write('{}\n'.format(input_file))
print '\nExiting...\n\n'
misc.iota_exit()
# If fewer images than requested processors are supplied, set the number of
# processors to the number of images
if self.params.n_processors > len(self.input_list):
self.params.n_processors = len(self.input_list)
# Generate base folder paths
self.conv_base = misc.set_base_dir('converted_pickles', out_dir = self.params.output)
self.int_base = misc.set_base_dir('integration', out_dir = self.params.output)
self.obj_base = os.path.join(self.int_base, 'image_objects')
self.fin_base = os.path.join(self.int_base, 'final')
self.tmp_base = os.path.join(self.int_base, 'tmp')
self.viz_base = os.path.join(self.int_base, 'visualization')
# Generate base folders
os.makedirs(self.int_base)
os.makedirs(self.obj_base)
os.makedirs(self.fin_base)
os.makedirs(self.tmp_base)
# Determine input base
self.input_base = os.path.abspath(os.path.dirname(os.path.commonprefix(self.input_list)))
# Initialize main log
self.logfile = os.path.abspath(os.path.join(self.int_base, 'iota.log'))
# Log starting info
misc.main_log(self.logfile, '{:=^80} \n'.format(' IOTA MAIN LOG '))
misc.main_log(self.logfile, '{:-^80} \n'.format(' SETTINGS FOR THIS RUN '))
misc.main_log(self.logfile, self.txt_out)
if self.params.advanced.integrate_with == 'cctbx':
target_file = self.params.cctbx.target
elif self.params.advanced.integrate_with == 'dials':
target_file = self.params.dials.target
misc.main_log(self.logfile, '{:-^80} \n\n'
''.format(' TARGET FILE ({}) CONTENTS '
''.format(target_file)))
with open(target_file, 'r') as phil_file:
phil_file_contents = phil_file.read()
misc.main_log(self.logfile, phil_file_contents)
