def update_settings(self):
idxs = self.inp_box.ctr.GetItemCount()
items = [self.inp_box.ctr.GetItemData(i) for i in range(idxs)]
inputs = []
reference = None
sequence = None
for i in items:
inp_type = i.type.type.GetString(i.type_selection)
if inp_type in ('processed pickle list', 'processed pickle folder',
'processed pickle'):
inputs.append(i.path)
elif inp_type == 'reference MTZ':
reference = i.path
elif inp_type == 'sequence':
sequence = i.path
self.pparams.data = inputs
if reference is not None:
self.pparams.hklisoin = reference
if self.opt_chk_useref.GetValue():
self.pparams.hklrefin = reference
self.out_dir = self.out_box.ctr.GetValue()
self.pparams.run_no = util.set_base_dir(
out_dir=self.out_dir) # Need to change
self.pparams.title = self.project_title.ctr.GetValue()
self.pparams.n_residues = self.opt_spc_nres.ctr.GetValue()
self.pparams.n_processors = self.opt_spc_nproc.ctr.GetValue()
update_phil = master_phil.format(python_object=self.pparams)
self.regenerate_params(update_phil)
def initialize_output(self):
try:
self.int_base = util.set_base_dir('integration',
out_dir=self.params.output)
self.obj_base = os.path.join(self.int_base, 'image_objects')
self.fin_base = os.path.join(self.int_base, 'final')
self.log_base = os.path.join(self.int_base, 'logs')
self.viz_base = os.path.join(self.int_base, 'visualization')
if not self.params.advanced.temporary_output_folder:
self.tmp_base = os.path.join(self.int_base, 'tmp')
else:
self.tmp_base = os.path.join(self.params.advanced.temporary_output_folder)
# Determine input base
common_pfx = os.path.abspath(os.path.dirname(os.path.commonprefix(self.input_list)))
if len(self.params.input) == 1:
self.input_base = os.path.commonprefix([self.params.input[0], common_pfx])
else:
self.input_base = common_pfx
# Generate base folders
os.makedirs(self.int_base)
os.makedirs(self.obj_base)
os.makedirs(self.fin_base)
os.makedirs(self.log_base)
try:
if not os.path.isdir(self.tmp_base):
os.makedirs(self.tmp_base)
except OSError:
self.tmp_base = os.path.join(self.int_base, 'tmp')
if not os.path.isdir(self.tmp_base):
os.makedirs(self.tmp_base)
# Designate files for init and image list iterable
self.init_file = os.path.join(self.int_base, 'init.cfg')
self.iter_file = os.path.join(self.int_base, 'iter.cfg')
# Initialize info object
self.info = ProcessInfo(int_base=self.int_base, tmp_base=self.tmp_base)
self.info_file = os.path.join(self.int_base, 'proc.info')
return True, 'IOTA_INIT: OUTPUT INITIALIZED'
except Exception as e:
return False, 'IOTA_INIT_ERROR: OUTPUT NOT INITIALIZED: {}'.format(e)
def make_int_object_list(self):
""" Generates list of image objects from previous grid search """
from libtbx import easy_pickle as ep
if self.params.cctbx_ha14.selection.select_only.grid_search_path is None:
int_dir = util.set_base_dir('integration', True)
else:
int_dir = self.params.cctbx_ha14.selection.select_only.grid_search_path
img_objects = []
for root, dirs, files in os.walk(int_dir):
for filename in files:
found_file = os.path.join(root, filename)
if found_file.endswith(('int')):
obj = ep.load(found_file)
img_objects.append(obj)
return img_objects
def make_prime_input(self, filename='prime.phil', run_zero=False):
""" Imports default PRIME input parameters, modifies correct entries and
prints out a starting PHIL file to be used with PRIME
"""
assert self.info
pixel_size = self.info.pixel_size
hres = self.info.stats['res']
lres = self.info.stats['lres']
# If symmetry / unit cell were not overridden from GUI, set from INFO
if not self.best_pg:
try:
self.best_pg = self.info.best_pg.replace(" ", '')
except AttributeError as e:
print('PRIME INPUT ERROR, SPACE GROUP: ', e)
self.best_pg = 'P1'
if not self.best_uc:
self.best_uc = self.info.best_uc
# Determine crystal system from crystal symmetry
sym = crystal.symmetry(space_group_symbol=self.best_pg)
crystal_system = str(sym.space_group().crystal_system())
# Determine number of images for indexing ambiguity resolution
# My default: 1/2 of images or 300, whichever is smaller
if len(self.info.categories['integrated']) >= 600:
idx_ambiguity_sample = 300
idx_ambiguity_selected = 100
else:
idx_ambiguity_sample = int(
round(len(self.info.categories['integrated']) / 2))
idx_ambiguity_selected = int(round(idx_ambiguity_sample / 3))
# Set run number to 000 if running LivePRIME
out_dir = os.path.join(os.path.dirname(self.prime_data_path), 'prime')
if run_zero:
run_path = os.path.join(out_dir, '000')
else:
run_path = util.set_base_dir(out_dir=out_dir)
# Populate pertinent data parameters
prime_params = mod_input.master_phil.extract()
prime_params.run_no = run_path
prime_params.data = [self.prime_data_path]
prime_params.title = 'Auto-generated by IOTA v{} on {}' \
''.format(iota_version, now)
prime_params.scale.d_min = hres['mean']
prime_params.scale.d_max = 8
prime_params.postref.scale.d_min = hres['mean']
prime_params.postref.scale.d_max = lres['max']
prime_params.postref.crystal_orientation.d_min = hres['mean']
prime_params.postref.crystal_orientation.d_max = lres['max']
prime_params.postref.reflecting_range.d_min = hres['mean']
prime_params.postref.reflecting_range.d_max = lres['max']
prime_params.postref.unit_cell.d_min = hres['mean']
prime_params.postref.unit_cell.d_max = lres['max']
prime_params.postref.allparams.d_min = hres['mean']
prime_params.postref.allparams.d_max = lres['max']
prime_params.merge.d_min = hres['mean']
prime_params.merge.d_max = lres['max']
prime_params.target_unit_cell = uctbx.unit_cell(self.best_uc)
prime_params.target_space_group = self.best_pg
prime_params.target_crystal_system = crystal_system
prime_params.pixel_size_mm = pixel_size
prime_params.n_residues = 500
prime_params.indexing_ambiguity.n_sample_frames = idx_ambiguity_sample
prime_params.indexing_ambiguity.n_selected_frames = idx_ambiguity_selected
# Determine which queue to run on (i.e. match IOTA queue)
# Modify specific options based in IOTA settings
# Queue options
if (self.params.mp.method == 'lsf'
and self.params.mp.queue is not None):
prime_params.queue.mode = 'bsub'
prime_params.queue.qname = self.params.mp.queue
# Number of processors (automatically, 1/2 of IOTA procs)
prime_params.n_processors = int(self.params.mp.n_processors / 2)
# Generate PRIME param PHIL
prime_phil = mod_input.master_phil.format(python_object=prime_params)
prime_file = os.path.join(self.info.int_base, filename)
with open(prime_file, 'w') as pf:
pf.write(prime_phil.as_str())
return prime_phil
def initialize_new_run(phil, input_dict=None, target_phil=None):
''' Create base integration folder; safe phil, input, and info to file '''
try:
params = phil.extract()
int_base, run_no = util.set_base_dir(dirname='integration',
out_dir=params.output,
get_run_no=True)
if not os.path.isdir(int_base):
os.makedirs(int_base)
# Create input list file and populate param input line
if input_dict:
if len(input_dict['imagepaths']) >= 10:
input_list_file = os.path.join(int_base, 'input.lst')
with open(input_list_file, 'w') as lf:
for f in input_dict['imagefiles']:
lf.write('{}\n'.format(f))
params.input = [input_list_file]
else:
input_list_file = None
params.input = input_dict['imagepaths']
else:
input_list_file = None
# Generate default backend PHIL, write to file, and update params
target_fp = os.path.join(int_base,'target.phil')
if target_phil:
target_phil = inp.write_phil(phil_str=target_phil,
dest_file=target_fp,
write_target_file=True)
else:
if params.cctbx_xfel.target:
target_phil = inp.write_phil(phil_file=params.cctbx_xfel.target,
dest_file=target_fp,
write_target_file=True)
else:
method = params.advanced.processing_backend
target_phil, _ = inp.write_defaults(method=method,
write_param_file=False,
filepath=target_fp)
params.cctbx_xfel.target = target_fp
# Save PHIL for this run in base integration folder
paramfile = os.path.join(int_base, 'iota_r{}.param'.format(run_no))
phil = phil.format(python_object=params)
with open(paramfile, 'w') as philf:
philf.write(phil.as_str())
# Initialize main log
logfile = os.path.abspath(os.path.join(int_base, 'iota.log'))
# Initialize proc.info object and save to file
info = ProcInfo.from_args(
iota_phil=phil.as_str(),
target_phil=target_phil.as_str(),
int_base=int_base,
input_list_file=input_list_file,
info_file=os.path.join(int_base, 'proc.info'),
cluster_info_file=os.path.join(int_base, 'cluster.info'),
paramfile=paramfile,
logfile=logfile,
run_number=run_no,
description=params.description,
status='initialized',
have_results=False,
errors=[],
init_proc=False)
info.export_json()
return True, info, 'IOTA_XTERM_INIT: Initialization complete!'
except Exception as e:
import traceback
traceback.print_exc()
msg = 'IOTA_INIT_ERROR: Could not initialize run! {}'.format(e)
return False, None, msg
def initialize_new_run(phil, input_dict=None, target_phil=None):
''' Create base integration folder; safe phil, input, and info to file '''
try:
params = phil.extract()
int_base, run_no = util.set_base_dir(dirname='integration',
out_dir=params.output,
get_run_no=True)
if not os.path.isdir(int_base):
os.makedirs(int_base)
# Create input list file and populate param input line
if input_dict:
if len(input_dict['imagepaths']) >= 25:
input_list_file = os.path.join(int_base, 'input.lst')
with open(input_list_file, 'w') as lf:
for f in input_dict['imagefiles']:
lf.write('{}\n'.format(f))
params.input = [input_list_file]
else:
# If there are too many imagefiles, re-constitute the "glob" format
# by matching filepaths and replacing non-matching characters with
# asterisks
if len(input_dict['imagefiles']) >= 25:
input_paths = []
for path in input_dict['imagepaths']:
fileset = [
os.path.basename(i)
for i in input_dict['imagefiles'] if path in i
]
zips = [list(set(i)) for i in zip(*fileset)]
chars = [i[0] if len(i) == 1 else '*' for i in zips]
fname = ''.join(chars)
while "*" * 2 in fname:
fname = fname.replace("*" * 2, "*")
input_paths.append(os.path.join(path, fname))
params.input = input_paths
else:
params.input = input_dict['imagefiles']
input_list_file = None
else:
input_list_file = None
# Generate default backend PHIL, write to file, and update params
target_fp = os.path.join(int_base, 'target.phil')
if target_phil:
target_phil = inp.write_phil(phil_str=target_phil,
dest_file=target_fp,
write_target_file=True)
else:
if params.cctbx_xfel.target:
target_phil = inp.write_phil(
phil_file=params.cctbx_xfel.target,
dest_file=target_fp,
write_target_file=True)
else:
method = params.advanced.processing_backend
target_phil, _ = inp.write_defaults(method=method,
write_param_file=False,
filepath=target_fp)
params.cctbx_xfel.target = target_fp
# Save PHIL for this run in base integration folder
paramfile = os.path.join(int_base, 'iota_r{}.param'.format(run_no))
phil = phil.format(python_object=params)
with open(paramfile, 'w') as philf:
philf.write(phil.as_str())
# Initialize main log
logfile = os.path.abspath(os.path.join(int_base, 'iota.log'))
# Initialize proc.info object and save to file
info = ProcInfo.from_args(
iota_phil=phil.as_str(),
target_phil=target_phil.as_str(),
int_base=int_base,
input_list_file=input_list_file,
info_file=os.path.join(int_base, 'proc.info'),
cluster_info_file=os.path.join(int_base, 'cluster.info'),
paramfile=paramfile,
logfile=logfile,
run_number=run_no,
description=params.description,
status='initialized',
have_results=False,
errors=[],
init_proc=False)
info.export_json()
return True, info, 'IOTA_XTERM_INIT: Initialization complete!'
except Exception as e:
msg = 'IOTA_INIT_ERROR: Could not initialize run! {}'.format(e)
return False, None, msg
def __init__(self, init):
ImageImporterBase.__init__(self, init=init)
self.params = init.params
self.modify = True
self.conv_base = util.set_base_dir('converted_pickles',
out_dir=self.params.output)
