def initialize_interface(args, phil_args=None, gui=False):
""" Read and process input, create PHIL """
msg = []
input_dict = util.ginp.process_mixed_input(args.path)
if input_dict and not gui and not input_dict['imagefiles']:
return None, None, "IOTA_INIT_ERROR: No readable image files in path(s)!"
# Move args that were included in paths and not processed into phil_args,
# to try and interpret them as PHIL args
if input_dict['badpaths']:
phil_args.extend(input_dict['badpaths'])
# Read in parameters, make IOTA PHIL
iota_phil, bad_args = inp.process_input(args=args,
phil_args=phil_args,
paramfile=input_dict['paramfile'],
gui=gui)
# Check if any PHIL args not read into the PHIL were in fact bad paths
if bad_args:
input_dict['badpaths'] = [a for a in bad_args if a in input_dict['badpaths']]
if input_dict['badpaths']:
msg.append('Files or directories not found:')
for badpath in input_dict['badpaths']:
msg += '\n{}'.format(badpath)
bad_args = [a for a in bad_args if a not in input_dict['badpaths']]
if bad_args:
msg += '\nThese arguments could not be interpreted: '
for arg in bad_args:
msg += '\n{}'.format(arg)
return input_dict, iota_phil, msg
def run(self):
self.args, self.phil_args = parse_command_args(
self.iver, self.help_message).parse_known_args()
# Check for type of input
if len(self.args.path) == 0 or self.args.path is None: # No input
parse_command_args(self.iver, self.help_message).print_help()
if self.args.default: # Write out default params and exit
help_out, txt_out = inp.print_params()
print '\n{:-^70}\n'.format('IOTA Parameters')
print help_out
inp.write_defaults(os.path.abspath(os.path.curdir), txt_out)
misc.iota_exit()
elif len(self.args.path) > 1: # If multiple paths / wildcards
file_list = ginp.make_input_list(self.args.path)
list_file = os.path.join(os.path.abspath(os.path.curdir),
'input.lst')
with open(list_file, 'w') as lf:
lf.write('\n'.join(file_list))
msg = "\nIOTA will run in AUTO mode using wildcard datapath:\n" \
"{} files found, compiled in {}\n".format(len(file_list), list_file)
self.params, self.txt_out = inp.process_input(
self.args, self.phil_args, list_file, 'auto', self.now)
else: # If single path, check type
carg = os.path.abspath(self.args.path[0])
if os.path.exists(carg):
# If user provided a parameter file
if os.path.isfile(carg) and os.path.basename(carg).endswith(
'.param'):
msg = ''
self.params, self.txt_out = inp.process_input(
self.args, self.phil_args, carg, 'file')
# If user provided a list of input files
elif os.path.isfile(carg) and os.path.basename(carg).endswith(
'.lst'):
msg = "\nIOTA will run in AUTO mode using {}:\n".format(
carg)
self.params, self.txt_out = inp.process_input(
self.args, self.phil_args, carg, 'auto', self.now)
# If user provided a single filepath
elif os.path.isfile(
carg) and not os.path.basename(carg).endswith('.lst'):
msg = "\nIOTA will run in SINGLE-FILE mode using {}:\n".format(
carg)
self.params, self.txt_out = inp.process_input(
self.args, self.phil_args, carg, 'auto', self.now)
# If user provided a data folder
elif os.path.isdir(carg):
msg = "\nIOTA will run in AUTO mode using {}:\n".format(
carg)
self.params, self.txt_out = inp.process_input(
self.args, self.phil_args, carg, 'auto', self.now)
# If user provided gibberish
else:
print self.logo
print "ERROR: Invalid input! Need parameter filename or data folder."
misc.iota_exit()
# Identify indexing / integration program
if self.params.advanced.integrate_with == 'cctbx':
prg = " with CCTBX.XFEL\n"
elif self.params.advanced.integrate_with == 'dials':
prg = " with DIALS\n"
self.logo += prg
print self.logo
print '\n{}\n'.format(self.now)
if msg != '':
print msg
if self.args.analyze != None:
print 'ANALYSIS ONLY will be performed (analyzing run #{})'.format(
self.args.analyze)
self.analyze_prior_results('{:003d}'.format(int(
self.args.analyze)))
misc.iota_exit()
if self.params.mp_method == 'mpi':
rank, size = misc.get_mpi_rank_and_size()
self.master_process = rank == 0
else:
self.master_process = True
# Call function to read input folder structure (or input file) and
# generate list of image file paths
if self.params.cctbx.selection.select_only.flag_on:
self.gs_img_objects = self.make_int_object_list()
self.input_list = [i.conv_img for i in self.gs_img_objects]
else:
self.input_list = self.make_input_list()
# Check for -l option, output list of input files and exit
if self.args.list:
list_file = os.path.abspath("{}/input.lst".format(os.curdir))
# Check if other files of this name exist under the current folder
list_folder = os.path.dirname(list_file)
list_files = [
i for i in os.listdir(list_folder) if i.endswith(".lst")
]
if len(list_files) > 0:
list_file = os.path.join(
list_folder, "input_{}.lst".format(len(list_files)))
print '\nINPUT LIST ONLY option selected'
print 'Input list in {} \n\n'.format(list_file)
with open(list_file, "w") as lf:
for i, input_file in enumerate(self.input_list, 1):
lf.write('{}\n'.format(input_file))
print "{}: {}".format(i, input_file)
lf.write('{}\n'.format(input_file))
print '\nExiting...\n\n'
misc.iota_exit()
# If fewer images than requested processors are supplied, set the number of
# processors to the number of images
if self.params.n_processors > len(self.input_list):
self.params.n_processors = len(self.input_list)
# Generate base folder paths
self.conv_base = misc.set_base_dir('converted_pickles',
out_dir=self.params.output)
self.int_base = misc.set_base_dir('integration',
out_dir=self.params.output)
self.obj_base = os.path.join(self.int_base, 'image_objects')
self.fin_base = os.path.join(self.int_base, 'final')
self.log_base = os.path.join(self.int_base, 'logs')
self.viz_base = os.path.join(self.int_base, 'visualization')
self.tmp_base = os.path.join(
'/tmp', '{}_{}'.format(os.getlogin(), time.time()))
# Generate base folders
os.makedirs(self.int_base)
os.makedirs(self.obj_base)
os.makedirs(self.fin_base)
os.makedirs(self.log_base)
os.makedirs(self.tmp_base)
# Determine input base
self.input_base = os.path.abspath(
os.path.dirname(os.path.commonprefix(self.input_list)))
# Initialize main log
self.logfile = os.path.abspath(os.path.join(self.int_base, 'iota.log'))
# Log starting info
misc.main_log(self.logfile, '{:=^80} \n'.format(' IOTA MAIN LOG '))
misc.main_log(self.logfile,
'{:-^80} \n'.format(' SETTINGS FOR THIS RUN '))
misc.main_log(self.logfile, self.txt_out)
if self.params.advanced.integrate_with == 'cctbx':
target_file = self.params.cctbx.target
elif self.params.advanced.integrate_with == 'dials':
target_file = self.params.dials.target
misc.main_log(
self.logfile, '{:-^80} \n\n'
''.format(' TARGET FILE ({}) CONTENTS '
''.format(target_file)))
with open(target_file, 'r') as phil_file:
phil_file_contents = phil_file.read()
misc.main_log(self.logfile, phil_file_contents)
# Write target file and record its location in params
local_target_file = os.path.join(self.int_base, 'target.phil')
with open(local_target_file, 'w') as tf:
tf.write(phil_file_contents)
def initialize_interface(self):
""" Initialize properties specific for command line
:return: True if successful, False if fails
"""
self.args, self.phil_args = parse_command_args(
self.iver, self.help_message).parse_known_args()
ginp = InputFinder()
# Check for type of input
if not self.args.path: # No input
parse_command_args(self.iver, self.help_message).print_help()
if self.args.default: # Write out default params and exit
help_out, txt_out = inp.print_params()
print('\n{:-^70}\n'.format('IOTA Parameters'))
print(help_out)
return False, 'IOTA_XTERM_INIT: OUTPUT PARAMETERS ONLY'
elif len(self.args.path) > 1: # If multiple paths / wildcards
file_list = ginp.make_input_list(self.args.path)
list_file = os.path.join(os.path.abspath(os.path.curdir),
'input.lst')
with open(list_file, 'w') as lf:
lf.write('\n'.join(file_list))
msg = "\nIOTA will run in AUTO mode using wildcard datapath:\n" \
"{} files found, compiled in {}\n".format(len(file_list), list_file)
self.iota_phil = inp.process_input(self.args, self.phil_args,
list_file, 'auto', self.now)
self.params = self.iota_phil.extract()
else: # If single path, check type
carg = os.path.abspath(self.args.path[0])
if os.path.isfile(carg):
ptype = ginp.get_file_type(carg)
if ptype.lower() in ('raw image', 'image pickle'):
msg = "\nIOTA will run in SINGLE-FILE mode using {}:\n".format(
carg)
mode = 'auto'
elif ('iota' and 'settings' in ptype.lower()):
msg = '\nIOTA will run in SCRIPT mode using {}:\n'.format(
carg)
mode = 'file'
elif 'list' in ptype.lower():
msg = "\nIOTA will run in AUTO mode using {}:\n".format(
carg)
mode = 'auto'
else:
pr = 'WARNING! File format not recognized. Proceed anyway? [Y/N] '
unknown_file = raw_input(pr)
if 'y' in unknown_file.lower():
ftype = raw_input(
"File type? [image, list, or parameters] ")
msg = "\nIOTA will run WITH DODGY input using {}:\n".format(
carg)
if 'par' in ftype:
mode = 'file'
else:
mode = 'auto'
else:
print('Exiting...')
return False, 'IOTA_XTERM_INIT_ERROR: Unrecognizable input!'
elif os.path.isdir(carg):
ptype = ginp.get_folder_type(carg)
if ('image' and 'folder' in ptype.lower()):
msg = "\nIOTA will run in AUTO mode using {}:\n".format(
carg)
mode = 'auto'
else:
msg = "IOTA_XTERM_INIT_ERROR: No images in {}!".format(
carg)
print(self.logo)
print(msg)
return False, msg
# If user provided gibberish
else:
msg = "IOTA_XTERM_INIT_ERROR: Invalid input! Need parameter filename " \
"or data folder."
print(self.logo)
print(msg)
return False, msg
# Initialize parameters for this command-line run
self.iota_phil = inp.process_input(self.args, self.phil_args, carg,
mode, self.now)
self.params = self.iota_phil.extract()
# Identify indexing / integration program and add to logo
b_end = " with {}".format(
str(self.params.advanced.processing_backend).upper())
prg = "{:>{w}}".format(b_end, w=76)
self.logo += prg
print(self.logo)
print('\n{}\n'.format(self.now))
if msg != '':
print(msg)
if self.args.analyze is not None:
print('ANALYSIS ONLY will be performed (analyzing run #{})'.format(
self.args.analyze))
self.analyze_prior_results('{:003d}'.format(int(
self.args.analyze)))
return False
if self.params.mp.method == 'mpi':
rank, size = get_mpi_rank_and_size()
self.master_process = rank == 0
else:
self.master_process = True
# Call function to read input folder structure (or input file) and
# generate list of image file paths
if self.params.cctbx_ha14.selection.select_only.flag_on:
cmd.Command.start("Importing saved grid search results")
self.gs_img_objects = self.make_int_object_list()
self.input_list = [i.conv_img for i in self.gs_img_objects]
cmd.Command.end("Importing saved grid search results -- DONE")
else:
cmd.Command.start("Reading input files")
self.input_list = self.make_input_list()
cmd.Command.end("Reading input files -- DONE")
# Select range of images/objects if turned on
if self.params.advanced.image_range.flag_on:
self.input_list = self.select_image_range(self.input_list)
# Pick a randomized subset of images/objects if turned on
if self.params.advanced.random_sample.flag_on and \
self.params.advanced.random_sample.number < len(self.input_list):
cmd.Command.start("Selecting {} random images out of {} found"
"".format(
self.params.advanced.random_sample.number,
len(self.input_list)))
self.input_list = self.select_random_subset(self.input_list)
cmd.Command.end(
"Selecting {} random images out of {} found -- DONE"
"".format(self.params.advanced.random_sample.number,
len(self.input_list)))
# Check for -l option, output list of input files and exit
if self.args.list:
list_file = os.path.abspath("{}/input.lst".format(os.curdir))
# Check if other files of this name exist under the current folder
list_folder = os.path.dirname(list_file)
list_files = [
i for i in os.listdir(list_folder) if i.endswith(".lst")
]
if len(list_files) > 0:
list_file = os.path.join(
list_folder, "input_{}.lst".format(len(list_files)))
msg = 'IOTA_XTERM_INIT: INPUT LIST ONLY option selected'
print('\n{}'.format(msg))
print('Input list in {} \n\n'.format(list_file))
with open(list_file, "w") as lf:
for i, input_file in enumerate(self.input_list, 1):
lf.write('{}\n'.format(input_file))
print("{}: {}".format(i, input_file))
lf.write('{}\n'.format(input_file))
print('\nExiting...\n\n')
return False, msg
return True, 'IOTA_XTERM_INIT: Initialization complete!'
def run(self):
self.args, self.phil_args = parse_command_args(self.iver,
self.help_message).parse_known_args()
# Check for type of input
if self.args.path == None: # No input
parse_command_args(self.iver, self.help_message).print_help()
if self.args.default: # Write out default params and exit
help_out, txt_out = inp.print_params()
print '\n{:-^70}\n'.format('IOTA Parameters')
print help_out
inp.write_defaults(os.path.abspath(os.path.curdir), txt_out)
misc.iota_exit()
else: # If input exists, check type
carg = os.path.abspath(self.args.path)
if os.path.exists(carg):
# If user provided a parameter file
if os.path.isfile(carg) and os.path.basename(carg).endswith('.param'):
msg = ''
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'file')
# If user provided a list of input files
elif os.path.isfile(carg) and os.path.basename(carg).endswith('.lst'):
msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg)
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'auto', self.now)
# If user provided a single filepath
elif os.path.isfile(carg) and not os.path.basename(carg).endswith('.lst'):
msg = "\nIOTA will run in SINGLE-FILE mode using {}:\n".format(carg)
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'auto', self.now)
# If user provided a data folder
elif os.path.isdir(carg):
msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg)
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'auto', self.now)
# If user provided gibberish
else:
print self.logo
print "ERROR: Invalid input! Need parameter filename or data folder."
misc.iota_exit()
# Identify indexing / integration program
if self.params.advanced.integrate_with == 'cctbx':
prg = " with CCTBX.XFEL\n"
elif self.params.advanced.integrate_with == 'dials':
prg = " with DIALS\n"
self.logo += prg
print self.logo
print '\n{}\n'.format(self.now)
if msg != '':
print msg
if self.args.analyze != None:
self.analyze_prior_results('{:003d}'.format(int(self.args.analyze)))
misc.iota_exit()
if self.params.mp_method == 'mpi':
rank, size = misc.get_mpi_rank_and_size()
self.master_process = rank == 0
else:
self.master_process = True
# Call function to read input folder structure (or input file) and
# generate list of image file paths
if self.params.cctbx.selection.select_only.flag_on:
self.gs_img_objects = self.make_int_object_list()
self.input_list = [i.conv_img for i in self.gs_img_objects]
else:
self.input_list = self.make_input_list()
# Check for -l option, output list of input files and exit
if self.args.list:
if list_file == None:
list_file = os.path.abspath("{}/input.lst".format(os.curdir))
print '\nINPUT LIST ONLY option selected'
print 'Input list in {} \n\n'.format(list_file)
with open(list_file, "w") as lf:
for i, input_file in enumerate(self.input_list, 1):
lf.write('{}\n'.format(input_file))
print "{}: {}".format(i, input_file)
lf.write('{}\n'.format(input_file))
print '\nExiting...\n\n'
misc.iota_exit()
# If fewer images than requested processors are supplied, set the number of
# processors to the number of images
if self.params.n_processors > len(self.input_list):
self.params.n_processors = len(self.input_list)
# Generate base folder paths
self.conv_base = misc.set_base_dir('converted_pickles', out_dir = self.params.output)
self.int_base = misc.set_base_dir('integration', out_dir = self.params.output)
self.obj_base = os.path.join(self.int_base, 'image_objects')
self.fin_base = os.path.join(self.int_base, 'final')
self.tmp_base = os.path.join(self.int_base, 'tmp')
self.viz_base = os.path.join(self.int_base, 'visualization')
# Generate base folders
os.makedirs(self.int_base)
os.makedirs(self.obj_base)
os.makedirs(self.fin_base)
os.makedirs(self.tmp_base)
# Determine input base
self.input_base = os.path.abspath(os.path.dirname(os.path.commonprefix(self.input_list)))
# Initialize main log
self.logfile = os.path.abspath(os.path.join(self.int_base, 'iota.log'))
# Log starting info
misc.main_log(self.logfile, '{:=^80} \n'.format(' IOTA MAIN LOG '))
misc.main_log(self.logfile, '{:-^80} \n'.format(' SETTINGS FOR THIS RUN '))
misc.main_log(self.logfile, self.txt_out)
if self.params.advanced.integrate_with == 'cctbx':
target_file = self.params.cctbx.target
elif self.params.advanced.integrate_with == 'dials':
target_file = self.params.dials.target
misc.main_log(self.logfile, '{:-^80} \n\n'
''.format(' TARGET FILE ({}) CONTENTS '
''.format(target_file)))
with open(target_file, 'r') as phil_file:
phil_file_contents = phil_file.read()
misc.main_log(self.logfile, phil_file_contents)