def initialize_single_image(img, paramfile, output_file=None, output_dir=None,
min_bragg=10):
phil, _ = inp.get_input_phil(paramfile=paramfile)
params = phil.extract()
params.input = [img]
params.mp.n_processors = 1
params.data_selection.image_triage.minimum_Bragg_peaks = min_bragg
info = ProcInfo.from_args(iota_phil=phil.as_str(),
paramfile=paramfile)
# Initialize output
if output_file is not None:
if output_dir is not None:
output = os.path.join(os.path.abspath(output_dir), output_file)
else:
output = os.path.abspath(output_file)
else:
output = None
info.obj_list_file = output
info.generate_input_list(params=params)
info = generate_stat_containers(info=info, params=params)
return info, params
def process_mixed_input(self, paths):
input_dict = dict(paramfile=None,
imagefiles=[], imagepaths=[],
objectfiles=[], objectpaths=[],
neither=[], badpaths=[])
if type(paths) == str:
paths = [paths]
for path in paths:
if os.path.exists(path):
if 'IOTA settings' in self.get_file_type(path):
input_dict['paramfile'] = path
# If there's a paramfile, get data from it first (will always be
# imagefiles, never objects!)
from iota.components.iota_input import get_input_phil
phil, _ = get_input_phil(paramfile=input_dict['paramfile'])
prm = phil.extract()
input_dict['imagefiles'].extend(self.make_input_list(prm.input))
input_dict['imagepaths'].extend(prm.input)
else:
contents, ctype = self.get_input(path, filter_type='self')
if not contents:
continue
if ctype is None:
ctype = ''
if type(contents) == str:
contents = [contents]
if 'object' in ctype:
input_dict['objectfiles'].extend(contents)
if 'folder' in ctype:
input_dict['objectpaths'].append(os.path.abspath(path))
else:
input_dict['objectpaths'].append(os.path.abspath(os.path.dirname(path)))
elif 'image' in ctype:
input_dict['imagefiles'].extend(contents)
if 'folder' in ctype:
input_dict['imagepaths'].append(os.path.abspath(path))
else:
input_dict['imagepaths'].append(os.path.abspath(os.path.dirname(path)))
else:
input_dict['neither'].extend(contents)
input_dict['badpaths'].append(os.path.abspath(path))
else:
input_dict['badpaths'].append(os.path.abspath(path))
# Make paths unique
if input_dict['imagefiles']:
input_dict['imagefiles'] = list(set(input_dict['imagefiles']))
if input_dict['objectfiles']:
input_dict['objectfiles'] = list(set(input_dict['objectfiles']))
if input_dict['imagepaths']:
input_dict['imagepaths'] = list(set(input_dict['imagepaths']))
if input_dict['objectpaths']:
input_dict['objectpaths'] = list(set(input_dict['objectpaths']))
return input_dict
def resume_processing(info):
''' Initialize run parameters for an existing run (e.g. for resuming a
terminated run or re-submitting with new images)
:param info: INFO object
:return: info: Updated INFO object
params: IOTA params
'''
if not info.init_proc:
return initialize_processing(info.paramfile, info.run_number)
else:
try:
phil, _ = inp.get_input_phil(paramfile=info.paramfile)
except Exception as e:
return None, None
else:
info.status = 'processing'
return info, phil.extract()
def initialize_processing(paramfile, run_no):
''' Initialize processing for a set of images
:param paramfile: text file with IOTA parameters
:param run_no: number of the processing run
:return: info: INFO object
params: IOTA params
'''
try:
phil, _ = inp.get_input_phil(paramfile=paramfile)
except Exception as e:
msg = 'IOTA_PROC_ERROR: Cannot import IOTA parameters! {}'.format(e)
return False, msg
else:
params = phil.extract()
# Reconstruct integration base path and get info object
int_base = os.path.join(params.output, 'integration/{:03d}'.format(run_no))
try:
info_file = os.path.join(int_base, 'proc.info')
info = ProcInfo.from_json(filepath=info_file)
except Exception as e:
msg = 'IOTA_PROC_ERROR: Cannot import INFO object! {}'.format(e)
return False, msg
# Generate input list and input base
if not hasattr(info, 'input_list'):
info.generate_input_list(params=params)
common_pfx = os.path.abspath(
os.path.dirname(os.path.commonprefix(info.input_list)))
input_base = common_pfx
if os.path.isdir(os.path.abspath(params.input[0])):
new_common_pfx = os.path.commonprefix([os.path.abspath(params.input[0]), common_pfx])
if new_common_pfx not in ('', '.'):
input_base = new_common_pfx
# Generate subfolder paths
paths = dict(obj_base=os.path.join(int_base, 'image_objects'),
fin_base=os.path.join(int_base, 'final'),
log_base=os.path.join(int_base, 'logs'),
viz_base=os.path.join(int_base, 'visualization'),
tmp_base=os.path.join(int_base, 'tmp'),
input_base=input_base)
# FIXME: ordering of items in dictionaries changes depending on python version
for bkey, bvalue in paths.items():
if bkey == "input_base": # I think this is what the original code wanted to do
continue
if not os.path.isdir(bvalue):
os.makedirs(bvalue)
info.update(paths)
# Generate filepaths for various info files
info_files = dict(
obj_list_file=os.path.join(info.tmp_base, 'finished_objects.lst'),
idx_file=os.path.join(info.int_base, 'observations.pickle')
)
info.update(info_files)
# Initialize stat containers
info = generate_stat_containers(info=info, params=params)
# Initialize main log
util.main_log(info.logfile, '{:*^80} \n'.format(' IOTA MAIN LOG '))
util.main_log(info.logfile, '{:-^80} \n'.format(' SETTINGS FOR THIS RUN '))
util.main_log(info.logfile, info.iota_phil)
util.main_log(info.logfile, '{:-^80} \n'.format('BACKEND SETTINGS'))
util.main_log(info.logfile, info.target_phil)
info.export_json()
return info, params
def process_mixed_input(self, paths):
input_dict = dict(
paramfile=None,
imagefiles=[],
imagepaths=[],
objectfiles=[],
objectpaths=[],
neither=[],
badpaths=[],
)
if type(paths) == str:
raw_paths = [paths]
elif isinstance(paths, list) or isinstance(paths, tuple):
raw_paths = paths
paths = []
for path in raw_paths:
paths.extend(glob(path))
for path in paths:
path = os.path.abspath(path)
if os.path.exists(path):
if "IOTA settings" in self.get_file_type(path):
input_dict["paramfile"] = path
# If there's a paramfile, get data from it first (will always be
# imagefiles, never objects!)
from iota.components.iota_input import get_input_phil
phil, _ = get_input_phil(paramfile=input_dict["paramfile"])
prm = phil.extract()
input_dict["imagefiles"].extend(self.make_input_list(prm.input)[0])
input_dict["imagepaths"].extend(prm.input)
else:
contents, ctype, _ = self.get_input(path, filter_type="self")
if not contents:
continue
if ctype is None:
ctype = ""
if type(contents) == str:
contents = [contents]
if "object" in ctype:
input_dict["objectfiles"].extend(contents)
if "folder" in ctype:
input_dict["objectpaths"].append(os.path.abspath(path))
else:
input_dict["objectpaths"].append(
os.path.abspath(os.path.dirname(path))
)
elif "image" in ctype:
input_dict["imagefiles"].extend(contents)
if "folder" in ctype:
input_dict["imagepaths"].append(os.path.abspath(path))
else:
input_dict["imagepaths"].append(
os.path.abspath(os.path.dirname(path))
)
else:
input_dict["neither"].extend(contents)
input_dict["badpaths"].append(path)
else:
input_dict["badpaths"].append(path)
# Make paths unique
if input_dict["imagefiles"]:
input_dict["imagefiles"] = list(set(input_dict["imagefiles"]))
if input_dict["objectfiles"]:
input_dict["objectfiles"] = list(set(input_dict["objectfiles"]))
if input_dict["imagepaths"]:
input_dict["imagepaths"] = list(set(input_dict["imagepaths"]))
if input_dict["objectpaths"]:
input_dict["objectpaths"] = list(set(input_dict["objectpaths"]))
return input_dict
def initialize_processing(paramfile, run_no):
"""Initialize processing for a set of images.
:param paramfile: text file with IOTA parameters
:param run_no: number of the processing run
:return: info: INFO object
params: IOTA params
"""
try:
phil, _ = inp.get_input_phil(paramfile=paramfile)
except Exception as e:
msg = "IOTA_PROC_ERROR: Cannot import IOTA parameters! {}".format(e)
return False, msg
else:
params = phil.extract()
# Reconstruct integration base path and get info object
int_base = os.path.join(params.output, "integration/{:03d}".format(run_no))
try:
info_file = os.path.join(int_base, "proc.info")
info = ProcInfo.from_json(filepath=info_file)
except Exception as e:
msg = "IOTA_PROC_ERROR: Cannot import INFO object! {}".format(e)
return False, msg
# Generate input list and input base
if not hasattr(info, "input_list"):
info.generate_input_list(params=params)
filepath_list = []
for item in info.input_list:
if isinstance(item, list) or isinstance(item, tuple):
fp = [i for i in item if os.path.exists(str(i))]
if fp and len(fp) == 1:
filepath_list.append(fp[0])
else:
if os.path.exists(item):
filepath_list.append(item)
common_pfx = os.path.abspath(
os.path.dirname(os.path.commonprefix(filepath_list)))
input_base = common_pfx
if os.path.isdir(os.path.abspath(params.input[0])):
new_common_pfx = os.path.commonprefix(
[os.path.abspath(params.input[0]), common_pfx])
if new_common_pfx not in ("", "."):
input_base = new_common_pfx
# Generate subfolder paths
paths = dict(
obj_base=os.path.join(int_base, "image_objects"),
fin_base=os.path.join(int_base, "final"),
log_base=os.path.join(int_base, "logs"),
dials_log_base=os.path.join(int_base, "logs/dials_logs"),
viz_base=os.path.join(int_base, "visualization"),
tmp_base=os.path.join(int_base, "tmp"),
input_base=input_base,
)
for bkey, bvalue in paths.items():
if bkey == "input_base":
continue
if not os.path.isdir(bvalue):
os.makedirs(bvalue)
info.update(paths)
# Generate filepaths for various info files
info_files = dict(
obj_list_file=os.path.join(info.tmp_base, "finished_objects.lst"),
idx_file=os.path.join(info.int_base, "observations.pickle"),
)
info.update(info_files)
# Initialize stat containers
info = generate_stat_containers(info=info, params=params)
# Initialize main log
util.main_log(info.logfile, "{:*^80} \n".format(" IOTA MAIN LOG "))
util.main_log(info.logfile, "{:-^80} \n".format(" SETTINGS FOR THIS RUN "))
util.main_log(info.logfile, info.iota_phil)
util.main_log(info.logfile, "{:-^80} \n".format("BACKEND SETTINGS"))
util.main_log(info.logfile, info.target_phil)
info.export_json()
return info, params
