def initialize_single_image(img, paramfile, output_file=None, output_dir=None, min_bragg=10): phil, _ = inp.get_input_phil(paramfile=paramfile) params = phil.extract() params.input = [img] params.mp.n_processors = 1 params.data_selection.image_triage.minimum_Bragg_peaks = min_bragg info = ProcInfo.from_args(iota_phil=phil.as_str(), paramfile=paramfile) # Initialize output if output_file is not None: if output_dir is not None: output = os.path.join(os.path.abspath(output_dir), output_file) else: output = os.path.abspath(output_file) else: output = None info.obj_list_file = output info.generate_input_list(params=params) info = generate_stat_containers(info=info, params=params) return info, params
def process_mixed_input(self, paths): input_dict = dict(paramfile=None, imagefiles=[], imagepaths=[], objectfiles=[], objectpaths=[], neither=[], badpaths=[]) if type(paths) == str: paths = [paths] for path in paths: if os.path.exists(path): if 'IOTA settings' in self.get_file_type(path): input_dict['paramfile'] = path # If there's a paramfile, get data from it first (will always be # imagefiles, never objects!) from iota.components.iota_input import get_input_phil phil, _ = get_input_phil(paramfile=input_dict['paramfile']) prm = phil.extract() input_dict['imagefiles'].extend(self.make_input_list(prm.input)) input_dict['imagepaths'].extend(prm.input) else: contents, ctype = self.get_input(path, filter_type='self') if not contents: continue if ctype is None: ctype = '' if type(contents) == str: contents = [contents] if 'object' in ctype: input_dict['objectfiles'].extend(contents) if 'folder' in ctype: input_dict['objectpaths'].append(os.path.abspath(path)) else: input_dict['objectpaths'].append(os.path.abspath(os.path.dirname(path))) elif 'image' in ctype: input_dict['imagefiles'].extend(contents) if 'folder' in ctype: input_dict['imagepaths'].append(os.path.abspath(path)) else: input_dict['imagepaths'].append(os.path.abspath(os.path.dirname(path))) else: input_dict['neither'].extend(contents) input_dict['badpaths'].append(os.path.abspath(path)) else: input_dict['badpaths'].append(os.path.abspath(path)) # Make paths unique if input_dict['imagefiles']: input_dict['imagefiles'] = list(set(input_dict['imagefiles'])) if input_dict['objectfiles']: input_dict['objectfiles'] = list(set(input_dict['objectfiles'])) if input_dict['imagepaths']: input_dict['imagepaths'] = list(set(input_dict['imagepaths'])) if input_dict['objectpaths']: input_dict['objectpaths'] = list(set(input_dict['objectpaths'])) return input_dict
def resume_processing(info): ''' Initialize run parameters for an existing run (e.g. for resuming a terminated run or re-submitting with new images) :param info: INFO object :return: info: Updated INFO object params: IOTA params ''' if not info.init_proc: return initialize_processing(info.paramfile, info.run_number) else: try: phil, _ = inp.get_input_phil(paramfile=info.paramfile) except Exception as e: return None, None else: info.status = 'processing' return info, phil.extract()
def initialize_processing(paramfile, run_no): ''' Initialize processing for a set of images :param paramfile: text file with IOTA parameters :param run_no: number of the processing run :return: info: INFO object params: IOTA params ''' try: phil, _ = inp.get_input_phil(paramfile=paramfile) except Exception as e: msg = 'IOTA_PROC_ERROR: Cannot import IOTA parameters! {}'.format(e) return False, msg else: params = phil.extract() # Reconstruct integration base path and get info object int_base = os.path.join(params.output, 'integration/{:03d}'.format(run_no)) try: info_file = os.path.join(int_base, 'proc.info') info = ProcInfo.from_json(filepath=info_file) except Exception as e: msg = 'IOTA_PROC_ERROR: Cannot import INFO object! {}'.format(e) return False, msg # Generate input list and input base if not hasattr(info, 'input_list'): info.generate_input_list(params=params) common_pfx = os.path.abspath( os.path.dirname(os.path.commonprefix(info.input_list))) input_base = common_pfx if os.path.isdir(os.path.abspath(params.input[0])): new_common_pfx = os.path.commonprefix([os.path.abspath(params.input[0]), common_pfx]) if new_common_pfx not in ('', '.'): input_base = new_common_pfx # Generate subfolder paths paths = dict(obj_base=os.path.join(int_base, 'image_objects'), fin_base=os.path.join(int_base, 'final'), log_base=os.path.join(int_base, 'logs'), viz_base=os.path.join(int_base, 'visualization'), tmp_base=os.path.join(int_base, 'tmp'), input_base=input_base) # FIXME: ordering of items in dictionaries changes depending on python version for bkey, bvalue in paths.items(): if bkey == "input_base": # I think this is what the original code wanted to do continue if not os.path.isdir(bvalue): os.makedirs(bvalue) info.update(paths) # Generate filepaths for various info files info_files = dict( obj_list_file=os.path.join(info.tmp_base, 'finished_objects.lst'), idx_file=os.path.join(info.int_base, 'observations.pickle') ) info.update(info_files) # Initialize stat containers info = generate_stat_containers(info=info, params=params) # Initialize main log util.main_log(info.logfile, '{:*^80} \n'.format(' IOTA MAIN LOG ')) util.main_log(info.logfile, '{:-^80} \n'.format(' SETTINGS FOR THIS RUN ')) util.main_log(info.logfile, info.iota_phil) util.main_log(info.logfile, '{:-^80} \n'.format('BACKEND SETTINGS')) util.main_log(info.logfile, info.target_phil) info.export_json() return info, params
def process_mixed_input(self, paths): input_dict = dict( paramfile=None, imagefiles=[], imagepaths=[], objectfiles=[], objectpaths=[], neither=[], badpaths=[], ) if type(paths) == str: raw_paths = [paths] elif isinstance(paths, list) or isinstance(paths, tuple): raw_paths = paths paths = [] for path in raw_paths: paths.extend(glob(path)) for path in paths: path = os.path.abspath(path) if os.path.exists(path): if "IOTA settings" in self.get_file_type(path): input_dict["paramfile"] = path # If there's a paramfile, get data from it first (will always be # imagefiles, never objects!) from iota.components.iota_input import get_input_phil phil, _ = get_input_phil(paramfile=input_dict["paramfile"]) prm = phil.extract() input_dict["imagefiles"].extend(self.make_input_list(prm.input)[0]) input_dict["imagepaths"].extend(prm.input) else: contents, ctype, _ = self.get_input(path, filter_type="self") if not contents: continue if ctype is None: ctype = "" if type(contents) == str: contents = [contents] if "object" in ctype: input_dict["objectfiles"].extend(contents) if "folder" in ctype: input_dict["objectpaths"].append(os.path.abspath(path)) else: input_dict["objectpaths"].append( os.path.abspath(os.path.dirname(path)) ) elif "image" in ctype: input_dict["imagefiles"].extend(contents) if "folder" in ctype: input_dict["imagepaths"].append(os.path.abspath(path)) else: input_dict["imagepaths"].append( os.path.abspath(os.path.dirname(path)) ) else: input_dict["neither"].extend(contents) input_dict["badpaths"].append(path) else: input_dict["badpaths"].append(path) # Make paths unique if input_dict["imagefiles"]: input_dict["imagefiles"] = list(set(input_dict["imagefiles"])) if input_dict["objectfiles"]: input_dict["objectfiles"] = list(set(input_dict["objectfiles"])) if input_dict["imagepaths"]: input_dict["imagepaths"] = list(set(input_dict["imagepaths"])) if input_dict["objectpaths"]: input_dict["objectpaths"] = list(set(input_dict["objectpaths"])) return input_dict
def initialize_processing(paramfile, run_no): """Initialize processing for a set of images. :param paramfile: text file with IOTA parameters :param run_no: number of the processing run :return: info: INFO object params: IOTA params """ try: phil, _ = inp.get_input_phil(paramfile=paramfile) except Exception as e: msg = "IOTA_PROC_ERROR: Cannot import IOTA parameters! {}".format(e) return False, msg else: params = phil.extract() # Reconstruct integration base path and get info object int_base = os.path.join(params.output, "integration/{:03d}".format(run_no)) try: info_file = os.path.join(int_base, "proc.info") info = ProcInfo.from_json(filepath=info_file) except Exception as e: msg = "IOTA_PROC_ERROR: Cannot import INFO object! {}".format(e) return False, msg # Generate input list and input base if not hasattr(info, "input_list"): info.generate_input_list(params=params) filepath_list = [] for item in info.input_list: if isinstance(item, list) or isinstance(item, tuple): fp = [i for i in item if os.path.exists(str(i))] if fp and len(fp) == 1: filepath_list.append(fp[0]) else: if os.path.exists(item): filepath_list.append(item) common_pfx = os.path.abspath( os.path.dirname(os.path.commonprefix(filepath_list))) input_base = common_pfx if os.path.isdir(os.path.abspath(params.input[0])): new_common_pfx = os.path.commonprefix( [os.path.abspath(params.input[0]), common_pfx]) if new_common_pfx not in ("", "."): input_base = new_common_pfx # Generate subfolder paths paths = dict( obj_base=os.path.join(int_base, "image_objects"), fin_base=os.path.join(int_base, "final"), log_base=os.path.join(int_base, "logs"), dials_log_base=os.path.join(int_base, "logs/dials_logs"), viz_base=os.path.join(int_base, "visualization"), tmp_base=os.path.join(int_base, "tmp"), input_base=input_base, ) for bkey, bvalue in paths.items(): if bkey == "input_base": continue if not os.path.isdir(bvalue): os.makedirs(bvalue) info.update(paths) # Generate filepaths for various info files info_files = dict( obj_list_file=os.path.join(info.tmp_base, "finished_objects.lst"), idx_file=os.path.join(info.int_base, "observations.pickle"), ) info.update(info_files) # Initialize stat containers info = generate_stat_containers(info=info, params=params) # Initialize main log util.main_log(info.logfile, "{:*^80} \n".format(" IOTA MAIN LOG ")) util.main_log(info.logfile, "{:-^80} \n".format(" SETTINGS FOR THIS RUN ")) util.main_log(info.logfile, info.iota_phil) util.main_log(info.logfile, "{:-^80} \n".format("BACKEND SETTINGS")) util.main_log(info.logfile, info.target_phil) info.export_json() return info, params