Esempio n. 1
0
def initialize_processing(paramfile, run_no):
  ''' Initialize processing for a set of images
  :param paramfile: text file with IOTA parameters
  :param run_no: number of the processing run
  :return: info: INFO object
           params: IOTA params
  '''
  try:
    phil, _ = inp.get_input_phil(paramfile=paramfile)
  except Exception as e:
    msg = 'IOTA_PROC_ERROR: Cannot import IOTA parameters! {}'.format(e)
    return False, msg
  else:
    params = phil.extract()

  # Reconstruct integration base path and get info object
  int_base = os.path.join(params.output, 'integration/{:03d}'.format(run_no))
  try:
    info_file = os.path.join(int_base, 'proc.info')
    info = ProcInfo.from_json(filepath=info_file)
  except Exception as e:
    msg = 'IOTA_PROC_ERROR: Cannot import INFO object! {}'.format(e)
    return False, msg

  # Generate input list and input base
  if not hasattr(info, 'input_list'):
    info.generate_input_list(params=params)
  common_pfx = os.path.abspath(
    os.path.dirname(os.path.commonprefix(info.input_list)))

  input_base = common_pfx
  if os.path.isdir(os.path.abspath(params.input[0])):
    new_common_pfx = os.path.commonprefix([os.path.abspath(params.input[0]), common_pfx])
    if new_common_pfx not in ('', '.'):
      input_base = new_common_pfx

  # Generate subfolder paths
  paths = dict(obj_base=os.path.join(int_base, 'image_objects'),
               fin_base=os.path.join(int_base, 'final'),
               log_base=os.path.join(int_base, 'logs'),
               viz_base=os.path.join(int_base, 'visualization'),
               tmp_base=os.path.join(int_base, 'tmp'),
               input_base=input_base)
  # FIXME: ordering of items in dictionaries changes depending on python version
  for bkey, bvalue in paths.items():
    if bkey == "input_base":  # I think this is what the original code wanted to do
      continue
    if not os.path.isdir(bvalue):
      os.makedirs(bvalue)
  info.update(paths)

  # Generate filepaths for various info files
  info_files = dict(
    obj_list_file=os.path.join(info.tmp_base, 'finished_objects.lst'),
    idx_file=os.path.join(info.int_base, 'observations.pickle')
  )
  info.update(info_files)

  # Initialize stat containers
  info = generate_stat_containers(info=info, params=params)

    # Initialize main log
  util.main_log(info.logfile, '{:*^80} \n'.format(' IOTA MAIN LOG '))
  util.main_log(info.logfile, '{:-^80} \n'.format(' SETTINGS FOR THIS RUN '))
  util.main_log(info.logfile, info.iota_phil)
  util.main_log(info.logfile, '{:-^80} \n'.format('BACKEND SETTINGS'))
  util.main_log(info.logfile, info.target_phil)

  info.export_json()

  return info, params
Esempio n. 2
0
def initialize_processing(paramfile, run_no):
    """Initialize processing for a set of images.

    :param paramfile: text file with IOTA parameters
    :param run_no: number of the processing run
    :return: info: INFO object
             params: IOTA params
    """
    try:
        phil, _ = inp.get_input_phil(paramfile=paramfile)
    except Exception as e:
        msg = "IOTA_PROC_ERROR: Cannot import IOTA parameters! {}".format(e)
        return False, msg
    else:
        params = phil.extract()

    # Reconstruct integration base path and get info object
    int_base = os.path.join(params.output, "integration/{:03d}".format(run_no))
    try:
        info_file = os.path.join(int_base, "proc.info")
        info = ProcInfo.from_json(filepath=info_file)
    except Exception as e:
        msg = "IOTA_PROC_ERROR: Cannot import INFO object! {}".format(e)
        return False, msg

    # Generate input list and input base
    if not hasattr(info, "input_list"):
        info.generate_input_list(params=params)
    filepath_list = []
    for item in info.input_list:
        if isinstance(item, list) or isinstance(item, tuple):
            fp = [i for i in item if os.path.exists(str(i))]
            if fp and len(fp) == 1:
                filepath_list.append(fp[0])
        else:
            if os.path.exists(item):
                filepath_list.append(item)
    common_pfx = os.path.abspath(
        os.path.dirname(os.path.commonprefix(filepath_list)))

    input_base = common_pfx
    if os.path.isdir(os.path.abspath(params.input[0])):
        new_common_pfx = os.path.commonprefix(
            [os.path.abspath(params.input[0]), common_pfx])
        if new_common_pfx not in ("", "."):
            input_base = new_common_pfx

    # Generate subfolder paths
    paths = dict(
        obj_base=os.path.join(int_base, "image_objects"),
        fin_base=os.path.join(int_base, "final"),
        log_base=os.path.join(int_base, "logs"),
        dials_log_base=os.path.join(int_base, "logs/dials_logs"),
        viz_base=os.path.join(int_base, "visualization"),
        tmp_base=os.path.join(int_base, "tmp"),
        input_base=input_base,
    )
    for bkey, bvalue in paths.items():
        if bkey == "input_base":
            continue
        if not os.path.isdir(bvalue):
            os.makedirs(bvalue)
    info.update(paths)

    # Generate filepaths for various info files
    info_files = dict(
        obj_list_file=os.path.join(info.tmp_base, "finished_objects.lst"),
        idx_file=os.path.join(info.int_base, "observations.pickle"),
    )
    info.update(info_files)

    # Initialize stat containers
    info = generate_stat_containers(info=info, params=params)

    # Initialize main log
    util.main_log(info.logfile, "{:*^80} \n".format(" IOTA MAIN LOG "))
    util.main_log(info.logfile, "{:-^80} \n".format(" SETTINGS FOR THIS RUN "))
    util.main_log(info.logfile, info.iota_phil)
    util.main_log(info.logfile, "{:-^80} \n".format("BACKEND SETTINGS"))
    util.main_log(info.logfile, info.target_phil)

    info.export_json()

    return info, params