def testDequoteUnmatched(self): self.assertTrue(dequote('"' + '(0, 1)') == '"(0, 1)')
def testDequote(self): self.assertTrue(dequote('"(0, 1)"') == '(0, 1)')
def testNoDequoteNeeded(self): self.assertTrue(dequote("(0, 1)") == '(0, 1)')
def process_raw_cfg(raw_cfg, args): """ Add default information and perform error checking @param raw_cfg: configuration read by config parser @param args: command-line arguments @return: processed, error-checked config """ # boolean to specify if an additional output (resid) is in the fit.best resid_in_best = args.resid # None or file location last_best_file = args.file # None or file location summary_file_name = args.summary_file cfgs = { FIT_OPT: { HIJ_FIT: args.hij_fit, VII_SEC: args.vii_fit, REP1_SEC: args.rep1_fit, REP_EXPON2_SEC: args.rep_exp2_fit } } # Process raw cfg values for section in raw_cfg: section_dict = {} if section == MAIN_SEC: for entry in raw_cfg[section]: section_dict[entry[0]] = entry[1] else: section_dict = {} for entry in raw_cfg[section]: if entry[0] == GROUP_NAMES: section_dict[entry[0]] = entry[1] else: vals = [x.strip() for x in entry[1].split(',')] if len(vals) == 2: vals.append(DEF_DESCRIP) try: section_dict[entry[0]] = { LOW: float(vals[0]), HIGH: float(vals[1]), DESCRIP: vals[2] } except ValueError: raise InvalidDataError( "Check input. In configuration file section '{}', expected " "comma-separated numerical lower range value, upper-range value, and " "(optional) description (i.e. '-10,10,d_OO') for key '{}'. " "Found: {}".format(section, entry[0], entry[1])) cfgs[section] = section_dict # Check for defaults for section in cfgs: if section == MAIN_SEC: for cfg in MAIN_SEC_DEF_CFG_VALS: if cfg not in cfgs[section]: cfgs[section][cfg] = MAIN_SEC_DEF_CFG_VALS[cfg] else: if section in PARAM_SECS: for param in PARAM_SEC_REQ_CFG_VALS: if param not in cfgs[section]: raise InvalidDataError( "In configuration file section '{}', missing parameter '{}'." "".format(section, param)) else: if section != FIT_OPT: raise InvalidDataError( "Found section '{}' in the configuration file. This program expects only " "sections: {}".format(section, [MAIN_SEC, FIT_OPT] + PARAM_SECS)) # Add main section with defaults if the optional main section is missing if MAIN_SEC not in cfgs: cfgs[MAIN_SEC] = MAIN_SEC_DEF_CFG_VALS if resid_in_best: cfgs[MAIN_SEC][PARAM_NUM] = 1 cfgs[MAIN_SEC][RESID_IN_BEST] = True else: cfgs[MAIN_SEC][PARAM_NUM] = 0 cfgs[MAIN_SEC][RESID_IN_BEST] = False # Ensure all required info is present for specified sections. for section in cfgs: if section == MAIN_SEC: for param in cfgs[section]: if param not in MAIN_SEC_DEF_CFG_VALS: raise InvalidDataError( "The configuration file contains parameter '{}' in section '{}'; expected " "only the following parameters for this section: {}" "".format(param, section, MAIN_SEC_DEF_CFG_VALS.keys())) if len(cfgs[MAIN_SEC][SUM_HEAD_SUFFIX]) > 1: cfgs[MAIN_SEC][param] = dequote(cfgs[MAIN_SEC][param]) if cfgs[section][OUT_BASE_DIR] is None: cfgs[section][OUT_BASE_DIR] = "" cfgs[section][INP_FILE] = os.path.abspath( os.path.join(cfgs[section][OUT_BASE_DIR], cfgs[section][INP_FILE])) if not os.path.exists(os.path.dirname(cfgs[section][INP_FILE])): raise IOError( "Invalid directory provided in configuration section '{}' " "parameter '{}': {}".format(section, OUT_BASE_DIR, cfgs[section][OUT_BASE_DIR])) elif section != FIT_OPT: for param in FIT_PARAMS[section]: cfgs[MAIN_SEC][PARAM_NUM] += 1 if param not in cfgs[section]: raise InvalidDataError( "The configuration file is missing parameter '{}' in section '{}'. " "Check input.".format(param, section)) for param in cfgs[section]: if param not in FIT_PARAMS[section] and param != GROUP_NAMES: raise InvalidDataError( "The configuration file contains parameter '{}' in section '{}'; expected " "only the following parameters for this section: {}" "".format(param, section, FIT_PARAMS[section])) if last_best_file is not None: if os.path.exists(last_best_file): cfgs[MAIN_SEC][BEST_FILE] = last_best_file else: raise IOError("Invalid '{}' provided".format(last_best_file)) if summary_file_name is not None: if cfgs[MAIN_SEC][BEST_FILE] is None: raise InvalidDataError( "No '{}' specified, which is required with a specified '{}'" "".format(BEST_FILE, SUMMARY_FILE)) if os.path.exists(summary_file_name): cfgs[MAIN_SEC][SUMMARY_FILE] = summary_file_name else: # can create a new file, but make sure directory exists dir_name = os.path.dirname(summary_file_name) if os.path.exists(dir_name) or dir_name == '': warning( "Will create a new summary file, as specified '{}' not found: {}" "".format(SUMMARY_FILE, summary_file_name)) cfgs[MAIN_SEC][SUMMARY_FILE] = summary_file_name else: raise IOError( "Invalid '{}' provided. Neither file nor directory found for " "specified file: {}".format(SUMMARY_FILE, summary_file_name)) return cfgs
def make_summary(cfg): """ If the option is specified, add the last best fit output file to the list of outputs and evaluate changes @param cfg: configuration for the run @return: """ best_file = cfg[MAIN_SEC][BEST_FILE] summary_file = cfg[MAIN_SEC][SUMMARY_FILE] low, high, headers = get_param_info(cfg) latest_output = np.loadtxt(best_file, dtype=np.float64) if os.path.isfile(summary_file): last_row = None percent_diffs = [] previous_output = np.loadtxt(summary_file, dtype=np.float64) all_output = np.vstack((previous_output, latest_output)) for row in all_output: if last_row is not None: diff = row - last_row percent_diff = {} # Check data for small values, hitting upper or lower bound, and calc % diff for index, val in enumerate(np.nditer(row)): if abs(val) < TOL: warning( "Small value ({}) encountered for parameter {} (col {})" "".format(val, headers[index], index)) if abs(diff[index]) > TOL: if abs(last_row[index]) > TOL: percent_diff[headers[index]] = round( diff[index] / last_row[index] * 100, 2) else: if abs(diff[index]) > TOL: percent_diff[headers[index]] = np.inf if abs(val - low[index]) < TOL: warning( "Value ({}) near lower bound ({}) encountered for parameter {} (col {})." "".format(val, low[index], headers[index], index)) if abs(val - high[index]) < TOL: warning( "Value ({}) near upper bound ({}) encountered for parameter {} (col {})." "".format(val, high[index], headers[index], index)) else: percent_diff[headers[index]] = np.nan percent_diffs.append(percent_diff) last_row = row if len(percent_diffs) > 0: max_percent_diff = 0 max_diff_param = None for param, val in percent_diffs[-1].items(): if abs(val) > abs(max_percent_diff): max_percent_diff = val max_diff_param = param print( "Maximum (absolute value) percent difference from last read line is {} % for parameter '{}'." "".format(max_percent_diff, max_diff_param)) if cfg[MAIN_SEC][RESID_IN_BEST]: print("Percent change in residual: {} %" "".format( percent_diffs[-1][RESIDUAL + cfg[MAIN_SEC][SUM_HEAD_SUFFIX]])) # format for gnuplot and np.loadtxt f_out = create_out_fname(summary_file, suffix='_perc_diff', ext='.csv', base_dir=cfg[MAIN_SEC][OUT_BASE_DIR]) write_csv(percent_diffs, f_out, headers, extrasaction="ignore") f_out = create_out_fname(summary_file, ext='.csv', base_dir=cfg[MAIN_SEC][OUT_BASE_DIR]) header_str = ','.join([dequote(header) for header in headers]) print(headers) print(header_str) with open(f_out, 'w') as s_file: s_file.write(','.join(headers) + '\n') np.savetxt(f_out, all_output, fmt='%8.6f', delimiter=',') print(header_str) np.savetxt(f_out, all_output, fmt='%8.6f', delimiter=',', header=header_str, comments='') print('Wrote file: {}'.format(f_out)) # in addition to csv (above), print format for gnuplot and np.loadtxt # with open(summary_file, 'w') as s_file: # np.savetxt(s_file, all_output, fmt='%12.6f') np.savetxt(summary_file, all_output, fmt='%12.6f') print("Wrote file: {}".format(summary_file)) else: # have this as sep statement, because now printing a 1D array, handled differently than 2D array (newline=' ') np.savetxt(summary_file, latest_output, fmt='%12.6f', newline=' ') print("Wrote results from {} to new summary file {}".format( best_file, summary_file))