def testDequoteUnmatched(self):
self.assertTrue(dequote('"' + '(0, 1)') == '"(0, 1)')
def testDequote(self):
self.assertTrue(dequote('"(0, 1)"') == '(0, 1)')
def testNoDequoteNeeded(self):
self.assertTrue(dequote("(0, 1)") == '(0, 1)')
def process_raw_cfg(raw_cfg, args):
"""
Add default information and perform error checking
@param raw_cfg: configuration read by config parser
@param args: command-line arguments
@return: processed, error-checked config
"""
# boolean to specify if an additional output (resid) is in the fit.best
resid_in_best = args.resid
# None or file location
last_best_file = args.file
# None or file location
summary_file_name = args.summary_file
cfgs = {
FIT_OPT: {
HIJ_FIT: args.hij_fit,
VII_SEC: args.vii_fit,
REP1_SEC: args.rep1_fit,
REP_EXPON2_SEC: args.rep_exp2_fit
}
}
# Process raw cfg values
for section in raw_cfg:
section_dict = {}
if section == MAIN_SEC:
for entry in raw_cfg[section]:
section_dict[entry[0]] = entry[1]
else:
section_dict = {}
for entry in raw_cfg[section]:
if entry[0] == GROUP_NAMES:
section_dict[entry[0]] = entry[1]
else:
vals = [x.strip() for x in entry[1].split(',')]
if len(vals) == 2:
vals.append(DEF_DESCRIP)
try:
section_dict[entry[0]] = {
LOW: float(vals[0]),
HIGH: float(vals[1]),
DESCRIP: vals[2]
}
except ValueError:
raise InvalidDataError(
"Check input. In configuration file section '{}', expected "
"comma-separated numerical lower range value, upper-range value, and "
"(optional) description (i.e. '-10,10,d_OO') for key '{}'. "
"Found: {}".format(section, entry[0], entry[1]))
cfgs[section] = section_dict
# Check for defaults
for section in cfgs:
if section == MAIN_SEC:
for cfg in MAIN_SEC_DEF_CFG_VALS:
if cfg not in cfgs[section]:
cfgs[section][cfg] = MAIN_SEC_DEF_CFG_VALS[cfg]
else:
if section in PARAM_SECS:
for param in PARAM_SEC_REQ_CFG_VALS:
if param not in cfgs[section]:
raise InvalidDataError(
"In configuration file section '{}', missing parameter '{}'."
"".format(section, param))
else:
if section != FIT_OPT:
raise InvalidDataError(
"Found section '{}' in the configuration file. This program expects only "
"sections: {}".format(section, [MAIN_SEC, FIT_OPT] +
PARAM_SECS))
# Add main section with defaults if the optional main section is missing
if MAIN_SEC not in cfgs:
cfgs[MAIN_SEC] = MAIN_SEC_DEF_CFG_VALS
if resid_in_best:
cfgs[MAIN_SEC][PARAM_NUM] = 1
cfgs[MAIN_SEC][RESID_IN_BEST] = True
else:
cfgs[MAIN_SEC][PARAM_NUM] = 0
cfgs[MAIN_SEC][RESID_IN_BEST] = False
# Ensure all required info is present for specified sections.
for section in cfgs:
if section == MAIN_SEC:
for param in cfgs[section]:
if param not in MAIN_SEC_DEF_CFG_VALS:
raise InvalidDataError(
"The configuration file contains parameter '{}' in section '{}'; expected "
"only the following parameters for this section: {}"
"".format(param, section,
MAIN_SEC_DEF_CFG_VALS.keys()))
if len(cfgs[MAIN_SEC][SUM_HEAD_SUFFIX]) > 1:
cfgs[MAIN_SEC][param] = dequote(cfgs[MAIN_SEC][param])
if cfgs[section][OUT_BASE_DIR] is None:
cfgs[section][OUT_BASE_DIR] = ""
cfgs[section][INP_FILE] = os.path.abspath(
os.path.join(cfgs[section][OUT_BASE_DIR],
cfgs[section][INP_FILE]))
if not os.path.exists(os.path.dirname(cfgs[section][INP_FILE])):
raise IOError(
"Invalid directory provided in configuration section '{}' "
"parameter '{}': {}".format(section, OUT_BASE_DIR,
cfgs[section][OUT_BASE_DIR]))
elif section != FIT_OPT:
for param in FIT_PARAMS[section]:
cfgs[MAIN_SEC][PARAM_NUM] += 1
if param not in cfgs[section]:
raise InvalidDataError(
"The configuration file is missing parameter '{}' in section '{}'. "
"Check input.".format(param, section))
for param in cfgs[section]:
if param not in FIT_PARAMS[section] and param != GROUP_NAMES:
raise InvalidDataError(
"The configuration file contains parameter '{}' in section '{}'; expected "
"only the following parameters for this section: {}"
"".format(param, section, FIT_PARAMS[section]))
if last_best_file is not None:
if os.path.exists(last_best_file):
cfgs[MAIN_SEC][BEST_FILE] = last_best_file
else:
raise IOError("Invalid '{}' provided".format(last_best_file))
if summary_file_name is not None:
if cfgs[MAIN_SEC][BEST_FILE] is None:
raise InvalidDataError(
"No '{}' specified, which is required with a specified '{}'"
"".format(BEST_FILE, SUMMARY_FILE))
if os.path.exists(summary_file_name):
cfgs[MAIN_SEC][SUMMARY_FILE] = summary_file_name
else:
# can create a new file, but make sure directory exists
dir_name = os.path.dirname(summary_file_name)
if os.path.exists(dir_name) or dir_name == '':
warning(
"Will create a new summary file, as specified '{}' not found: {}"
"".format(SUMMARY_FILE, summary_file_name))
cfgs[MAIN_SEC][SUMMARY_FILE] = summary_file_name
else:
raise IOError(
"Invalid '{}' provided. Neither file nor directory found for "
"specified file: {}".format(SUMMARY_FILE,
summary_file_name))
return cfgs
def make_summary(cfg):
"""
If the option is specified, add the last best fit output file to the list of outputs and evaluate changes
@param cfg: configuration for the run
@return:
"""
best_file = cfg[MAIN_SEC][BEST_FILE]
summary_file = cfg[MAIN_SEC][SUMMARY_FILE]
low, high, headers = get_param_info(cfg)
latest_output = np.loadtxt(best_file, dtype=np.float64)
if os.path.isfile(summary_file):
last_row = None
percent_diffs = []
previous_output = np.loadtxt(summary_file, dtype=np.float64)
all_output = np.vstack((previous_output, latest_output))
for row in all_output:
if last_row is not None:
diff = row - last_row
percent_diff = {}
# Check data for small values, hitting upper or lower bound, and calc % diff
for index, val in enumerate(np.nditer(row)):
if abs(val) < TOL:
warning(
"Small value ({}) encountered for parameter {} (col {})"
"".format(val, headers[index], index))
if abs(diff[index]) > TOL:
if abs(last_row[index]) > TOL:
percent_diff[headers[index]] = round(
diff[index] / last_row[index] * 100, 2)
else:
if abs(diff[index]) > TOL:
percent_diff[headers[index]] = np.inf
if abs(val - low[index]) < TOL:
warning(
"Value ({}) near lower bound ({}) encountered for parameter {} (col {})."
"".format(val, low[index], headers[index],
index))
if abs(val - high[index]) < TOL:
warning(
"Value ({}) near upper bound ({}) encountered for parameter {} (col {})."
"".format(val, high[index], headers[index],
index))
else:
percent_diff[headers[index]] = np.nan
percent_diffs.append(percent_diff)
last_row = row
if len(percent_diffs) > 0:
max_percent_diff = 0
max_diff_param = None
for param, val in percent_diffs[-1].items():
if abs(val) > abs(max_percent_diff):
max_percent_diff = val
max_diff_param = param
print(
"Maximum (absolute value) percent difference from last read line is {} % for parameter '{}'."
"".format(max_percent_diff, max_diff_param))
if cfg[MAIN_SEC][RESID_IN_BEST]:
print("Percent change in residual: {} %"
"".format(
percent_diffs[-1][RESIDUAL +
cfg[MAIN_SEC][SUM_HEAD_SUFFIX]]))
# format for gnuplot and np.loadtxt
f_out = create_out_fname(summary_file,
suffix='_perc_diff',
ext='.csv',
base_dir=cfg[MAIN_SEC][OUT_BASE_DIR])
write_csv(percent_diffs, f_out, headers, extrasaction="ignore")
f_out = create_out_fname(summary_file,
ext='.csv',
base_dir=cfg[MAIN_SEC][OUT_BASE_DIR])
header_str = ','.join([dequote(header) for header in headers])
print(headers)
print(header_str)
with open(f_out, 'w') as s_file:
s_file.write(','.join(headers) + '\n')
np.savetxt(f_out, all_output, fmt='%8.6f', delimiter=',')
print(header_str)
np.savetxt(f_out,
all_output,
fmt='%8.6f',
delimiter=',',
header=header_str,
comments='')
print('Wrote file: {}'.format(f_out))
# in addition to csv (above), print format for gnuplot and np.loadtxt
# with open(summary_file, 'w') as s_file:
# np.savetxt(s_file, all_output, fmt='%12.6f')
np.savetxt(summary_file, all_output, fmt='%12.6f')
print("Wrote file: {}".format(summary_file))
else:
# have this as sep statement, because now printing a 1D array, handled differently than 2D array (newline=' ')
np.savetxt(summary_file, latest_output, fmt='%12.6f', newline=' ')
print("Wrote results from {} to new summary file {}".format(
best_file, summary_file))
def testDequoteUnmatched(self):
self.assertTrue(dequote('"' + '(0, 1)') == '"(0, 1)')
def testNoDequoteNeeded(self):
self.assertTrue(dequote("(0, 1)") == '(0, 1)')
def testDequote(self):
self.assertTrue(dequote('"(0, 1)"') == '(0, 1)')
