def analyze(self, output_dir):
logging.info('Analyzing value...')
# gets mean values and outliers
self.values = np.array([datapoint.value for datapoint in self.data])
self.value_mean = np.mean(self.values, axis=0)
self.value_std = np.std(self.values, axis=0)
all_outliers = set(
get_outliers_dist_mean(self.values, self.config.value_outlier_stds,
True, True))
# registers outliers
self.low_values = []
self.high_values = []
for t in range(1, len(self.data) - 1):
# tests for above outlier
if t in all_outliers and self.values[t] > self.value_mean and \
self.values[t - 1] <= self.values[t] > self.values[t + 1]:
self.high_values.append(t)
# tests for below outlier
elif t in all_outliers and self.values[t] < self.value_mean and \
self.values[t - 1] >= self.values[t] < self.values[t + 1]:
self.low_values.append(t)
# sorts outliers
self.high_values.sort(key=lambda i: self.values[i], reverse=True)
self.low_values.sort(key=lambda i: self.values[i])
# summary of elements
logging.info('Finished')
logging.info(f'\tMean value received over {len(self.data)} timesteps: '
f'{self.value_mean:.3f} ± {self.value_std:.3f}')
logging.info('\tFound {} high values (stds={}): {}'.format(
len(self.high_values), self.config.value_outlier_stds,
self.high_values))
logging.info('\tFound {} low values (stds={}): {}'.format(
len(self.low_values), self.config.value_outlier_stds,
self.low_values))
logging.info('Saving report in {}...'.format(output_dir))
# saves analysis report
self.save(os.path.join(output_dir, 'value.pkl.gz'))
value_std = self.values.std(0)
self._save_time_dataset_csv(
self.values, 'Value', os.path.join(output_dir, 'values-time.csv'))
self._plot_elements_sp(
self.values,
self.value_mean + self.config.value_outlier_stds * value_std,
self.value_mean - self.config.value_outlier_stds * value_std,
os.path.join(output_dir, 'value-time.{}'.format(self.img_fmt)),
'High value threshold', 'Low value threshold', 'Value', 'Value')
save_list_csv(self.low_values,
os.path.join(output_dir, 'low-values.csv'))
save_list_csv(self.high_values,
os.path.join(output_dir, 'high-values.csv'))
def analyze(self, output_dir):
logging.info('Analyzing reward...')
# gets mean rewards and outliers
self.all_rewards = np.array(
[datapoint.reward for datapoint in self.data])
self.mean_reward = self.all_rewards.mean(0)
all_outliers = set(
get_outliers_dist_mean(self.all_rewards,
self.config.rwd_outlier_stds, True, True))
# registers outliers
self.low_rewards = []
self.high_rewards = []
for t in range(1, len(self.data) - 1):
# tests for above outlier
if t in all_outliers and self.all_rewards[t] > self.mean_reward and \
self.all_rewards[t - 1] <= self.all_rewards[t] > self.all_rewards[t + 1]:
self.high_rewards.append(t)
# tests for below outlier
elif t in all_outliers and self.all_rewards[t] < self.mean_reward and \
self.all_rewards[t - 1] >= self.all_rewards[t] < self.all_rewards[t + 1]:
self.low_rewards.append(t)
# sorts outliers
self.high_rewards.sort(key=lambda i: self.all_rewards[i], reverse=True)
self.low_rewards.sort(key=lambda i: self.all_rewards[i])
# summary of elements
logging.info('Finished')
logging.info('\tMean reward received over {} timesteps: {:.3f}'.format(
len(self.data), self.mean_reward))
logging.info('\tFound {} high rewards (stds={}): {}'.format(
len(self.high_rewards), self.config.rwd_outlier_stds,
self.high_rewards))
logging.info('\tFound {} low rewards (stds={}): {}'.format(
len(self.low_rewards), self.config.rwd_outlier_stds,
self.low_rewards))
logging.info('Saving report in {}...'.format(output_dir))
# saves analysis report
self.save(os.path.join(output_dir, 'reward.pkl.gz'))
rwd_std = self.all_rewards.std(0)
save_list_csv(list(self.all_rewards),
os.path.join(output_dir, 'all_rewards.csv'))
self._plot_elements(
self.all_rewards, self.high_rewards, self.low_rewards,
self.mean_reward + self.config.rwd_outlier_stds * rwd_std,
self.mean_reward - self.config.rwd_outlier_stds * rwd_std,
os.path.join(output_dir, 'reward-time.{}'.format(self.img_fmt)),
'High reward threshold', 'Low reward threshold', 'Reward',
'Reward')
save_list_csv(self.low_rewards,
os.path.join(output_dir, 'low_rewards.csv'))
save_list_csv(self.high_rewards,
os.path.join(output_dir, 'high_rewards.csv'))
def analyze(self, output_dir):
logging.info(
'Analyzing action execution value (maximum minimum differences)...'
)
# prepares multiprocessing
pool = self._get_mp_pool()
# gets action-factor exec max differences for each timestep
data = [datapoint.action_probs for datapoint in self.data]
self.all_execution_diffs = np.array(
pool.map(_get_action_max_differences, data))
# registers mean max diff outliers
self.mean_execution_diffs = self.all_execution_diffs.mean(axis=1)
self.mean_execution_diff = self.mean_execution_diffs.mean(0)
self.high_exec_differences = []
for t in range(1, len(self.data) - 1):
# tests for high difference element (local maximum)
if self.mean_execution_diffs[t] >= self.config.uncertain_exec_min_div and \
self.mean_execution_diffs[t - 1] <= self.mean_execution_diffs[t] > self.mean_execution_diffs[t + 1]:
self.high_exec_differences.append(t)
# sorts outliers
self.high_exec_differences.sort(
key=lambda i: self.mean_execution_diffs[i], reverse=True)
# gets mean action factor max execution difference
self.mean_action_factor_diffs = self.all_execution_diffs.mean(axis=0)
# summary of elements
logging.info('Finished')
logging.info(
'\tMean action-execution differences (peak-to-peak) over {} timesteps: {:.3f}'
.format(len(self.data), self.mean_execution_diff))
logging.info(
'\tFound {} high execution differences (min diff={}): {}'.format(
len(self.high_exec_differences),
self.config.uncertain_exec_min_div,
self.high_exec_differences))
logging.info('Saving report in {}...'.format(output_dir))
# saves analysis report
self.save(os.path.join(output_dir, 'execution-value.pkl.gz'))
np.savetxt(os.path.join(output_dir, 'all-execution-diffs.csv'),
self.all_execution_diffs,
'%s',
',',
comments='')
save_list_csv(self.mean_execution_diffs,
os.path.join(output_dir, 'mean-exec-diff-time.csv'))
self._plot_elements(
self.mean_execution_diffs, self.high_exec_differences, [],
self.config.certain_exec_max_div, np.nan,
os.path.join(output_dir,
'mean-exec-diff-time.{}'.format(self.img_fmt)),
'High exec. diff. threshold', '',
'Action Execution Max. Difference', 'Max. Prob. Difference')
save_list_csv(self.mean_action_factor_diffs,
os.path.join(output_dir, 'mean-action-diffs.csv'))
self._plot_action_factor_divs(
self.mean_action_factor_diffs,
os.path.join(output_dir,
'mean-action-diffs.{}'.format(self.img_fmt)),
'Mean Action-Factor Execution Diff', 'Max. Prob. Difference')
save_list_csv(self.high_exec_differences,
os.path.join(output_dir, 'high-exec-differences.csv'))
for t in self.high_exec_differences:
self._plot_action_factor_divs(
self.all_execution_diffs[t],
os.path.join(
output_dir,
'high-exec-{}-action-diffs.{}'.format(t, self.img_fmt)),
'Mean Action-Factor Execution Diff', 'Max. Prob. Difference')
def analyze(self, output_dir):
if len(self.data) == 0 or self.data[0].next_obs is None:
logging.info('Epistemic uncertainty: nothing to analyze, skipping')
return
logging.info(
'Analyzing epistemic uncertainty via variance in models\' predictions...'
)
# gets mean prediction variance and outliers, where shape is (sample+mean+var (3), num_nets, obs_dim)
next_obs = np.array(
[datapoint.next_obs[MODEL_MEAN_IDX] for datapoint in self.data])
self.all_pred_vars = np.mean(np.square(
np.linalg.norm(np.swapaxes(next_obs, 0, 1) - next_obs.mean(axis=1),
axis=-1)),
axis=0)
self.mean_pred_var = self.all_pred_vars.mean(0)
all_outliers = set(
get_outliers_dist_mean(self.all_pred_vars,
self.config.epistemic_outlier_stds, True,
True))
# registers outliers
self.low_pred_vars = []
self.high_pred_vars = []
for t in range(1, len(self.data) - 1):
# tests for above outlier
if t in all_outliers and self.all_pred_vars[t] > self.mean_pred_var and \
self.all_pred_vars[t - 1] <= self.all_pred_vars[t] > self.all_pred_vars[t + 1]:
self.high_pred_vars.append(t)
# tests for below outlier
elif t in all_outliers and self.all_pred_vars[t] < self.mean_pred_var and \
self.all_pred_vars[t - 1] >= self.all_pred_vars[t] < self.all_pred_vars[t + 1]:
self.low_pred_vars.append(t)
# sorts outliers
self.high_pred_vars.sort(key=lambda i: self.all_pred_vars[i],
reverse=True)
self.low_pred_vars.sort(key=lambda i: self.all_pred_vars[i])
# summary of elements
logging.info('Finished')
logging.info(
'\tMean epistemic uncertainty over {} timesteps: {:.3f}'.format(
len(self.data), self.mean_pred_var))
logging.info(
'\tFound {} situations with high epistemic uncertainty (stds={}): {}'
.format(len(self.high_pred_vars),
self.config.epistemic_outlier_stds, self.high_pred_vars))
logging.info(
'\tFound {} situations with low epistemic uncertainty (stds={}): {}'
.format(len(self.low_pred_vars),
self.config.epistemic_outlier_stds, self.low_pred_vars))
logging.info('Saving report in {}...'.format(output_dir))
# saves analysis report
self.save(os.path.join(output_dir, 'epistemic-uncert-var.pkl.gz'))
rwd_std = self.all_pred_vars.std(0)
save_list_csv(list(self.all_pred_vars),
os.path.join(output_dir, 'all-epistemic-uncert-var.csv'))
self._plot_elements(
self.all_pred_vars, self.high_pred_vars, self.low_pred_vars,
self.mean_pred_var + self.config.epistemic_outlier_stds * rwd_std,
self.mean_pred_var - self.config.epistemic_outlier_stds * rwd_std,
os.path.join(output_dir,
'epistemic-uncert-var-time.{}'.format(self.img_fmt)),
'High epistemic uncert. threshold',
'Low epistemic uncert. threshold', 'Epistemic Uncertainty',
'Predictive Models\' Observation Variance')
save_list_csv(self.low_pred_vars,
os.path.join(output_dir, 'low-epistemic-uncert-var.csv'))
save_list_csv(
self.high_pred_vars,
os.path.join(output_dir, 'high-epistemic-uncert-var.csv'))
def analyze(self, output_dir):
logging.info('Analyzing maximum value difference...')
# gets mean ptps and outliers
self.all_action_diffs = np.array([
np.mean([
np.ptp(factor_values)
for factor_values in datapoint.action_values
]) for datapoint in self.data
])
self.mean_diff = self.all_action_diffs.mean(0)
all_outliers = set(
get_outliers_dist_mean(self.all_action_diffs,
self.config.value_outlier_stds, True, True))
# registers outliers
self.low_action_diffs = []
self.high_action_diffs = []
for t in range(1, len(self.data) - 1):
# tests for above outlier
if t in all_outliers and self.all_action_diffs[t] > self.mean_diff and \
self.all_action_diffs[t - 1] <= self.all_action_diffs[t] > self.all_action_diffs[t + 1]:
self.high_action_diffs.append(t)
# tests for below outlier
elif t in all_outliers and self.all_action_diffs[t] < self.mean_diff and \
self.all_action_diffs[t - 1] >= self.all_action_diffs[t] < self.all_action_diffs[t + 1]:
self.low_action_diffs.append(t)
# sorts outliers
self.high_action_diffs.sort(key=lambda i: self.all_action_diffs[i],
reverse=True)
self.low_action_diffs.sort(key=lambda i: self.all_action_diffs[i])
# summary of elements
logging.info('Finished')
logging.info(
'\tMean max action-value difference over {} timesteps: {:.3f}'.
format(len(self.data), self.mean_diff))
logging.info(
'\tFound {} high action-value differences (stds={}): {}'.format(
len(self.high_action_diffs), self.config.value_outlier_stds,
self.high_action_diffs))
logging.info(
'\tFound {} low action-value differences (stds={}): {}'.format(
len(self.low_action_diffs), self.config.value_outlier_stds,
self.low_action_diffs))
logging.info('Saving report in {}...'.format(output_dir))
# saves analysis report
self.save(os.path.join(output_dir, 'action-value.pkl.gz'))
value_std = self.all_action_diffs.std(0)
save_list_csv(list(self.all_action_diffs),
os.path.join(output_dir, 'all-action-diffs.csv'))
self._plot_elements(
self.all_action_diffs, self.high_action_diffs,
self.low_action_diffs,
self.mean_diff + self.config.value_outlier_stds * value_std,
self.mean_diff - self.config.value_outlier_stds * value_std,
os.path.join(output_dir,
'action-diff-time.{}'.format(self.img_fmt)),
'High action-diff. threshold', 'Low action-diff. threshold',
'Maximum Action-Value Difference', 'Action-Value Diff.')
save_list_csv(self.low_action_diffs,
os.path.join(output_dir, 'low-action-diffs.csv'))
save_list_csv(self.high_action_diffs,
os.path.join(output_dir, 'high-action-diffs.csv'))
def analyze(self, output_dir):
if len(self.data) == 0 or self.data[0].next_obs is None:
logging.info('Epistemic uncertainty: nothing to analyze, skipping')
return
logging.info(
'Analyzing epistemic uncertainty via JRD in models\' prob. predictions...'
)
# gets mean prediction JRD and outliers, where next_obs shape is (sample+mean+var (3), num_nets, obs_dim)
# based on https://github.com/nnaisense/MAX/blob/master/utilities.py
# shape: (steps, ensemble_size, d_state)
mu = np.array(
[datapoint.next_obs[MODEL_MEAN_IDX] for datapoint in self.data],
dtype=np.double)
var = np.array([
datapoint.next_obs[MODEL_VARIANCE_IDX] for datapoint in self.data
],
dtype=np.double)
steps, es, d_s = mu.shape
# entropy of the mean
entropy_mean = np.zeros(steps, dtype=np.double)
for i in range(es):
for j in range(es):
mu_i, mu_j = mu[:, i], mu[:, j] # shape: both (steps, d_state)
var_i, var_j = var[:,
i], var[:,
j] # shape: both (steps, d_state)
mu_diff = mu_j - mu_i # shape: (steps, d_state)
var_sum = var_i + var_j # shape: (steps, d_state)
pre_exp = (mu_diff * 1 / var_sum * mu_diff
) # shape: (steps, d_state)
pre_exp = np.sum(pre_exp, axis=-1) # shape: (steps)
exp = np.exp(-1 / 2 * pre_exp) # shape: (steps)
den = np.sqrt(np.prod(var_sum, axis=-1)) # shape: (steps)
entropy_mean += exp / den # shape: (steps)
entropy_mean = -np.log(entropy_mean / (((2 * np.pi)**(d_s / 2)) *
(es * es))) # shape: (steps)
# mean of entropies
total_entropy = np.prod(var, axis=-1) # shape: (steps, ensemble_size)
total_entropy = np.log(((2 * np.pi)**d_s) *
total_entropy) # shape: (steps, ensemble_size)
total_entropy = 1 / 2 * total_entropy + (d_s / 2) * np.log(
2) # shape: (steps, ensemble_size)
mean_entropy = total_entropy.mean(axis=1)
# Jensen-Renyi divergence
self.all_pred_jrds = entropy_mean - mean_entropy # shape: (steps)
self.mean_pred_jrd = self.all_pred_jrds.mean(0)
all_outliers = set(
get_outliers_dist_mean(self.all_pred_jrds,
self.config.epistemic_outlier_stds, True,
True))
# registers outliers
self.low_pred_jrds = []
self.high_pred_jrds = []
for t in range(1, len(self.data) - 1):
# tests for above outlier
if t in all_outliers and self.all_pred_jrds[t] > self.mean_pred_jrd and \
self.all_pred_jrds[t - 1] <= self.all_pred_jrds[t] > self.all_pred_jrds[t + 1]:
self.high_pred_jrds.append(t)
# tests for below outlier
elif t in all_outliers and self.all_pred_jrds[t] < self.mean_pred_jrd and \
self.all_pred_jrds[t - 1] >= self.all_pred_jrds[t] < self.all_pred_jrds[t + 1]:
self.low_pred_jrds.append(t)
# sorts outliers
self.high_pred_jrds.sort(key=lambda i: self.all_pred_jrds[i],
reverse=True)
self.low_pred_jrds.sort(key=lambda i: self.all_pred_jrds[i])
# summary of elements
logging.info('Finished')
logging.info(
'\tMean epistemic uncertainty over {} timesteps: {:.3f}'.format(
len(self.data), self.mean_pred_jrd))
logging.info(
'\tFound {} situations with high epistemic uncertainty (stds={}): {}'
.format(len(self.high_pred_jrds),
self.config.epistemic_outlier_stds, self.high_pred_jrds))
logging.info(
'\tFound {} situations with low epistemic uncertainty (stds={}): {}'
.format(len(self.low_pred_jrds),
self.config.epistemic_outlier_stds, self.low_pred_jrds))
logging.info('Saving report in {}...'.format(output_dir))
# saves analysis report
self.save(os.path.join(output_dir, 'epistemic-uncert-jrd.pkl.gz'))
rwd_std = self.all_pred_jrds.std(0)
save_list_csv(list(self.all_pred_jrds),
os.path.join(output_dir, 'all-epistemic-uncert-jrd.csv'))
self._plot_elements(
self.all_pred_jrds, self.high_pred_jrds, self.low_pred_jrds,
self.mean_pred_jrd + self.config.epistemic_outlier_stds * rwd_std,
self.mean_pred_jrd - self.config.epistemic_outlier_stds * rwd_std,
os.path.join(output_dir,
'epistemic-uncert-jrd-time.{}'.format(self.img_fmt)),
'High epistemic uncert. threshold',
'Low epistemic uncert. threshold', 'Epistemic Uncertainty',
'Predictive Models\' Observation JRD')
save_list_csv(self.low_pred_jrds,
os.path.join(output_dir, 'low-epistemic-uncert-jrd.csv'))
save_list_csv(
self.high_pred_jrds,
os.path.join(output_dir, 'high-epistemic-uncert-jrd.csv'))
def analyze(self, output_dir):
logging.info('Analyzing action execution uncertainty...')
# prepares multiprocessing
pool = self._get_mp_pool()
# gets action-factor exec diversities for each timestep
data = [datapoint.action_probs for datapoint in self.data]
self.all_execution_divs = np.array(
pool.map(_get_action_dist_evenness, data))
# registers mean exec outliers
self.mean_execution_divs = self.all_execution_divs.mean(axis=1)
self.mean_execution_div = self.mean_execution_divs.mean(0)
self.uncertain_executions = []
self.certain_executions = []
for t in range(1, len(self.data) - 1):
# tests for uncertain element (local maximum)
if self.mean_execution_divs[t] >= self.config.uncertain_exec_min_div and \
self.mean_execution_divs[t - 1] <= self.mean_execution_divs[t] > self.mean_execution_divs[t + 1]:
self.uncertain_executions.append(t)
# tests for certain element (local minimum)
elif self.mean_execution_divs[t] <= self.config.certain_exec_max_div and \
self.mean_execution_divs[t - 1] >= self.mean_execution_divs[t] < self.mean_execution_divs[t + 1]:
self.certain_executions.append(t)
# sorts outliers
self.uncertain_executions.sort(
key=lambda i: self.mean_execution_divs[i], reverse=True)
self.certain_executions.sort(key=lambda i: self.mean_execution_divs[i])
# gets mean action factor execution diversity
self.mean_action_factor_divs = self.all_execution_divs.mean(axis=0)
# summary of elements
logging.info('Finished')
logging.info(
'\tMean action-execution certainty over {} timesteps: {:.3f}'.
format(len(self.data), self.mean_execution_div))
logging.info(
'\tFound {} uncertain action executions (min div={}): {}'.format(
len(self.uncertain_executions),
self.config.uncertain_exec_min_div, self.uncertain_executions))
logging.info(
'\tFound {} certain action executions (max div={}): {}'.format(
len(self.certain_executions), self.config.certain_exec_max_div,
self.certain_executions))
logging.info('Saving report in {}...'.format(output_dir))
# saves analysis report
self.save(os.path.join(output_dir, 'execution-certainty.pkl.gz'))
np.savetxt(os.path.join(output_dir, 'all-execution-divs.csv'),
self.all_execution_divs,
'%s',
',',
comments='')
self._save_time_dataset_csv(
self.mean_execution_divs, 'Execution Uncertainty',
os.path.join(output_dir, 'mean-exec-div-time.csv'))
self._plot_elements_sp(
self.mean_execution_divs, self.config.uncertain_exec_min_div,
self.config.certain_exec_max_div,
os.path.join(output_dir,
'mean-exec-div-time.{}'.format(self.img_fmt)),
'Uncert. exec. threshold', 'Cert. exec. threshold',
'Action Execution Uncertainty', 'Norm. True Diversity')
save_list_csv(self.mean_action_factor_divs,
os.path.join(output_dir, 'mean-action-divs.csv'))
self._plot_action_factor_divs(
self.mean_action_factor_divs,
os.path.join(output_dir,
'mean-action-divs.{}'.format(self.img_fmt)),
'Mean Action-Factor Execution Uncertainty', 'Norm. True Diversity')
save_list_csv(self.certain_executions,
os.path.join(output_dir, 'certain-executions.csv'))
save_list_csv(self.uncertain_executions,
os.path.join(output_dir, 'uncertain-executions.csv'))
for t in self.certain_executions:
self._plot_action_factor_divs(
self.all_execution_divs[t],
os.path.join(
output_dir,
'cert-exec-{}-action-divs.{}'.format(t, self.img_fmt)),
'Mean Action-Factor Execution Uncertainty',
'Norm. True Diversity')
for t in self.uncertain_executions:
self._plot_action_factor_divs(
self.all_execution_divs[t],
os.path.join(
output_dir,
'uncert-exec-{}-action-divs.{}'.format(t, self.img_fmt)),
'Mean Action-Factor Execution Uncertainty',
'Norm. True Diversity')
def analyze(self, output_dir):
if len(self.data) == 0 or self.data[0].next_obs is None:
logging.info('Aleatoric uncertainty: nothing to analyze, skipping')
return
logging.info('Analyzing aleatoric uncertainty...')
# gets mean prediction variance and outliers, where next_obs shape is (sample+mean+var (3), num_nets, obs_dim)
self.all_pred_vars = np.array([
mean_entropy(datapoint.next_obs[MODEL_VARIANCE_IDX])
for datapoint in self.data
])
self.mean_pred_var = self.all_pred_vars.mean(0)
all_outliers = set(
get_outliers_dist_mean(self.all_pred_vars,
self.config.aleatoric_outlier_stds, True,
True))
# registers outliers
self.low_pred_vars = []
self.high_pred_vars = []
for t in range(1, len(self.data) - 1):
# tests for above outlier
if t in all_outliers and self.all_pred_vars[t] > self.mean_pred_var and \
self.all_pred_vars[t - 1] <= self.all_pred_vars[t] > self.all_pred_vars[t + 1]:
self.high_pred_vars.append(t)
# tests for below outlier
elif t in all_outliers and self.all_pred_vars[t] < self.mean_pred_var and \
self.all_pred_vars[t - 1] >= self.all_pred_vars[t] < self.all_pred_vars[t + 1]:
self.low_pred_vars.append(t)
# sorts outliers
self.high_pred_vars.sort(key=lambda i: self.all_pred_vars[i],
reverse=True)
self.low_pred_vars.sort(key=lambda i: self.all_pred_vars[i])
# summary of elements
logging.info('Finished')
logging.info(
'\tMean aleatoric uncertainty over {} timesteps: {:.3f}'.format(
len(self.data), self.mean_pred_var))
logging.info(
'\tFound {} situations with high aleatoric uncertainty (stds={}): {}'
.format(len(self.high_pred_vars),
self.config.aleatoric_outlier_stds, self.high_pred_vars))
logging.info(
'\tFound {} situations with low aleatoric uncertainty (stds={}): {}'
.format(len(self.low_pred_vars),
self.config.aleatoric_outlier_stds, self.low_pred_vars))
logging.info('Saving report in {}...'.format(output_dir))
# saves analysis report
self.save(os.path.join(output_dir, 'aleatoric-uncertainty.pkl.gz'))
rwd_std = self.all_pred_vars.std(0)
save_list_csv(list(self.all_pred_vars),
os.path.join(output_dir, 'all-aleatoric-uncert.csv'))
self._plot_elements(
self.all_pred_vars, self.high_pred_vars, self.low_pred_vars,
self.mean_pred_var + self.config.aleatoric_outlier_stds * rwd_std,
self.mean_pred_var - self.config.aleatoric_outlier_stds * rwd_std,
os.path.join(output_dir,
'aleatoric-uncert-time.{}'.format(self.img_fmt)),
'High aleatoric uncert. threshold',
'Low aleatoric uncert. threshold', 'Aleatoric Uncertainty',
'Mean Prediction Variance')
save_list_csv(self.low_pred_vars,
os.path.join(output_dir, 'low-aleatoric-uncert.csv'))
save_list_csv(self.high_pred_vars,
os.path.join(output_dir, 'high-aleatoric-uncert.csv'))