def report_phase_I(master_df):
df = None
print("macro average")
for agg, od in product([True, False], repeat=2):
for base_size in [200, 300, 400, 500]:
df = fixed(master_df, {
'aggregation': agg,
'order-dependence': od,
'base size': base_size
})
df[f'precision'] = df[f'true positives'] / (df[f'true positives'] +
df[f'false positives'])
df[f'recall'] = df[f'true positives'] / df['dependencies']
df[f'F-measure'] = 2 * df[f'precision'] * df[f'recall'] / (
df[f'precision'] + df[f'recall'])
print(
f"{agg} & {not od} & {base_size} & {df['precision'].mean() * 100:.2f}\\% & {df['recall'].mean() * 100:.2f}\\% \\\\"
)
print()
print()
print("micro average")
for agg, od in product([True, False], repeat=2):
for base_size in [200, 300, 400, 500]:
df = fixed(master_df, {
'aggregation': agg,
'order-dependence': od,
'base size': base_size
})
precision = df[f'true positives'].sum() / (
df[f'true positives'].sum() + df[f'false positives'].sum())
recall = df[f'true positives'].sum() / df['dependencies'].sum()
print(
f"{agg} & {not od} & {base_size} & {precision*100:.2f}\\% & {recall*100:.2f}\\% \\\\"
)
print()
print()
print("average TP, FP")
for agg, od in product([True, False], repeat=2):
for base_size in [200, 300, 400, 500]:
df = fixed(master_df, {
'aggregation': agg,
'order-dependence': od,
'base size': base_size
})
print(
f"{agg} & {not od} & {base_size} & {df[f'true positives'].mean():.2f}\\% & {df[f'false positives'].mean():.2f}\\% \\\\"
)
return df
def check_number_of_tests(folder, master_df: pd.DataFrame):
working_df = fixed(master_df, {'aggregation': False, 'orientation rule': 'majority'})
working_df = working_df[np.logical_or(working_df['base size'] == 200, working_df['base size'] == 500)]
working_df['detect'] = np.logical_and(working_df['detect rbo'], working_df['detect post rbo'])
working_df = working_df[working_df['detect rbo'] == working_df['detect post rbo']] # both on or both of
working_df['number of tests'] = (working_df['RBO test'] + working_df['post-RBO test'])
working_df["criteria"] = working_df["sepset rule"] + ' ' + working_df["detect"].map(lambda x: 'w/ detect' if x else 'w/o detect')
hue_order = [sr + ' ' + det for sr in ['first', 'minimal', 'full'] for det in ['w/o detect', 'w/ detect']]
paper_rc = {'lines.linewidth': 1, 'lines.markersize': 2}
sns.set(style='white', font_scale=1.2, palette=sns.color_palette('Paired', 6), context='paper', rc=paper_rc, color_codes=False)
plt.figure()
g = sns.catplot(x='base size',
y='number of tests',
hue='criteria',
data=working_df,
hue_order=hue_order,
kind='box', legend=False, legend_out=False,
height=3.25, aspect=1.2
)
plt.legend(bbox_to_anchor=(1.05, 0.75), loc=2, borderaxespad=0.)
g.axes.flat[0].set_yscale('log', basey=2)
sns.despine()
plt.tight_layout()
plt.savefig(f'{working_dir}{folder}/number_of_tests.pdf', transparent=True, bbox_inches='tight', pad_inches=0.02)
plt.close('all')
def check_number_of_cases(folder, master_df: pd.DataFrame):
working_df = fixed(master_df, {'aggregation': False, 'orientation rule': 'majority'})
working_df = working_df[np.logical_or(working_df['base size'] == 200, working_df['base size'] == 500)]
working_df['detect'] = np.logical_and(working_df['detect rbo'], working_df['detect post rbo'])
working_df = working_df[working_df['detect rbo'] == working_df['detect post rbo']] # both on or both of
working_df['number of cases'] = (working_df['RBO case'] + working_df['post-RBO case'])
working_df["criteria"] = working_df["sepset rule"] + ' ' + working_df["detect"].map(lambda x: 'w/ detect' if x else 'w/o detect')
hue_order = [sr + ' ' + det for sr in ['first', 'minimal', 'full'] for det in ['w/o detect', 'w/ detect']]
paper_rc = {'lines.linewidth': 1, 'lines.markersize': 2}
sns.set(style='white', font_scale=1.2, palette=sns.color_palette('Paired', 6), context='paper', rc=paper_rc)
plt.figure()
sns.catplot(x='base size',
y='number of cases',
hue='criteria',
data=working_df,
hue_order=hue_order,
dodge=True,
kind='strip',
jitter=1,
alpha=0.5,
)
sns.despine()
plt.tight_layout()
plt.savefig(f'{working_dir}{folder}/number_of_cases.pdf', transparent=True, bbox_inches='tight', pad_inches=0.02)
plt.close('all')
def how_aggregation(folder, master_df: pd.DataFrame):
df = master_df.rename(
columns={
'right direction (aggregated)': 'right direction',
'reverse direction (aggregated)': 'reverse direction',
'false positives (aggregated)': 'false positive'
})
df['counts'] = df['right direction'] + df['reverse direction'] + df[
'false positive']
how_aggregated_worked = list()
for size in sorted(df['base size'].unique()):
subdf = fixed(df, {'base size': size})
counts = subdf['right direction'].sum(
) + subdf['reverse direction'].sum() + subdf['false positive'].sum()
how_aggregated_worked.append([
size, counts, subdf['right direction'].mean(),
subdf['reverse direction'].mean(), subdf['false positive'].mean()
])
agg_df = pd.DataFrame(how_aggregated_worked,
columns=[
'base size', 'counts', 'right direction',
'reverse direction', 'false positive'
])
draw_how_aggregated_works(folder, agg_df, df)
def _find_worse_settings(master_df, criteria, columns, alpha=0.01, verbose=False, worst=False, the_larger_the_better=True, test_threshold=20):
""" Find values of settings to be excluded """
justlen = max(len(str(v)) for v in columns) + 2
vallen = max(len(str(val)) for v in columns for val in master_df[v].unique()) + 2
performances = list()
for settings in product(*[list(master_df[var].unique()) for var in columns]):
criteria_values = fixed(master_df, dict(zip(columns, settings)))[criteria].mean()
performances.append((settings, criteria_values))
performances = sorted(performances, key=lambda _perf: _perf[1], reverse=the_larger_the_better)
if len(performances) <= test_threshold:
return set()
rankings = defaultdict(lambda: defaultdict(list))
perfvals = defaultdict(lambda: defaultdict(list))
_print_(''.join([str(f).center(20) for f in columns]), verbose=verbose)
_print_('--------------------------------------------', verbose=verbose)
for ranking, perf in enumerate(performances):
settings, f2 = perf
_print_(''.join([str(f).center(20) for f in settings]), end='', verbose=verbose)
_print_(f'{f2:.3f} -- ({ranking + 1})', verbose=verbose)
for f, s in zip(columns, settings):
rankings[f][s].append(ranking)
perfvals[f][s].append(f2)
to_remove = set()
lowest_pval = None
for col in columns:
keep_vals = set(master_df[col].unique())
for val1, val2 in combinations(master_df[col].unique(), 2):
# let val1 ranksum is small (better)
if sum(rankings[col][val1]) > sum(rankings[col][val2]):
val1, val2 = val2, val1
p_val = scipy.stats.mannwhitneyu(rankings[col][val1], rankings[col][val2], alternative='less')[1]
# p_val = scipy.stats.ttest_ind(perfvals[col][val1], perfvals[col][val2], equal_var=False)[1]
if p_val < alpha:
_print_(f'{str(col).rjust(justlen)}: {str(val1).rjust(vallen)} > {str(val2).ljust(vallen)} (p-value: {p_val:.3f} < {alpha})', verbose=verbose)
if val2 in keep_vals:
keep_vals.remove(val2)
to_remove.add((col, val2))
if lowest_pval is None:
lowest_pval = {(col, val2, p_val)}
elif next(iter(lowest_pval))[-1] == p_val:
lowest_pval.add((col, val2, p_val))
elif next(iter(lowest_pval))[-1] > p_val:
lowest_pval = {(col, val2, p_val)}
if worst:
if lowest_pval is not None:
_print_(f"worst settings: {lowest_pval}", verbose=verbose)
return {(x, y) for x, y, _ in lowest_pval}
else:
return set()
else:
return to_remove
def check_splits2(folder, master_df: pd.DataFrame):
for sepset_rule in ['minimal']:
paper_rc = {'lines.linewidth': 1, 'lines.markersize': 2}
sns.set(style='white', font_scale=1.2, context='paper', rc=paper_rc)
col_fail_noncollider, col_noncollider, col_fail_collider, col_collider = sns.color_palette('Paired', 4)
col_weak = sns.color_palette("Greys", 6)[2]
col_weak2 = sns.color_palette("Greys", 6)[3]
fig, axes = plt.subplots(1, 2, figsize=(6, 3))
axes[0].set_title('RBO')
axes[1].set_title('non-RBO')
for col, sublabel in enumerate(['RBO', 'post-RBO']):
working_df = fixed(master_df, {f'detect rbo': False, 'detect post rbo': False, 'sepset rule': sepset_rule}) # first?
working_df['correct non-collider'] = working_df[f'{sublabel} correct non-collider']
working_df['wrong non-collider'] = working_df[f'{sublabel} non-collider'] - working_df[f'{sublabel} correct non-collider']
working_df['correct collider'] = working_df[f'{sublabel} correct collider']
working_df['wrong collider'] = working_df[f'{sublabel} collider'] - working_df[f'{sublabel} correct collider']
working_df['correct collider-fail'] = working_df[f'{sublabel} correct collider-fail']
working_df['wrong collider-fail'] = working_df[f'{sublabel} collider-fail'] - working_df[f'{sublabel} correct collider-fail']
working_df = working_df.groupby(['base size']).mean().reset_index()
working_df = pd.melt(working_df, id_vars=['base size'],
value_vars=['correct non-collider', 'wrong non-collider', 'correct collider', 'wrong collider', 'correct collider-fail', 'wrong collider-fail'],
var_name='detection type',
value_name='counts')
df = working_df
colors = [col_noncollider, col_fail_noncollider, col_collider, col_fail_collider, col_weak2, col_weak]
ax = axes[col]
margin_bottom = np.zeros(len(working_df['base size'].unique()))
for num, detection_type in enumerate(['correct non-collider', 'wrong non-collider', 'correct collider', 'wrong collider']):
values = list(df[df['detection type'] == detection_type].loc[:, 'counts'])
df[df['detection type'] == detection_type].plot.bar(x='base size', y='counts', ax=ax, stacked=True,
bottom=margin_bottom, color=colors[num], label=detection_type)
margin_bottom += values
if col == 1:
handles, labels = ax.get_legend_handles_labels()
ax.legend(handles[::-1], labels[::-1], loc='center left', bbox_to_anchor=(1, 0.5))
else:
ax.legend().remove()
_, max1 = axes[0].get_ylim()
_, max2 = axes[1].get_ylim()
axes[0].set_ylim(0, 9) # max(max1, max2))
axes[1].set_ylim(0, 9) # max(max1, max2))
axes[1].set_yticks([])
axes[1].set_yticklabels([])
axes[0].set_ylabel('normal tests')
sns.despine()
plt.tight_layout()
plt.savefig(f'{working_dir}{folder}/1-by-2-rbo-post-rbo-statistics-no-detect_{sepset_rule}.pdf', transparent=True, bbox_inches='tight', pad_inches=0.02)
plt.close('all')
def draw_phase_I(folder, master_df: pd.DataFrame):
""" Macro average """
if True:
draw_phase_1_macro_average(
folder,
fixed(master_df, {
'order-dependence': False,
'aggregation': True
}), sns.color_palette("Paired", 6))
if True:
draw_order_independence(folder, master_df)
if True:
how_aggregation(
folder,
fixed(master_df, {
'order-dependence': False,
'aggregation': True
}))
draw_aggregation(folder, master_df)
def perform_check_best_settings(label: str, folder, master_df: pd.DataFrame, verbose=False):
# not to choose base size
fixables = sorted(['base size', 'aggregation', 'sepset rule', 'orientation rule', 'detect rbo', 'detect post rbo'])
# known_order = {'orientation rule': ['majority', 'conservative'], 'sepset rule': ['first', 'minimal', 'full']}
known_order = {'orientation rule': ['majority'], 'sepset rule': ['minimal']}
for col in fixables:
_print_(col, verbose=verbose)
for val in known_order[col] if col in known_order else sorted(master_df[col].unique()):
working_df = fixed(master_df, {col: val})
boot_precision = np.zeros((2000,))
boot_recall = np.zeros((2000,))
boot_f = np.zeros((2000,))
for boot in range(2000):
sampling = np.random.randint(len(working_df), size=len(working_df))
subsampled = working_df.iloc[sampling]
precision = subsampled['correct directed'].sum() / (subsampled['directed'].sum() + 1e-20)
recall = subsampled['correct directed'].sum() / subsampled['directables'].sum()
f_measure = 2 * precision * recall / (precision + recall)
boot_precision[boot] = precision
boot_recall[boot] = recall
boot_f[boot] = f_measure
precision = working_df['correct directed'].sum() / working_df['directed'].sum()
recall = working_df['correct directed'].sum() / working_df['directables'].sum()
f_measure = 2 * precision * recall / (precision + recall)
low_p, high_p = np.percentile(boot_precision, [5, 95])
low_r, high_r = np.percentile(boot_recall, [5, 95])
low_f, high_f = np.percentile(boot_f, [5, 95])
_print_(f' {str(val).rjust(15)} {precision:.4f} ({low_p:.4f} ~ {high_p:.4f}) {recall:.4f} ({low_r:.4f} ~ {high_r:.4f}) {f_measure:.4f} ({low_f:.4f} ~ {high_f:.4f})',
verbose=verbose)
macro = list()
for setting in product(*[list(master_df[var].unique()) for var in fixables]):
subsampled = fixed(master_df, dict(zip(fixables, setting)))
if len(subsampled) == 0:
continue
precision = subsampled['correct directed'].sum() / subsampled['directed'].sum()
recall = subsampled['correct directed'].sum() / subsampled['directables'].sum()
f_measure = 2 * precision * recall / (precision + recall)
macro.append([*setting, precision, recall, f_measure])
newdf = pd.DataFrame(macro, columns=[*fixables, 'precision', 'recall', 'F-measure'])
print(newdf)
_print_('precision', verbose=verbose)
_print_(newdf.sort_values('precision', ascending=False).iloc[:10], verbose=verbose)
_print_('recall', verbose=verbose)
_print_(newdf.sort_values('recall', ascending=False).iloc[:10], verbose=verbose)
_print_('F-measure', verbose=verbose)
_print_(newdf.sort_values('F-measure', ascending=False).iloc[:10], verbose=verbose)
performances = np.zeros((len(newdf), 4))
performances[:, 0] = newdf['precision']
performances[:, 1] = newdf['recall']
performances[:, 2] = newdf['F-measure']
performances[:, 3] = newdf['base size']
# dumb
to_retain = list()
for i in range(len(performances)):
for j in range(len(performances)):
if i == j:
continue
if performances[i, 0] < performances[j, 0] and performances[i, 1] < performances[j, 1] and performances[i, 3] == performances[j, 3]:
break
else:
to_retain.append(i)
best_performances = performances[to_retain, :]
colors = {100 * (i + 2): c for i, c in enumerate(sns.color_palette("YlGn", 9)[3:])}
scatter_colors = [colors[size] for size in performances[:, 3]]
paper_rc = {'lines.linewidth': 1, 'lines.markersize': 2}
sns.set(style='white', font_scale=1.2, context='paper', rc=paper_rc)
plt.figure(figsize=(4, 4))
plt.scatter(performances[:, 0], performances[:, 1],
c=scatter_colors,
alpha=0.6,
lw=0,
s=25)
best_df = pd.DataFrame(best_performances, columns=['precision', 'recall', 'F-measure', 'base size'])
for key, gdf in best_df.groupby('base size'):
sortedf = gdf.sort_values(['precision', 'recall'])
plt.gca().plot(sortedf['precision'], sortedf['recall'], color=colors[key], label=int(key))
if label == 'random':
plt.gca().legend()
if label == 'random':
plt.gca().set_xlim(0.75, 1.0)
plt.gca().set_ylim(0.1, 0.85)
else:
plt.gca().set_xlim(0.8, 1.0)
plt.gca().set_ylim(0.3, 0.95)
plt.gca().set_xlabel('precision')
plt.gca().set_ylabel('recall')
sns.despine()
plt.tight_layout()
plt.savefig(f'{working_dir}{folder}/settings_phase_II.pdf', transparent=True, bbox_inches='tight', pad_inches=0.02)
plt.close('all')
def _draw_general(master_df: pd.DataFrame, *,
to_fix=None,
x=None,
y=None,
row=None,
col=None,
hue=None,
row_order=None,
col_order=None,
hue_order=None,
to_melt=None,
melt_id_vars=None,
figsize=(6, 4),
font_scale=1.2,
palette=None,
filename=None,
y_lim=None,
factor_plot_hook=None,
legend_locs=(-1, -1, 4), **kwargs):
time.sleep(0.1)
print(f' drawing: {filename}', file=sys.stderr, flush=True)
fixed_found = {col: master_df[col].unique()[0] for col in master_df.columns if len(master_df[col].unique()) == 1}
if fixed_found:
print(' Found Fixed::', fixed_found, file=sys.stderr, flush=True)
time.sleep(0.1)
# prepare data
working_df = master_df
if to_fix is not None:
working_df = fixed(working_df, to_fix)
if to_melt:
working_df = pd.melt(working_df, id_vars=melt_id_vars,
value_vars=hue_order, var_name=hue, value_name=y)
# prepare
paper_rc = {'lines.linewidth': 1, 'lines.markersize': 2}
sns.set(style='white', font_scale=font_scale, context='paper', rc=paper_rc)
plt.figure(figsize=figsize)
if palette is not None:
sns.set_palette(palette)
# prepare for options
factor_plot_dict = {'x': x, 'y': y, 'data': working_df, 'kind': 'point'}
if hue is not None:
factor_plot_dict.update(hue=hue)
if hue_order is not None:
factor_plot_dict.update(hue_order=hue_order)
if row is not None:
factor_plot_dict.update(row=row)
if row_order is not None:
factor_plot_dict.update(row_order=row_order)
if col is not None:
factor_plot_dict.update(col=col)
if col_order is not None:
factor_plot_dict.update(col_order=col_order)
if kwargs is not None:
factor_plot_dict.update(**kwargs)
g = sns.catplot(**factor_plot_dict)
if y_lim is not None:
for ax in g.axes.flat:
ax.set_ylim(*y_lim)
if legend_locs:
if legend_locs == 'out':
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
else:
g.axes[legend_locs[0]][legend_locs[1]].legend(loc=legend_locs[2])
if factor_plot_hook is not None:
if isinstance(factor_plot_hook, list):
for hook in factor_plot_hook:
hook(g)
# saving and closing
sns.despine()
plt.tight_layout()
plt.savefig(filename, transparent=True, bbox_inches='tight', pad_inches=0.02)
plt.close('all')
def perform_check_best_settings(label: str,
folder,
master_df: pd.DataFrame,
verbose=True):
# not to choose base size
fixables = sorted(['base size'])
for col in fixables:
_print_(col, verbose=verbose)
for val in sorted(master_df[col].unique()):
working_df = fixed(master_df, {col: val})
boot_precision = np.zeros((2000, ))
boot_recall = np.zeros((2000, ))
boot_f = np.zeros((2000, ))
for boot in range(2000):
sampling = np.random.randint(len(working_df),
size=len(working_df))
subsampled = working_df.iloc[sampling]
precision = subsampled['correct directed'].sum() / (
subsampled['directed'].sum() + 1e-20)
recall = subsampled['correct directed'].sum(
) / subsampled['directables'].sum()
f_measure = 2 * precision * recall / (precision + recall)
boot_precision[boot] = precision
boot_recall[boot] = recall
boot_f[boot] = f_measure
precision = working_df['correct directed'].sum(
) / working_df['directed'].sum()
recall = working_df['correct directed'].sum(
) / working_df['directables'].sum()
f_measure = 2 * precision * recall / (precision + recall)
low_p, high_p = np.percentile(boot_precision, [5, 95])
low_r, high_r = np.percentile(boot_recall, [5, 95])
low_f, high_f = np.percentile(boot_f, [5, 95])
_print_(
f' {str(val).rjust(15)} {precision:.4f} ({low_p:.4f} ~ {high_p:.4f}) {recall:.4f} ({low_r:.4f} ~ {high_r:.4f}) {f_measure:.4f} ({low_f:.4f} ~ {high_f:.4f})',
verbose=verbose)
macro = list()
for setting in product(
*[list(master_df[var].unique()) for var in fixables]):
subsampled = fixed(master_df, dict(zip(fixables, setting)))
if len(subsampled) == 0:
continue
precision = subsampled['correct directed'].sum(
) / subsampled['directed'].sum()
recall = subsampled['correct directed'].sum(
) / subsampled['directables'].sum()
f_measure = 2 * precision * recall / (precision + recall)
macro.append([*setting, precision, recall, f_measure])
newdf = pd.DataFrame(
macro, columns=[*fixables, 'precision', 'recall', 'F-measure'])
_print_('precision', verbose=verbose)
_print_(newdf.sort_values('precision', ascending=False).iloc[:10],
verbose=verbose)
_print_('recall', verbose=verbose)
_print_(newdf.sort_values('recall', ascending=False).iloc[:10],
verbose=verbose)
_print_('F-measure', verbose=verbose)
_print_(newdf.sort_values('F-measure', ascending=False).iloc[:10],
verbose=verbose)