from tables.common import parse_file, EVAL_RESULTS_TEMPLATE
import matplotlib.pyplot as plt
import numpy as np
for sparsity in [0, .1, .2, .3, .4, .5, .6, .7, .8, .9]:
fig, ax = plt.subplots()
for task in ['MNLI', 'QQP', 'QNLI', 'SST-2', 'CoLA']:
evals = []
mask_diffs = []
for lr in ['2e-5', '3e-5', '4e-5', '5e-5']:
downstream_results = parse_file(
f'models/{task}/downstream_prune_{int(100*sparsity)}_lr_{lr}/eval_results.txt',
EVAL_RESULTS_TEMPLATE)
if downstream_results:
evals.append(downstream_results['eval_accuracy'])
mask_diffs.append(
parse_file(
f'models/{task}/downstream_prune_{int(100*sparsity)}_lr_{lr}/mask_diff.txt',
'{diff:f}')['diff'])
else:
print(f'Missing {task} {sparsity} {lr}')
mask_diffs = np.array(mask_diffs)
evals = np.array(evals)
ax.plot(np.sort(mask_diffs),
evals[np.argsort(mask_diffs)],
label=f'{task}')
ax.set_title('Mask Diff vs. Eval Acc')
ax.set_xlabel('Mask Diff')
ax.set_ylabel('Dev Acc')
ax.legend()
masked_lm_accuracy = {masked_lm_accuracy:f}
masked_lm_loss = {masked_lm_loss:f}
next_sentence_accuracy = {next_sentence_accuracy:f}
next_sentence_loss = {next_sentence_loss:f}
"""
table_rows = []
for sparsity in [0, .1, .2, .3, .4, .5, .6, .7, .8, .9]:
eval_entries, train_losses = grid_search_eval(
lambda task, lr:
f'models/{task}/burned_in_random_prune_{int(sparsity*100)}_lr_{lr}/eval_results.txt'
)
pretrain_results = parse_file(
f'models/pretrain/burned_in_random_prune_{int(sparsity*100)}/eval_results.txt',
PRETRAIN_EVAL_RESULTS_TEMPLATE)
pretrain_loss = pretrain_results['loss'] if pretrain_results else '?'
avg_eval = sum(eval_entries) / len(eval_entries)
avg_loss = sum(train_losses) / len(train_losses)
table_rows.append(f"{sparsity*100:.0f} & {pretrain_loss:.2f} & " +
" & ".join([
f"{acc*100:.1f}$|$\\small{{{loss:.2f}}}"
for acc, loss in zip(eval_entries, train_losses)
]) +
f" & {avg_eval*100:.1f}$|$\\small{{{avg_loss*100:.1f}}}")
rows = "\\\\\n".join(table_rows)
print(f"""
\\begin{{tabular}}{{ccccccccccc}}
# Note: parse doesn't like scientific notation strings, so we manually convert to floats
DIFF_TEMPLATE = """Average: {avg}
Std: {std}
"""
fig, ax = plt.subplots()
for task in ['MNLI', 'QQP', 'QNLI', 'SST-2', 'CoLA']:
print(task)
lr = '5e-5'
epochs = list(range(13))
avgs = []
stds = []
for epoch in epochs:
diff_results = parse_file(
f'graphs/weight_change/trained_downstream/{task}_{epoch}.txt',
DIFF_TEMPLATE)
if not diff_results:
print(f'Missing epoch {epoch}', end=' ')
avgs.append(float('inf'))
stds.append(0)
else:
avgs.append(float(diff_results['avg']) * 100)
stds.append(float(diff_results['std']) * 100)
ax.plot(epochs, avgs, label=task)
ax.set_title('Weight Sort Order Movement During Downstream Training')
ax.set_xlabel('Downstream Epochs Trained')
ax.set_ylabel('Avg % Movement of Weights in Sort Order')
ax.set_ylim(0)
from tables.common import parse_file
plt.rcParams.update({'font.size': 12})
fig, ax = plt.subplots()
for task in ['MNLI', 'QQP', 'QNLI', 'SST-2', 'CoLA']: # 'STS-B'
print()
print(task)
x = []
y = []
for sparsity in [0, .1, .2, .3, .4, .5, .6, .7, .8, .9]:
print()
print(f'SPARSITY {sparsity}')
for lr in ['2e-5','3e-5','4e-5','5e-5']:
mask_diff_res = parse_file(
f'models/{task}/downstream_prune_{int(sparsity*100)}_lr_{lr}/mask_diff.txt',
'{diff:f}')
if not mask_diff_res:
try:
y.append(
diff_masks(
f'models/{task}/downstream_prune_{int(sparsity*100)}_lr_{lr}',
f'models/pretrain/gradual_prune_{int(sparsity*100)}'
) * 100)
x.append(sparsity)
with open(f'models/{task}/downstream_prune_{int(sparsity*100)}_lr_{lr}/mask_diff.txt', 'w+') as out_f:
print(y[-1], file=out_f)
except FileNotFoundError:
print(f'Missing {lr}', end=' ')
else:
import matplotlib.pyplot as plt
from tables.common import parse_file
FEATURE_COMPARE_TEMPLATE = "{total} / {count} = {cosine}\n"
sparsities = [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
differences = []
for sparsity in sparsities:
cosine = parse_file(f'models/pretrain/gradual_prune_{int(sparsity*100)}/features_compare_0.txt',
FEATURE_COMPARE_TEMPLATE)['cosine']
differences.append(float(cosine))
fig, ax = plt.subplots()
ax.set_title('Average Feature Cosine Sim with Prune 0')
ax.set_xlabel('Prune Percentage')
ax.set_ylabel('Cosine Sim')
ax.set_ylim(0)
ax.plot(sparsities, differences)
fig.savefig('cosine.png')
from tables.common import parse_file, grid_search_eval
PRETRAIN_EVAL_RESULTS_TEMPLATE = """global_step = {global_step:d}
loss = {loss:f}
masked_lm_accuracy = {masked_lm_accuracy:f}
masked_lm_loss = {masked_lm_loss:f}
next_sentence_accuracy = {next_sentence_accuracy:f}
next_sentence_loss = {next_sentence_loss:f}
"""
table_rows = []
for sparsity in [0, .1, .2, .3, .4, .5, .6, .7, .8, .9]:
eval_entries, train_losses = grid_search_eval(lambda task, lr: f'models/{task}/gradual_prune_{int(sparsity*100)}_less_embed_lr_{lr}/eval_results.txt')
pretrain_results = parse_file(f'models/pretrain/gradual_prune_0_head_pruned_{int(sparsity*100)}/eval_results.txt', PRETRAIN_EVAL_RESULTS_TEMPLATE)
pretrain_loss = pretrain_results['loss'] if pretrain_results else float('inf')
avg_eval = sum(eval_entries) / len(eval_entries)
avg_loss = sum(train_losses) / len(train_losses)
table_rows.append(
f"{sparsity} & {pretrain_loss:.2f} & " #+ " & ".join([f"{acc:.2f}|{loss:.2f}" for acc, loss in zip(eval_entries, train_losses)]) + f" & {avg_eval:.2f}|{avg_loss:.2f}"
)
rows = "\\\\\n".join(table_rows)
print(f"""
\\begin{{tabular}}{{ccccccccccc}}
Pruned & Pre-train Loss & MNLI & QQP & QNLI & SST-2 & CoLA & STS-B & MRPC & RTE & AVG\\\\
\hline
{rows}\\\\
\\end{{tabular}}
plt.rcParams.update({'font.size': 14})
fig, eval_ax = plt.subplots()
diff_ax = eval_ax.twinx()
all_diffs = []
all_evals = []
for task in ['MNLI', 'QQP', 'QNLI', 'SST-2', 'CoLA']:
print(task)
lr = '5e-5'
epochs = [0, 1, 3, 6, 9, 12]
mask_diffs = []
evals = []
for epoch in epochs:
model_name = f'models/{task}/trained_downstream_lr_{lr}_epoch_{epoch}_pruned_60_lr_{lr}'
downstream_results = parse_file(f'{model_name}/eval_results.txt',
EVAL_RESULTS_TEMPLATE)
if downstream_results:
evals.append(downstream_results['eval_accuracy'])
mask_diff_res = parse_file(f'{model_name}/mask_diff.txt',
'{diff:f}')
if not mask_diff_res:
mask_diff = diff_masks(
model_name,
f'models/{task}/trained_downstream_lr_{lr}_epoch_0_pruned_60_lr_{lr}'
)
mask_diffs.append(mask_diff * 100)
with open(f'{model_name}/mask_diff.txt', 'w+') as out_f:
print(mask_diff, file=out_f)
else:
mask_diffs.append(mask_diff_res['diff'] * 100)
else: