def create_csv_table(items, round_digits=1):
row_groups = collections.defaultdict(list)
for item in items:
group_key = (item["task"], item["intermediate_task"])
row_groups[group_key].append(item)
header = [
"task",
"int task",
"score",
"stddev",
"num trials",
]
body = []
for (task, int_task), row_items in row_groups.items():
scores = [get_single_score(item["score"]) for item in row_items]
# scores = [s for s in scores if s > 60.0]
scores = np.array(scores)
print(scores)
row = [
task,
int_task,
round(np.mean(scores), round_digits),
round(np.std(scores), round_digits),
len(scores),
]
body.append(row)
body = sorted(body, key=lambda r: r[:2])
rows = [header] + body
return result_utils.csv_to_str(rows)
def create_csv_table(items, round_digits=1):
row_groups = collections.defaultdict(list)
for item in items:
group_key = (item["task"], item["mlm_examples"])
row_groups[group_key].append(item)
header = [
"task",
"mlm ex",
"task f1",
"stddev",
"num trials",
]
body = []
for (task, mlm_examples), row_items in row_groups.items():
scores = np.array(
[get_single_score(item["score"]) for item in row_items])
row = [
task,
mlm_examples,
round(np.mean(scores), round_digits),
round(np.std(scores), round_digits),
len(row_items),
]
body.append(row)
body = sorted(body, key=lambda r: r[:1])
rows = [header] + body
return result_utils.csv_to_str(rows)
def create_csv_table(filepath, round_digits=1):
items = result_utils.load_json(filepath)
row_groups = collections.defaultdict(list)
for item in items:
group_key = hashabledict({
"target_task": item["target_task"],
"donor_task": item["donor_task"],
})
row_groups[group_key].append(item)
header = [
"task",
"donor",
"merged score",
"stddev",
"orig score",
"stddev",
"mean boost",
"stddev",
"max boost",
"min boost",
"num trials",
]
body = []
for hp, row_items in row_groups.items():
og_scores = np.array(
[get_single_score(item["original_score"]) for item in row_items])
merged_scores = np.array(
[get_single_score(item["merged_score"]) for item in row_items])
row = [
hp["target_task"],
hp["donor_task"],
round(np.mean(merged_scores), round_digits),
round(np.std(merged_scores), round_digits),
#
round(np.mean(og_scores), round_digits),
round(np.std(og_scores), round_digits),
#
round(np.mean(merged_scores - og_scores), round_digits),
round(np.std(merged_scores - og_scores), round_digits),
#
round(np.max(merged_scores - og_scores), round_digits),
round(np.min(merged_scores - og_scores), round_digits),
len(row_items),
]
body.append(row)
body = sorted(body, key=lambda r: r[:2])
rows = [header] + body
return result_utils.csv_to_str(rows)
def create_csv_table(filepath, round_digits=1):
items = result_utils.load_json(filepath)
row_groups = collections.defaultdict(list)
for item in items:
group_key = hashabledict(item["hyperparams"])
row_groups[group_key].append(item)
header = [
"MNLI ckpt",
"RTE ckpt",
"merged RTE acc",
"merged stddev",
"orig RTE acc",
"orig stddev",
"MNLI body acc",
"MNLI body stddev",
"num trials",
]
body = []
for hp, row_items in row_groups.items():
og_scores = [
get_single_score(item["original_score"]) for item in row_items
]
merged_scores = [
get_single_score(item["merged_score"]) for item in row_items
]
donor_body_scores = [
get_single_score(item["donor_body_score"]) for item in row_items
]
row = [
hp["donor_ckpt_index"],
hp["target_ckpt_index"],
round(np.mean(merged_scores), round_digits),
round(np.std(merged_scores), round_digits),
#
round(np.mean(og_scores), round_digits),
round(np.std(og_scores), round_digits),
#
round(np.mean(donor_body_scores), round_digits),
round(np.std(donor_body_scores), round_digits),
len(row_items),
]
body.append(row)
body = sorted(body, key=lambda r: r[:2])
rows = [header] + body
return result_utils.csv_to_str(rows)
def create_csv_table(filepath,
round_digits=1,
group_by_ckpt_index=True,
best_per_finetuned_model=False):
items = result_utils.load_json(filepath)
row_groups = collections.defaultdict(list)
for item in items:
group_key = hashabledict(item["hyperparams"])
row_groups[group_key].append(item)
if best_per_finetuned_model:
new_row_groups = collections.defaultdict(list)
for hp, row_items in row_groups.items():
# TODO: get best original score as well
best = max(row_items,
key=lambda r: get_single_score(r["merged_score"]))
new_key = dict(hp)
del new_key["train_run_uuid"]
new_key = hashabledict(new_key)
new_row_groups[new_key].append(best)
row_groups = new_row_groups
header = [
"task",
"task ckpt",
"merged task f1",
"stddev",
"orig task f1",
"stddev",
"mean boost",
"stddev",
"max boost",
"min boost",
"num trials",
]
body = []
for hp, row_items in row_groups.items():
og_scores = np.array(
[get_single_score(item["original_score"]) for item in row_items])
merged_scores = np.array(
[get_single_score(item["merged_score"]) for item in row_items])
row = [
hp["task"],
hp["target_ckpt_index"] if group_by_ckpt_index else "-",
round(np.mean(merged_scores), round_digits),
round(np.std(merged_scores), round_digits),
#
round(np.mean(og_scores), round_digits),
round(np.std(og_scores), round_digits),
#
round(np.mean(merged_scores - og_scores), round_digits),
round(np.std(merged_scores - og_scores), round_digits),
#
round(np.max(merged_scores - og_scores), round_digits),
round(np.min(merged_scores - og_scores), round_digits),
len(row_items),
]
body.append(row)
body = sorted(body, key=lambda r: r[:2])
rows = [header] + body
return result_utils.csv_to_str(rows)
def create_csv_table(filepath, round_digits=1):
items = result_utils.load_json(filepath)
row_groups = collections.defaultdict(list)
for item in items:
group_key = hashabledict(item["hyperparams"])
row_groups[group_key].append(item)
row_groups2 = collections.defaultdict(list)
for hp, row_items in row_groups.items():
best_og = max(
row_items,
key=lambda item: get_single_score(item["original_score"]))
best_merged = max(
row_items, key=lambda item: get_single_score(item["merged_score"]))
hp = dict(hp)
del hp["train_run_uuid"]
group_key = hashabledict(hp)
row_groups2[group_key].append({
"original_score":
best_og["original_score"],
"merged_score":
best_merged["merged_score"],
})
header = [
"task",
"mlm train ex",
"mlm reg str",
"merged task f1",
"stddev",
"orig task f1",
"stddev",
"mean boost",
"stddev",
"max boost",
"min boost",
"num trials",
]
body = []
for hp, row_items in row_groups2.items():
og_scores = np.array(
[get_single_score(item["original_score"]) for item in row_items])
merged_scores = np.array(
[get_single_score(item["merged_score"]) for item in row_items])
row = [
hp["task"],
hp["pretrained_examples"],
hp["pretrained_reg_strength"],
# q,
round(np.mean(merged_scores), round_digits),
round(np.std(merged_scores), round_digits),
#
round(np.mean(og_scores), round_digits),
round(np.std(og_scores), round_digits),
#
round(np.mean(merged_scores - og_scores), round_digits),
round(np.std(merged_scores - og_scores), round_digits),
#
round(np.max(merged_scores - og_scores), round_digits),
round(np.min(merged_scores - og_scores), round_digits),
len(row_items),
]
body.append(row)
body = sorted(body, key=lambda r: r[:3])
rows = [header] + body
return result_utils.csv_to_str(rows)
