def run_hidden_layers_experiment():
runs: List[Run] = []
for hidden_layers in [0, 2, 4, 8]:
experiment_name = f'our_approach_our_data_hidden_layers'
runs.extend((
run(run_name=
f'our_approach_our_data_sc_hidden_layers_{hidden_layers}',
experiment_name=experiment_name,
data_set_path=
f'ucca4bpm/data/transformed/ours_qian_sc_postags_fine.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=5,
optimizer=Adam(lr=1e-5),
early_stopping=False,
num_gcn_hidden_layers=hidden_layers,
none_class_id=None,
num_folds=num_folds),
run(run_name=
f'our_approach_our_data_ssr_hidden_layers_{hidden_layers}',
experiment_name=experiment_name,
data_set_path=
f'ucca4bpm/data/transformed/ours_qian_ssr_postags_fine.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=10,
optimizer=Adam(lr=5e-5),
early_stopping=False,
num_gcn_hidden_layers=hidden_layers,
none_class_id=None,
num_folds=num_folds),
run(run_name=
f'our_approach_our_data_srl_hidden_layers_{hidden_layers}',
experiment_name=experiment_name,
data_set_path=
f'ucca4bpm/data/transformed/ours_qian_srl_postags_fine.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=15,
optimizer=Adam(lr=5e-5),
early_stopping=False,
num_gcn_hidden_layers=hidden_layers,
none_class_id=None,
num_folds=num_folds),
run(run_name=
f'our_approach_our_data_quishpi_hidden_layers_{hidden_layers}',
experiment_name=experiment_name,
data_set_path=
f'ucca4bpm/data/transformed/ours_quishpi_postags_fine.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=15,
optimizer=Adam(lr=5e-5),
early_stopping=False,
num_gcn_hidden_layers=hidden_layers,
none_class_id=0,
num_folds=num_folds),
))
plot_f1_metrics(runs, 'Importance of Hidden Layers',
{'f1_fragment': 'F1 (Fragment)'})
def run_feature_experiment():
runs: List[Run] = []
for feature_name in [
'featureless', 'google300', 'glove25', 'postags_fine'
]:
experiment_name = f'our_approach_our_data_features'
runs.extend((
run(run_name=f'our_approach_our_data_sc_features{feature_name}',
experiment_name=experiment_name,
data_set_path=
f'ucca4bpm/data/transformed/ours_qian_sc_{feature_name}.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=5,
optimizer=Adam(lr=1e-5),
early_stopping=False,
num_gcn_hidden_layers=2,
none_class_id=None,
num_folds=num_folds),
run(run_name=f'our_approach_our_data_ssr_features{feature_name}',
experiment_name=experiment_name,
data_set_path=
f'ucca4bpm/data/transformed/ours_qian_ssr_{feature_name}.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=10,
optimizer=Adam(lr=5e-5),
early_stopping=False,
num_gcn_hidden_layers=2,
none_class_id=None,
num_folds=num_folds),
run(run_name=f'our_approach_our_data_srl_features{feature_name}',
experiment_name=experiment_name,
data_set_path=
f'ucca4bpm/data/transformed/ours_qian_srl_{feature_name}.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=15,
optimizer=Adam(lr=5e-5),
early_stopping=False,
num_gcn_hidden_layers=2,
none_class_id=None,
num_folds=num_folds),
run(run_name=
f'our_approach_our_data_quishpi_features{feature_name}',
experiment_name=experiment_name,
data_set_path=
f'ucca4bpm/data/transformed/ours_quishpi_{feature_name}.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=15,
optimizer=Adam(lr=5e-5),
early_stopping=False,
num_gcn_hidden_layers=2,
none_class_id=0,
num_folds=num_folds),
))
plot_f1_metrics(runs, 'Importance of Feature Choice',
{'f1_fragment': 'F1 (Fragment)'})
def run_lr_experiment():
runs: List[Run] = []
for lr in [5e-3, 5e-4, 5e-5, 5e-6]:
experiment_name = f'our_approach_our_data_lr'
lr_formatted = f'{lr}'.replace('.', '-')
runs.extend((
run(run_name=f'our_approach_our_data_sc_lr_{lr_formatted}',
experiment_name=experiment_name,
data_set_path=
f'ucca4bpm/data/transformed/ours_qian_sc_postags_fine.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=5,
optimizer=Adam(lr=lr),
early_stopping=False,
num_gcn_hidden_layers=2,
none_class_id=None,
num_folds=num_folds),
run(run_name=f'our_approach_our_data_ssr_lr_{lr_formatted}',
experiment_name=experiment_name,
data_set_path=
f'ucca4bpm/data/transformed/ours_qian_ssr_postags_fine.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=10,
optimizer=Adam(lr=lr),
early_stopping=False,
num_gcn_hidden_layers=2,
none_class_id=None,
num_folds=num_folds),
run(run_name=f'our_approach_our_data_srl_lr_{lr_formatted}',
experiment_name=experiment_name,
data_set_path=
f'ucca4bpm/data/transformed/ours_qian_srl_postags_fine.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=15,
optimizer=Adam(lr=lr),
early_stopping=False,
num_gcn_hidden_layers=2,
none_class_id=None,
num_folds=num_folds),
run(run_name=f'our_approach_our_data_quishpi_lr_{lr_formatted}',
experiment_name=experiment_name,
data_set_path=
f'ucca4bpm/data/transformed/ours_quishpi_postags_fine.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=15,
optimizer=Adam(lr=lr),
early_stopping=False,
num_gcn_hidden_layers=2,
none_class_id=0,
num_folds=num_folds),
))
plot_f1_metrics(runs, 'Importance of Learning Rate',
{'f1_fragment': 'F1 (Fragment)'})
plot_f1_metrics([
run(run_name='SC_COR',
experiment_name=experiment_name,
data_set_path='ucca4bpm/data/transformed/qian_sc_cor_postags_fine.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=1,
optimizer=Adam(lr=5e-4),
early_stopping=False,
num_gcn_hidden_layers=2,
none_class_id=None,
num_folds=num_folds),
run(run_name='SC_MAM',
experiment_name=experiment_name,
data_set_path='ucca4bpm/data/transformed/qian_sc_mam_postags_fine.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=1,
optimizer=Adam(lr=5e-4),
early_stopping=False,
num_gcn_hidden_layers=2,
none_class_id=None,
num_folds=num_folds),
run(run_name='SSR_COR',
experiment_name=experiment_name,
data_set_path='ucca4bpm/data/transformed/qian_ssr_cor_postags_fine.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=2,
optimizer=Adam(lr=5e-5),
early_stopping=False,
none_class_id=None,
num_gcn_hidden_layers=4,
num_folds=num_folds),
run(run_name='SSR_MAM',
experiment_name=experiment_name,
data_set_path='ucca4bpm/data/transformed/qian_ssr_mam_postags_fine.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=2,
optimizer=Adam(lr=5e-5),
early_stopping=False,
num_gcn_hidden_layers=4,
none_class_id=None,
num_folds=num_folds),
run(run_name='SRL_COR',
experiment_name=experiment_name,
data_set_path='ucca4bpm/data/transformed/qian_srl_cor_postags_fine.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=2,
optimizer=Adam(lr=5e-5),
early_stopping=False,
num_gcn_hidden_layers=4,
none_class_id=None,
num_folds=num_folds),
run(run_name='SRL_MAM',
experiment_name=experiment_name,
data_set_path='ucca4bpm/data/transformed/qian_srl_mam_postags_fine.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=2,
optimizer=Adam(lr=5e-5),
early_stopping=False,
num_gcn_hidden_layers=4,
none_class_id=None,
num_folds=num_folds),
run(run_name='ours_sc',
experiment_name=experiment_name,
data_set_path='ucca4bpm/data/transformed/ours_qian_sc_postags_fine.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=10,
optimizer=Adam(lr=5e-4),
early_stopping=False,
num_gcn_hidden_layers=2,
none_class_id=None,
num_folds=num_folds),
run(run_name='ours_ssr',
experiment_name=experiment_name,
data_set_path='ucca4bpm/data/transformed/ours_qian_ssr_postags_fine.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=15,
optimizer=Adam(lr=5e-4),
early_stopping=False,
num_gcn_hidden_layers=2,
none_class_id=None,
num_folds=num_folds),
run(run_name='ours_srl',
experiment_name=experiment_name,
data_set_path='ucca4bpm/data/transformed/ours_qian_srl_postags_fine.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=20,
optimizer=Adam(lr=5e-4),
early_stopping=False,
num_gcn_hidden_layers=2,
none_class_id=None,
num_folds=num_folds)
], 'Our approach on data by Qian et al', {f'f1_{mode}': label for mode, label in span_matching_modes.items()})
experiment_name = f'our_approach_quishpi_data'
plot_f1_metrics(
[
# run(run_name=f'quishpi_their_data',
# experiment_name=experiment_name,
# data_set_path='ucca4bpm/data/transformed/quishpi_postags_fine.pickle',
# span_matching_modes=list(span_matching_modes.keys()),
# gcn_hidden_len=64,
# num_epochs=15,
# optimizer=Adam(lr=5e-5),
# early_stopping=False,
# num_gcn_hidden_layers=2,
# none_class_id=None,
# num_folds=num_folds),
run(run_name=f'quishpi_our_data',
experiment_name=experiment_name,
data_set_path=
'ucca4bpm/data/transformed/ours_quishpi_postags_fine.pickle',
span_matching_modes=list(span_matching_modes.keys()),
gcn_hidden_len=64,
num_epochs=15,
optimizer=Adam(lr=5e-5),
early_stopping=False,
num_gcn_hidden_layers=2,
none_class_id=None,
num_folds=num_folds),
],
'Our Approach in Optimal Configuration on Data by Quishpi et al.',
{f'f1_{mode}': label
for mode, label in span_matching_modes.items()})