"""
Experiment configuration for:
Model: BERT trained on Squad2.0
Benchmark: Tacred
"""
from reflex.qa_runner import QARunner
from reflex.utils import setup_experiment
ex = setup_experiment('BERT Squad2.0 TACRED')
@ex.config
def conf():
qa_path = '/Users/ankur/Projects/RE-Flex/weights/squad2' # Path to trained weights
relations_filepath = '/Users/ankur/Projects/RE-Flex/data/tacred_relations.jsonl' # Path to relations file
data_directory = '/Users/ankur/Projects/RE-Flex/data/tacred' # Path to underlying data
batch_size = 16
must_choose_answer = True
@ex.automain
def main(qa_path, relations_filepath, data_directory, batch_size,
must_choose_answer):
runner = QARunner(qa_path, relations_filepath, data_directory, batch_size,
must_choose_answer)
em, f1, per_relation_metrics = runner.predict()
return {'em': em, 'f1': f1, 'per_relation_metrics': per_relation_metrics}
"""
Experiment configuration for:
Model: BERT trained on ZSRE
Benchmark: ZSRE
"""
from reflex.qa_runner import QARunner
from reflex.utils import setup_experiment
import os
import pickle
ex = setup_experiment('BERT ZSRE ZSRE')
@ex.config
def conf():
qa_path = os.path.join(os.environ['BASE_PATH'],
'weights/bert-zsre') # Path to trained weights
relations_filepath = os.path.join(
os.environ['BASE_PATH'],
'data/zsre_relations.jsonl') # Path to relations file
data_directory = os.path.join(os.environ['BASE_PATH'],
'data/zsre/test') # Path to underlying data
batch_size = 16
must_choose_answer = False
device = 'cuda'
trained_to_reject = True
@ex.automain
def main(qa_path, relations_filepath, data_directory, batch_size,
must_choose_answer, device, trained_to_reject):
"""
Experiment configuration for:
Model: Naive Roberta (no context seed)
Benchmark: Google-RE
"""
from reflex.lm_runner import LMRunner
from reflex.utils import setup_experiment
import os
ex = setup_experiment('Naive GoogleRE')
@ex.config
def conf():
model_dir = os.path.join(
os.environ['BASE_PATH'],
'weights/roberta_large') # Path to trained weights
model_name = os.path.join(os.environ['BASE_PATH'],
'weights/roberta_large/model.pt')
relations_filepath = os.path.join(
os.environ['BASE_PATH'],
'data/googlere_relations.jsonl') # Path to relations file
data_directory = os.path.join(os.environ['BASE_PATH'],
'data/googlere') # Path to underlying data
batch_size = 16
must_choose_answer = True
use_context = False
device = 'cuda'
cap = 0
"""
Experiment configuration for:
Model: BERT Trained on Squad2.0
Benchmark: T-REx
"""
from reflex.qa_runner import QARunner
from reflex.utils import setup_experiment, save_se_list
import os
ex = setup_experiment('BERT Squad2.0 T-REx')
@ex.config
def conf():
qa_path = os.path.join(os.environ['BASE_PATH'], 'weights/squad2') # Path to trained weights
relations_filepath = os.path.join(os.environ['BASE_PATH'], 'data/trex_relations.jsonl') # Path to relations file
data_directory = os.path.join(os.environ['BASE_PATH'], 'data/trex') # Path to underlying data
error_path = os.path.join(os.environ['BASE_PATH'], 'figures', 'bsquad_trex.csv')
batch_size = 16
must_choose_answer = True
device = 'cuda'
trained_to_reject = True
@ex.automain
def main(qa_path, relations_filepath, data_directory, batch_size, must_choose_answer, device, trained_to_reject, error_path):
runner = QARunner(qa_path, relations_filepath, data_directory, batch_size, must_choose_answer, device, trained_to_reject)
em, f1, per_relation_metrics = runner.predict()
save_se_list(runner.se_list, error_path)
return {'em': em, 'f1': f1, 'per_relation_metrics': per_relation_metrics}
"""
Experiment configuration for:
Model: BERT trained on ZSRE
Benchmark: TACRED
"""
from reflex.qa_runner import QARunner
from reflex.utils import setup_experiment
ex = setup_experiment('BERT ZSRE TACREd')
@ex.config
def conf():
qa_path = '/Users/ankur/Projects/RE-Flex/weights/zsre' # Path to trained weights
relations_filepath = '/Users/ankur/Projects/RE-Flex/data/tacred_relations.jsonl' # Path to relations file
data_directory = '/Users/ankur/Projects/RE-Flex/data/tacred' # Path to underlying data
batch_size = 16
must_choose_answer = True
@ex.automain
def main(qa_path, relations_filepath, data_directory, batch_size,
must_choose_answer):
runner = QARunner(qa_path, relations_filepath, data_directory, batch_size,
must_choose_answer)
em, f1, per_relation_metrics = runner.predict()
return {'em': em, 'f1': f1, 'per_relation_metrics': per_relation_metrics}
"""
Experiment configuration for:
Model: Dhingra et al 2018 -- https://arxiv.org/abs/1804.00720
Benchmark: Google-RE
"""
from reflex.qa_runner import QARunner
from reflex.utils import setup_experiment
ex = setup_experiment('Dhingra GoogleRE')
@ex.config
def conf():
qa_path = '/Users/ankur/Projects/RE-Flex/weights/dhingra-latest' # Path to trained weights
relations_filepath = '/Users/ankur/Projects/RE-Flex/data/googlere_relations.jsonl' # Path to relations file
data_directory = '/Users/ankur/Projects/RE-Flex/data/Google_RE2' # Path to underlying data
batch_size = 16
must_choose_answer = True
@ex.automain
def main(qa_path, relations_filepath, data_directory, batch_size, must_choose_answer):
runner = QARunner(qa_path, relations_filepath, data_directory, batch_size, must_choose_answer)
em, f1, per_relation_metrics = runner.predict()
return {'em': em, 'f1': f1, 'per_relation_metrics': per_relation_metrics}
"""
Experiment configuration for:
Model: Naive RoBERTa
Benchmark: T-REx
"""
from reflex.lm_runner import LMRunner
from reflex.utils import setup_experiment
import os
ex = setup_experiment('Naive T-REx')
@ex.config
def conf():
model_dir = os.path.join(
os.environ['BASE_PATH'],
'weights/roberta_large') # Path to trained weights
model_name = os.path.join(os.environ['BASE_PATH'],
'weights/roberta_large/model.pt')
relations_filepath = os.path.join(
os.environ['BASE_PATH'],
'data/trex_relations.jsonl') # Path to relations file
data_directory = os.path.join(os.environ['BASE_PATH'],
'data/trex') # Path to underlying data
batch_size = 16
must_choose_answer = True
use_context = False
device = 'cuda'
cap = 0
"""
Experiment configuration for:
Model: RE-Flex
Benchmark: Google-RE
"""
import fasttext
import spacy
from reflex.reflex_runner import ReflexRunner
from reflex.utils import setup_experiment
import os
ex = setup_experiment('RE-Flex Google-RE')
@ex.config
def conf():
model_dir = os.path.join(
os.environ['BASE_PATH'],
'weights/roberta_large') # Path to trained weights
model_name = os.path.join(os.environ['BASE_PATH'],
'weights/roberta_large/model.pt')
relations_filepath = os.path.join(
os.environ['BASE_PATH'],
'data/googlere_relations.jsonl') # Path to relations file
data_directory = os.path.join(os.environ['BASE_PATH'],
'data/googlere') # Path to underlying data
batch_size = 16
must_choose_answer = True
device = 'cuda'
k = 16
word_embeddings_path = os.path.join(os.environ['BASE_PATH'],
"""
Experiment configuration for:
Model: BiDAF trained on Squad2.0
Benchmark: Tacred
"""
from reflex.bidaf_runner import BidafRunner
from reflex.utils import setup_experiment, save_se_list
import os
ex = setup_experiment('BiDAF GoogleRE')
@ex.config
def conf():
relations_filepath = os.path.join(
os.environ['BASE_PATH'],
'data/googlere_relations.jsonl') # Path to relations file
data_directory = os.path.join(os.environ['BASE_PATH'],
'data/googlere') # Path to underlying data
error_path = os.path.join(os.environ['BASE_PATH'], 'figures',
'bidaf_googlere.csv')
must_choose_answer = True
calculate_single_error = True
@ex.automain
def main(relations_filepath, data_directory, must_choose_answer, error_path,
calculate_single_error):
runner = BidafRunner(relations_filepath, data_directory,
must_choose_answer, calculate_single_error)
em, f1, per_relation_metrics = runner.predict()
"""
Experiment configuration for:
Model: BERT trained on ZSRE
Benchmark: Google-RE
"""
from reflex.qa_runner import QARunner
from reflex.utils import setup_experiment, save_se_list
import os
ex = setup_experiment('BERT ZSRE GoogleRE')
@ex.config
def conf():
qa_path = os.path.join(os.environ['BASE_PATH'], 'weights/bert-zsre') # Path to trained weights
relations_filepath = os.path.join(os.environ['BASE_PATH'], 'data/googlere_relations.jsonl') # Path to relations file
data_directory = os.path.join(os.environ['BASE_PATH'], 'data/googlere') # Path to underlying data
error_path = os.path.join(os.environ['BASE_PATH'], 'figures', 'bzsre_googlere.csv')
batch_size = 16
must_choose_answer = True
device = 'cuda'
trained_to_reject = True
@ex.automain
def main(qa_path, relations_filepath, data_directory, batch_size, must_choose_answer, device, trained_to_reject, error_path):
runner = QARunner(qa_path, relations_filepath, data_directory, batch_size, must_choose_answer, device, trained_to_reject)
em, f1, per_relation_metrics = runner.predict()
save_se_list(runner.se_list, error_path)
return {'em': em, 'f1': f1, 'per_relation_metrics': per_relation_metrics}
"""
Experiment configuration for:
Model: Radford Roberta (Naive w/ context seed)
Benchmark: T-REx
"""
from reflex.lm_runner import LMRunner
from reflex.utils import setup_experiment
ex = setup_experiment('Radford T-REx')
@ex.config
def conf():
model_dir = '/Users/ankur/Projects/RE-Flex/weights/roberta_large' # Path to trained weights
model_name = '/Users/ankur/Projects/RE-Flex/weights/roberta_large/model.pt'
relations_filepath = '/Users/ankur/Projects/RE-Flex/data/trex_relations.jsonl' # Path to relations file
data_directory = '/Users/ankur/Projects/RE-Flex/data/trex' # Path to underlying data
batch_size = 16
must_choose_answer = True
use_context = True
device = 'cpu'
@ex.automain
def main(model_dir, model_name, device, relations_filepath, data_directory,
batch_size, must_choose_answer, use_context):
runner = LMRunner(model_dir, model_name, device, relations_filepath,
data_directory, batch_size, must_choose_answer)
em, f1, per_relation_metrics = runner.predict_naive(use_context=False)
return {'em': em, 'f1': f1, 'per_relation_metrics': per_relation_metrics}
"""
Experiment configuration for:
Model: BERT trained squad
Benchmark: ZSRE
"""
from reflex.qa_runner import QARunner
from reflex.utils import setup_experiment
import os
import pickle
ex = setup_experiment('BERT Squad2.0 ZSRE')
@ex.config
def conf():
qa_path = os.path.join(os.environ['BASE_PATH'], 'weights/squad2') # Path to trained weights
relations_filepath = os.path.join(os.environ['BASE_PATH'], 'data/zsre_relations.jsonl') # Path to relations file
data_directory = os.path.join(os.environ['BASE_PATH'], 'data/zsre/test') # Path to underlying data
batch_size = 16
must_choose_answer = False
device = 'cuda'
trained_to_reject = True
@ex.automain
def main(qa_path, relations_filepath, data_directory, batch_size, must_choose_answer, device, trained_to_reject):
runner = QARunner(qa_path, relations_filepath, data_directory, batch_size, must_choose_answer, device, trained_to_reject, calculate_single_error=False)
em, f1, per_relation_metrics = runner.predict()
with open('BERTSQUADZSRE.pkl', 'wb') as wf:
pickle.dump(per_relation_metrics, wf)
return {'em': em, 'f1': f1, 'per_relation_metrics': per_relation_metrics}
"""
Experiment configuration for:
Model: BERT trained on ZSRE
Benchmark: T-REx
"""
from reflex.qa_runner import QARunner
from reflex.utils import setup_experiment
ex = setup_experiment('BERT ZSRE T-REx')
@ex.config
def conf():
qa_path = '/Users/ankur/Projects/RE-Flex/weights/lewis-latest' # Path to trained weights
relations_filepath = '/Users/ankur/Projects/RE-Flex/data/trex_relations.jsonl' # Path to relations file
data_directory = '/Users/ankur/Projects/RE-Flex/data/trex' # Path to underlying data
batch_size = 16
must_choose_answer = True
@ex.automain
def main(qa_path, relations_filepath, data_directory, batch_size,
must_choose_answer):
runner = QARunner(qa_path, relations_filepath, data_directory, batch_size,
must_choose_answer)
em, f1, per_relation_metrics = runner.predict()
return {'em': em, 'f1': f1, 'per_relation_metrics': per_relation_metrics}
"""
Experiment configuration for:
Model: Dhingra et al 2018 -- https://arxiv.org/abs/1804.00720
Benchmark: Tacred
"""
from reflex.qa_runner import QARunner
from reflex.utils import setup_experiment
import os
ex = setup_experiment('Dhingra Tacred')
@ex.config
def conf():
qa_path = os.path.join(os.environ['BASE_PATH'],
'weights/dhingra-latest') # Path to trained weights
relations_filepath = os.path.join(
os.environ['BASE_PATH'],
'data/tacred_relations.jsonl') # Path to relations file
data_directory = os.path.join(
os.environ['BASE_PATH'], 'data/tacred/test') # Path to underlying data
batch_size = 16
must_choose_answer = True
device = 'cuda'
trained_to_reject = False
@ex.automain
def main(qa_path, relations_filepath, data_directory, batch_size,
must_choose_answer, device, trained_to_reject):
runner = QARunner(qa_path,
"""
Experiment configuration for:
Model: Lewis et al 2019 -- https://arxiv.org/abs/1906.04980
Benchmark: T-REx
"""
from reflex.qa_runner import QARunner
from reflex.utils import setup_experiment
ex = setup_experiment('Lewis T-REx')
@ex.config
def conf():
qa_path = '/Users/ankur/Projects/RE-Flex/weights/lewis-latest' # Path to trained weights
relations_filepath = '/Users/ankur/Projects/RE-Flex/data/trex_relations.jsonl' # Path to relations file
data_directory = '/Users/ankur/Projects/RE-Flex/data/trex' # Path to underlying data
batch_size = 16
must_choose_answer = True
@ex.automain
def main(qa_path, relations_filepath, data_directory, batch_size, must_choose_answer):
runner = QARunner(qa_path, relations_filepath, data_directory, batch_size, must_choose_answer)
em, f1, per_relation_metrics = runner.predict()
return {'em': em, 'f1': f1, 'per_relation_metrics': per_relation_metrics}
"""
Experiment configuration for:
Model: RE-Flex
Benchmark: Test
"""
import fasttext
import spacy
from reflex.reflex_runner import ReflexRunner
from reflex.utils import setup_experiment
from reflex.metrics import calculate_final_em_f1
import os
ex = setup_experiment('RE-Flex Test')
@ex.config
def conf():
model_dir = os.path.join(
os.environ['BASE_PATH'],
'weights/roberta_large') # Path to trained weights
model_name = os.path.join(os.environ['BASE_PATH'],
'weights/roberta_large/model.pt')
relations_filepath = os.path.join(
os.environ['BASE_PATH'],
'data/test_relations.jsonl') # Path to relations file
data_directory = os.path.join(os.environ['BASE_PATH'],
'data/Test') # Path to underlying data
batch_size = 16
must_choose_answer = False
device = 'cpu'
ls = [-3, -2, -1, -0.5, 0, 0.5, 1, 2, 3]
"""
Experiment configuration for:
Model: Naive Roberta (no context seed)
Benchmark: Tacred
"""
from reflex.lm_runner import LMRunner
from reflex.utils import setup_experiment
import os
ex = setup_experiment('Naive TACRED')
@ex.config
def conf():
model_dir = os.path.join(
os.environ['BASE_PATH'],
'weights/roberta_large') # Path to trained weights
model_name = os.path.join(os.environ['BASE_PATH'],
'weights/roberta_large/model.pt')
relations_filepath = os.path.join(
os.environ['BASE_PATH'],
'data/tacred_relations.jsonl') # Path to relations file
data_directory = os.path.join(
os.environ['BASE_PATH'], 'data/tacred/test') # Path to underlying data
batch_size = 16
must_choose_answer = True
use_context = False
device = 'cuda'
cap = 0
"""
Experiment configuration for:
Model: RE-Flex
Benchmark: T-REx
"""
import fasttext
import spacy
from reflex.reflex_runner import ReflexRunner
from reflex.utils import setup_experiment
from reflex.metrics import calculate_final_em_f1
import os
ex = setup_experiment('RE-Flex T-REx')
@ex.config
def conf():
model_dir = os.path.join(os.environ['BASE_PATH'], 'weights/roberta_large') # Path to trained weights
model_name = os.path.join(os.environ['BASE_PATH'], 'weights/roberta_large/model.pt')
relations_filepath = os.path.join(os.environ['BASE_PATH'], 'data/trex_relations.jsonl') # Path to relations file
data_directory = os.path.join(os.environ['BASE_PATH'], 'data/trex') # Path to underlying data
batch_size = 16
must_choose_answer = True
device = 'cpu'
ls = [-3, -2, -1, -0.5, 0, 0.5, 1, 2, 3]
k = 16
word_embeddings_path = os.path.join(os.environ['BASE_PATH'], 'weights/crawl-300d-2M-subword.bin')
@ex.automain
def main(model_dir, model_name, device, relations_filepath, data_directory, batch_size, must_choose_answer, word_embeddings_path, ls, k):
spacy_model = spacy.load('en_core_web_lg')
we_model = fasttext.load_model(word_embeddings_path)
"""
Experiment configuration for:
Model: Dhingra et al 2018 -- https://arxiv.org/abs/1804.00720
Benchmark: ZSRE
"""
from reflex.qa_runner import QARunner
from reflex.utils import setup_experiment
import os
ex = setup_experiment('Dhingra ZSRE')
@ex.config
def conf():
qa_path = os.path.join(os.environ['BASE_PATH'],
'weights/dhingra-latest') # Path to trained weights
relations_filepath = os.path.join(
os.environ['BASE_PATH'],
'data/zsre_relations.jsonl') # Path to relations file
data_directory = os.path.join(os.environ['BASE_PATH'],
'data/zsre/test') # Path to underlying data
batch_size = 16
must_choose_answer = True
device = 'cuda'
trained_to_reject = False
@ex.automain
def main(qa_path, relations_filepath, data_directory, batch_size,
must_choose_answer, device, trained_to_reject):
runner = QARunner(qa_path,
"""
Experiment configuration for:
Model: BiDAF trained on Squad2.0
Benchmark: Tacred
"""
from reflex.bidaf_runner import BidafRunner
from reflex.utils import setup_experiment
import os
ex = setup_experiment('BiDAF TACRED')
@ex.config
def conf():
relations_filepath = os.path.join(os.environ['BASE_PATH'], 'data/tacred_relations.jsonl') # Path to relations file
data_directory = os.path.join(os.environ['BASE_PATH'], 'data/tacred/test') # Path to underlying data
must_choose_answer = False
calculate_single_error = False
@ex.automain
def main(relations_filepath, data_directory, must_choose_answer, calculate_single_error):
runner = BidafRunner(relations_filepath, data_directory, must_choose_answer, calculate_single_error)
em, f1, per_relation_metrics = runner.predict()
return {'em': em, 'f1': f1, 'per_relation_metrics': per_relation_metrics}
"""
Experiment configuration for:
Model: BERT trained squad
Benchmark: Google-RE
"""
from reflex.qa_runner import QARunner
from reflex.utils import setup_experiment, save_se_list
import os
ex = setup_experiment('BERT Squad2.0 GoogleRE')
@ex.config
def conf():
qa_path = os.path.join(os.environ['BASE_PATH'],
'weights/squad2') # Path to trained weights
relations_filepath = os.path.join(
os.environ['BASE_PATH'],
'data/googlere_relations.jsonl') # Path to relations file
data_directory = os.path.join(os.environ['BASE_PATH'],
'data/googlere') # Path to underlying data
error_path = os.path.join(os.environ['BASE_PATH'], 'figures',
'bsquad_googlere.csv')
batch_size = 16
must_choose_answer = True
device = 'cuda'
trained_to_reject = True
@ex.automain
def main(qa_path, relations_filepath, data_directory, batch_size,
"""
Experiment configuration for:
Model: Lewis et al 2019 -- https://arxiv.org/abs/1906.04980
Benchmark: Google-RE
"""
from reflex.qa_runner import QARunner
from reflex.utils import setup_experiment
import os
ex = setup_experiment('Lewis Google-RE')
@ex.config
def conf():
qa_path = os.path.join(os.environ['BASE_PATH'],
'weights/lewis-latest') # Path to trained weights
relations_filepath = os.path.join(
os.environ['BASE_PATH'],
'data/googlere_relations.jsonl') # Path to relations file
data_directory = os.path.join(os.environ['BASE_PATH'],
'data/Google_RE') # Path to underlying data
batch_size = 16
must_choose_answer = True
device = 'cpu'
trained_to_reject = False
@ex.automain
def main(qa_path, relations_filepath, data_directory, batch_size,
must_choose_answer, device, trained_to_reject):
runner = QARunner(qa_path, relations_filepath, data_directory, batch_size,
"""
Experiment configuration for:
Model: Radford Roberta (Naive w/ context seed)
Benchmark: ZSRE
"""
from reflex.lm_runner import LMRunner
from reflex.utils import setup_experiment
import os
ex = setup_experiment('Radford ZSRE')
@ex.config
def conf():
model_dir = os.path.join(os.environ['BASE_PATH'], 'weights/roberta_large') # Path to trained weights
model_name = os.path.join(os.environ['BASE_PATH'], 'weights/roberta_large/model.pt')
relations_filepath = os.path.join(os.environ['BASE_PATH'], 'data/zsre_relations.jsonl') # Path to relations file
data_directory = os.path.join(os.environ['BASE_PATH'], 'data/zsre/test') # Path to underlying data
batch_size = 16
must_choose_answer = False
use_context = True
device = 'cuda'
cap = 10
@ex.automain
def main(model_dir, model_name, device, relations_filepath, data_directory, batch_size, must_choose_answer, use_context, cap):
runner = LMRunner(model_dir, model_name, device, relations_filepath, data_directory, batch_size, must_choose_answer, cap)
em, f1, per_relation_metrics = runner.predict_naive(use_context=use_context)
return {'em': em, 'f1': f1, 'per_relation_metrics': per_relation_metrics}
"""
Experiment configuration for:
Model: BiDAF trained on Squad2.0
Benchmark: ZSRE
"""
from reflex.bidaf_runner import BidafRunner
from reflex.utils import setup_experiment
import os
ex = setup_experiment('BiDAF ZSRE')
@ex.config
def conf():
relations_filepath = os.path.join(os.environ['BASE_PATH'], 'data/zsre_relations.jsonl') # Path to relations file
data_directory = os.path.join(os.environ['BASE_PATH'], 'data/zsre/test') # Path to underlying data
must_choose_answer = False
@ex.automain
def main(relations_filepath, data_directory, must_choose_answer):
runner = BidafRunner(relations_filepath, data_directory, must_choose_answer)
em, f1, per_relation_metrics = runner.predict()
return {'em': em, 'f1': f1, 'per_relation_metrics': per_relation_metrics}
"""
Experiment configuration for:
Model: Radford Roberta (Naive w/ context seed)
Benchmark: Google-RE
"""
from reflex.lm_runner import LMRunner
from reflex.utils import setup_experiment
ex = setup_experiment('Radford Google-RE')
@ex.config
def conf():
model_dir = '/Users/ankur/Projects/RE-Flex/weights/roberta_large' # Path to trained weights
model_name = '/Users/ankur/Projects/RE-Flex/weights/roberta_large/model.pt'
relations_filepath = '/Users/ankur/Projects/RE-Flex/data/googlere_relations.jsonl' # Path to relations file
data_directory = '/Users/ankur/Projects/RE-Flex/data/Google_RE2' # Path to underlying data
batch_size = 16
must_choose_answer = True
use_context = True
device = 'cpu'
@ex.automain
def main(model_dir, model_name, device, relations_filepath, data_directory, batch_size, must_choose_answer, use_context):
runner = LMRunner(model_dir, model_name, device, relations_filepath, data_directory, batch_size, must_choose_answer)
em, f1, per_relation_metrics = runner.predict_naive(use_context=False)
return {'em': em, 'f1': f1, 'per_relation_metrics': per_relation_metrics}
"""
Experiment configuration for:
Model: RE-Flex
Benchmark: TACRED
"""
import fasttext
import spacy
import pickle
from reflex.reflex_runner import ReflexRunner
from reflex.utils import setup_experiment, save_reflex_e_list
import os
ex = setup_experiment('RE-Flex TACRED')
@ex.config
def conf():
model_dir = os.path.join(
os.environ['BASE_PATH'],
'weights/roberta_large') # Path to trained weights
model_name = os.path.join(os.environ['BASE_PATH'],
'weights/roberta_large/model.pt')
relations_filepath = os.path.join(
os.environ['BASE_PATH'],
'data/tacred_relations.jsonl') # Path to relations file
data_directory = os.path.join(
os.environ['BASE_PATH'], 'data/tacred/test') # Path to underlying data
hyperparam_path = os.path.join(os.environ['BASE_PATH'], 'tacred_tune.pkl')
error_path = os.path.join(os.environ['BASE_PATH'], 'figures',
'reflex_tacred.csv')
batch_size = 16
"""
Experiment configuration for:
Model: Radford Roberta (Naive w/ context seed)
Benchmark: Tacred
"""
from reflex.lm_runner import LMRunner
from reflex.utils import setup_experiment
ex = setup_experiment('Radford TACRED')
@ex.config
def conf():
model_dir = '/Users/ankur/Projects/RE-Flex/weights/roberta_large' # Path to trained weights
model_name = '/Users/ankur/Projects/RE-Flex/weights/roberta_large/model.pt'
relations_filepath = '/Users/ankur/Projects/RE-Flex/data/tacred_relations.jsonl' # Path to relations file
data_directory = '/Users/ankur/Projects/RE-Flex/data/tacred' # Path to underlying data
batch_size = 16
must_choose_answer = True
use_context = True
device = 'cpu'
@ex.automain
def main(model_dir, model_name, device, relations_filepath, data_directory,
batch_size, must_choose_answer, use_context):
runner = LMRunner(model_dir, model_name, device, relations_filepath,
data_directory, batch_size, must_choose_answer)
em, f1, per_relation_metrics = runner.predict_naive(use_context=False)
return {'em': em, 'f1': f1, 'per_relation_metrics': per_relation_metrics}
"""
Experiment configuration for:
Model: Lewis et al 2019 -- https://arxiv.org/abs/1906.04980
Benchmark: ZSRE
"""
from reflex.qa_runner import QARunner
from reflex.utils import setup_experiment
ex = setup_experiment('Lewis ZSRE')
@ex.config
def conf():
qa_path = '/Users/ankur/Projects/RE-Flex/weights/lewis-latest' # Path to trained weights
relations_filepath = '/Users/ankur/Projects/RE-Flex/data/zsre_relations.jsonl' # Path to relations file
data_directory = '/Users/ankur/Projects/RE-Flex/data/zsre' # Path to underlying data
batch_size = 16
must_choose_answer = True
@ex.automain
def main(qa_path, relations_filepath, data_directory, batch_size,
must_choose_answer):
runner = QARunner(qa_path, relations_filepath, data_directory, batch_size,
must_choose_answer)
em, f1, per_relation_metrics = runner.predict()
return {'em': em, 'f1': f1, 'per_relation_metrics': per_relation_metrics}
"""
Experiment configuration for:
Model: Lewis et al 2019 -- https://arxiv.org/abs/1906.04980
Benchmark: Tacred
"""
from reflex.qa_runner import QARunner
from reflex.utils import setup_experiment
ex = setup_experiment('Lewis TACRED')
@ex.config
def conf():
qa_path = os.path.join(os.environ['BASE_PATH'],
'weights/lewis-latest') # Path to trained weights
relations_filepath = os.path.join(
os.environ['BASE_PATH'],
'data/tacred_relations.jsonl') # Path to relations file
data_directory = os.path.join(os.environ['BASE_PATH'],
'data/Tacred') # Path to underlying data
batch_size = 16
must_choose_answer = True
device = 'cpu'
trained_to_reject = False
@ex.automain
def main(qa_path, relations_filepath, data_directory, batch_size,
must_choose_answer):
runner = QARunner(qa_path, relations_filepath, data_directory, batch_size,
must_choose_answer)
"""
Experiment configuration for:
Model: RE-Flex
Benchmark: TACRED
"""
import fasttext
import spacy
from reflex.reflex_runner import ReflexRunner
from reflex.utils import setup_experiment
from reflex.metrics import calculate_final_em_f1_dev
import pickle
import os
ex = setup_experiment('RE-Flex TACRED Dev tuning')
@ex.config
def conf():
model_dir = os.path.join(os.environ['BASE_PATH'], 'weights/roberta_large') # Path to trained weights
model_name = os.path.join(os.environ['BASE_PATH'], 'weights/roberta_large/model.pt')
relations_filepath = os.path.join(os.environ['BASE_PATH'], 'data/tacred_relations.jsonl') # Path to relations file
data_directory = os.path.join(os.environ['BASE_PATH'], 'data/tacred/dev') # Path to underlying data
batch_size = 16
must_choose_answer = False
device = 'cuda'
ls = [-1.5, -1, -0.5, 0, 0.5, 1, 1.5, 2]
k = 16
word_embeddings_path = os.path.join(os.environ['BASE_PATH'], 'weights/crawl-300d-2M-subword.bin')
@ex.automain
def main(model_dir, model_name, device, relations_filepath, data_directory, batch_size, must_choose_answer, word_embeddings_path, ls, k):
spacy_model = spacy.load('en_core_web_lg')