def make_model(model_name, example_store):
if model_name == "SeaCR-Rulebased":
return seacr.make_rulebased_seacr(example_store)
elif model_name == "SeaCR":
return seacr.make_default_seacr(example_store)
elif model_name == "SeaCR-NoSearch":
return seacr.make_default_seacr_no_search(example_store)
elif model_name == "SeaCR-OracleCompare":
return seacr.make_default_seacr_with_oracle_comparer(example_store)
elif model_name == "EncDec":
return encdecmodel.get_default_encdec_model(example_store,
standard_size=16)
elif model_name == "EncDecRetrieval":
return encdecmodel.get_default_encdec_model(example_store,
replacer=None,
use_retrieval_decoder=True)
else:
raise ValueError("Unrecognized model type ", type)
def get_examples_per_sec(batch_size, examples_sample_size=1e4):
index = get_index()
num_docs = index.get_num_x_values()
print("num docs", num_docs)
model = get_default_encdec_model(index, standard_size=64)
trainer = train.TypeTranslateCFTrainer(model, index, batch_size=batch_size)
train_time = datetime.datetime.now()
print("train time", train_time)
epochs = math.ceil(examples_sample_size / num_docs)
print("going to perform", epochs, "epochs")
trainer.train(epochs)
seconds_taken = (datetime.datetime.now() - train_time).total_seconds()
examples_done = num_docs * epochs
examples_per_second = examples_done / seconds_taken
return examples_per_second
def get_multi_train_time(batch_size, procs):
os.environ["OMP_NUM_THREADS"] = "1"
torch.set_num_threads = 1
index = get_index()
print("num docs", index.get_num_x_values())
model = get_default_encdec_model(examples=index, standard_size=64)
# train
mptrainer = MultiprocTrainer(model,
make_default_trainer_fac(model, batch_size))
num_docs = index.get_num_x_values()
time = mptrainer.train(workers_count=procs,
epochs_per_worker=50,
index=index,
batch_size=batch_size,
converge_check_val=0.75)
return time
def train_to_threshold_st(batch_size, threshold=0.7, max_epochs=100):
index = get_index()
num_docs = index.get_num_x_values()
print("num docs", num_docs)
model = get_default_encdec_model(index, standard_size=64)
trainer = train.TypeTranslateCFTrainer(model, index, batch_size=batch_size)
time_spent = 0
for epoch in range(max_epochs):
start_time = datetime.datetime.now()
trainer.train(1)
time_spent += (datetime.datetime.now() - start_time).total_seconds()
logger = EvaluateLogger()
trainer.evaluate(logger)
acc = logger.stats['ExactMatch'].true_frac
print(f"Curr acc {acc}")
if acc >= threshold:
break
return time_spent
def get_examples_per_sec_multi(batch_size, procs, examples_sample_size=1e4):
os.environ["OMP_NUM_THREADS"] = "1"
torch.set_num_threads = 1
index = get_index()
print("num docs", index.get_num_x_values())
model = get_default_encdec_model(examples=index, standard_size=64)
# train
mptrainer = MultiprocTrainer(model,
make_default_trainer_fac(model, batch_size))
num_docs = index.get_num_x_values()
epochs = math.ceil(examples_sample_size / num_docs / procs)
print("going to perform", epochs, "epochs")
train_time = datetime.datetime.now()
mptrainer.train(workers_count=procs,
epochs_per_worker=epochs,
index=index,
batch_size=batch_size)
seconds_taken = (datetime.datetime.now() - train_time).total_seconds()
examples_done = num_docs * epochs * procs
examples_per_second = examples_done / seconds_taken
return examples_per_second
p.join()
print("done!")
return val_accum.value
if __name__ == "__main__":
index_fac = example_store_fac([
'../../builtin_types/numbers.ainix.yaml',
'../../builtin_types/generic_parsers.ainix.yaml'
], [
"../../builtin_types/numbers_examples.ainix.yaml"
])
batch_size = 4
index = index_fac()
model = get_default_encdec_model(examples=index)
# Try before
print("before:")
trainer = TypeTranslateCFTrainer(model, index, batch_size)
logger = EvaluateLogger()
trainer.evaluate(logger)
print_ast_eval_log(logger)
# train
mptrainer = MultiprocTrainer(
model,
make_default_trainer_fac(
model,
batch_size
)
(1 - train_frac, DataSplits.VALIDATION))
import ainix_kernel.indexing.exampleindex
from ainix_kernel.indexing import exampleloader
import datetime
print("start time", datetime.datetime.now())
type_context, index, replacers, loader = get_examples(
split_proportions, args.randomize_seed)
from ainix_kernel.models.EncoderDecoder.encdecmodel import \
get_default_encdec_model
model = get_default_encdec_model(index,
standard_size=128,
use_retrieval_decoder=False,
replacer=replacers,
pretrain_checkpoint=cookie_monster_path()
if args.use_pretrain else None,
learn_on_top_pretrained=args.use_pretrain)
#model = make_rulebased_seacr(index)
trainer = TypeTranslateCFTrainer(model,
index,
replacer=replacers,
loader=loader)
train_time = datetime.datetime.now()
print("train time", train_time)
epochs = 35
trainer.train(epochs,
eval_every_n_epochs=3,
intermitted_save_path="./checkpoints/chkp",