def __init__(self):
self.loaded_exp = load_best_model_exp(conf.model)
self.__dict__.update(self.loaded_exp)
self.best_model = self.loaded_exp.train_eval_model_factory.load_estimator(
self.best_model_dir, self.best_model_params)
self.data_loader = create_data_loader()
self.data_loader.load_data()
def create_load_experiment():
data_loader = create_data_loader()
exp_factory_clazz = getattr(experiment.factory, FLAGS.model)
exp_factory = exp_factory_clazz()
exp = None
if exp_factory.is_compatible_with(data_loader):
exp = exp_factory.create(data_loader, FLAGS.num_workers,
FLAGS.num_samples,
FLAGS.num_samples_best_eval)
exp.load_best_model()
return exp
def run_true_metrics(data_set):
chdir_data_set(data_set)
FLAGS.data_set = data_set
data_loader = create_data_loader()
if data_loader.can_compute_ll():
train_ll = data_loader.ll(data_loader.train_x, data_loader.train_y)
validation_ll=data_loader.ll(data_loader.validation_x, data_loader.validation_y)
test_ll=data_loader.ll(data_loader.test_x, data_loader.test_y)
with open('real_metrics.pkl', 'wb') as f:
data = {}
data["train_ll"] = train_ll
data["validation_ll"] = validation_ll
data["test_ll"] = test_ll
pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
store_data_seeds(data_sample_random_seeds)
print("Data sample seeds: %s"%data_sample_random_seeds)
for data_set in [data_set.strip() for data_set in FLAGS.data_sets.split(",")]:
funcs.append(Callable(run_true_metrics, data_set=data_set))
for model in [model.strip() for model in FLAGS.models.split(",")]:
funcs.append(Callable(run,model=model, data_set=data_set,data_sample_random_seeds = data_sample_random_seeds))
# First prefetch all data
for data_set in FLAGS.data_sets.split(","):
FLAGS.data_set = data_set
target_dir = os.path.join(FLAGS.dir, data_set)
os.makedirs(target_dir, exist_ok=True)
os.chdir(target_dir)
data_loader = create_data_loader()
done = 0.0
futures = []
res = [None] * len(funcs)
with concurrent.futures.ProcessPoolExecutor(FLAGS.num_parallel_experiments) as executor:
for i, fun in enumerate(funcs):
inserted = False
while not inserted:
if len(futures) < FLAGS.num_parallel_experiments:
futures.append((i, executor.submit(fun)))
inserted = True
for fut in list(futures):
try:
res[fut[0]] = fut[1].result(0)
def __init__(self):
loaded = load_best_model_exp(FLAGS.model)
self.__dict__.update(loaded)
self.best_model = load_model(self.best_model_dir, self.best_model_params)
self.data_loader = create_data_loader()
self.data_loader.load_data()