def run_for_class(clas, it=5): print('work on ' + clas) torch.cuda.empty_cache() encoder_name = 'encoder_' + best_lm_exp_id drop_mult = 1 ex = Experiment(db_name + '_' + clas) ex.observers.append(MongoObserver.create(db_name=db_name + '_' + clas)) @ex.config def my_config(): exp_id = datetime.datetime.now().strftime("%Y_%_m_%d_%H_%M_%S_%f") factor = 2.6 wd = 1e-7 moms = (0.8, 0.7) full_epochs = 20 bs = 50 embed_prevent = 6 lm_model_type = 'trained_6_embed_prevent' @ex.main def run_exp(exp_id, drop_mult, moms, wd, factor, full_epochs): for prev in range(0, 10): data_clas = setup_data(clas, prev) learn = text_classifier_learner(data_clas, drop_mult=drop_mult, embed_prevent_first=6) if prev == 0: learn.load_encoder(encoder_name) else: learn.load(exp_id) learn.unfreeze() lr = news_utils.fastai.get_optimal_lr(learn, runs=1, num_it=500) lrs = [lr / (factor**(4 - x)) for x in range(4)] + [lr] learn.metrics += [ news_utils.fastai.F1Macro(), news_utils.fastai.F1Weighted(), news_utils.fastai.PrecisionMacro(), news_utils.fastai.RecallMacro() ] learn.callbacks += [ SaveModelCallback(learn, name=exp_id), news_utils.fastai.SacredLogger(learn, ex), ] max_i = 2 if prev == 0: max_i = 4 for i in range(1, 3): epochs = 1 if i in [1, 2]: learn.freeze_to(-i) else: learn.unfreeze() # epochs = full_epochs learn.fit_one_cycle(epochs, np.array(lrs), wd=wd, moms=moms) for _ in range(it): ex.run(config_updates={"drop_mult": drop_mult})
def test_named_config_not_found_raises(): ex = Experiment('exp') ex.main(lambda: None) with pytest.raises(NamedConfigNotFoundError, match='Named config not found: "not_there". ' 'Available config values are:'): ex.run(named_configs=('not_there',))
def test_named_config_not_found_raises(): ex = Experiment('exp') ex.main(lambda: None) with pytest.raises(NamedConfigNotFoundError, match='Named config not found: "not_there". ' 'Available config values are:'): ex.run(named_configs=('not_there', ))
def run_for_class(it=1): # print('work on ' + clas) torch.cuda.empty_cache() data_clas = setup_data() encoder_name = 'encoder_' + best_lm_exp_id drop_mult = 1.0 #learn = text_classifier_learner(data_clas, drop_mult=drop_mult, embed_prevent_first=0) #earn.load_encoder(encoder_name) all_lrs = [] #for _ in range(3): # all_lrs.append(news_utils.fastai.get_optimal_lr(learn, runs=1)) #optim_lr = max(all_lrs) ex = Experiment(db_name) ex.observers.append(MongoObserver.create(db_name=db_name)) @ex.config def my_config(): exp_id = datetime.datetime.now().strftime("%Y_%_m_%d_%H_%M_%S_%f") factor = 2.6 wd = 0.01 moms = (0.8, 0.7) full_epochs =200 bs = 64 embed_prevent=0 mod='simle_fit' input_p = 0.3 lr = 0.001 embed_p = 0.1 @ex.main def run_exp(exp_id, drop_mult, input_p, embed_p, lr, moms, wd, factor, full_epochs): lrs = [lr / (factor ** (4 - x)) for x in range(4)] + [lr] learn = text_classifier_learner(data_clas, drop_mult=drop_mult, embed_prevent_first=0) learn.load_encoder(encoder_name) learn.metrics += [KappaScore(), news_utils.fastai.F1Macro(), news_utils.fastai.F1Weighted(), news_utils.fastai.PrecisionMacro(), news_utils.fastai.RecallMacro()] learn.callbacks += [ SaveModelCallback(learn, name=exp_id, monitor='kappa_score'), EarlyStoppingCallback(learn, monitor='kappa_score', patience=20, mode='max'), news_utils.fastai.SacredLogger(learn, ex), ] for i in range(1, 5): epochs = 1 if i in [1, 2]: learn.freeze_to(-i) else: learn.freeze_to(-2) epochs = full_epochs learn.fit(epochs, np.array(lrs)) for _ in range(it): ex.run(config_updates={ "drop_mult": drop_mult})
def test_save_config(): ex = Experiment(name="experiment") config = {"a": 1, "b": 2, "seed": 42} ex.add_config(config) @ex.main def main(): pass cwd = os.getcwd() try: with tempfile.TemporaryDirectory() as dirname: os.chdir(dirname) ex.run("save_config") with open(os.path.join(dirname, "config.json")) as f: saved_config = json.load(f) assert config == saved_config if opt.has_yaml: ex.run("save_config", {"config_filename": "config.yaml"}) with open(os.path.join(dirname, "config.yaml")) as f: saved_config = opt.yaml.load(f, Loader=opt.yaml.FullLoader) assert config == saved_config finally: os.chdir(cwd)
def _run_experiment(experiment: sacred.Experiment, config: str, observer: str, num_trials: int = 1): experiment.add_config(config) experiment.observers.append( file_storage.FileStorageObserver.create(observer)) for trial in range(num_trials): experiment.run()
def run_for_class(clas, it=1): print('work on ' + clas) torch.cuda.empty_cache() data_clas = setup_data(clas) encoder_name = 'encoder_' + best_lm_exp_id drop_mult = args.do learn = text_classifier_learner(data_clas, drop_mult=drop_mult) learn.load_encoder(encoder_name) # optim_lr = news_utils.fastai.get_optimal_lr(learn, runs=3) optim_lr = 0.001 ex = Experiment(db_name + '_' + clas) ex.observers.append(MongoObserver.create(db_name=db_name + '_' + clas)) @ex.config def my_config(): exp_id = datetime.datetime.now().strftime("%Y_%_m_%d_%H_%M_%S_%f") factor = 2.6 wd = 1e-7 moms = (0.8, 0.7) full_epochs = 200 @ex.main def run_exp(exp_id, drop_mult, lr, moms, wd, factor, full_epochs): lrs = [lr / (factor**(4 - x)) for x in range(4)] + [lr] learn = text_classifier_learner(data_clas, drop_mult=drop_mult) learn.load_encoder(encoder_name) learn.metrics += [ KappaScore(), news_utils.fastai.F1Macro(), news_utils.fastai.F1Weighted() ] learn.callbacks += [ EarlyStoppingCallback(learn, monitor='kappa_score', patience=20, mode='max'), SaveModelCallback(learn, name=exp_id, monitor='kappa_score'), news_utils.fastai.SacredLogger(learn, ex), ] for i in range(1, 4): epochs = 1 if i in [1, 2]: learn.freeze_to(-i) else: learn.unfreeze() epochs = full_epochs learn.fit(epochs, np.array(lrs)) for _ in range(it): ex.run(config_updates={"lr": optim_lr, "drop_mult": drop_mult})
def fetch_parents(current_path, parents=[]): tmp_ex = Experiment('treeqn') tmp_ex.add_config(current_path) with suppress_stdout(): tmp_ex.run("print_config") if tmp_ex.current_run is not None and "parent_config" in tmp_ex.current_run.config: return fetch_parents(tmp_ex.current_run.config["parent_config"], [current_path] + parents) else: return [current_path] + parents
def test_experiment_builder(config): ex = Experiment() ex = config_builder.build(ex) @ex.automain def main(_run): pretty(_run.config) ex.run(named_configs=[config])
def test_circular_dependency_raises(): # create experiment with circular dependency ing = Ingredient('ing') ex = Experiment('exp', ingredients=[ing]) ex.main(lambda: None) ing.ingredients.append(ex) # run and see if it raises with pytest.raises(CircularDependencyError, match='exp->ing->exp'): ex.run()
def test_config_added_raises(): ex = Experiment('exp') ex.main(lambda: None) with pytest.raises( ConfigAddedError, match=r'Added new config entry that is not used anywhere.*\n' r'\s*Conflicting configuration values:\n' r'\s*a=42'): ex.run(config_updates={'a': 42})
def run_for_class(it=1): train_df = pd.read_pickle(Path('/mnt/data/group07/johannes/ompc/data_ann_pp')/cat/fold/'train.pkl') test_df = pd.read_pickle(Path('/mnt/data/group07/johannes/ompc/data_ann_pp')/cat/fold/'test.pkl') if cat == 'SentimentNeutral': train_df['Value'] = train_df['Value'].apply(lambda x: 1 if x == 0 else 0) test_df['Value'] = test_df['Value'].apply(lambda x: 1 if x == 0 else 0) print('fixing sentimal neutral') data = TextClasDataBunch.from_ids(pad_idx=25000, bs=64,path=exp_path, vocab=data_lm_ft.vocab, classes=[0, 1], train_lbls=train_df['Value'], valid_lbls=test_df['Value'], train_ids=train_df['res'], valid_ids=test_df['res']) drop_mult = 1 exp_name = 'pp_' + cat + '_' + fold + '_' + str(drop_mult).replace('.', '_') ex = Experiment(exp_name) ex.observers.append(MongoObserver.create(db_name=exp_name)) @ex.config def my_config(): exp_id = datetime.datetime.now().strftime("%Y_%_m_%d_%H_%M_%S_%f") factor = 2.6 full_epochs = 100 bs = 64 mod='simple_fit' lr = 0.001 @ex.main def run_exp(exp_id, drop_mult, lr, factor, full_epochs): lrs = [lr / (factor ** (4 - x)) for x in range(4)] + [lr] learn = text_classifier_learner(data, drop_mult=drop_mult) learn.load_encoder('enc5') learn.metrics += [KappaScore(), news_utils.fastai.F1Bin(), news_utils.fastai.PrecBin(), news_utils.fastai.RecaBin()] learn.loss_func = torch.nn.CrossEntropyLoss(torch.FloatTensor(sklearn.utils.class_weight.compute_class_weight('balanced', [0, 1], train_df['Value'])).cuda()) learn.callbacks += [ SaveModelCallback(learn, name=exp_id, monitor='F1_bin'), EarlyStoppingCallback(learn, monitor='F1_bin', patience=80, mode='max'), news_utils.fastai.SacredLogger(learn, ex), ] for i in range(1, 5): epochs = 1 if i in [1, 2, 3]: learn.freeze_to(-i) else: learn.unfreeze() epochs = full_epochs learn.fit(epochs, np.array(lrs)) for _ in range(it): ex.run(config_updates={ "drop_mult": drop_mult})
def test_config_added_raises(): ex = Experiment("exp") ex.main(lambda: None) with pytest.raises( ConfigAddedError, match=r"Added new config entry that is not used anywhere.*\n" r"\s*Conflicting configuration values:\n" r"\s*a=42", ): ex.run(config_updates={"a": 42})
def run_for_class(clas, it=5): print('work on ' + clas) torch.cuda.empty_cache() data_clas = setup_data(clas) drop_mult = 1 learn = text_classifier_learner(data_clas, drop_mult=drop_mult) learn.load_encoder('no_ft') optim_lr = news_utils.fastai.get_optimal_lr(learn, runs=3) ex = Experiment(db_name + '_' + clas) ex.observers.append(MongoObserver.create(db_name=db_name + '_' + clas)) @ex.config def my_config(): exp_id = datetime.datetime.now().strftime("%Y_%_m_%d_%H_%M_%S_%f") factor = 2.6 wd = 1e-7 moms = (0.8, 0.7) full_epochs = 10 bs = 50 @ex.main def run_exp(exp_id, drop_mult, lr, moms, wd, factor, full_epochs): lrs = [lr / (factor**(4 - x)) for x in range(4)] + [lr] learn = text_classifier_learner(data_clas, drop_mult=drop_mult) learn.load_encoder('no_ft') learn.metrics += [ news_utils.fastai.F1Macro(), news_utils.fastai.F1Weighted(), news_utils.fastai.PrecisionMacro(), news_utils.fastai.RecallMacro() ] learn.callbacks += [ SaveModelCallback(learn, name=exp_id), news_utils.fastai.SacredLogger(learn, ex), ] for i in range(1, 4): epochs = 1 if i in [1, 2]: learn.freeze_to(-i) else: learn.unfreeze() epochs = full_epochs learn.fit_one_cycle(epochs, np.array(lrs), wd=wd, moms=moms) for _ in range(it): ex.run(config_updates={"lr": optim_lr, "drop_mult": drop_mult})
def test_world_tick(benchmark): ex = Experiment('pytest', ingredients=[mapgen.ing]) @ex.command def f(): np.random.seed(1) params = np.random.randn(mapgen.count_params()) world = mapgen.create_world(map_seed=1) mapgen.add_agents(world, params=params) benchmark(world.tick) conf = {'mapgen.world_size': 128} ex.run(command_name='f', config_updates=conf)
def run_for_class(clas, it=5): torch.cuda.empty_cache() data_clas = setup_data(clas) encoder_name = 'encoder_' + model_id drop_mult = 1 learn = text_classifier_learner(data_clas, drop_mult=drop_mult) learn.load_encoder(encoder_name) optim_lr = news_utils.fastai.get_optimal_lr(learn, runs=7) ex = Experiment(db_name + '_' + clas) ex.observers.append(MongoObserver.create(db_name=db_name)) @ex.config def my_config(): exp_id = datetime.datetime.now().strftime("%Y_%_m_%d_%H_%M_%S_%f") factor = 3 wd = 1e-7 moms = (0.8, 0.7) full_epochs = 10 @ex.main def run_exp(exp_id, drop_mult, lr, moms, wd, factor, full_epochs): lrs = [lr / (factor**(4 - x)) for x in range(4)] + [lr] learn = text_classifier_learner(data_clas, drop_mult=drop_mult) learn.load_encoder(encoder_name) learn.metrics += [ news_utils.fastai.F1Macro(), news_utils.fastai.F1Weighted() ] learn.callbacks += [ SaveModelCallback(learn, name=exp_id), ] for i in range(1, 4): epochs = 1 if i in [1, 2]: learn.freeze_to(-i) else: learn.unfreeze() # learn.callbacks += [EarlyStoppingCallback(learn, patience=5)] epochs = full_epochs # learn.fit_one_cycle(epochs, np.array(lrs) * 1 / (i ** 4), wd=wd, moms=moms) learn.fit_one_cycle(epochs, np.array(lrs), wd=wd, moms=moms) for _ in range(it): ex.run(config_updates={"lr": optim_lr, "drop_mult": drop_mult})
def test_run_waits_for_running_queue_observer(): queue_observer_with_long_interval = QueueObserver(mock.MagicMock(), interval=1, retry_interval=0.01) ex = Experiment("ator3000") ex.observers.append(queue_observer_with_long_interval) @ex.main def main(): print("do nothing") ex.run() assert (queue_observer_with_long_interval._covered_observer. method_calls[-1][0] == "completed_event")
def run(config_updates, mongodb_url="localhost:27017"): """Run a single experiment with the given configuration Args: config_updates (dict): Configuration updates mongodb_url (str): MongoDB URL, or None if no Mongo observer should be used for this run """ # Dynamically bind experiment config and main function ex = Experiment() ex.config(base_config) ex.main(element_world_v4) # Attach MongoDB observer if necessary if mongodb_url is not None and not ex.observers: ex.observers.append(MongoObserver(url=mongodb_url)) # Suppress warnings about padded MPDs with warnings.catch_warnings(): warnings.filterwarnings(action="ignore", category=PaddedMDPWarning) # Run the experiment run = ex.run(config_updates=config_updates ) # , options={"--loglevel": "ERROR"}) # Return the result return run.result
def sacred_experiment_with_config(config, name, main_fcn, db_name, base_dir, checkpoint_dir, sources=[], tune_config={}): """Launch a sacred experiment.""" # creating a sacred experiment # https://github.com/IDSIA/sacred/issues/492 from sacred import Experiment, SETTINGS SETTINGS.CONFIG.READ_ONLY_CONFIG = False ex = Experiment(name, base_dir=base_dir) ex.observers.append(MongoObserver(db_name=db_name)) for f in sources: if isinstance(f, str): f_py = f + '.py' shutil.copy(f, f_py) ex.add_source_file(f_py) export_config = dict(config) export_config.update(tune_config) ex.add_config(config=tune_config, **tune_config) @ex.main def run_train(): return main_fcn(config=config, checkpoint_dir=checkpoint_dir, ex=ex) return ex.run()
def objective(args_): # arguments to pass as config_updates dict global args # result to pass to hyperopt global result # command-line arguments global parse_args try: ex = Experiment('Hyperopt') logger.debug("Adding observer for {}, DB {}".format(parse_args.mongo_db_address,parse_args.mongo_db_name)) ex.observers.append(MongoObserver.create(url=parse_args.mongo_db_address, db_name=parse_args.mongo_db_name)) pythia_args = make_args_for_pythia(args_) args = mp.get_args(**pythia_args) ex.main(run_with_global_args) r = ex.run(config_updates=pythia_args) logger.debug("Experiment result: {}\n" "Report to hyperopt: {}".format(r.result, result)) return result except: raise #If we somehow cannot get to the MongoDB server, then continue with the experiment logger.warning("Running without Sacred") run_with_global_args()
def objective(args_): # arguments to pass as config_updates dict global args # result to pass to hyperopt global result # command-line arguments global parse_args try: ex = Experiment('Hyperopt') logger.debug("Adding observer for {}, DB {}".format( parse_args.mongo_db_address, parse_args.mongo_db_name)) ex.observers.append( MongoObserver.create(url=parse_args.mongo_db_address, db_name=parse_args.mongo_db_name)) pythia_args = make_args_for_pythia(args_) args = mp.get_args(**pythia_args) ex.main(run_with_global_args) r = ex.run(config_updates=pythia_args) logger.debug("Experiment result: {}\n" "Report to hyperopt: {}".format(r.result, result)) return result except: raise #If we somehow cannot get to the MongoDB server, then continue with the experiment logger.warning("Running without Sacred") run_with_global_args()
def test_mlproject_global_iteration(tmpdir): N_GLOBAL_ITERATIONS = 3 ex = Experiment() ex.add_config({ 'batch_size': 5, 'n_global_iterations': N_GLOBAL_ITERATIONS, 'tensorboard_dir': None, 'device': 'cuda:0' if torch.cuda.is_available() else 'cpu', 'model_dir': str(tmpdir.join('models')), }) @ex.automain def main(_run): proj = MyProject.from_run(_run) proj.train() assert proj.global_step == N_GLOBAL_ITERATIONS ex.run()
def test_delete(delete_db_loader, delete_mongo_observer): # Add experiment to db. ex = Experiment("to be deleted") ex.observers.append(delete_mongo_observer) ex.add_config({"value": 1}) def run(value, _run): _run.log_scalar("test_metric", 1) _run.add_artifact(__file__) return value ex.main(run) ex.run() # Retrieve and delete experiment. exp = delete_db_loader.find_by_id(1) exp.delete(confirmed=True) # Make sure experiment cannot be retrieved again. with raises(ValueError): exp = delete_db_loader.find_by_id(1)
def test_cifar(tmpdir): ex = Experiment() ex.add_config({ 'batch_size': 5, 'n_global_iterations': 3, 'tensorboard_dir': None, 'device': 'cuda:0' if torch.cuda.is_available() else 'cpu', 'model_dir': str(tmpdir.join('models')), }) @ex.automain def main(_run): proj = CifarProject.from_run(_run) print(proj.model._device_args, proj.model._device_kwargs) proj.test() proj.train() print(proj.model._device_args, proj.model._device_kwargs) proj.test() ex.run()
def test_run_waits_for_running_queue_observer_after_failure(): queue_observer_with_long_interval = QueueObserver(mock.MagicMock(), interval=1, retry_interval=0.01) ex = Experiment("ator3000") ex.observers.append(queue_observer_with_long_interval) @ex.main def main(): raise Exception("fatal error") try: ex.run() except: pass assert (queue_observer_with_long_interval._covered_observer. method_calls[-1][0] == "failed_event")
def test_mlproject_log_iterations(tmpdir): # TODO: N_GLOBAL_ITERATIONS = 10 ex = Experiment() ex.add_config({ 'batch_size': 5, 'n_global_iterations': N_GLOBAL_ITERATIONS, 'log_iteration_scalars': 2, 'log_iteration_all': 5, 'tensorboard_dir': None, 'device': 'cuda:0' if torch.cuda.is_available() else 'cpu', 'model_dir': str(tmpdir.join('models')), }) @ex.automain def main(_run): class MyModel(SimpleModel): def __init__(self): model = LinearModel() super().__init__('test', model, torch.optim.SGD(model.parameters(), lr=0.1), nn.CrossEntropyLoss()) def train_batch(self, batch): if proj.epoch_step % proj.config['log_iteration_scalars'] == 0: self.log = LogLevel.SCALARS if proj.epoch_step % proj.config['log_iteration_all'] == 0: self.log = LogLevel.ALL super().train_batch(batch) class TestProject(MyProject): @staticmethod def get_model(config): return MyModel() proj = MyProject.from_run(_run) proj.train() assert proj.global_step == N_GLOBAL_ITERATIONS ex.run()
def test_find_latest__for_multiple_with_newly_added_experiments(recent_db_loader, recent_mongo_observer): ex = SacredExperiment("most recent") ex.observers.append(recent_mongo_observer) ex.add_config({"value": 1}) def run(value, _run): return value ex.main(run) ex.run() ex = SacredExperiment("new most recent") ex.observers.append(recent_mongo_observer) ex.add_config({"value": 2}) ex.main(run) ex.run() exps = recent_db_loader.find_latest(2) assert exps[0].config.value == 2 assert exps[1].config.value == 1
def sacred_hyperopt_objective(params): """ Objective to call with hyperopt that uses sacred to log the experiment results """ ex = Experiment('example') ex.config(expt_config) ex.main(difficult_optimization_objective) ex.observers.append( MongoObserver.create( url=f"{os.environ['MONGO_WRITE_IP']}:{os.environ['MONGO_PORT']}")) run = ex.run(config_updates=params, options={"--loglevel": 40}) return {"loss": run.result, "status": hyperopt.STATUS_OK}
def inner(delete_mongo_observer, config_value): ex = SacredExperiment("name") ex.observers.append(delete_mongo_observer) ex.add_config({"value": config_value}) def run_fn(value, _run): _run.log_scalar("test_metric", 1) _run.add_artifact(__file__) return value ex.main(run_fn) run = ex.run() return run._id
def experiment_with_numpy_in_info(info_mongo_observer, info_db_loader) -> sacred.run.Run: # Add experiment to db. ex = Experiment("info experiment") ex.observers.append(info_mongo_observer) ex.add_config({"value": 1}) def run(value, _run): _run.info["array"] = np.array([1, 2, 3]) return value ex.main(run) yield ex.run() # Retrieve and delete experiment. info_db_loader.find_by_id(1).delete(confirmed=True)
def test_info(info_db_loader, info_db_loader_pickled, info_mongo_observer): # Add experiment to db. ex = Experiment("info experiment") ex.observers.append(info_mongo_observer) ex.add_config({"value": 1}) def run(value, _run): _run.info["number"] = 13 _run.info["list"] = [1, 2] _run.info["object"] = Fraction(3, 4) return value ex.main(run) ex.run() # Retrieve and delete experiment. exp_unpickled = info_db_loader.find_by_id(1) exp_pickled = info_db_loader_pickled.find_by_id(1) assert exp_unpickled.info["number"] == exp_pickled.info["number"] == 13 assert exp_unpickled.info["list"] == exp_pickled.info["list"] == [1, 2] assert exp_unpickled.info["object"] == Fraction(3, 4) assert isinstance(exp_pickled.info["object"], collections.abc.Mapping) exp_unpickled.delete(confirmed=True)
def experiment_with_pandas_in_info(info_mongo_observer, info_db_loader) -> sacred.run.Run: # Add experiment to db. ex = Experiment("info experiment") ex.observers.append(info_mongo_observer) ex.add_config({"value": 1}) def run(value, _run): _run.info["dataframe"] = pd.DataFrame({"a": [1, 2, 3], "b": ["1", "2", "3"]}) return value ex.main(run) yield ex.run() # Retrieve and delete experiment. info_db_loader.find_by_id(1).delete(confirmed=True)
def objective(args_): global args global result global parse_args args=args_ try: ex = Experiment('Hyperopt') ex.observers.append(MongoObserver.create(url=parse_args.mongo_db_address, db_name=parse_args.mongo_db_name)) ex.main(lambda: run_with_global_args()) r = ex.run(config_updates=args) print(r) return result except: raise #If we somehow cannot get to the MongoDB server, then continue with the experiment print("Running without Sacred") run_with_global_args()
print 'Mean training accuracy: %s' % training_accuracy print for hashtag_name in testing_hashtag_names: accuracy, _ = predict_on_hashtag(sess, model_vars, hashtag_name, HUMOR_TRIAL_TWEET_PAIR_EMBEDDING_DIR, trial_hashtag_datas) print 'Hashtag %s accuracy: %s' % (hashtag_name, accuracy) accuracies.append(accuracy) test_accuracy = np.mean(accuracies) print 'Final test accuracy: %s' % test_accuracy return {'train_accuracy': training_accuracy, 'test_accuracy': test_accuracy} @ex.main def main(learning_rate, num_epochs, dropout, use_emb_model, use_char_model, model_save_dir, hidden_dim_size, leave_out_hashtags=[]): load_build_train_and_predict(learning_rate, num_epochs, dropout, use_emb_model, use_char_model, model_save_dir, hidden_dim_size, leave_out_hashtags=[]) if __name__ == '__main__': num_experiments_run = 1 for index in range(num_experiments_run): print 'Experiment: %s' % index r = ex.run()
def main(): import argparse parser = argparse.ArgumentParser(description='Two layer linear regression') parser.add_argument("image_feature_file_train", type=str, help="Image Feature file for the training set") parser.add_argument("text_feature_file_train", type=str, help="Text Feature file for the training set") parser.add_argument("image_feature_file_test", type=str, help="Image Feature file for the test set") parser.add_argument("text_feature_file_test", type=str, help="Text Feature file for the test set") parser.add_argument("word_vector_file", type=str, help="Text file containing the word vectors") # Optional Args parser.add_argument("--learning_rate", type=float, default=0.001, help="Learning Rate") parser.add_argument("--num_epochs", type=int, default=200, help="Number of epochs to run for") parser.add_argument("--batch_size", type=int, default=128, help="Batch size to use for training") parser.add_argument("--model_type", type=str, default="multihot", choices=['multihot', 'naivesum', 'wdv', 'negsampling', 'fast0tag'], help="Loss function to use for training") parser.add_argument("--in_memory", action='store_true', default="store_false", help="Load training image features into memory for faster training") parser.add_argument("--model_input_path", type=str, default=None, help="Model input path (to continue training)") parser.add_argument("--model_output_path", type=str, default=None, help="Model output path (to save training)") # new args parser.add_argument("--hidden_units", type=str, default="200", help="Define a neural network as comma separated layer sizes. If log-reg, then set to '0'.") parser.add_argument("--cross_eval", action="store_true", default=False, help="Use if test dataset is different from training dataset") parser.add_argument("--word_vector_type", type=str, choices=[t.name for t in WordVectorTypes], help="Format of word_vector_file") parser.add_argument("--epoch_verbosity", type=int, default=10, help="Epoch verbosity rate") parser.add_argument("--verbose_eval", action="store_true", default=False, help="Use to run evaluation against test data every epoch_verbosity") parser.add_argument("--optim_words", action="store_true", default=False, help="If using negsampling model_type, use to jointly optimize words") parser.add_argument("--ignore_posbatch", action="store_true", default=False, help="Sample, ignoring from positive batch instead of examples. This should be taken out in future iters.") parser.add_argument("--joint_factor", type=float, default=1.0, help="Multiplier for learning rate in updating joint optimization") parser.add_argument("--use_batch_norm", action="store_true", default=False, help="Do we want to use batch normalization? Default is False") parser.add_argument("--opt_type", type=str, default="adam", help="What type of optimizer would you like? Choices are (adam,sgd)") parser.add_argument("--weight_decay", type=float, default=0.0, help="Weight decay to manually decay every 10 epochs. Default=0 for no decay.") parser.add_argument("--scale_words", type=float, default=1.0, help="Scale the word vectors. If set to zero, scale by L2-norm. Otherwise, wordvec=scale x wordvec") parser.add_argument("--scale_images", type=float, default=1.0, help="Scale the word vectors. If set to zero, scale by L2-norm. Otherwise, imvec=scale x imvec. ") parser.add_argument("--fast_sample", action="store_true", default=False, help="Fast sample based on distribution, only use in large dictionaries") args = parser.parse_args() try: # Sacred Imports from sacred import Experiment from sacred.observers import MongoObserver from sacred.initialize import Scaffold # Monkey patch to avoid having to declare all our variables def noop(item): pass Scaffold._warn_about_suspicious_changes = noop ex = Experiment('Attalos') ex.observers.append(MongoObserver.create(url=os.environ['MONGO_DB_URI'], db_name='attalos_experiment')) ex.main(lambda: convert_args_and_call_model(args)) ex.run(config_updates=args.__dict__) except ImportError: # We don't have sacred, just run the script logger.warn('Not using Sacred. Your results will not be saved') convert_args_and_call_model(args)
def test_missing_config_raises(): ex = Experiment('exp') ex.main(lambda a: None) with pytest.raises(MissingConfigError): ex.run()
def main(): import argparse parser = argparse.ArgumentParser(description='Two layer linear regression') parser.add_argument("image_feature_file_train", type=str, help="Image Feature file for the training set") parser.add_argument("text_feature_file_train", type=str, help="Text Feature file for the training set") parser.add_argument("image_feature_file_test", type=str, help="Image Feature file for the test set") parser.add_argument("text_feature_file_test", type=str, help="Text Feature file for the test set") parser.add_argument("word_vector_file", type=str, help="Text file containing the word vectors") # Optional Args parser.add_argument("--learning_rate", type=float, default=.001, help="Learning Rate") parser.add_argument("--epochs", type=int, default=200, help="Number of epochs to run for") parser.add_argument("--batch_size", type=int, default=128, help="Batch size to use for training") parser.add_argument("--network", type=str, default="200,200", help="Define a neural network as comma separated layer sizes") parser.add_argument("--model_type", type=str, default="mse", choices=['mse', 'negsampling', 'fast0tag'], help="Loss function to use for training") parser.add_argument("--in_memory", action='store_true', default="store_false", help="Load training image features into memory for faster training") parser.add_argument("--model_input_path", type=str, default=None, help="Model input path (to continue training)") parser.add_argument("--model_output_path", type=str, default=None, help="Model output path (to save training)") parser.add_argument("--max_pos", type=int, default=5, help="Max number of positive examples") parser.add_argument("--max_neg", type=int, default=10, help="Max number of negative examples") global args args = parser.parse_args() try: # Sacred Imports from sacred import Experiment from sacred.observers import MongoObserver from sacred.initialize import Scaffold # Monkey patch to avoid having to declare all our variables def noop(item): pass Scaffold._warn_about_suspicious_changes = noop ex = Experiment('Regress2sum') ex.observers.append(MongoObserver.create(url=os.environ['MONGO_DB_URI'], db_name='attalos_experiment')) ex.main(lambda: convert_args_and_call_model()) ex.run(config_updates=args.__dict__) except ImportError: # We don't have sacred, just run the script convert_args_and_call_model()
from sklearn import svm, datasets, model_selection ex = Experiment("svm") ex.observers.append( FileStorageObserver.create("my_runs") ) ex.add_config({ # Configuration is explicitly defined as dictionary. "C": 1.0, "gamma": 0.7, "kernel": "rbf", "seed": 42 }) def get_model(C, gamma, kernel): return svm.SVC(C=C, kernel=kernel, gamma=gamma) @ex.main # Using main, command-line arguments will not be interpreted in any special way. def run(_config): X, y = datasets.load_breast_cancer(return_X_y=True) X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.2) clf = get_model(_config["C"], _config["gamma"], _config["kernel"]) # Parameters are passed explicitly. clf.fit(X_train, y_train) return clf.score(X_test, y_test) if __name__ == "__main__": ex.run()