def run_for_class(clas, it=5):
print('work on ' + clas)
torch.cuda.empty_cache()
encoder_name = 'encoder_' + best_lm_exp_id
drop_mult = 1
ex = Experiment(db_name + '_' + clas)
ex.observers.append(MongoObserver.create(db_name=db_name + '_' + clas))
@ex.config
def my_config():
exp_id = datetime.datetime.now().strftime("%Y_%_m_%d_%H_%M_%S_%f")
factor = 2.6
wd = 1e-7
moms = (0.8, 0.7)
full_epochs = 20
bs = 50
embed_prevent = 6
lm_model_type = 'trained_6_embed_prevent'
@ex.main
def run_exp(exp_id, drop_mult, moms, wd, factor, full_epochs):
for prev in range(0, 10):
data_clas = setup_data(clas, prev)
learn = text_classifier_learner(data_clas,
drop_mult=drop_mult,
embed_prevent_first=6)
if prev == 0:
learn.load_encoder(encoder_name)
else:
learn.load(exp_id)
learn.unfreeze()
lr = news_utils.fastai.get_optimal_lr(learn, runs=1, num_it=500)
lrs = [lr / (factor**(4 - x)) for x in range(4)] + [lr]
learn.metrics += [
news_utils.fastai.F1Macro(),
news_utils.fastai.F1Weighted(),
news_utils.fastai.PrecisionMacro(),
news_utils.fastai.RecallMacro()
]
learn.callbacks += [
SaveModelCallback(learn, name=exp_id),
news_utils.fastai.SacredLogger(learn, ex),
]
max_i = 2
if prev == 0:
max_i = 4
for i in range(1, 3):
epochs = 1
if i in [1, 2]:
learn.freeze_to(-i)
else:
learn.unfreeze()
# epochs = full_epochs
learn.fit_one_cycle(epochs, np.array(lrs), wd=wd, moms=moms)
for _ in range(it):
ex.run(config_updates={"drop_mult": drop_mult})
def test_named_config_not_found_raises():
ex = Experiment('exp')
ex.main(lambda: None)
with pytest.raises(NamedConfigNotFoundError,
match='Named config not found: "not_there". '
'Available config values are:'):
ex.run(named_configs=('not_there',))
def test_named_config_not_found_raises():
ex = Experiment('exp')
ex.main(lambda: None)
with pytest.raises(NamedConfigNotFoundError,
match='Named config not found: "not_there". '
'Available config values are:'):
ex.run(named_configs=('not_there', ))
def run_for_class(it=1):
# print('work on ' + clas)
torch.cuda.empty_cache()
data_clas = setup_data()
encoder_name = 'encoder_' + best_lm_exp_id
drop_mult = 1.0
#learn = text_classifier_learner(data_clas, drop_mult=drop_mult, embed_prevent_first=0)
#earn.load_encoder(encoder_name)
all_lrs = []
#for _ in range(3):
# all_lrs.append(news_utils.fastai.get_optimal_lr(learn, runs=1))
#optim_lr = max(all_lrs)
ex = Experiment(db_name)
ex.observers.append(MongoObserver.create(db_name=db_name))
@ex.config
def my_config():
exp_id = datetime.datetime.now().strftime("%Y_%_m_%d_%H_%M_%S_%f")
factor = 2.6
wd = 0.01
moms = (0.8, 0.7)
full_epochs =200
bs = 64
embed_prevent=0
mod='simle_fit'
input_p = 0.3
lr = 0.001
embed_p = 0.1
@ex.main
def run_exp(exp_id, drop_mult, input_p, embed_p, lr, moms, wd, factor, full_epochs):
lrs = [lr / (factor ** (4 - x)) for x in range(4)] + [lr]
learn = text_classifier_learner(data_clas, drop_mult=drop_mult, embed_prevent_first=0)
learn.load_encoder(encoder_name)
learn.metrics += [KappaScore(), news_utils.fastai.F1Macro(),
news_utils.fastai.F1Weighted(), news_utils.fastai.PrecisionMacro(), news_utils.fastai.RecallMacro()]
learn.callbacks += [
SaveModelCallback(learn, name=exp_id, monitor='kappa_score'),
EarlyStoppingCallback(learn, monitor='kappa_score', patience=20, mode='max'),
news_utils.fastai.SacredLogger(learn, ex),
]
for i in range(1, 5):
epochs = 1
if i in [1, 2]:
learn.freeze_to(-i)
else:
learn.freeze_to(-2)
epochs = full_epochs
learn.fit(epochs, np.array(lrs))
for _ in range(it):
ex.run(config_updates={ "drop_mult": drop_mult})
def test_save_config():
ex = Experiment(name="experiment")
config = {"a": 1, "b": 2, "seed": 42}
ex.add_config(config)
@ex.main
def main():
pass
cwd = os.getcwd()
try:
with tempfile.TemporaryDirectory() as dirname:
os.chdir(dirname)
ex.run("save_config")
with open(os.path.join(dirname, "config.json")) as f:
saved_config = json.load(f)
assert config == saved_config
if opt.has_yaml:
ex.run("save_config", {"config_filename": "config.yaml"})
with open(os.path.join(dirname, "config.yaml")) as f:
saved_config = opt.yaml.load(f, Loader=opt.yaml.FullLoader)
assert config == saved_config
finally:
os.chdir(cwd)
def _run_experiment(experiment: sacred.Experiment,
config: str,
observer: str,
num_trials: int = 1):
experiment.add_config(config)
experiment.observers.append(
file_storage.FileStorageObserver.create(observer))
for trial in range(num_trials):
experiment.run()
def run_for_class(clas, it=1):
print('work on ' + clas)
torch.cuda.empty_cache()
data_clas = setup_data(clas)
encoder_name = 'encoder_' + best_lm_exp_id
drop_mult = args.do
learn = text_classifier_learner(data_clas, drop_mult=drop_mult)
learn.load_encoder(encoder_name)
# optim_lr = news_utils.fastai.get_optimal_lr(learn, runs=3)
optim_lr = 0.001
ex = Experiment(db_name + '_' + clas)
ex.observers.append(MongoObserver.create(db_name=db_name + '_' + clas))
@ex.config
def my_config():
exp_id = datetime.datetime.now().strftime("%Y_%_m_%d_%H_%M_%S_%f")
factor = 2.6
wd = 1e-7
moms = (0.8, 0.7)
full_epochs = 200
@ex.main
def run_exp(exp_id, drop_mult, lr, moms, wd, factor, full_epochs):
lrs = [lr / (factor**(4 - x)) for x in range(4)] + [lr]
learn = text_classifier_learner(data_clas, drop_mult=drop_mult)
learn.load_encoder(encoder_name)
learn.metrics += [
KappaScore(),
news_utils.fastai.F1Macro(),
news_utils.fastai.F1Weighted()
]
learn.callbacks += [
EarlyStoppingCallback(learn,
monitor='kappa_score',
patience=20,
mode='max'),
SaveModelCallback(learn, name=exp_id, monitor='kappa_score'),
news_utils.fastai.SacredLogger(learn, ex),
]
for i in range(1, 4):
epochs = 1
if i in [1, 2]:
learn.freeze_to(-i)
else:
learn.unfreeze()
epochs = full_epochs
learn.fit(epochs, np.array(lrs))
for _ in range(it):
ex.run(config_updates={"lr": optim_lr, "drop_mult": drop_mult})
def fetch_parents(current_path, parents=[]):
tmp_ex = Experiment('treeqn')
tmp_ex.add_config(current_path)
with suppress_stdout():
tmp_ex.run("print_config")
if tmp_ex.current_run is not None and "parent_config" in tmp_ex.current_run.config:
return fetch_parents(tmp_ex.current_run.config["parent_config"], [current_path] + parents)
else:
return [current_path] + parents
def test_experiment_builder(config):
ex = Experiment()
ex = config_builder.build(ex)
@ex.automain
def main(_run):
pretty(_run.config)
ex.run(named_configs=[config])
def fetch_parents(current_path, parents=[]):
tmp_ex = Experiment('treeqn')
tmp_ex.add_config(current_path)
with suppress_stdout():
tmp_ex.run("print_config")
if tmp_ex.current_run is not None and "parent_config" in tmp_ex.current_run.config:
return fetch_parents(tmp_ex.current_run.config["parent_config"],
[current_path] + parents)
else:
return [current_path] + parents
def test_circular_dependency_raises():
# create experiment with circular dependency
ing = Ingredient('ing')
ex = Experiment('exp', ingredients=[ing])
ex.main(lambda: None)
ing.ingredients.append(ex)
# run and see if it raises
with pytest.raises(CircularDependencyError, match='exp->ing->exp'):
ex.run()
def test_config_added_raises():
ex = Experiment('exp')
ex.main(lambda: None)
with pytest.raises(
ConfigAddedError,
match=r'Added new config entry that is not used anywhere.*\n'
r'\s*Conflicting configuration values:\n'
r'\s*a=42'):
ex.run(config_updates={'a': 42})
def test_config_added_raises():
ex = Experiment('exp')
ex.main(lambda: None)
with pytest.raises(
ConfigAddedError,
match=r'Added new config entry that is not used anywhere.*\n'
r'\s*Conflicting configuration values:\n'
r'\s*a=42'):
ex.run(config_updates={'a': 42})
def test_circular_dependency_raises():
# create experiment with circular dependency
ing = Ingredient('ing')
ex = Experiment('exp', ingredients=[ing])
ex.main(lambda: None)
ing.ingredients.append(ex)
# run and see if it raises
with pytest.raises(CircularDependencyError, match='exp->ing->exp'):
ex.run()
def run_for_class(it=1):
train_df = pd.read_pickle(Path('/mnt/data/group07/johannes/ompc/data_ann_pp')/cat/fold/'train.pkl')
test_df = pd.read_pickle(Path('/mnt/data/group07/johannes/ompc/data_ann_pp')/cat/fold/'test.pkl')
if cat == 'SentimentNeutral':
train_df['Value'] = train_df['Value'].apply(lambda x: 1 if x == 0 else 0)
test_df['Value'] = test_df['Value'].apply(lambda x: 1 if x == 0 else 0)
print('fixing sentimal neutral')
data = TextClasDataBunch.from_ids(pad_idx=25000, bs=64,path=exp_path, vocab=data_lm_ft.vocab, classes=[0, 1], train_lbls=train_df['Value'], valid_lbls=test_df['Value'], train_ids=train_df['res'], valid_ids=test_df['res'])
drop_mult = 1
exp_name = 'pp_' + cat + '_' + fold + '_' + str(drop_mult).replace('.', '_')
ex = Experiment(exp_name)
ex.observers.append(MongoObserver.create(db_name=exp_name))
@ex.config
def my_config():
exp_id = datetime.datetime.now().strftime("%Y_%_m_%d_%H_%M_%S_%f")
factor = 2.6
full_epochs = 100
bs = 64
mod='simple_fit'
lr = 0.001
@ex.main
def run_exp(exp_id, drop_mult, lr, factor, full_epochs):
lrs = [lr / (factor ** (4 - x)) for x in range(4)] + [lr]
learn = text_classifier_learner(data, drop_mult=drop_mult)
learn.load_encoder('enc5')
learn.metrics += [KappaScore(), news_utils.fastai.F1Bin(), news_utils.fastai.PrecBin(), news_utils.fastai.RecaBin()]
learn.loss_func = torch.nn.CrossEntropyLoss(torch.FloatTensor(sklearn.utils.class_weight.compute_class_weight('balanced', [0, 1], train_df['Value'])).cuda())
learn.callbacks += [
SaveModelCallback(learn, name=exp_id, monitor='F1_bin'),
EarlyStoppingCallback(learn, monitor='F1_bin', patience=80, mode='max'),
news_utils.fastai.SacredLogger(learn, ex),
]
for i in range(1, 5):
epochs = 1
if i in [1, 2, 3]:
learn.freeze_to(-i)
else:
learn.unfreeze()
epochs = full_epochs
learn.fit(epochs, np.array(lrs))
for _ in range(it):
ex.run(config_updates={ "drop_mult": drop_mult})
def test_config_added_raises():
ex = Experiment("exp")
ex.main(lambda: None)
with pytest.raises(
ConfigAddedError,
match=r"Added new config entry that is not used anywhere.*\n"
r"\s*Conflicting configuration values:\n"
r"\s*a=42",
):
ex.run(config_updates={"a": 42})
def run_for_class(clas, it=5):
print('work on ' + clas)
torch.cuda.empty_cache()
data_clas = setup_data(clas)
drop_mult = 1
learn = text_classifier_learner(data_clas, drop_mult=drop_mult)
learn.load_encoder('no_ft')
optim_lr = news_utils.fastai.get_optimal_lr(learn, runs=3)
ex = Experiment(db_name + '_' + clas)
ex.observers.append(MongoObserver.create(db_name=db_name + '_' + clas))
@ex.config
def my_config():
exp_id = datetime.datetime.now().strftime("%Y_%_m_%d_%H_%M_%S_%f")
factor = 2.6
wd = 1e-7
moms = (0.8, 0.7)
full_epochs = 10
bs = 50
@ex.main
def run_exp(exp_id, drop_mult, lr, moms, wd, factor, full_epochs):
lrs = [lr / (factor**(4 - x)) for x in range(4)] + [lr]
learn = text_classifier_learner(data_clas, drop_mult=drop_mult)
learn.load_encoder('no_ft')
learn.metrics += [
news_utils.fastai.F1Macro(),
news_utils.fastai.F1Weighted(),
news_utils.fastai.PrecisionMacro(),
news_utils.fastai.RecallMacro()
]
learn.callbacks += [
SaveModelCallback(learn, name=exp_id),
news_utils.fastai.SacredLogger(learn, ex),
]
for i in range(1, 4):
epochs = 1
if i in [1, 2]:
learn.freeze_to(-i)
else:
learn.unfreeze()
epochs = full_epochs
learn.fit_one_cycle(epochs, np.array(lrs), wd=wd, moms=moms)
for _ in range(it):
ex.run(config_updates={"lr": optim_lr, "drop_mult": drop_mult})
def test_world_tick(benchmark):
ex = Experiment('pytest', ingredients=[mapgen.ing])
@ex.command
def f():
np.random.seed(1)
params = np.random.randn(mapgen.count_params())
world = mapgen.create_world(map_seed=1)
mapgen.add_agents(world, params=params)
benchmark(world.tick)
conf = {'mapgen.world_size': 128}
ex.run(command_name='f', config_updates=conf)
def run_for_class(clas, it=5):
torch.cuda.empty_cache()
data_clas = setup_data(clas)
encoder_name = 'encoder_' + model_id
drop_mult = 1
learn = text_classifier_learner(data_clas, drop_mult=drop_mult)
learn.load_encoder(encoder_name)
optim_lr = news_utils.fastai.get_optimal_lr(learn, runs=7)
ex = Experiment(db_name + '_' + clas)
ex.observers.append(MongoObserver.create(db_name=db_name))
@ex.config
def my_config():
exp_id = datetime.datetime.now().strftime("%Y_%_m_%d_%H_%M_%S_%f")
factor = 3
wd = 1e-7
moms = (0.8, 0.7)
full_epochs = 10
@ex.main
def run_exp(exp_id, drop_mult, lr, moms, wd, factor, full_epochs):
lrs = [lr / (factor**(4 - x)) for x in range(4)] + [lr]
learn = text_classifier_learner(data_clas, drop_mult=drop_mult)
learn.load_encoder(encoder_name)
learn.metrics += [
news_utils.fastai.F1Macro(),
news_utils.fastai.F1Weighted()
]
learn.callbacks += [
SaveModelCallback(learn, name=exp_id),
]
for i in range(1, 4):
epochs = 1
if i in [1, 2]:
learn.freeze_to(-i)
else:
learn.unfreeze()
# learn.callbacks += [EarlyStoppingCallback(learn, patience=5)]
epochs = full_epochs
# learn.fit_one_cycle(epochs, np.array(lrs) * 1 / (i ** 4), wd=wd, moms=moms)
learn.fit_one_cycle(epochs, np.array(lrs), wd=wd, moms=moms)
for _ in range(it):
ex.run(config_updates={"lr": optim_lr, "drop_mult": drop_mult})
def test_run_waits_for_running_queue_observer():
queue_observer_with_long_interval = QueueObserver(mock.MagicMock(),
interval=1,
retry_interval=0.01)
ex = Experiment("ator3000")
ex.observers.append(queue_observer_with_long_interval)
@ex.main
def main():
print("do nothing")
ex.run()
assert (queue_observer_with_long_interval._covered_observer.
method_calls[-1][0] == "completed_event")
def run(config_updates, mongodb_url="localhost:27017"):
"""Run a single experiment with the given configuration
Args:
config_updates (dict): Configuration updates
mongodb_url (str): MongoDB URL, or None if no Mongo observer should be used for
this run
"""
# Dynamically bind experiment config and main function
ex = Experiment()
ex.config(base_config)
ex.main(element_world_v4)
# Attach MongoDB observer if necessary
if mongodb_url is not None and not ex.observers:
ex.observers.append(MongoObserver(url=mongodb_url))
# Suppress warnings about padded MPDs
with warnings.catch_warnings():
warnings.filterwarnings(action="ignore", category=PaddedMDPWarning)
# Run the experiment
run = ex.run(config_updates=config_updates
) # , options={"--loglevel": "ERROR"})
# Return the result
return run.result
def sacred_experiment_with_config(config, name, main_fcn, db_name, base_dir, checkpoint_dir, sources=[], tune_config={}):
"""Launch a sacred experiment."""
# creating a sacred experiment
# https://github.com/IDSIA/sacred/issues/492
from sacred import Experiment, SETTINGS
SETTINGS.CONFIG.READ_ONLY_CONFIG = False
ex = Experiment(name, base_dir=base_dir)
ex.observers.append(MongoObserver(db_name=db_name))
for f in sources:
if isinstance(f, str):
f_py = f + '.py'
shutil.copy(f, f_py)
ex.add_source_file(f_py)
export_config = dict(config)
export_config.update(tune_config)
ex.add_config(config=tune_config, **tune_config)
@ex.main
def run_train():
return main_fcn(config=config, checkpoint_dir=checkpoint_dir, ex=ex)
return ex.run()
def objective(args_):
# arguments to pass as config_updates dict
global args
# result to pass to hyperopt
global result
# command-line arguments
global parse_args
try:
ex = Experiment('Hyperopt')
logger.debug("Adding observer for {}, DB {}".format(parse_args.mongo_db_address,parse_args.mongo_db_name))
ex.observers.append(MongoObserver.create(url=parse_args.mongo_db_address, db_name=parse_args.mongo_db_name))
pythia_args = make_args_for_pythia(args_)
args = mp.get_args(**pythia_args)
ex.main(run_with_global_args)
r = ex.run(config_updates=pythia_args)
logger.debug("Experiment result: {}\n"
"Report to hyperopt: {}".format(r.result, result))
return result
except:
raise
#If we somehow cannot get to the MongoDB server, then continue with the experiment
logger.warning("Running without Sacred")
run_with_global_args()
def objective(args_):
# arguments to pass as config_updates dict
global args
# result to pass to hyperopt
global result
# command-line arguments
global parse_args
try:
ex = Experiment('Hyperopt')
logger.debug("Adding observer for {}, DB {}".format(
parse_args.mongo_db_address, parse_args.mongo_db_name))
ex.observers.append(
MongoObserver.create(url=parse_args.mongo_db_address,
db_name=parse_args.mongo_db_name))
pythia_args = make_args_for_pythia(args_)
args = mp.get_args(**pythia_args)
ex.main(run_with_global_args)
r = ex.run(config_updates=pythia_args)
logger.debug("Experiment result: {}\n"
"Report to hyperopt: {}".format(r.result, result))
return result
except:
raise
#If we somehow cannot get to the MongoDB server, then continue with the experiment
logger.warning("Running without Sacred")
run_with_global_args()
def test_mlproject_global_iteration(tmpdir):
N_GLOBAL_ITERATIONS = 3
ex = Experiment()
ex.add_config({
'batch_size': 5,
'n_global_iterations': N_GLOBAL_ITERATIONS,
'tensorboard_dir': None,
'device': 'cuda:0' if torch.cuda.is_available() else 'cpu',
'model_dir': str(tmpdir.join('models')),
})
@ex.automain
def main(_run):
proj = MyProject.from_run(_run)
proj.train()
assert proj.global_step == N_GLOBAL_ITERATIONS
ex.run()
def test_delete(delete_db_loader, delete_mongo_observer):
# Add experiment to db.
ex = Experiment("to be deleted")
ex.observers.append(delete_mongo_observer)
ex.add_config({"value": 1})
def run(value, _run):
_run.log_scalar("test_metric", 1)
_run.add_artifact(__file__)
return value
ex.main(run)
ex.run()
# Retrieve and delete experiment.
exp = delete_db_loader.find_by_id(1)
exp.delete(confirmed=True)
# Make sure experiment cannot be retrieved again.
with raises(ValueError):
exp = delete_db_loader.find_by_id(1)
def test_cifar(tmpdir):
ex = Experiment()
ex.add_config({
'batch_size': 5,
'n_global_iterations': 3,
'tensorboard_dir': None,
'device': 'cuda:0' if torch.cuda.is_available() else 'cpu',
'model_dir': str(tmpdir.join('models')),
})
@ex.automain
def main(_run):
proj = CifarProject.from_run(_run)
print(proj.model._device_args, proj.model._device_kwargs)
proj.test()
proj.train()
print(proj.model._device_args, proj.model._device_kwargs)
proj.test()
ex.run()
def test_run_waits_for_running_queue_observer_after_failure():
queue_observer_with_long_interval = QueueObserver(mock.MagicMock(),
interval=1,
retry_interval=0.01)
ex = Experiment("ator3000")
ex.observers.append(queue_observer_with_long_interval)
@ex.main
def main():
raise Exception("fatal error")
try:
ex.run()
except:
pass
assert (queue_observer_with_long_interval._covered_observer.
method_calls[-1][0] == "failed_event")
def test_mlproject_log_iterations(tmpdir):
# TODO:
N_GLOBAL_ITERATIONS = 10
ex = Experiment()
ex.add_config({
'batch_size': 5,
'n_global_iterations': N_GLOBAL_ITERATIONS,
'log_iteration_scalars': 2,
'log_iteration_all': 5,
'tensorboard_dir': None,
'device': 'cuda:0' if torch.cuda.is_available() else 'cpu',
'model_dir': str(tmpdir.join('models')),
})
@ex.automain
def main(_run):
class MyModel(SimpleModel):
def __init__(self):
model = LinearModel()
super().__init__('test', model,
torch.optim.SGD(model.parameters(), lr=0.1),
nn.CrossEntropyLoss())
def train_batch(self, batch):
if proj.epoch_step % proj.config['log_iteration_scalars'] == 0:
self.log = LogLevel.SCALARS
if proj.epoch_step % proj.config['log_iteration_all'] == 0:
self.log = LogLevel.ALL
super().train_batch(batch)
class TestProject(MyProject):
@staticmethod
def get_model(config):
return MyModel()
proj = MyProject.from_run(_run)
proj.train()
assert proj.global_step == N_GLOBAL_ITERATIONS
ex.run()
def test_find_latest__for_multiple_with_newly_added_experiments(recent_db_loader, recent_mongo_observer):
ex = SacredExperiment("most recent")
ex.observers.append(recent_mongo_observer)
ex.add_config({"value": 1})
def run(value, _run):
return value
ex.main(run)
ex.run()
ex = SacredExperiment("new most recent")
ex.observers.append(recent_mongo_observer)
ex.add_config({"value": 2})
ex.main(run)
ex.run()
exps = recent_db_loader.find_latest(2)
assert exps[0].config.value == 2
assert exps[1].config.value == 1
def sacred_hyperopt_objective(params):
"""
Objective to call with hyperopt
that uses sacred to log the experiment results
"""
ex = Experiment('example')
ex.config(expt_config)
ex.main(difficult_optimization_objective)
ex.observers.append(
MongoObserver.create(
url=f"{os.environ['MONGO_WRITE_IP']}:{os.environ['MONGO_PORT']}"))
run = ex.run(config_updates=params, options={"--loglevel": 40})
return {"loss": run.result, "status": hyperopt.STATUS_OK}
def inner(delete_mongo_observer, config_value):
ex = SacredExperiment("name")
ex.observers.append(delete_mongo_observer)
ex.add_config({"value": config_value})
def run_fn(value, _run):
_run.log_scalar("test_metric", 1)
_run.add_artifact(__file__)
return value
ex.main(run_fn)
run = ex.run()
return run._id
def experiment_with_numpy_in_info(info_mongo_observer, info_db_loader) -> sacred.run.Run:
# Add experiment to db.
ex = Experiment("info experiment")
ex.observers.append(info_mongo_observer)
ex.add_config({"value": 1})
def run(value, _run):
_run.info["array"] = np.array([1, 2, 3])
return value
ex.main(run)
yield ex.run()
# Retrieve and delete experiment.
info_db_loader.find_by_id(1).delete(confirmed=True)
def test_info(info_db_loader, info_db_loader_pickled, info_mongo_observer):
# Add experiment to db.
ex = Experiment("info experiment")
ex.observers.append(info_mongo_observer)
ex.add_config({"value": 1})
def run(value, _run):
_run.info["number"] = 13
_run.info["list"] = [1, 2]
_run.info["object"] = Fraction(3, 4)
return value
ex.main(run)
ex.run()
# Retrieve and delete experiment.
exp_unpickled = info_db_loader.find_by_id(1)
exp_pickled = info_db_loader_pickled.find_by_id(1)
assert exp_unpickled.info["number"] == exp_pickled.info["number"] == 13
assert exp_unpickled.info["list"] == exp_pickled.info["list"] == [1, 2]
assert exp_unpickled.info["object"] == Fraction(3, 4)
assert isinstance(exp_pickled.info["object"], collections.abc.Mapping)
exp_unpickled.delete(confirmed=True)
def experiment_with_pandas_in_info(info_mongo_observer, info_db_loader) -> sacred.run.Run:
# Add experiment to db.
ex = Experiment("info experiment")
ex.observers.append(info_mongo_observer)
ex.add_config({"value": 1})
def run(value, _run):
_run.info["dataframe"] = pd.DataFrame({"a": [1, 2, 3], "b": ["1", "2", "3"]})
return value
ex.main(run)
yield ex.run()
# Retrieve and delete experiment.
info_db_loader.find_by_id(1).delete(confirmed=True)
def objective(args_):
global args
global result
global parse_args
args=args_
try:
ex = Experiment('Hyperopt')
ex.observers.append(MongoObserver.create(url=parse_args.mongo_db_address, db_name=parse_args.mongo_db_name))
ex.main(lambda: run_with_global_args())
r = ex.run(config_updates=args)
print(r)
return result
except:
raise
#If we somehow cannot get to the MongoDB server, then continue with the experiment
print("Running without Sacred")
run_with_global_args()
print 'Mean training accuracy: %s' % training_accuracy
print
for hashtag_name in testing_hashtag_names:
accuracy, _ = predict_on_hashtag(sess,
model_vars,
hashtag_name,
HUMOR_TRIAL_TWEET_PAIR_EMBEDDING_DIR,
trial_hashtag_datas)
print 'Hashtag %s accuracy: %s' % (hashtag_name, accuracy)
accuracies.append(accuracy)
test_accuracy = np.mean(accuracies)
print 'Final test accuracy: %s' % test_accuracy
return {'train_accuracy': training_accuracy,
'test_accuracy': test_accuracy}
@ex.main
def main(learning_rate, num_epochs, dropout, use_emb_model,
use_char_model, model_save_dir, hidden_dim_size, leave_out_hashtags=[]):
load_build_train_and_predict(learning_rate, num_epochs, dropout, use_emb_model,
use_char_model, model_save_dir, hidden_dim_size, leave_out_hashtags=[])
if __name__ == '__main__':
num_experiments_run = 1
for index in range(num_experiments_run):
print 'Experiment: %s' % index
r = ex.run()
def main():
import argparse
parser = argparse.ArgumentParser(description='Two layer linear regression')
parser.add_argument("image_feature_file_train",
type=str,
help="Image Feature file for the training set")
parser.add_argument("text_feature_file_train",
type=str,
help="Text Feature file for the training set")
parser.add_argument("image_feature_file_test",
type=str,
help="Image Feature file for the test set")
parser.add_argument("text_feature_file_test",
type=str,
help="Text Feature file for the test set")
parser.add_argument("word_vector_file",
type=str,
help="Text file containing the word vectors")
# Optional Args
parser.add_argument("--learning_rate",
type=float,
default=0.001,
help="Learning Rate")
parser.add_argument("--num_epochs",
type=int,
default=200,
help="Number of epochs to run for")
parser.add_argument("--batch_size",
type=int,
default=128,
help="Batch size to use for training")
parser.add_argument("--model_type",
type=str,
default="multihot",
choices=['multihot', 'naivesum', 'wdv', 'negsampling', 'fast0tag'],
help="Loss function to use for training")
parser.add_argument("--in_memory",
action='store_true',
default="store_false",
help="Load training image features into memory for faster training")
parser.add_argument("--model_input_path",
type=str,
default=None,
help="Model input path (to continue training)")
parser.add_argument("--model_output_path",
type=str,
default=None,
help="Model output path (to save training)")
# new args
parser.add_argument("--hidden_units",
type=str,
default="200",
help="Define a neural network as comma separated layer sizes. If log-reg, then set to '0'.")
parser.add_argument("--cross_eval",
action="store_true",
default=False,
help="Use if test dataset is different from training dataset")
parser.add_argument("--word_vector_type",
type=str,
choices=[t.name for t in WordVectorTypes],
help="Format of word_vector_file")
parser.add_argument("--epoch_verbosity",
type=int,
default=10,
help="Epoch verbosity rate")
parser.add_argument("--verbose_eval",
action="store_true",
default=False,
help="Use to run evaluation against test data every epoch_verbosity")
parser.add_argument("--optim_words",
action="store_true",
default=False,
help="If using negsampling model_type, use to jointly optimize words")
parser.add_argument("--ignore_posbatch",
action="store_true",
default=False,
help="Sample, ignoring from positive batch instead of examples. This should be taken out in future iters.")
parser.add_argument("--joint_factor",
type=float,
default=1.0,
help="Multiplier for learning rate in updating joint optimization")
parser.add_argument("--use_batch_norm",
action="store_true",
default=False,
help="Do we want to use batch normalization? Default is False")
parser.add_argument("--opt_type",
type=str,
default="adam",
help="What type of optimizer would you like? Choices are (adam,sgd)")
parser.add_argument("--weight_decay",
type=float,
default=0.0,
help="Weight decay to manually decay every 10 epochs. Default=0 for no decay.")
parser.add_argument("--scale_words",
type=float,
default=1.0,
help="Scale the word vectors. If set to zero, scale by L2-norm. Otherwise, wordvec=scale x wordvec")
parser.add_argument("--scale_images",
type=float,
default=1.0,
help="Scale the word vectors. If set to zero, scale by L2-norm. Otherwise, imvec=scale x imvec. ")
parser.add_argument("--fast_sample",
action="store_true",
default=False,
help="Fast sample based on distribution, only use in large dictionaries")
args = parser.parse_args()
try:
# Sacred Imports
from sacred import Experiment
from sacred.observers import MongoObserver
from sacred.initialize import Scaffold
# Monkey patch to avoid having to declare all our variables
def noop(item):
pass
Scaffold._warn_about_suspicious_changes = noop
ex = Experiment('Attalos')
ex.observers.append(MongoObserver.create(url=os.environ['MONGO_DB_URI'],
db_name='attalos_experiment'))
ex.main(lambda: convert_args_and_call_model(args))
ex.run(config_updates=args.__dict__)
except ImportError:
# We don't have sacred, just run the script
logger.warn('Not using Sacred. Your results will not be saved')
convert_args_and_call_model(args)
def test_missing_config_raises():
ex = Experiment('exp')
ex.main(lambda a: None)
with pytest.raises(MissingConfigError):
ex.run()
def main():
import argparse
parser = argparse.ArgumentParser(description='Two layer linear regression')
parser.add_argument("image_feature_file_train",
type=str,
help="Image Feature file for the training set")
parser.add_argument("text_feature_file_train",
type=str,
help="Text Feature file for the training set")
parser.add_argument("image_feature_file_test",
type=str,
help="Image Feature file for the test set")
parser.add_argument("text_feature_file_test",
type=str,
help="Text Feature file for the test set")
parser.add_argument("word_vector_file",
type=str,
help="Text file containing the word vectors")
# Optional Args
parser.add_argument("--learning_rate",
type=float,
default=.001,
help="Learning Rate")
parser.add_argument("--epochs",
type=int,
default=200,
help="Number of epochs to run for")
parser.add_argument("--batch_size",
type=int,
default=128,
help="Batch size to use for training")
parser.add_argument("--network",
type=str,
default="200,200",
help="Define a neural network as comma separated layer sizes")
parser.add_argument("--model_type",
type=str,
default="mse",
choices=['mse', 'negsampling', 'fast0tag'],
help="Loss function to use for training")
parser.add_argument("--in_memory",
action='store_true',
default="store_false",
help="Load training image features into memory for faster training")
parser.add_argument("--model_input_path",
type=str,
default=None,
help="Model input path (to continue training)")
parser.add_argument("--model_output_path",
type=str,
default=None,
help="Model output path (to save training)")
parser.add_argument("--max_pos",
type=int,
default=5,
help="Max number of positive examples")
parser.add_argument("--max_neg",
type=int,
default=10,
help="Max number of negative examples")
global args
args = parser.parse_args()
try:
# Sacred Imports
from sacred import Experiment
from sacred.observers import MongoObserver
from sacred.initialize import Scaffold
# Monkey patch to avoid having to declare all our variables
def noop(item):
pass
Scaffold._warn_about_suspicious_changes = noop
ex = Experiment('Regress2sum')
ex.observers.append(MongoObserver.create(url=os.environ['MONGO_DB_URI'],
db_name='attalos_experiment'))
ex.main(lambda: convert_args_and_call_model())
ex.run(config_updates=args.__dict__)
except ImportError:
# We don't have sacred, just run the script
convert_args_and_call_model()
from sklearn import svm, datasets, model_selection
ex = Experiment("svm")
ex.observers.append(
FileStorageObserver.create("my_runs")
)
ex.add_config({ # Configuration is explicitly defined as dictionary.
"C": 1.0,
"gamma": 0.7,
"kernel": "rbf",
"seed": 42
})
def get_model(C, gamma, kernel):
return svm.SVC(C=C, kernel=kernel, gamma=gamma)
@ex.main # Using main, command-line arguments will not be interpreted in any special way.
def run(_config):
X, y = datasets.load_breast_cancer(return_X_y=True)
X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.2)
clf = get_model(_config["C"], _config["gamma"], _config["kernel"]) # Parameters are passed explicitly.
clf.fit(X_train, y_train)
return clf.score(X_test, y_test)
if __name__ == "__main__":
ex.run()
