def run(config_updates, mongodb_url="localhost:27017"):
"""Run a single experiment with the given configuration
Args:
config_updates (dict): Configuration updates
mongodb_url (str): MongoDB URL, or None if no Mongo observer should be used for
this run
"""
# Dynamically bind experiment config and main function
ex = Experiment()
ex.config(base_config)
ex.main(element_world_v4)
# Attach MongoDB observer if necessary
if mongodb_url is not None and not ex.observers:
ex.observers.append(MongoObserver(url=mongodb_url))
# Suppress warnings about padded MPDs
with warnings.catch_warnings():
warnings.filterwarnings(action="ignore", category=PaddedMDPWarning)
# Run the experiment
run = ex.run(config_updates=config_updates
) # , options={"--loglevel": "ERROR"})
# Return the result
return run.result
def objective(args_):
# arguments to pass as config_updates dict
global args
# result to pass to hyperopt
global result
# command-line arguments
global parse_args
try:
ex = Experiment('Hyperopt')
logger.debug("Adding observer for {}, DB {}".format(parse_args.mongo_db_address,parse_args.mongo_db_name))
ex.observers.append(MongoObserver.create(url=parse_args.mongo_db_address, db_name=parse_args.mongo_db_name))
pythia_args = make_args_for_pythia(args_)
args = mp.get_args(**pythia_args)
ex.main(run_with_global_args)
r = ex.run(config_updates=pythia_args)
logger.debug("Experiment result: {}\n"
"Report to hyperopt: {}".format(r.result, result))
return result
except:
raise
#If we somehow cannot get to the MongoDB server, then continue with the experiment
logger.warning("Running without Sacred")
run_with_global_args()
def test_named_config_not_found_raises():
ex = Experiment('exp')
ex.main(lambda: None)
with pytest.raises(NamedConfigNotFoundError,
match='Named config not found: "not_there". '
'Available config values are:'):
ex.run(named_configs=('not_there', ))
def sacred_run(command, name='train', default_configs_root='default_configs'):
ex = Experiment(name)
def default_config(config, command_name, logger):
default_config = {}
# loading modules default config
fn = join(default_configs_root, 'modules.yaml')
default_modules_args = load_config_file(fn)
default_config['modules'] = {}
for module, module_config in config['modules'].items():
default_args = default_modules_args[module_config['_name']]
default_config['modules'][module] = default_args
# loading datasets default config
fn = join(default_configs_root, 'datasets.yaml')
default_datasets_args = load_config_file(fn)
default_config['datasets'] = {}
for dataset, dataset_config in config['datasets'].items():
default_args = default_datasets_args[dataset_config['_name']]
default_config['datasets'][dataset] = default_args
# loading experiment default configs
fn = join(default_configs_root, 'experiment.yaml')
default_experiment_args = load_config_file(fn)
default_args = default_experiment_args[config['experiment']['_name']]
default_config['experiment'] = default_args
return default_config
ex.config_hook(default_config)
ex.main(command)
ex.run_commandline()
def test_named_config_not_found_raises():
ex = Experiment('exp')
ex.main(lambda: None)
with pytest.raises(NamedConfigNotFoundError,
match='Named config not found: "not_there". '
'Available config values are:'):
ex.run(named_configs=('not_there',))
def sacred_run(command, name='train', default_configs_root='default_configs'):
ex = Experiment(name)
def default_config(config, command_name, logger):
default_config = {}
components = ['datasets', 'modules', 'optimizers']
for comp in components:
file_name = '{}.yaml'.format(comp)
default_file_path = join(default_configs_root, file_name)
default_config[comp] = get_component_configs(config, comp, default_file_path)
# loading experiment default configs
fn = join(default_configs_root, 'experiment.yaml')
default_experiment_args = load_config_file(fn)
default_args = default_experiment_args[config['experiment']['_name']]
default_config['experiment'] = default_args
return default_config
ex.config_hook(default_config)
ex.main(command)
ex.run_commandline()
def objective(args_):
# arguments to pass as config_updates dict
global args
# result to pass to hyperopt
global result
# command-line arguments
global parse_args
try:
ex = Experiment('Hyperopt')
logger.debug("Adding observer for {}, DB {}".format(
parse_args.mongo_db_address, parse_args.mongo_db_name))
ex.observers.append(
MongoObserver.create(url=parse_args.mongo_db_address,
db_name=parse_args.mongo_db_name))
pythia_args = make_args_for_pythia(args_)
args = mp.get_args(**pythia_args)
ex.main(run_with_global_args)
r = ex.run(config_updates=pythia_args)
logger.debug("Experiment result: {}\n"
"Report to hyperopt: {}".format(r.result, result))
return result
except:
raise
#If we somehow cannot get to the MongoDB server, then continue with the experiment
logger.warning("Running without Sacred")
run_with_global_args()
def test_circular_dependency_raises():
# create experiment with circular dependency
ing = Ingredient('ing')
ex = Experiment('exp', ingredients=[ing])
ex.main(lambda: None)
ing.ingredients.append(ex)
# run and see if it raises
with pytest.raises(CircularDependencyError, match='exp->ing->exp'):
ex.run()
def test_config_added_raises():
ex = Experiment('exp')
ex.main(lambda: None)
with pytest.raises(
ConfigAddedError,
match=r'Added new config entry that is not used anywhere.*\n'
r'\s*Conflicting configuration values:\n'
r'\s*a=42'):
ex.run(config_updates={'a': 42})
def test_config_added_raises():
ex = Experiment('exp')
ex.main(lambda: None)
with pytest.raises(
ConfigAddedError,
match=r'Added new config entry that is not used anywhere.*\n'
r'\s*Conflicting configuration values:\n'
r'\s*a=42'):
ex.run(config_updates={'a': 42})
def test_circular_dependency_raises():
# create experiment with circular dependency
ing = Ingredient('ing')
ex = Experiment('exp', ingredients=[ing])
ex.main(lambda: None)
ing.ingredients.append(ex)
# run and see if it raises
with pytest.raises(CircularDependencyError, match='exp->ing->exp'):
ex.run()
def test_config_added_raises():
ex = Experiment("exp")
ex.main(lambda: None)
with pytest.raises(
ConfigAddedError,
match=r"Added new config entry that is not used anywhere.*\n"
r"\s*Conflicting configuration values:\n"
r"\s*a=42",
):
ex.run(config_updates={"a": 42})
def inner(delete_mongo_observer, config_value):
ex = SacredExperiment("name")
ex.observers.append(delete_mongo_observer)
ex.add_config({"value": config_value})
def run_fn(value, _run):
_run.log_scalar("test_metric", 1)
_run.add_artifact(__file__)
return value
ex.main(run_fn)
run = ex.run()
return run._id
def sacred_hyperopt_objective(params):
"""
Objective to call with hyperopt
that uses sacred to log the experiment results
"""
ex = Experiment('example')
ex.config(expt_config)
ex.main(difficult_optimization_objective)
ex.observers.append(
MongoObserver.create(
url=f"{os.environ['MONGO_WRITE_IP']}:{os.environ['MONGO_PORT']}"))
run = ex.run(config_updates=params, options={"--loglevel": 40})
return {"loss": run.result, "status": hyperopt.STATUS_OK}
def train(build_model, dataset, hparams, logdir, expname, observer):
# Location to save trained models
output_dir = path.join(logdir, "test")
# Create the actual train function to run
def train(_run):
model = build_model(hparams, **dataset.preprocessing.kwargs)
# Make optimizer
optimizer = tf.keras.optimizers.Adam(hparams.lr,
beta_1=hparams.beta_1,
beta_2=hparams.beta_2)
model.compile(optimizer=optimizer,
loss="categorical_crossentropy",
metrics=["categorical_accuracy"])
# model.summary()
train_log = model.fit(
dataset.train_data(hparams.batch_size),
epochs=hparams.epochs,
steps_per_epoch=dataset.train_examples // hparams.batch_size,
validation_data=dataset.validation_data(hparams.batch_size),
validation_steps=dataset.validation_examples // hparams.batch_size)
# Log the performace values to sacred
for (metric, values) in train_log.history.items():
for (idx, value) in enumerate(values):
_run.log_scalar(metric, value, idx)
# TODO: Save model
# Build config
config = {}
for (k, v) in hparams.items():
config[k] = v
config["model"] = build_model.__name__
config["dataset"] = dataset.dataset_name
# Setup sacred experiment
ex = Experiment(expname)
# Disable capturing the output from window
ex.captured_out_filter = lambda captured_output: "Output capturing turned off."
ex.main(train)
ex.add_config(config)
# build argv for sacred -- hacky way!
_argv = f"{ex.default_command} --{observer}"
ex.run_commandline(argv=_argv)
def create_base_experiment(
sacred_args,
name=None): #path=Config.LOG_DIR, db_name=Config.DB_NAME):
ex = Experiment(name)
print(name)
ex.capture(set_device)
ex.main(main)
ex.capture(run_train)
ex.command(evaluate_experiment)
ex.command(test_config)
ex.command(show_options)
ex.command(evaluate_checkpoint)
# set default values
ex = config_builder.build(ex)
# set observers but check if maybe sacred will create them
# on its own
""" TODO
Problem is that we create the experiment before the command line is parsed by sacred.
But then we cannot set default values without using a shell script. Or
modify the file path for the logger."""
if Config.LOG_DIR is not None and '-F' not in sacred_args:
path = Config.LOG_DIR
if name is not None:
path = os.path.join(path, name)
file_ob = FileStorageObserver.create(path)
ex.observers.append(file_ob)
if Config.SLACK_WEBHOOK_URL != "":
slack_ob = SlackObserver(Config.SLACK_WEBHOOK_URL)
ex.observers.append(slack_ob)
if (Config.MONGO_DB_NAME is not None and Config.MONGO_DB_NAME != "") \
and '-m' not in sacred_args:
if Config.MONGO_USER != "":
mongo_ob = MongoObserver.create(username=Config.MONGO_USER,
password=Config.MONGO_PW,
url=Config.MONGO_URL,
authMechanism="SCRAM-SHA-256",
db_name=Config.MONGO_DB_NAME)
else:
mongo_ob = MongoObserver.create(url=Config.MONGO_URL,
db_name=Config.MONGO_DB_NAME)
ex.observers.append(mongo_ob)
return ex
def experiment_with_pandas_in_info(info_mongo_observer, info_db_loader) -> sacred.run.Run:
# Add experiment to db.
ex = Experiment("info experiment")
ex.observers.append(info_mongo_observer)
ex.add_config({"value": 1})
def run(value, _run):
_run.info["dataframe"] = pd.DataFrame({"a": [1, 2, 3], "b": ["1", "2", "3"]})
return value
ex.main(run)
yield ex.run()
# Retrieve and delete experiment.
info_db_loader.find_by_id(1).delete(confirmed=True)
def sacred_run(command, default_configs_root='configs/default'):
ex = Experiment('default')
@ex.config_hook
def default_config(config, command_name, logger):
default_config = {}
for comp, conf in config.items():
default_file_path = os.path.join(default_configs_root,
f'{comp}.yaml')
default_config[comp] = get_component_configs(
config, comp, default_file_path)
return default_config
ex.main(command)
ex.run_commandline()
def experiment_with_numpy_in_info(info_mongo_observer, info_db_loader) -> sacred.run.Run:
# Add experiment to db.
ex = Experiment("info experiment")
ex.observers.append(info_mongo_observer)
ex.add_config({"value": 1})
def run(value, _run):
_run.info["array"] = np.array([1, 2, 3])
return value
ex.main(run)
yield ex.run()
# Retrieve and delete experiment.
info_db_loader.find_by_id(1).delete(confirmed=True)
def sacred_run(command, default_configs_root='configs/default'):
ex = Experiment('default')
files = glob.glob('./src/**/*.py', recursive=True)
for f in files:
ex.add_source_file(f)
@ex.config_hook
def default_config(config, command_name, logger):
default_config = {}
for comp, conf in config.items():
default_file_path = os.path.join(default_configs_root, f'{comp}.yaml')
default_config[comp] = get_component_configs(config, comp, default_file_path)
return default_config
ex.main(command)
ex.run_commandline()
def test_delete(delete_db_loader, delete_mongo_observer):
# Add experiment to db.
ex = Experiment("to be deleted")
ex.observers.append(delete_mongo_observer)
ex.add_config({"value": 1})
def run(value, _run):
_run.log_scalar("test_metric", 1)
_run.add_artifact(__file__)
return value
ex.main(run)
ex.run()
# Retrieve and delete experiment.
exp = delete_db_loader.find_by_id(1)
exp.delete(confirmed=True)
# Make sure experiment cannot be retrieved again.
with raises(ValueError):
exp = delete_db_loader.find_by_id(1)
def objective(args_):
global args
global result
global parse_args
args=args_
try:
ex = Experiment('Hyperopt')
ex.observers.append(MongoObserver.create(url=parse_args.mongo_db_address, db_name=parse_args.mongo_db_name))
ex.main(lambda: run_with_global_args())
r = ex.run(config_updates=args)
print(r)
return result
except:
raise
#If we somehow cannot get to the MongoDB server, then continue with the experiment
print("Running without Sacred")
run_with_global_args()
def run(config, mongodb_url="localhost:27017"):
"""Run a single experiment with the given configuration"""
# Dynamically bind experiment config and main function
ex = Experiment()
ex.config(base_config)
ex.main(canonical_puddle_world)
# Attach MongoDB observer if necessary
if not ex.observers:
ex.observers.append(MongoObserver(url=mongodb_url))
# Suppress warnings about padded MPDs
with warnings.catch_warnings():
warnings.filterwarnings(action="ignore", category=PaddedMDPWarning)
# Run the experiment
run = ex.run(config_updates=config, options={"--loglevel": "ERROR"})
# Return the result
return run.result
class SacredExperiment(object):
def __init__(
self,
experiment_name,
experiment_dir,
observer_type="file_storage",
mongo_url=None,
db_name=None,
):
"""__init__
:param experiment_name: The name of the experiments.
:param experiment_dir: The directory to store all the results of the experiments(This is for file_storage).
:param observer_type: The observer to record the results: the `file_storage` or `mongo`
:param mongo_url: The mongo url(for mongo observer)
:param db_name: The mongo url(for mongo observer)
"""
self.experiment_name = experiment_name
self.experiment = Experiment(self.experiment_name)
self.experiment_dir = experiment_dir
self.experiment.logger = get_module_logger("Sacred")
self.observer_type = observer_type
self.mongo_db_url = mongo_url
self.mongo_db_name = db_name
self._setup_experiment()
def _setup_experiment(self):
if self.observer_type == "file_storage":
file_storage_observer = FileStorageObserver.create(
basedir=self.experiment_dir)
self.experiment.observers.append(file_storage_observer)
elif self.observer_type == "mongo":
mongo_observer = MongoObserver.create(url=self.mongo_db_url,
db_name=self.mongo_db_name)
self.experiment.observers.append(mongo_observer)
else:
raise NotImplementedError("Unsupported observer type: {}".format(
self.observer_type))
def add_artifact(self, filename):
self.experiment.add_artifact(filename)
def add_info(self, key, value):
self.experiment.info[key] = value
def main_wrapper(self, func):
return self.experiment.main(func)
def config_wrapper(self, func):
return self.experiment.config(func)
def test_find_latest__for_multiple_with_newly_added_experiments(recent_db_loader, recent_mongo_observer):
ex = SacredExperiment("most recent")
ex.observers.append(recent_mongo_observer)
ex.add_config({"value": 1})
def run(value, _run):
return value
ex.main(run)
ex.run()
ex = SacredExperiment("new most recent")
ex.observers.append(recent_mongo_observer)
ex.add_config({"value": 2})
ex.main(run)
ex.run()
exps = recent_db_loader.find_latest(2)
assert exps[0].config.value == 2
assert exps[1].config.value == 1
def objective(args_):
global args
global result
global parse_args
args = args_
try:
ex = Experiment('Hyperopt')
ex.observers.append(
MongoObserver.create(url=parse_args.mongo_db_address,
db_name=parse_args.mongo_db_name))
ex.main(lambda: run_with_global_args())
r = ex.run(config_updates=args)
print(r)
return result
except:
raise
#If we somehow cannot get to the MongoDB server, then continue with the experiment
print("Running without Sacred")
run_with_global_args()
def test_info(info_db_loader, info_db_loader_pickled, info_mongo_observer):
# Add experiment to db.
ex = Experiment("info experiment")
ex.observers.append(info_mongo_observer)
ex.add_config({"value": 1})
def run(value, _run):
_run.info["number"] = 13
_run.info["list"] = [1, 2]
_run.info["object"] = Fraction(3, 4)
return value
ex.main(run)
ex.run()
# Retrieve and delete experiment.
exp_unpickled = info_db_loader.find_by_id(1)
exp_pickled = info_db_loader_pickled.find_by_id(1)
assert exp_unpickled.info["number"] == exp_pickled.info["number"] == 13
assert exp_unpickled.info["list"] == exp_pickled.info["list"] == [1, 2]
assert exp_unpickled.info["object"] == Fraction(3, 4)
assert isinstance(exp_pickled.info["object"], collections.abc.Mapping)
exp_unpickled.delete(confirmed=True)
def main():
import argparse
parser = argparse.ArgumentParser(description='Two layer linear regression')
parser.add_argument("image_feature_file_train",
type=str,
help="Image Feature file for the training set")
parser.add_argument("text_feature_file_train",
type=str,
help="Text Feature file for the training set")
parser.add_argument("image_feature_file_test",
type=str,
help="Image Feature file for the test set")
parser.add_argument("text_feature_file_test",
type=str,
help="Text Feature file for the test set")
parser.add_argument("word_vector_file",
type=str,
help="Text file containing the word vectors")
# Optional Args
parser.add_argument("--learning_rate",
type=float,
default=.001,
help="Learning Rate")
parser.add_argument("--epochs",
type=int,
default=200,
help="Number of epochs to run for")
parser.add_argument("--batch_size",
type=int,
default=128,
help="Batch size to use for training")
parser.add_argument("--network",
type=str,
default="200,200",
help="Define a neural network as comma separated layer sizes")
parser.add_argument("--model_type",
type=str,
default="mse",
choices=['mse', 'negsampling', 'fast0tag'],
help="Loss function to use for training")
parser.add_argument("--in_memory",
action='store_true',
default="store_false",
help="Load training image features into memory for faster training")
parser.add_argument("--model_input_path",
type=str,
default=None,
help="Model input path (to continue training)")
parser.add_argument("--model_output_path",
type=str,
default=None,
help="Model output path (to save training)")
parser.add_argument("--max_pos",
type=int,
default=5,
help="Max number of positive examples")
parser.add_argument("--max_neg",
type=int,
default=10,
help="Max number of negative examples")
global args
args = parser.parse_args()
try:
# Sacred Imports
from sacred import Experiment
from sacred.observers import MongoObserver
from sacred.initialize import Scaffold
# Monkey patch to avoid having to declare all our variables
def noop(item):
pass
Scaffold._warn_about_suspicious_changes = noop
ex = Experiment('Regress2sum')
ex.observers.append(MongoObserver.create(url=os.environ['MONGO_DB_URI'],
db_name='attalos_experiment'))
ex.main(lambda: convert_args_and_call_model())
ex.run(config_updates=args.__dict__)
except ImportError:
# We don't have sacred, just run the script
convert_args_and_call_model()
def test_missing_config_raises():
ex = Experiment('exp')
ex.main(lambda a: None)
with pytest.raises(MissingConfigError):
ex.run()
def main():
import argparse
parser = argparse.ArgumentParser(description='Two layer linear regression')
parser.add_argument("image_feature_file_train",
type=str,
help="Image Feature file for the training set")
parser.add_argument("text_feature_file_train",
type=str,
help="Text Feature file for the training set")
parser.add_argument("image_feature_file_test",
type=str,
help="Image Feature file for the test set")
parser.add_argument("text_feature_file_test",
type=str,
help="Text Feature file for the test set")
parser.add_argument("word_vector_file",
type=str,
help="Text file containing the word vectors")
# Optional Args
parser.add_argument("--learning_rate",
type=float,
default=.001,
help="Learning Rate")
parser.add_argument("--epochs",
type=int,
default=200,
help="Number of epochs to run for")
parser.add_argument("--batch_size",
type=int,
default=128,
help="Batch size to use for training")
parser.add_argument(
"--network",
type=str,
default="200,200",
help="Define a neural network as comma separated layer sizes")
parser.add_argument("--model_type",
type=str,
default="mse",
choices=['mse', 'negsampling', 'fast0tag'],
help="Loss function to use for training")
parser.add_argument(
"--in_memory",
action='store_true',
default="store_false",
help="Load training image features into memory for faster training")
parser.add_argument("--model_input_path",
type=str,
default=None,
help="Model input path (to continue training)")
parser.add_argument("--model_output_path",
type=str,
default=None,
help="Model output path (to save training)")
parser.add_argument("--max_pos",
type=int,
default=5,
help="Max number of positive examples")
parser.add_argument("--max_neg",
type=int,
default=10,
help="Max number of negative examples")
global args
args = parser.parse_args()
try:
# Sacred Imports
from sacred import Experiment
from sacred.observers import MongoObserver
from sacred.initialize import Scaffold
# Monkey patch to avoid having to declare all our variables
def noop(item):
pass
Scaffold._warn_about_suspicious_changes = noop
ex = Experiment('Regress2sum')
ex.observers.append(
MongoObserver.create(url=os.environ['MONGO_DB_URI'],
db_name='attalos_experiment'))
ex.main(lambda: convert_args_and_call_model())
ex.run(config_updates=args.__dict__)
except ImportError:
# We don't have sacred, just run the script
convert_args_and_call_model()
def main():
import argparse
parser = argparse.ArgumentParser(description='Two layer linear regression')
parser.add_argument("image_feature_file_train",
type=str,
help="Image Feature file for the training set")
parser.add_argument("text_feature_file_train",
type=str,
help="Text Feature file for the training set")
parser.add_argument("image_feature_file_test",
type=str,
help="Image Feature file for the test set")
parser.add_argument("text_feature_file_test",
type=str,
help="Text Feature file for the test set")
parser.add_argument("word_vector_file",
type=str,
help="Text file containing the word vectors")
# Optional Args
parser.add_argument("--learning_rate",
type=float,
default=0.001,
help="Learning Rate")
parser.add_argument("--num_epochs",
type=int,
default=200,
help="Number of epochs to run for")
parser.add_argument("--batch_size",
type=int,
default=128,
help="Batch size to use for training")
parser.add_argument("--model_type",
type=str,
default="multihot",
choices=['multihot', 'naivesum', 'wdv', 'negsampling', 'fast0tag'],
help="Loss function to use for training")
parser.add_argument("--in_memory",
action='store_true',
default="store_false",
help="Load training image features into memory for faster training")
parser.add_argument("--model_input_path",
type=str,
default=None,
help="Model input path (to continue training)")
parser.add_argument("--model_output_path",
type=str,
default=None,
help="Model output path (to save training)")
# new args
parser.add_argument("--hidden_units",
type=str,
default="200",
help="Define a neural network as comma separated layer sizes. If log-reg, then set to '0'.")
parser.add_argument("--cross_eval",
action="store_true",
default=False,
help="Use if test dataset is different from training dataset")
parser.add_argument("--word_vector_type",
type=str,
choices=[t.name for t in WordVectorTypes],
help="Format of word_vector_file")
parser.add_argument("--epoch_verbosity",
type=int,
default=10,
help="Epoch verbosity rate")
parser.add_argument("--verbose_eval",
action="store_true",
default=False,
help="Use to run evaluation against test data every epoch_verbosity")
parser.add_argument("--optim_words",
action="store_true",
default=False,
help="If using negsampling model_type, use to jointly optimize words")
parser.add_argument("--ignore_posbatch",
action="store_true",
default=False,
help="Sample, ignoring from positive batch instead of examples. This should be taken out in future iters.")
parser.add_argument("--joint_factor",
type=float,
default=1.0,
help="Multiplier for learning rate in updating joint optimization")
parser.add_argument("--use_batch_norm",
action="store_true",
default=False,
help="Do we want to use batch normalization? Default is False")
parser.add_argument("--opt_type",
type=str,
default="adam",
help="What type of optimizer would you like? Choices are (adam,sgd)")
parser.add_argument("--weight_decay",
type=float,
default=0.0,
help="Weight decay to manually decay every 10 epochs. Default=0 for no decay.")
parser.add_argument("--scale_words",
type=float,
default=1.0,
help="Scale the word vectors. If set to zero, scale by L2-norm. Otherwise, wordvec=scale x wordvec")
parser.add_argument("--scale_images",
type=float,
default=1.0,
help="Scale the word vectors. If set to zero, scale by L2-norm. Otherwise, imvec=scale x imvec. ")
parser.add_argument("--fast_sample",
action="store_true",
default=False,
help="Fast sample based on distribution, only use in large dictionaries")
args = parser.parse_args()
try:
# Sacred Imports
from sacred import Experiment
from sacred.observers import MongoObserver
from sacred.initialize import Scaffold
# Monkey patch to avoid having to declare all our variables
def noop(item):
pass
Scaffold._warn_about_suspicious_changes = noop
ex = Experiment('Attalos')
ex.observers.append(MongoObserver.create(url=os.environ['MONGO_DB_URI'],
db_name='attalos_experiment'))
ex.main(lambda: convert_args_and_call_model(args))
ex.run(config_updates=args.__dict__)
except ImportError:
# We don't have sacred, just run the script
logger.warn('Not using Sacred. Your results will not be saved')
convert_args_and_call_model(args)
FLAGS.output_path = os.path.join(FLAGS.output_path,
FLAGS.experiment_name + "_" + time_str)
FLAGS.log_path = os.path.join(FLAGS.output_path, "logs")
FLAGS.checkpoints_path = os.path.join(FLAGS.output_path, "checkpoints")
ex = Experiment(FLAGS.experiment_name)
if not os.path.exists(FLAGS.output_path):
makedirs(FLAGS.output_path)
makedirs(os.path.join(FLAGS.log_path))
makedirs(os.path.join(FLAGS.checkpoints_path))
mongo_conf = FLAGS.mongodb
if mongo_conf != None:
mongo_conf = FLAGS.mongodb.split(":")
ex.observers.append(
MongoObserver.create(url=":".join(mongo_conf[:-1]),
db_name=mongo_conf[-1]))
logging(str(FLAGS), FLAGS.log_path)
ex.main(run_experiment)
cfg = vars(FLAGS)
cfg["FLAGS"] = FLAGS
ex.add_config(cfg)
r = ex.run()
# KD-GIP and KP-GS-domain
# kernels, followed closely by KD-GIP and KP-SW+ k
def test_missing_config_raises():
ex = Experiment('exp')
ex.main(lambda a: None)
with pytest.raises(MissingConfigError):
ex.run()
def main():
import argparse
parser = argparse.ArgumentParser(description='Two layer linear regression')
parser.add_argument("image_feature_file_train",
type=str,
help="Image Feature file for the training set")
parser.add_argument("text_feature_file_train",
type=str,
help="Text Feature file for the training set")
parser.add_argument("image_feature_file_test",
type=str,
help="Image Feature file for the test set")
parser.add_argument("text_feature_file_test",
type=str,
help="Text Feature file for the test set")
parser.add_argument("word_vector_file",
type=str,
help="Text file containing the word vectors")
# Optional Args
parser.add_argument("--learning_rate",
type=float,
default=0.001,
help="Learning Rate")
parser.add_argument("--num_epochs",
type=int,
default=200,
help="Number of epochs to run for")
parser.add_argument("--batch_size",
type=int,
default=128,
help="Batch size to use for training")
parser.add_argument(
"--model_type",
type=str,
default="multihot",
choices=['multihot', 'naivesum', 'wdv', 'negsampling', 'fast0tag'],
help="Loss function to use for training")
parser.add_argument(
"--in_memory",
action='store_true',
default="store_false",
help="Load training image features into memory for faster training")
parser.add_argument("--model_input_path",
type=str,
default=None,
help="Model input path (to continue training)")
parser.add_argument("--model_output_path",
type=str,
default=None,
help="Model output path (to save training)")
# new args
parser.add_argument(
"--hidden_units",
type=str,
default="200",
help=
"Define a neural network as comma separated layer sizes. If log-reg, then set to '0'."
)
parser.add_argument(
"--cross_eval",
action="store_true",
default=False,
help="Use if test dataset is different from training dataset")
parser.add_argument("--word_vector_type",
type=str,
choices=[t.name for t in WordVectorTypes],
help="Format of word_vector_file")
parser.add_argument("--epoch_verbosity",
type=int,
default=10,
help="Epoch verbosity rate")
parser.add_argument(
"--verbose_eval",
action="store_true",
default=False,
help="Use to run evaluation against test data every epoch_verbosity")
parser.add_argument(
"--optim_words",
action="store_true",
default=False,
help="If using negsampling model_type, use to jointly optimize words")
parser.add_argument(
"--ignore_posbatch",
action="store_true",
default=False,
help=
"Sample, ignoring from positive batch instead of examples. This should be taken out in future iters."
)
parser.add_argument(
"--joint_factor",
type=float,
default=1.0,
help="Multiplier for learning rate in updating joint optimization")
parser.add_argument(
"--use_batch_norm",
action="store_true",
default=False,
help="Do we want to use batch normalization? Default is False")
parser.add_argument(
"--opt_type",
type=str,
default="adam",
help="What type of optimizer would you like? Choices are (adam,sgd)")
parser.add_argument(
"--weight_decay",
type=float,
default=0.0,
help=
"Weight decay to manually decay every 10 epochs. Default=0 for no decay."
)
parser.add_argument(
"--scale_words",
type=float,
default=1.0,
help=
"Scale the word vectors. If set to zero, scale by L2-norm. Otherwise, wordvec=scale x wordvec"
)
parser.add_argument(
"--scale_images",
type=float,
default=1.0,
help=
"Scale the word vectors. If set to zero, scale by L2-norm. Otherwise, imvec=scale x imvec. "
)
parser.add_argument(
"--fast_sample",
action="store_true",
default=False,
help="Fast sample based on distribution, only use in large dictionaries"
)
args = parser.parse_args()
try:
# Sacred Imports
from sacred import Experiment
from sacred.observers import MongoObserver
from sacred.initialize import Scaffold
# Monkey patch to avoid having to declare all our variables
def noop(item):
pass
Scaffold._warn_about_suspicious_changes = noop
ex = Experiment('Attalos')
ex.observers.append(
MongoObserver.create(url=os.environ['MONGO_DB_URI'],
db_name='attalos_experiment'))
ex.main(lambda: convert_args_and_call_model(args))
ex.run(config_updates=args.__dict__)
except ImportError:
# We don't have sacred, just run the script
logger.warn('Not using Sacred. Your results will not be saved')
convert_args_and_call_model(args)
def wrap_with_sacred(command, ex_hook):
ex = Experiment()
ex_hook(ex)
ex.config_hook(load_default_config)
ex.main(command)
ex.run_commandline()
def main():
import argparse
parser = argparse.ArgumentParser(description='Two layer linear regression')
parser.add_argument("image_feature_file_train",
type=str,
help="Image Feature file for the training set")
parser.add_argument("text_feature_file_train",
type=str,
help="Text Feature file for the training set")
parser.add_argument("image_feature_file_test",
type=str,
help="Image Feature file for the test set")
parser.add_argument("text_feature_file_test",
type=str,
help="Text Feature file for the test set")
parser.add_argument("word_vector_file",
type=str,
help="Text file containing the word vectors")
# Optional Args
parser.add_argument("--learning_rate",
type=float,
default=0.001,
help="Learning Rate")
parser.add_argument("--num_epochs",
type=int,
default=200,
help="Number of epochs to run for")
parser.add_argument("--batch_size",
type=int,
default=128,
help="Batch size to use for training")
parser.add_argument(
"--model_type",
type=str,
default="multihot",
choices=['multihot', 'naivesum', 'wdv', 'negsampling', 'fast0tag'],
help="Loss function to use for training")
parser.add_argument(
"--in_memory",
action='store_true',
default="store_false",
help="Load training image features into memory for faster training")
parser.add_argument("--model_input_path",
type=str,
default=None,
help="Model input path (to continue training)")
parser.add_argument("--model_output_path",
type=str,
default=None,
help="Model output path (to save training)")
# new args
parser.add_argument(
"--hidden_units",
type=str,
default="200,200",
help="Define a neural network as comma separated layer sizes")
parser.add_argument(
"--cross_eval",
action="store_true",
default=False,
help="Use if test dataset is different from training dataset")
parser.add_argument("--word_vector_type",
type=str,
choices=[t.name for t in WordVectorTypes],
help="Format of word_vector_file")
parser.add_argument("--epoch_verbosity",
type=int,
default=10,
help="Epoch verbosity rate")
parser.add_argument(
"--verbose_eval",
action="store_true",
default=False,
help="Use to run evaluation against test data every epoch_verbosity")
parser.add_argument(
"--optim_words",
action="store_true",
default=False,
help="If using negsampling model_type, use to jointly optimize words")
args = parser.parse_args()
try:
# Sacred Imports
from sacred import Experiment
from sacred.observers import MongoObserver
from sacred.initialize import Scaffold
# Monkey patch to avoid having to declare all our variables
def noop(item):
pass
Scaffold._warn_about_suspicious_changes = noop
ex = Experiment('Attalos')
ex.observers.append(
MongoObserver.create(url=os.environ['MONGO_DB_URI'],
db_name='attalos_experiment'))
ex.main(lambda: convert_args_and_call_model(args))
ex.run(config_updates=args.__dict__)
except ImportError:
# We don't have sacred, just run the script
logger.warn('Not using Sacred. Your results will not be saved')
convert_args_and_call_model(args)
