from sacred import Ingredient from .init import init_model model = Ingredient('model') init_model = model.capture(init_model)
""" :param builder_type: str. Defaults to 'energy_builder'. Possible values include: 'energy_builder', 'force_builder', 'ts_builder'. :param kwargs: kwargs passed directly to Builder classes :return: Builder object specified by 'builder_type' """ kwargs["radial_factory"] = get_radial_factory( kwargs.get("radial_factory", "multi_dense"), kwargs.get("radial_kwargs", None) ) if builder_type == "energy_builder": return EnergyBuilder(**kwargs) elif builder_type == "force_builder": return ForceBuilder(**kwargs) elif builder_type == "cartesian_builder": return CartesianBuilder(**kwargs) elif builder_type == "siamese_builder": return SiameseBuilder(**kwargs) elif builder_type == "classifier_builder": return ClassifierBuilder(**kwargs) else: raise ValueError( "arg `builder_type` had value: {} which is not supported. Check " "ingredient docs for supported string identifiers".format(builder_type) ) # ===== Logger Ingredient(s) ===== # logger_ingredient = Ingredient("metric_logger") get_logger = logger_ingredient.capture(SacredMetricLogger)
bxent_loss = {'name': 'bxent', 'params': {}} xent_loss = {'name': 'xent', 'params': {}} accuracy = {'name': 'acc', 'params': {'output_transform': thresholded_output_transform}} mse_loss = {'name': 'mse', 'params': {}} kl_div = {'name': 'kl-div', 'params': {}} training = Ingredient('training') training.add_named_config('vae', loss=vae_loss, metrics=[reconstruction_loss, kl_div]) training.add_named_config('bvae', loss=bvae_loss, metrics=[reconstruction_loss, kl_div]) training.add_named_config('capconst', loss=cap_const, metrics=[reconstruction_loss, kl_div]) training.add_named_config('2afc', loss=bxent_loss, metrics=[bxent_loss, accuracy]) training.add_named_config('mafc', loss=xent_loss, metrics=[xent_loss, accuracy]) training.add_named_config('recons_nll', loss=reconstruction_loss, metrics=[reconstruction_loss]) init_optimizer = training.capture(init_optimizer) @training.capture def init_metrics(loss, metrics): metrics = list(map(dict.copy, metrics)) return _init_metrics(loss, metrics)
import sys from sacred import Ingredient if sys.path[0] != '../src': sys.path.insert(0, '../src') from model.rnn import init_rnn model = Ingredient('model') init_rnn = model.capture(init_rnn) @model.capture def init_model(device): model = init_rnn() return model.to(device=device) @model.command(unobserved=True) def show(): model = init_model() print(model)
from sacred import Ingredient from .setup import setup_training, set_seed_and_device from .engine import run_training training = Ingredient('training') set_seed_and_device = training.capture(set_seed_and_device) setup_training = training.capture(setup_training) run_training = training.capture(run_training)
import sys from sacred import Ingredient from ignite.engine import Events if sys.path[0] != '../src': sys.path.insert(0, '../src') from training.handlers import * from training.loss import init_metrics from training.optimizer import init_optimizer from training.engine import create_rnn_trainer, create_rnn_evaluator training = Ingredient('training') init_metrics = training.capture(init_metrics) init_optimizer = training.capture(training_optimizer) create_rnn_trainer = training.capture(create_rnn_trainer) create_rnn_evaluator = training.capture(create_rnn_evalautor)
def experiment( dataset: Callable[..., Bunch], estimator: Callable[..., BaseEstimator], *, save_train: bool = False, ) -> Experiment: """ Prepare a Scikit-learn experiment as a Sacred experiment. Prepare a Scikit-learn experiment indicating a dataset and an estimator and return it as a Sacred experiment. Parameters ---------- dataset : function Dataset fetch function. Might receive any argument. Must return a :external:obj:`Bunch` with ``data``, ``target`` (might be ``None``), ``inner_cv`` (might be ``None``) and ``outer_cv`` (might be ``None``). estimator : function Estimator initialization function. Might receive any keyword argument. Must return an initialized sklearn-compatible estimator. Returns ------- experiment : Experiment Sacred experiment, ready to be run. """ dataset_ingredient = Ingredient("dataset") dataset = dataset_ingredient.capture(dataset) estimator_ingredient = Ingredient("estimator") estimator = estimator_ingredient.capture(estimator) experiment = Experiment(ingredients=( dataset_ingredient, estimator_ingredient, ), ) @experiment.automain def run() -> None: """Run the experiment.""" data = dataset() # Metaparameter search cv = getattr(data, "inner_cv", None) try: e = estimator(cv=cv) except TypeError as exception: warn(f"The estimator does not accept cv: {exception}") e = estimator() # Model assessment if getattr(data, "test_indices", None): _benchmark_one( experiment=experiment, estimator=e, data=data, save_train=save_train, ) elif getattr(data, "outer_cv", None) is not None: _benchmark_partitions( experiment=experiment, estimator=e, data=data, save_train=save_train, ) return experiment
def experiment(dataset, estimator): """Prepare a Scikit-learn experiment as a Sacred experiment. Prepare a Scikit-learn experiment indicating a dataset and an estimator and return it as a Sacred experiment. Parameters ---------- dataset : function Dataset fetch function. Might receive any argument. Must return a Bunch with data, target (might be None), inner_cv (might be None) and outer_cv (might be None). estimator : function Estimator initialization function. Might receive any keyword argument. Must return an initialized sklearn-compatible estimator. Returns ------- experiment : Experiment Sacred experiment, ready to be run. """ _dataset = Ingredient('dataset') dataset = _dataset.capture(dataset) _estimator = Ingredient('estimator') estimator = _estimator.capture(estimator) experiment = Experiment(ingredients=(_dataset, _estimator)) @experiment.automain def run(): """Run the experiment.""" data = dataset() # Metaparameter search X = data.data y = data.target cv = None explicit_cv_folds = False if hasattr(data, 'inner_cv'): cv = data.inner_cv explicit_cv_folds = hasattr(data.inner_cv, '__iter__') if explicit_cv_folds: # Explicit CV folds X = np.array([]).reshape((0, *data.inner_cv[0][0].shape[1:])) y = np.array([]).reshape((0, *data.inner_cv[0][1].shape[1:])) cv = [] for i, (X_, y_, X_test_, y_test_) in enumerate(data.inner_cv): X = np.concatenate((X, X_, X_test_)) y = np.concatenate((y, y_, y_test_)) cv = cv + [-1]*len(X_) + [i]*len(X_test_) cv = PredefinedSplit(cv) try: e = estimator(cv=cv) except Exception as exception: warn(f'The estimator does not accept cv: {exception}') e = estimator() if explicit_cv_folds: e.fit(X, y=y) e.fit = e.best_estimator_.fit # Model assessment if hasattr(data, 'data_test') and (data.data_test is not None): # Test partition e.fit(X, y=y) try: with NamedTemporaryFile() as tmpfile: joblib.dump(e, tmpfile.name) experiment.add_artifact(tmpfile.name, name='estimator.joblib') except Exception as exception: warn(f'Artifact save failed: {exception}') experiment.log_scalar('score_mean', e.score(data.data_test, y=data.target_test)) experiment.log_scalar('score_std', 0.0) for output in ('transform', 'predict'): if hasattr(e, output): with open(os.path.join(mkdtemp(), f'{output}.npy'), 'wb+') as tmpfile: np.save(tmpfile, getattr(e, output)(data.data_test)) experiment.add_artifact(tmpfile.name) elif hasattr(data, 'outer_cv'): # Outer CV if hasattr(data.outer_cv, '__iter__'): # Explicit CV folds scores = {'test_score': list(), 'train_score': list(), 'fit_time': list(), 'score_time': list(), 'estimator': list()} outputs = {'transform': list(), 'predict': list()} for X, y, X_test, y_test in data.outer_cv: t0 = process_time() e.fit(X, y=y) t1 = process_time() test_score = e.score(X_test, y=y_test) t2 = process_time() scores['test_score'].append(test_score) scores['train_score'].append(e.score(X, y=y)) scores['fit_time'].append(t1 - t0) scores['score_time'].append(t2 - t1) scores['estimator'].append(e) for output in ('transform', 'predict'): if hasattr(e, output): outputs[output].append([getattr(e, output)(X_test)]) for output in ('transform', 'predict'): if outputs[output]: with open(os.path.join(mkdtemp(), f'{output}.npy'), 'wb+') as tmpfile: np.save(tmpfile, np.array(outputs[output])) experiment.add_artifact(tmpfile.name) else: # Automatic/indexed CV folds scores = cross_validate(e, data.data, y=data.target, cv=data.outer_cv, return_train_score=True, return_estimator=True) try: with NamedTemporaryFile() as tmpfile: joblib.dump(e, tmpfile.name) experiment.add_artifact(tmpfile.name, name='scores.joblib') except Exception as exception: warn(f'Artifact save failed: {exception}') experiment.log_scalar('score_mean', np.nanmean(scores['test_score'])) experiment.log_scalar('score_std', np.nanstd(scores['test_score'])) return experiment
import matplotlib.pyplot as plt if sys.path[0] != '../src': sys.path.insert(0, '../src') # from dataset.tdisc import load_tdata from dataset.sprites import load_sprites from dataset.shapes3d import load_shapes3d from dataset.mpi import load_mpi3d from dataset.transforms import Triplets import configs.datasplits as splits dataset = Ingredient('dataset') load_sprites = dataset.capture(load_sprites) load_shapes3d = dataset.capture(load_shapes3d) load_mpi3d = dataset.capture(load_mpi3d) load_composition = dataset.capture(Triplets) dataset.add_config(setting='unsupervised') dataset.add_named_config('unsupervised', setting='unsupervised') dataset.add_named_config('supervised', setting='supervised') @dataset.capture def get_dataset(dataset): if dataset == 'dsprites': dataset_loader = load_sprites elif dataset == 'shapes3d': dataset_loader = load_shapes3d
from sacred import Ingredient from .dummy import load_data dataset = Ingredient('dataset') load_delayed_addition = dataset.capture(load_data)