def run_experiment(experiment_config: Dict, save_weights: bool, gpu_ind: int, use_wandb: bool = True): """Run a training experiment.""" print(f'Running experiment with config {experiment_config} on GPU {gpu_ind}') datasets_module = importlib.import_module('text_recognizer.datasets') dataset_class_ = getattr(datasets_module, experiment_config['dataset']) dataset_args = experiment_config.get('dataset_args', {}) dataset = dataset_class_(**dataset_args) dataset.load_or_generate_data() print(dataset) models_module = importlib.import_module('text_recognizer.models') model_class_ = getattr(models_module, experiment_config['model']) networks_module = importlib.import_module('text_recognizer.networks') network_fn_ = getattr(networks_module, experiment_config['network']) network_args = experiment_config.get('network_args', {}) model = model_class_(dataset_cls=dataset_class_, network_fn=network_fn_, dataset_args=dataset_args, network_args=network_args) print(model) experiment_config['train_args'] = {**DEFAULT_TRAIN_ARGS, **experiment_config.get('train_args', {})} experiment_config['experiment_group'] = experiment_config.get('experiment_group', None) experiment_config['gpu_ind'] = gpu_ind train_model(model, dataset, epochs=experiment_config['train_args']['epochs'], batch_size=experiment_config['train_args']['batch_size'], use_wandb=use_wandb) score = model.evaluate(dataset.x_test, dataset.y_test) print(f'Test evaluation: {score}') if save_weights: model.save_weights()
def run_experiment(experiment_config: Dict, save_weights: bool, gpu_ind: int, use_wandb: bool=True): """ experiment_config is of the form { "dataset": "EmnistLinesDataset", "dataset_args": { "max_overlap": 0.4 }, "model": "LineModel", "network": "line_cnn_sliding_window", "network_args": { "window_width": 14, "window_stride": 7 }, "train_args": { "batch_size": 128, "epochs": 10 } } save_weights: if True, will save the final model weights to a canonical location (see Model in models/base.py) gpu_ind: integer specifying which gpu to use """ print(f'Running experiment with config {experiment_config} on GPU {gpu_ind}') datasets_module = importlib.import_module('text_recognizer.datasets') dataset_class_ = getattr(datasets_module, experiment_config['dataset']) dataset_args = experiment_config.get('dataset_args', {}) dataset = dataset_class_(**dataset_args) dataset.load_or_generate_data() print(dataset) models_module = importlib.import_module('text_recognizer.models') model_class_ = getattr(models_module, experiment_config['model']) networks_module = importlib.import_module('text_recognizer.networks') network_fn_ = getattr(networks_module, experiment_config['network']) network_args = experiment_config.get('network_args', {}) model = model_class_(dataset_cls=dataset_class_, network_fn=network_fn_, dataset_args=dataset_args, network_args=network_args) print(model) experiment_config['train_args'] = {**DEFAULT_TRAIN_ARGS, **experiment_config.get('train_args', {})} experiment_config['experiment_group'] = experiment_config.get('experiment_group', None) experiment_config['gpu_ind'] = gpu_ind train_model( model, dataset, epochs=experiment_config['train_args']['epochs'], batch_size=experiment_config['train_args']['batch_size'], gpu_ind=gpu_ind, use_wandb=use_wandb ) score = model.evaluate(dataset.x_test, dataset.y_test) print(f'Test evaluation: {score}') if save_weights: model.save_weights()
def run_experiment(experiment_config: Dict, save_weights: bool, gpu_ind: int, use_wandb: bool = True): """ Run a training experiment. Parameters ---------- experiment_config (dict) Of the form { "dataset": "EmnistLinesDataset", "dataset_args": { "max_overlap": 0.4, "subsample_fraction": 0.2 }, "model": "LineModel", "network": "line_cnn_all_conv", "network_args": { "window_width": 14, "window_stride": 7 }, "train_args": { "batch_size": 128, "epochs": 10 } } save_weights (bool) If True, will save the final model weights to a canonical location (see Model in models/base.py) gpu_ind (int) specifies which gpu to use (or -1 for first available) use_wandb (bool) sync training run to wandb """ print( f"Running experiment with config {experiment_config} on GPU {gpu_ind}") datasets_module = importlib.import_module("text_recognizer.datasets") dataset_class_ = getattr(datasets_module, experiment_config["dataset"]) dataset_args = experiment_config.get("dataset_args", {}) dataset = dataset_class_(**dataset_args) dataset.load_or_generate_data() print(dataset) models_module = importlib.import_module("text_recognizer.models") model_class_ = getattr(models_module, experiment_config["model"]) networks_module = importlib.import_module("text_recognizer.networks") network_fn_ = getattr(networks_module, experiment_config["network"]) network_args = experiment_config.get("network_args", {}) model = model_class_( dataset_cls=dataset_class_, network_fn=network_fn_, dataset_args=dataset_args, network_args=network_args, ) print(model) experiment_config["train_args"] = { **DEFAULT_TRAIN_ARGS, **experiment_config.get("train_args", {}), } experiment_config["experiment_group"] = experiment_config.get( "experiment_group", None) experiment_config["gpu_ind"] = gpu_ind # Hide lines below until Lab 4 if use_wandb: wandb.init(config=experiment_config) # Hide lines above until Lab 4 train_model( model, dataset, epochs=experiment_config["train_args"]["epochs"], batch_size=experiment_config["train_args"]["batch_size"], use_wandb=use_wandb, ) score = model.evaluate(dataset.x_test, dataset.y_test) print(f"Test evaluation: {score}") # Hide lines below until Lab 4 if use_wandb: wandb.log({"test_metric": score}) # Hide lines above until Lab 4 if save_weights: model.save_weights()
def run_experiment(experiment_config: Dict, save_weights: bool, gpu_ind: int, use_wandb: bool = True): """ Run a training experiment. Parameters ---------- experiment_config (dict) Of the form { "dataset": "EmnistLinesDataset", "dataset_args": { "max_overlap": 0.4, "subsample_fraction": 0.2 }, "model": "LineModel", "network": "line_cnn_all_conv", "network_args": { "window_width": 14, "window_stride": 7 }, "train_args": { "batch_size": 128, "epochs": 10 } } save_weights (bool) If True, will save the final model weights to a canonical location (see Model in models/base.py) gpu_ind (int) specifies which gpu to use (or -1 for first available) use_wandb (bool) sync training run to wandb """ print( f'Running experiment with config {experiment_config} on GPU {gpu_ind}') tf.enable_eager_execution() datasets_module = importlib.import_module('mss.datasets') dataset_class_ = getattr(datasets_module, experiment_config['dataset']) dataset_args = { **DEFAULT_DATASET_ARGS, **experiment_config.get('dataset_args', {}) } models_module = importlib.import_module('mss.models') model_class_ = getattr(models_module, experiment_config['model']) model_args = { **DEFAULT_MODEL_ARGS, **experiment_config.get('model_args', {}) } networks_module = importlib.import_module('mss.networks') network_fn_ = getattr(networks_module, experiment_config['network']) network_args = experiment_config.get('network_args', {}) train_args = { **DEFAULT_TRAIN_ARGS, **experiment_config.get('train_args', {}) } model = model_class_(dataset_cls=dataset_class_, network_fn=network_fn_, dataset_args=dataset_args, network_args=network_args, train_args=train_args, model_args=model_args) experiment_config['experiment_group'] = experiment_config.get( 'experiment_group', None) experiment_config['gpu_ind'] = gpu_ind # Hide lines below until Lab 4 #if use_wandb: # wandb.init() # wandb.config.update(experiment_config) # Hide lines above until Lab 4 train_model(model, epochs=train_args['epochs'], batch_size=train_args['batch_size'], gpu_ind=gpu_ind, use_wandb=use_wandb) if save_weights: model.save_weights() score = model.evaluate() print(f'Test evaluation: {score}')
def run_experiment(experiment_config: Dict, save_weights: bool, gpu_ind: int, use_wandb: bool = True): print( f'Running experiment with config {experiment_config}, on GPU {gpu_ind}' ) datasets_module = importlib.import_module('architecture.datasets') dataset_class_ = getattr(datasets_module, experiment_config['dataset']) dataset_args = experiment_config.get('dataset_args', {}) dataset = dataset_class_(**dataset_args) dataset.load_or_generate_data() print(dataset) models_module = importlib.import_module('architecture.models') model_class_ = getattr(models_module, experiment_config['model']) networks_module = importlib.import_module('architecture.networks') network_fn_ = getattr(networks_module, experiment_config['network']) network_args = experiment_config.get('network_args', {}) opt_args_ = experiment_config.get('opt_args', DEFAULT_OPT_ARGS) model = model_class_(dataset_cls=dataset_class_, network_fn=network_fn_, dataset_args=dataset_args, network_args=network_args, opt_args=opt_args_) experiment_config["train_args"] = { **DEFAULT_TRAIN_ARGS, **experiment_config.get("train_args", {}), } experiment_config["opt_args"] = { **DEFAULT_OPT_ARGS, **experiment_config.get("opt_args", {}), } experiment_config["experiment_group"] = experiment_config.get( "experiment_group", None) experiment_config["gpu_ind"] = gpu_ind if use_wandb: dataset_name = { 'AlzheimerT2SmallDataset': 't2mini', 'AlzheimerT2StarSmallDataset': 't2starmini', 'AlzheimerT2StarFullDataset': 't2starfull', 'AlzheimerMPRageDeep': 'mprage_deep', 'AlzheimerMPRageNoDeep': 'mprage_nodeep', } tags = [] tags.append('-'.join(list(dataset.mapping.values())).lower()) if dataset.num_classes > 2: tags.append('binary') else: tags.append('multiclass') wandb.init( project='alzheimer-dl', config=experiment_config, name='{model} {dataset} {epochs}ep {batch_size}bs'.format( model=('multi ' if dataset.num_classes > 2 else 'bin ') + experiment_config['network'], dataset=dataset_name[experiment_config['dataset']], epochs=experiment_config["train_args"]["epochs"], batch_size=experiment_config["train_args"]["batch_size"], tags=tags)) with tf.device('/GPU:0'): train_model( model, dataset, epochs=experiment_config["train_args"]["epochs"], batch_size=experiment_config["train_args"]["batch_size"], use_wandb=use_wandb, ) if use_wandb: classes = list(dataset.mapping.values()) cm_val = plot_confusion_matrix( model, dataset.X_val, dataset.y_val, classes, experiment_config["train_args"]["batch_size"]) wandb.log({"confusion_matrix - validation data": cm_val}) cm_test = plot_confusion_matrix( model, dataset.X_test, dataset.y_test, classes, experiment_config["train_args"]["batch_size"]) wandb.log({"confusion_matrix - test": cm_test}) if save_weights: model.save_weights()
def run_experiment(experiment_config: Dict, save_weights: bool, gpu_ind: int, use_experiment_manager: bool = True): """ Run a training experiment. Parameters ---------- experiment_config (dict) Of the form { "dataset": "Dataset", "dataset_args": { "data_arg": 0.4, }, "model": "Model", "network": "neural_net", "network_args": { "hidden_size": 256, }, "train_args": { "batch_size": 128, "epochs": 10 } } save_weights (bool) If True, will save the final model weights to a canonical location (see Model in models/base.py) gpu_ind (int) specifies which gpu to use use_experiment_manager (bool) sync training run to wandb, tensorboard, etc. """ print(f"Running experiment with config {experiment_config} on GPU {gpu_ind}") datasets_module = importlib.import_module("model_core.datasets") dataset_class_ = getattr(datasets_module, experiment_config["dataset"]) dataset_args = experiment_config.get("dataset_args", {}) dataset = dataset_class_(**dataset_args) dataset.load_or_generate_data() print(dataset) models_module = importlib.import_module("model_core.models") model_class_ = getattr(models_module, experiment_config["model"]) networks_module = importlib.import_module("model_core.networks") network_fn_ = getattr(networks_module, experiment_config["network"]) network_args = experiment_config.get("network_args", {}) model = model_class_( dataset_cls=dataset_class_, network_fn=network_fn_, dataset_args=dataset_args, network_args=network_args, ) print(model) experiment_config["train_args"] = { **DEFAULT_TRAIN_ARGS, **experiment_config.get("train_args", {}), } experiment_config["experiment_group"] = experiment_config.get("experiment_group", None) experiment_config["gpu_ind"] = gpu_ind train_model( model, dataset, epochs=experiment_config["train_args"]["epochs"], batch_size=experiment_config["train_args"]["batch_size"], use_experiment_manager=use_experiment_manager, ) score = model.evaluate(dataset.x_test, dataset.y_test) print(f"Test evaluation: {score}") if save_weights: model.save_weights()
def run_experiment(experiment_config: Dict, save_weights: bool, gpu_ind: int, use_wandb: bool = True): """ Run a training experiment. For defaults see: base.fit, network, Parameters ---------- experiment_config (dict) Of the form { "dataset": "EmnistLinesDataset", "dataset_args": { "max_overlap": 0.4, "subsample_fraction": 0.2 }, "model": "LineModel", "network": "line_cnn_all_conv", "network_args": { "window_width": 14, "window_stride": 7 }, "train_args": { "batch_size": 128, "epochs": 10 } } save_weights (bool) If True, will save the final model weights to a canonical location (see Model in models/base.py) gpu_ind (int) specifies which gpu to use (or -1 for first available) use_wandb (bool) sync training run to wandb """ print(f'Running experiment with config {experiment_config} on GPU {gpu_ind}') datasets_module = importlib.import_module('text_recognizer.datasets') dataset_class_ = getattr(datasets_module, experiment_config['dataset']) dataset_args = experiment_config.get('dataset_args', {}) dataset = dataset_class_(**dataset_args) dataset.load_or_generate_data() print(dataset) models_module = importlib.import_module('text_recognizer.models') model_class_ = getattr(models_module, experiment_config['model']) networks_module = importlib.import_module('text_recognizer.networks') network_fn_ = getattr(networks_module, experiment_config['network']) network_args = experiment_config.get('network_args', {}) model = model_class_( dataset_cls=dataset_class_, network_fn=network_fn_, dataset_args=dataset_args, network_args=network_args ) print(model.network) experiment_config['train_args'] = {**DEFAULT_TRAIN_ARGS, **experiment_config.get('train_args', {})} experiment_config['minor_train_args'] = {k:v for k,v in experiment_config['train_args'].items() if k not in ('epochs', 'batch_size')} experiment_config['experiment_group'] = experiment_config.get('experiment_group', None) experiment_config['gpu_ind'] = gpu_ind train_model( model, dataset, epochs=experiment_config['train_args']['epochs'], batch_size=experiment_config['train_args']['batch_size'], gpu_ind=gpu_ind, use_wandb=use_wandb, **experiment_config['minor_train_args'] ) score = model.evaluate(dataset.x_test, dataset.y_test) print(f'Test evaluation: {score}') if save_weights: model.save_weights()
def run_experiment(experiment_config: Dict, save_weights: bool, gpu_ind: int, use_wandb: bool = True): """ experiment_config is of the form { "dataset": "EmnistLinesDataset", "dataset_args": { "max_overlap": 0.4 }, "model": "LineModel", "network": "line_cnn_sliding_window", "network_args": { "window_width": 14, "window_stride": 7 }, "train_args": { "batch_size": 128, "epochs": 10 } } save_weights: if True, will save the final model weights to a canonical location (see Model in models/base.py) gpu_ind: integer specifying which gpu to use """ print( f'Running experiment with config {experiment_config} on GPU {gpu_ind}') # import Datasets module from repo. Make sure Python path set to main repo datasets_module = importlib.import_module('text_recognizer.datasets') dataset_class_ = getattr(datasets_module, experiment_config['dataset']) dataset_args = experiment_config.get('dataset_args', {}) dataset = dataset_class_(**dataset_args) dataset.load_or_generate_data() print(dataset) # Import Models module, instantiate model object with dataset and network objects models_module = importlib.import_module('text_recognizer.models') # CharacterModel, LineModel...etc. model_class_ = getattr(models_module, experiment_config['model']) networks_module = importlib.import_module('text_recognizer.networks') # mlp, resnet...etc. network_fn_ = getattr(networks_module, experiment_config['network']) network_args = experiment_config.get('network_args', {}) # model object model = model_class_(dataset_cls=dataset_class_, network_fn=network_fn_, dataset_args=dataset_args, network_args=network_args) print(model) experiment_config['train_args'] = { **DEFAULT_TRAIN_ARGS, **experiment_config.get('train_args', {}) } experiment_config['experiment_group'] = experiment_config.get( 'experiment_group', None) experiment_config['gpu_ind'] = gpu_ind # Run training script using loaded objects train_model(model, dataset, epochs=experiment_config['train_args']['epochs'], batch_size=experiment_config['train_args']['batch_size'], gpu_ind=gpu_ind, use_wandb=use_wandb) # Compute performance using evaluation metric on the test set score = model.evaluate(dataset.x_test, dataset.y_test) # Print performance score print(f'Test evaluation: {score}') # Save model weights if save_weights: model.save_weights()